From ae90679f88e00eff3e56166478591c047d04db0b Mon Sep 17 00:00:00 2001
From: Dmitrii Golovanov <dmitrii.golovanov@intel.com>
Date: Fri, 10 Jan 2025 14:50:46 +0100
Subject: [PATCH] tests: benchmarks: thread_metric: Record measurements

Extentd 'benchmark.thread_metric' (tests/benchmarks/thread_metric)
test suite to collect benchmark measurements in Twister reports
as recordings parsed from the test's output: time period values
as well as errors.

Additionally, each test is executed until it makes at least 3
measurements to estimate variance.

Signed-off-by: Dmitrii Golovanov <dmitrii.golovanov@intel.com>
---
 tests/benchmarks/thread_metric/testcase.yaml  | 21 ++++++++++++++---
 .../thread_metric/thread_metric_readme.txt    | 23 ++++++++++++++++++-
 2 files changed, 40 insertions(+), 4 deletions(-)

diff --git a/tests/benchmarks/thread_metric/testcase.yaml b/tests/benchmarks/thread_metric/testcase.yaml
index dbe5f7442c7..43fef683dd0 100644
--- a/tests/benchmarks/thread_metric/testcase.yaml
+++ b/tests/benchmarks/thread_metric/testcase.yaml
@@ -9,8 +9,10 @@ common:
   # time does not pass while the CPU executes. So the benchmark just appears as if hung.
   arch_exclude:
     - posix
-  # qemu_nios2 excluded as it is slow
+  # some slow qemu_* excluded
   platform_exclude:
+    - qemu_malta/qemu_malta
+    - qemu_malta/qemu_malta/be
     - qemu_nios2
   integration_platforms:
     - qemu_x86
@@ -18,9 +20,22 @@ common:
   timeout: 300
   harness: console
   harness_config:
-    type: one_line
+    type: multi_line
+    ordered: true
     regex:
-      - "(.*) Relative Time: (.*)"
+      # Collect at least 3 measurements for each benchmark:
+      - "(.*) Thread-Metric(.+) Relative Time:[ ]*[0-9]+(.*)"
+      - "(.*)Time Period Total:[ ]*[0-9]+(.*)"
+      - "(.*) Thread-Metric(.+) Relative Time:[ ]*[0-9]+(.*)"
+      - "(.*)Time Period Total:[ ]*[0-9]+(.*)"
+      - "(.*) Thread-Metric(.+) Relative Time:[ ]*[0-9]+(.*)"
+      - "(.*)Time Period Total:[ ]*[0-9]+(.*)"
+    record:
+      regex:
+        - "Time Period Total:[ ]*(?P<total_time_period>[0-9]+)"
+        - "ERROR:[ ]*(?P<error_message>.*)"
+        - "[ ]+Average:(?P<error_details>.*)"
+      merge: true
 
 tests:
   benchmark.thread_metric.basic:
diff --git a/tests/benchmarks/thread_metric/thread_metric_readme.txt b/tests/benchmarks/thread_metric/thread_metric_readme.txt
index c2216ddb9b7..a4ab6a52a86 100644
--- a/tests/benchmarks/thread_metric/thread_metric_readme.txt
+++ b/tests/benchmarks/thread_metric/thread_metric_readme.txt
@@ -123,6 +123,27 @@ tm_memory_allocation_test.c                   Basic memory allocation test
 tm_porting_layer_zephyr.c                     Specific porting layer source
                                                  code for Zephyr
 
+2.5. Test execution with Twister tool
+
+When the test suite is executed by Twister it takes parameters from testcase.yaml
+file, in particular:
+
+    * check expected benchmark output presence at least three times to collect
+      measurements from 3 consequtive intervals for each of the benchmark tests.
+
+    * use 300 sec. timeout on each benchmark test from this suite;
+      it is expected to be at least twice bigger than normally needed
+      to collect measurements 3 times with 30 sec. intervals on most of the
+      platforms except some simulators.
+
+    * parse benchmark output to extract measurements and errors when
+      it happens e.g. on counters diverged from average; Twister records
+      this data in twister.json and recording.csv report files for analysis.
+
+For more details see Twister testcase.yaml documentation and 'harness_config:'
+parameters.
+
+
 3 Porting
 
 3.1 Porting Layer
@@ -216,7 +237,7 @@ measurement tests:
     of memory. If successful, a TM_SUCCESS is returned.
 
 
-2.2 Porting Requirements Checklist
+3.2 Porting Requirements Checklist
 
 The following requirements are made in order to ensure fair benchmarks
 are achieved on each RTOS performing the test: