fix(testing): Checkout proper branch for Wokwi tests and small QoL improvements (#10435)

* fix(tests): Add missing newlines to output

* fix(tests): Improve fibonacci test

* fix(tests): Remove redundant targets from json

* fix(wokwi): Checkout proper base branch for wokwi tests

* feat(logging): Add logging to some tests to improve debugging

* fix(ci): Make CI more permissive and improve messages

* fix(tests): Bump pytest-embedded version to support P4

.github/scripts/tests_matrix.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+build_types="'validation'"
+hw_types="'validation'"
+wokwi_types="'validation'"
+qemu_types="'validation'"
+
+if [[ $IS_PR != 'true' ]] || [[ $PERFORMANCE_ENABLED == 'true' ]]; then
+  build_types+=",'performance'"
+  hw_types+=",'performance'"
+  #wokwi_types+=",'performance'"
+  #qemu_types+=",'performance'"
+fi
+
+targets="'esp32','esp32s2','esp32s3','esp32c3','esp32c6','esp32h2'"
+
+mkdir -p info
+
+echo "[$wokwi_types]" > info/wokwi_types.txt
+echo "[$targets]" > info/targets.txt
+
+echo "build-types=[$build_types]" >> $GITHUB_OUTPUT
+echo "hw-types=[$hw_types]" >> $GITHUB_OUTPUT
+echo "wokwi-types=[$wokwi_types]" >> $GITHUB_OUTPUT
+echo "qemu-types=[$qemu_types]" >> $GITHUB_OUTPUT
+echo "targets=[$targets]" >> $GITHUB_OUTPUT

.github/workflows/dangerjs.yml

@@ -19,4 +19,6 @@ jobs:
     - name: DangerJS pull request linter
       uses: espressif/shared-github-dangerjs@v1
       env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        rule-max-commits: 'false'
+        commit-messages-min-summary-length: '10'

.github/workflows/tests.yml

@@ -57,29 +57,25 @@ jobs:
       hw-types: ${{ steps.set-matrix.outputs.hw-types }}
       wokwi-types: ${{ steps.set-matrix.outputs.wokwi-types }}
       qemu-types: ${{ steps.set-matrix.outputs.qemu-types }}
+      targets: ${{ steps.set-matrix.outputs.targets }}
+    env:
+      IS_PR: ${{ github.event.pull_request.number != null }}
+      PERFORMANCE_ENABLED: ${{ contains(github.event.pull_request.labels.*.name, 'perf_test') }}
     steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          sparse-checkout: .github/scripts/tests_matrix.sh
+
       - name: Set matrix
         id: set-matrix
-        run: |
-          build_types='["validation"'
-          hw_types='["validation"'
-          wokwi_types='["validation"'
-          qemu_types='["validation"'
+        run: bash .github/scripts/tests_matrix.sh
 
-          is_pr=${{ github.event.pull_request.number != null }}
-          is_performance_enabled=${{ contains(github.event.pull_request.labels.*.name, 'perf_test') }}
-
-          if [[ $is_pr != 'true' ]] || [[ $is_performance_enabled == 'true' ]]; then
-            build_types+=',"performance"'
-            hw_types+=',"performance"'
-            #wokwi_types+=',"performance"'
-            #qemu_types+=',"performance"'
-          fi
-
-          echo "build-types=$build_types]" >> $GITHUB_OUTPUT
-          echo "hw-types=$hw_types]" >> $GITHUB_OUTPUT
-          echo "wokwi-types=$wokwi_types]" >> $GITHUB_OUTPUT
-          echo "qemu-types=$qemu_types]" >> $GITHUB_OUTPUT
+      - name: Upload
+        uses: actions/upload-artifact@v4
+        with:
+          name: matrix_info
+          path: info/*
 
   call-build-tests:
     name: Build
@@ -88,7 +84,7 @@ jobs:
     strategy:
       matrix:
         type: ${{ fromJson(needs.gen-matrix.outputs.build-types) }}
-        chip: ['esp32', 'esp32s2', 'esp32s3', 'esp32c3', 'esp32c6', 'esp32h2']
+        chip: ${{ fromJson(needs.gen-matrix.outputs.targets) }}
     with:
       type: ${{ matrix.type }}
       chip: ${{ matrix.chip }}
@@ -105,7 +101,7 @@ jobs:
       fail-fast: false
       matrix:
         type: ${{ fromJson(needs.gen-matrix.outputs.hw-types) }}
-        chip: ['esp32', 'esp32s2', 'esp32s3', 'esp32c3', 'esp32c6', 'esp32h2']
+        chip: ${{ fromJson(needs.gen-matrix.outputs.targets) }}
     with:
       type: ${{ matrix.type }}
       chip: ${{ matrix.chip }}

.github/workflows/tests_results.yml

@@ -79,6 +79,7 @@ jobs:
           event_name: ${{ env.original_event }}
           files: ./artifacts/**/*.xml
           action_fail: true
+          compare_to_earlier_commit: false
 
       - name: Fail if tests failed
         if: ${{ env.original_conclusion == 'failure' || env.original_conclusion == 'timed_out' || github.event.workflow_run.conclusion == 'failure' || github.event.workflow_run.conclusion == 'timed_out' }}

.github/workflows/tests_wokwi.yml

@@ -22,6 +22,9 @@ jobs:
     outputs:
       pr_num: ${{ steps.set-ref.outputs.pr_num }}
       ref: ${{ steps.set-ref.outputs.ref }}
+      base: ${{ steps.set-ref.outputs.base }}
+      targets: ${{ steps.set-ref.outputs.targets }}
+      types: ${{ steps.set-ref.outputs.types }}
     steps:
       - name: Report pending
         uses: actions/github-script@v7
@@ -51,10 +54,18 @@ jobs:
           name: event_file
           path: artifacts/event_file
 
+      - name: Download and extract matrix info
+        uses: actions/download-artifact@v4
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          run-id: ${{ github.event.workflow_run.id }}
+          name: matrix_info
+          path: artifacts/matrix_info
+
       - name: Try to read PR number
         id: set-ref
         run: |
-          pr_num=$(jq -r '.pull_request.number' artifacts/event_file/event.json)
+          pr_num=$(jq -r '.pull_request.number' artifacts/event_file/event.json | tr -cd "[:digit:]")
           if [ -z "$pr_num" ] || [ "$pr_num" == "null" ]; then
             pr_num=""
           fi
@@ -64,11 +75,22 @@ jobs:
             ref=${{ github.ref }}
           fi
 
-          action=$(jq -r '.action' artifacts/event_file/event.json)
+          action=$(jq -r '.action' artifacts/event_file/event.json | tr -cd "[:alpha:]_")
           if [ "$action" == "null" ]; then
             action=""
           fi
 
+          base=$(jq -r '.pull_request.base.ref' artifacts/event_file/event.json | tr -cd "[:alnum:]/_.-")
+          if [ -z "$base" ] || [ "$base" == "null" ]; then
+            base=${{ github.ref }}
+          fi
+
+          types=$(cat artifacts/matrix_info/wokwi_types.txt | tr -cd "[:alpha:],[]'")
+          targets=$(cat artifacts/matrix_info/targets.txt | tr -cd "[:alnum:],[]'")
+
+          echo "base = $base"
+          echo "targets = $targets"
+          echo "types = $types"
           echo "pr_num = $pr_num"
 
           printf "$ref" >> artifacts/ref.txt
@@ -98,6 +120,9 @@ jobs:
           cat artifacts/conclusion.txt
 
           echo "pr_num=$pr_num" >> $GITHUB_OUTPUT
+          echo "base=$base" >> $GITHUB_OUTPUT
+          echo "targets=$targets" >> $GITHUB_OUTPUT
+          echo "types=$types" >> $GITHUB_OUTPUT
           echo "ref=$ref" >> $GITHUB_OUTPUT
 
       - name: Download and extract parent hardware results
@@ -164,8 +189,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        type: ['validation']
-        chip: ['esp32', 'esp32s2', 'esp32s3', 'esp32c3', 'esp32c6', 'esp32h2']
+        type: ${{ fromJson(needs.get-artifacts.outputs.types) }}
+        chip: ${{ fromJson(needs.get-artifacts.outputs.targets) }}
     steps:
       - name: Report pending
         uses: actions/github-script@v7
@@ -211,9 +236,12 @@ jobs:
           echo "enabled=$enabled" >> $GITHUB_OUTPUT
 
       # Note that changes to the workflows and tests will only be picked up after the PR is merged
+      # DO NOT CHECKOUT THE USER'S REPOSITORY IN THIS WORKFLOW. IT HAS HIGH SECURITY RISKS.
       - name: Checkout repository
         if: ${{ steps.check-tests.outputs.enabled == 'true' }}
         uses: actions/checkout@v4
+        with:
+          ref: ${{ needs.get-artifacts.outputs.base || github.ref }}
 
       - uses: actions/setup-python@v5
         if: ${{ steps.check-tests.outputs.enabled == 'true' }}

tests/performance/coremark/coremark.ino

@@ -40,7 +40,7 @@ void setup() {
   Serial.printf("Cores: %d\n", CONFIG_SOC_CPU_CORES_NUM);
   Serial.flush();
   for (int i = 0; i < N_RUNS; i++) {
-    Serial.printf("Run %d", i);
+    Serial.printf("Run %d\n", i);
     coremark_main();
     Serial.flush();
   }

tests/performance/fibonacci/fibonacci.ino

@@ -31,7 +31,7 @@ void setup() {
   Serial.printf("N: %d\n", FIB_N);
   Serial.flush();
   for (int i = 0; i < N_RUNS; i++) {
-    Serial.printf("Run %d", i);
+    Serial.printf("Run %d\n", i);
     unsigned long start = millis();
     fibonacci = fib(FIB_N);
     unsigned long elapsed = millis() - start;

tests/performance/fibonacci/test_fibonacci.py

@@ -2,24 +2,21 @@ import json
 import logging
 import os
 
+fib_results = {}
+
+def fib(n):
+    if n < 2:
+        return n
+    elif str(n) in fib_results:
+        return fib_results[str(n)]
+    else:
+        fib_results[str(n)] = fib(n - 1) + fib(n - 2)
+        return fib_results[str(n)]
+
 
 def test_fibonacci(dut, request):
     LOGGER = logging.getLogger(__name__)
 
-    # Fibonacci results starting from fib(35) to fib(45)
-    fib_results = [
-        9227465,
-        14930352,
-        24157817,
-        39088169,
-        63245986,
-        102334155,
-        165580141,
-        267914296,
-        433494437,
-        701408733,
-    ]
-
     # Match "Runs: %d"
     res = dut.expect(r"Runs: (\d+)", timeout=60)
     runs = int(res.group(0).decode("utf-8").split(" ")[1])
@@ -30,7 +27,11 @@ def test_fibonacci(dut, request):
     res = dut.expect(r"N: (\d+)", timeout=300)
     fib_n = int(res.group(0).decode("utf-8").split(" ")[1])
     LOGGER.info("Calculating Fibonacci({})".format(fib_n))
-    assert fib_n > 30 and fib_n < 50, "Invalid Fibonacci number"
+    assert fib_n > 0, "Invalid Fibonacci number"
+
+    # Calculate Fibonacci results
+    expected_result = fib(fib_n)
+    LOGGER.info("Expected Fibonacci result: {}".format(expected_result))
 
     list_time = []
 
@@ -48,7 +49,7 @@ def test_fibonacci(dut, request):
         assert fib_result > 0, "Invalid Fibonacci result"
 
         # Check if the result is correct
-        assert fib_result == fib_results[fib_n - 35]
+        assert fib_result == expected_result
 
         # Match "Time: %lu.%03lu s"
         res = dut.expect(r"Time: (\d+)\.(\d+) s", timeout=300)

tests/performance/psramspeed/ci.json

@@ -3,9 +3,7 @@
     "qemu": false,
     "wokwi": false
   },
-  "targets": {
-    "esp32c3": false,
-    "esp32c6": false,
-    "esp32h2": false
-  }
+  "requires": [
+    "CONFIG_SPIRAM=y"
+  ]
 }

tests/performance/psramspeed/psramspeed.ino

@@ -252,7 +252,7 @@ void setup() {
   Serial.printf("Max test size: %d\n", MAX_TEST_SIZE);
   Serial.flush();
   for (int i = 0; i < N_RUNS; i++) {
-    Serial.printf("Run %d", i);
+    Serial.printf("Run %d\n", i);
     memcpy_speed_test(dest, src, MAX_TEST_SIZE, N_COPIES);
     Serial.flush();
     memset_speed_test(dest, FILL_VALUE, MAX_TEST_SIZE, N_COPIES);

tests/performance/ramspeed/ramspeed.ino

@@ -248,7 +248,7 @@ void setup() {
   Serial.printf("Max test size: %d\n", MAX_TEST_SIZE);
   Serial.flush();
   for (int i = 0; i < N_RUNS; i++) {
-    Serial.printf("Run %d", i);
+    Serial.printf("Run %d\n", i);
     memcpy_speed_test(dest, src, MAX_TEST_SIZE, N_COPIES);
     Serial.flush();
     memset_speed_test(dest, FILL_VALUE, MAX_TEST_SIZE, N_COPIES);

tests/performance/superpi/superpi.ino

@@ -25,7 +25,7 @@ void setup() {
   Serial.printf("Digits: %d\n", DIGITS);
   Serial.flush();
   for (int i = 0; i < N_RUNS; i++) {
-    Serial.printf("Run %d", i);
+    Serial.printf("Run %d\n", i);
     unsigned long start = millis();
     pi_calc(DIGITS);
     unsigned long elapsed = millis() - start;

tests/requirements.txt

@@ -1,7 +1,7 @@
 cryptography==43.0.1
 --only-binary cryptography
 pytest-cov==5.0.0
-pytest-embedded-serial-esp==1.11.5
-pytest-embedded-arduino==1.11.5
-pytest-embedded-wokwi==1.11.5
-pytest-embedded-qemu==1.11.5
+pytest-embedded-serial-esp==1.11.6
+pytest-embedded-arduino==1.11.6
+pytest-embedded-wokwi==1.11.6
+pytest-embedded-qemu==1.11.6

tests/validation/gpio/test_gpio.py

@@ -1,5 +1,15 @@
+import logging
+
 def test_gpio(dut):
+    LOGGER = logging.getLogger(__name__)
+
     dut.expect_exact("Button test")
+
+    LOGGER.info("Expecting button press 1")
     dut.expect_exact("Button pressed 1 times")
+
+    LOGGER.info("Expecting button press 2")
     dut.expect_exact("Button pressed 2 times")
+
+    LOGGER.info("Expecting button press 3")
     dut.expect_exact("Button pressed 3 times")

tests/validation/nvs/test_nvs.py

@@ -1,4 +1,13 @@
+import logging
+
 def test_nvs(dut):
+    LOGGER = logging.getLogger(__name__)
+
+    LOGGER.info("Expecting counter value 0")
     dut.expect_exact("Current counter value: 0")
+
+    LOGGER.info("Expecting counter value 1")
     dut.expect_exact("Current counter value: 1")
+
+    LOGGER.info("Expecting counter value 2")
     dut.expect_exact("Current counter value: 2")

tests/validation/periman/test_periman.py

@@ -1,4 +1,7 @@
+import logging
+
 def test_periman(dut):
+    LOGGER = logging.getLogger(__name__)
     peripherals = [
         "GPIO",
         "SigmaDelta",
@@ -29,9 +32,10 @@ def test_periman(dut):
 
         if peripheral in peripherals:
             if "not" in console_output:
-                assert False, f"Peripheral {peripheral} printed when it should not"
+                assert False, f"Output printed when it should not after peripheral {peripheral}"
+            LOGGER.info(f"Correct output after peripheral: {peripheral}")
             peripherals.remove(peripheral)
         else:
             assert False, f"Unknown peripheral: {peripheral}"
 
-    assert peripherals == [], f"Missing peripherals output: {peripherals}"
+    assert peripherals == [], f"Missing output after peripherals: {peripherals}"

tests/validation/psram/ci.json

@@ -5,10 +5,5 @@
   },
   "requires": [
     "CONFIG_SPIRAM=y"
-  ],
-  "targets": {
-    "esp32c3": false,
-    "esp32c6": false,
-    "esp32h2": false
-  }
+  ]
 }

tests/validation/wifi/test_wifi.py

@@ -1,6 +1,15 @@
+import logging
+
 def test_wifi(dut):
+    LOGGER = logging.getLogger(__name__)
+
+    LOGGER.info("Starting WiFi Scan")
     dut.expect_exact("Scan start")
     dut.expect_exact("Scan done")
     dut.expect_exact("Wokwi-GUEST")
+    LOGGER.info("WiFi Scan done")
+
+    LOGGER.info("Connecting to WiFi")
     dut.expect_exact("WiFi connected")
     dut.expect_exact("IP address:")
+    LOGGER.info("WiFi connected")