ci(report): Add runtime tests report (#10764)

.github/scripts/tests_run.sh

@@ -11,9 +11,11 @@ function run_test {
     local error=0
     local sdkconfig_path
     local extra_args
+    local test_type
 
     sketchdir=$(dirname "$sketch")
     sketchname=$(basename "$sketchdir")
+    test_type=$(basename "$(dirname "$sketchdir")")
 
     if [ "$options" -eq 0 ] && [ -f "$sketchdir"/ci.json ]; then
         len=$(jq -r --arg target "$target" '.fqbn[$target] | length' "$sketchdir"/ci.json)
@@ -113,14 +115,14 @@ function run_test {
         rm "$sketchdir"/diagram.json 2>/dev/null || true
 
         result=0
-        printf "\033[95mpytest \"%s/test_%s.py\" --build-dir \"%s\" --junit-xml=\"%s\" %s\033[0m\n" "$sketchdir" "$sketchname" "$build_dir" "$report_file" "${extra_args[*]@Q}"
+        printf "\033[95mpytest \"%s/test_%s.py\" --build-dir \"%s\" --junit-xml=\"%s\" -o junit_suite_name=%s_%s_%s_%s%s %s\033[0m\n" "$sketchdir" "$sketchname" "$build_dir" "$report_file" "$test_type" "$platform" "$target" "$sketchname" "$i" "${extra_args[*]@Q}"
-        bash -c "set +e; pytest \"$sketchdir/test_$sketchname.py\" --build-dir \"$build_dir\" --junit-xml=\"$report_file\" ${extra_args[*]@Q}; exit \$?" || result=$?
+        bash -c "set +e; pytest \"$sketchdir/test_$sketchname.py\" --build-dir \"$build_dir\" --junit-xml=\"$report_file\" -o junit_suite_name=${test_type}_${platform}_${target}_${sketchname}${i} ${extra_args[*]@Q}; exit \$?" || result=$?
         printf "\n"
         if [ $result -ne 0 ]; then
             result=0
             printf "\033[95mRetrying test: %s -- Config: %s\033[0m\n" "$sketchname" "$i"
-            printf "\033[95mpytest \"%s/test_%s.py\" --build-dir \"%s\" --junit-xml=\"%s\" %s\033[0m\n" "$sketchdir" "$sketchname" "$build_dir" "$report_file" "${extra_args[*]@Q}"
+            printf "\033[95mpytest \"%s/test_%s.py\" --build-dir \"%s\" --junit-xml=\"%s\" -o junit_suite_name=%s_%s_%s_%s%s %s\033[0m\n" "$sketchdir" "$sketchname" "$build_dir" "$report_file" "$test_type" "$platform" "$target" "$sketchname" "$i" "${extra_args[*]@Q}"
-            bash -c "set +e; pytest \"$sketchdir/test_$sketchname.py\" --build-dir \"$build_dir\" --junit-xml=\"$report_file\" ${extra_args[*]@Q}; exit \$?" || result=$?
+            bash -c "set +e; pytest \"$sketchdir/test_$sketchname.py\" --build-dir \"$build_dir\" --junit-xml=\"$report_file\" -o junit_suite_name=${test_type}_${platform}_${target}_${sketchname}${i} ${extra_args[*]@Q}; exit \$?" || result=$?
             printf "\n"
             if [ $result -ne 0 ]; then
                 printf "\033[91mFailed test: %s -- Config: %s\033[0m\n\n" "$sketchname" "$i"

.github/workflows/tests_results.yml

@@ -41,6 +41,7 @@ jobs:
           original_sha=$(cat ./artifacts/parent-artifacts/sha.txt)
           original_ref=$(cat ./artifacts/parent-artifacts/ref.txt)
           original_conclusion=$(cat ./artifacts/parent-artifacts/conclusion.txt)
+          original_run_id=$(cat ./artifacts/parent-artifacts/run_id.txt)
 
           # Sanitize the values to avoid security issues
 
@@ -59,17 +60,27 @@ jobs:
           # Conclusion: Allow alphabetical characters and underscores
           original_conclusion=$(echo "$original_conclusion" | tr -cd '[:alpha:]_')
 
+          # Run ID: Allow numeric characters
+          original_run_id=$(echo "$original_run_id" | tr -cd '[:digit:]')
+
           echo "original_event=$original_event" >> $GITHUB_ENV
           echo "original_action=$original_action" >> $GITHUB_ENV
           echo "original_sha=$original_sha" >> $GITHUB_ENV
           echo "original_ref=$original_ref" >> $GITHUB_ENV
           echo "original_conclusion=$original_conclusion" >> $GITHUB_ENV
+          echo "original_run_id=$original_run_id" >> $GITHUB_ENV
 
           echo "original_event = $original_event"
           echo "original_action = $original_action"
           echo "original_sha = $original_sha"
           echo "original_ref = $original_ref"
           echo "original_conclusion = $original_conclusion"
+          echo "original_run_id = $original_run_id"
+
+      - name: Print links to other runs
+        run: |
+          echo "Build, Hardware and QEMU tests: https://github.com/${{ github.repository }}/actions/runs/${{ env.original_run_id }}"
+          echo "Wokwi tests: https://github.com/${{ github.repository }}/actions/runs/${{ github.event.workflow_run.id }}"
 
       - name: Publish Unit Test Results
         uses: EnricoMi/publish-unit-test-result-action@v2
@@ -80,6 +91,17 @@ jobs:
           files: ./artifacts/**/*.xml
           action_fail: true
           compare_to_earlier_commit: false
+          json_file: ./unity_results.json
+          json_suite_details: true
+
+      - name: Upload JSON
+        uses: actions/upload-artifact@v4
+        if: ${{ always() }}
+        with:
+          name: unity_results
+          overwrite: true
+          path: |
+            ./unity_results.json
 
       - name: Fail if tests failed
         if: ${{ env.original_conclusion == 'failure' || env.original_conclusion == 'timed_out' || github.event.workflow_run.conclusion == 'failure' || github.event.workflow_run.conclusion == 'timed_out' }}
@@ -138,11 +160,18 @@ jobs:
             })).data;
             core.info(`${name} is ${state}`);
 
-      - name: Create output folder
+      - name: Generate report
         if: ${{ !cancelled() && (env.original_event == 'schedule' || env.original_event == 'workflow_dispatch') }} # codespell:ignore cancelled
+        env:
+          REPORT_FILE: ./runtime-tests-results/RUNTIME_TESTS_REPORT.md
+          WOKWI_RUN_ID: ${{ github.event.workflow_run.id }}
+          BUILD_RUN_ID: ${{ env.original_run_id }}
+          IS_FAILING: ${{ env.original_conclusion == 'failure' || env.original_conclusion == 'timed_out' || github.event.workflow_run.conclusion == 'failure' || github.event.workflow_run.conclusion == 'timed_out' || job.status == 'failure' }}
         run: |
-          rm -rf artifacts
+          rm -rf artifacts $REPORT_FILE
-          mkdir -p runtime-tests-results
+          mv -f ./unity_results.json ./runtime-tests-results/unity_results.json
+          touch $REPORT_FILE
+          python3 ./runtime-tests-results/table_generator.py ./runtime-tests-results/unity_results.json >> $REPORT_FILE
 
       - name: Generate badge
         if: ${{ !cancelled() && (env.original_event == 'schedule' || env.original_event == 'workflow_dispatch') }} # codespell:ignore cancelled
@@ -161,6 +190,6 @@ jobs:
           git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
           if [[ `git status --porcelain` ]]; then
             git add --all
-            git commit -m "Updated runtime tests badge"
+            git commit -m "Updated runtime tests report"
             git push origin HEAD:gh-pages
           fi

.github/workflows/tests_wokwi.yml

@@ -109,6 +109,10 @@ jobs:
           printf "\nAction = "
           cat artifacts/action.txt
 
+          printf "${{ github.event.workflow_run.id }}" >> artifacts/run_id.txt
+          printf "\nRun ID = "
+          cat artifacts/run_id.txt
+
           if [ -z "$ref" ] || [ "$ref" == "null" ]; then
             echo "Failed to get PR number or ref"
             exit 1

README.md

@@ -3,7 +3,7 @@
 [![Build Status](https://img.shields.io/github/actions/workflow/status/espressif/arduino-esp32/push.yml?branch=master&event=push&label=Compilation%20Tests)](https://github.com/espressif/arduino-esp32/actions/workflows/push.yml?query=branch%3Amaster+event%3Apush)
 [![Verbose Build Status](https://img.shields.io/github/actions/workflow/status/espressif/arduino-esp32/push.yml?branch=master&event=schedule&label=Compilation%20Tests%20(Verbose))](https://github.com/espressif/arduino-esp32/actions/workflows/push.yml?query=branch%3Amaster+event%3Aschedule)
 [![External Libraries Test](https://img.shields.io/github/actions/workflow/status/espressif/arduino-esp32/lib.yml?branch=master&event=schedule&label=External%20Libraries%20Test)](https://github.com/espressif/arduino-esp32/blob/gh-pages/LIBRARIES_TEST.md)
-[![Runtime Tests](https://github.com/espressif/arduino-esp32/blob/gh-pages/runtime-tests-results/badge.svg)](https://github.com/espressif/arduino-esp32/actions/workflows/tests_results.yml)
+[![Runtime Tests](https://github.com/espressif/arduino-esp32/blob/gh-pages/runtime-tests-results/badge.svg)](https://github.com/espressif/arduino-esp32/blob/gh-pages/runtime-tests-results/RUNTIME_TESTS_REPORT.md)
 
 ### Need help or have a question? Join the chat at [Discord](https://discord.gg/8xY6e9crwv) or [open a new Discussion](https://github.com/espressif/arduino-esp32/discussions)