adafruit-mirror/arduino-esp32
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

ci(refactor): Fix workflows and small improvements (#9793)

Browse source 
* ci(fix): Fix new workflows

* ci(pre-commit): Properly cache python modules

* fix(get.py): Verify files after extraction and make it more readable

* ci(idf): Compile as component when libs change

* ci(wokwi): Re-enable cache for wokwi tests

* ci(tests): Run scripts only from master

* Revert "fix(get.py): Verify files after extraction and make it more readable"

This reverts commit cc40edb40766ea8a013c0d76326bc57314418bff.

* ci(compilation): Fix changed files check

* ci(wokwi): Disable tests that use scenarios
This commit is contained in:
Lucas Saavedra Vaz 2024-06-07 12:29:27 -03:00 committed by GitHub
parent 5d9cd9ac58
commit e3fedc5e8c
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
10 changed files with 121 additions and 65 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
.github/scripts/tests_run.sh vendored
View file

@ -101,7 +101,7 @@ function run_test() {
fi fi
fi fi
done done
printf "\n" printf "Test return code: $error\n"
return $error return $error
} }
@ -250,6 +250,7 @@ else
exit_code=0 exit_code=0
run_test $target $sketch $options $erase || exit_code=$? run_test $target $sketch $options $erase || exit_code=$?
echo "Sketch $sketch exit code: $exit_code"
if [ $exit_code -ne 0 ]; then if [ $exit_code -ne 0 ]; then
error=$exit_code error=$exit_code
fi fi

17
.github/workflows/build_tests.yml vendored
View file

@ -47,11 +47,24 @@ jobs:
echo "enabled=$enabled" >> $GITHUB_OUTPUT echo "enabled=$enabled" >> $GITHUB_OUTPUT
- name: Checkout Repository - name: Checkout user repository
uses: actions/checkout@v4
if: ${{ steps.check-build.outputs.enabled == 'true' }} if: ${{ steps.check-build.outputs.enabled == 'true' }}
uses: actions/checkout@v4
with: with:
ref: ${{ github.event.pull_request.head.sha || github.sha }} ref: ${{ github.event.pull_request.head.sha || github.sha }}
persist-credentials: false
sparse-checkout-cone-mode: false
sparse-checkout: |
/*
!.github
# To avoid giving unknown scripts elevated permissions, download them from the master branch
- name: Get CI scripts from master
if: ${{ steps.check-build.outputs.enabled == 'true' }}
run: |
mkdir -p .github
cd .github
curl https://codeload.github.com/${{ github.repository }}/tar.gz/master | tar -xz --strip=2 arduino-esp32-master/.github
- name: Get libs cache - name: Get libs cache
uses: actions/cache@v4 uses: actions/cache@v4

28
.github/workflows/hw.yml vendored
View file

@ -49,18 +49,32 @@ jobs:
echo "enabled=$enabled" >> $GITHUB_OUTPUT echo "enabled=$enabled" >> $GITHUB_OUTPUT
- name: Checkout repository - name: Checkout user repository
if: ${{ steps.check-tests.outputs.enabled == 'true' }} if: ${{ steps.check-tests.outputs.enabled == 'true' }}
uses: actions/checkout@v4 uses: actions/checkout@v4
with: with:
ref: ${{ github.event.pull_request.head.sha || github.sha }} ref: ${{ github.event.pull_request.head.sha || github.sha }}
persist-credentials: false
sparse-checkout-cone-mode: false
sparse-checkout: |
/*
!.github
- uses: actions/setup-python@v5 # To avoid giving unknown scripts elevated permissions, download them from the master branch
if: steps.check-tests.outputs.enabled == 'true' - name: Get CI scripts from master
with: if: ${{ steps.check-tests.outputs.enabled == 'true' }}
cache-dependency-path: tests/requirements.txt run: |
cache: 'pip' mkdir -p .github
python-version: '3.10.1' cd .github
curl https://codeload.github.com/${{ github.repository }}/tar.gz/master | tar -xz --strip=2 arduino-esp32-master/.github
# setup-python currently only works on ubuntu images
# - uses: actions/setup-python@v5
# if: ${{ steps.check-tests.outputs.enabled == 'true' }}
# with:
# cache-dependency-path: tests/requirements.txt
# cache: 'pip'
# python-version: '3.10.1'
- name: Install dependencies - name: Install dependencies
if: ${{ steps.check-tests.outputs.enabled == 'true' }} if: ${{ steps.check-tests.outputs.enabled == 'true' }}

8
.github/workflows/pre-commit.yml vendored
View file

@ -28,6 +28,8 @@ jobs:
- name: Set up Python 3 - name: Set up Python 3
uses: actions/setup-python@v5 uses: actions/setup-python@v5
with: with:
cache-dependency-path: tools/pre-commit/requirements.txt
cache: 'pip'
python-version: "3.x" python-version: "3.x"
- name: Get Python version hash - name: Get Python version hash
@ -41,11 +43,10 @@ jobs:
with: with:
path: | path: |
~/.cache/pre-commit ~/.cache/pre-commit
~/.cache/pip key: pre-commit-${{ env.PY_HASH }}-${{ hashFiles('.pre-commit-config.yaml', '.github/workflows/pre-commit.yml', 'tools/pre-commit/requirements.txt') }}
key: pre-commit|${{ env.PY_HASH }}|${{ hashFiles('.pre-commit-config.yaml', '.github/workflows/pre-commit.yml') }}
- name: Install python dependencies - name: Install python dependencies
run: python -m pip install pre-commit docutils run: python -m pip install -r tools/pre-commit/requirements.txt
- name: Get changed files - name: Get changed files
id: changed-files id: changed-files
@ -61,7 +62,6 @@ jobs:
with: with:
path: | path: |
~/.cache/pre-commit ~/.cache/pre-commit
~/.cache/pip
key: ${{ steps.restore-cache.outputs.cache-primary-key }} key: ${{ steps.restore-cache.outputs.cache-primary-key }}
- name: Push changes using pre-commit-ci-lite - name: Push changes using pre-commit-ci-lite

12
.github/workflows/push.yml vendored
View file

@ -48,6 +48,7 @@ jobs:
runs-on: ubuntu-latest runs-on: ubuntu-latest
outputs: outputs:
build_all: ${{ steps.set-chunks.outputs.build_all }} build_all: ${{ steps.set-chunks.outputs.build_all }}
build_libraries: ${{ steps.set-chunks.outputs.build_libraries }}
build_static_sketches: ${{ steps.set-chunks.outputs.build_static_sketches }} build_static_sketches: ${{ steps.set-chunks.outputs.build_static_sketches }}
build_idf: ${{ steps.set-chunks.outputs.build_idf }} build_idf: ${{ steps.set-chunks.outputs.build_idf }}
build_platformio: ${{ steps.set-chunks.outputs.build_platformio }} build_platformio: ${{ steps.set-chunks.outputs.build_platformio }}
@ -103,6 +104,7 @@ jobs:
build_platformio=${{ steps.changed-files.outputs.platformio_any_changed == 'true' }} build_platformio=${{ steps.changed-files.outputs.platformio_any_changed == 'true' }}
build_idf=${{ steps.changed-files.outputs.idf_any_changed == 'true' }} build_idf=${{ steps.changed-files.outputs.idf_any_changed == 'true' }}
build_libraries=${{ steps.changed-files.outputs.libraries_any_changed == 'true' }}
build_static_sketches=${{ steps.changed-files.outputs.static_sketeches_any_changed == 'true' }} build_static_sketches=${{ steps.changed-files.outputs.static_sketeches_any_changed == 'true' }}
core_changed=${{ steps.changed-files.outputs.core_any_changed == 'true' }} core_changed=${{ steps.changed-files.outputs.core_any_changed == 'true' }}
@ -136,9 +138,6 @@ jobs:
fi fi
echo "" echo ""
done done
else
echo "Unhandled change triggered the build. This should not happen."
exit 1
fi fi
if [[ -n $sketches ]]; then if [[ -n $sketches ]]; then
@ -164,6 +163,7 @@ jobs:
chunks+="]" chunks+="]"
echo "build_all=$build_all" >> $GITHUB_OUTPUT echo "build_all=$build_all" >> $GITHUB_OUTPUT
echo "build_libraries=$build_libraries" >> $GITHUB_OUTPUT
echo "build_static_sketches=$build_static_sketches" >> $GITHUB_OUTPUT echo "build_static_sketches=$build_static_sketches" >> $GITHUB_OUTPUT
echo "build_idf=$build_idf" >> $GITHUB_OUTPUT echo "build_idf=$build_idf" >> $GITHUB_OUTPUT
echo "build_platformio=$build_platformio" >> $GITHUB_OUTPUT echo "build_platformio=$build_platformio" >> $GITHUB_OUTPUT
@ -182,6 +182,7 @@ jobs:
# Ubuntu # Ubuntu
build-arduino-linux: build-arduino-linux:
name: Arduino ${{ matrix.chunk }} on ubuntu-latest name: Arduino ${{ matrix.chunk }} on ubuntu-latest
if: ${{ needs.gen-chunks.outputs.build_all == 'true' || needs.gen-chunks.outputs.build_libraries == 'true' }}
needs: gen-chunks needs: gen-chunks
runs-on: ubuntu-latest runs-on: ubuntu-latest
strategy: strategy:
@ -274,7 +275,10 @@ jobs:
build-esp-idf-component: build-esp-idf-component:
name: Build with ESP-IDF ${{ matrix.idf_ver }} for ${{ matrix.idf_target }} name: Build with ESP-IDF ${{ matrix.idf_ver }} for ${{ matrix.idf_target }}
needs: gen-chunks needs: gen-chunks
if: ${{ needs.gen-chunks.outputs.build_all == 'true' || needs.gen-chunks.outputs.build_idf == 'true' }} if: |
needs.gen-chunks.outputs.build_all == 'true' ||
needs.gen-chunks.outputs.build_libraries == 'true' ||
needs.gen-chunks.outputs.build_idf == 'true'
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
strategy: strategy:
fail-fast: false fail-fast: false

33
.github/workflows/qemu.yml vendored
View file

@ -45,35 +45,48 @@ jobs:
echo "enabled=$enabled" >> $GITHUB_OUTPUT echo "enabled=$enabled" >> $GITHUB_OUTPUT
- name: Checkout repository - name: Checkout user repository
if: ${{ steps.check-tests.outputs.enabled == 'true' }}
uses: actions/checkout@v4 uses: actions/checkout@v4
if: steps.check-tests.outputs.enabled == 'true'
with: with:
ref: ${{ github.event.pull_request.head.sha || github.sha }} ref: ${{ github.event.pull_request.head.sha || github.sha }}
persist-credentials: false
sparse-checkout-cone-mode: false
sparse-checkout: |
/*
!.github
# To avoid giving unknown scripts elevated permissions, download them from the master branch
- name: Get CI scripts from master
if: ${{ steps.check-tests.outputs.enabled == 'true' }}
run: |
mkdir -p .github
cd .github
curl https://codeload.github.com/${{ github.repository }}/tar.gz/master | tar -xz --strip=2 arduino-esp32-master/.github
- uses: actions/setup-python@v5 - uses: actions/setup-python@v5
if: steps.check-tests.outputs.enabled == 'true' if: ${{ steps.check-tests.outputs.enabled == 'true' }}
with: with:
cache-dependency-path: tests/requirements.txt cache-dependency-path: tests/requirements.txt
cache: 'pip' cache: 'pip'
python-version: '3.x' python-version: '3.x'
- name: Install Python dependencies - name: Install Python dependencies
if: steps.check-tests.outputs.enabled == 'true' if: ${{ steps.check-tests.outputs.enabled == 'true' }}
run: | run: |
pip install -U pip pip install -U pip
pip install -r tests/requirements.txt --extra-index-url https://dl.espressif.com/pypi pip install -r tests/requirements.txt --extra-index-url https://dl.espressif.com/pypi
- name: Install APT dependencies - name: Install APT dependencies
uses: awalsh128/cache-apt-pkgs-action@v1.4.2 uses: awalsh128/cache-apt-pkgs-action@v1.4.2
if: steps.check-tests.outputs.enabled == 'true' if: ${{ steps.check-tests.outputs.enabled == 'true' }}
with: with:
packages: libpixman-1-0 libnuma1 libglib2.0-0 libslirp0 libsdl2-2.0-0 packages: libpixman-1-0 libnuma1 libglib2.0-0 libslirp0 libsdl2-2.0-0
version: 1.0 version: 1.0
- name: Get QEMU version - name: Get QEMU version
uses: pozetroninc/github-action-get-latest-release@v0.7.0 uses: pozetroninc/github-action-get-latest-release@v0.7.0
if: steps.check-tests.outputs.enabled == 'true' if: ${{ steps.check-tests.outputs.enabled == 'true' }}
id: get-qemu-version id: get-qemu-version
with: with:
token: ${{secrets.GITHUB_TOKEN}} token: ${{secrets.GITHUB_TOKEN}}
@ -84,14 +97,14 @@ jobs:
- name: Cache QEMU - name: Cache QEMU
id: cache-qemu id: cache-qemu
uses: actions/cache@v4 uses: actions/cache@v4
if: steps.check-tests.outputs.enabled == 'true' if: ${{ steps.check-tests.outputs.enabled == 'true' }}
with: with:
path: | path: |
~/qemu ~/qemu
key: qemu-${{ steps.get-qemu-version.outputs.release }}-${{ hashFiles('.github/workflows/qemu.yml') }} key: qemu-${{ steps.get-qemu-version.outputs.release }}-${{ hashFiles('.github/workflows/qemu.yml') }}
- name: Download QEMU - name: Download QEMU
if: steps.cache-qemu.outputs.cache-hit != 'true' && steps.check-tests.outputs.enabled == 'true' if: ${{ steps.cache-qemu.outputs.cache-hit != 'true' && steps.check-tests.outputs.enabled == 'true' }}
run: | run: |
cd ${{ env.QEMU_INSTALL_PATH }} cd ${{ env.QEMU_INSTALL_PATH }}
underscore_release=$(echo ${{ steps.get-qemu-version.outputs.release }} | sed 's/\-/_/g') underscore_release=$(echo ${{ steps.get-qemu-version.outputs.release }} | sed 's/\-/_/g')
@ -103,7 +116,7 @@ jobs:
echo "QEMU_PATH=${{ env.QEMU_INSTALL_PATH }}/qemu" >> $GITHUB_ENV echo "QEMU_PATH=${{ env.QEMU_INSTALL_PATH }}/qemu" >> $GITHUB_ENV
- name: Get binaries - name: Get binaries
if: steps.check-tests.outputs.enabled == 'true' if: ${{ steps.check-tests.outputs.enabled == 'true' }}
id: cache-build-binaries id: cache-build-binaries
uses: actions/cache/restore@v4 uses: actions/cache/restore@v4
with: with:
@ -115,7 +128,7 @@ jobs:
~/.arduino/tests/**/build*.tmp/*.json ~/.arduino/tests/**/build*.tmp/*.json
- name: Run Tests - name: Run Tests
if: steps.check-tests.outputs.enabled == 'true' if: ${{ steps.check-tests.outputs.enabled == 'true' }}
run: QEMU_PATH="${{ env.QEMU_INSTALL_PATH }}" bash .github/scripts/tests_run.sh -c -type ${{inputs.type}} -t ${{inputs.chip}} -i 0 -m 1 -Q run: QEMU_PATH="${{ env.QEMU_INSTALL_PATH }}" bash .github/scripts/tests_run.sh -c -type ${{inputs.type}} -t ${{inputs.chip}} -i 0 -m 1 -Q
- name: Upload ${{ inputs.chip }} ${{ inputs.type }} QEMU results as cache - name: Upload ${{ inputs.chip }} ${{ inputs.type }} QEMU results as cache

15
.github/workflows/tests.yml vendored
View file

@ -12,14 +12,6 @@ on:
- '!libraries/**.txt' - '!libraries/**.txt'
- '!libraries/**.properties' - '!libraries/**.properties'
- 'package/**' - 'package/**'
- '.github/workflows/tests.yml'
- '.github/workflows/build_tests.yml'
- '.github/workflows/hw.yml'
- '.github/workflows/wokwi.yml'
- '.github/workflows/qemu.yml'
- '.github/scripts/install-*.sh'
- '.github/scripts/tests_*.sh'
- '.github/scripts/sketch_utils.sh'
schedule: schedule:
- cron: '0 2 * * *' - cron: '0 2 * * *'
@ -167,7 +159,7 @@ jobs:
clean: clean:
name: Clean objects name: Clean objects
needs: unit-test-results needs: unit-test-results
if: always() && ${{ github.event_name }} == 'pull_request_target' && ${{ github.event.action }} != 'closed' if: always()
permissions: permissions:
actions: write actions: write
runs-on: ubuntu-latest runs-on: ubuntu-latest
@ -179,6 +171,11 @@ jobs:
const ref = '${{ github.event.pull_request.number || github.ref }}'; const ref = '${{ github.event.pull_request.number || github.ref }}';
const key_prefix = 'tests-' + ref + '-'; const key_prefix = 'tests-' + ref + '-';
if ('${{ github.event_name }}' == 'pull_request_target' && '${{ github.event.action }}' != 'closed') {
console.log('Skipping cache cleanup for open PR');
return;
}
await github.paginate(github.rest.actions.getActionsCacheList, { await github.paginate(github.rest.actions.getActionsCacheList, {
owner: context.repo.owner, owner: context.repo.owner,
repo: context.repo.repo, repo: context.repo.repo,

65
.github/workflows/wokwi.yml vendored
View file

@ -30,16 +30,15 @@ jobs:
id: ${{ github.event.pull_request.number || github.ref }}-${{ github.event.pull_request.head.sha || github.sha }}-${{ inputs.chip }}-${{ inputs.type }} id: ${{ github.event.pull_request.number || github.ref }}-${{ github.event.pull_request.head.sha || github.sha }}-${{ inputs.chip }}-${{ inputs.type }}
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
# Disabled as Wokwi infrastrucutre is not stable (so we can re-trigger the tests manually) - name: Check if already run
# - name: Check if already run if: ${{ github.event.pull_request.number != null }}
# if: ${{ github.event.pull_request.number != null }} id: get-cache-results
# id: get-cache-results uses: actions/cache/restore@v4
# uses: actions/cache/restore@v4 with:
# with: key: tests-${{ env.id }}-results-wokwi
# key: tests-${{ env.id }}-results-wokwi path: |
# path: | tests/**/*.xml
# tests/**/*.xml tests/**/result_*.json
# tests/**/result_*.json
- name: Evaluate if tests should be run - name: Evaluate if tests should be run
id: check-tests id: check-tests
@ -54,35 +53,48 @@ jobs:
echo "enabled=$enabled" >> $GITHUB_OUTPUT echo "enabled=$enabled" >> $GITHUB_OUTPUT
- name: Checkout Repository - name: Checkout user repository
if: ${{ steps.check-tests.outputs.enabled == 'true' }}
uses: actions/checkout@v4 uses: actions/checkout@v4
if: steps.check-tests.outputs.enabled == 'true'
with: with:
ref: ${{ github.event.pull_request.head.sha || github.sha }} ref: ${{ github.event.pull_request.head.sha || github.sha }}
persist-credentials: false
sparse-checkout-cone-mode: false
sparse-checkout: |
/*
!.github
# To avoid giving unknown scripts elevated permissions, download them from the master branch
- name: Get CI scripts from master
if: ${{ steps.check-tests.outputs.enabled == 'true' }}
run: |
mkdir -p .github
cd .github
curl https://codeload.github.com/${{ github.repository }}/tar.gz/master | tar -xz --strip=2 arduino-esp32-master/.github
- uses: actions/setup-python@v5 - uses: actions/setup-python@v5
if: steps.check-tests.outputs.enabled == 'true' if: ${{ steps.check-tests.outputs.enabled == 'true' }}
with: with:
cache-dependency-path: tests/requirements.txt cache-dependency-path: tests/requirements.txt
cache: 'pip' cache: 'pip'
python-version: '3.x' python-version: '3.x'
- name: Install dependencies - name: Install dependencies
if: steps.check-tests.outputs.enabled == 'true' if: ${{ steps.check-tests.outputs.enabled == 'true' }}
run: | run: |
pip install -U pip pip install -U pip
pip install -r tests/requirements.txt --extra-index-url https://dl.espressif.com/pypi pip install -r tests/requirements.txt --extra-index-url https://dl.espressif.com/pypi
- name: Install Wokwi CLI - name: Install Wokwi CLI
if: steps.check-tests.outputs.enabled == 'true' if: ${{ steps.check-tests.outputs.enabled == 'true' }}
run: curl -L https://wokwi.com/ci/install.sh | sh run: curl -L https://wokwi.com/ci/install.sh | sh
- name: Wokwi CI Server - name: Wokwi CI Server
if: steps.check-tests.outputs.enabled == 'true' if: ${{ steps.check-tests.outputs.enabled == 'true' }}
uses: wokwi/wokwi-ci-server-action@v1 uses: wokwi/wokwi-ci-server-action@v1
- name: Get binaries - name: Get binaries
if: steps.check-tests.outputs.enabled == 'true' if: ${{ steps.check-tests.outputs.enabled == 'true' }}
id: cache-build-binaries id: cache-build-binaries
uses: actions/cache/restore@v4 uses: actions/cache/restore@v4
with: with:
@ -94,21 +106,20 @@ jobs:
~/.arduino/tests/**/build*.tmp/*.json ~/.arduino/tests/**/build*.tmp/*.json
- name: Run Tests - name: Run Tests
if: steps.check-tests.outputs.enabled == 'true' if: ${{ steps.check-tests.outputs.enabled == 'true' }}
env: env:
WOKWI_CLI_TOKEN: ${{ secrets.WOKWI_CLI_TOKEN }} WOKWI_CLI_TOKEN: ${{ secrets.WOKWI_CLI_TOKEN }}
run: | run: |
bash .github/scripts/tests_run.sh -c -type ${{ inputs.type }} -t ${{inputs.chip}} -i 0 -m 1 -W ${{env.WOKWI_TIMEOUT}} bash .github/scripts/tests_run.sh -c -type ${{ inputs.type }} -t ${{inputs.chip}} -i 0 -m 1 -W ${{env.WOKWI_TIMEOUT}}
# Disabled as Wokwi infrastrucutre is not stable (so we can re-trigger the tests manually) - name: Upload ${{ inputs.chip }} ${{ inputs.type }} Wokwi results as cache
# - name: Upload ${{ inputs.chip }} ${{ inputs.type }} Wokwi results as cache uses: actions/cache/save@v4
# uses: actions/cache/save@v4 if: ${{ always() && steps.check-tests.outputs.enabled == 'true' }}
# if: ${{ always() && steps.check-tests.outputs.enabled == 'true' }} with:
# with: key: tests-${{ env.id }}-results-wokwi
# key: tests-${{ env.id }}-results-wokwi path: |
# path: | tests/**/*.xml
# tests/**/*.xml tests/**/result_*.json
# tests/**/result_*.json
- name: Upload ${{ inputs.chip }} ${{ inputs.type }} Wokwi results as artifacts - name: Upload ${{ inputs.chip }} ${{ inputs.type }} Wokwi results as artifacts
uses: actions/upload-artifact@v4 uses: actions/upload-artifact@v4

3
tests/validation/gpio/ci.json
View file

@ -1,6 +1,7 @@
{ {
"platforms": { "platforms": {
"qemu": false, "qemu": false,
"hardware": false "hardware": false,
"wokwi": false
} }
} }

2
tools/pre-commit/requirements.txt Normal file
View file

@ -0,0 +1,2 @@
pre-commit==3.7.1
docutils==0.21.2