adafruit-mirror/arduino-esp32
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

ci(pre-commit): Add bash script formatter and linter (#10681)

Browse source 
* ci(pre-commit): Add check for bash scripts

* fix(formatting): Fix bash scripts

* docs(pre-commit): Add info about the included hooks
This commit is contained in:
Lucas Saavedra Vaz 2024-12-09 10:25:11 -03:00 committed by GitHub
parent 7a0775d697
commit cd772f1437
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
22 changed files with 741 additions and 574 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
.github/scripts/check-cmakelists.sh vendored
View file

@ -1,4 +1,5 @@
#!/bin/bash
#
# This script is used in the CI workflow. It checks all non-examples source files in libraries/ and cores/ are listed in
# CMakeLists.txt for the cmake-based IDF component
@ -12,10 +13,10 @@ set -e
git submodule update --init --recursive
# find all source files in repo
REPO_SRCS=`find cores/esp32/ libraries/ -name 'examples' -prune -o -name '*.c' -print -o -name '*.cpp' -print | sort`
REPO_SRCS=$(find cores/esp32/ libraries/ -name 'examples' -prune -o -name '*.c' -print -o -name '*.cpp' -print | sort)
# find all source files named in CMakeLists.txt COMPONENT_SRCS
CMAKE_SRCS=`cmake --trace-expand -P CMakeLists.txt 2>&1 | grep set\(srcs | cut -d'(' -f3 | sed 's/ )//' | sed 's/srcs //' | tr ' ;' '\n' | sort`
CMAKE_SRCS=$(cmake --trace-expand -P CMakeLists.txt 2>&1 | grep set\(srcs | cut -d'(' -f3 | sed 's/ )//' | sed 's/srcs //' | tr ' ;' '\n' | sort)
if ! diff -u0 --label "Repo Files" --label "srcs" <(echo "$REPO_SRCS") <(echo "$CMAKE_SRCS"); then
echo "Source files in repo (-) and source files in CMakeLists.txt (+) don't match"

25
.github/scripts/find_all_boards.sh vendored
View file

@ -3,7 +3,9 @@
# Get all boards
boards_array=()
for line in `grep '.tarch=' boards.txt`; do
boards_list=$(grep '.tarch=' boards.txt)
while read -r line; do
board_name=$(echo "$line" | cut -d '.' -f1 | cut -d '#' -f1)
# skip esp32c2 as we dont build libs for it
if [ "$board_name" == "esp32c2" ]; then
@ -12,29 +14,26 @@ for line in `grep '.tarch=' boards.txt`; do
fi
boards_array+=("espressif:esp32:$board_name")
echo "Added 'espressif:esp32:$board_name' to array"
done
done <<< "$boards_list"
# Create JSON like string with all boards found and pass it to env variable
board_count=${#boards_array[@]}
echo "Boards found: $board_count"
echo "BOARD-COUNT=$board_count" >> $GITHUB_ENV
echo "BOARD-COUNT=$board_count" >> "$GITHUB_ENV"
if [ $board_count -gt 0 ]
then
if [ "$board_count" -gt 0 ]; then
json_matrix='['
for board in ${boards_array[@]}
do
for board in "${boards_array[@]}"; do
json_matrix+='"'$board'"'
if [ $board_count -gt 1 ]
then
if [ "$board_count" -gt 1 ]; then
json_matrix+=","
fi
board_count=$(($board_count - 1))
board_count=$((board_count - 1))
done
json_matrix+=']'
echo $json_matrix
echo "FQBNS=${json_matrix}" >> $GITHUB_ENV
echo "$json_matrix"
echo "FQBNS=${json_matrix}" >> "$GITHUB_ENV"
else
echo "FQBNS=" >> $GITHUB_ENV
echo "FQBNS=" >> "$GITHUB_ENV"
fi

37
.github/scripts/find_new_boards.sh vendored
View file

@ -5,14 +5,13 @@ owner_repository=$1
base_ref=$2
# Download the boards.txt file from the base branch
curl -L -o boards_base.txt https://raw.githubusercontent.com/$owner_repository/$base_ref/boards.txt
curl -L -o boards_base.txt https://raw.githubusercontent.com/"$owner_repository"/"$base_ref"/boards.txt
# Compare boards.txt file in the repo with the modified file from PR
diff=$(diff -u boards_base.txt boards.txt)
# Check if the diff is empty
if [ -z "$diff" ]
then
if [ -z "$diff" ]; then
echo "No changes in boards.txt file"
echo "FQBNS="
exit 0
@ -21,7 +20,7 @@ fi
# Extract added or modified lines (lines starting with '+' or '-')
modified_lines=$(echo "$diff" | grep -E '^[+-][^+-]')
# Print the modified lines for debugging
# Print the modified lines for debugging
echo "Modified lines:"
echo "$modified_lines"
@ -29,15 +28,12 @@ boards_array=()
previous_board=""
# Extract board names from the modified lines, and add them to the boards_array
while read -r line
do
while read -r line; do
board_name=$(echo "$line" | cut -d '.' -f1 | cut -d '#' -f1)
# remove + or - from the board name at the beginning
board_name=$(echo "$board_name" | sed 's/^[+-]//')
if [ "$board_name" != "" ] && [ "$board_name" != "+" ] && [ "$board_name" != "-" ] && [ "$board_name" != "esp32_family" ]
then
if [ "$board_name" != "$previous_board" ]
then
board_name=${board_name#[-+]}
if [ "$board_name" != "" ] && [ "$board_name" != "+" ] && [ "$board_name" != "-" ] && [ "$board_name" != "esp32_family" ]; then
if [ "$board_name" != "$previous_board" ]; then
boards_array+=("espressif:esp32:$board_name")
previous_board="$board_name"
echo "Added 'espressif:esp32:$board_name' to array"
@ -48,22 +44,19 @@ done <<< "$modified_lines"
# Create JSON like string with all boards found and pass it to env variable
board_count=${#boards_array[@]}
if [ $board_count -gt 0 ]
then
if [ "$board_count" -gt 0 ]; then
json_matrix='{"fqbn": ['
for board in ${boards_array[@]}
do
for board in "${boards_array[@]}"; do
json_matrix+='"'$board'"'
if [ $board_count -gt 1 ]
then
if [ "$board_count" -gt 1 ]; then
json_matrix+=","
fi
board_count=$(($board_count - 1))
board_count=$((board_count - 1))
done
json_matrix+=']}'
echo $json_matrix
echo "FQBNS=${json_matrix}" >> $GITHUB_ENV
echo "$json_matrix"
echo "FQBNS=${json_matrix}" >> "$GITHUB_ENV"
else
echo "FQBNS=" >> $GITHUB_ENV
fi
echo "FQBNS=" >> "$GITHUB_ENV"
fi

3
.github/scripts/install-arduino-cli.sh vendored
View file

@ -1,6 +1,6 @@
#!/bin/bash
OSBITS=`uname -m`
OSBITS=$(uname -m)
if [[ "$OSTYPE" == "linux"* ]]; then
export OS_IS_LINUX="1"
if [[ "$OSBITS" == "i686" ]]; then
@ -49,4 +49,3 @@ if [ ! -d "$ARDUINO_IDE_PATH" ] || [ ! -f "$ARDUINO_IDE_PATH/arduino-cli" ]; the
curl -fsSL https://raw.githubusercontent.com/arduino/arduino-cli/master/install.sh | BINDIR="$ARDUINO_IDE_PATH" sh
fi
fi

14
.github/scripts/install-arduino-core-esp32.sh vendored
View file

@ -5,7 +5,7 @@ if [ ! -d "$ARDUINO_ESP32_PATH" ]; then
echo "Installing ESP32 Arduino Core ..."
script_init_path="$PWD"
mkdir -p "$ARDUINO_USR_PATH/hardware/espressif"
cd "$ARDUINO_USR_PATH/hardware/espressif"
cd "$ARDUINO_USR_PATH/hardware/espressif" || exit
echo "Installing Python Serial ..."
pip install pyserial > /dev/null
@ -15,25 +15,25 @@ if [ ! -d "$ARDUINO_ESP32_PATH" ]; then
pip install requests > /dev/null
fi
if [ ! -z "$GITHUB_REPOSITORY" ]; then
if [ -n "$GITHUB_REPOSITORY" ]; then
echo "Linking Core..."
ln -s $GITHUB_WORKSPACE esp32
ln -s "$GITHUB_WORKSPACE" esp32
else
echo "Cloning Core Repository..."
git clone https://github.com/espressif/arduino-esp32.git esp32 > /dev/null 2>&1
fi
#echo "Updating Submodules ..."
cd esp32
cd esp32 || exit
#git submodule update --init --recursive > /dev/null 2>&1
echo "Installing Platform Tools ..."
if [ "$OS_IS_WINDOWS" == "1" ]; then
cd tools && ./get.exe
cd tools && ./get.exe
else
cd tools && python get.py
cd tools && python get.py
fi
cd $script_init_path
cd "$script_init_path" || exit
echo "ESP32 Arduino has been installed in '$ARDUINO_ESP32_PATH'"
echo ""

3
.github/scripts/install-arduino-ide.sh vendored
View file

@ -4,7 +4,7 @@
#OSTYPE: 'msys', ARCH: 'x86_64' => win32
#OSTYPE: 'darwin18', ARCH: 'i386' => macos
OSBITS=`uname -m`
OSBITS=$(uname -m)
if [[ "$OSTYPE" == "linux"* ]]; then
export OS_IS_LINUX="1"
ARCHIVE_FORMAT="tar.xz"
@ -77,4 +77,3 @@ if [ ! -d "$ARDUINO_IDE_PATH" ]; then
echo "Arduino IDE Installed in '$ARDUINO_IDE_PATH'"
echo ""
fi

54
.github/scripts/install-platformio-esp32.sh vendored
View file

@ -52,7 +52,7 @@ python -c "$replace_script"
if [ "$GITHUB_REPOSITORY" == "espressif/arduino-esp32" ]; then
echo "Linking Core..."
ln -s $GITHUB_WORKSPACE "$PLATFORMIO_ESP32_PATH"
ln -s "$GITHUB_WORKSPACE" "$PLATFORMIO_ESP32_PATH"
else
echo "Cloning Core Repository ..."
git clone --recursive https://github.com/espressif/arduino-esp32.git "$PLATFORMIO_ESP32_PATH" > /dev/null 2>&1
@ -61,7 +61,7 @@ fi
echo "PlatformIO for ESP32 has been installed"
echo ""
function build_pio_sketch(){ # build_pio_sketch <board> <options> <path-to-ino>
function build_pio_sketch { # build_pio_sketch <board> <options> <path-to-ino>
if [ "$#" -lt 3 ]; then
echo "ERROR: Illegal number of parameters"
echo "USAGE: build_pio_sketch <board> <options> <path-to-ino>"
@ -71,13 +71,15 @@ function build_pio_sketch(){ # build_pio_sketch <board> <options> <path-to-ino>
local board="$1"
local options="$2"
local sketch="$3"
local sketch_dir=$(dirname "$sketch")
local sketch_dir
sketch_dir=$(dirname "$sketch")
echo ""
echo "Compiling '"$(basename "$sketch")"' ..."
echo "Compiling '$(basename "$sketch")' ..."
python -m platformio ci --board "$board" "$sketch_dir" --project-option="$options"
}
function build_pio_sketches(){ # build_pio_sketches <board> <options> <examples-path> <chunk> <total-chunks>
function build_pio_sketches { # build_pio_sketches <board> <options> <examples-path> <chunk> <total-chunks>
if [ "$#" -lt 3 ]; then
echo "ERROR: Illegal number of parameters"
echo "USAGE: build_pio_sketches <board> <options> <examples-path> [<chunk> <total-chunks>]"
@ -108,27 +110,34 @@ function build_pio_sketches(){ # build_pio_sketches <board> <options> <examples-
${COUNT_SKETCHES} "$examples" "esp32"
local sketchcount=$?
set -e
local sketches=$(cat sketches.txt)
local sketches
sketches=$(cat sketches.txt)
rm -rf sketches.txt
local chunk_size=$(( $sketchcount / $chunks_num ))
local all_chunks=$(( $chunks_num * $chunk_size ))
local chunk_size
local all_chunks
local start_index
local end_index
local start_num
chunk_size=$(( sketchcount / chunks_num ))
all_chunks=$(( chunks_num * chunk_size ))
if [ "$all_chunks" -lt "$sketchcount" ]; then
chunk_size=$(( $chunk_size + 1 ))
chunk_size=$(( chunk_size + 1 ))
fi
local start_index=$(( $chunk_idex * $chunk_size ))
start_index=$(( chunk_idex * chunk_size ))
if [ "$sketchcount" -le "$start_index" ]; then
echo "Skipping job"
return 0
fi
local end_index=$(( $(( $chunk_idex + 1 )) * $chunk_size ))
end_index=$(( $(( chunk_idex + 1 )) * chunk_size ))
if [ "$end_index" -gt "$sketchcount" ]; then
end_index=$sketchcount
fi
local start_num=$(( $start_index + 1 ))
start_num=$(( start_index + 1 ))
echo "Found $sketchcount Sketches";
echo "Chunk Count : $chunks_num"
echo "Chunk Size : $chunk_size"
@ -137,25 +146,32 @@ function build_pio_sketches(){ # build_pio_sketches <board> <options> <examples-
local sketchnum=0
for sketch in $sketches; do
local sketchdir=$(dirname $sketch)
local sketchdirname=$(basename $sketchdir)
local sketchname=$(basename $sketch)
local sketchdir
local sketchdirname
local sketchname
local is_target
local has_requirements
sketchdir=$(dirname "$sketch")
sketchdirname=$(basename "$sketchdir")
sketchname=$(basename "$sketch")
if [[ "$sketchdirname.ino" != "$sketchname" ]]; then
continue
elif [ -f $sketchdir/ci.json ]; then
elif [ -f "$sketchdir"/ci.json ]; then
# If the target is listed as false, skip the sketch. Otherwise, include it.
is_target=$(jq -r '.targets[esp32]' $sketchdir/ci.json)
is_target=$(jq -r '.targets[esp32]' "$sketchdir"/ci.json)
if [[ "$is_target" == "false" ]]; then
continue
fi
local has_requirements=$(${CHECK_REQUIREMENTS} $sketchdir "$SDKCONFIG_DIR/esp32/sdkconfig")
has_requirements=$(${CHECK_REQUIREMENTS} "$sketchdir" "$SDKCONFIG_DIR/esp32/sdkconfig")
if [ "$has_requirements" == "0" ]; then
continue
fi
fi
sketchnum=$(($sketchnum + 1))
sketchnum=$((sketchnum + 1))
if [ "$sketchnum" -le "$start_index" ] \
|| [ "$sketchnum" -gt "$end_index" ]; then
continue

69
.github/scripts/on-pages.sh vendored
View file

@ -1,12 +1,13 @@
#/bin/bash
#!/bin/bash
set -e
function get_file_size(){
function get_file_size {
local file="$1"
if [[ "$OSTYPE" == "darwin"* ]]; then
eval `stat -s "$file"`
eval "$(stat -s "$file")"
local res="$?"
echo "$st_size"
echo "${st_size:?}"
return $res
else
stat --printf="%s" "$file"
@ -15,25 +16,32 @@ function get_file_size(){
}
#git_remove_from_pages <file>
function git_remove_from_pages(){
function git_remove_from_pages {
local path=$1
local info=`curl -s -k -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.v3.object+json" -X GET "https://api.github.com/repos/$GITHUB_REPOSITORY/contents/$path?ref=gh-pages"`
local type=`echo "$info" | jq -r '.type'`
if [ ! $type == "file" ]; then
if [ ! $type == "null" ]; then
local info
local type
local sha
local message
info=$(curl -s -k -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.v3.object+json" -X GET "https://api.github.com/repos/$GITHUB_REPOSITORY/contents/$path?ref=gh-pages")
type=$(echo "$info" | jq -r '.type')
if [ ! "$type" == "file" ]; then
if [ ! "$type" == "null" ]; then
echo "Wrong type '$type'"
else
echo "File is not on Pages"
fi
return 0
fi
local sha=`echo "$info" | jq -r '.sha'`
local message="Deleting "$(basename $path)
sha=$(echo "$info" | jq -r '.sha')
message="Deleting "$(basename "$path")
local json="{\"branch\":\"gh-pages\",\"message\":\"$message\",\"sha\":\"$sha\"}"
echo "$json" | curl -s -k -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.v3.raw+json" -X DELETE --data @- "https://api.github.com/repos/$GITHUB_REPOSITORY/contents/$path"
}
function git_upload_to_pages(){
function git_upload_to_pages {
local path=$1
local src=$2
@ -42,41 +50,50 @@ function git_upload_to_pages(){
return 1
fi
local info=`curl -s -k -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.v3.object+json" -X GET "https://api.github.com/repos/$GITHUB_REPOSITORY/contents/$path?ref=gh-pages"`
local type=`echo "$info" | jq -r '.type'`
local message=$(basename $path)
local info
local type
local message
local sha=""
local content=""
if [ $type == "file" ]; then
sha=`echo "$info" | jq -r '.sha'`
info=$(curl -s -k -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.v3.object+json" -X GET "https://api.github.com/repos/$GITHUB_REPOSITORY/contents/$path?ref=gh-pages")
type=$(echo "$info" | jq -r '.type')
message=$(basename "$path")
if [ "$type" == "file" ]; then
sha=$(echo "$info" | jq -r '.sha')
sha=",\"sha\":\"$sha\""
message="Updating $message"
elif [ ! $type == "null" ]; then
elif [ ! "$type" == "null" ]; then
>&2 echo "Wrong type '$type'"
return 1
else
message="Creating $message"
fi
content=`base64 -i "$src"`
content=$(base64 -i "$src")
data="{\"branch\":\"gh-pages\",\"message\":\"$message\",\"content\":\"$content\"$sha}"
echo "$data" | curl -s -k -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.v3.raw+json" -X PUT --data @- "https://api.github.com/repos/$GITHUB_REPOSITORY/contents/$path"
}
function git_safe_upload_to_pages(){
function git_safe_upload_to_pages {
local path=$1
local file="$2"
local name=$(basename "$file")
local size=`get_file_size "$file"`
local upload_res=`git_upload_to_pages "$path" "$file"`
if [ $? -ne 0 ]; then
local name
local size
local upload_res
name=$(basename "$file")
size=$(get_file_size "$file")
if ! upload_res=$(git_upload_to_pages "$path" "$file"); then
>&2 echo "ERROR: Failed to upload '$name' ($?)"
return 1
fi
up_size=`echo "$upload_res" | jq -r '.content.size'`
if [ $up_size -ne $size ]; then
up_size=$(echo "$upload_res" | jq -r '.content.size')
if [ "$up_size" -ne "$size" ]; then
>&2 echo "ERROR: Uploaded size does not match! $up_size != $size"
#git_delete_asset
return 1

80
.github/scripts/on-push.sh vendored
View file

@ -4,43 +4,44 @@ set -e
export ARDUINO_BUILD_DIR="$HOME/.arduino/build.tmp"
function build(){
function build {
local target=$1
local chunk_index=$2
local chunks_cnt=$3
local build_log=$4
local sketches_file=$5
shift; shift; shift; shift; shift;
local sketches=$*
shift 5
local sketches=("$@")
local BUILD_SKETCH="${SCRIPTS_DIR}/sketch_utils.sh build"
local BUILD_SKETCHES="${SCRIPTS_DIR}/sketch_utils.sh chunk_build"
local args="-ai $ARDUINO_IDE_PATH -au $ARDUINO_USR_PATH"
args+=" -t $target"
local args=("-ai" "$ARDUINO_IDE_PATH" "-au" "$ARDUINO_USR_PATH" "-t" "$target")
if [ "$OS_IS_LINUX" == "1" ]; then
args+=" -p $ARDUINO_ESP32_PATH/libraries"
args+=" -i $chunk_index -m $chunks_cnt"
args+=("-p" "$ARDUINO_ESP32_PATH/libraries" "-i" "$chunk_index" "-m" "$chunks_cnt")
if [ -n "$sketches_file" ]; then
args+=" -f $sketches_file"
args+=("-f" "$sketches_file")
fi
if [ $build_log -eq 1 ]; then
args+=" -l $build_log"
if [ "$build_log" -eq 1 ]; then
args+=("-l" "$build_log")
fi
${BUILD_SKETCHES} ${args}
${BUILD_SKETCHES} "${args[@]}"
else
for sketch in ${sketches}; do
local sargs="$args -s $(dirname $sketch)"
for sketch in "${sketches[@]}"; do
local sargs=("${args[@]}")
local ctags_version
local preprocessor_version
sargs+=("-s" "$(dirname "$sketch")")
if [ "$OS_IS_WINDOWS" == "1" ] && [ -d "$ARDUINO_IDE_PATH/tools-builder" ]; then
local ctags_version=`ls "$ARDUINO_IDE_PATH/tools-builder/ctags/"`
local preprocessor_version=`ls "$ARDUINO_IDE_PATH/tools-builder/arduino-preprocessor/"`
win_opts="-prefs=runtime.tools.ctags.path=$ARDUINO_IDE_PATH/tools-builder/ctags/$ctags_version
-prefs=runtime.tools.arduino-preprocessor.path=$ARDUINO_IDE_PATH/tools-builder/arduino-preprocessor/$preprocessor_version"
sargs+=" ${win_opts}"
ctags_version=$(ls "$ARDUINO_IDE_PATH/tools-builder/ctags/")
preprocessor_version=$(ls "$ARDUINO_IDE_PATH/tools-builder/arduino-preprocessor/")
sargs+=(
"-prefs=runtime.tools.ctags.path=$ARDUINO_IDE_PATH/tools-builder/ctags/$ctags_version"
"-prefs=runtime.tools.arduino-preprocessor.path=$ARDUINO_IDE_PATH/tools-builder/arduino-preprocessor/$preprocessor_version"
)
fi
${BUILD_SKETCH} ${sargs}
${BUILD_SKETCH} "${sargs[@]}"
done
fi
}
@ -73,40 +74,40 @@ fi
SCRIPTS_DIR="./.github/scripts"
if [ "$BUILD_PIO" -eq 0 ]; then
source ${SCRIPTS_DIR}/install-arduino-cli.sh
source ${SCRIPTS_DIR}/install-arduino-core-esp32.sh
source "${SCRIPTS_DIR}/install-arduino-cli.sh"
source "${SCRIPTS_DIR}/install-arduino-core-esp32.sh"
SKETCHES_ESP32="\
$ARDUINO_ESP32_PATH/libraries/NetworkClientSecure/examples/WiFiClientSecure/WiFiClientSecure.ino\
$ARDUINO_ESP32_PATH/libraries/BLE/examples/Server/Server.ino\
$ARDUINO_ESP32_PATH/libraries/ESP32/examples/Camera/CameraWebServer/CameraWebServer.ino\
$ARDUINO_ESP32_PATH/libraries/Insights/examples/MinimalDiagnostics/MinimalDiagnostics.ino\
"
SKETCHES_ESP32=(
"$ARDUINO_ESP32_PATH/libraries/NetworkClientSecure/examples/WiFiClientSecure/WiFiClientSecure.ino"
"$ARDUINO_ESP32_PATH/libraries/BLE/examples/Server/Server.ino"
"$ARDUINO_ESP32_PATH/libraries/ESP32/examples/Camera/CameraWebServer/CameraWebServer.ino"
"$ARDUINO_ESP32_PATH/libraries/Insights/examples/MinimalDiagnostics/MinimalDiagnostics.ino"
)
#create sizes_file
sizes_file="$GITHUB_WORKSPACE/cli_compile_$CHUNK_INDEX.json"
if [ "$BUILD_LOG" -eq 1 ]; then
#create sizes_file and echo start of JSON array with "boards" key
echo "{\"boards\": [" > $sizes_file
echo "{\"boards\": [" > "$sizes_file"
fi
#build sketches for different targets
build "esp32p4" "$CHUNK_INDEX" "$CHUNKS_CNT" "$BUILD_LOG" "$SKETCHES_FILE" "$SKETCHES_ESP32"
build "esp32s3" "$CHUNK_INDEX" "$CHUNKS_CNT" "$BUILD_LOG" "$SKETCHES_FILE" "$SKETCHES_ESP32"
build "esp32s2" "$CHUNK_INDEX" "$CHUNKS_CNT" "$BUILD_LOG" "$SKETCHES_FILE" "$SKETCHES_ESP32"
build "esp32c3" "$CHUNK_INDEX" "$CHUNKS_CNT" "$BUILD_LOG" "$SKETCHES_FILE" "$SKETCHES_ESP32"
build "esp32c6" "$CHUNK_INDEX" "$CHUNKS_CNT" "$BUILD_LOG" "$SKETCHES_FILE" "$SKETCHES_ESP32"
build "esp32h2" "$CHUNK_INDEX" "$CHUNKS_CNT" "$BUILD_LOG" "$SKETCHES_FILE" "$SKETCHES_ESP32"
build "esp32" "$CHUNK_INDEX" "$CHUNKS_CNT" "$BUILD_LOG" "$SKETCHES_FILE" "$SKETCHES_ESP32"
build "esp32p4" "$CHUNK_INDEX" "$CHUNKS_CNT" "$BUILD_LOG" "$SKETCHES_FILE" "${SKETCHES_ESP32[@]}"
build "esp32s3" "$CHUNK_INDEX" "$CHUNKS_CNT" "$BUILD_LOG" "$SKETCHES_FILE" "${SKETCHES_ESP32[@]}"
build "esp32s2" "$CHUNK_INDEX" "$CHUNKS_CNT" "$BUILD_LOG" "$SKETCHES_FILE" "${SKETCHES_ESP32[@]}"
build "esp32c3" "$CHUNK_INDEX" "$CHUNKS_CNT" "$BUILD_LOG" "$SKETCHES_FILE" "${SKETCHES_ESP32[@]}"
build "esp32c6" "$CHUNK_INDEX" "$CHUNKS_CNT" "$BUILD_LOG" "$SKETCHES_FILE" "${SKETCHES_ESP32[@]}"
build "esp32h2" "$CHUNK_INDEX" "$CHUNKS_CNT" "$BUILD_LOG" "$SKETCHES_FILE" "${SKETCHES_ESP32[@]}"
build "esp32" "$CHUNK_INDEX" "$CHUNKS_CNT" "$BUILD_LOG" "$SKETCHES_FILE" "${SKETCHES_ESP32[@]}"
if [ "$BUILD_LOG" -eq 1 ]; then
#remove last comma from the last JSON object
sed -i '$ s/,$//' "$sizes_file"
#echo end of JSON array
echo "]}" >> $sizes_file
echo "]}" >> "$sizes_file"
fi
else
source ${SCRIPTS_DIR}/install-platformio-esp32.sh
source "${SCRIPTS_DIR}/install-platformio-esp32.sh"
# PlatformIO ESP32 Test
BOARD="esp32dev"
OPTIONS="board_build.partitions = huge_app.csv"
@ -117,8 +118,7 @@ else
build_pio_sketch "$BOARD" "$OPTIONS" "$PLATFORMIO_ESP32_PATH/libraries/ESP32/examples/Camera/CameraWebServer/CameraWebServer.ino"
# Basic sanity testing for other series
for board in "esp32-c3-devkitm-1" "esp32-s2-saola-1" "esp32-s3-devkitc-1"
do
for board in "esp32-c3-devkitm-1" "esp32-s2-saola-1" "esp32-s3-devkitc-1"; do
python -m platformio ci --board "$board" "$PLATFORMIO_ESP32_PATH/libraries/WiFi/examples/WiFiClient" --project-option="board_build.partitions = huge_app.csv"
done

215
.github/scripts/on-release.sh vendored
View file

@ -1,29 +1,34 @@
#!/bin/bash
# Disable shellcheck warning about using 'cat' to read a file.
# Disable shellcheck warning about using individual redirections for each command.
# Disable shellcheck warning about $? uses.
# shellcheck disable=SC2002,SC2129,SC2181,SC2319
if [ ! $GITHUB_EVENT_NAME == "release" ]; then
if [ ! "$GITHUB_EVENT_NAME" == "release" ]; then
echo "Wrong event '$GITHUB_EVENT_NAME'!"
exit 1
fi
EVENT_JSON=`cat $GITHUB_EVENT_PATH`
EVENT_JSON=$(cat "$GITHUB_EVENT_PATH")
action=`echo $EVENT_JSON | jq -r '.action'`
if [ ! $action == "published" ]; then
action=$(echo "$EVENT_JSON" | jq -r '.action')
if [ ! "$action" == "published" ]; then
echo "Wrong action '$action'. Exiting now..."
exit 0
fi
draft=`echo $EVENT_JSON | jq -r '.release.draft'`
if [ $draft == "true" ]; then
draft=$(echo "$EVENT_JSON" | jq -r '.release.draft')
if [ "$draft" == "true" ]; then
echo "It's a draft release. Exiting now..."
exit 0
fi
RELEASE_PRE=`echo $EVENT_JSON | jq -r '.release.prerelease'`
RELEASE_TAG=`echo $EVENT_JSON | jq -r '.release.tag_name'`
RELEASE_BRANCH=`echo $EVENT_JSON | jq -r '.release.target_commitish'`
RELEASE_ID=`echo $EVENT_JSON | jq -r '.release.id'`
RELEASE_PRE=$(echo "$EVENT_JSON" | jq -r '.release.prerelease')
RELEASE_TAG=$(echo "$EVENT_JSON" | jq -r '.release.tag_name')
RELEASE_BRANCH=$(echo "$EVENT_JSON" | jq -r '.release.target_commitish')
RELEASE_ID=$(echo "$EVENT_JSON" | jq -r '.release.id')
SCRIPTS_DIR="./.github/scripts"
OUTPUT_DIR="$GITHUB_WORKSPACE/build"
PACKAGE_NAME="esp32-$RELEASE_TAG"
PACKAGE_JSON_MERGE="$GITHUB_WORKSPACE/.github/scripts/merge_packages.py"
@ -36,17 +41,23 @@ echo "Action: $action, Branch: $RELEASE_BRANCH, ID: $RELEASE_ID"
echo "Tag: $RELEASE_TAG, Draft: $draft, Pre-Release: $RELEASE_PRE"
# Try extracting something like a JSON with a "boards" array/element and "vendor" fields
BOARDS=`echo $RELEASE_BODY | grep -Pzo '(?s){.*}' | jq -r '.boards[]? // .boards? // empty' | xargs echo -n 2>/dev/null`
VENDOR=`echo $RELEASE_BODY | grep -Pzo '(?s){.*}' | jq -r '.vendor? // empty' | xargs echo -n 2>/dev/null`
if ! [ -z "${BOARDS}" ]; then echo "Releasing board(s): $BOARDS" ; fi
if ! [ -z "${VENDOR}" ]; then echo "Setting packager: $VENDOR" ; fi
BOARDS=$(echo "$RELEASE_BODY" | grep -Pzo '(?s){.*}' | jq -r '.boards[]? // .boards? // empty' | xargs echo -n 2>/dev/null)
VENDOR=$(echo "$RELEASE_BODY" | grep -Pzo '(?s){.*}' | jq -r '.vendor? // empty' | xargs echo -n 2>/dev/null)
function get_file_size(){
if [ -n "${BOARDS}" ]; then
echo "Releasing board(s): $BOARDS"
fi
if [ -n "${VENDOR}" ]; then
echo "Setting packager: $VENDOR"
fi
function get_file_size {
local file="$1"
if [[ "$OSTYPE" == "darwin"* ]]; then
eval `stat -s "$file"`
eval "$(stat -s "$file")"
local res="$?"
echo "$st_size"
echo "${st_size:?}"
return $res
else
stat --printf="%s" "$file"
@ -54,23 +65,29 @@ function get_file_size(){
fi
}
function git_upload_asset(){
local name=$(basename "$1")
function git_upload_asset {
local name
name=$(basename "$1")
# local mime=$(file -b --mime-type "$1")
curl -k -X POST -sH "Authorization: token $GITHUB_TOKEN" -H "Content-Type: application/octet-stream" --data-binary @"$1" "https://uploads.github.com/repos/$GITHUB_REPOSITORY/releases/$RELEASE_ID/assets?name=$name"
}
function git_safe_upload_asset(){
function git_safe_upload_asset {
local file="$1"
local name=$(basename "$file")
local size=`get_file_size "$file"`
local upload_res=`git_upload_asset "$file"`
if [ $? -ne 0 ]; then
local name
local size
local upload_res
name=$(basename "$file")
size=$(get_file_size "$file")
if ! upload_res=$(git_upload_asset "$file"); then
>&2 echo "ERROR: Failed to upload '$name' ($?)"
return 1
fi
up_size=`echo "$upload_res" | jq -r '.size'`
if [ $up_size -ne $size ]; then
up_size=$(echo "$upload_res" | jq -r '.size')
if [ "$up_size" -ne "$size" ]; then
>&2 echo "ERROR: Uploaded size does not match! $up_size != $size"
#git_delete_asset
return 1
@ -79,7 +96,7 @@ function git_safe_upload_asset(){
return $?
}
function git_upload_to_pages(){
function git_upload_to_pages {
local path=$1
local src=$2
@ -88,41 +105,50 @@ function git_upload_to_pages(){
return 1
fi
local info=`curl -s -k -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.v3.object+json" -X GET "https://api.github.com/repos/$GITHUB_REPOSITORY/contents/$path?ref=gh-pages"`
local type=`echo "$info" | jq -r '.type'`
local message=$(basename $path)
local info
local type
local message
local sha=""
local content=""
if [ $type == "file" ]; then
sha=`echo "$info" | jq -r '.sha'`
info=$(curl -s -k -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.v3.object+json" -X GET "https://api.github.com/repos/$GITHUB_REPOSITORY/contents/$path?ref=gh-pages")
type=$(echo "$info" | jq -r '.type')
message=$(basename "$path")
if [ "$type" == "file" ]; then
sha=$(echo "$info" | jq -r '.sha')
sha=",\"sha\":\"$sha\""
message="Updating $message"
elif [ ! $type == "null" ]; then
elif [ ! "$type" == "null" ]; then
>&2 echo "Wrong type '$type'"
return 1
else
message="Creating $message"
fi
content=`base64 -i "$src"`
content=$(base64 -i "$src")
data="{\"branch\":\"gh-pages\",\"message\":\"$message\",\"content\":\"$content\"$sha}"
echo "$data" | curl -s -k -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.v3.raw+json" -X PUT --data @- "https://api.github.com/repos/$GITHUB_REPOSITORY/contents/$path"
}
function git_safe_upload_to_pages(){
function git_safe_upload_to_pages {
local path=$1
local file="$2"
local name=$(basename "$file")
local size=`get_file_size "$file"`
local upload_res=`git_upload_to_pages "$path" "$file"`
if [ $? -ne 0 ]; then
local name
local size
local upload_res
name=$(basename "$file")
size=$(get_file_size "$file")
if ! upload_res=$(git_upload_to_pages "$path" "$file"); then
>&2 echo "ERROR: Failed to upload '$name' ($?)"
return 1
fi
up_size=`echo "$upload_res" | jq -r '.content.size'`
if [ $up_size -ne $size ]; then
up_size=$(echo "$upload_res" | jq -r '.content.size')
if [ "$up_size" -ne "$size" ]; then
>&2 echo "ERROR: Uploaded size does not match! $up_size != $size"
#git_delete_asset
return 1
@ -131,15 +157,20 @@ function git_safe_upload_to_pages(){
return $?
}
function merge_package_json(){
function merge_package_json {
local jsonLink=$1
local jsonOut=$2
local old_json=$OUTPUT_DIR/oldJson.json
local merged_json=$OUTPUT_DIR/mergedJson.json
local error_code=0
echo "Downloading previous JSON $jsonLink ..."
curl -L -o "$old_json" "https://github.com/$GITHUB_REPOSITORY/releases/download/$jsonLink?access_token=$GITHUB_TOKEN" 2>/dev/null
if [ $? -ne 0 ]; then echo "ERROR: Download Failed! $?"; exit 1; fi
error_code=$?
if [ $error_code -ne 0 ]; then
echo "ERROR: Download Failed! $error_code"
exit 1
fi
echo "Creating new JSON ..."
set +e
@ -147,7 +178,7 @@ function merge_package_json(){
set -e
set -v
if [ ! -s $merged_json ]; then
if [ ! -s "$merged_json" ]; then
rm -f "$merged_json"
echo "Nothing to merge"
else
@ -188,9 +219,10 @@ else
done
# Copy only relevant variant files
mkdir "$PKG_DIR/variants/"
for variant in `cat ${PKG_DIR}/boards.txt | grep "\.variant=" | cut -d= -f2` ; do
board_list=$(cat "${PKG_DIR}"/boards.txt | grep "\.variant=" | cut -d= -f2)
while IFS= read -r variant; do
cp -Rf "$GITHUB_WORKSPACE/variants/${variant}" "$PKG_DIR/variants/"
done
done <<< "$board_list"
fi
cp -f "$GITHUB_WORKSPACE/CMakeLists.txt" "$PKG_DIR/"
cp -f "$GITHUB_WORKSPACE/idf_component.yml" "$PKG_DIR/"
@ -233,34 +265,36 @@ sed "s/{runtime\.platform\.path}.tools.xtensa-esp-elf/\\{runtime.tools.$X32TC_NE
sed 's/{runtime\.platform\.path}.tools.riscv32-esp-elf-gdb/\{runtime.tools.riscv32-esp-elf-gdb.path\}/g' | \
sed "s/{runtime\.platform\.path}.tools.riscv32-esp-elf/\\{runtime.tools.$RVTC_NEW_NAME.path\\}/g" | \
sed 's/{runtime\.platform\.path}.tools.esptool/\{runtime.tools.esptool_py.path\}/g' | \
sed 's/{runtime\.platform\.path}.tools.openocd-esp32/\{runtime.tools.openocd-esp32.path\}/g' \
> "$PKG_DIR/platform.txt"
sed 's/{runtime\.platform\.path}.tools.openocd-esp32/\{runtime.tools.openocd-esp32.path\}/g' > "$PKG_DIR/platform.txt"
if ! [ -z ${VENDOR} ]; then
if [ -n "${VENDOR}" ]; then
# Append vendor name to platform.txt to create a separate section
sed -i "/^name=.*/s/$/ ($VENDOR)/" "$PKG_DIR/platform.txt"
fi
# Add header with version information
echo "Generating core_version.h ..."
ver_define=`echo $RELEASE_TAG | tr "[:lower:].\055" "[:upper:]_"`
ver_hex=`git -C "$GITHUB_WORKSPACE" rev-parse --short=8 HEAD 2>/dev/null`
echo \#define ARDUINO_ESP32_GIT_VER 0x$ver_hex > "$PKG_DIR/cores/esp32/core_version.h"
echo \#define ARDUINO_ESP32_GIT_DESC `git -C "$GITHUB_WORKSPACE" describe --tags 2>/dev/null` >> "$PKG_DIR/cores/esp32/core_version.h"
echo \#define ARDUINO_ESP32_RELEASE_$ver_define >> "$PKG_DIR/cores/esp32/core_version.h"
echo \#define ARDUINO_ESP32_RELEASE \"$ver_define\" >> "$PKG_DIR/cores/esp32/core_version.h"
ver_define=$(echo "$RELEASE_TAG" | tr "[:lower:].\055" "[:upper:]_")
ver_hex=$(git -C "$GITHUB_WORKSPACE" rev-parse --short=8 HEAD 2>/dev/null)
echo \#define ARDUINO_ESP32_GIT_VER 0x"$ver_hex" > "$PKG_DIR/cores/esp32/core_version.h"
echo \#define ARDUINO_ESP32_GIT_DESC "$(git -C "$GITHUB_WORKSPACE" describe --tags 2>/dev/null)" >> "$PKG_DIR/cores/esp32/core_version.h"
echo \#define ARDUINO_ESP32_RELEASE_"$ver_define" >> "$PKG_DIR/cores/esp32/core_version.h"
echo \#define ARDUINO_ESP32_RELEASE \""$ver_define"\" >> "$PKG_DIR/cores/esp32/core_version.h"
# Compress package folder
echo "Creating ZIP ..."
pushd "$OUTPUT_DIR" >/dev/null
zip -qr "$PACKAGE_ZIP" "$PACKAGE_NAME"
if [ $? -ne 0 ]; then echo "ERROR: Failed to create $PACKAGE_ZIP ($?)"; exit 1; fi
if [ $? -ne 0 ]; then
echo "ERROR: Failed to create $PACKAGE_ZIP ($?)"
exit 1
fi
# Calculate SHA-256
echo "Calculating SHA sum ..."
PACKAGE_PATH="$OUTPUT_DIR/$PACKAGE_ZIP"
PACKAGE_SHA=`shasum -a 256 "$PACKAGE_ZIP" | cut -f 1 -d ' '`
PACKAGE_SIZE=`get_file_size "$PACKAGE_ZIP"`
PACKAGE_SHA=$(shasum -a 256 "$PACKAGE_ZIP" | cut -f 1 -d ' ')
PACKAGE_SIZE=$(get_file_size "$PACKAGE_ZIP")
popd >/dev/null
rm -rf "$PKG_DIR"
echo "'$PACKAGE_ZIP' Created! Size: $PACKAGE_SIZE, SHA-256: $PACKAGE_SHA"
@ -268,7 +302,7 @@ echo
# Upload package to release page
echo "Uploading package to release page ..."
PACKAGE_URL=`git_safe_upload_asset "$PACKAGE_PATH"`
PACKAGE_URL=$(git_safe_upload_asset "$PACKAGE_PATH")
echo "Package Uploaded"
echo "Download URL: $PACKAGE_URL"
echo
@ -276,9 +310,9 @@ echo
##
## TEMP WORKAROUND FOR RV32 LONG PATH ON WINDOWS
##
RVTC_VERSION=`cat $PACKAGE_JSON_TEMPLATE | jq -r ".packages[0].platforms[0].toolsDependencies[] | select(.name == \"$RVTC_NAME\") | .version" | cut -d '_' -f 2`
RVTC_VERSION=$(cat "$PACKAGE_JSON_TEMPLATE" | jq -r ".packages[0].platforms[0].toolsDependencies[] | select(.name == \"$RVTC_NAME\") | .version" | cut -d '_' -f 2)
# RVTC_VERSION=`date -j -f '%Y%m%d' "$RVTC_VERSION" '+%y%m'` # MacOS
RVTC_VERSION=`date -d "$RVTC_VERSION" '+%y%m'`
RVTC_VERSION=$(date -d "$RVTC_VERSION" '+%y%m')
rvtc_jq_arg="\
(.packages[0].platforms[0].toolsDependencies[] | select(.name==\"$RVTC_NAME\")).version = \"$RVTC_VERSION\" |\
(.packages[0].platforms[0].toolsDependencies[] | select(.name==\"$RVTC_NAME\")).name = \"$RVTC_NEW_NAME\" |\
@ -303,17 +337,20 @@ jq_arg=".packages[0].platforms[0].version = \"$RELEASE_TAG\" | \
.packages[0].platforms[0].checksum = \"SHA-256:$PACKAGE_SHA\""
# Generate package JSONs
echo "Genarating $PACKAGE_JSON_DEV ..."
echo "Generating $PACKAGE_JSON_DEV ..."
cat "$PACKAGE_JSON_TEMPLATE" | jq "$jq_arg" > "$OUTPUT_DIR/$PACKAGE_JSON_DEV"
if [ "$RELEASE_PRE" == "false" ]; then
echo "Genarating $PACKAGE_JSON_REL ..."
echo "Generating $PACKAGE_JSON_REL ..."
cat "$PACKAGE_JSON_TEMPLATE" | jq "$jq_arg" > "$OUTPUT_DIR/$PACKAGE_JSON_REL"
fi
# Figure out the last release or pre-release
echo "Getting previous releases ..."
releasesJson=`curl -sH "Authorization: token $GITHUB_TOKEN" "https://api.github.com/repos/$GITHUB_REPOSITORY/releases" 2>/dev/null`
if [ $? -ne 0 ]; then echo "ERROR: Get Releases Failed! ($?)"; exit 1; fi
releasesJson=$(curl -sH "Authorization: token $GITHUB_TOKEN" "https://api.github.com/repos/$GITHUB_REPOSITORY/releases" 2>/dev/null)
if [ $? -ne 0 ]; then
echo "ERROR: Get Releases Failed! ($?)"
exit 1
fi
set +e
prev_release=$(echo "$releasesJson" | jq -e -r ". | map(select(.draft == false and .prerelease == false)) | sort_by(.published_at | - fromdateiso8601) | .[0].tag_name")
@ -333,13 +370,13 @@ echo "Previous (any)release: $prev_any_release"
echo
# Merge package JSONs with previous releases
if [ ! -z "$prev_any_release" ] && [ "$prev_any_release" != "null" ]; then
if [ -n "$prev_any_release" ] && [ "$prev_any_release" != "null" ]; then
echo "Merging with JSON from $prev_any_release ..."
merge_package_json "$prev_any_release/$PACKAGE_JSON_DEV" "$OUTPUT_DIR/$PACKAGE_JSON_DEV"
fi
if [ "$RELEASE_PRE" == "false" ]; then
if [ ! -z "$prev_release" ] && [ "$prev_release" != "null" ]; then
if [ -n "$prev_release" ] && [ "$prev_release" != "null" ]; then
echo "Merging with JSON from $prev_release ..."
merge_package_json "$prev_release/$PACKAGE_JSON_REL" "$OUTPUT_DIR/$PACKAGE_JSON_REL"
fi
@ -349,21 +386,30 @@ fi
echo "Installing arduino-cli ..."
export PATH="/home/runner/bin:$PATH"
source ./.github/scripts/install-arduino-cli.sh
source "${SCRIPTS_DIR}/install-arduino-cli.sh"
echo "Testing $PACKAGE_JSON_DEV install ..."
echo "Installing esp32 ..."
arduino-cli core install esp32:esp32 --additional-urls "file://$OUTPUT_DIR/$PACKAGE_JSON_DEV"
if [ $? -ne 0 ]; then echo "ERROR: Failed to install esp32 ($?)"; exit 1; fi
if [ $? -ne 0 ]; then
echo "ERROR: Failed to install esp32 ($?)"
exit 1
fi
echo "Compiling example ..."
arduino-cli compile --fqbn esp32:esp32:esp32 $GITHUB_WORKSPACE/libraries/ESP32/examples/CI/CIBoardsTest/CIBoardsTest.ino
if [ $? -ne 0 ]; then echo "ERROR: Failed to compile example ($?)"; exit 1; fi
arduino-cli compile --fqbn esp32:esp32:esp32 "$GITHUB_WORKSPACE"/libraries/ESP32/examples/CI/CIBoardsTest/CIBoardsTest.ino
if [ $? -ne 0 ]; then
echo "ERROR: Failed to compile example ($?)"
exit 1
fi
echo "Uninstalling esp32 ..."
arduino-cli core uninstall esp32:esp32
if [ $? -ne 0 ]; then echo "ERROR: Failed to uninstall esp32 ($?)"; exit 1; fi
if [ $? -ne 0 ]; then
echo "ERROR: Failed to uninstall esp32 ($?)"
exit 1
fi
echo "Test successful!"
@ -372,15 +418,24 @@ if [ "$RELEASE_PRE" == "false" ]; then
echo "Installing esp32 ..."
arduino-cli core install esp32:esp32 --additional-urls "file://$OUTPUT_DIR/$PACKAGE_JSON_REL"
if [ $? -ne 0 ]; then echo "ERROR: Failed to install esp32 ($?)"; exit 1; fi
if [ $? -ne 0 ]; then
echo "ERROR: Failed to install esp32 ($?)"
exit 1
fi
echo "Compiling example ..."
arduino-cli compile --fqbn esp32:esp32:esp32 $GITHUB_WORKSPACE/libraries/ESP32/examples/CI/CIBoardsTest/CIBoardsTest.ino
if [ $? -ne 0 ]; then echo "ERROR: Failed to compile example ($?)"; exit 1; fi
arduino-cli compile --fqbn esp32:esp32:esp32 "$GITHUB_WORKSPACE"/libraries/ESP32/examples/CI/CIBoardsTest/CIBoardsTest.ino
if [ $? -ne 0 ]; then
echo "ERROR: Failed to compile example ($?)"
exit 1
fi
echo "Uninstalling esp32 ..."
arduino-cli core uninstall esp32:esp32
if [ $? -ne 0 ]; then echo "ERROR: Failed to uninstall esp32 ($?)"; exit 1; fi
if [ $? -ne 0 ]; then
echo "ERROR: Failed to uninstall esp32 ($?)"
exit 1
fi
echo "Test successful!"
fi
@ -388,13 +443,13 @@ fi
# Upload package JSONs
echo "Uploading $PACKAGE_JSON_DEV ..."
echo "Download URL: "`git_safe_upload_asset "$OUTPUT_DIR/$PACKAGE_JSON_DEV"`
echo "Pages URL: "`git_safe_upload_to_pages "$PACKAGE_JSON_DEV" "$OUTPUT_DIR/$PACKAGE_JSON_DEV"`
echo "Download URL: $(git_safe_upload_asset "$OUTPUT_DIR/$PACKAGE_JSON_DEV")"
echo "Pages URL: $(git_safe_upload_to_pages "$PACKAGE_JSON_DEV" "$OUTPUT_DIR/$PACKAGE_JSON_DEV")"
echo
if [ "$RELEASE_PRE" == "false" ]; then
echo "Uploading $PACKAGE_JSON_REL ..."
echo "Download URL: "`git_safe_upload_asset "$OUTPUT_DIR/$PACKAGE_JSON_REL"`
echo "Pages URL: "`git_safe_upload_to_pages "$PACKAGE_JSON_REL" "$OUTPUT_DIR/$PACKAGE_JSON_REL"`
echo "Download URL: $(git_safe_upload_asset "$OUTPUT_DIR/$PACKAGE_JSON_REL")"
echo "Pages URL: $(git_safe_upload_to_pages "$PACKAGE_JSON_REL" "$OUTPUT_DIR/$PACKAGE_JSON_REL")"
echo
fi

48
.github/scripts/set_push_chunks.sh vendored
View file

@ -11,23 +11,23 @@ elif [[ $LIB_CHANGED == 'true' ]]; then
echo "Libraries changed. Building only affected sketches."
if [[ $NETWORKING_CHANGED == 'true' ]]; then
echo "Networking libraries changed. Building networking related sketches."
networking_sketches="$(find libraries/WiFi -name *.ino) "
networking_sketches+="$(find libraries/Ethernet -name *.ino) "
networking_sketches+="$(find libraries/PPP -name *.ino) "
networking_sketches+="$(find libraries/NetworkClientSecure -name *.ino) "
networking_sketches+="$(find libraries/WebServer -name *.ino) "
networking_sketches="$(find libraries/WiFi -name '*.ino') "
networking_sketches+="$(find libraries/Ethernet -name '*.ino') "
networking_sketches+="$(find libraries/PPP -name '*.ino') "
networking_sketches+="$(find libraries/NetworkClientSecure -name '*.ino') "
networking_sketches+="$(find libraries/WebServer -name '*.ino') "
fi
if [[ $FS_CHANGED == 'true' ]]; then
echo "FS libraries changed. Building FS related sketches."
fs_sketches="$(find libraries/SD -name *.ino) "
fs_sketches+="$(find libraries/SD_MMC -name *.ino) "
fs_sketches+="$(find libraries/SPIFFS -name *.ino) "
fs_sketches+="$(find libraries/LittleFS -name *.ino) "
fs_sketches+="$(find libraries/FFat -name *.ino) "
fs_sketches="$(find libraries/SD -name '*.ino') "
fs_sketches+="$(find libraries/SD_MMC -name '*.ino') "
fs_sketches+="$(find libraries/SPIFFS -name '*.ino') "
fs_sketches+="$(find libraries/LittleFS -name '*.ino') "
fs_sketches+="$(find libraries/FFat -name '*.ino') "
fi
sketches="$networking_sketches $fs_sketches"
for file in $LIB_FILES; do
lib=$(echo $file | awk -F "/" '{print $1"/"$2}')
lib=$(echo "$file" | awk -F "/" '{print $1"/"$2}')
if [[ "$file" == *.ino ]]; then
# If file ends with .ino, add it to the list of sketches
echo "Sketch found: $file"
@ -36,14 +36,14 @@ elif [[ $LIB_CHANGED == 'true' ]]; then
# If file is inside the src directory, find all sketches in the lib/examples directory
echo "Library src file found: $file"
if [[ -d $lib/examples ]]; then
lib_sketches=$(find $lib/examples -name *.ino)
lib_sketches=$(find "$lib"/examples -name '*.ino')
sketches+="$lib_sketches "
echo "Library sketches: $lib_sketches"
fi
else
# If file is in a example folder but it is not a sketch, find all sketches in the current directory
echo "File in example folder found: $file"
sketch=$(find $(dirname $file) -name *.ino)
sketch=$(find "$(dirname "$file")" -name '*.ino')
sketches+="$sketch "
echo "Sketch in example folder: $sketch"
fi
@ -53,9 +53,9 @@ fi
if [[ -n $sketches ]]; then
# Remove duplicates
sketches=$(echo $sketches | tr ' ' '\n' | sort | uniq)
sketches=$(echo "$sketches" | tr ' ' '\n' | sort | uniq)
for sketch in $sketches; do
echo $sketch >> sketches_found.txt
echo "$sketch" >> sketches_found.txt
chunks_count=$((chunks_count+1))
done
echo "Number of sketches found: $chunks_count"
@ -69,15 +69,17 @@ if [[ -n $sketches ]]; then
fi
chunks='["0"'
for i in $(seq 1 $(( $chunks_count - 1 )) ); do
for i in $(seq 1 $(( chunks_count - 1 )) ); do
chunks+=",\"$i\""
done
chunks+="]"
echo "build_all=$build_all" >> $GITHUB_OUTPUT
echo "build_libraries=$BUILD_LIBRARIES" >> $GITHUB_OUTPUT
echo "build_static_sketches=$BUILD_STATIC_SKETCHES" >> $GITHUB_OUTPUT
echo "build_idf=$BUILD_IDF" >> $GITHUB_OUTPUT
echo "build_platformio=$BUILD_PLATFORMIO" >> $GITHUB_OUTPUT
echo "chunk_count=$chunks_count" >> $GITHUB_OUTPUT
echo "chunks=$chunks" >> $GITHUB_OUTPUT
{
echo "build_all=$build_all"
echo "build_libraries=$BUILD_LIBRARIES"
echo "build_static_sketches=$BUILD_STATIC_SKETCHES"
echo "build_idf=$BUILD_IDF"
echo "build_platformio=$BUILD_PLATFORMIO"
echo "chunk_count=$chunks_count"
echo "chunks=$chunks"
} >> "$GITHUB_OUTPUT"

215
.github/scripts/sketch_utils.sh vendored
View file

@ -8,10 +8,12 @@ else
SDKCONFIG_DIR="tools/esp32-arduino-libs"
fi
function check_requirements(){ # check_requirements <sketchdir> <sdkconfig_path>
function check_requirements { # check_requirements <sketchdir> <sdkconfig_path>
local sketchdir=$1
local sdkconfig_path=$2
local has_requirements=1
local requirements
local requirements_or
if [ ! -f "$sdkconfig_path" ] || [ ! -f "$sketchdir/ci.json" ]; then
echo "ERROR: sdkconfig or ci.json not found" 1>&2
@ -19,10 +21,10 @@ function check_requirements(){ # check_requirements <sketchdir> <sdkconfig_path>
# CI will fail and the user will know that the sketch has a problem.
else
# Check if the sketch requires any configuration options (AND)
local requirements=$(jq -r '.requires[]? // empty' "$sketchdir/ci.json")
requirements=$(jq -r '.requires[]? // empty' "$sketchdir/ci.json")
if [[ "$requirements" != "null" && "$requirements" != "" ]]; then
for requirement in $requirements; do
requirement=$(echo $requirement | xargs)
requirement=$(echo "$requirement" | xargs)
found_line=$(grep -E "^$requirement" "$sdkconfig_path")
if [[ "$found_line" == "" ]]; then
has_requirements=0
@ -31,11 +33,11 @@ function check_requirements(){ # check_requirements <sketchdir> <sdkconfig_path>
fi
# Check if the sketch requires any configuration options (OR)
local requirements_or=$(jq -r '.requires_any[]? // empty' "$sketchdir/ci.json")
requirements_or=$(jq -r '.requires_any[]? // empty' "$sketchdir/ci.json")
if [[ "$requirements_or" != "null" && "$requirements_or" != "" ]]; then
local found=false
for requirement in $requirements_or; do
requirement=$(echo $requirement | xargs)
requirement=$(echo "$requirement" | xargs)
found_line=$(grep -E "^$requirement" "$sdkconfig_path")
if [[ "$found_line" != "" ]]; then
found=true
@ -51,8 +53,8 @@ function check_requirements(){ # check_requirements <sketchdir> <sdkconfig_path>
echo $has_requirements
}
function build_sketch(){ # build_sketch <ide_path> <user_path> <path-to-ino> [extra-options]
while [ ! -z "$1" ]; do
function build_sketch { # build_sketch <ide_path> <user_path> <path-to-ino> [extra-options]
while [ -n "$1" ]; do
case "$1" in
-ai )
shift
@ -97,10 +99,10 @@ function build_sketch(){ # build_sketch <ide_path> <user_path> <path-to-ino> [ex
shift
done
xtra_opts=$*
xtra_opts=("$@")
len=0
if [ -z $sketchdir ]; then
if [ -z "$sketchdir" ]; then
echo "ERROR: Sketch directory not provided"
echo "$USAGE"
exit 1
@ -108,8 +110,8 @@ function build_sketch(){ # build_sketch <ide_path> <user_path> <path-to-ino> [ex
# No FQBN was passed, try to get it from other options
if [ -z $fqbn ]; then
if [ -z $target ]; then
if [ -z "$fqbn" ]; then
if [ -z "$target" ]; then
echo "ERROR: Unspecified chip"
echo "$USAGE"
exit 1
@ -120,25 +122,25 @@ function build_sketch(){ # build_sketch <ide_path> <user_path> <path-to-ino> [ex
# precedence. Note that the following logic also falls to the default
# parameters if no arguments were passed and no file was found.
if [ -z $options ] && [ -f $sketchdir/ci.json ]; then
if [ -z "$options" ] && [ -f "$sketchdir"/ci.json ]; then
# The config file could contain multiple FQBNs for one chip. If
# that's the case we build one time for every FQBN.
len=`jq -r --arg target $target '.fqbn[$target] | length' $sketchdir/ci.json`
if [ $len -gt 0 ]; then
fqbn=`jq -r --arg target $target '.fqbn[$target] | sort' $sketchdir/ci.json`
len=$(jq -r --arg target "$target" '.fqbn[$target] | length' "$sketchdir"/ci.json)
if [ "$len" -gt 0 ]; then
fqbn=$(jq -r --arg target "$target" '.fqbn[$target] | sort' "$sketchdir"/ci.json)
fi
fi
if [ ! -z $options ] || [ $len -eq 0 ]; then
if [ -n "$options" ] || [ "$len" -eq 0 ]; then
# Since we are passing options, we will end up with only one FQBN to
# build.
len=1
if [ -f $sketchdir/ci.json ]; then
fqbn_append=`jq -r '.fqbn_append' $sketchdir/ci.json`
if [ $fqbn_append == "null" ]; then
if [ -f "$sketchdir"/ci.json ]; then
fqbn_append=$(jq -r '.fqbn_append' "$sketchdir"/ci.json)
if [ "$fqbn_append" == "null" ]; then
fqbn_append=""
fi
fi
@ -189,6 +191,10 @@ function build_sketch(){ # build_sketch <ide_path> <user_path> <path-to-ino> [ex
[ -n "${options:-$esp32p4_opts}" ] && opt=":${options:-$esp32p4_opts}"
fqbn="espressif:esp32:esp32p4$opt"
;;
*)
echo "ERROR: Invalid chip: $target"
exit 1
;;
esac
# Make it look like a JSON array.
@ -207,7 +213,7 @@ function build_sketch(){ # build_sketch <ide_path> <user_path> <path-to-ino> [ex
exit 1
fi
# The directory that will hold all the artifcats (the build directory) is
# The directory that will hold all the artifacts (the build directory) is
# provided through:
# 1. An env variable called ARDUINO_BUILD_DIR.
# 2. Created at the sketch level as "build" in the case of a single
@ -215,17 +221,18 @@ function build_sketch(){ # build_sketch <ide_path> <user_path> <path-to-ino> [ex
# 3. Created at the sketch level as "buildX" where X is the number
# of configuration built in case of a multiconfiguration test.
sketchname=$(basename $sketchdir)
sketchname=$(basename "$sketchdir")
local has_requirements
if [ -f $sketchdir/ci.json ]; then
if [ -f "$sketchdir"/ci.json ]; then
# If the target is listed as false, skip the sketch. Otherwise, include it.
is_target=$(jq -r --arg target $target '.targets[$target]' $sketchdir/ci.json)
is_target=$(jq -r --arg target "$target" '.targets[$target]' "$sketchdir"/ci.json)
if [[ "$is_target" == "false" ]]; then
echo "Skipping $sketchname for target $target"
exit 0
fi
local has_requirements=$(check_requirements "$sketchdir" "$SDKCONFIG_DIR/$target/sdkconfig")
has_requirements=$(check_requirements "$sketchdir" "$SDKCONFIG_DIR/$target/sdkconfig")
if [ "$has_requirements" == "0" ]; then
echo "Target $target does not meet the requirements for $sketchname. Skipping."
exit 0
@ -235,7 +242,7 @@ function build_sketch(){ # build_sketch <ide_path> <user_path> <path-to-ino> [ex
ARDUINO_CACHE_DIR="$HOME/.arduino/cache.tmp"
if [ -n "$ARDUINO_BUILD_DIR" ]; then
build_dir="$ARDUINO_BUILD_DIR"
elif [ $len -eq 1 ]; then
elif [ "$len" -eq 1 ]; then
# build_dir="$sketchdir/build"
build_dir="$HOME/.arduino/tests/$sketchname/build.tmp"
fi
@ -244,51 +251,49 @@ function build_sketch(){ # build_sketch <ide_path> <user_path> <path-to-ino> [ex
sizes_file="$GITHUB_WORKSPACE/cli_compile_$chunk_index.json"
mkdir -p "$ARDUINO_CACHE_DIR"
for i in `seq 0 $(($len - 1))`
do
if [ $len -ne 1 ]; then
# build_dir="$sketchdir/build$i"
build_dir="$HOME/.arduino/tests/$sketchname/build$i.tmp"
for i in $(seq 0 $((len - 1))); do
if [ "$len" -ne 1 ]; then
# build_dir="$sketchdir/build$i"
build_dir="$HOME/.arduino/tests/$sketchname/build$i.tmp"
fi
rm -rf $build_dir
mkdir -p $build_dir
rm -rf "$build_dir"
mkdir -p "$build_dir"
currfqbn=`echo $fqbn | jq -r --argjson i $i '.[$i]'`
currfqbn=$(echo "$fqbn" | jq -r --argjson i "$i" '.[$i]')
if [ -f "$ide_path/arduino-cli" ]; then
echo "Building $sketchname with arduino-cli and FQBN=$currfqbn"
curroptions=`echo "$currfqbn" | cut -d':' -f4`
currfqbn=`echo "$currfqbn" | cut -d':' -f1-3`
$ide_path/arduino-cli compile \
curroptions=$(echo "$currfqbn" | cut -d':' -f4)
currfqbn=$(echo "$currfqbn" | cut -d':' -f1-3)
"$ide_path"/arduino-cli compile \
--fqbn "$currfqbn" \
--board-options "$curroptions" \
--warnings "all" \
--build-property "compiler.warning_flags.all=-Wall -Werror=all -Wextra" \
--build-cache-path "$ARDUINO_CACHE_DIR" \
--build-path "$build_dir" \
$xtra_opts "${sketchdir}" \
2>&1 | tee $output_file
"${xtra_opts[@]}" "${sketchdir}" \
2>&1 | tee "$output_file"
exit_status=${PIPESTATUS[0]}
if [ $exit_status -ne 0 ]; then
if [ "$exit_status" -ne 0 ]; then
echo "ERROR: Compilation failed with error code $exit_status"
exit $exit_status
exit "$exit_status"
fi
if [ $log_compilation ]; then
if [ -n "$log_compilation" ]; then
#Extract the program storage space and dynamic memory usage in bytes and percentage in separate variables from the output, just the value without the string
flash_bytes=$(grep -oE 'Sketch uses ([0-9]+) bytes' $output_file | awk '{print $3}')
flash_percentage=$(grep -oE 'Sketch uses ([0-9]+) bytes \(([0-9]+)%\)' $output_file | awk '{print $5}' | tr -d '(%)')
ram_bytes=$(grep -oE 'Global variables use ([0-9]+) bytes' $output_file | awk '{print $4}')
ram_percentage=$(grep -oE 'Global variables use ([0-9]+) bytes \(([0-9]+)%\)' $output_file | awk '{print $6}' | tr -d '(%)')
flash_bytes=$(grep -oE 'Sketch uses ([0-9]+) bytes' "$output_file" | awk '{print $3}')
flash_percentage=$(grep -oE 'Sketch uses ([0-9]+) bytes \(([0-9]+)%\)' "$output_file" | awk '{print $5}' | tr -d '(%)')
ram_bytes=$(grep -oE 'Global variables use ([0-9]+) bytes' "$output_file" | awk '{print $4}')
ram_percentage=$(grep -oE 'Global variables use ([0-9]+) bytes \(([0-9]+)%\)' "$output_file" | awk '{print $6}' | tr -d '(%)')
# Extract the directory path excluding the filename
directory_path=$(dirname "$sketch")
# Define the constant part
constant_part="/home/runner/Arduino/hardware/espressif/esp32/libraries/"
# Extract the desired substring using sed
lib_sketch_name=$(echo "$directory_path" | sed "s|$constant_part||")
# Extract the desired substring
lib_sketch_name="${directory_path#"$constant_part"}"
#append json file where key is fqbn, sketch name, sizes -> extracted values
echo "{\"name\": \"$lib_sketch_name\",
\"sizes\": [{
@ -304,15 +309,15 @@ function build_sketch(){ # build_sketch <ide_path> <user_path> <path-to-ino> [ex
echo "Building $sketchname with arduino-builder and FQBN=$currfqbn"
echo "Build path = $build_dir"
$ide_path/arduino-builder -compile -logger=human -core-api-version=10810 \
-fqbn=\"$currfqbn\" \
"$ide_path"/arduino-builder -compile -logger=human -core-api-version=10810 \
-fqbn=\""$currfqbn"\" \
-warnings="all" \
-tools "$ide_path/tools-builder" \
-hardware "$user_path/hardware" \
-libraries "$user_path/libraries" \
-build-cache "$ARDUINO_CACHE_DIR" \
-build-path "$build_dir" \
$xtra_opts "${sketchdir}/${sketchname}.ino"
"${xtra_opts[@]}" "${sketchdir}/${sketchname}.ino"
exit_status=$?
if [ $exit_status -ne 0 ]; then
@ -339,15 +344,16 @@ function build_sketch(){ # build_sketch <ide_path> <user_path> <path-to-ino> [ex
unset options
}
function count_sketches(){ # count_sketches <path> [target] [file] [ignore-requirements]
function count_sketches { # count_sketches <path> [target] [file] [ignore-requirements]
local path=$1
local target=$2
local ignore_requirements=$3
local file=$4
local sketches
if [ $# -lt 1 ]; then
echo "ERROR: Illegal number of parameters"
echo "USAGE: ${0} count <path> [target]"
echo "ERROR: Illegal number of parameters"
echo "USAGE: ${0} count <path> [target]"
fi
rm -rf sketches.txt
@ -357,42 +363,47 @@ function count_sketches(){ # count_sketches <path> [target] [file] [ignore-requi
fi
if [ -f "$file" ]; then
local sketches=$(cat $file)
sketches=$(cat "$file")
else
local sketches=$(find $path -name *.ino | sort)
sketches=$(find "$path" -name '*.ino' | sort)
fi
local sketchnum=0
for sketch in $sketches; do
local sketchdir=$(dirname $sketch)
local sketchdirname=$(basename $sketchdir)
local sketchname=$(basename $sketch)
local sketchdir
local sketchdirname
local sketchname
local has_requirements
sketchdir=$(dirname "$sketch")
sketchdirname=$(basename "$sketchdir")
sketchname=$(basename "$sketch")
if [[ "$sketchdirname.ino" != "$sketchname" ]]; then
continue
elif [[ -n $target ]] && [[ -f $sketchdir/ci.json ]]; then
# If the target is listed as false, skip the sketch. Otherwise, include it.
is_target=$(jq -r --arg target $target '.targets[$target]' $sketchdir/ci.json)
is_target=$(jq -r --arg target "$target" '.targets[$target]' "$sketchdir"/ci.json)
if [[ "$is_target" == "false" ]]; then
continue
fi
if [ "$ignore_requirements" != "1" ]; then
local has_requirements=$(check_requirements "$sketchdir" "$SDKCONFIG_DIR/$target/sdkconfig")
has_requirements=$(check_requirements "$sketchdir" "$SDKCONFIG_DIR/$target/sdkconfig")
if [ "$has_requirements" == "0" ]; then
continue
fi
fi
fi
echo $sketch >> sketches.txt
sketchnum=$(($sketchnum + 1))
echo "$sketch" >> sketches.txt
sketchnum=$((sketchnum + 1))
done
return $sketchnum
}
function build_sketches(){ # build_sketches <ide_path> <user_path> <target> <path> <chunk> <total-chunks> [extra-options]
local args=""
while [ ! -z "$1" ]; do
function build_sketches { # build_sketches <ide_path> <user_path> <target> <path> <chunk> <total-chunks> [extra-options]
local args=()
while [ -n "$1" ]; do
case $1 in
-ai )
shift
@ -405,12 +416,12 @@ function build_sketches(){ # build_sketches <ide_path> <user_path> <target> <pat
-t )
shift
target=$1
args+=" -t $target"
args+=("-t" "$target")
;;
-fqbn )
shift
fqbn=$1
args+=" -fqbn $fqbn"
args+=("-fqbn" "$fqbn")
;;
-p )
shift
@ -439,10 +450,10 @@ function build_sketches(){ # build_sketches <ide_path> <user_path> <target> <pat
shift
done
local xtra_opts=$*
local xtra_opts=("$@")
if [ -z "$chunk_index" ] || [ -z "$chunk_max" ]; then
echo "ERROR: Invalid chunk paramters"
echo "ERROR: Invalid chunk parameters"
echo "$USAGE"
exit 1
fi
@ -465,13 +476,16 @@ function build_sketches(){ # build_sketches <ide_path> <user_path> <target> <pat
local sketchcount=$?
fi
set -e
local sketches=$(cat sketches.txt)
local sketches
sketches=$(cat sketches.txt)
rm -rf sketches.txt
local chunk_size=$(( $sketchcount / $chunk_max ))
local all_chunks=$(( $chunk_max * $chunk_size ))
local chunk_size
local all_chunks
chunk_size=$(( sketchcount / chunk_max ))
all_chunks=$(( chunk_max * chunk_size ))
if [ "$all_chunks" -lt "$sketchcount" ]; then
chunk_size=$(( $chunk_size + 1 ))
chunk_size=$(( chunk_size + 1 ))
fi
local start_index=0
@ -480,19 +494,20 @@ function build_sketches(){ # build_sketches <ide_path> <user_path> <target> <pat
start_index=$chunk_index
end_index=$sketchcount
else
start_index=$(( $chunk_index * $chunk_size ))
start_index=$(( chunk_index * chunk_size ))
if [ "$sketchcount" -le "$start_index" ]; then
echo "No sketches to build for $target in this chunk"
return 0
fi
end_index=$(( $(( $chunk_index + 1 )) * $chunk_size ))
end_index=$(( $(( chunk_index + 1 )) * chunk_size ))
if [ "$end_index" -gt "$sketchcount" ]; then
end_index=$sketchcount
fi
fi
local start_num=$(( $start_index + 1 ))
local start_num
start_num=$(( start_index + 1 ))
echo "Found $sketchcount Sketches for target '$target'";
echo "Chunk Index : $chunk_index"
echo "Chunk Count : $chunk_max"
@ -501,14 +516,14 @@ function build_sketches(){ # build_sketches <ide_path> <user_path> <target> <pat
echo "End Sketch : $end_index"
#if fqbn is not passed then set it to default for compilation log
if [ -z $fqbn ]; then
if [ -z "$fqbn" ]; then
log_fqbn="espressif:esp32:$target"
else
log_fqbn=$fqbn
fi
sizes_file="$GITHUB_WORKSPACE/cli_compile_$chunk_index.json"
if [ $log_compilation ]; then
if [ -n "$log_compilation" ]; then
#echo board,target and start of sketches to sizes_file json
echo "{ \"board\": \"$log_fqbn\",
\"target\": \"$target\",
@ -516,30 +531,34 @@ function build_sketches(){ # build_sketches <ide_path> <user_path> <target> <pat
fi
local sketchnum=0
args+=" -ai $ide_path -au $user_path -i $chunk_index"
if [ $log_compilation ]; then
args+=" -l $log_compilation"
args+=("-ai" "$ide_path" "-au" "$user_path" "-i" "$chunk_index")
if [ -n "$log_compilation" ]; then
args+=("-l" "$log_compilation")
fi
for sketch in $sketches; do
local sketchdir=$(dirname $sketch)
local sketchdirname=$(basename $sketchdir)
sketchnum=$(($sketchnum + 1))
local sketchdir
local sketchdirname
sketchdir=$(dirname "$sketch")
sketchdirname=$(basename "$sketchdir")
sketchnum=$((sketchnum + 1))
if [ "$sketchnum" -le "$start_index" ] \
|| [ "$sketchnum" -gt "$end_index" ]; then
continue
fi
echo ""
echo "Building Sketch Index $sketchnum - $sketchdirname"
build_sketch $args -s $sketchdir $xtra_opts
build_sketch "${args[@]}" -s "$sketchdir" "${xtra_opts[@]}"
local result=$?
if [ $result -ne 0 ]; then
return $result
fi
done
if [ $log_compilation ]; then
if [ -n "$log_compilation" ]; then
#remove last comma from json
if [ $i -eq $(($len - 1)) ]; then
if [ "$i" -eq $((len - 1)) ]; then
sed -i '$ s/.$//' "$sizes_file"
fi
#echo end of sketches sizes_file json
@ -554,28 +573,28 @@ function build_sketches(){ # build_sketches <ide_path> <user_path> <target> <pat
USAGE="
USAGE: ${0} [command] [options]
Available commands:
count: Count sketches.
build: Build a sketch.
chunk_build: Build a chunk of sketches.
check_requirements: Check if target meets sketch requirements.
count: Count sketches.
build: Build a sketch.
chunk_build: Build a chunk of sketches.
check_requirements: Check if target meets sketch requirements.
"
cmd=$1
shift
if [ -z $cmd ]; then
if [ -z "$cmd" ]; then
echo "ERROR: No command supplied"
echo "$USAGE"
exit 2
fi
case "$cmd" in
"count") count_sketches $*
"count") count_sketches "$@"
;;
"build") build_sketch $*
"build") build_sketch "$@"
;;
"chunk_build") build_sketches $*
"chunk_build") build_sketches "$@"
;;
"check_requirements") check_requirements $*
"check_requirements") check_requirements "$@"
;;
*)
echo "ERROR: Unrecognized command"

33
.github/scripts/tests_build.sh vendored
View file

@ -3,14 +3,14 @@
USAGE="
USAGE:
${0} -c -type <test_type> <chunk_build_opts>
Example: ${0} -c -type validation -t esp32 -i 0 -m 15
Example: ${0} -c -type validation -t esp32 -i 0 -m 15
${0} -s sketch_name <build_opts>
Example: ${0} -s hello_world -t esp32
Example: ${0} -s hello_world -t esp32
${0} -clean
Remove build and test generated files
Remove build and test generated files
"
function clean(){
function clean {
rm -rf tests/.pytest_cache
find tests/ -type d -name 'build*' -exec rm -rf "{}" \+
find tests/ -type d -name '__pycache__' -exec rm -rf "{}" \+
@ -23,7 +23,7 @@ BUILD_CMD=""
chunk_build=0
while [ ! -z "$1" ]; do
while [ -n "$1" ]; do
case $1 in
-c )
chunk_build=1
@ -45,25 +45,25 @@ while [ ! -z "$1" ]; do
exit 0
;;
* )
break
;;
break
;;
esac
shift
done
source ${SCRIPTS_DIR}/install-arduino-cli.sh
source ${SCRIPTS_DIR}/install-arduino-core-esp32.sh
source "${SCRIPTS_DIR}/install-arduino-cli.sh"
source "${SCRIPTS_DIR}/install-arduino-core-esp32.sh"
args="-ai $ARDUINO_IDE_PATH -au $ARDUINO_USR_PATH"
args=("-ai" "$ARDUINO_IDE_PATH" "-au" "$ARDUINO_USR_PATH")
if [[ $test_type == "all" ]] || [[ -z $test_type ]]; then
if [ -n "$sketch" ]; then
tmp_sketch_path=$(find tests -name $sketch.ino)
test_type=$(basename $(dirname $(dirname "$tmp_sketch_path")))
tmp_sketch_path=$(find tests -name "$sketch".ino)
test_type=$(basename "$(dirname "$(dirname "$tmp_sketch_path")")")
echo "Sketch $sketch test type: $test_type"
test_folder="$PWD/tests/$test_type"
else
test_folder="$PWD/tests"
test_folder="$PWD/tests"
fi
else
test_folder="$PWD/tests/$test_type"
@ -71,11 +71,10 @@ fi
if [ $chunk_build -eq 1 ]; then
BUILD_CMD="${SCRIPTS_DIR}/sketch_utils.sh chunk_build"
args+=" -p $test_folder -i 0 -m 1"
args+=("-p" "$test_folder" "-i" "0" "-m" "1")
else
BUILD_CMD="${SCRIPTS_DIR}/sketch_utils.sh build"
args+=" -s $test_folder/$sketch"
args+=("-s" "$test_folder/$sketch")
fi
${BUILD_CMD} ${args} $*
${BUILD_CMD} "${args[@]}" "$@"

20
.github/scripts/tests_matrix.sh vendored
View file

@ -6,10 +6,10 @@ wokwi_types="'validation'"
qemu_types="'validation'"
if [[ $IS_PR != 'true' ]] || [[ $PERFORMANCE_ENABLED == 'true' ]]; then
build_types+=",'performance'"
hw_types+=",'performance'"
#wokwi_types+=",'performance'"
#qemu_types+=",'performance'"
build_types+=",'performance'"
hw_types+=",'performance'"
#wokwi_types+=",'performance'"
#qemu_types+=",'performance'"
fi
targets="'esp32','esp32s2','esp32s3','esp32c3','esp32c6','esp32h2','esp32p4'"
@ -19,8 +19,10 @@ mkdir -p info
echo "[$wokwi_types]" > info/wokwi_types.txt
echo "[$targets]" > info/targets.txt
echo "build-types=[$build_types]" >> $GITHUB_OUTPUT
echo "hw-types=[$hw_types]" >> $GITHUB_OUTPUT
echo "wokwi-types=[$wokwi_types]" >> $GITHUB_OUTPUT
echo "qemu-types=[$qemu_types]" >> $GITHUB_OUTPUT
echo "targets=[$targets]" >> $GITHUB_OUTPUT
{
echo "build-types=[$build_types]"
echo "hw-types=[$hw_types]"
echo "wokwi-types=[$wokwi_types]"
echo "qemu-types=[$qemu_types]"
echo "targets=[$targets]"
} >> "$GITHUB_OUTPUT"

223
.github/scripts/tests_run.sh vendored
View file

@ -1,126 +1,130 @@
#!/bin/bash
function run_test() {
function run_test {
local target=$1
local sketch=$2
local options=$3
local erase_flash=$4
local sketchdir=$(dirname $sketch)
local sketchname=$(basename $sketchdir)
local sketchdir
local sketchname
local result=0
local error=0
local sdkconfig_path
local extra_args
if [ $options -eq 0 ] && [ -f $sketchdir/ci.json ]; then
len=`jq -r --arg target $target '.fqbn[$target] | length' $sketchdir/ci.json`
if [ $len -eq 0 ]; then
sketchdir=$(dirname "$sketch")
sketchname=$(basename "$sketchdir")
if [ "$options" -eq 0 ] && [ -f "$sketchdir"/ci.json ]; then
len=$(jq -r --arg target "$target" '.fqbn[$target] | length' "$sketchdir"/ci.json)
if [ "$len" -eq 0 ]; then
len=1
fi
else
len=1
fi
if [ $len -eq 1 ]; then
if [ "$len" -eq 1 ]; then
sdkconfig_path="$HOME/.arduino/tests/$sketchname/build.tmp/sdkconfig"
else
sdkconfig_path="$HOME/.arduino/tests/$sketchname/build0.tmp/sdkconfig"
fi
if [ -f $sketchdir/ci.json ]; then
if [ -f "$sketchdir"/ci.json ]; then
# If the target or platform is listed as false, skip the sketch. Otherwise, include it.
is_target=$(jq -r --arg target $target '.targets[$target]' $sketchdir/ci.json)
selected_platform=$(jq -r --arg platform $platform '.platforms[$platform]' $sketchdir/ci.json)
is_target=$(jq -r --arg target "$target" '.targets[$target]' "$sketchdir"/ci.json)
selected_platform=$(jq -r --arg platform "$platform" '.platforms[$platform]' "$sketchdir"/ci.json)
if [[ $is_target == "false" ]] || [[ $selected_platform == "false" ]]; then
printf "\033[93mSkipping $sketchname test for $target, platform: $platform\033[0m\n"
printf "\033[93mSkipping %s test for %s, platform: %s\033[0m\n" "$sketchname" "$target" "$platform"
printf "\n\n\n"
return 0
fi
fi
if [ ! -f $sdkconfig_path ]; then
printf "\033[93mSketch $sketchname not built\nMight be due to missing target requirements or build failure\033[0m\n"
if [ ! -f "$sdkconfig_path" ]; then
printf "\033[93mSketch %s not built\nMight be due to missing target requirements or build failure\033[0m\n" "$sketchname"
printf "\n\n\n"
return 0
fi
local right_target=$(grep -E "^CONFIG_IDF_TARGET=\"$target\"$" "$sdkconfig_path")
local right_target
right_target=$(grep -E "^CONFIG_IDF_TARGET=\"$target\"$" "$sdkconfig_path")
if [ -z "$right_target" ]; then
printf "\033[91mError: Sketch $sketchname compiled for different target\n\033[0m\n"
printf "\033[91mError: Sketch %s compiled for different target\n\033[0m\n" "$sketchname"
printf "\n\n\n"
return 1
fi
if [ $len -eq 1 ]; then
# build_dir="$sketchdir/build"
build_dir="$HOME/.arduino/tests/$sketchname/build.tmp"
report_file="$sketchdir/$target/$sketchname.xml"
if [ "$len" -eq 1 ]; then
# build_dir="$sketchdir/build"
build_dir="$HOME/.arduino/tests/$sketchname/build.tmp"
report_file="$sketchdir/$target/$sketchname.xml"
fi
for i in `seq 0 $(($len - 1))`
do
for i in $(seq 0 $((len - 1))); do
fqbn="Default"
if [ $len -ne 1 ]; then
fqbn=`jq -r --arg target $target --argjson i $i '.fqbn[$target] | sort | .[$i]' $sketchdir/ci.json`
elif [ -f $sketchdir/ci.json ]; then
has_fqbn=`jq -r --arg target $target '.fqbn[$target]' $sketchdir/ci.json`
if [ "$len" -ne 1 ]; then
fqbn=$(jq -r --arg target "$target" --argjson i "$i" '.fqbn[$target] | sort | .[$i]' "$sketchdir"/ci.json)
elif [ -f "$sketchdir"/ci.json ]; then
has_fqbn=$(jq -r --arg target "$target" '.fqbn[$target]' "$sketchdir"/ci.json)
if [ "$has_fqbn" != "null" ]; then
fqbn=`jq -r --arg target $target '.fqbn[$target] | .[0]' $sketchdir/ci.json`
fqbn=$(jq -r --arg target "$target" '.fqbn[$target] | .[0]' "$sketchdir"/ci.json)
fi
fi
printf "\033[95mRunning test: $sketchname -- Config: $fqbn\033[0m\n"
if [ $erase_flash -eq 1 ]; then
esptool.py -c $target erase_flash
printf "\033[95mRunning test: %s -- Config: %s\033[0m\n" "$sketchname" "$fqbn"
if [ "$erase_flash" -eq 1 ]; then
esptool.py -c "$target" erase_flash
fi
if [ $len -ne 1 ]; then
if [ "$len" -ne 1 ]; then
# build_dir="$sketchdir/build$i"
build_dir="$HOME/.arduino/tests/$sketchname/build$i.tmp"
report_file="$sketchdir/$target/$sketchname$i.xml"
fi
if [ $platform == "wokwi" ]; then
extra_args="--target $target --embedded-services arduino,wokwi --wokwi-timeout=$wokwi_timeout"
extra_args=("--target" "$target" "--embedded-services" "arduino,wokwi" "--wokwi-timeout=$wokwi_timeout")
if [[ -f "$sketchdir/scenario.yaml" ]]; then
extra_args+=" --wokwi-scenario $sketchdir/scenario.yaml"
extra_args+=("--wokwi-scenario" "$sketchdir/scenario.yaml")
fi
if [[ -f "$sketchdir/diagram.$target.json" ]]; then
extra_args+=" --wokwi-diagram $sketchdir/diagram.$target.json"
extra_args+=("--wokwi-diagram" "$sketchdir/diagram.$target.json")
fi
elif [ $platform == "qemu" ]; then
PATH=$HOME/qemu/bin:$PATH
extra_args="--embedded-services qemu --qemu-image-path $build_dir/$sketchname.ino.merged.bin"
extra_args=("--embedded-services" "qemu" "--qemu-image-path" "$build_dir/$sketchname.ino.merged.bin")
if [ $target == "esp32" ] || [ $target == "esp32s3" ]; then
extra_args+=" --qemu-prog-path qemu-system-xtensa --qemu-cli-args=\"-machine $target -m 4M -nographic\""
elif [ $target == "esp32c3" ]; then
extra_args+=" --qemu-prog-path qemu-system-riscv32 --qemu-cli-args=\"-machine $target -icount 3 -nographic\""
if [ "$target" == "esp32" ] || [ "$target" == "esp32s3" ]; then
extra_args+=("--qemu-prog-path" "qemu-system-xtensa" "--qemu-cli-args=\"-machine $target -m 4M -nographic\"")
elif [ "$target" == "esp32c3" ]; then
extra_args+=("--qemu-prog-path" "qemu-system-riscv32" "--qemu-cli-args=\"-machine $target -icount 3 -nographic\"")
else
printf "\033[91mUnsupported QEMU target: $target\033[0m\n"
printf "\033[91mUnsupported QEMU target: %s\033[0m\n" "$target"
exit 1
fi
else
extra_args="--embedded-services esp,arduino"
extra_args=("--embedded-services" "esp,arduino")
fi
rm $sketchdir/diagram.json 2>/dev/null || true
rm "$sketchdir"/diagram.json 2>/dev/null || true
result=0
printf "\033[95mpytest $sketchdir/test_$sketchname.py --build-dir $build_dir --junit-xml=$report_file $extra_args\033[0m\n"
bash -c "set +e; pytest $sketchdir/test_$sketchname.py --build-dir $build_dir --junit-xml=$report_file $extra_args; exit \$?" || result=$?
printf "\033[95mpytest \"%s/test_%s.py\" --build-dir \"%s\" --junit-xml=\"%s\" %s\033[0m\n" "$sketchdir" "$sketchname" "$build_dir" "$report_file" "${extra_args[*]@Q}"
bash -c "set +e; pytest \"$sketchdir/test_$sketchname.py\" --build-dir \"$build_dir\" --junit-xml=\"$report_file\" ${extra_args[*]@Q}; exit \$?" || result=$?
printf "\n"
if [ $result -ne 0 ]; then
result=0
printf "\033[95mRetrying test: $sketchname -- Config: $i\033[0m\n"
printf "\033[95mpytest $sketchdir/test_$sketchname.py --build-dir $build_dir --junit-xml=$report_file $extra_args\033[0m\n"
bash -c "set +e; pytest $sketchdir/test_$sketchname.py --build-dir $build_dir --junit-xml=$report_file $extra_args; exit \$?" || result=$?
printf "\033[95mRetrying test: %s -- Config: %s\033[0m\n" "$sketchname" "$i"
printf "\033[95mpytest \"%s/test_%s.py\" --build-dir \"%s\" --junit-xml=\"%s\" %s\033[0m\n" "$sketchdir" "$sketchname" "$build_dir" "$report_file" "${extra_args[*]@Q}"
bash -c "set +e; pytest \"$sketchdir/test_$sketchname.py\" --build-dir \"$build_dir\" --junit-xml=\"$report_file\" ${extra_args[*]@Q}; exit \$?" || result=$?
printf "\n"
if [ $result -ne 0 ]; then
printf "\033[91mFailed test: $sketchname -- Config: $i\033[0m\n\n"
error=$result
printf "\033[91mFailed test: %s -- Config: %s\033[0m\n\n" "$sketchname" "$i"
error=$result
fi
fi
done
@ -136,13 +140,13 @@ chunk_run=0
options=0
erase=0
while [ ! -z "$1" ]; do
while [ -n "$1" ]; do
case $1 in
-c )
chunk_run=1
;;
-Q )
if [ ! -d $QEMU_PATH ]; then
if [ ! -d "$QEMU_PATH" ]; then
echo "QEMU path $QEMU_PATH does not exist"
exit 1
fi
@ -188,98 +192,97 @@ while [ ! -z "$1" ]; do
test_type=$1
;;
* )
break
;;
break
;;
esac
shift
done
if [ ! $platform == "qemu" ]; then
source ${SCRIPTS_DIR}/install-arduino-ide.sh
source "${SCRIPTS_DIR}/install-arduino-ide.sh"
fi
# If sketch is provided and test type is not, test type is inferred from the sketch path
if [[ $test_type == "all" ]] || [[ -z $test_type ]]; then
if [ -n "$sketch" ]; then
tmp_sketch_path=$(find tests -name $sketch.ino)
test_type=$(basename $(dirname $(dirname "$tmp_sketch_path")))
tmp_sketch_path=$(find tests -name "$sketch".ino)
test_type=$(basename "$(dirname "$(dirname "$tmp_sketch_path")")")
echo "Sketch $sketch test type: $test_type"
test_folder="$PWD/tests/$test_type"
else
test_folder="$PWD/tests"
test_folder="$PWD/tests"
fi
else
test_folder="$PWD/tests/$test_type"
fi
if [ $chunk_run -eq 0 ]; then
if [ -z $sketch ]; then
if [ -z "$sketch" ]; then
echo "ERROR: Sketch name is required for single test run"
exit 1
fi
run_test $target $test_folder/$sketch/$sketch.ino $options $erase
run_test "$target" "$test_folder"/"$sketch"/"$sketch".ino $options $erase
exit $?
else
if [ "$chunk_max" -le 0 ]; then
echo "ERROR: Chunks count must be positive number"
exit 1
fi
if [ "$chunk_max" -le 0 ]; then
echo "ERROR: Chunks count must be positive number"
exit 1
fi
if [ "$chunk_index" -ge "$chunk_max" ] && [ "$chunk_max" -ge 2 ]; then
echo "ERROR: Chunk index must be less than chunks count"
exit 1
fi
if [ "$chunk_index" -ge "$chunk_max" ] && [ "$chunk_max" -ge 2 ]; then
echo "ERROR: Chunk index must be less than chunks count"
exit 1
fi
set +e
# Ignore requirements as we don't have the libs. The requirements will be checked in the run_test function
${COUNT_SKETCHES} "$test_folder" "$target" "1"
sketchcount=$?
set -e
sketches=$(cat sketches.txt)
rm -rf sketches.txt
set +e
# Ignore requirements as we don't have the libs. The requirements will be checked in the run_test function
${COUNT_SKETCHES} "$test_folder" "$target" "1"
sketchcount=$?
set -e
sketches=$(cat sketches.txt)
rm -rf sketches.txt
chunk_size=$(( $sketchcount / $chunk_max ))
all_chunks=$(( $chunk_max * $chunk_size ))
if [ "$all_chunks" -lt "$sketchcount" ]; then
chunk_size=$(( $chunk_size + 1 ))
fi
chunk_size=$(( sketchcount / chunk_max ))
all_chunks=$(( chunk_max * chunk_size ))
if [ "$all_chunks" -lt "$sketchcount" ]; then
chunk_size=$(( chunk_size + 1 ))
fi
start_index=0
end_index=0
if [ "$chunk_index" -ge "$chunk_max" ]; then
start_index=$chunk_index
end_index=$sketchcount
else
start_index=$(( $chunk_index * $chunk_size ))
if [ "$sketchcount" -le "$start_index" ]; then
exit 0
fi
start_index=0
end_index=0
if [ "$chunk_index" -ge "$chunk_max" ]; then
start_index=$chunk_index
end_index=$sketchcount
else
start_index=$(( chunk_index * chunk_size ))
if [ "$sketchcount" -le "$start_index" ]; then
exit 0
fi
end_index=$(( $(( $chunk_index + 1 )) * $chunk_size ))
if [ "$end_index" -gt "$sketchcount" ]; then
end_index=$sketchcount
fi
fi
end_index=$(( $(( chunk_index + 1 )) * chunk_size ))
if [ "$end_index" -gt "$sketchcount" ]; then
end_index=$sketchcount
fi
fi
start_num=$(( $start_index + 1 ))
sketchnum=0
error=0
sketchnum=0
error=0
for sketch in $sketches; do
for sketch in $sketches; do
sketchnum=$(($sketchnum + 1))
if [ "$sketchnum" -le "$start_index" ] \
|| [ "$sketchnum" -gt "$end_index" ]; then
continue
fi
sketchnum=$((sketchnum + 1))
if [ "$sketchnum" -le "$start_index" ] \
|| [ "$sketchnum" -gt "$end_index" ]; then
continue
fi
printf "\033[95mSketch Index $(($sketchnum - 1))\033[0m\n"
printf "\033[95mSketch Index %s\033[0m\n" "$((sketchnum - 1))"
exit_code=0
run_test $target $sketch $options $erase || exit_code=$?
if [ $exit_code -ne 0 ]; then
error=$exit_code
fi
done
exit $error
exit_code=0
run_test "$target" "$sketch" $options $erase || exit_code=$?
if [ $exit_code -ne 0 ]; then
error=$exit_code
fi
done
exit $error
fi

24
.github/scripts/update-version.sh vendored
View file

@ -1,20 +1,21 @@
#!/bin/bash
# shellcheck disable=SC2002
# For reference: add tools for all boards by replacing one line in each board
# "[board].upload.tool=esptool_py" to "[board].upload.tool=esptool_py\n[board].upload.tool.default=esptool_py\n[board].upload.tool.network=esp_ota"
#cat boards.txt | sed "s/\([a-zA-Z0-9_\-]*\)\.upload\.tool\=esptool_py/\1\.upload\.tool\=esptool_py\\n\1\.upload\.tool\.default\=esptool_py\\n\1\.upload\.tool\.network\=esp_ota/"
if [ ! $# -eq 3 ]; then
echo "Bad number of arguments: $#" >&2
echo "usage: $0 <major> <minor> <patch>" >&2
exit 1
echo "Bad number of arguments: $#" >&2
echo "usage: $0 <major> <minor> <patch>" >&2
exit 1
fi
re='^[0-9]+$'
if [[ ! $1 =~ $re ]] || [[ ! $2 =~ $re ]] || [[ ! $3 =~ $re ]] ; then
echo "error: Not a valid version: $1.$2.$3" >&2
echo "usage: $0 <major> <minor> <patch>" >&2
exit 1
echo "error: Not a valid version: $1.$2.$3" >&2
echo "usage: $0 <major> <minor> <patch>" >&2
exit 1
fi
ESP_ARDUINO_VERSION_MAJOR="$1"
@ -36,11 +37,12 @@ sed "s/#define ESP_ARDUINO_VERSION_MAJOR.*/#define ESP_ARDUINO_VERSION_MAJOR $ES
sed "s/#define ESP_ARDUINO_VERSION_MINOR.*/#define ESP_ARDUINO_VERSION_MINOR $ESP_ARDUINO_VERSION_MINOR/g" | \
sed "s/#define ESP_ARDUINO_VERSION_PATCH.*/#define ESP_ARDUINO_VERSION_PATCH $ESP_ARDUINO_VERSION_PATCH/g" > __esp_arduino_version.h && mv __esp_arduino_version.h cores/esp32/esp_arduino_version.h
for lib in `ls libraries`; do
if [ -f "libraries/$lib/library.properties" ]; then
echo "Updating Library $lib..."
cat "libraries/$lib/library.properties" | sed "s/version=.*/version=$ESP_ARDUINO_VERSION/g" > "libraries/$lib/__library.properties" && mv "libraries/$lib/__library.properties" "libraries/$lib/library.properties"
fi
libraries=$(find libraries -maxdepth 1 -mindepth 1 -type d -exec basename {} \;)
for lib in $libraries; do
if [ -f "libraries/$lib/library.properties" ]; then
echo "Updating Library $lib..."
cat "libraries/$lib/library.properties" | sed "s/version=.*/version=$ESP_ARDUINO_VERSION/g" > "libraries/$lib/__library.properties" && mv "libraries/$lib/__library.properties" "libraries/$lib/library.properties"
fi
done
exit 0

3
.github/scripts/upload_py_tools.sh vendored
View file

@ -1,11 +1,12 @@
#!/bin/bash
CHANGED_FILES=$1
echo "Pushing '$CHANGED_FILES' as github-actions[bot]"
git config --global github.user "github-actions[bot]"
git config --global user.name "github-actions[bot]"
git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
for tool in $CHANGED_FILES; do
git add tools/$tool.exe
git add tools/"$tool".exe
done
git commit -m "change(tools): Push generated binaries to PR"
git push

30
.pre-commit-config.yaml
View file

@ -14,6 +14,7 @@ repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: "v5.0.0"
hooks:
# Generic checks
- id: check-case-conflict
- id: check-symlinks
- id: debug-statements
@ -25,6 +26,8 @@ repos:
args: [--fix=lf]
- id: trailing-whitespace
args: [--markdown-linebreak-ext=md]
# JSON formatting
- id: pretty-format-json
stages: [manual]
args: [--autofix]
@ -35,40 +38,67 @@ repos:
package\.json$|
^package\/.*$
)
- repo: https://github.com/codespell-project/codespell
rev: "v2.3.0"
hooks:
# Spell checking
- id: codespell
exclude: ^.*\.(svd|SVD)$
- repo: https://github.com/pre-commit/mirrors-clang-format
rev: "v18.1.3"
hooks:
# C/C++ formatting
- id: clang-format
types_or: [c, c++]
exclude: ^.*\/build_opt\.h$
- repo: https://github.com/psf/black-pre-commit-mirror
rev: "24.10.0"
hooks:
# Python formatting
- id: black
types_or: [python]
args: [--line-length=120] #From the arduino code style. Add as argument rather than creating a new config file.
- repo: https://github.com/PyCQA/flake8
rev: "7.1.1"
hooks:
# Python linting
- id: flake8
types_or: [python]
additional_dependencies:
- flake8-bugbear
- flake8-comprehensions
- flake8-simplify
- repo: https://github.com/pre-commit/mirrors-prettier
rev: "v3.1.0"
hooks:
# YAML formatting
- id: prettier
types_or: [yaml]
- repo: https://github.com/shellcheck-py/shellcheck-py
rev: "v0.10.0.1"
hooks:
# Bash linting
- id: shellcheck
types: [shell]
- repo: https://github.com/openstack/bashate
rev: "2.1.1"
hooks:
# Bash formatting
- id: bashate
types: [shell]
args: ["-i", "E006"] # Ignore E006: Line too long
- repo: https://github.com/errata-ai/vale
rev: "v3.9.1"
hooks:
# Sync vale styles and lint markdown and reStructuredText
- id: vale
name: vale-sync
language_version: "1.21.6"

11
.shellcheckrc Normal file
View file

@ -0,0 +1,11 @@
# Shellcheck configuration file for ESP32 Arduino core
# Optional checks. https://github.com/koalaman/shellcheck/wiki/optional
enable=add-default-case,deprecate-which,avoid-nullary-conditions
# Enable search for external sources
external-sources=true
# Search folder for sourced files.
# Set to the folder where the original script is located.
source-path=SCRIPTDIR

10
docs/en/contributing.rst
View file

@ -441,6 +441,16 @@ For checking the code style and other code quality checks, we use pre-commit hoo
These hooks will be automatically run by the CI when a Pull Request is marked as ``Status: Pending Merge``.
You can check which hooks are being run in the ``.pre-commit-config.yaml`` file.
Currently, we have hooks for the following tasks:
* Formatters for C, C++, Python, Bash, JSON, Markdown and ReStructuredText files;
* Linters for Python, Bash and prose (spoken language);
* Checking for spelling errors in the code and documentation;
* Removing trailing whitespaces and tabs in the code;
* Checking for the presence of private keys and other sensitive information in the code;
* Fixing the line endings and end of files (EOF) in the code;
* And more.
You can read more about the pre-commit hooks in the `pre-commit documentation <https://pre-commit.com/>`_.
If you want to run the pre-commit hooks locally, you first need to install the required dependencies by running:

27
docs/utils.sh
View file

@ -1,18 +1,19 @@
#!/bin/bash
# Bash helper functions for adding SSH keys
function add_ssh_keys() {
local key_string="${1}"
mkdir -p ~/.ssh
chmod 700 ~/.ssh
echo -n "${key_string}" >~/.ssh/id_rsa_base64
base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 >~/.ssh/id_rsa
chmod 600 ~/.ssh/id_rsa
function add_ssh_keys {
local key_string="${1}"
mkdir -p ~/.ssh
chmod 700 ~/.ssh
echo -n "${key_string}" >~/.ssh/id_rsa_base64
base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 >~/.ssh/id_rsa
chmod 600 ~/.ssh/id_rsa
}
function add_doc_server_ssh_keys() {
local key_string="${1}"
local server_url="${2}"
local server_user="${3}"
add_ssh_keys "${key_string}"
echo -e "Host ${server_url}\n\tStrictHostKeyChecking no\n\tUser ${server_user}\n" >>~/.ssh/config
function add_doc_server_ssh_keys {
local key_string="${1}"
local server_url="${2}"
local server_user="${3}"
add_ssh_keys "${key_string}"
echo -e "Host ${server_url}\n\tStrictHostKeyChecking no\n\tUser ${server_user}\n" >>~/.ssh/config
}

166
tools/add_lib.sh
View file

@ -1,4 +1,5 @@
#!/bin/bash
HELP="This script help to add library when using arduino-esp32 as an ESP-IDF component
The script accepts up to three arguments:
-n NEW: URL address to new library on GIThub (cannot be combined with -e)
@ -26,119 +27,126 @@ n_param=""
# Parse the command-line arguments using getopts
while getopts "he:l:n:" opt; do
case $opt in
h)
echo "$HELP"
exit 0
;;
e)
#e_param="$OPTARG"
e_param="${OPTARG/#~/$HOME}"
;;
l)
#l_param="$OPTARG"
l_param="${OPTARG/#~/$HOME}"
;;
n)
n_param=$OPTARG
;;
\?)
echo "Invalid option: -$OPTARG" >&2
echo $HELP
exit 1
;;
:)
echo "Option -$OPTARG requires an argument." >&2
echo $HELP
exit 1
;;
esac
case $opt in
h)
echo "$HELP"
exit 0
;;
e)
#e_param="$OPTARG"
e_param="${OPTARG/#~/$HOME}"
;;
l)
#l_param="$OPTARG"
l_param="${OPTARG/#~/$HOME}"
;;
n)
n_param=$OPTARG
;;
\?)
echo "Invalid option: -$OPTARG" >&2
echo "$HELP"
exit 1
;;
:)
echo "Option -$OPTARG requires an argument." >&2
echo "$HELP"
exit 1
;;
*)
echo "Invalid option: -$OPTARG" >&2
echo "$HELP"
exit 1
;;
esac
done
# No parameter check
if [[ -z "$e_param" ]] && [[ -z "$l_param" ]] && [[ -z "$n_param" ]]; then
echo "Error: No parameters" >&2
echo "$HELP"
exit 1
echo "Error: No parameters" >&2
echo "$HELP"
exit 1
fi
# Only local path check (not permitted)
if [[ -z "$e_param" ]] && [[ ! -z "$l_param" ]] && [[ -z "$n_param" ]]; then
echo "Error: -l parameter must be paired with -e or -n" >&2
echo "$HELP"
exit 1
if [[ -z "$e_param" ]] && [[ -n "$l_param" ]] && [[ -z "$n_param" ]]; then
echo "Error: -l parameter must be paired with -e or -n" >&2
echo "$HELP"
exit 1
fi
# Invalid combination check
if [[ ! -z $e_param ]] && [[ ! -z $n_param ]]; then
echo "ERROR: Cannot combine -n with -e" >&2
echo "$HELP"
exit 1
if [[ -n $e_param ]] && [[ -n $n_param ]]; then
echo "ERROR: Cannot combine -n with -e" >&2
echo "$HELP"
exit 1
fi
# Check existing lib
if [[ ! -z "$e_param" ]]; then
if [[ ! -d "${e_param/#~/$HOME}" ]]; then # this works!
echo "Error: existing library parameter - path does not exist" >&2
exit 1
fi
if [[ -n "$e_param" ]]; then
if [[ ! -d "${e_param/#~/$HOME}" ]]; then # this works!
echo "Error: existing library parameter - path does not exist" >&2
exit 1
fi
fi
LIBRARY=""
# Only existing library was supplied
if [[ ! -z $e_param ]] && [[ -z $l_param ]] && [[ -z $n_param ]]; then
LIBRARY=$e_param
if [[ -n $e_param ]] && [[ -z $l_param ]] && [[ -z $n_param ]]; then
LIBRARY=$e_param
fi
# Install new lib
if [ ! -z $n_param ]; then
INSTALL_TARGET=""
if [ -z $l_param ]; then
# If local path for project is not supplied - use as INSTALL_TARGET Arduino libraries path
INSTALL_TARGET=$ARDUINO_LIBS_PATH/$(basename "$n_param")
else
INSTALL_TARGET=$l_param/components/$(basename "$n_param")
if [ ! -d "$l_param/components" ]; then
echo "Folder components does not exist yet: mkdir -p "$l_param/components""
mkdir -p "$l_param/components"
if [ -n "$n_param" ]; then
INSTALL_TARGET=""
if [ -z "$l_param" ]; then
# If local path for project is not supplied - use as INSTALL_TARGET Arduino libraries path
INSTALL_TARGET=$ARDUINO_LIBS_PATH/$(basename "$n_param")
else
INSTALL_TARGET=$l_param/components/$(basename "$n_param")
if [ ! -d "$l_param/components" ]; then
echo "Folder components does not exist yet: mkdir -p \"$l_param/components\""
mkdir -p "$l_param/components"
fi
fi
fi
# clone the new lib
echo "Cloning: git clone --recursive $n_param $INSTALL_TARGET"
git clone --recursive $n_param $INSTALL_TARGET
LIBRARY=$INSTALL_TARGET
# clone the new lib
echo "Cloning: git clone --recursive $n_param $INSTALL_TARGET"
git clone --recursive "$n_param" "$INSTALL_TARGET"
LIBRARY=$INSTALL_TARGET
fi
# Copy existing lib to local project
if [[ ! -z $e_param ]] && [[ ! -z $l_param ]]; then
if [ ! -d "$l_param/components" ]; then
echo "Folder components does not exist yet: mkdir -p "$l_param/components""
mkdir -p "$l_param/components"
fi
echo "Copy from $e_param to $l_param"
echo "cp -r $e_param $l_param/components/$(basename "$e_param")"
cp -r $e_param $l_param/components/$(basename "$e_param")
LIBRARY=$l_param/components/$(basename "$e_param")
if [[ -n $e_param ]] && [[ -n $l_param ]]; then
if [ ! -d "$l_param/components" ]; then
echo "Folder components does not exist yet: mkdir -p \"$l_param/components\""
mkdir -p "$l_param/components"
fi
echo "Copy from $e_param to $l_param"
echo "cp -r $e_param $l_param/components/\"$(basename "$e_param")\""
cp -r "$e_param" "$l_param"/components/"$(basename "$e_param")"
LIBRARY=$l_param/components/"$(basename "$e_param")"
fi
if [ -z "$LIBRARY" ]; then
echo "ERROR: No library path" >&2
exit 1
echo "ERROR: No library path" >&2
exit 1
fi
# 1. get the source list:
FILES=$(find $LIBRARY -name '*.c' -o -name '*.cpp' | xargs -I{} basename {})
FILES=$(find "$LIBRARY" -print0 -name '*.c' -o -name '*.cpp' | xargs -0 -I{} basename {})
# Fresh start
if [ -f $LIBRARY/CMakeLists.txt ]; then
rm $LIBRARY/CMakeLists.txt
touch $LIBRARY/CMakeLists.txt
if [ -f "$LIBRARY"/CMakeLists.txt ]; then
rm "$LIBRARY"/CMakeLists.txt
touch "$LIBRARY"/CMakeLists.txt
fi
# Generate CMakeLists.txt
echo "idf_component_register(SRCS $(echo $FILES | sed -e 's/ /" "/g' | sed -e 's/^/"/' -e 's/$/"/')" >> $LIBRARY/CMakeLists.txt
echo " INCLUDE_DIRS \".\"" >> $LIBRARY/CMakeLists.txt
echo " REQUIRES \"arduino-esp32\"" >> $LIBRARY/CMakeLists.txt
echo " )" >> $LIBRARY/CMakeLists.txt
{
echo "idf_component_register(SRCS $(echo "$FILES" | sed -e 's/ /" "/g' | sed -e 's/^/"/' -e 's/$/"/')"
echo " INCLUDE_DIRS \".\""
echo " REQUIRES \"arduino-esp32\""
echo " )"
} >> "$LIBRARY"/CMakeLists.txt