Update PlatformIO build script (#7579)

This PR brings updates according to the latest changes in the Arduino core:
- Bootloader binaries are now generated from elf files
- Updated CI scripts
- Updated esptoolpy to v4.2.1
- Minor clean-up by removing obsolete code

Resolves #7572

.github/scripts/install-platformio-esp32.sh

@@ -4,7 +4,7 @@ export PLATFORMIO_ESP32_PATH="$HOME/.platformio/packages/framework-arduinoespres
 PLATFORMIO_ESP32_URL="https://github.com/platformio/platform-espressif32.git"
 
 TOOLCHAIN_VERSION="8.4.0+2021r2-patch3"
-ESPTOOLPY_VERSION="~1.30100.0"
+ESPTOOLPY_VERSION="~1.40201.0"
 ESPRESSIF_ORGANIZATION_NAME="espressif"
 
 echo "Installing Python Wheel ..."

.github/scripts/on-push.sh

@@ -93,26 +93,11 @@ else
     build_pio_sketch "$BOARD" "$OPTIONS" "$PLATFORMIO_ESP32_PATH/libraries/BLE/examples/BLE_server/BLE_server.ino" && \
     build_pio_sketch "$BOARD" "$OPTIONS" "$PLATFORMIO_ESP32_PATH/libraries/ESP32/examples/Camera/CameraWebServer/CameraWebServer.ino"
 
-    # PlatformIO ESP32 Test
-    # OPTIONS="board_build.mcu = esp32s2"
-    # build_pio_sketch "$BOARD" "$OPTIONS" "$PLATFORMIO_ESP32_PATH/libraries/WiFi/examples/WiFiClient/WiFiClient.ino" && \
-    # build_pio_sketch "$BOARD" "$OPTIONS" "$PLATFORMIO_ESP32_PATH/libraries/WiFiClientSecure/examples/WiFiClientSecure/WiFiClientSecure.ino"
-
-    python -m platformio ci --board "$BOARD" "$PLATFORMIO_ESP32_PATH/libraries/WiFi/examples/WiFiClient" --project-option="board_build.mcu = esp32s2" --project-option="board_build.partitions = huge_app.csv"
-    python -m platformio ci --board "$BOARD" "$PLATFORMIO_ESP32_PATH/libraries/WiFi/examples/WiFiClient" --project-option="board_build.mcu = esp32c3" --project-option="board_build.partitions = huge_app.csv"
-
-    echo "Hacking in S3 support ..."
-    replace_script="import json; import os;"
-    replace_script+="fp=open(os.path.expanduser('~/.platformio/platforms/espressif32/platform.json'), 'r+');"
-    replace_script+="data=json.load(fp);"
-    replace_script+="data['packages']['toolchain-xtensa-esp32']['optional']=True;"
-    replace_script+="data['packages']['toolchain-xtensa-esp32s3']['optional']=False;"
-    replace_script+="data['packages']['tool-esptoolpy']['owner']='tasmota';"
-    replace_script+="data['packages']['tool-esptoolpy']['version']='https://github.com/tasmota/esptool/releases/download/v4.2.1/esptool-4.2.1.zip';"
-    replace_script+="fp.seek(0);fp.truncate();json.dump(data, fp, indent=2);fp.close()"
-    python -c "$replace_script"
-
-    python -m platformio ci --board "$BOARD" "$PLATFORMIO_ESP32_PATH/libraries/WiFi/examples/WiFiClient" --project-option="board_build.mcu = esp32s3" --project-option="board_build.partitions = huge_app.csv"
+    # Basic sanity testing for other series
+    for board in "esp32-c3-devkitm-1" "esp32-s2-saola-1" "esp32-s3-devkitc-1"
+    do
+        python -m platformio ci --board "$board" "$PLATFORMIO_ESP32_PATH/libraries/WiFi/examples/WiFiClient" --project-option="board_build.partitions = huge_app.csv"
+    done
 
     #build_pio_sketches "$BOARD" "$OPTIONS" "$PLATFORMIO_ESP32_PATH/libraries"
 fi

tools/platformio-build.py

@@ -82,52 +82,41 @@ def get_bootloader_image(variants_dir):
     return (
         variant_bootloader
         if isfile(variant_bootloader)
-        else join(
-            FRAMEWORK_DIR,
-            "tools",
-            "sdk",
-            build_mcu,
-            "bin",
-            "bootloader_${__get_board_boot_mode(__env__)}_${__get_board_f_flash(__env__)}.bin",
+        else generate_bootloader_image(
+            join(
+                FRAMEWORK_DIR,
+                "tools",
+                "sdk",
+                build_mcu,
+                "bin",
+                "bootloader_${__get_board_boot_mode(__env__)}_${__get_board_f_flash(__env__)}.elf",
+            )
         )
     )
 
 
-def get_patched_bootloader_image(original_bootloader_image, bootloader_offset):
-    patched_bootloader_image = join(env.subst("$BUILD_DIR"), "patched_bootloader.bin")
+def generate_bootloader_image(bootloader_elf):
     bootloader_cmd = env.Command(
-        patched_bootloader_image,
-        original_bootloader_image,
-        env.VerboseAction(
-            " ".join(
-                [
-                    '"$PYTHONEXE"',
-                    join(
-                        platform.get_package_dir("tool-esptoolpy") or "", "esptool.py"
-                    ),
-                    "--chip",
-                    build_mcu,
-                    "merge_bin",
-                    "-o",
-                    "$TARGET",
-                    "--flash_mode",
-                    "${__get_board_flash_mode(__env__)}",
-                    "--flash_freq",
-                    "${__get_board_f_flash(__env__)}",
-                    "--flash_size",
-                    board_config.get("upload.flash_size", "4MB"),
-                    "--target-offset",
-                    bootloader_offset,
-                    bootloader_offset,
-                    "$SOURCE",
-                ]
-            ),
-            "Updating bootloader headers",
-        ),
+        join("$BUILD_DIR", "bootloader.bin"),
+        bootloader_elf,
+        env.VerboseAction(" ".join([
+            '"$PYTHONEXE" "$OBJCOPY"',
+            "--chip", build_mcu, "elf2image",
+            "--flash_mode", "${__get_board_flash_mode(__env__)}",
+            "--flash_freq", "${__get_board_f_flash(__env__)}",
+            "--flash_size", board_config.get("upload.flash_size", "4MB"),
+            "-o", "$TARGET", "$SOURCES"
+        ]), "Building $TARGET"),
     )
+
     env.Depends("$BUILD_DIR/$PROGNAME$PROGSUFFIX", bootloader_cmd)
 
-    return patched_bootloader_image
+    # Because the Command always returns a NodeList, we have to
+    # access the first element in the list to get the Node object
+    # that actually represents the bootloader image.
+    # Also, this file is later used in generic Python code, so the
+    # Node object in converted to a generic string
+    return str(bootloader_cmd[0])
 
 
 def add_tinyuf2_extra_image():
@@ -210,34 +199,13 @@ env.Prepend(LIBS=libs)
 # Process framework extra images
 #
 
-# Starting with v2.0.4 the Arduino core contains updated bootloader images that have
-# innacurate default headers. This results in bootloops if firmware is flashed via
-# OpenOCD (e.g. debugging or uploading via debug tools). For this reason, before
-# uploading or debugging we need to adjust the bootloader binary according to
-# the values of the --flash-size and --flash-mode arguments.
-# Note: This behavior doesn't occur if uploading is done via esptoolpy, as esptoolpy
-# overrides the binary image headers before flashing.
-
-bootloader_patch_required = bool(
-    env.get("PIOFRAMEWORK", []) == ["arduino"]
-    and (
-        "debug" in env.GetBuildType()
-        or env.subst("$UPLOAD_PROTOCOL") in board_config.get("debug.tools", {})
-        or env.IsIntegrationDump()
-    )
-)
-
-bootloader_image_path = get_bootloader_image(variants_dir)
-bootloader_offset = "0x1000" if build_mcu in ("esp32", "esp32s2") else "0x0000"
-if bootloader_patch_required:
-    bootloader_image_path = get_patched_bootloader_image(
-        bootloader_image_path, bootloader_offset
-    )
-
 env.Append(
     LIBSOURCE_DIRS=[join(FRAMEWORK_DIR, "libraries")],
     FLASH_EXTRA_IMAGES=[
-        (bootloader_offset, bootloader_image_path),
+        (
+            "0x1000" if build_mcu in ("esp32", "esp32s2") else "0x0000",
+            get_bootloader_image(variants_dir),
+        ),
         ("0x8000", join(env.subst("$BUILD_DIR"), "partitions.bin")),
         ("0xe000", join(FRAMEWORK_DIR, "tools", "partitions", "boot_app0.bin")),
     ]