feat(esptool): Upgrade to esptool v5 (#11433)

* feat(esptool): Upgrade to esptool v5

* fix(script): Update script for better handling of esptool

* fix(script): Get proper download url

* fix(script): Apply copilot suggestions
This commit is contained in:
Lucas Saavedra Vaz 2025-06-10 05:19:32 -03:00 committed by GitHub
parent e9813c6a52
commit 89ff465328
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 278 additions and 42 deletions

236
.github/scripts/update_esptool.py vendored Normal file
View file

@ -0,0 +1,236 @@
#!/usr/bin/env python3
# This script is used to re-package the esptool if needed and update the JSON file
# for the Arduino ESP32 platform.
#
# The script has only been tested on macOS.
#
# For regular esptool releases, the generated packages already contain the correct permissions,
# extensions and are uploaded to the GitHub release assets. In this case, the script will only
# update the JSON file with the information from the GitHub release.
#
# The script can be used in two modes:
# 1. Local build: The build artifacts must be already downloaded and extracted in the base_folder.
# This is useful for esptool versions that are not yet released and that are grabbed from the
# GitHub build artifacts.
# 2. Release build: The script will get the release information from GitHub and update the JSON file.
# This is useful for esptool versions that are already released and that are uploaded to the
# GitHub release assets.
#
# For local build, the artifacts must be already downloaded and extracted in the base_folder
# set with the -l option.
# For example, a base folder "esptool" should contain the following folders extracted directly
# from the GitHub build artifacts:
# esptool/esptool-linux-aarch64
# esptool/esptool-linux-amd64
# esptool/esptool-linux-armv7
# esptool/esptool-macos-amd64
# esptool/esptool-macos-arm64
# esptool/esptool-windows-amd64
import argparse
import json
import os
import shutil
import stat
import tarfile
import zipfile
import hashlib
import requests
from pathlib import Path
def compute_sha256(filepath):
sha256 = hashlib.sha256()
with open(filepath, "rb") as f:
for block in iter(lambda: f.read(4096), b""):
sha256.update(block)
return f"SHA-256:{sha256.hexdigest()}"
def get_file_size(filepath):
return os.path.getsize(filepath)
def update_json_for_host(tmp_json_path, version, host, url, archiveFileName, checksum, size):
with open(tmp_json_path) as f:
data = json.load(f)
for pkg in data.get("packages", []):
for tool in pkg.get("tools", []):
if tool.get("name") == "esptool_py":
tool["version"] = version
if url is None:
# If the URL is not set, we need to find the old URL and update it
for system in tool.get("systems", []):
if system.get("host") == host:
url = system.get("url").replace(system.get("archiveFileName"), archiveFileName)
break
else:
print(f"No old URL found for host {host}. Using empty URL.")
url = ""
# Preserve existing systems order and update or append the new system
systems = tool.get("systems", [])
system_updated = False
for i, system in enumerate(systems):
if system.get("host") == host:
systems[i] = {
"host": host,
"url": url,
"archiveFileName": archiveFileName,
"checksum": checksum,
"size": str(size),
}
system_updated = True
break
if not system_updated:
systems.append({
"host": host,
"url": url,
"archiveFileName": archiveFileName,
"checksum": checksum,
"size": str(size),
})
tool["systems"] = systems
with open(tmp_json_path, "w") as f:
json.dump(data, f, indent=2, sort_keys=False, ensure_ascii=False)
f.write("\n")
def update_tools_dependencies(tmp_json_path, version):
with open(tmp_json_path) as f:
data = json.load(f)
for pkg in data.get("packages", []):
for platform in pkg.get("platforms", []):
for dep in platform.get("toolsDependencies", []):
if dep.get("name") == "esptool_py":
dep["version"] = version
with open(tmp_json_path, "w") as f:
json.dump(data, f, indent=2, sort_keys=False, ensure_ascii=False)
f.write("\n")
def create_archives(version, base_folder):
archive_files = []
for dirpath in Path(base_folder).glob("esptool-*"):
if not dirpath.is_dir():
continue
base = dirpath.name[len("esptool-"):]
if "windows" in dirpath.name:
zipfile_name = f"esptool-v{version}-{base}.zip"
print(f"Creating {zipfile_name} from {dirpath} ...")
with zipfile.ZipFile(zipfile_name, "w", zipfile.ZIP_DEFLATED) as zipf:
for root, _, files in os.walk(dirpath):
for file in files:
full_path = os.path.join(root, file)
zipf.write(full_path, os.path.relpath(full_path, start=dirpath))
archive_files.append(zipfile_name)
else:
tarfile_name = f"esptool-v{version}-{base}.tar.gz"
print(f"Creating {tarfile_name} from {dirpath} ...")
for root, dirs, files in os.walk(dirpath):
for name in dirs + files:
os.chmod(os.path.join(root, name), stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR |
stat.S_IRGRP | stat.S_IXGRP |
stat.S_IROTH | stat.S_IXOTH)
with tarfile.open(tarfile_name, "w:gz") as tar:
tar.add(dirpath, arcname=dirpath.name)
archive_files.append(tarfile_name)
return archive_files
def determine_hosts(archive_name):
if "linux-amd64" in archive_name:
return ["x86_64-pc-linux-gnu"]
elif "linux-armv7" in archive_name:
return ["arm-linux-gnueabihf"]
elif "linux-aarch64" in archive_name:
return ["aarch64-linux-gnu"]
elif "macos-amd64" in archive_name:
return ["x86_64-apple-darwin"]
elif "macos-arm64" in archive_name:
return ["arm64-apple-darwin"]
elif "windows-amd64" in archive_name:
return ["x86_64-mingw32", "i686-mingw32"]
else:
return []
def update_json_from_local_build(tmp_json_path, version, base_folder, archive_files):
for archive in archive_files:
print(f"Processing archive: {archive}")
hosts = determine_hosts(archive)
if not hosts:
print(f"Skipping unknown archive type: {archive}")
continue
archive_path = Path(archive)
checksum = compute_sha256(archive_path)
size = get_file_size(archive_path)
for host in hosts:
update_json_for_host(tmp_json_path, version, host, None, archive_path.name, checksum, size)
def update_json_from_release(tmp_json_path, version, release_info):
assets = release_info.get("assets", [])
for asset in assets:
if (asset.get("name").endswith(".tar.gz") or asset.get("name").endswith(".zip")) and "esptool" in asset.get("name"):
asset_fname = asset.get("name")
print(f"Processing asset: {asset_fname}")
hosts = determine_hosts(asset_fname)
if not hosts:
print(f"Skipping unknown archive type: {asset_fname}")
continue
asset_url = asset.get("browser_download_url")
asset_checksum = asset.get("digest")
asset_size = asset.get("size")
if asset_checksum is None:
asset_checksum = ""
print(f"Asset {asset_fname} has no checksum. Please set the checksum in the JSON file.")
for host in hosts:
update_json_for_host(tmp_json_path, version, host, asset_url, asset_fname, asset_checksum, asset_size)
def get_release_info(version):
url = f"https://api.github.com/repos/espressif/esptool/releases/tags/v{version}"
response = requests.get(url)
response.raise_for_status()
return response.json()
def main():
parser = argparse.ArgumentParser(description="Repack esptool and update JSON metadata.")
parser.add_argument("version", help="Version of the esptool (e.g. 5.0.dev1)")
parser.add_argument("-l", "--local", dest="base_folder", help="Enable local build mode and set the base folder with unpacked artifacts")
args = parser.parse_args()
script_dir = Path(__file__).resolve().parent
json_path = (script_dir / "../../package/package_esp32_index.template.json").resolve()
tmp_json_path = Path(str(json_path) + ".tmp")
shutil.copy(json_path, tmp_json_path)
local_build = args.base_folder is not None
if local_build:
os.chdir(args.base_folder)
os.environ['COPYFILE_DISABLE'] = 'true' # this disables including resource forks in tar files on macOS
# Clear any existing archive files
for file in Path(args.base_folder).glob("esptool-*.*"):
file.unlink()
archive_files = create_archives(args.version, args.base_folder)
update_json_from_local_build(tmp_json_path, args.version, args.base_folder, archive_files)
else:
release_info = get_release_info(args.version)
update_json_from_release(tmp_json_path, args.version, release_info)
print(f"Updating esptool version fields to {args.version}")
update_tools_dependencies(tmp_json_path, args.version)
shutil.move(tmp_json_path, json_path)
print(f"Done. JSON updated at {json_path}")
if __name__ == "__main__":
main()

View file

@ -81,7 +81,7 @@
{
"packager": "esp32",
"name": "esptool_py",
"version": "4.9.dev3"
"version": "5.0.dev1"
},
{
"packager": "esp32",
@ -469,56 +469,56 @@
},
{
"name": "esptool_py",
"version": "4.9.dev3",
"version": "5.0.dev1",
"systems": [
{
"host": "aarch64-linux-gnu",
"url": "https://github.com/espressif/arduino-esp32/releases/download/3.2.0/esptool-v5.0.dev1-linux-aarch64.tar.gz",
"archiveFileName": "esptool-v5.0.dev1-linux-aarch64.tar.gz",
"checksum": "SHA-256:bfafa7a7723ebbabfd8b6e3ca5ae00bfead0331de923754aeddb43b2c116a078",
"size": "58241736"
},
{
"host": "x86_64-pc-linux-gnu",
"url": "https://github.com/espressif/arduino-esp32/releases/download/3.1.0-RC3/esptool-v4.9.dev3-linux-amd64.tar.gz",
"archiveFileName": "esptool-v4.9.dev3-linux-amd64.tar.gz",
"checksum": "SHA-256:4ecaf51836cbf4ea3c19840018bfef3b0b8cd8fc3c95f6e1e043ca5bbeab9bf0",
"size": "64958202"
"url": "https://github.com/espressif/arduino-esp32/releases/download/3.2.0/esptool-v5.0.dev1-linux-amd64.tar.gz",
"archiveFileName": "esptool-v5.0.dev1-linux-amd64.tar.gz",
"checksum": "SHA-256:acd0486e96586b99d053a1479acbbbfcae8667227c831cdc53a171f9ccfa27ee",
"size": "100740042"
},
{
"host": "arm-linux-gnueabihf",
"url": "https://github.com/espressif/arduino-esp32/releases/download/3.1.0-RC3/esptool-v4.9.dev3-linux-armv7.tar.gz",
"archiveFileName": "esptool-v4.9.dev3-linux-armv7.tar.gz",
"checksum": "SHA-256:fff818573bce483ee793ac83c8211f6abf764aa3350f198228859f696a0a0b36",
"size": "31530030"
},
{
"host": "aarch64-linux-gnu",
"url": "https://github.com/espressif/arduino-esp32/releases/download/3.1.0-RC3/esptool-v4.9.dev3-linux-aarch64.tar.gz",
"archiveFileName": "esptool-v4.9.dev3-linux-aarch64.tar.gz",
"checksum": "SHA-256:5b274bdff2f62e6a07c3c1dfa51b1128924621f661747eca3dbe0f77972f2f06",
"size": "33663882"
"url": "https://github.com/espressif/arduino-esp32/releases/download/3.2.0/esptool-v5.0.dev1-linux-armv7.tar.gz",
"archiveFileName": "esptool-v5.0.dev1-linux-armv7.tar.gz",
"checksum": "SHA-256:ea77a38681506761bbb7b0b39c130811ed565667b67ebbdb4d6dcc6cb6e07368",
"size": "53451939"
},
{
"host": "x86_64-apple-darwin",
"url": "https://github.com/espressif/arduino-esp32/releases/download/3.1.0-RC3/esptool-v4.9.dev3-macos-amd64.tar.gz",
"archiveFileName": "esptool-v4.9.dev3-macos-amd64.tar.gz",
"checksum": "SHA-256:c733c83b58fcf5f642fbb2fddb8ff24640c2c785126cba0821fb70c4a5ceea7a",
"size": "32767836"
"url": "https://github.com/espressif/arduino-esp32/releases/download/3.2.0/esptool-v5.0.dev1-macos-amd64.tar.gz",
"archiveFileName": "esptool-v5.0.dev1-macos-amd64.tar.gz",
"checksum": "SHA-256:900a8e90731208bee96647e0e207a43612b9452c2120c4fdc0ff4c6be226257b",
"size": "59631998"
},
{
"host": "arm64-apple-darwin",
"url": "https://github.com/espressif/arduino-esp32/releases/download/3.1.0-RC3/esptool-v4.9.dev3-macos-arm64.tar.gz",
"archiveFileName": "esptool-v4.9.dev3-macos-arm64.tar.gz",
"checksum": "SHA-256:83c195a15981e6a5e7a130db2ccfb21e2d8093912e5b003681f9a5abadd71af7",
"size": "30121441"
},
{
"host": "i686-mingw32",
"url": "https://github.com/espressif/arduino-esp32/releases/download/3.1.0-RC3/esptool-v4.9.dev3-win64.zip",
"archiveFileName": "esptool-v4.9.dev3-win64.zip",
"checksum": "SHA-256:890051a4fdc684ff6f4af18d0bb27d274ca940ee0eef716a9455f8c64b25b215",
"size": "36072564"
"url": "https://github.com/espressif/arduino-esp32/releases/download/3.2.0/esptool-v5.0.dev1-macos-arm64.tar.gz",
"archiveFileName": "esptool-v5.0.dev1-macos-arm64.tar.gz",
"checksum": "SHA-256:3653f4de73cb4fc6a25351eaf663708e91c65ae3265d75bd54ca4315a4350bb4",
"size": "56349992"
},
{
"host": "x86_64-mingw32",
"url": "https://github.com/espressif/arduino-esp32/releases/download/3.1.0-RC3/esptool-v4.9.dev3-win64.zip",
"archiveFileName": "esptool-v4.9.dev3-win64.zip",
"checksum": "SHA-256:890051a4fdc684ff6f4af18d0bb27d274ca940ee0eef716a9455f8c64b25b215",
"size": "36072564"
"url": "https://github.com/espressif/arduino-esp32/releases/download/3.2.0/esptool-v5.0.dev1-win64.zip",
"archiveFileName": "esptool-v5.0.dev1-win64.zip",
"checksum": "SHA-256:1e8fd89645daf94f2d4406ec73c9004e617ea921079515f9fd749205eece4d6d",
"size": "59102658"
},
{
"host": "i686-mingw32",
"url": "https://github.com/espressif/arduino-esp32/releases/download/3.2.0/esptool-v5.0.dev1-win64.zip",
"archiveFileName": "esptool-v5.0.dev1-win64.zip",
"checksum": "SHA-256:1e8fd89645daf94f2d4406ec73c9004e617ea921079515f9fd749205eece4d6d",
"size": "59102658"
}
]
},

View file

@ -120,7 +120,7 @@ recipe.hooks.prebuild.2.pattern.windows=cmd /c if not exist "{build.path}\partit
recipe.hooks.prebuild.3.pattern.windows=cmd /c if not exist "{build.path}\partitions.csv" COPY "{runtime.platform.path}\tools\partitions\{build.partitions}.csv" "{build.path}\partitions.csv"
# Check if custom bootloader exist: source > variant > build.boot
recipe.hooks.prebuild.4.pattern_args=--chip {build.mcu} elf2image --flash_mode {build.flash_mode} --flash_freq {build.img_freq} --flash_size {build.flash_size} -o
recipe.hooks.prebuild.4.pattern_args=--chip {build.mcu} elf2image --flash-mode {build.flash_mode} --flash-freq {build.img_freq} --flash-size {build.flash_size} -o
recipe.hooks.prebuild.4.pattern=/usr/bin/env bash -c "[ -f "{build.source.path}"/bootloader.bin ] && cp -f "{build.source.path}"/bootloader.bin "{build.path}"/{build.project_name}.bootloader.bin || ( [ -f "{build.variant.path}"/{build.custom_bootloader}.bin ] && cp "{build.variant.path}"/{build.custom_bootloader}.bin "{build.path}"/{build.project_name}.bootloader.bin || "{tools.esptool_py.path}"/{tools.esptool_py.cmd} {recipe.hooks.prebuild.4.pattern_args} "{build.path}"/{build.project_name}.bootloader.bin "{compiler.sdk.path}"/bin/bootloader_{build.boot}_{build.boot_freq}.elf )"
recipe.hooks.prebuild.4.pattern.windows=cmd /c IF EXIST "{build.source.path}\bootloader.bin" ( COPY /y "{build.source.path}\bootloader.bin" "{build.path}\{build.project_name}.bootloader.bin" ) ELSE ( IF EXIST "{build.variant.path}\{build.custom_bootloader}.bin" ( COPY "{build.variant.path}\{build.custom_bootloader}.bin" "{build.path}\{build.project_name}.bootloader.bin" ) ELSE ( "{tools.esptool_py.path}\{tools.esptool_py.cmd}" {recipe.hooks.prebuild.4.pattern_args} "{build.path}\{build.project_name}.bootloader.bin" "{compiler.sdk.path}\bin\bootloader_{build.boot}_{build.boot_freq}.elf" ) )
@ -164,7 +164,7 @@ recipe.c.combine.pattern="{compiler.path}{compiler.c.elf.cmd}" {compiler.c.elf.f
recipe.objcopy.partitions.bin.pattern={tools.gen_esp32part.cmd} -q "{build.path}/partitions.csv" "{build.path}/{build.project_name}.partitions.bin"
## Create bin
recipe.objcopy.bin.pattern_args=--chip {build.mcu} elf2image --flash_mode "{build.flash_mode}" --flash_freq "{build.img_freq}" --flash_size "{build.flash_size}" --elf-sha256-offset 0xb0 -o "{build.path}/{build.project_name}.bin" "{build.path}/{build.project_name}.elf"
recipe.objcopy.bin.pattern_args=--chip {build.mcu} elf2image --flash-mode "{build.flash_mode}" --flash-freq "{build.img_freq}" --flash-size "{build.flash_size}" --elf-sha256-offset 0xb0 -o "{build.path}/{build.project_name}.bin" "{build.path}/{build.project_name}.elf"
recipe.objcopy.bin.pattern="{tools.esptool_py.path}/{tools.esptool_py.cmd}" {recipe.objcopy.bin.pattern_args}
## Create Insights Firmware Package
@ -177,7 +177,7 @@ recipe.hooks.objcopy.postobjcopy.2.pattern=/usr/bin/env bash -c "[ ! -d "{build.
recipe.hooks.objcopy.postobjcopy.2.pattern.windows=cmd /c if exist "{build.path}\libraries\ESP_SR" if exist "{compiler.sdk.path}\esp_sr\srmodels.bin" COPY /y "{compiler.sdk.path}\esp_sr\srmodels.bin" "{build.path}\srmodels.bin"
# Create merged binary
recipe.hooks.objcopy.postobjcopy.3.pattern_args=--chip {build.mcu} merge_bin -o "{build.path}/{build.project_name}.merged.bin" --fill-flash-size {build.flash_size} --flash_mode keep --flash_freq keep --flash_size keep {build.bootloader_addr} "{build.path}/{build.project_name}.bootloader.bin" 0x8000 "{build.path}/{build.project_name}.partitions.bin" 0xe000 "{runtime.platform.path}/tools/partitions/boot_app0.bin" 0x10000 "{build.path}/{build.project_name}.bin"
recipe.hooks.objcopy.postobjcopy.3.pattern_args=--chip {build.mcu} merge-bin -o "{build.path}/{build.project_name}.merged.bin" --pad-to-size {build.flash_size} --flash-mode keep --flash-freq keep --flash-size keep {build.bootloader_addr} "{build.path}/{build.project_name}.bootloader.bin" 0x8000 "{build.path}/{build.project_name}.partitions.bin" 0xe000 "{runtime.platform.path}/tools/partitions/boot_app0.bin" 0x10000 "{build.path}/{build.project_name}.bin"
recipe.hooks.objcopy.postobjcopy.3.pattern="{tools.esptool_py.path}/{tools.esptool_py.cmd}" {recipe.hooks.objcopy.postobjcopy.3.pattern_args}
## Save bin
@ -294,14 +294,14 @@ debug.additional_config=debug_config.{build.mcu}
tools.esptool_py.upload.protocol=serial
tools.esptool_py.upload.params.verbose=
tools.esptool_py.upload.params.quiet=
tools.esptool_py.upload.pattern_args=--chip {build.mcu} --port "{serial.port}" --baud {upload.speed} {upload.flags} --before default_reset --after hard_reset write_flash {upload.erase_cmd} -z --flash_mode keep --flash_freq keep --flash_size keep {build.bootloader_addr} "{build.path}/{build.project_name}.bootloader.bin" 0x8000 "{build.path}/{build.project_name}.partitions.bin" 0xe000 "{runtime.platform.path}/tools/partitions/boot_app0.bin" 0x10000 "{build.path}/{build.project_name}.bin" {upload.extra_flags}
tools.esptool_py.upload.pattern_args=--chip {build.mcu} --port "{serial.port}" --baud {upload.speed} {upload.flags} --before default-reset --after hard-reset write-flash {upload.erase_cmd} -z --flash-mode keep --flash-freq keep --flash-size keep {build.bootloader_addr} "{build.path}/{build.project_name}.bootloader.bin" 0x8000 "{build.path}/{build.project_name}.partitions.bin" 0xe000 "{runtime.platform.path}/tools/partitions/boot_app0.bin" 0x10000 "{build.path}/{build.project_name}.bin" {upload.extra_flags}
tools.esptool_py.upload.pattern="{path}/{cmd}" {upload.pattern_args}
## Program Application
## -------------------
tools.esptool_py.program.params.verbose=
tools.esptool_py.program.params.quiet=
tools.esptool_py.program.pattern_args=--chip {build.mcu} --port "{serial.port}" --baud {upload.speed} {upload.flags} --before default_reset --after hard_reset write_flash -z --flash_mode keep --flash_freq keep --flash_size keep 0x10000 "{build.path}/{build.project_name}.bin"
tools.esptool_py.program.pattern_args=--chip {build.mcu} --port "{serial.port}" --baud {upload.speed} {upload.flags} --before default-reset --after hard-reset write-flash -z --flash-mode keep --flash-freq keep --flash-size keep 0x10000 "{build.path}/{build.project_name}.bin"
tools.esptool_py.program.pattern="{path}/{cmd}" {program.pattern_args}
## Erase Chip (before burning the bootloader)
@ -309,7 +309,7 @@ tools.esptool_py.program.pattern="{path}/{cmd}" {program.pattern_args}
tools.esptool_py.erase.protocol=serial
tools.esptool_py.erase.params.verbose=
tools.esptool_py.erase.params.quiet=
tools.esptool_py.erase.pattern_args=--chip {build.mcu} --port "{serial.port}" --baud {upload.speed} {upload.flags} --before default_reset --after hard_reset erase_flash
tools.esptool_py.erase.pattern_args=--chip {build.mcu} --port "{serial.port}" --baud {upload.speed} {upload.flags} --before default-reset --after hard-reset erase-flash
tools.esptool_py.erase.pattern="{path}/{cmd}" {erase.pattern_args}
## Burn Bootloader