From dfbbffa639ff3a139328cecf45a47d59c033b3a3 Mon Sep 17 00:00:00 2001 From: Melissa LeBlanc-Williams Date: Fri, 13 Dec 2024 14:48:55 -0800 Subject: [PATCH 1/6] Update to use esptool-js for the backend flash writing --- .github/workflows/build.yml | 2 + base_installer.js | 100 +++++++++++++++++++++--------- cpinstaller.js | 119 ++++++++++++++++++------------------ 3 files changed, 135 insertions(+), 86 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 72a14e8..af1f23b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -4,6 +4,8 @@ on: push: branches: - main + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: jobs: build: diff --git a/base_installer.js b/base_installer.js index a2f21d9..10db9e7 100644 --- a/base_installer.js +++ b/base_installer.js @@ -5,8 +5,7 @@ 'use strict'; import {html, render} from 'https://cdn.jsdelivr.net/npm/lit-html/+esm'; import {asyncAppend} from 'https://cdn.jsdelivr.net/npm/lit-html/directives/async-append/+esm'; -import * as esptoolPackage from "https://cdn.jsdelivr.net/npm/esp-web-flasher@5.1.2/dist/web/index.js/+esm" - +import { ESPLoader, Transport } from "https://unpkg.com/esptool-js@0.5.1/bundle.js"; export const ESP_ROM_BAUD = 115200; export class InstallButton extends HTMLButtonElement { @@ -15,12 +14,15 @@ export class InstallButton extends HTMLButtonElement { constructor() { super(); + this.baudRate = ESP_ROM_BAUD; this.dialogElements = {}; this.currentFlow = null; this.currentStep = 0; this.currentDialogElement = null; - this.port = null; - this.espStub = null; + this.device = null; + this.transport = null; + this.esploader = null; + this.chip = null; this.dialogCssClass = "install-dialog"; this.connected = this.connectionStates.DISCONNECTED; this.menuTitle = "Installer Menu"; @@ -230,7 +232,12 @@ export class InstallButton extends HTMLButtonElement { buttonElement.id = this.createIdFromLabel(button.label); buttonElement.addEventListener("click", async (e) => { e.preventDefault(); - await button.onClick.bind(this)(); + if (button.onClick instanceof Function) { + await button.onClick.bind(this)(); + } else if (button.onClick instanceof Array) { + let [func, ...params] = button.onClick; + await func.bind(this)(...params); + } }); buttonElement.addEventListener("update", async (e) => { if ("onUpdate" in button) { @@ -343,13 +350,17 @@ export class InstallButton extends HTMLButtonElement { return macAddr.map((value) => value.toString(16).toUpperCase().padStart(2, "0")).join(":"); } - async disconnect() { - if (this.espStub) { - await espStub.disconnect(); - await espStub.port.close(); - this.updateUIConnected(this.connectionStates.DISCONNECTED); - this.espStub = null; + async espDisconnect() { + if (this.transport) { + await this.transport.disconnect(); + await this.transport.waitForUnlock(1500); + this.updateEspConnected(this.connectionStates.DISCONNECTED); + this.transport = null; + this.device = null; + this.chip = null; + return true; } + return false; } async runFlow(flow) { @@ -390,6 +401,17 @@ export class InstallButton extends HTMLButtonElement { } } + async advanceSteps(stepCount) { + if (!this.currentFlow) { + return; + } + + if (this.currentStep <= this.currentFlow.steps.length + stepCount) { + this.currentStep += stepCount; + await this.currentFlow.steps[this.currentStep].bind(this)(); + } + } + async showMenu() { // Display Menu this.showDialog(this.dialogs.menu); @@ -405,38 +427,60 @@ export class InstallButton extends HTMLButtonElement { this.showDialog(this.dialogs.error, {message: message}); } - async setBaudRateIfChipSupports(chipType, baud) { - if (baud == ESP_ROM_BAUD) { return } // already the default - - if (chipType == esptoolPackage.CHIP_FAMILY_ESP32) { // only supports the default - this.logMsg(`ESP32 Chip only works at 115200 instead of the preferred ${baud}. Staying at 115200...`); - return - } + async setBaudRateIfChipSupports(baud) { + if (baud == this.baudRate) { return } // already the current setting await this.changeBaudRate(baud); } async changeBaudRate(baud) { - if (this.espStub && this.baudRates.includes(baud)) { - await this.espStub.setBaudrate(baud); + if (this.baudRates.includes(baud)) { + if (this.transport == null) { + this.baudRate = baud; + } else { + this.errorMsg("Cannot change baud rate while connected."); + } } } - async espHardReset(bootloader = false) { - if (this.espStub) { - await this.espStub.hardReset(bootloader); + async espHardReset() { + if (this.esploader) { + await this.esploader.hardReset(); } } async espConnect(logger) { - // - Request a port and open a connection. - this.port = await navigator.serial.requestPort(); - logger.log("Connecting..."); - await this.port.open({ baudRate: ESP_ROM_BAUD }); + + if (this.device === null) { + this.device = await navigator.serial.requestPort({}); + this.transport = new Transport(this.device, true); + } + + const espLoaderTerminal = { + clean() { + // Clear the terminal + }, + writeLine(data) { + logger.log(data); + }, + write(data) { + logger.log(data); + }, + }; + + const loaderOptions = { + transport: this.transport, + baudrate: this.baudRate, + terminal: espLoaderTerminal, + debugLogging: false, + }; + + this.esploader = new ESPLoader(loaderOptions); + this.chip = await this.esploader.main(); logger.log("Connected successfully."); - return new esptoolPackage.ESPLoader(this.port, logger); + return this.esploader; }; } \ No newline at end of file diff --git a/cpinstaller.js b/cpinstaller.js index eb14e9c..bc33514 100644 --- a/cpinstaller.js +++ b/cpinstaller.js @@ -7,8 +7,8 @@ import { html } from 'https://cdn.jsdelivr.net/npm/lit-html/+esm'; import { map } from 'https://cdn.jsdelivr.net/npm/lit-html/directives/map/+esm'; import * as toml from "https://cdn.jsdelivr.net/npm/iarna-toml-esm@3.0.5/+esm" import * as zip from "https://cdn.jsdelivr.net/npm/@zip.js/zip.js@2.6.65/+esm"; -import * as esptoolPackage from "https://cdn.jsdelivr.net/npm/esp-web-flasher@5.1.2/dist/web/index.js/+esm" -import { REPL } from 'https://cdn.jsdelivr.net/gh/adafruit/circuitpython-repl-js/repl.js'; +import { default as CryptoJS } from "https://cdn.jsdelivr.net/npm/crypto-js@4.1.1/+esm"; +import { REPL } from 'https://cdn.jsdelivr.net/gh/adafruit/circuitpython-repl-js@3.2.1/repl.js'; import { InstallButton, ESP_ROM_BAUD } from "./base_installer.js"; // TODO: Combine multiple steps together. For now it was easier to make them separate, @@ -19,6 +19,8 @@ import { InstallButton, ESP_ROM_BAUD } from "./base_installer.js"; // // TODO: Hide the log and make it accessible via the menu (future feature, output to console for now) // May need to deal with the fact that the ESPTool uses Web Serial and CircuitPython REPL uses Web Serial +// +// TODO: Update File Operations to take advantage of the REPL FileOps class to allow non-CIRCUITPY drive access const PREFERRED_BAUDRATE = 921600; const COPY_CHUNK_SIZE = 64 * 1024; // 64 KB Chunks @@ -26,12 +28,6 @@ const DEFAULT_RELEASE_LATEST = false; // Use the latest release or the stable const BOARD_DEFS = "https://adafruit-circuit-python.s3.amazonaws.com/esp32_boards.json"; const CSS_DIALOG_CLASS = "cp-installer-dialog"; -const FAMILY_TO_CHIP_MAP = { - 'esp32s2': esptoolPackage.CHIP_FAMILY_ESP32S2, - 'esp32s3': esptoolPackage.CHIP_FAMILY_ESP32S3, - 'esp32c3': esptoolPackage.CHIP_FAMILY_ESP32C3, - 'esp32': esptoolPackage.CHIP_FAMILY_ESP32 -} const attrMap = { "bootloader": "bootloaderUrl", @@ -214,12 +210,12 @@ export class CPInstallButton extends InstallButton { isEnabled: async () => { return !this.hasNativeUsb() && !!this.binFileUrl }, }, uf2Only: { // Upgrade when Bootloader is already installer - label: `Upgrade/Install CircuitPython [version] UF2 Only`, + label: `Install CircuitPython [version] UF2 Only`, steps: [this.stepWelcome, this.stepSelectBootDrive, this.stepCopyUf2, this.stepSelectCpyDrive, this.stepCredentials, this.stepSuccess], isEnabled: async () => { return this.hasNativeUsb() && !!this.uf2FileUrl }, }, binOnly: { - label: `Upgrade CircuitPython [version] Bin Only`, + label: `Install CircuitPython [version] Bin Only`, steps: [this.stepWelcome, this.stepSerialConnect, this.stepConfirm, this.stepEraseAll, this.stepFlashBin, this.stepSuccess], isEnabled: async () => { return !!this.binFileUrl }, }, @@ -311,6 +307,10 @@ export class CPInstallButton extends InstallButton { `, buttons: [ this.previousButton, + { + label: "Skip Erase", + onClick: [this.advanceSteps, 2], + }, { label: "Continue", onClick: this.nextStep, @@ -481,7 +481,7 @@ export class CPInstallButton extends InstallButton { action: "Erasing Flash", }); try { - await this.espStub.eraseFlash(); + await this.esploader.eraseFlash(); } catch (err) { this.errorMsg("Unable to finish erasing Flash memory. Please try again."); } @@ -550,8 +550,8 @@ export class CPInstallButton extends InstallButton { async stepSetupRepl() { // TODO: Try and reuse the existing connection so user doesn't need to select it again - /*if (this.port) { - this.replSerialDevice = this.port; + /*if (this.device) { + this.replSerialDevice = this.device; await this.setupRepl(); }*/ const serialPortName = await this.getSerialPortName(); @@ -579,6 +579,7 @@ export class CPInstallButton extends InstallButton { // TODO: Currently the control is just disabled and not used because we don't have anything to modify boot.py in place. // Setting mass_storage_disabled to true/false will display the checkbox with the appropriately checked state. //parameters.mass_storage_disabled = true; + // This can be updated to use FileOps for ease of implementation } // Display Credentials Request Dialog @@ -658,45 +659,33 @@ export class CPInstallButton extends InstallButton { async espToolConnectHandler(e) { await this.onReplDisconnected(e); await this.espDisconnect(); - let esploader; + await this.setBaudRateIfChipSupports(PREFERRED_BAUDRATE); try { - esploader = await this.espConnect({ + this.updateEspConnected(this.connectionStates.CONNECTING); + await this.espConnect({ log: (...args) => this.logMsg(...args), debug: (...args) => {}, error: (...args) => this.errorMsg(...args), }); + this.updateEspConnected(this.connectionStates.CONNECTED); } catch (err) { // It's possible the dialog was also canceled here + this.updateEspConnected(this.connectionStates.DISCONNECTED); this.errorMsg("Unable to open Serial connection to board. Make sure the port is not already in use by another application or in another browser tab."); return; } try { - this.updateEspConnected(this.connectionStates.CONNECTING); - await esploader.initialize(); - this.updateEspConnected(this.connectionStates.CONNECTED); - } catch (err) { - await esploader.disconnect(); - // Disconnection before complete - this.updateEspConnected(this.connectionStates.DISCONNECTED); - this.errorMsg("Unable to connect to the board. Make sure it is in bootloader mode by holding the boot0 button when powering on and try again.") - return; - } - - try { - this.logMsg(`Connected to ${esploader.chipName}`); - this.logMsg(`MAC Address: ${this.formatMacAddr(esploader.macAddr())}`); + this.logMsg(`Connected to ${this.esploader.chip}`); + this.logMsg(`MAC Address: ${this.formatMacAddr(this.esploader.chip.readMac(this.esploader))}`); // check chip compatibility - if (FAMILY_TO_CHIP_MAP[this.chipFamily] == esploader.chipFamily) { + if (this.chipFamily == `${this.esploader.chip}`) { this.logMsg("This chip checks out"); - this.espStub = await esploader.runStub(); - this.espStub.addEventListener("disconnect", () => { + this.esploader.addEventListener("disconnect", () => { this.updateEspConnected(this.connectionStates.DISCONNECTED); - this.espStub = null; }); - await this.setBaudRateIfChipSupports(esploader.chipFamily, PREFERRED_BAUDRATE); await this.nextStep(); return; } @@ -706,7 +695,9 @@ export class CPInstallButton extends InstallButton { await this.espDisconnect(); } catch (err) { - await esploader.disconnect(); + if (this.transport) { + await this.transport.disconnect(); + } // Disconnection before complete this.updateEspConnected(this.connectionStates.DISCONNECTED); this.errorMsg("Oops, we lost connection to your board before completing the install. Please check your USB connection and click Connect again. Refresh the browser if it becomes unresponsive.") @@ -1024,10 +1015,28 @@ export class CPInstallButton extends InstallButton { async downloadAndInstall(url, fileToExtract = null, cacheFile = false) { let [filename, fileBlob] = await this.downloadAndExtract(url, fileToExtract, cacheFile); + const fileArray = []; + + const readBlobAsBinaryString = (inputFile) => { + const reader = new FileReader(); + + return new Promise((resolve, reject) => { + reader.onerror = () => { + reader.abort(); + reject(new DOMException("Problem parsing input file.")); + }; + + reader.onload = () => { + resolve(reader.result); + }; + reader.readAsBinaryString(inputFile); + }); + }; // Update the Progress dialog if (fileBlob) { - const fileContents = (new Uint8Array(await fileBlob.arrayBuffer())).buffer; + fileArray.push({ data: await readBlobAsBinaryString(fileBlob), address: 0 }); + let lastPercent = 0; this.showDialog(this.dialogs.actionProgress, { action: `Flashing ${filename}` @@ -1037,14 +1046,22 @@ export class CPInstallButton extends InstallButton { progressElement.value = 0; try { - await this.espStub.flashData(fileContents, (bytesWritten, totalBytes) => { - let percentage = Math.round((bytesWritten / totalBytes) * 100); - if (percentage > lastPercent) { - progressElement.value = percentage; - this.logMsg(`${percentage}% (${bytesWritten}/${totalBytes})...`); - lastPercent = percentage; - } - }, 0, 0); + const flashOptions = { + fileArray: fileArray, + flashSize: "keep", + eraseAll: false, + compress: true, + reportProgress: (fileIndex, written, total) => { + let percentage = Math.round((written / total) * 100); + if (percentage > lastPercent) { + progressElement.value = percentage; + this.logMsg(`${percentage}% (${written}/${total})...`); + lastPercent = percentage; + } + }, + calculateMD5Hash: (image) => CryptoJS.MD5(CryptoJS.enc.Latin1.parse(image)), + }; + await this.esploader.writeFlash(flashOptions); } catch (err) { this.errorMsg(`Unable to flash file: ${filename}. Error Message: ${err}`); } @@ -1256,20 +1273,6 @@ export class CPInstallButton extends InstallButton { return {}; } - async espDisconnect() { - // Disconnect the ESPTool - if (this.espStub) { - await this.espStub.disconnect(); - this.espStub.removeEventListener("disconnect", this.espDisconnect.bind(this)); - this.updateEspConnected(this.connectionStates.DISCONNECTED); - this.espStub = null; - } - if (this.port) { - await this.port.close(); - this.port = null; - } - } - async serialTransmit(msg) { const encoder = new TextEncoder(); if (this.writer) { From ca8279779bd1352d75b7df305d1c226c6ed257ad Mon Sep 17 00:00:00 2001 From: Melissa LeBlanc-Williams Date: Tue, 17 Dec 2024 15:09:36 -0800 Subject: [PATCH 2/6] Updated to use esptool-js --- base_installer.js | 7 +------ cpinstaller.js | 21 +++++++++++---------- 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/base_installer.js b/base_installer.js index 10db9e7..8fd6e9e 100644 --- a/base_installer.js +++ b/base_installer.js @@ -232,12 +232,7 @@ export class InstallButton extends HTMLButtonElement { buttonElement.id = this.createIdFromLabel(button.label); buttonElement.addEventListener("click", async (e) => { e.preventDefault(); - if (button.onClick instanceof Function) { - await button.onClick.bind(this)(); - } else if (button.onClick instanceof Array) { - let [func, ...params] = button.onClick; - await func.bind(this)(...params); - } + await button.onClick.bind(this)(); }); buttonElement.addEventListener("update", async (e) => { if ("onUpdate" in button) { diff --git a/cpinstaller.js b/cpinstaller.js index bc33514..3fd0321 100644 --- a/cpinstaller.js +++ b/cpinstaller.js @@ -309,7 +309,7 @@ export class CPInstallButton extends InstallButton { this.previousButton, { label: "Skip Erase", - onClick: [this.advanceSteps, 2], + onClick: async (e) => { if (confirm("Skipping the erase step may cause issues and is not recommended. Continue?")) { await this.advanceSteps(2); }}, }, { label: "Continue", @@ -675,16 +675,17 @@ export class CPInstallButton extends InstallButton { return; } - try { - this.logMsg(`Connected to ${this.esploader.chip}`); - this.logMsg(`MAC Address: ${this.formatMacAddr(this.esploader.chip.readMac(this.esploader))}`); + //try { + this.logMsg(`Connected to ${this.esploader.chip.CHIP_NAME}`); // check chip compatibility - if (this.chipFamily == `${this.esploader.chip}`) { + if (this.chipFamily == `${this.esploader.chip.CHIP_NAME}`.toLowerCase()) { this.logMsg("This chip checks out"); - this.esploader.addEventListener("disconnect", () => { - this.updateEspConnected(this.connectionStates.DISCONNECTED); - }); + + // esploader-js doesn't have a disconnect event, so we can't use this + //this.esploader.addEventListener("disconnect", () => { + // this.updateEspConnected(this.connectionStates.DISCONNECTED); + //}); await this.nextStep(); return; @@ -694,14 +695,14 @@ export class CPInstallButton extends InstallButton { this.errorMsg("Oops, this is the wrong firmware for your board.") await this.espDisconnect(); - } catch (err) { + /*} catch (err) { if (this.transport) { await this.transport.disconnect(); } // Disconnection before complete this.updateEspConnected(this.connectionStates.DISCONNECTED); this.errorMsg("Oops, we lost connection to your board before completing the install. Please check your USB connection and click Connect again. Refresh the browser if it becomes unresponsive.") - } + }*/ } async onSerialReceive(e) { From 3e3e8641376da6040bf7c3c24c52315f8023af95 Mon Sep 17 00:00:00 2001 From: Melissa LeBlanc-Williams Date: Tue, 17 Dec 2024 15:26:45 -0800 Subject: [PATCH 3/6] Reenable a try/catch --- cpinstaller.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cpinstaller.js b/cpinstaller.js index 3fd0321..71df7c3 100644 --- a/cpinstaller.js +++ b/cpinstaller.js @@ -675,7 +675,7 @@ export class CPInstallButton extends InstallButton { return; } - //try { + try { this.logMsg(`Connected to ${this.esploader.chip.CHIP_NAME}`); // check chip compatibility @@ -695,14 +695,14 @@ export class CPInstallButton extends InstallButton { this.errorMsg("Oops, this is the wrong firmware for your board.") await this.espDisconnect(); - /*} catch (err) { + } catch (err) { if (this.transport) { await this.transport.disconnect(); } // Disconnection before complete this.updateEspConnected(this.connectionStates.DISCONNECTED); this.errorMsg("Oops, we lost connection to your board before completing the install. Please check your USB connection and click Connect again. Refresh the browser if it becomes unresponsive.") - }*/ + } } async onSerialReceive(e) { From 78afec369e2867d94583014ad6e51a273c371096 Mon Sep 17 00:00:00 2001 From: Melissa LeBlanc-Williams Date: Tue, 31 Dec 2024 10:52:03 -0800 Subject: [PATCH 4/6] Update to esptool-js 0.5.3 --- base_installer.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base_installer.js b/base_installer.js index 8fd6e9e..bbaf823 100644 --- a/base_installer.js +++ b/base_installer.js @@ -5,7 +5,7 @@ 'use strict'; import {html, render} from 'https://cdn.jsdelivr.net/npm/lit-html/+esm'; import {asyncAppend} from 'https://cdn.jsdelivr.net/npm/lit-html/directives/async-append/+esm'; -import { ESPLoader, Transport } from "https://unpkg.com/esptool-js@0.5.1/bundle.js"; +import { ESPLoader, Transport } from "https://unpkg.com/esptool-js@0.5.3/bundle.js"; export const ESP_ROM_BAUD = 115200; export class InstallButton extends HTMLButtonElement { From af10e21877800b7cea04402e79c496572c22979e Mon Sep 17 00:00:00 2001 From: Melissa LeBlanc-Williams Date: Tue, 31 Dec 2024 13:27:32 -0800 Subject: [PATCH 5/6] Fix ESP32-C6 --- cpinstaller.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cpinstaller.js b/cpinstaller.js index 71df7c3..233c5ac 100644 --- a/cpinstaller.js +++ b/cpinstaller.js @@ -679,7 +679,8 @@ export class CPInstallButton extends InstallButton { this.logMsg(`Connected to ${this.esploader.chip.CHIP_NAME}`); // check chip compatibility - if (this.chipFamily == `${this.esploader.chip.CHIP_NAME}`.toLowerCase()) { + console.log(this.chipFamily, `${this.esploader.chip.CHIP_NAME}`.toLowerCase()); + if (this.chipFamily == `${this.esploader.chip.CHIP_NAME}`.toLowerCase().replaceAll("-", "")) { this.logMsg("This chip checks out"); // esploader-js doesn't have a disconnect event, so we can't use this From 83d9ff7cb752b293719af62d4f2dc586c843f962 Mon Sep 17 00:00:00 2001 From: Melissa LeBlanc-Williams Date: Fri, 3 Jan 2025 14:46:37 -0800 Subject: [PATCH 6/6] Remove console.log from 1 line --- cpinstaller.js | 1 - 1 file changed, 1 deletion(-) diff --git a/cpinstaller.js b/cpinstaller.js index 233c5ac..a09d391 100644 --- a/cpinstaller.js +++ b/cpinstaller.js @@ -679,7 +679,6 @@ export class CPInstallButton extends InstallButton { this.logMsg(`Connected to ${this.esploader.chip.CHIP_NAME}`); // check chip compatibility - console.log(this.chipFamily, `${this.esploader.chip.CHIP_NAME}`.toLowerCase()); if (this.chipFamily == `${this.esploader.chip.CHIP_NAME}`.toLowerCase().replaceAll("-", "")) { this.logMsg("This chip checks out");