Update to use esptool-js for the backend flash writing

.github/workflows/build.yml

@@ -4,6 +4,8 @@ on:
   push:
     branches:
       - main
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
 
 jobs:
   build:

base_installer.js

@@ -5,8 +5,7 @@
 'use strict';
 import {html, render} from 'https://cdn.jsdelivr.net/npm/lit-html/+esm';
 import {asyncAppend} from 'https://cdn.jsdelivr.net/npm/lit-html/directives/async-append/+esm';
-import * as esptoolPackage from "https://cdn.jsdelivr.net/npm/esp-web-flasher@5.1.2/dist/web/index.js/+esm"
+import { ESPLoader, Transport } from "https://unpkg.com/esptool-js@0.5.1/bundle.js";
-
 export const ESP_ROM_BAUD = 115200;
 
 export class InstallButton extends HTMLButtonElement {
@@ -15,12 +14,15 @@ export class InstallButton extends HTMLButtonElement {
 
     constructor() {
         super();
+        this.baudRate = ESP_ROM_BAUD;
         this.dialogElements = {};
         this.currentFlow = null;
         this.currentStep = 0;
         this.currentDialogElement = null;
-        this.port = null;
+        this.device = null;
-        this.espStub = null;
+        this.transport = null;
+        this.esploader = null;
+        this.chip = null;
         this.dialogCssClass = "install-dialog";
         this.connected = this.connectionStates.DISCONNECTED;
         this.menuTitle = "Installer Menu";
@@ -230,7 +232,12 @@ export class InstallButton extends HTMLButtonElement {
             buttonElement.id = this.createIdFromLabel(button.label);
             buttonElement.addEventListener("click", async (e) => {
                 e.preventDefault();
-                await button.onClick.bind(this)();
+                if (button.onClick instanceof Function) {
+                    await button.onClick.bind(this)();
+                } else if (button.onClick instanceof Array) {
+                    let [func, ...params] = button.onClick;
+                    await func.bind(this)(...params);
+                }
             });
             buttonElement.addEventListener("update", async (e) => {
                 if ("onUpdate" in button) {
@@ -343,13 +350,17 @@ export class InstallButton extends HTMLButtonElement {
         return macAddr.map((value) => value.toString(16).toUpperCase().padStart(2, "0")).join(":");
     }
 
-    async disconnect() {
+    async espDisconnect() {
-        if (this.espStub) {
+        if (this.transport) {
-            await espStub.disconnect();
+            await this.transport.disconnect();
-            await espStub.port.close();
+            await this.transport.waitForUnlock(1500);
-            this.updateUIConnected(this.connectionStates.DISCONNECTED);
+            this.updateEspConnected(this.connectionStates.DISCONNECTED);
-            this.espStub = null;
+            this.transport = null;
+            this.device = null;
+            this.chip = null;
+            return true;
         }
+        return false;
     }
 
     async runFlow(flow) {
@@ -390,6 +401,17 @@ export class InstallButton extends HTMLButtonElement {
         }
     }
 
+    async advanceSteps(stepCount) {
+        if (!this.currentFlow) {
+            return;
+        }
+
+        if (this.currentStep <= this.currentFlow.steps.length + stepCount) {
+            this.currentStep += stepCount;
+            await this.currentFlow.steps[this.currentStep].bind(this)();
+        }
+    }
+
     async showMenu() {
         // Display Menu
         this.showDialog(this.dialogs.menu);
@@ -405,38 +427,60 @@ export class InstallButton extends HTMLButtonElement {
         this.showDialog(this.dialogs.error, {message: message});
     }
 
-    async setBaudRateIfChipSupports(chipType, baud) {
+    async setBaudRateIfChipSupports(baud) {
-        if (baud == ESP_ROM_BAUD) { return } // already the default
+        if (baud == this.baudRate) { return } // already the current setting
-
-        if (chipType == esptoolPackage.CHIP_FAMILY_ESP32) { // only supports the default
-            this.logMsg(`ESP32 Chip only works at 115200 instead of the preferred ${baud}. Staying at 115200...`);
-            return
-        }
 
         await this.changeBaudRate(baud);
     }
 
     async changeBaudRate(baud) {
-        if (this.espStub && this.baudRates.includes(baud)) {
+        if (this.baudRates.includes(baud)) {
-            await this.espStub.setBaudrate(baud);
+            if (this.transport == null) {
+                this.baudRate = baud;
+            } else {
+                this.errorMsg("Cannot change baud rate while connected.");
+            }
         }
     }
 
-    async espHardReset(bootloader = false) {
+    async espHardReset() {
-        if (this.espStub) {
+        if (this.esploader) {
-            await this.espStub.hardReset(bootloader);
+            await this.esploader.hardReset();
         }
     }
 
     async espConnect(logger) {
-        // - Request a port and open a connection.
-        this.port = await navigator.serial.requestPort();
-
         logger.log("Connecting...");
-        await this.port.open({ baudRate: ESP_ROM_BAUD });
+
+        if (this.device === null) {
+            this.device = await navigator.serial.requestPort({});
+            this.transport = new Transport(this.device, true);
+        }
+
+        const espLoaderTerminal = {
+            clean() {
+                // Clear the terminal
+            },
+            writeLine(data) {
+                logger.log(data);
+            },
+            write(data) {
+                logger.log(data);
+            },
+        };
+
+        const loaderOptions = {
+            transport: this.transport,
+            baudrate: this.baudRate,
+            terminal: espLoaderTerminal,
+            debugLogging: false,
+        };
+
+        this.esploader = new ESPLoader(loaderOptions);
+        this.chip = await this.esploader.main();
 
         logger.log("Connected successfully.");
 
-        return new esptoolPackage.ESPLoader(this.port, logger);
+        return this.esploader;
     };
 }

cpinstaller.js

@@ -7,8 +7,8 @@ import { html } from 'https://cdn.jsdelivr.net/npm/lit-html/+esm';
 import { map } from 'https://cdn.jsdelivr.net/npm/lit-html/directives/map/+esm';
 import * as toml from "https://cdn.jsdelivr.net/npm/iarna-toml-esm@3.0.5/+esm"
 import * as zip from "https://cdn.jsdelivr.net/npm/@zip.js/zip.js@2.6.65/+esm";
-import * as esptoolPackage from "https://cdn.jsdelivr.net/npm/esp-web-flasher@5.1.2/dist/web/index.js/+esm"
+import { default as CryptoJS } from "https://cdn.jsdelivr.net/npm/crypto-js@4.1.1/+esm";
-import { REPL } from 'https://cdn.jsdelivr.net/gh/adafruit/circuitpython-repl-js/repl.js';
+import { REPL } from 'https://cdn.jsdelivr.net/gh/adafruit/circuitpython-repl-js@3.2.1/repl.js';
 import { InstallButton, ESP_ROM_BAUD } from "./base_installer.js";
 
 // TODO: Combine multiple steps together. For now it was easier to make them separate,
@@ -19,6 +19,8 @@ import { InstallButton, ESP_ROM_BAUD } from "./base_installer.js";
 //
 // TODO: Hide the log and make it accessible via the menu (future feature, output to console for now)
 // May need to deal with the fact that the ESPTool uses Web Serial and CircuitPython REPL uses Web Serial
+//
+// TODO: Update File Operations to take advantage of the REPL FileOps class to allow non-CIRCUITPY drive access
 
 const PREFERRED_BAUDRATE = 921600;
 const COPY_CHUNK_SIZE = 64 * 1024; // 64 KB Chunks
@@ -26,12 +28,6 @@ const DEFAULT_RELEASE_LATEST = false;   // Use the latest release or the stable
 const BOARD_DEFS = "https://adafruit-circuit-python.s3.amazonaws.com/esp32_boards.json";
 
 const CSS_DIALOG_CLASS = "cp-installer-dialog";
-const FAMILY_TO_CHIP_MAP = {
-    'esp32s2': esptoolPackage.CHIP_FAMILY_ESP32S2,
-    'esp32s3': esptoolPackage.CHIP_FAMILY_ESP32S3,
-    'esp32c3': esptoolPackage.CHIP_FAMILY_ESP32C3,
-    'esp32': esptoolPackage.CHIP_FAMILY_ESP32
-}
 
 const attrMap = {
     "bootloader": "bootloaderUrl",
@@ -214,12 +210,12 @@ export class CPInstallButton extends InstallButton {
             isEnabled: async () => { return !this.hasNativeUsb() && !!this.binFileUrl },
         },
         uf2Only: { // Upgrade when Bootloader is already installer
-            label: `Upgrade/Install CircuitPython [version] UF2 Only`,
+            label: `Install CircuitPython [version] UF2 Only`,
             steps: [this.stepWelcome, this.stepSelectBootDrive, this.stepCopyUf2, this.stepSelectCpyDrive, this.stepCredentials, this.stepSuccess],
             isEnabled: async () => { return this.hasNativeUsb() && !!this.uf2FileUrl },
         },
         binOnly: {
-            label: `Upgrade CircuitPython [version] Bin Only`,
+            label: `Install CircuitPython [version] Bin Only`,
             steps: [this.stepWelcome, this.stepSerialConnect, this.stepConfirm, this.stepEraseAll, this.stepFlashBin, this.stepSuccess],
             isEnabled: async () => { return !!this.binFileUrl },
         },
@@ -311,6 +307,10 @@ export class CPInstallButton extends InstallButton {
             `,
             buttons: [
                 this.previousButton,
+                {
+                    label: "Skip Erase",
+                    onClick: [this.advanceSteps, 2],
+                },
                 {
                     label: "Continue",
                     onClick: this.nextStep,
@@ -481,7 +481,7 @@ export class CPInstallButton extends InstallButton {
             action: "Erasing Flash",
         });
         try {
-            await this.espStub.eraseFlash();
+            await this.esploader.eraseFlash();
         } catch (err) {
             this.errorMsg("Unable to finish erasing Flash memory. Please try again.");
         }
@@ -550,8 +550,8 @@ export class CPInstallButton extends InstallButton {
 
     async stepSetupRepl() {
         // TODO: Try and reuse the existing connection so user doesn't need to select it again
-        /*if (this.port) {
+        /*if (this.device) {
-            this.replSerialDevice = this.port;
+            this.replSerialDevice = this.device;
             await this.setupRepl();
         }*/
         const serialPortName = await this.getSerialPortName();
@@ -579,6 +579,7 @@ export class CPInstallButton extends InstallButton {
             // TODO: Currently the control is just disabled and not used because we don't have anything to modify boot.py in place.
             // Setting mass_storage_disabled to true/false will display the checkbox with the appropriately checked state.
             //parameters.mass_storage_disabled = true;
+            // This can be updated to use FileOps for ease of implementation
         }
 
         // Display Credentials Request Dialog
@@ -658,45 +659,33 @@ export class CPInstallButton extends InstallButton {
     async espToolConnectHandler(e) {
         await this.onReplDisconnected(e);
         await this.espDisconnect();
-        let esploader;
+        await this.setBaudRateIfChipSupports(PREFERRED_BAUDRATE);
         try {
-            esploader = await this.espConnect({
+            this.updateEspConnected(this.connectionStates.CONNECTING);
+            await this.espConnect({
                 log: (...args) => this.logMsg(...args),
                 debug: (...args) => {},
                 error: (...args) => this.errorMsg(...args),
             });
+            this.updateEspConnected(this.connectionStates.CONNECTED);
         } catch (err) {
             // It's possible the dialog was also canceled here
+            this.updateEspConnected(this.connectionStates.DISCONNECTED);
             this.errorMsg("Unable to open Serial connection to board. Make sure the port is not already in use by another application or in another browser tab.");
             return;
         }
 
         try {
-            this.updateEspConnected(this.connectionStates.CONNECTING);
+            this.logMsg(`Connected to ${this.esploader.chip}`);
-            await esploader.initialize();
+            this.logMsg(`MAC Address: ${this.formatMacAddr(this.esploader.chip.readMac(this.esploader))}`);
-            this.updateEspConnected(this.connectionStates.CONNECTED);
-        } catch (err) {
-            await esploader.disconnect();
-            // Disconnection before complete
-            this.updateEspConnected(this.connectionStates.DISCONNECTED);
-            this.errorMsg("Unable to connect to the board. Make sure it is in bootloader mode by holding the boot0 button when powering on and try again.")
-            return;
-        }
-
-        try {
-            this.logMsg(`Connected to ${esploader.chipName}`);
-            this.logMsg(`MAC Address: ${this.formatMacAddr(esploader.macAddr())}`);
 
             // check chip compatibility
-            if (FAMILY_TO_CHIP_MAP[this.chipFamily] == esploader.chipFamily) {
+            if (this.chipFamily == `${this.esploader.chip}`) {
                 this.logMsg("This chip checks out");
-                this.espStub = await esploader.runStub();
+                this.esploader.addEventListener("disconnect", () => {
-                this.espStub.addEventListener("disconnect", () => {
                     this.updateEspConnected(this.connectionStates.DISCONNECTED);
-                    this.espStub = null;
                 });
 
-                await this.setBaudRateIfChipSupports(esploader.chipFamily, PREFERRED_BAUDRATE);
                 await this.nextStep();
                 return;
             }
@@ -706,7 +695,9 @@ export class CPInstallButton extends InstallButton {
             await this.espDisconnect();
 
         } catch (err) {
-            await esploader.disconnect();
+            if (this.transport) {
+                await this.transport.disconnect();
+            }
             // Disconnection before complete
             this.updateEspConnected(this.connectionStates.DISCONNECTED);
             this.errorMsg("Oops, we lost connection to your board before completing the install. Please check your USB connection and click Connect again. Refresh the browser if it becomes unresponsive.")
@@ -1024,10 +1015,28 @@ export class CPInstallButton extends InstallButton {
 
     async downloadAndInstall(url, fileToExtract = null, cacheFile = false) {
         let [filename, fileBlob] = await this.downloadAndExtract(url, fileToExtract, cacheFile);
+        const fileArray = [];
+
+        const readBlobAsBinaryString = (inputFile) => {
+            const reader = new FileReader();
+
+            return new Promise((resolve, reject) => {
+                reader.onerror = () => {
+                    reader.abort();
+                    reject(new DOMException("Problem parsing input file."));
+                };
+
+                reader.onload = () => {
+                    resolve(reader.result);
+                };
+                reader.readAsBinaryString(inputFile);
+            });
+        };
 
         // Update the Progress dialog
         if (fileBlob) {
-            const fileContents = (new Uint8Array(await fileBlob.arrayBuffer())).buffer;
+            fileArray.push({ data: await readBlobAsBinaryString(fileBlob), address: 0 });
+
             let lastPercent = 0;
             this.showDialog(this.dialogs.actionProgress, {
                 action: `Flashing ${filename}`
@@ -1037,14 +1046,22 @@ export class CPInstallButton extends InstallButton {
             progressElement.value = 0;
 
             try {
-                await this.espStub.flashData(fileContents, (bytesWritten, totalBytes) => {
+                const flashOptions = {
-                    let percentage = Math.round((bytesWritten / totalBytes) * 100);
+                    fileArray: fileArray,
-                    if (percentage > lastPercent) {
+                    flashSize: "keep",
-                        progressElement.value = percentage;
+                    eraseAll: false,
-                        this.logMsg(`${percentage}% (${bytesWritten}/${totalBytes})...`);
+                    compress: true,
-                        lastPercent = percentage;
+                    reportProgress: (fileIndex, written, total) => {
-                    }
+                        let percentage = Math.round((written / total) * 100);
-                }, 0, 0);
+                        if (percentage > lastPercent) {
+                            progressElement.value = percentage;
+                            this.logMsg(`${percentage}% (${written}/${total})...`);
+                            lastPercent = percentage;
+                        }
+                    },
+                    calculateMD5Hash: (image) => CryptoJS.MD5(CryptoJS.enc.Latin1.parse(image)),
+                };
+                await this.esploader.writeFlash(flashOptions);
             } catch (err) {
                 this.errorMsg(`Unable to flash file: ${filename}. Error Message: ${err}`);
             }
@@ -1256,20 +1273,6 @@ export class CPInstallButton extends InstallButton {
         return {};
     }
 
-    async espDisconnect() {
-        // Disconnect the ESPTool
-        if (this.espStub) {
-            await this.espStub.disconnect();
-            this.espStub.removeEventListener("disconnect", this.espDisconnect.bind(this));
-            this.updateEspConnected(this.connectionStates.DISCONNECTED);
-            this.espStub = null;
-        }
-        if (this.port) {
-            await this.port.close();
-            this.port = null;
-        }
-    }
-
     async serialTransmit(msg) {
         const encoder = new TextEncoder();
         if (this.writer) {