add esp32_programFlashDefl() with File32 from SD card

- upgrade tool/esp_compress.py with --sd option to generate metadata for
use with files on sdcard
- add new program_esp32_uart_from_sdcard that tested on metro m4 (and
pyportal)
This commit is contained in:
hathach 2023-10-12 23:24:29 +07:00
parent c25bc452fd
commit 9a1477f407
11 changed files with 337 additions and 330 deletions

2
.gitignore vendored Normal file
View file

@ -0,0 +1,2 @@
.idea
.pio

View file

@ -1,128 +0,0 @@
// This sketch program ESP32 by flashing bin file from SD Card
// Hardware wiring:
// - Brain GPIO28 <-> ESP32 IO0
// - Brain Reset <-> ESP32 Enable
// - Brain TX/RX <-> ESP32 RX/TX
// required for Host MSC block device
#include "SdFat.h"
// required for USB host
#include "pio_usb.h"
#include "Adafruit_TinyUSB.h"
#include "Adafruit_TestBed_Brains.h"
#define ESP32_RESET 27
#define ESP32_IO0 28
#define ESP32_BAUDRATE 2000000
//#define ESP32_BAUDRATE 921600
//#define ESP32_BAUDRATE 115200
// Bin files on SDCard to program
// These files can be generated by any of Arduino sketches
// The example use WiFiAccessPoint sketch, which we can verify its running image
// by simply scan for existence of ssid "YourAP"
struct {
uint32_t addr;
const char* fpath;
} bin_list [] = {
{ 0x1000 , "esp32/WiFiAccessPoint/WiFiAccessPoint.ino.bootloader.bin" },
{ 0x8000 , "esp32/WiFiAccessPoint/WiFiAccessPoint.ino.partitions.bin" },
{ 0xe000 , "esp32/WiFiAccessPoint/boot_app0.bin" },
{ 0x10000 , "esp32/WiFiAccessPoint/WiFiAccessPoint.ino.bin" },
};
enum {
BIN_LIST_COUNT = sizeof(bin_list)/sizeof(bin_list[0])
};
// Defined an boot rom object that use UART Serial1
ESP32BootROMClass ESP32BootROM(Serial1, ESP32_IO0, ESP32_RESET);
//--------------------------------------------------------------------+
// Setup and Loop on Core0
//--------------------------------------------------------------------+
void prepare_sd(void) {
if (!Brain.SD_detected()) {
Brain.LCD_printf(0, "No SD Card");
while ( !Brain.SD_detected() ) delay(10);
}
if ( !Brain.SD_begin(SD_SCK_MHZ(16)) ) {
Brain.LCD_printf(0, "SD init failed");
while(1) delay(10);
}
Brain.LCD_printf(0, "SD mounted");
// Print out file on SD if Serial is connected
if (Serial) {
Serial.println();
Serial.println("SD Contents:");
Serial.printf("Card size = %0.1f GB\n", 0.000000512 * Brain.SD.card()->sectorCount());
Brain.SD.ls(LS_R | LS_DATE | LS_SIZE);
}
}
void print_speed(size_t count, uint32_t ms) {
Brain.LCD_printf(0, "%.01fKB %.01fs", count/1000.0F, ms / 1000.0F);
Serial.printf("Completed %u bytes in %.02f seconds.\r\n", count, ms / 1000.0F);
Serial.printf("Speed : %.02f KB/s\r\n", (count / 1000.0F) / (ms / 1000.0F));
}
void setup() {
Serial.begin(115200);
while (!Serial) delay(10);
Serial.println("Tester Brains: Programming ESP32 with UART!");
Brain.begin();
// prepare SD Card
prepare_sd();
while ( !Brain.esp32_begin(&ESP32BootROM, ESP32_BAUDRATE) ) {
// retry syncing
delay(100);
}
// Writing bin files
size_t total_bytes = 0;
uint32_t ms = millis();
for(size_t i=0; i<BIN_LIST_COUNT; i++) {
Brain.LCD_printf("Flashing file %u", i);
size_t wr_count = Brain.esp32_programFlash(bin_list[i].fpath, bin_list[i].addr);
total_bytes += wr_count;
if (!wr_count) {
Brain.LCD_printf_error("Failed to flash");
}
}
print_speed(total_bytes, millis() - ms);
Brain.esp32_end(false);
// reset ESP32 to run new firmware
Brain.targetReset();
}
void loop() {
}
//--------------------------------------------------------------------+
// Setup and Loop on Core1
//--------------------------------------------------------------------+
// call usbh_begin() here to make pio usb background task run on core1
// NOTE: Brain.begin() should be called here as well to prevent race condition
void setup1() {
}
// core1's loop: process usb host task on core1
void loop1() {
yield();
}

View file

@ -0,0 +1,30 @@
// Generated by tools/esp_compress.py
#define ESP_BINARIES_COUNT (1)
// const esp32_zipfile_t NINA_W102_1_7_5
const esp32_zipfile_t NINA_W102_1_7_5 = {
.name = "NINA_W102-1.7.5.bin.gz",
.data = NULL,
.compressed_len = 634996,
.uncompressed_len = 1160192,
.md5 =
{
0x07,
0x7c,
0xec,
0x6d,
0xaa,
0x68,
0x13,
0xc5,
0xa4,
0x68,
0x0f,
0x45,
0xc9,
0xd8,
0x77,
0x6b,
},
};

View file

@ -0,0 +1,146 @@
/* This sketch run on SAMD21/SAMD51 to program ESP32 by flashing bin file from SD Card
* Supported/tested ESP MCU are: ESP32, ESP32-S2, ESP32-S3, ESP8266
* Hardware wiring:
* - TB D2 <-> ESP32 IO0
* - TB D3 <-> ESP32 Enable
* - TB TX/RX <-> ESP32 RX/TX
*
* How to run this example:
* 0. Define ESP32_RESET, ESP32_IO0, SD_CS, SD_DETECT in this sketch according to your hardware setup
* 1. Generate compressed binary and its metadata (len, md5 etc..) by running:
* python3 tools/esp_compress.py --sd <directory_of_bin_files>
* For example: python tools/esp_compress.py --sd .
* 2. .bin.gz (e.g NINA_W102-1.7.5.bin.gz) and esp_binaries.h will be generated in the same directory
* 3. Copy esp_binaries.h to this example directory
* 4. Copy .bin.gz files to SD Card within BIN_DIR (defined in this sketch)
* 5. Insert SD Card to board
* 6. Upload this sketch to board
*
* Note: for convenience, this example included generated 'NINA_W102-1.7.5.bin.gz' and 'esp_binaries.h'
* from https://github.com/adafruit/nina-fw/releases/tag/1.7.5 in this example directory.
*/
#include "SdFat.h"
// #include "Adafruit_TinyUSB.h"
#include "Adafruit_TestBed.h"
//--------------------------------------------------------------------+
// Hardware Configuration
//--------------------------------------------------------------------+
// These are defined in airlift-capable board such PyPortal, PyBadge, etc.
#if defined(ESP32_GPIO0) && defined(ESP32_RESETN)
#define ESP32_RESET ESP32_RESETN
#define ESP32_IO0 ESP32_GPIO0
#else
#define ESP32_RESET 2
#define ESP32_IO0 3
#endif
#if defined(ARDUINO_PYPORTAL_M4)
#define SD_CS 32
#define SD_DETECT 33
#else
#define SD_CS 4
#define SD_DETECT 5 // optional
#endif
#define ESP32_BAUDRATE 2000000
//#define ESP32_BAUDRATE 1500000
//#define ESP32_BAUDRATE 921600
//#define ESP32_BAUDRATE 115200
//--------------------------------------------------------------------+
// Binaries and Path
//--------------------------------------------------------------------+
#define BIN_DIR "/" // SD card's directory that contains bin files
#include "esp_binaries.h"
struct {
uint32_t addr;
esp32_zipfile_t const *zfile;
} bin_files[] = {
{0x00000, &NINA_W102_1_7_5}
};
enum { BIN_FILES_COUNT = sizeof(bin_files) / sizeof(bin_files[0]) };
//--------------------------------------------------------------------+
// Defined an boot rom object that use UART Serial1
ESP32BootROMClass ESP32BootROM(Serial1, ESP32_IO0, ESP32_RESET);
SdFat SD;
File32 fbin;
//--------------------------------------------------------------------+
// Implementation
//--------------------------------------------------------------------+
void print_speed(size_t count, uint32_t ms) {
float count_k = count / 1000.0F;
float sec = ms / 1000.0F;
float speed = count_k / sec;
Serial.print(count_k); Serial.print("KB "); Serial.print(sec); Serial.println("s");
Serial.print("Spd: "); Serial.print(speed); Serial.println(" KB/s");
}
void setup() {
Serial.begin(115200);
while (!Serial) delay(10);
Serial.println("TestBed: Programming ESP32 with UART!");
#ifdef SD_DETECT
pinMode(SD_DETECT, INPUT_PULLUP);
if (digitalRead(SD_DETECT) == LOW) {
Serial.println("SD Card not inserted");
while (1) delay(10);
}
#endif
if (!SD.begin(SD_CS, SD_SCK_MHZ(12))) {
Serial.println("SD Card init failed");
while (1) delay(10);
}
Serial.println("SD Card init OK");
TB.begin();
while ( !TB.esp32_begin(&ESP32BootROM, ESP32_BAUDRATE) ) {
// retry syncing
delay(100);
}
// Writing bin files
size_t total_bytes = 0;
uint32_t ms = millis();
for(size_t i=0; i<BIN_FILES_COUNT; i++) {
Serial.printf("Flashing file %u\r\n", i);
Serial.printf("File: %s\r\n", bin_files[i].zfile->name);
char bin_path[128] = BIN_DIR;
strcat(bin_path, bin_files[i].zfile->name);
fbin.open(&SD, bin_path);
if (!fbin) {
Serial.printf("Failed to open file %s\r\n", bin_files[i].zfile->name);
continue;
}
size_t wr_count = TB.esp32_programFlashDefl(bin_files[i].zfile, bin_files[i].addr, &fbin);
total_bytes += wr_count;
if (!wr_count) {
Serial.printf("Failed to flash");
}
fbin.close();
}
print_speed(total_bytes, millis() - ms);
TB.esp32_end();
// reset ESP32 to run new firmware
TB.targetReset();
}
void loop() {
}

View file

@ -486,8 +486,9 @@ bool Adafruit_TestBed::esp32_s3_inReset(void) { return _esp32s3_in_reset; }
void Adafruit_TestBed::esp32_s3_clearReset(void) { _esp32s3_in_reset = false; }
size_t Adafruit_TestBed::esp32_programFlashDefl(const esp32_zipfile_t *zfile,
uint32_t addr) {
size_t
Adafruit_TestBed::_esp32_programFlashDefl_impl(const esp32_zipfile_t *zfile,
uint32_t addr, File32 *fsrc) {
if (!esp32boot) {
return 0;
}
@ -511,168 +512,113 @@ size_t Adafruit_TestBed::esp32_programFlashDefl(const esp32_zipfile_t *zfile,
// Write Size is different depending on ROM (1K) or Stub (16KB)
uint32_t const block_size = esp32boot->getFlashWriteSize();
uint8_t *buf = NULL;
bool const use_sdcard = (fsrc != NULL);
if (use_sdcard) {
buf = (uint8_t *)malloc(block_size);
if (!buf) {
Serial.printf("No memory %lu\n", block_size);
return 0;
}
}
Serial.printf("Compressed %lu bytes to %lu\r\n", zfile->uncompressed_len,
zfile->compressed_len);
if (!esp32boot->beginFlashDefl(addr, zfile->uncompressed_len,
zfile->compressed_len)) {
Serial.println("beginFlash failed!");
} else {
_esp32_flash_defl = true;
if (buf) {
free(buf);
}
return 0;
}
uint32_t const block_num = div_ceil(zfile->compressed_len, block_size);
_esp32_flash_defl = true;
//------------- Flashing -------------//
uint8_t const *data = zfile->data;
uint32_t remain = zfile->compressed_len;
//------------- Flashing -------------//
uint32_t written = 0;
for (uint32_t i = 0; i < block_num; i++) {
setLED(HIGH);
Serial.printf("Pckt %u/%u", i + 1, block_num);
uint32_t const block_num = div_ceil(zfile->compressed_len, block_size);
for (uint32_t i = 0; i < block_num; i++) {
setLED(HIGH);
Serial.printf("Pckt %lu/%lu", i + 1, block_num);
uint32_t const wr_count = MIN(block_size, remain);
uint32_t const remain = zfile->compressed_len - written;
uint32_t const wr_count = (remain < block_size) ? remain : block_size;
// Note: flash deflat does not need padding
if (!esp32boot->dataFlashDefl(data, wr_count)) {
Serial.println("Failed to flash");
break;
uint8_t const *data;
if (!use_sdcard) {
// file contents is stored in internal flash (e.g rp2040)
data = zfile->data + written;
} else {
// file contents is stored in sdcard
memset(buf, 0xff, block_size); // empty it out
if (wr_count != fsrc->read(buf, wr_count)) {
Serial.println("File contents does not matched with compressed_len");
free(buf);
return 0;
}
setLED(LOW);
data = buf;
}
data += wr_count;
remain -= wr_count;
// Note: flash deflat does not need padding
if (!esp32boot->dataFlashDefl(data, wr_count)) {
Serial.println("Failed to flash");
break;
}
written += wr_count;
setLED(LOW);
}
Serial.println();
// Stub only writes each block to flash after 'ack'ing the receive,
// so do a final dummy operation which will not be 'ack'ed
// until the last block has actually been written out to flash
if (esp32boot->isRunningStub()) {
Serial.println("Dummy read chip detect after final block");
(void)esp32boot->read_chip_detect();
}
//------------- MD5 verification -------------//
Serial.println("Verifying MD5");
esp32boot->md5Flash(addr, zfile->uncompressed_len, esp_md5);
if (0 == memcmp(zfile->md5, esp_md5, 16)) {
Serial.println("MD5 matched");
} else {
Serial.println("MD5 mismatched!!");
Serial.printf("File: ");
for (size_t i = 0; i < 16; i++) {
Serial.printf("%02X ", zfile->md5[i]);
}
Serial.println();
// Stub only writes each block to flash after 'ack'ing the receive,
// so do a final dummy operation which will not be 'ack'ed
// until the last block has actually been written out to flash
if (esp32boot->isRunningStub()) {
Serial.println("Dummy read chip detect after final block");
(void)esp32boot->read_chip_detect();
Serial.printf("ESP : ");
for (size_t i = 0; i < 16; i++) {
Serial.printf("%02X ", esp_md5[i]);
}
Serial.println();
}
//------------- MD5 verification -------------//
Serial.println("Verifying MD5");
esp32boot->md5Flash(addr, zfile->uncompressed_len, esp_md5);
if (0 == memcmp(zfile->md5, esp_md5, 16)) {
Serial.println("MD5 matched");
} else {
Serial.println("MD5 mismatched!!");
Serial.printf("File: ");
for (size_t i = 0; i < 16; i++) {
Serial.printf("%02X ", zfile->md5[i]);
}
Serial.println();
Serial.printf("ESP : ");
for (size_t i = 0; i < 16; i++) {
Serial.printf("%02X ", esp_md5[i]);
}
Serial.println();
}
if (buf) {
free(buf);
}
return zfile->uncompressed_len;
}
#if 0
size_t Adafruit_TestBed::esp32_programFlash(const char *fpath,
uint32_t addr) {
if (!esp32boot) {
return 0;
}
// Write Size is different depending on ROM (1K) or Stub (16KB)
uint32_t const block_size = esp32boot->getFlashWriteSize();
uint8_t *buf = (uint8_t *)malloc(block_size);
if (!buf) {
Serial.printf("No memory %u\n", block_size);
return 0;
}
File32 fsrc = SD.open(fpath);
if (!fsrc) {
Serial.printf("SD: cannot open file: %s\r\n", fpath);
return 0;
}
uint32_t fsize = fsrc.fileSize();
uint32_t total_count = 0;
Serial.printf("fsize = %lu, block size = %lu\r\n", fsize, block_size);
if (!esp32boot->beginFlash(addr, fsize, block_size)) {
Serial.println("beginFlash failed!");
} else {
LCD_printf("#Packets %u", div_ceil(fsize, block_size));
MD5Builder md5;
md5.begin();
//------------- Flashing -------------//
while (fsrc.available()) {
memset(buf, 0xff, block_size); // empty it out
uint32_t const rd_count = fsrc.read(buf, block_size);
setLED(HIGH);
Serial.printf("#");
if (!esp32boot->dataFlash(buf, block_size)) {
Serial.println("Failed to flash");
break;
}
setLED(LOW);
md5.add(buf, rd_count);
total_count += rd_count;
}
Serial.println();
// Stub only writes each block to flash after 'ack'ing the receive,
// so do a final dummy operation which will not be 'ack'ed
// until the last block has actually been written out to flash
if (esp32boot->isRunningStub()) {
(void)esp32boot->read_chip_detect();
}
//------------- MD5 verification -------------//
md5.calculate();
Serial.printf("md5 = %s\r\n", md5.toString().c_str());
uint8_t file_md5[16];
md5.getBytes(file_md5);
uint8_t esp_md5[16];
esp32boot->md5Flash(addr, fsize, esp_md5);
if (0 == memcmp(file_md5, esp_md5, 16)) {
Serial.println("MD5 matched");
} else {
Serial.println("MD5 mismatched!!");
Serial.printf("File: ");
for (size_t i = 0; i < 16; i++) {
Serial.printf("%02X ", file_md5[i]);
}
Serial.println();
Serial.printf("ESP : ");
for (size_t i = 0; i < 16; i++) {
Serial.printf("%02X ", esp_md5[i]);
}
Serial.println();
}
}
free(buf);
fsrc.close();
return total_count;
size_t Adafruit_TestBed::esp32_programFlashDefl(const esp32_zipfile_t *zfile,
uint32_t addr) {
return _esp32_programFlashDefl_impl(zfile, addr, NULL);
}
size_t Adafruit_TestBed::esp32_programFlashDefl(const esp32_zipfile_t *zfile,
uint32_t addr, File32 *fsrc) {
return _esp32_programFlashDefl_impl(zfile, addr, fsrc);
}
#endif
Adafruit_TestBed TB;

View file

@ -6,6 +6,7 @@
#include "Wire.h"
#include "ESP32BootROM.h"
#include "SdFat.h"
#define RED 0xFF0000
#define YELLOW 0xFFFF00
@ -68,13 +69,13 @@ public:
bool esp32_begin(ESP32BootROMClass *bootrom, uint32_t baudrate);
void esp32_end(bool reset_esp = false);
// program esp32 target with file from SDCard
// return number of programmed bytes
// size_t esp32_programFlash(const char *fpath, uint32_t addr);
// program flash with compressed using zipfile struct
// program esp32 target with compressed data stored in internal flash
size_t esp32_programFlashDefl(const esp32_zipfile_t *zfile, uint32_t addr);
// program esp32 target with compressed file from SDCard
size_t esp32_programFlashDefl(const esp32_zipfile_t *zfile, uint32_t addr,
File32 *fsrc);
bool esp32_s3_inReset(void);
void esp32_s3_clearReset(void);
@ -106,6 +107,9 @@ private:
bool _esp32_flash_defl;
uint32_t _esp32_chip_detect;
bool _esp32s3_in_reset;
size_t _esp32_programFlashDefl_impl(const esp32_zipfile_t *zfile,
uint32_t addr, File32 *fsrc);
};
extern Adafruit_TestBed TB;

View file

@ -136,7 +136,6 @@ bool Adafruit_TestBed_Brains::inited(void) { return _inited; }
void Adafruit_TestBed_Brains::rp2040_targetResetBootRom(int bootsel_pin,
uint32_t reset_ms) {
pinMode(bootsel_pin, OUTPUT);
digitalWrite(bootsel_pin, LOW);
targetReset(reset_ms);

View file

@ -3,8 +3,9 @@
#ifdef ARDUINO_RASPBERRY_PI_PICO
#include "Adafruit_TestBed.h"
#include "SdFat.h"
#include "Adafruit_TestBed.h"
#include <LiquidCrystal.h>
#include "Adafruit_DAP.h"

View file

@ -1,21 +1,7 @@
import sys
import zlib
import hashlib
from pathlib import Path
import os.path
output_header = 'esp_binaries.h'
# Argument is path to folder containing .bin files
# This script will create .bin.gz file with the same name to the folder
# and also print out C array of the gz along with MD5 checksum
dir = ''
if len(sys.argv) == 1:
print("No [DIRS] arguments, use current directory as default")
dir = Path('./')
else:
# only take 1 argument
dir = Path(sys.argv[1])
import click
def print_carray(f, payload):
while len(payload) > 0:
@ -25,50 +11,71 @@ def print_carray(f, payload):
payload = payload[16:]
f.write('\n')
with open(output_header, 'w') as fc:
# print typedef struct
fc.write('// Generated by tools/esp_compress.py\n')
file_list = sorted(dir.glob('*.bin'))
fc.write('#define ESP_BINARIES_COUNT ({})\n\n'.format(len(file_list)))
# print list of struct first
for fname in file_list:
var = fname.stem
var = var.replace('.', '_')
var = var.replace('-', '_')
fc.write('// const esp32_zipfile_t {}\n'.format(var))
fc.write('\n'.format(var))
@click.command()
@click.argument('dpath', type=click.Path(exists=True), default='.')
@click.option('--sd', is_flag=True, default=False, help='Generate header for using with SDCard')
def main(dpath, sd):
"""
This script takes a directory path (default is current directory) and
create compressed .bin.gz file along 'esp_binaries.h' containing metadata in esp32_zipfile_t struct.
Which can be used to program ESP32 with TestBed or Brain.
If '--sd' option is specified, it will skip generating C array data for gz file.
"""
output_header = 'esp_binaries.h'
with open(output_header, 'w') as fc:
# print typedef struct
fc.write('// Generated by tools/esp_compress.py\n')
file_list = sorted(Path(dpath).glob('*.bin'))
fc.write(f'#define ESP_BINARIES_COUNT ({len(file_list)})\n\n')
for fname in file_list:
with open(fname, 'rb') as fi:
image = fi.read()
zimage = zlib.compress(image, 9)
fzname = fname.with_suffix('.bin.gz')
md5 = hashlib.md5(image)
# write .gz file
with open(fzname, 'wb') as fz:
fz.write(zimage)
# write to c header file
# print list of struct first
for fname in file_list:
var = fname.stem
var = var.replace('.', '_')
var = var.replace('-', '_')
fc.write(f'// const esp32_zipfile_t {var}\n')
fc.write('\n')
# bin gz contents
fc.write('const uint8_t _{}_gz[{}] = {{'.format(var, len(zimage)))
print_carray(fc, zimage)
fc.write('};\n\n')
for fname in file_list:
with open(fname, 'rb') as fi:
image = fi.read()
zimage = zlib.compress(image, 9)
fzname = fname.with_suffix('.bin.gz')
md5 = hashlib.md5(image)
fc.write('const esp32_zipfile_t {} = {{\n'.format(var))
fc.write(' .name = "{}",\n'.format(var))
fc.write(' .data = _{}_gz,\n'.format(var))
fc.write(' .compressed_len = {},\n'.format(len(zimage)))
fc.write(' .uncompressed_len = {},\n'.format(len(image)))
fc.write(' .md5 = {')
print_carray(fc, md5.digest())
fc.write(' },\n')
fc.write('};\n')
fc.write('\n')
# write .gz file
with open(fzname, 'wb') as fz:
fz.write(zimage)
print("Compressed {}: {} to {} bytes, deflation: {:.2f}%".format(var, len(image), len(zimage), 100*(1-len(zimage)/len(image))))
# write to c header file
var = fname.stem
var = var.replace('.', '_')
var = var.replace('-', '_')
# bin gz contents
if not sd:
fc.write(f'const uint8_t _{var}_gz[{len(zimage)}] = {{')
print_carray(fc, zimage)
fc.write('};\n\n')
fc.write(f'const esp32_zipfile_t {var} = {{\n')
fc.write(f' .name = "{fzname}",\n')
if sd:
fc.write(f' .data = NULL,\n')
else:
fc.write(f' .data = _{var}_gz,\n')
fc.write(f' .compressed_len = {len(zimage)},\n')
fc.write(f' .uncompressed_len = {len(image)},\n')
fc.write(' .md5 = {')
print_carray(fc, md5.digest())
fc.write(' },\n')
fc.write('};\n')
fc.write('\n')
deflation = 100*(1-len(zimage)/len(image))
print(f"Compressed {var}: {len(image)} to {len(zimage)} bytes, deflation: {deflation:.2f}%")
if __name__ == '__main__':
main()