Merge pull request #8952 from dhalbert/memento-storage-crash

Espressif: fix handling of single-partition CIRCUITPY (non-CIRCUITPY_STORAGE_EXTEND) - fixes MEMENTO bug

ports/espressif/Makefile

@@ -417,7 +417,12 @@ endif
 do-sdkconfig: $(BUILD)/esp-idf/config/sdkconfig.h
 QSTR_GLOBAL_REQUIREMENTS += $(BUILD)/esp-idf/config/sdkconfig.h
 $(BUILD)/esp-idf/config/sdkconfig.h: boards/$(BOARD)/sdkconfig boards/$(BOARD)/mpconfigboard.mk CMakeLists.txt | $(BUILD)/esp-idf
-	IDF_PATH=$(IDF_PATH) cmake -S . -B $(BUILD)/esp-idf -DSDKCONFIG=$(BUILD)/esp-idf/sdkconfig -DSDKCONFIG_DEFAULTS="$(SDKCONFIGS)" -DCMAKE_TOOLCHAIN_FILE=$(IDF_PATH)/tools/cmake/toolchain-$(IDF_TARGET).cmake -DIDF_TARGET=$(IDF_TARGET) -GNinja
+	$(STEPECHO) "LINK $@"
+	$(Q)env IDF_PATH=$(IDF_PATH) cmake -S . -B $(BUILD)/esp-idf -DSDKCONFIG=$(BUILD)/esp-idf/sdkconfig -DSDKCONFIG_DEFAULTS="$(SDKCONFIGS)" -DCMAKE_TOOLCHAIN_FILE=$(IDF_PATH)/tools/cmake/toolchain-$(IDF_TARGET).cmake -DIDF_TARGET=$(IDF_TARGET) -GNinja
+	$(Q)$(PYTHON) tools/check-sdkconfig.py \
+		CIRCUITPY_DUALBANK=$(CIRCUITPY_DUALBANK) \
+		CIRCUITPY_STORAGE_EXTEND=$(CIRCUITPY_STORAGE_EXTEND) \
+		$@
 
 # build a lib
 # Adding -d explain -j 1 -v to the ninja line will output debug info

ports/espressif/boards/adafruit_esp32s3_camera/mpconfigboard.mk

@@ -12,10 +12,13 @@ CIRCUITPY_ESP_FLASH_FREQ = 80m
 CIRCUITPY_ESP_PSRAM_SIZE = 2MB
 CIRCUITPY_ESP_PSRAM_MODE = qio
 CIRCUITPY_ESP_PSRAM_FREQ = 80m
-FLASH_SDKCONFIG = esp-idf-config/sdkconfig-4MB-1ota.defaults
+
+# No OTA partition: larger firmware partition
+FLASH_SIZE_SDKCONFIG = esp-idf-config/sdkconfig-flash-$(CIRCUITPY_ESP_FLASH_SIZE)-no-ota.defaults
 
 CIRCUITPY_AUDIOBUSIO = 0
 CIRCUITPY_CANIO = 0
+CIRCUITPY_DUALBANK = 0
 CIRCUITPY_ESPCAMERA = 1
 CIRCUITPY_FRAMEBUFFERIO = 0
 CIRCUITPY_KEYPAD = 0

ports/espressif/esp-idf-config/sdkconfig-flash-4MB-no-ota.defaults

@@ -1,4 +1,7 @@
 #
+# Espressif IoT Development Framework Configuration
+#
+#
 # Serial flasher config
 #
 # CONFIG_ESPTOOLPY_FLASHSIZE_1MB is not set
@@ -6,13 +9,17 @@
 CONFIG_ESPTOOLPY_FLASHSIZE_4MB=y
 # CONFIG_ESPTOOLPY_FLASHSIZE_8MB is not set
 # CONFIG_ESPTOOLPY_FLASHSIZE_16MB is not set
+# CONFIG_ESPTOOLPY_FLASHSIZE_32MB is not set
+# CONFIG_ESPTOOLPY_FLASHSIZE_64MB is not set
+# CONFIG_ESPTOOLPY_FLASHSIZE_128MB is not set
 CONFIG_ESPTOOLPY_FLASHSIZE="4MB"
-CONFIG_ESPTOOLPY_FLASHSIZE_DETECT=y
 # end of Serial flasher config
 
-CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="esp-idf-config/partitions-4MB-1ota.csv"
 #
 # Partition Table
 #
-CONFIG_PARTITION_TABLE_FILENAME="esp-idf-config/partitions-4MB-1ota.csv"
+CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="esp-idf-config/partitions-4MB-no-ota.csv"
+CONFIG_PARTITION_TABLE_FILENAME="esp-idf-config/partitions-4MB-no-ota.csv"
 # end of Partition Table
+
+# end of Espressif IoT Development Framework Configuration

ports/espressif/mpconfigport.mk

@@ -26,7 +26,6 @@ CIRCUITPY_AUDIOMP3 ?= 0
 CIRCUITPY_BLEIO_HCI = 0
 CIRCUITPY_CANIO ?= 1
 CIRCUITPY_COUNTIO ?= 1
-CIRCUITPY_DOTCLOCKFRAMEBUFFER_USES_SUPERVISOR_ALLOCATION = 0
 CIRCUITPY_DUALBANK ?= 1
 CIRCUITPY_ESPCAMERA ?= 1
 CIRCUITPY_ESPIDF ?= 1
@@ -113,14 +112,12 @@ else ifeq ($(IDF_TARGET),esp32s2)
 # Modules
 # No BLE in hw
 CIRCUITPY_BLEIO = 0
-CIRCUITPY_RGBMATRIX_USES_SUPERVISOR_ALLOCATION = 0
 
 CIRCUITPY_ESP_USB_SERIAL_JTAG ?= 0
 
 else ifeq ($(IDF_TARGET),esp32s3)
 # Modules
 CIRCUITPY_BITMAPFILTER ?= $(CIRCUITPY_ESPCAMERA)
-CIRCUITPY_RGBMATRIX_USES_SUPERVISOR_ALLOCATION = 0
 CIRCUITPY_ESP_USB_SERIAL_JTAG ?= 0
 
 # No room for _bleio on boards with 4MB flash

ports/espressif/supervisor/internal_flash.c

@@ -168,7 +168,7 @@ mp_uint_t supervisor_flash_write_blocks(const uint8_t *src, uint32_t lba, uint32
         #if CIRCUITPY_STORAGE_EXTEND
         multi_partition_rw(_cache, sector_offset, SECTOR_SIZE, OP_WRITE);
         #else
-        single_partition_rw(_partition[0], _cache, sector_offset, SECTOR_SIZE, OP_READ);
+        single_partition_rw(_partition[0], _cache, sector_offset, SECTOR_SIZE, OP_WRITE);
         #endif
     }
     return 0; // success

ports/espressif/tools/check-sdkconfig.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python3
+import sys
+
+import click
+
+
+def int_or_string(s):
+    try:
+        return int(s)
+    except ValueError:
+        return s.strip('"')
+
+
+def collect_definitions(file):
+    """Collect all definitions in supplied sdkconfig.h."""
+    sdk_config = {}
+    for line in file:
+        if line.startswith("#define "):
+            _, k, v = line.strip().split(None, 2)
+            # Handle transitive definitions like '#define CONFIG_TCP_MSL CONFIG_LWIP_TCP_MSL'
+            v = sdk_config.get(k, v)
+            sdk_config[k] = int_or_string(v)
+    return sdk_config
+
+
+def validate(sdk_config, circuitpy_config):
+    partition_table = sdk_config.get("CONFIG_PARTITION_TABLE_FILENAME")
+    for var in ("CIRCUITPY_STORAGE_EXTEND", "CIRCUITPY_DUALBANK"):
+        if circuitpy_config.get(var):
+            with open(partition_table) as f:
+                content = f.read()
+                if not "ota_1" in content:
+                    raise SystemExit(
+                        f"{var} is incompatible with {partition_table=} (no ota_1 partition)"
+                    )
+
+    # Add more checks here for other things we want to verify.
+    return
+
+
+@click.command()
+@click.argument("definitions", nargs=-1, metavar="CIRCUITPY_X=1 CIRCUITPY_Y=0 ...")
+@click.argument(
+    "sdkconfig_h", required=True, nargs=1, type=click.File("r"), metavar="<path to sdkconfig.h>"
+)
+def run(definitions, sdkconfig_h):
+    sdk_config = collect_definitions(sdkconfig_h)
+
+    # Parse definitions arguments
+    circuitpy_config = {}
+    for definition in definitions:
+        k, v = definition.split("=", 1)
+        circuitpy_config[k] = int_or_string(v)
+
+    # Validate.
+    validate(sdk_config, circuitpy_config)
+
+
+if __name__ == "__main__":
+    run()

py/circuitpy_mpconfig.h

@@ -415,9 +415,6 @@ extern const struct _mp_obj_module_t nvm_module;
 
 #define MP_STATE_PORT MP_STATE_VM
 
-// From supervisor/memory.c
-struct _supervisor_allocation_node;
-
 void background_callback_run_all(void);
 #define RUN_BACKGROUND_TASKS (background_callback_run_all())
 

py/circuitpy_mpconfig.mk

@@ -237,8 +237,6 @@ CFLAGS += -DCIRCUITPY_PARALLELDISPLAYBUS=$(CIRCUITPY_PARALLELDISPLAYBUS)
 
 CIRCUITPY_DOTCLOCKFRAMEBUFFER ?= 0
 CFLAGS += -DCIRCUITPY_DOTCLOCKFRAMEBUFFER=$(CIRCUITPY_DOTCLOCKFRAMEBUFFER)
-CIRCUITPY_DOTCLOCKFRAMEBUFFER_USES_SUPERVISOR_ALLOCATION ?= 1
-CFLAGS += -DCIRCUITPY_DOTCLOCKFRAMEBUFFER_USES_SUPERVISOR_ALLOCATION=$(CIRCUITPY_DOTCLOCKFRAMEBUFFER_USES_SUPERVISOR_ALLOCATION)
 
 # bitmaptools and framebufferio rely on displayio and are not on small boards
 CIRCUITPY_BITMAPTOOLS ?= $(call enable-if-all,$(CIRCUITPY_FULL_BUILD) $(CIRCUITPY_DISPLAYIO))
@@ -435,8 +433,6 @@ CFLAGS += -DCIRCUITPY_RP2PIO=$(CIRCUITPY_RP2PIO)
 
 CIRCUITPY_RGBMATRIX ?= 0
 CFLAGS += -DCIRCUITPY_RGBMATRIX=$(CIRCUITPY_RGBMATRIX)
-CIRCUITPY_RGBMATRIX_USES_SUPERVISOR_ALLOCATION ?= 1
-CFLAGS += -DCIRCUITPY_RGBMATRIX_USES_SUPERVISOR_ALLOCATION=$(CIRCUITPY_RGBMATRIX_USES_SUPERVISOR_ALLOCATION)
 
 CIRCUITPY_ROTARYIO ?= 1
 CFLAGS += -DCIRCUITPY_ROTARYIO=$(CIRCUITPY_ROTARYIO)