MPy v1.22 merge: initial merge; not compiled yet

.git-blame-ignore-revs

@@ -1,3 +1,6 @@
+# all: Update Python formatting to ruff-format.
+bbd8760bd9a2302e5abee29db279102bb11d7732
+
 # all: Fix various spelling mistakes found by codespell 2.2.6.
 cf490a70917a1b2d38ba9b58e763e0837d0f7ca7
 

.gitattributes

@@ -10,12 +10,15 @@
 *.props text eol=crlf
 *.bat text eol=crlf
 
+# CIRCUITPY-CHANGE: add some more binary types.
 # These are binary so should never be modified by git.
 *.a binary
+*.ico binary
 *.png binary
 *.jpg binary
 *.dxf binary
 *.mpy binary
+*.der binary
 *.deb binary
 *.zip binary
 *.pdf binary
@@ -24,11 +27,4 @@
 # These should also not be modified by git.
 tests/basics/string_cr_conversion.py -text
 tests/basics/string_crlf_conversion.py -text
-ports/stm32/pybcdc.inf_template -text
+# CIRCUITPY-CHANGE: remove non-CircuitPython tests
-ports/stm32/usbhost/** -text
-ports/cc3200/hal/aes.c -text
-ports/cc3200/hal/aes.h -text
-ports/cc3200/hal/des.c -text
-ports/cc3200/hal/i2s.c -text
-ports/cc3200/hal/i2s.h -text
-ports/cc3200/version.h -text

.gitignore

@@ -2,6 +2,8 @@
 #
 # SPDX-License-Identifier: MIT
 
+# CIRCUITPY-CHANGES: many additions
+
 # Compiled Sources
 ###################
 *.o

.pre-commit-config.yaml

@@ -2,6 +2,8 @@
 #
 # SPDX-License-Identifier: Unlicense
 
+# CIRCUITPY-CHANGE: CircuitPython-specific.
+
 repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.0.1

LICENSE

@@ -17,5 +17,15 @@ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-SOFTWARE.
+THE SOFTWARE.
+
+--------------------------------------------------------------------------------
+
+#CIRCUITPY-CHANGE:
+
+Unless specified otherwise (see below), the above license and copyright applies
+to all files derived from MicroPython in this repository.
+
+Individual files may include additional copyright holders and specify other licenses.
+See the comments and SPDX headers.

docs/library/sys.rst

@@ -49,7 +49,7 @@ Constants
 .. data:: maxsize
 
    Maximum value which a native integer type can hold on the current platform,
-   or maximum value representable by CircuitPython integer type, if it's smaller
+   or maximum value representable by the CircuitPython integer type, if it's smaller
    than platform max value (that is the case for CircuitPython ports without
    long int support).
 

docs/readthedocs/settings/local_settings.py

@@ -1,9 +1,10 @@
 import os
 
 # Directory that the project lives in, aka ../..
-SITE_ROOT = '/'.join(os.path.dirname(__file__).split('/')[0:-2])
+SITE_ROOT = "/".join(os.path.dirname(__file__).split("/")[0:-2])
 
 TEMPLATE_DIRS = (
-    "%s/templates/" % SITE_ROOT, # Your custom template directory, before the RTD one to override it.
+    "%s/templates/"
-    "%s/readthedocs/templates/" % SITE_ROOT, # Default RTD template dir
+    % SITE_ROOT,  # Your custom template directory, before the RTD one to override it.
+    "%s/readthedocs/templates/" % SITE_ROOT,  # Default RTD template dir
 )

docs/reference/micropython2_migration.rst

@@ -0,0 +1,74 @@
+.. _micropython2_migration:
+
+MicroPython 2.0 Migration Guide
+===============================
+
+MicroPython 2.0 is the (currently in development, not yet available) next major
+release of MicroPython.
+
+After maintaining API compatibility for almost a decade with the ``1.x`` series, in
+order to unblock some project-wide improvements MicroPython 2.0 will introduce a
+small number of breaking API changes that will require some programs to be
+updated. This guide explains how to update your Python code to accommodate these
+changes.
+
+This document is a work-in-progress. As more work is done on MicroPython 2.0,
+more items will be added to the lists below.
+
+**Note:** There are currently no MicroPython 2.0 firmware builds available for
+download. You can build it yourself by enabling the ``MICROPY_PREVIEW_VERSION_2``
+config option. As it gets closer to being ready for release, builds will be
+provided for both ``1.x.y`` and ``2.0.0-preview``.
+
+Hardware and peripherals
+------------------------
+
+Overview
+~~~~~~~~
+
+The goal is to improve consistency in the :mod:`machine` APIs across different
+ports, making it easier to write code, documentation, and tutorials that work on
+any supported microcontroller.
+
+This means that some ports' APIs need to change to match other ports.
+
+Changes
+~~~~~~~
+
+*None yet*
+
+OS & filesystem
+---------------
+
+Overview
+~~~~~~~~
+
+The primary goal is to support the ability to execute :term:`.mpy files <.mpy
+file>` directly from the filesystem without first copying them into RAM. This
+improves code deployment time and reduces memory overhead and fragmentation.
+
+Additionally, a further goal is to support a more flexible way of configuring
+partitions, filesystem types, and options like USB mass storage.
+
+Changes
+~~~~~~~
+
+*None yet*
+
+CPython compatibility
+---------------------
+
+Overview
+~~~~~~~~
+
+The goal is to improve compatibility with CPython by removing MicroPython
+extensions from CPython APIs. In most cases this means moving existing
+MicroPython-specific functions or classes to new modules.
+
+This makes it easier to write code that works on both CPython and MicroPython,
+which is useful for development and testing.
+
+Changes
+~~~~~~~
+
+*None yet*

examples/usercmodule/cppexample/example.cpp

@@ -1,9 +1,16 @@
 extern "C" {
 #include <examplemodule.h>
+#include <py/objstr.h>
 
 // Here we implement the function using C++ code, but since it's
 // declaration has to be compatible with C everything goes in extern "C" scope.
 mp_obj_t cppfunc(mp_obj_t a_obj, mp_obj_t b_obj) {
+    // The following no-ops are just here to verify the static assertions used in
+    // the public API all compile with C++.
+    MP_STATIC_ASSERT_STR_ARRAY_COMPATIBLE;
+    if (mp_obj_is_type(a_obj, &mp_type_BaseException)) {
+    }
+
     // Prove we have (at least) C++11 features.
     const auto a = mp_obj_get_int(a_obj);
     const auto b = mp_obj_get_int(b_obj);

extmod/extmod.mk

@@ -2,6 +2,7 @@
 # and provides rules to build 3rd-party components for extmod modules.
 
 # CIRCUITPY-CHANGE: many extmod modules removed
+# CIRCUITPY-CHANGE: modzlib.c still used
 SRC_EXTMOD_C += \
 	extmod/modasyncio.c \
 	extmod/modbinascii.c \
@@ -37,6 +38,96 @@ SRC_QSTR += $(SRC_EXTMOD_C)
 CFLAGS += $(CFLAGS_EXTMOD) $(CFLAGS_THIRDPARTY)
 LDFLAGS += $(LDFLAGS_EXTMOD) $(LDFLAGS_THIRDPARTY)
 
+################################################################################
+# libm/libm_dbl math library
+
+# Single-precision math library.
+SRC_LIB_LIBM_C += $(addprefix lib/libm/,\
+	acoshf.c \
+	asinfacosf.c \
+	asinhf.c \
+	atan2f.c \
+	atanf.c \
+	atanhf.c \
+	ef_rem_pio2.c \
+	erf_lgamma.c \
+	fmodf.c \
+	kf_cos.c \
+	kf_rem_pio2.c \
+	kf_sin.c \
+	kf_tan.c \
+	log1pf.c \
+	math.c \
+	nearbyintf.c \
+	roundf.c \
+	sf_cos.c \
+	sf_erf.c \
+	sf_frexp.c \
+	sf_ldexp.c \
+	sf_modf.c \
+	sf_sin.c \
+	sf_tan.c \
+	wf_lgamma.c \
+	wf_tgamma.c \
+	)
+
+# Choose only one of these sqrt implementations, software or hardware.
+SRC_LIB_LIBM_SQRT_SW_C += lib/libm/ef_sqrt.c
+SRC_LIB_LIBM_SQRT_HW_C += lib/libm/thumb_vfp_sqrtf.c
+
+# Double-precision math library.
+SRC_LIB_LIBM_DBL_C += $(addprefix lib/libm_dbl/,\
+	__cos.c \
+	__expo2.c \
+	__fpclassify.c \
+	__rem_pio2.c \
+	__rem_pio2_large.c \
+	__signbit.c \
+	__sin.c \
+	__tan.c \
+	acos.c \
+	acosh.c \
+	asin.c \
+	asinh.c \
+	atan.c \
+	atan2.c \
+	atanh.c \
+	ceil.c \
+	cos.c \
+	cosh.c \
+	copysign.c \
+	erf.c \
+	exp.c \
+	expm1.c \
+	floor.c \
+	fmod.c \
+	frexp.c \
+	ldexp.c \
+	lgamma.c \
+	log.c \
+	log10.c \
+	log1p.c \
+	modf.c \
+	nearbyint.c \
+	pow.c \
+	rint.c \
+	round.c \
+	scalbn.c \
+	sin.c \
+	sinh.c \
+	tan.c \
+	tanh.c \
+	tgamma.c \
+	trunc.c \
+	)
+
+# Choose only one of these sqrt implementations, software or hardware.
+SRC_LIB_LIBM_DBL_SQRT_SW_C += lib/libm_dbl/sqrt.c
+SRC_LIB_LIBM_DBL_SQRT_HW_C += lib/libm_dbl/thumb_vfp_sqrt.c
+
+# Too many warnings in libm_dbl, disable for now.
+$(BUILD)/lib/libm_dbl/%.o: CFLAGS += -Wno-double-promotion -Wno-float-conversion
+
 ################################################################################
 # VFS FAT FS
 
@@ -75,6 +166,7 @@ SRC_THIRDPARTY_C += $(addprefix $(LITTLEFS_DIR)/,\
 	lfs2_util.c \
 	)
 
+# CIRCUITPY-CHANGE: -Wno-missing-field-initializers instead of -Wno-shadow
 $(BUILD)/$(LITTLEFS_DIR)/lfs2.o: CFLAGS += -Wno-missing-field-initializers
 endif
 
@@ -235,6 +327,9 @@ SRC_THIRDPARTY_C += $(addprefix $(LWIP_DIR)/,\
 	core/ipv6/nd6.c \
 	netif/ethernet.c \
 	)
+ifeq ($(MICROPY_PY_LWIP_LOOPBACK),1)
+CFLAGS_EXTMOD += -DLWIP_NETIF_LOOPBACK=1
+endif
 ifeq ($(MICROPY_PY_LWIP_SLIP),1)
 CFLAGS_EXTMOD += -DMICROPY_PY_LWIP_SLIP=1
 SRC_THIRDPARTY_C += $(LWIP_DIR)/netif/slipif.c

extmod/lwip-include/lwipopts.h

@@ -23,6 +23,7 @@ typedef uint32_t sys_prot_t;
 #define LWIP_NETCONN 0
 #define LWIP_SOCKET 0
 
+// CIRCUITPY-CHANGE: #if instead of #ifdef
 #if MICROPY_PY_LWIP_SLIP
 #define LWIP_HAVE_SLIPIF 1
 #endif

extmod/misc.h

@@ -39,10 +39,12 @@ bool mp_os_dupterm_is_builtin_stream(mp_const_obj_t stream);
 void mp_os_dupterm_stream_detached_attached(mp_obj_t stream_detached, mp_obj_t stream_attached);
 uintptr_t mp_os_dupterm_poll(uintptr_t poll_flags);
 int mp_os_dupterm_rx_chr(void);
-void mp_os_dupterm_tx_strn(const char *str, size_t len);
+int mp_os_dupterm_tx_strn(const char *str, size_t len);
 void mp_os_deactivate(size_t dupterm_idx, const char *msg, mp_obj_t exc);
 #else
-#define mp_os_dupterm_tx_strn(s, l)
+static inline int mp_os_dupterm_tx_strn(const char *s, size_t l) {
+    return -1;
+}
 #endif
 
 #endif // MICROPY_INCLUDED_EXTMOD_MISC_H

extmod/modasyncio.c

@@ -31,6 +31,7 @@
 
 #if MICROPY_PY_ASYNCIO
 
+// CIRCUITPY-CHANGE
 #if CIRCUITPY && !(defined(__unix__) || defined(__APPLE__))
 #include "shared-bindings/supervisor/__init__.h"
 #endif
@@ -308,6 +309,7 @@ STATIC mp_obj_t task_getiter(mp_obj_t self_in, mp_obj_iter_buf_t *iter_buf) {
 STATIC mp_obj_t task_iternext(mp_obj_t self_in) {
     mp_obj_task_t *self = MP_OBJ_TO_PTR(self_in);
     if (TASK_IS_DONE(self)) {
+        // CIRCUITPY-CHANGE
         if (self->data == mp_const_none) {
             // Task finished but has already been sent to the loop's exception handler.
             mp_raise_StopIteration(MP_OBJ_NULL);

extmod/modbinascii.c

@@ -34,6 +34,7 @@
 
 #if MICROPY_PY_BINASCII
 
+// CIRCUITPY-CHANGE: added
 static void check_not_unicode(const mp_obj_t arg) {
     #if MICROPY_CPYTHON_COMPAT
     if (mp_obj_is_str(arg)) {

extmod/modhashlib.c

@@ -114,6 +114,7 @@ STATIC mp_obj_t hashlib_sha256_digest(mp_obj_t self_in) {
 
 #else
 
+// CIRCUITPY-CHNAGE
 static void check_not_unicode(const mp_obj_t arg) {
     #if MICROPY_CPYTHON_COMPAT
     if (mp_obj_is_str(arg)) {
@@ -136,6 +137,7 @@ STATIC mp_obj_t hashlib_sha256_make_new(const mp_obj_type_t *type, size_t n_args
 }
 
 STATIC mp_obj_t hashlib_sha256_update(mp_obj_t self_in, mp_obj_t arg) {
+    // CIRCUITPY-CHANGE
     check_not_unicode(arg);
     mp_obj_hash_t *self = MP_OBJ_TO_PTR(self_in);
     hashlib_ensure_not_final(self);
@@ -306,7 +308,7 @@ STATIC mp_obj_t hashlib_md5_digest(mp_obj_t self_in) {
 
 #if MICROPY_SSL_MBEDTLS
 
-#if MBEDTLS_VERSION_NUMBER < 0x02070000
+#if MBEDTLS_VERSION_NUMBER < 0x02070000 || MBEDTLS_VERSION_NUMBER >= 0x03000000
 #define mbedtls_md5_starts_ret mbedtls_md5_starts
 #define mbedtls_md5_update_ret mbedtls_md5_update
 #define mbedtls_md5_finish_ret mbedtls_md5_finish

extmod/modjson.c

@@ -26,6 +26,7 @@
 
 #include <stdio.h>
 
+// CIRCUITPY-CHANGE
 #include "py/binary.h"
 #include "py/objarray.h"
 #include "py/objlist.h"
@@ -111,6 +112,7 @@ STATIC MP_DEFINE_CONST_FUN_OBJ_1(mod_json_dumps_obj, mod_json_dumps);
 
 #endif
 
+// CIRCUITPY-CHANGE
 #define JSON_DEBUG(...) (void)0
 // #define JSON_DEBUG(...) mp_printf(&mp_plat_print __VA_OPT__(,) __VA_ARGS__)
 
@@ -147,6 +149,7 @@ typedef struct _json_stream_t {
 
 STATIC byte json_stream_next(json_stream_t *s) {
     mp_uint_t ret = s->read(s->stream_obj, &s->cur, 1, &s->errcode);
+    // CIRCUITPY-CHANGE
     JSON_DEBUG("  usjon_stream_next err:%2d cur: %c \n", s->errcode, s->cur);
     if (s->errcode != 0) {
         mp_raise_OSError(s->errcode);
@@ -427,6 +430,7 @@ fail:
     mp_raise_ValueError(MP_ERROR_TEXT("syntax error in JSON"));
 }
 
+// CIRCUITPY-CHANGE
 STATIC mp_obj_t mod_json_load(mp_obj_t stream_obj) {
     return _mod_json_load(stream_obj, true);
 }
@@ -437,7 +441,7 @@ STATIC mp_obj_t mod_json_loads(mp_obj_t obj) {
     mp_get_buffer_raise(obj, &bufinfo, MP_BUFFER_READ);
     vstr_t vstr = {bufinfo.len, bufinfo.len, (char *)bufinfo.buf, true};
     mp_obj_stringio_t sio = {{&mp_type_stringio}, &vstr, 0, MP_OBJ_NULL};
-    return _mod_json_load(MP_OBJ_FROM_PTR(&sio), false);
+    return mod_json_load(MP_OBJ_FROM_PTR(&sio), false);
 }
 STATIC MP_DEFINE_CONST_FUN_OBJ_1(mod_json_loads_obj, mod_json_loads);
 

extmod/modmachine.c

@@ -0,0 +1,242 @@
+/*
+ * This file is part of the MicroPython project, http://micropython.org/
+ *
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2023 Damien P. George
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "py/runtime.h"
+
+#if MICROPY_PY_MACHINE
+
+#include "extmod/modmachine.h"
+#include "shared/runtime/pyexec.h"
+
+#if MICROPY_PY_MACHINE_DHT_READINTO
+#include "drivers/dht/dht.h"
+#endif
+
+// The port must provide implementations of these low-level machine functions.
+
+STATIC void mp_machine_idle(void);
+
+#if MICROPY_PY_MACHINE_BOOTLOADER
+NORETURN void mp_machine_bootloader(size_t n_args, const mp_obj_t *args);
+#endif
+
+#if MICROPY_PY_MACHINE_BARE_METAL_FUNCS
+STATIC mp_obj_t mp_machine_unique_id(void);
+NORETURN STATIC void mp_machine_reset(void);
+STATIC mp_int_t mp_machine_reset_cause(void);
+STATIC mp_obj_t mp_machine_get_freq(void);
+STATIC void mp_machine_set_freq(size_t n_args, const mp_obj_t *args);
+STATIC void mp_machine_lightsleep(size_t n_args, const mp_obj_t *args);
+NORETURN STATIC void mp_machine_deepsleep(size_t n_args, const mp_obj_t *args);
+#endif
+
+// The port can provide additional machine-module implementation in this file.
+#ifdef MICROPY_PY_MACHINE_INCLUDEFILE
+#include MICROPY_PY_MACHINE_INCLUDEFILE
+#endif
+
+STATIC mp_obj_t machine_soft_reset(void) {
+    pyexec_system_exit = PYEXEC_FORCED_EXIT;
+    mp_raise_type(&mp_type_SystemExit);
+}
+STATIC MP_DEFINE_CONST_FUN_OBJ_0(machine_soft_reset_obj, machine_soft_reset);
+
+#if MICROPY_PY_MACHINE_BOOTLOADER
+NORETURN mp_obj_t machine_bootloader(size_t n_args, const mp_obj_t *args) {
+    mp_machine_bootloader(n_args, args);
+}
+MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(machine_bootloader_obj, 0, 1, machine_bootloader);
+#endif
+
+STATIC mp_obj_t machine_idle(void) {
+    mp_machine_idle();
+    return mp_const_none;
+}
+STATIC MP_DEFINE_CONST_FUN_OBJ_0(machine_idle_obj, machine_idle);
+
+#if MICROPY_PY_MACHINE_BARE_METAL_FUNCS
+
+STATIC mp_obj_t machine_unique_id(void) {
+    return mp_machine_unique_id();
+}
+MP_DEFINE_CONST_FUN_OBJ_0(machine_unique_id_obj, machine_unique_id);
+
+NORETURN STATIC mp_obj_t machine_reset(void) {
+    mp_machine_reset();
+}
+MP_DEFINE_CONST_FUN_OBJ_0(machine_reset_obj, machine_reset);
+
+STATIC mp_obj_t machine_reset_cause(void) {
+    return MP_OBJ_NEW_SMALL_INT(mp_machine_reset_cause());
+}
+MP_DEFINE_CONST_FUN_OBJ_0(machine_reset_cause_obj, machine_reset_cause);
+
+STATIC mp_obj_t machine_freq(size_t n_args, const mp_obj_t *args) {
+    if (n_args == 0) {
+        return mp_machine_get_freq();
+    } else {
+        mp_machine_set_freq(n_args, args);
+        return mp_const_none;
+    }
+}
+MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(machine_freq_obj, 0, 1, machine_freq);
+
+STATIC mp_obj_t machine_lightsleep(size_t n_args, const mp_obj_t *args) {
+    mp_machine_lightsleep(n_args, args);
+    return mp_const_none;
+}
+MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(machine_lightsleep_obj, 0, 1, machine_lightsleep);
+
+NORETURN STATIC mp_obj_t machine_deepsleep(size_t n_args, const mp_obj_t *args) {
+    mp_machine_deepsleep(n_args, args);
+}
+MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(machine_deepsleep_obj, 0, 1, machine_deepsleep);
+
+#endif
+
+#if MICROPY_PY_MACHINE_DISABLE_IRQ_ENABLE_IRQ
+
+STATIC mp_obj_t machine_disable_irq(void) {
+    uint32_t state = MICROPY_BEGIN_ATOMIC_SECTION();
+    return mp_obj_new_int(state);
+}
+STATIC MP_DEFINE_CONST_FUN_OBJ_0(machine_disable_irq_obj, machine_disable_irq);
+
+STATIC mp_obj_t machine_enable_irq(mp_obj_t state_in) {
+    uint32_t state = mp_obj_get_int(state_in);
+    MICROPY_END_ATOMIC_SECTION(state);
+    return mp_const_none;
+}
+STATIC MP_DEFINE_CONST_FUN_OBJ_1(machine_enable_irq_obj, machine_enable_irq);
+
+#endif
+
+STATIC const mp_rom_map_elem_t machine_module_globals_table[] = {
+    { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_machine) },
+
+    // Memory access objects.
+    { MP_ROM_QSTR(MP_QSTR_mem8), MP_ROM_PTR(&machine_mem8_obj) },
+    { MP_ROM_QSTR(MP_QSTR_mem16), MP_ROM_PTR(&machine_mem16_obj) },
+    { MP_ROM_QSTR(MP_QSTR_mem32), MP_ROM_PTR(&machine_mem32_obj) },
+
+    // Miscellaneous functions.
+    #if MICROPY_PY_MACHINE_BARE_METAL_FUNCS
+    { MP_ROM_QSTR(MP_QSTR_unique_id), MP_ROM_PTR(&machine_unique_id_obj) },
+    #endif
+
+    // Reset related functions.
+    { MP_ROM_QSTR(MP_QSTR_soft_reset), MP_ROM_PTR(&machine_soft_reset_obj) },
+    #if MICROPY_PY_MACHINE_BOOTLOADER
+    { MP_ROM_QSTR(MP_QSTR_bootloader), MP_ROM_PTR(&machine_bootloader_obj) },
+    #endif
+    #if MICROPY_PY_MACHINE_BARE_METAL_FUNCS
+    { MP_ROM_QSTR(MP_QSTR_reset), MP_ROM_PTR(&machine_reset_obj) },
+    { MP_ROM_QSTR(MP_QSTR_reset_cause), MP_ROM_PTR(&machine_reset_cause_obj) },
+    #endif
+
+    // Power related functions.
+    { MP_ROM_QSTR(MP_QSTR_idle), MP_ROM_PTR(&machine_idle_obj) },
+    #if MICROPY_PY_MACHINE_BARE_METAL_FUNCS
+    { MP_ROM_QSTR(MP_QSTR_freq), MP_ROM_PTR(&machine_freq_obj) },
+    { MP_ROM_QSTR(MP_QSTR_lightsleep), MP_ROM_PTR(&machine_lightsleep_obj) },
+    { MP_ROM_QSTR(MP_QSTR_deepsleep), MP_ROM_PTR(&machine_deepsleep_obj) },
+    #endif
+
+    // Interrupt related functions.
+    #if MICROPY_PY_MACHINE_DISABLE_IRQ_ENABLE_IRQ
+    { MP_ROM_QSTR(MP_QSTR_disable_irq), MP_ROM_PTR(&machine_disable_irq_obj) },
+    { MP_ROM_QSTR(MP_QSTR_enable_irq), MP_ROM_PTR(&machine_enable_irq_obj) },
+    #endif
+
+    // Functions for bit protocols.
+    #if MICROPY_PY_MACHINE_BITSTREAM
+    { MP_ROM_QSTR(MP_QSTR_bitstream), MP_ROM_PTR(&machine_bitstream_obj) },
+    #endif
+    #if MICROPY_PY_MACHINE_DHT_READINTO
+    { MP_ROM_QSTR(MP_QSTR_dht_readinto), MP_ROM_PTR(&dht_readinto_obj) },
+    #endif
+    #if MICROPY_PY_MACHINE_PULSE
+    { MP_ROM_QSTR(MP_QSTR_time_pulse_us), MP_ROM_PTR(&machine_time_pulse_us_obj) },
+    #endif
+
+    // Classes for PinBase and Signal.
+    #if MICROPY_PY_MACHINE_PIN_BASE
+    { MP_ROM_QSTR(MP_QSTR_PinBase), MP_ROM_PTR(&machine_pinbase_type) },
+    #endif
+    { MP_ROM_QSTR(MP_QSTR_Signal), MP_ROM_PTR(&machine_signal_type) },
+
+    // Classes for software bus protocols.
+    #if MICROPY_PY_MACHINE_SOFTI2C
+    { MP_ROM_QSTR(MP_QSTR_SoftI2C), MP_ROM_PTR(&mp_machine_soft_i2c_type) },
+    #endif
+    #if MICROPY_PY_MACHINE_SOFTSPI
+    { MP_ROM_QSTR(MP_QSTR_SoftSPI), MP_ROM_PTR(&mp_machine_soft_spi_type) },
+    #endif
+
+    // Classes for hardware peripherals.
+    #if MICROPY_PY_MACHINE_ADC
+    { MP_ROM_QSTR(MP_QSTR_ADC), MP_ROM_PTR(&machine_adc_type) },
+    #endif
+    #if MICROPY_PY_MACHINE_ADC_BLOCK
+    { MP_ROM_QSTR(MP_QSTR_ADCBlock), MP_ROM_PTR(&machine_adc_block_type) },
+    #endif
+    #if MICROPY_PY_MACHINE_DAC
+    { MP_ROM_QSTR(MP_QSTR_DAC), MP_ROM_PTR(&machine_dac_type) },
+    #endif
+    #if MICROPY_PY_MACHINE_I2C
+    { MP_ROM_QSTR(MP_QSTR_I2C), MP_ROM_PTR(&machine_i2c_type) },
+    #endif
+    #if MICROPY_PY_MACHINE_I2S
+    { MP_ROM_QSTR(MP_QSTR_I2S), MP_ROM_PTR(&machine_i2s_type) },
+    #endif
+    #if MICROPY_PY_MACHINE_PWM
+    { MP_ROM_QSTR(MP_QSTR_PWM), MP_ROM_PTR(&machine_pwm_type) },
+    #endif
+    #if MICROPY_PY_MACHINE_SPI
+    { MP_ROM_QSTR(MP_QSTR_SPI), MP_ROM_PTR(&machine_spi_type) },
+    #endif
+    #if MICROPY_PY_MACHINE_UART
+    { MP_ROM_QSTR(MP_QSTR_UART), MP_ROM_PTR(&machine_uart_type) },
+    #endif
+    #if MICROPY_PY_MACHINE_WDT
+    { MP_ROM_QSTR(MP_QSTR_WDT), MP_ROM_PTR(&machine_wdt_type) },
+    #endif
+
+    // A port can add extra entries to the module by defining the following macro.
+    #ifdef MICROPY_PY_MACHINE_EXTRA_GLOBALS
+    MICROPY_PY_MACHINE_EXTRA_GLOBALS
+    #endif
+};
+STATIC MP_DEFINE_CONST_DICT(machine_module_globals, machine_module_globals_table);
+
+const mp_obj_module_t mp_module_machine = {
+    .base = { &mp_type_module },
+    .globals = (mp_obj_dict_t *)&machine_module_globals,
+};
+
+MP_REGISTER_EXTENSIBLE_MODULE(MP_QSTR_machine, mp_module_machine);
+
+#endif // MICROPY_PY_MACHINE

extmod/modmachine.h

@@ -0,0 +1,269 @@
+/*
+ * This file is part of the MicroPython project, http://micropython.org/
+ *
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2023 Damien P. George
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#ifndef MICROPY_INCLUDED_EXTMOD_MODMACHINE_H
+#define MICROPY_INCLUDED_EXTMOD_MODMACHINE_H
+
+#include "py/mphal.h"
+#include "py/obj.h"
+
+#if MICROPY_PY_MACHINE
+
+#include "drivers/bus/spi.h"
+
+// Whether to enable the ADC.init() method.
+// Requires a port to implement mp_machine_adc_init_helper().
+#ifndef MICROPY_PY_MACHINE_ADC_INIT
+#define MICROPY_PY_MACHINE_ADC_INIT (0)
+#endif
+
+// Whether to enable the ADC.deinit() method.
+// Requires a port to implement mp_machine_adc_deinit().
+#ifndef MICROPY_PY_MACHINE_ADC_DEINIT
+#define MICROPY_PY_MACHINE_ADC_DEINIT (0)
+#endif
+
+// Whether to enable the ADC.block() method.
+// Requires a port to implement mp_machine_adc_block().
+#ifndef MICROPY_PY_MACHINE_ADC_BLOCK
+#define MICROPY_PY_MACHINE_ADC_BLOCK (0)
+#endif
+
+// Whether to enable the ADC.read_uv() method.
+// Requires a port to implement mp_machine_adc_read_uv().
+#ifndef MICROPY_PY_MACHINE_ADC_READ_UV
+#define MICROPY_PY_MACHINE_ADC_READ_UV (0)
+#endif
+
+// Whether to enable the ADC.atten() and ADC.width() methods.
+// Note: these are legacy and should not be used on new ports.
+#ifndef MICROPY_PY_MACHINE_ADC_ATTEN_WIDTH
+#define MICROPY_PY_MACHINE_ADC_ATTEN_WIDTH (0)
+#endif
+
+// Whether to enable the ADC.read() method.
+// Note: this is legacy and should not be used on new ports.
+#ifndef MICROPY_PY_MACHINE_ADC_READ
+#define MICROPY_PY_MACHINE_ADC_READ (0)
+#endif
+
+// Whether to enable the UART.sendbreak() method.
+// Requires a port to implement mp_machine_uart_sendbreak().
+#ifndef MICROPY_PY_MACHINE_UART_SENDBREAK
+#define MICROPY_PY_MACHINE_UART_SENDBREAK (0)
+#endif
+
+// Whether to enable the UART.readchar() and UART.writechar() methods.
+// Requires a port to implement mp_machine_uart_readchar() and mp_machine_uart_writechar().
+#ifndef MICROPY_PY_MACHINE_UART_READCHAR_WRITECHAR
+#define MICROPY_PY_MACHINE_UART_READCHAR_WRITECHAR (0)
+#endif
+
+// Whether to enable the UART.irq() method.
+// Requires a port to implement mp_machine_uart_irq().
+#ifndef MICROPY_PY_MACHINE_UART_IRQ
+#define MICROPY_PY_MACHINE_UART_IRQ (0)
+#endif
+
+// Temporary support for legacy construction of SoftI2C via I2C type.
+#define MP_MACHINE_I2C_CHECK_FOR_LEGACY_SOFTI2C_CONSTRUCTION(n_args, n_kw, all_args) \
+    do { \
+        if (n_args == 0 || all_args[0] == MP_OBJ_NEW_SMALL_INT(-1)) { \
+            mp_print_str(MICROPY_ERROR_PRINTER, "Warning: I2C(-1, ...) is deprecated, use SoftI2C(...) instead\n"); \
+            if (n_args != 0) { \
+                --n_args; \
+                ++all_args; \
+            } \
+            return MP_OBJ_TYPE_GET_SLOT(&mp_machine_soft_i2c_type, make_new)(&mp_machine_soft_i2c_type, n_args, n_kw, all_args); \
+        } \
+    } while (0)
+
+// Temporary support for legacy construction of SoftSPI via SPI type.
+#define MP_MACHINE_SPI_CHECK_FOR_LEGACY_SOFTSPI_CONSTRUCTION(n_args, n_kw, all_args) \
+    do { \
+        if (n_args == 0 || all_args[0] == MP_OBJ_NEW_SMALL_INT(-1)) { \
+            mp_print_str(MICROPY_ERROR_PRINTER, "Warning: SPI(-1, ...) is deprecated, use SoftSPI(...) instead\n"); \
+            if (n_args != 0) { \
+                --n_args; \
+                ++all_args; \
+            } \
+            return MP_OBJ_TYPE_GET_SLOT(&mp_machine_soft_spi_type, make_new)(&mp_machine_soft_spi_type, n_args, n_kw, all_args); \
+        } \
+    } while (0)
+
+#if MICROPY_PY_MACHINE_I2C || MICROPY_PY_MACHINE_SOFTI2C
+
+#define MP_MACHINE_I2C_FLAG_READ (0x01) // if not set then it's a write
+#define MP_MACHINE_I2C_FLAG_STOP (0x02)
+
+#if MICROPY_PY_MACHINE_I2C_TRANSFER_WRITE1
+// If set, the first mp_machine_i2c_buf_t in a transfer is a write.
+#define MP_MACHINE_I2C_FLAG_WRITE1 (0x04)
+#endif
+
+#endif
+
+// A port must provide these types, but they are otherwise opaque.
+typedef struct _machine_adc_obj_t machine_adc_obj_t;
+typedef struct _machine_adc_block_obj_t machine_adc_block_obj_t;
+typedef struct _machine_i2s_obj_t machine_i2s_obj_t;
+typedef struct _machine_pwm_obj_t machine_pwm_obj_t;
+typedef struct _machine_uart_obj_t machine_uart_obj_t;
+typedef struct _machine_wdt_obj_t machine_wdt_obj_t;
+
+typedef struct _machine_mem_obj_t {
+    mp_obj_base_t base;
+    unsigned elem_size; // in bytes
+} machine_mem_obj_t;
+
+#if MICROPY_PY_MACHINE_I2C || MICROPY_PY_MACHINE_SOFTI2C
+
+typedef struct _mp_machine_i2c_buf_t {
+    size_t len;
+    uint8_t *buf;
+} mp_machine_i2c_buf_t;
+
+// I2C protocol:
+// - init must be non-NULL
+// - start/stop/read/write can be NULL, meaning operation is not supported
+// - transfer must be non-NULL
+// - transfer_single only needs to be set if transfer=mp_machine_i2c_transfer_adaptor
+typedef struct _mp_machine_i2c_p_t {
+    #if MICROPY_PY_MACHINE_I2C_TRANSFER_WRITE1
+    bool transfer_supports_write1;
+    #endif
+    void (*init)(mp_obj_base_t *obj, size_t n_args, const mp_obj_t *pos_args, mp_map_t *kw_args);
+    int (*start)(mp_obj_base_t *obj);
+    int (*stop)(mp_obj_base_t *obj);
+    int (*read)(mp_obj_base_t *obj, uint8_t *dest, size_t len, bool nack);
+    int (*write)(mp_obj_base_t *obj, const uint8_t *src, size_t len);
+    int (*transfer)(mp_obj_base_t *obj, uint16_t addr, size_t n, mp_machine_i2c_buf_t *bufs, unsigned int flags);
+    int (*transfer_single)(mp_obj_base_t *obj, uint16_t addr, size_t len, uint8_t *buf, unsigned int flags);
+} mp_machine_i2c_p_t;
+
+// SoftI2C object.
+typedef struct _mp_machine_soft_i2c_obj_t {
+    mp_obj_base_t base;
+    uint32_t us_delay;
+    uint32_t us_timeout;
+    mp_hal_pin_obj_t scl;
+    mp_hal_pin_obj_t sda;
+} mp_machine_soft_i2c_obj_t;
+
+#endif
+
+#if MICROPY_PY_MACHINE_SPI || MICROPY_PY_MACHINE_SOFTSPI
+
+// SPI protocol.
+typedef struct _mp_machine_spi_p_t {
+    void (*init)(mp_obj_base_t *obj, size_t n_args, const mp_obj_t *pos_args, mp_map_t *kw_args);
+    void (*deinit)(mp_obj_base_t *obj); // can be NULL
+    void (*transfer)(mp_obj_base_t *obj, size_t len, const uint8_t *src, uint8_t *dest);
+} mp_machine_spi_p_t;
+
+// SoftSPI object.
+typedef struct _mp_machine_soft_spi_obj_t {
+    mp_obj_base_t base;
+    mp_soft_spi_obj_t spi;
+} mp_machine_soft_spi_obj_t;
+
+#endif
+
+// Objects for machine.mem8, machine.mem16 and machine.mem32.
+extern const machine_mem_obj_t machine_mem8_obj;
+extern const machine_mem_obj_t machine_mem16_obj;
+extern const machine_mem_obj_t machine_mem32_obj;
+
+// These classes correspond to machine.Type entries in the machine module.
+// Their Python bindings are implemented in extmod, and their implementation
+// is provided by a port.
+extern const mp_obj_type_t machine_adc_type;
+extern const mp_obj_type_t machine_adc_block_type;
+extern const mp_obj_type_t machine_i2c_type;
+extern const mp_obj_type_t machine_i2s_type;
+extern const mp_obj_type_t machine_mem_type;
+extern const mp_obj_type_t machine_pin_type;
+extern const mp_obj_type_t machine_pinbase_type;
+extern const mp_obj_type_t machine_pwm_type;
+extern const mp_obj_type_t machine_rtc_type;
+extern const mp_obj_type_t machine_signal_type;
+extern const mp_obj_type_t machine_spi_type;
+extern const mp_obj_type_t machine_timer_type;
+extern const mp_obj_type_t machine_uart_type;
+extern const mp_obj_type_t machine_wdt_type;
+
+#if MICROPY_PY_MACHINE_SOFTI2C
+extern const mp_obj_type_t mp_machine_soft_i2c_type;
+#endif
+#if MICROPY_PY_MACHINE_I2C || MICROPY_PY_MACHINE_SOFTI2C
+extern const mp_obj_dict_t mp_machine_i2c_locals_dict;
+#endif
+
+#if MICROPY_PY_MACHINE_SOFTSPI
+extern const mp_obj_type_t mp_machine_soft_spi_type;
+extern const mp_machine_spi_p_t mp_machine_soft_spi_p;
+#endif
+#if MICROPY_PY_MACHINE_SPI || MICROPY_PY_MACHINE_SOFTSPI
+extern const mp_obj_dict_t mp_machine_spi_locals_dict;
+#endif
+
+#if defined(MICROPY_MACHINE_MEM_GET_READ_ADDR)
+uintptr_t MICROPY_MACHINE_MEM_GET_READ_ADDR(mp_obj_t addr_o, uint align);
+#endif
+#if defined(MICROPY_MACHINE_MEM_GET_WRITE_ADDR)
+uintptr_t MICROPY_MACHINE_MEM_GET_WRITE_ADDR(mp_obj_t addr_o, uint align);
+#endif
+
+NORETURN mp_obj_t machine_bootloader(size_t n_args, const mp_obj_t *args);
+void machine_bitstream_high_low(mp_hal_pin_obj_t pin, uint32_t *timing_ns, const uint8_t *buf, size_t len);
+mp_uint_t machine_time_pulse_us(mp_hal_pin_obj_t pin, int pulse_level, mp_uint_t timeout_us);
+
+MP_DECLARE_CONST_FUN_OBJ_0(machine_unique_id_obj);
+MP_DECLARE_CONST_FUN_OBJ_0(machine_reset_obj);
+MP_DECLARE_CONST_FUN_OBJ_0(machine_reset_cause_obj);
+MP_DECLARE_CONST_FUN_OBJ_VAR_BETWEEN(machine_freq_obj);
+MP_DECLARE_CONST_FUN_OBJ_VAR_BETWEEN(machine_lightsleep_obj);
+MP_DECLARE_CONST_FUN_OBJ_VAR_BETWEEN(machine_deepsleep_obj);
+MP_DECLARE_CONST_FUN_OBJ_VAR_BETWEEN(machine_bootloader_obj);
+MP_DECLARE_CONST_FUN_OBJ_VAR_BETWEEN(machine_bitstream_obj);
+MP_DECLARE_CONST_FUN_OBJ_VAR_BETWEEN(machine_time_pulse_us_obj);
+
+#if MICROPY_PY_MACHINE_I2C
+int mp_machine_i2c_transfer_adaptor(mp_obj_base_t *self, uint16_t addr, size_t n, mp_machine_i2c_buf_t *bufs, unsigned int flags);
+int mp_machine_soft_i2c_transfer(mp_obj_base_t *self, uint16_t addr, size_t n, mp_machine_i2c_buf_t *bufs, unsigned int flags);
+#endif
+
+#if MICROPY_PY_MACHINE_SPI
+mp_obj_t mp_machine_spi_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args);
+MP_DECLARE_CONST_FUN_OBJ_VAR_BETWEEN(mp_machine_spi_read_obj);
+MP_DECLARE_CONST_FUN_OBJ_VAR_BETWEEN(mp_machine_spi_readinto_obj);
+MP_DECLARE_CONST_FUN_OBJ_2(mp_machine_spi_write_obj);
+MP_DECLARE_CONST_FUN_OBJ_3(mp_machine_spi_write_readinto_obj);
+#endif
+
+#endif // MICROPY_PY_MACHINE
+
+#endif // MICROPY_INCLUDED_EXTMOD_MODMACHINE_H

extmod/modos.c

@@ -24,6 +24,7 @@
  * THE SOFTWARE.
  */
 
+#include "py/mphal.h"
 #include "py/objstr.h"
 #include "py/runtime.h"
 
@@ -48,6 +49,13 @@
 #include "extmod/vfs_posix.h"
 #endif
 
+#if MICROPY_MBFS
+#if MICROPY_VFS
+#error "MICROPY_MBFS requires MICROPY_VFS to be disabled"
+#endif
+#include "ports/nrf/modules/os/microbitfs.h"
+#endif
+
 #if MICROPY_PY_OS_UNAME
 #include "genhdr/mpversion.h"
 #endif
@@ -121,6 +129,21 @@ STATIC MP_DEFINE_CONST_FUN_OBJ_0(mp_os_uname_obj, mp_os_uname);
 
 #endif
 
+#if MICROPY_PY_OS_DUPTERM_NOTIFY
+STATIC mp_obj_t mp_os_dupterm_notify(mp_obj_t obj_in) {
+    (void)obj_in;
+    for (;;) {
+        int c = mp_os_dupterm_rx_chr();
+        if (c < 0) {
+            break;
+        }
+        ringbuf_put(&stdin_ringbuf, c);
+    }
+    return mp_const_none;
+}
+STATIC MP_DEFINE_CONST_FUN_OBJ_1(mp_os_dupterm_notify_obj, mp_os_dupterm_notify);
+#endif
+
 STATIC const mp_rom_map_elem_t os_module_globals_table[] = {
     { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_os) },
 
@@ -187,6 +210,14 @@ STATIC const mp_rom_map_elem_t os_module_globals_table[] = {
     { MP_ROM_QSTR(MP_QSTR_VfsPosix), MP_ROM_PTR(&mp_type_vfs_posix) },
     #endif
     #endif
+
+    #if MICROPY_MBFS
+    // For special micro:bit filesystem only.
+    { MP_ROM_QSTR(MP_QSTR_listdir), MP_ROM_PTR(&os_mbfs_listdir_obj) },
+    { MP_ROM_QSTR(MP_QSTR_ilistdir), MP_ROM_PTR(&os_mbfs_ilistdir_obj) },
+    { MP_ROM_QSTR(MP_QSTR_stat), MP_ROM_PTR(&os_mbfs_stat_obj) },
+    { MP_ROM_QSTR(MP_QSTR_remove), MP_ROM_PTR(&os_mbfs_remove_obj) },
+    #endif
 };
 STATIC MP_DEFINE_CONST_DICT(os_module_globals, os_module_globals_table);
 

extmod/modre.c

@@ -238,12 +238,12 @@ STATIC mp_obj_t re_exec(bool is_anchored, uint n_args, const mp_obj_t *args) {
     }
     #endif
     int caps_num = (self->re.sub + 1) * 2;
-    mp_obj_match_t *match = m_new_obj_var(mp_obj_match_t, char *, caps_num);
+    mp_obj_match_t *match = m_new_obj_var(mp_obj_match_t, caps, char *, caps_num);
     // cast is a workaround for a bug in msvc: it treats const char** as a const pointer instead of a pointer to pointer to const char
     memset((char *)match->caps, 0, caps_num * sizeof(char *));
     int res = re1_5_recursiveloopprog(&self->re, &subj, match->caps, caps_num, is_anchored);
     if (res == 0) {
-        m_del_var(mp_obj_match_t, char *, caps_num, match);
+        m_del_var(mp_obj_match_t, caps, char *, caps_num, match);
         return mp_const_none;
     }
 

extmod/modselect.c

@@ -32,6 +32,7 @@
 #include "py/stream.h"
 #include "py/mperrno.h"
 #include "py/mphal.h"
+// CIRCUITPY-CHANGE
 #include "shared/runtime/interrupt_char.h"
 
 #if MICROPY_PY_SELECT
@@ -42,6 +43,7 @@
 
 #if MICROPY_PY_SELECT_POSIX_OPTIMISATIONS
 
+#include <string.h>
 #include <poll.h>
 
 #if !((MP_STREAM_POLL_RD) == (POLLIN) && \
@@ -143,14 +145,47 @@ STATIC void poll_obj_set_revents(poll_obj_t *poll_obj, mp_uint_t revents) {
     }
 }
 
+// How much (in pollfds) to grow the allocation for poll_set->pollfds by.
+#define POLL_SET_ALLOC_INCREMENT (4)
+
 STATIC struct pollfd *poll_set_add_fd(poll_set_t *poll_set, int fd) {
     struct pollfd *free_slot = NULL;
 
     if (poll_set->used == poll_set->max_used) {
         // No free slots below max_used, so expand max_used (and possibly allocate).
         if (poll_set->max_used >= poll_set->alloc) {
-            poll_set->pollfds = m_renew(struct pollfd, poll_set->pollfds, poll_set->alloc, poll_set->alloc + 4);
+            size_t new_alloc = poll_set->alloc + POLL_SET_ALLOC_INCREMENT;
-            poll_set->alloc += 4;
+            // Try to grow in-place.
+            struct pollfd *new_fds = m_renew_maybe(struct pollfd, poll_set->pollfds, poll_set->alloc, new_alloc, false);
+            if (!new_fds) {
+                // Failed to grow in-place. Do a new allocation and copy over the pollfd values.
+                new_fds = m_new(struct pollfd, new_alloc);
+                memcpy(new_fds, poll_set->pollfds, sizeof(struct pollfd) * poll_set->alloc);
+
+                // Update existing poll_obj_t to update their pollfd field to
+                // point to the same offset inside the new allocation.
+                for (mp_uint_t i = 0; i < poll_set->map.alloc; ++i) {
+                    if (!mp_map_slot_is_filled(&poll_set->map, i)) {
+                        continue;
+                    }
+
+                    poll_obj_t *poll_obj = MP_OBJ_TO_PTR(poll_set->map.table[i].value);
+                    if (!poll_obj) {
+                        // This is the one we're currently adding,
+                        // poll_set_add_obj doesn't assign elem->value until
+                        // afterwards.
+                        continue;
+                    }
+
+                    poll_obj->pollfd = new_fds + (poll_obj->pollfd - poll_set->pollfds);
+                }
+
+                // Delete the old allocation.
+                m_del(struct pollfd, poll_set->pollfds, poll_set->alloc);
+            }
+
+            poll_set->pollfds = new_fds;
+            poll_set->alloc = new_alloc;
         }
         free_slot = &poll_set->pollfds[poll_set->max_used++];
     } else {
@@ -307,6 +342,7 @@ STATIC mp_uint_t poll_set_poll_once(poll_set_t *poll_set, size_t *rwx_num) {
 
 STATIC mp_uint_t poll_set_poll_until_ready_or_timeout(poll_set_t *poll_set, size_t *rwx_num, mp_uint_t timeout) {
     mp_uint_t start_ticks = mp_hal_ticks_ms();
+    bool has_timeout = timeout != (mp_uint_t)-1;
 
     #if MICROPY_PY_SELECT_POSIX_OPTIMISATIONS
 
@@ -351,12 +387,12 @@ STATIC mp_uint_t poll_set_poll_until_ready_or_timeout(poll_set_t *poll_set, size
         }
 
         // Return if an object is ready, or if the timeout expired.
-        if (n_ready > 0 || (timeout != (mp_uint_t)-1 && mp_hal_ticks_ms() - start_ticks >= timeout)) {
+        if (n_ready > 0 || (has_timeout && mp_hal_ticks_ms() - start_ticks >= timeout)) {
             return n_ready;
         }
 
-        // This would be MICROPY_EVENT_POLL_HOOK but the call to poll() above already includes a delay.
+        // This would be mp_event_wait_ms() but the call to poll() above already includes a delay.
-        mp_handle_pending(true);
+        mp_event_handle_nowait();
     }
 
     #else
@@ -364,17 +400,20 @@ STATIC mp_uint_t poll_set_poll_until_ready_or_timeout(poll_set_t *poll_set, size
     for (;;) {
         // poll the objects
         mp_uint_t n_ready = poll_set_poll_once(poll_set, rwx_num);
-        if (n_ready > 0 || (timeout != (mp_uint_t)-1 && mp_hal_ticks_ms() - start_ticks >= timeout)) {
+        uint32_t elapsed = mp_hal_ticks_ms() - start_ticks;
+        if (n_ready > 0 || (has_timeout && elapsed >= timeout)) {
             return n_ready;
         }
-        // CIRCUITPY-CHANGE
+        // CIRCUITPY-CHANGE: check for ctrl-C interrupt
-        RUN_BACKGROUND_TASKS;
         if (mp_hal_is_interrupted()) {
             return 0;
+
+        // CIRCUITPY-CHANGE: mp_event_wait_* do RUN_BACKGROUND_TASKS
+        if (has_timeout) {
+            mp_event_wait_ms(timeout - elapsed);
+        } else {
+            mp_event_wait_indefinite();
         }
-        #ifdef MICROPY_EVENT_POLL_HOOK
-        MICROPY_EVENT_POLL_HOOK;
-        #endif
     }
 
     #endif

extmod/moductypes.c

@@ -96,7 +96,7 @@ STATIC NORETURN void syntax_error(void) {
 STATIC mp_obj_t uctypes_struct_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args) {
     mp_arg_check_num(n_args, n_kw, 2, 3, false);
     mp_obj_uctypes_struct_t *o = mp_obj_malloc(mp_obj_uctypes_struct_t, type);
-    o->addr = (void *)(uintptr_t)mp_obj_int_get_truncated(args[0]);
+    o->addr = (void *)(uintptr_t)mp_obj_get_int_truncated(args[0]);
     o->desc = args[1];
     o->flags = LAYOUT_NATIVE;
     if (n_args == 3) {

extmod/vfs.c

@@ -42,6 +42,7 @@
 #include "extmod/vfs_lfs.h"
 #endif
 
+// CIRCUITPY-CHANGE: handle undefined without a warning
 #if defined(MICROPY_VFS_POSIX) && MICROPY_VFS_POSIX
 #include "extmod/vfs_posix.h"
 #endif
@@ -310,6 +311,7 @@ mp_obj_t mp_vfs_open(size_t n_args, const mp_obj_t *pos_args, mp_map_t *kw_args)
     mp_arg_val_t args[MP_ARRAY_SIZE(allowed_args)];
     mp_arg_parse_all(n_args, pos_args, kw_args, MP_ARRAY_SIZE(allowed_args), allowed_args, args);
 
+    // CIRCUITPY-CHANGE: handle undefined without a warning
     #if defined(MICROPY_VFS_POSIX) && MICROPY_VFS_POSIX
     // If the file is an integer then delegate straight to the POSIX handler
     if (mp_obj_is_small_int(args[ARG_file].u_obj)) {
@@ -434,6 +436,7 @@ mp_obj_t mp_vfs_listdir(size_t n_args, const mp_obj_t *args) {
 MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_vfs_listdir_obj, 0, 1, mp_vfs_listdir);
 
 mp_obj_t mp_vfs_mkdir(mp_obj_t path_in) {
+    // CIRCUITPY-CHANGE: initialize
     mp_obj_t path_out = mp_const_none;
     mp_vfs_mount_t *vfs = lookup_path(path_in, &path_out);
     if (vfs == MP_VFS_ROOT || (vfs != MP_VFS_NONE && !strcmp(mp_obj_str_get_str(path_out), "/"))) {

extmod/vfs.h

@@ -28,6 +28,7 @@
 
 #include "py/builtin.h"
 #include "py/obj.h"
+// CIRCUITPY-CHANGE
 #include "py/proto.h"
 
 // return values of mp_vfs_lookup_path
@@ -44,6 +45,7 @@
 #define MP_BLOCKDEV_FLAG_FREE_OBJ       (0x0002) // fs_user_mount_t obj should be freed on umount
 #define MP_BLOCKDEV_FLAG_HAVE_IOCTL     (0x0004) // new protocol with ioctl
 #define MP_BLOCKDEV_FLAG_NO_FILESYSTEM  (0x0008) // the block device has no filesystem on it
+// CIRCUITPY-CHANGE: additional flags
 // Device is writable over USB and read-only to MicroPython.
 #define MP_BLOCKDEV_FLAG_USB_WRITABLE             (0x0010)
 // Bit set when the above flag is checked before opening a file for write.
@@ -59,9 +61,9 @@
 #define MP_BLOCKDEV_IOCTL_BLOCK_SIZE    (5)
 #define MP_BLOCKDEV_IOCTL_BLOCK_ERASE   (6)
 
-
 // At the moment the VFS protocol just has import_stat, but could be extended to other methods
 typedef struct _mp_vfs_proto_t {
+    // CIRCUITPY-CHANGE
     MP_PROTOCOL_HEAD
     mp_import_stat_t (*import_stat)(void *self, const char *path);
 } mp_vfs_proto_t;
@@ -88,6 +90,7 @@ typedef struct _mp_vfs_mount_t {
     struct _mp_vfs_mount_t *next;
 } mp_vfs_mount_t;
 
+// CIRCUITPY-CHANGE: allow outside use of ilistdir_it_iternext
 typedef struct _mp_vfs_ilistdir_it_t {
     mp_obj_base_t base;
     mp_fun_1_t iternext;
@@ -124,7 +127,6 @@ mp_obj_t mp_vfs_stat(mp_obj_t path_in);
 mp_obj_t mp_vfs_statvfs(mp_obj_t path_in);
 
 int mp_vfs_mount_and_chdir_protected(mp_obj_t bdev, mp_obj_t mount_point);
-mp_obj_t mp_vfs_ilistdir_it_iternext(mp_obj_t self_in);
 
 MP_DECLARE_CONST_FUN_OBJ_KW(mp_vfs_mount_obj);
 MP_DECLARE_CONST_FUN_OBJ_1(mp_vfs_umount_obj);

extmod/vfs_blockdev.c

@@ -30,6 +30,7 @@
 #include "py/mperrno.h"
 #include "extmod/vfs.h"
 
+// CIRCUITPY-CHANGE
 #if CIRCUITPY_SDCARDIO
 #include "shared-bindings/sdcardio/SDCard.h"
 #endif

extmod/vfs_fat.c

@@ -36,6 +36,7 @@
 #error "with MICROPY_VFS_FAT enabled, must also enable MICROPY_VFS"
 #endif
 
+// CIRCUITPY-CHANGE: extra includes
 #include <string.h>
 #include "py/obj.h"
 #include "py/objproperty.h"
@@ -54,6 +55,7 @@
 
 #define mp_obj_fat_vfs_t fs_user_mount_t
 
+// CIRCUITPY-CHANGE
 // Factoring this common call saves about 90 bytes.
 STATIC NORETURN void mp_raise_OSError_fresult(FRESULT res) {
     mp_raise_OSError(fresult_to_errno_table[res]);
@@ -92,12 +94,14 @@ STATIC mp_obj_t fat_vfs_make_new(const mp_obj_type_t *type, size_t n_args, size_
         // don't error out if no filesystem, to let mkfs()/mount() create one if wanted
         vfs->blockdev.flags |= MP_BLOCKDEV_FLAG_NO_FILESYSTEM;
     } else if (res != FR_OK) {
+        // CIRCUITPY-CHANGE
         mp_raise_OSError_fresult(res);
     }
 
     return MP_OBJ_FROM_PTR(vfs);
 }
 
+// CIRCUITPY-CHANGE
 STATIC void verify_fs_writable(fs_user_mount_t *vfs) {
     if (!filesystem_is_writable_by_python(vfs)) {
         mp_raise_OSError(MP_EROFS);
@@ -125,6 +129,7 @@ STATIC mp_obj_t fat_vfs_mkfs(mp_obj_t bdev_in) {
         res = f_mkfs(&vfs->fatfs, FM_FAT32, 0, working_buf, sizeof(working_buf));
     }
     if (res != FR_OK) {
+        // CIRCUITPY-CHANGE
         mp_raise_OSError_fresult(res);
     }
 
@@ -209,6 +214,7 @@ STATIC mp_obj_t fat_vfs_ilistdir_func(size_t n_args, const mp_obj_t *args) {
     iter->is_str = is_str_type;
     FRESULT res = f_opendir(&self->fatfs, &iter->dir, path);
     if (res != FR_OK) {
+        // CIRCUITPY-CHANGE
         mp_raise_OSError_fresult(res);
     }
 
@@ -218,6 +224,7 @@ STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(fat_vfs_ilistdir_obj, 1, 2, fat_vfs_i
 
 STATIC mp_obj_t fat_vfs_remove_internal(mp_obj_t vfs_in, mp_obj_t path_in, mp_int_t attr) {
     mp_obj_fat_vfs_t *self = MP_OBJ_TO_PTR(vfs_in);
+    // CIRCUITPY-CHANGE
     verify_fs_writable(self);
     const char *path = mp_obj_str_get_str(path_in);
 
@@ -225,6 +232,7 @@ STATIC mp_obj_t fat_vfs_remove_internal(mp_obj_t vfs_in, mp_obj_t path_in, mp_in
     FRESULT res = f_stat(&self->fatfs, path, &fno);
 
     if (res != FR_OK) {
+        // CIRCUITPY-CHANGE
         mp_raise_OSError_fresult(res);
     }
 
@@ -233,6 +241,7 @@ STATIC mp_obj_t fat_vfs_remove_internal(mp_obj_t vfs_in, mp_obj_t path_in, mp_in
         res = f_unlink(&self->fatfs, path);
 
         if (res != FR_OK) {
+            // CIRCUITPY-CHANGE
             mp_raise_OSError_fresult(res);
         }
         return mp_const_none;
@@ -253,10 +262,10 @@ STATIC MP_DEFINE_CONST_FUN_OBJ_2(fat_vfs_rmdir_obj, fat_vfs_rmdir);
 
 STATIC mp_obj_t fat_vfs_rename(mp_obj_t vfs_in, mp_obj_t path_in, mp_obj_t path_out) {
     mp_obj_fat_vfs_t *self = MP_OBJ_TO_PTR(vfs_in);
+    // CIRCUITPY-CHANGE
     verify_fs_writable(self);
     const char *old_path = mp_obj_str_get_str(path_in);
     const char *new_path = mp_obj_str_get_str(path_out);
-
     FRESULT res = f_rename(&self->fatfs, old_path, new_path);
     if (res == FR_EXIST) {
         // if new_path exists then try removing it (but only if it's a file)
@@ -267,6 +276,7 @@ STATIC mp_obj_t fat_vfs_rename(mp_obj_t vfs_in, mp_obj_t path_in, mp_obj_t path_
     if (res == FR_OK) {
         return mp_const_none;
     } else {
+        // CIRCUITPY-CHANGE
         mp_raise_OSError_fresult(res);
     }
 
@@ -281,6 +291,7 @@ STATIC mp_obj_t fat_vfs_mkdir(mp_obj_t vfs_in, mp_obj_t path_o) {
     if (res == FR_OK) {
         return mp_const_none;
     } else {
+        // CIRCUITPY-CHANGE
         mp_raise_OSError_fresult(res);
     }
 }
@@ -295,6 +306,7 @@ STATIC mp_obj_t fat_vfs_chdir(mp_obj_t vfs_in, mp_obj_t path_in) {
     FRESULT res = f_chdir(&self->fatfs, path);
 
     if (res != FR_OK) {
+        // CIRCUITPY-CHANGE
         mp_raise_OSError_fresult(res);
     }
 
@@ -308,6 +320,7 @@ STATIC mp_obj_t fat_vfs_getcwd(mp_obj_t vfs_in) {
     char buf[MICROPY_ALLOC_PATH_MAX + 1];
     FRESULT res = f_getcwd(&self->fatfs, buf, sizeof(buf));
     if (res != FR_OK) {
+        // CIRCUITPY-CHANGE
         mp_raise_OSError_fresult(res);
     }
     return mp_obj_new_str(buf, strlen(buf));
@@ -329,6 +342,7 @@ STATIC mp_obj_t fat_vfs_stat(mp_obj_t vfs_in, mp_obj_t path_in) {
     } else {
         FRESULT res = f_stat(&self->fatfs, path, &fno);
         if (res != FR_OK) {
+            // CIRCUITPY-CHANGE
             mp_raise_OSError_fresult(res);
         }
     }
@@ -340,6 +354,7 @@ STATIC mp_obj_t fat_vfs_stat(mp_obj_t vfs_in, mp_obj_t path_in) {
     } else {
         mode |= MP_S_IFREG;
     }
+    // CIRCUITPY-CHANGE
     #if MICROPY_LONGINT_IMPL == MICROPY_LONGINT_IMPL_NONE
     // On non-longint builds, the number of seconds since 1970 (epoch) is too
     // large to fit in a smallint, so just return 31-DEC-1999 (0).
@@ -362,9 +377,9 @@ STATIC mp_obj_t fat_vfs_stat(mp_obj_t vfs_in, mp_obj_t path_in) {
     t->items[4] = MP_OBJ_NEW_SMALL_INT(0); // st_uid
     t->items[5] = MP_OBJ_NEW_SMALL_INT(0); // st_gid
     t->items[6] = mp_obj_new_int_from_uint(fno.fsize); // st_size
-    t->items[7] = seconds; // st_atime
+    t->items[7] = mp_obj_new_int_from_uint(seconds); // st_atime
-    t->items[8] = seconds; // st_mtime
+    t->items[8] = mp_obj_new_int_from_uint(seconds); // st_mtime
-    t->items[9] = seconds; // st_ctime
+    t->items[9] = mp_obj_new_int_from_uint(seconds); // st_ctime
 
     return MP_OBJ_FROM_PTR(t);
 }
@@ -379,6 +394,7 @@ STATIC mp_obj_t fat_vfs_statvfs(mp_obj_t vfs_in, mp_obj_t path_in) {
     FATFS *fatfs = &self->fatfs;
     FRESULT res = f_getfree(fatfs, &nclst);
     if (FR_OK != res) {
+        // CIRCUITPY-CHANGE
         mp_raise_OSError_fresult(res);
     }
 
@@ -417,6 +433,7 @@ STATIC mp_obj_t vfs_fat_mount(mp_obj_t self_in, mp_obj_t readonly, mp_obj_t mkfs
         res = f_mkfs(&self->fatfs, FM_FAT | FM_SFD, 0, working_buf, sizeof(working_buf));
     }
     if (res != FR_OK) {
+        // CIRCUITPY-CHANGE
         mp_raise_OSError_fresult(res);
     }
     self->blockdev.flags &= ~MP_BLOCKDEV_FLAG_NO_FILESYSTEM;
@@ -432,6 +449,7 @@ STATIC mp_obj_t vfs_fat_umount(mp_obj_t self_in) {
 }
 STATIC MP_DEFINE_CONST_FUN_OBJ_1(fat_vfs_umount_obj, vfs_fat_umount);
 
+// CIRCUITPY-CHANGE
 STATIC mp_obj_t vfs_fat_utime(mp_obj_t vfs_in, mp_obj_t path_in, mp_obj_t times_in) {
     mp_obj_fat_vfs_t *self = MP_OBJ_TO_PTR(vfs_in);
     const char *path = mp_obj_str_get_str(path_in);
@@ -524,6 +542,7 @@ STATIC const mp_rom_map_elem_t fat_vfs_locals_dict_table[] = {
     { MP_ROM_QSTR(MP_QSTR_statvfs), MP_ROM_PTR(&fat_vfs_statvfs_obj) },
     { MP_ROM_QSTR(MP_QSTR_mount), MP_ROM_PTR(&vfs_fat_mount_obj) },
     { MP_ROM_QSTR(MP_QSTR_umount), MP_ROM_PTR(&fat_vfs_umount_obj) },
+    // CIRCUITPY-CHANGE: added
     { MP_ROM_QSTR(MP_QSTR_utime), MP_ROM_PTR(&fat_vfs_utime_obj) },
     { MP_ROM_QSTR(MP_QSTR_readonly), MP_ROM_PTR(&fat_vfs_readonly_obj) },
     #if MICROPY_FATFS_USE_LABEL
@@ -533,6 +552,7 @@ STATIC const mp_rom_map_elem_t fat_vfs_locals_dict_table[] = {
 STATIC MP_DEFINE_CONST_DICT(fat_vfs_locals_dict, fat_vfs_locals_dict_table);
 
 STATIC const mp_vfs_proto_t fat_vfs_proto = {
+    // CIRCUITPY-CHANGE
     MP_PROTO_IMPLEMENT(MP_QSTR_protocol_vfs)
     .import_stat = fat_vfs_import_stat,
 };
@@ -540,6 +560,7 @@ STATIC const mp_vfs_proto_t fat_vfs_proto = {
 MP_DEFINE_CONST_OBJ_TYPE(
     mp_fat_vfs_type,
     MP_QSTR_VfsFat,
+    // CIRCUITPY-CHANGE: has properties
     MP_TYPE_FLAG_HAS_SPECIAL_ACCESSORS,
     make_new, fat_vfs_make_new,
     protocol, &fat_vfs_proto,

extmod/vfs_fat.h

@@ -47,6 +47,7 @@ extern const mp_obj_type_t mp_type_vfs_fat_textio;
 
 MP_DECLARE_CONST_FUN_OBJ_3(fat_vfs_open_obj);
 
+// CIRCUITPY-CHANGE
 typedef struct _pyb_file_obj_t {
     mp_obj_base_t base;
     FIL fp;

extmod/vfs_fat_file.c

@@ -27,6 +27,7 @@
 #include "py/mpconfig.h"
 #if MICROPY_VFS && MICROPY_VFS_FAT
 
+// CIRCUITPY-CHANGE: more includes
 #include <stdio.h>
 #include <string.h>
 
@@ -63,6 +64,7 @@ const byte fresult_to_errno_table[20] = {
 
 STATIC void file_obj_print(const mp_print_t *print, mp_obj_t self_in, mp_print_kind_t kind) {
     (void)kind;
+    // CIRCUITPY-CHANGE
     mp_printf(print, "<io.%q %p>", mp_obj_get_type_qstr(self_in), MP_OBJ_TO_PTR(self_in));
 }
 
@@ -198,6 +200,7 @@ STATIC mp_obj_t fat_vfs_open(mp_obj_t self_in, mp_obj_t path_in, mp_obj_t mode_i
     const mp_obj_type_t *type = &mp_type_vfs_fat_textio;
     int mode = 0;
     const char *mode_s = mp_obj_str_get_str(mode_in);
+    // CIRCUITPY-CHANGE: validate mode
     uint32_t rwxa_count = 0;
     uint32_t bt_count = 0;
     uint32_t plus_count = 0;

extmod/vfs_lfsx.c

@@ -99,7 +99,7 @@ STATIC void MP_VFS_LFSx(init_config)(MP_OBJ_VFS_LFSx * self, mp_obj_t bdev, size
     config->lookahead_buffer = m_new(uint8_t, config->lookahead / 8);
     #else
     config->block_cycles = 100;
-    config->cache_size = 4 * MAX(read_size, prog_size);
+    config->cache_size = MIN(config->block_size, (4 * MAX(read_size, prog_size)));
     config->lookahead_size = lookahead;
     config->read_buffer = m_new(uint8_t, config->cache_size);
     config->prog_buffer = m_new(uint8_t, config->cache_size);

extmod/vfs_lfsx_file.c

@@ -90,9 +90,9 @@ mp_obj_t MP_VFS_LFSx(file_open)(mp_obj_t self_in, mp_obj_t path_in, mp_obj_t mod
     }
 
     #if LFS_BUILD_VERSION == 1
-    MP_OBJ_VFS_LFSx_FILE *o = m_new_obj_var_with_finaliser(MP_OBJ_VFS_LFSx_FILE, uint8_t, self->lfs.cfg->prog_size);
+    MP_OBJ_VFS_LFSx_FILE *o = m_new_obj_var_with_finaliser(MP_OBJ_VFS_LFSx_FILE, file_buffer, uint8_t, self->lfs.cfg->prog_size);
     #else
-    MP_OBJ_VFS_LFSx_FILE *o = m_new_obj_var_with_finaliser(MP_OBJ_VFS_LFSx_FILE, uint8_t, self->lfs.cfg->cache_size);
+    MP_OBJ_VFS_LFSx_FILE *o = m_new_obj_var_with_finaliser(MP_OBJ_VFS_LFSx_FILE, file_buffer, uint8_t, self->lfs.cfg->cache_size);
     #endif
     o->base.type = type;
     o->vfs = self;

extmod/vfs_posix.c

@@ -46,6 +46,9 @@
 #ifdef _MSC_VER
 #include <direct.h> // For mkdir etc.
 #endif
+#ifdef _WIN32
+#include <windows.h>
+#endif
 
 typedef struct _mp_obj_vfs_posix_t {
     mp_obj_base_t base;
@@ -55,21 +58,23 @@ typedef struct _mp_obj_vfs_posix_t {
 } mp_obj_vfs_posix_t;
 
 STATIC const char *vfs_posix_get_path_str(mp_obj_vfs_posix_t *self, mp_obj_t path) {
-    if (self->root_len == 0) {
+    const char *path_str = mp_obj_str_get_str(path);
-        return mp_obj_str_get_str(path);
+    if (self->root_len == 0 || path_str[0] != '/') {
+        return path_str;
     } else {
-        self->root.len = self->root_len;
+        self->root.len = self->root_len - 1;
-        vstr_add_str(&self->root, mp_obj_str_get_str(path));
+        vstr_add_str(&self->root, path_str);
         return vstr_null_terminated_str(&self->root);
     }
 }
 
 STATIC mp_obj_t vfs_posix_get_path_obj(mp_obj_vfs_posix_t *self, mp_obj_t path) {
-    if (self->root_len == 0) {
+    const char *path_str = mp_obj_str_get_str(path);
+    if (self->root_len == 0 || path_str[0] != '/') {
         return path;
     } else {
-        self->root.len = self->root_len;
+        self->root.len = self->root_len - 1;
-        vstr_add_str(&self->root, mp_obj_str_get_str(path));
+        vstr_add_str(&self->root, path_str);
         return mp_obj_new_str(self->root.buf, self->root.len);
     }
 }
@@ -107,7 +112,28 @@ STATIC mp_obj_t vfs_posix_make_new(const mp_obj_type_t *type, size_t n_args, siz
     mp_obj_vfs_posix_t *vfs = mp_obj_malloc(mp_obj_vfs_posix_t, type);
     vstr_init(&vfs->root, 0);
     if (n_args == 1) {
-        vstr_add_str(&vfs->root, mp_obj_str_get_str(args[0]));
+        const char *root = mp_obj_str_get_str(args[0]);
+        // if the root is relative, make it absolute, otherwise we'll get confused by chdir
+        #ifdef _WIN32
+        char buf[MICROPY_ALLOC_PATH_MAX + 1];
+        DWORD result = GetFullPathNameA(root, sizeof(buf), buf, NULL);
+        if (result > 0 && result < sizeof(buf)) {
+            vstr_add_str(&vfs->root, buf);
+        } else {
+            mp_raise_OSError(GetLastError());
+        }
+        #else
+        if (root[0] != '\0' && root[0] != '/') {
+            char buf[MICROPY_ALLOC_PATH_MAX + 1];
+            const char *cwd = getcwd(buf, sizeof(buf));
+            if (cwd == NULL) {
+                mp_raise_OSError(errno);
+            }
+            vstr_add_str(&vfs->root, cwd);
+            vstr_add_char(&vfs->root, '/');
+        }
+        vstr_add_str(&vfs->root, root);
+        #endif
         vstr_add_char(&vfs->root, '/');
     }
     vfs->root_len = vfs->root.len;
@@ -160,7 +186,14 @@ STATIC mp_obj_t vfs_posix_getcwd(mp_obj_t self_in) {
     if (ret == NULL) {
         mp_raise_OSError(errno);
     }
-    ret += self->root_len;
+    if (self->root_len > 0) {
+        ret += self->root_len - 1;
+        #ifdef _WIN32
+        if (*ret == '\\') {
+            *(char *)ret = '/';
+        }
+        #endif
+    }
     return mp_obj_new_str(ret, strlen(ret));
 }
 STATIC MP_DEFINE_CONST_FUN_OBJ_1(vfs_posix_getcwd_obj, vfs_posix_getcwd);

extmod/vfs_posix.h

@@ -27,7 +27,6 @@
 #define MICROPY_INCLUDED_EXTMOD_VFS_POSIX_H
 
 #include "py/lexer.h"
-#include "py/mpconfig.h"
 #include "py/obj.h"
 
 extern const mp_obj_type_t mp_type_vfs_posix;

extmod/vfs_posix_file.c

@@ -279,14 +279,14 @@ STATIC const mp_stream_p_t vfs_posix_textio_stream_p = {
 
 #if MICROPY_PY_SYS_STDIO_BUFFER
 
-const mp_obj_vfs_posix_file_t mp_sys_stdin_buffer_obj = {{&mp_type_vfs_posix_fileio}, STDIN_FILENO};
+mp_obj_vfs_posix_file_t mp_sys_stdin_buffer_obj = {{&mp_type_vfs_posix_fileio}, STDIN_FILENO};
-const mp_obj_vfs_posix_file_t mp_sys_stdout_buffer_obj = {{&mp_type_vfs_posix_fileio}, STDOUT_FILENO};
+mp_obj_vfs_posix_file_t mp_sys_stdout_buffer_obj = {{&mp_type_vfs_posix_fileio}, STDOUT_FILENO};
-const mp_obj_vfs_posix_file_t mp_sys_stderr_buffer_obj = {{&mp_type_vfs_posix_fileio}, STDERR_FILENO};
+mp_obj_vfs_posix_file_t mp_sys_stderr_buffer_obj = {{&mp_type_vfs_posix_fileio}, STDERR_FILENO};
 
 // Forward declarations.
-const mp_obj_vfs_posix_file_t mp_sys_stdin_obj;
+mp_obj_vfs_posix_file_t mp_sys_stdin_obj;
-const mp_obj_vfs_posix_file_t mp_sys_stdout_obj;
+mp_obj_vfs_posix_file_t mp_sys_stdout_obj;
-const mp_obj_vfs_posix_file_t mp_sys_stderr_obj;
+mp_obj_vfs_posix_file_t mp_sys_stderr_obj;
 
 STATIC void vfs_posix_textio_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) {
     if (dest[0] != MP_OBJ_NULL) {
@@ -332,8 +332,8 @@ MP_DEFINE_CONST_OBJ_TYPE(
     locals_dict, &vfs_posix_rawfile_locals_dict
     );
 
-const mp_obj_vfs_posix_file_t mp_sys_stdin_obj = {{&mp_type_vfs_posix_textio}, STDIN_FILENO};
+mp_obj_vfs_posix_file_t mp_sys_stdin_obj = {{&mp_type_vfs_posix_textio}, STDIN_FILENO};
-const mp_obj_vfs_posix_file_t mp_sys_stdout_obj = {{&mp_type_vfs_posix_textio}, STDOUT_FILENO};
+mp_obj_vfs_posix_file_t mp_sys_stdout_obj = {{&mp_type_vfs_posix_textio}, STDOUT_FILENO};
-const mp_obj_vfs_posix_file_t mp_sys_stderr_obj = {{&mp_type_vfs_posix_textio}, STDERR_FILENO};
+mp_obj_vfs_posix_file_t mp_sys_stderr_obj = {{&mp_type_vfs_posix_textio}, STDERR_FILENO};
 
 #endif // MICROPY_VFS_POSIX

extmod/vfs_reader.c

@@ -26,6 +26,7 @@
 
 #include <stdio.h>
 #include <string.h>
+#include <stdlib.h>
 
 #include "py/runtime.h"
 #include "py/stream.h"
@@ -34,33 +35,39 @@
 
 #if MICROPY_READER_VFS
 
+#ifndef MICROPY_READER_VFS_DEFAULT_BUFFER_SIZE
+#define MICROPY_READER_VFS_DEFAULT_BUFFER_SIZE (2 * MICROPY_BYTES_PER_GC_BLOCK - offsetof(mp_reader_vfs_t, buf))
+#endif
+#define MICROPY_READER_VFS_MIN_BUFFER_SIZE (MICROPY_BYTES_PER_GC_BLOCK - offsetof(mp_reader_vfs_t, buf))
+#define MICROPY_READER_VFS_MAX_BUFFER_SIZE (255)
+
 typedef struct _mp_reader_vfs_t {
     mp_obj_t file;
-    uint16_t len;
+    uint8_t bufpos;
-    uint16_t pos;
+    uint8_t buflen;
-    byte buf[24];
+    uint8_t bufsize;
+    byte buf[];
 } mp_reader_vfs_t;
 
 STATIC mp_uint_t mp_reader_vfs_readbyte(void *data) {
     mp_reader_vfs_t *reader = (mp_reader_vfs_t *)data;
-    if (reader->pos >= reader->len) {
+    if (reader->bufpos >= reader->buflen) {
-        if (reader->len < sizeof(reader->buf)) {
+        if (reader->buflen < reader->bufsize) {
             return MP_READER_EOF;
         } else {
             int errcode;
-            reader->len = mp_stream_rw(reader->file, reader->buf, sizeof(reader->buf),
+            reader->buflen = mp_stream_rw(reader->file, reader->buf, reader->bufsize, &errcode, MP_STREAM_RW_READ | MP_STREAM_RW_ONCE);
-                &errcode, MP_STREAM_RW_READ | MP_STREAM_RW_ONCE);
             if (errcode != 0) {
                 // TODO handle errors properly
                 return MP_READER_EOF;
             }
-            if (reader->len == 0) {
+            if (reader->buflen == 0) {
                 return MP_READER_EOF;
             }
-            reader->pos = 0;
+            reader->bufpos = 0;
         }
     }
-    return reader->buf[reader->pos++];
+    return reader->buf[reader->bufpos++];
 }
 
 STATIC void mp_reader_vfs_close(void *data) {
@@ -69,19 +76,32 @@ STATIC void mp_reader_vfs_close(void *data) {
     m_del_obj(mp_reader_vfs_t, reader);
 }
 
-void mp_reader_new_file(mp_reader_t *reader, const char *filename) {
+void mp_reader_new_file(mp_reader_t *reader, qstr filename) {
-    mp_reader_vfs_t *rf = m_new_obj(mp_reader_vfs_t);
     mp_obj_t args[2] = {
-        mp_obj_new_str(filename, strlen(filename)),
+        MP_OBJ_NEW_QSTR(filename),
         MP_OBJ_NEW_QSTR(MP_QSTR_rb),
     };
-    rf->file = mp_vfs_open(MP_ARRAY_SIZE(args), &args[0], (mp_map_t *)&mp_const_empty_map);
+    mp_obj_t file = mp_vfs_open(MP_ARRAY_SIZE(args), &args[0], (mp_map_t *)&mp_const_empty_map);
-    int errcode;
+
-    rf->len = mp_stream_rw(rf->file, rf->buf, sizeof(rf->buf), &errcode, MP_STREAM_RW_READ | MP_STREAM_RW_ONCE);
+    const mp_stream_p_t *stream_p = mp_get_stream(file);
+    int errcode = 0;
+    mp_uint_t bufsize = stream_p->ioctl(file, MP_STREAM_GET_BUFFER_SIZE, 0, &errcode);
+    if (bufsize == MP_STREAM_ERROR || bufsize == 0) {
+        // bufsize == 0 is included here to support mpremote v1.21 and older where mount file ioctl
+        // returned 0 by default.
+        bufsize = MICROPY_READER_VFS_DEFAULT_BUFFER_SIZE;
+    } else {
+        bufsize = MIN(MICROPY_READER_VFS_MAX_BUFFER_SIZE, MAX(MICROPY_READER_VFS_MIN_BUFFER_SIZE, bufsize));
+    }
+
+    mp_reader_vfs_t *rf = m_new_obj_var(mp_reader_vfs_t, buf, byte, bufsize);
+    rf->file = file;
+    rf->bufsize = bufsize;
+    rf->buflen = mp_stream_rw(rf->file, rf->buf, rf->bufsize, &errcode, MP_STREAM_RW_READ | MP_STREAM_RW_ONCE);
     if (errcode != 0) {
         mp_raise_OSError(errcode);
     }
-    rf->pos = 0;
+    rf->bufpos = 0;
     reader->data = rf;
     reader->readbyte = mp_reader_vfs_readbyte;
     reader->close = mp_reader_vfs_close;

extmod/virtpin.h

@@ -27,6 +27,7 @@
 #define MICROPY_INCLUDED_EXTMOD_VIRTPIN_H
 
 #include "py/obj.h"
+// CIRCUITPY-CHANGE
 #include "py/proto.h"
 
 #define MP_PIN_READ   (1)
@@ -36,6 +37,7 @@
 
 // Pin protocol
 typedef struct _mp_pin_p_t {
+    // CIRCUITPY-CHANGE
     MP_PROTOCOL_HEAD
     mp_uint_t (*ioctl)(mp_obj_t obj, mp_uint_t request, uintptr_t arg, int *errcode);
 } mp_pin_p_t;

lib/littlefs/lfs2.h

@@ -8,8 +8,6 @@
 #ifndef LFS2_H
 #define LFS2_H
 
-#include <stdint.h>
-#include <stdbool.h>
 #include "lfs2_util.h"
 
 #ifdef __cplusplus
@@ -23,14 +21,14 @@ extern "C"
 // Software library version
 // Major (top-nibble), incremented on backwards incompatible changes
 // Minor (bottom-nibble), incremented on feature additions
-#define LFS2_VERSION 0x00020005
+#define LFS2_VERSION 0x00020008
 #define LFS2_VERSION_MAJOR (0xffff & (LFS2_VERSION >> 16))
 #define LFS2_VERSION_MINOR (0xffff & (LFS2_VERSION >>  0))
 
 // Version of On-disk data structures
 // Major (top-nibble), incremented on backwards incompatible changes
 // Minor (bottom-nibble), incremented on feature additions
-#define LFS2_DISK_VERSION 0x00020000
+#define LFS2_DISK_VERSION 0x00020001
 #define LFS2_DISK_VERSION_MAJOR (0xffff & (LFS2_DISK_VERSION >> 16))
 #define LFS2_DISK_VERSION_MINOR (0xffff & (LFS2_DISK_VERSION >>  0))
 
@@ -114,6 +112,8 @@ enum lfs2_type {
     LFS2_TYPE_SOFTTAIL       = 0x600,
     LFS2_TYPE_HARDTAIL       = 0x601,
     LFS2_TYPE_MOVESTATE      = 0x7ff,
+    LFS2_TYPE_CCRC           = 0x500,
+    LFS2_TYPE_FCRC           = 0x5ff,
 
     // internal chip sources
     LFS2_FROM_NOOP           = 0x000,
@@ -263,6 +263,14 @@ struct lfs2_config {
     // can help bound the metadata compaction time. Must be <= block_size.
     // Defaults to block_size when zero.
     lfs2_size_t metadata_max;
+
+#ifdef LFS2_MULTIVERSION
+    // On-disk version to use when writing in the form of 16-bit major version
+    // + 16-bit minor version. This limiting metadata to what is supported by
+    // older minor versions. Note that some features will be lost. Defaults to 
+    // to the most recent minor version when zero.
+    uint32_t disk_version;
+#endif
 };
 
 // File info structure
@@ -280,6 +288,27 @@ struct lfs2_info {
     char name[LFS2_NAME_MAX+1];
 };
 
+// Filesystem info structure
+struct lfs2_fsinfo {
+    // On-disk version.
+    uint32_t disk_version;
+
+    // Size of a logical block in bytes.
+    lfs2_size_t block_size;
+
+    // Number of logical blocks in filesystem.
+    lfs2_size_t block_count;
+
+    // Upper limit on the length of file names in bytes.
+    lfs2_size_t name_max;
+
+    // Upper limit on the size of files in bytes.
+    lfs2_size_t file_max;
+
+    // Upper limit on the size of custom attributes in bytes.
+    lfs2_size_t attr_max;
+};
+
 // Custom attribute structure, used to describe custom attributes
 // committed atomically during file writes.
 struct lfs2_attr {
@@ -410,6 +439,7 @@ typedef struct lfs2 {
     } free;
 
     const struct lfs2_config *cfg;
+    lfs2_size_t block_count;
     lfs2_size_t name_max;
     lfs2_size_t file_max;
     lfs2_size_t attr_max;
@@ -534,8 +564,8 @@ int lfs2_file_open(lfs2_t *lfs2, lfs2_file_t *file,
 // are values from the enum lfs2_open_flags that are bitwise-ored together.
 //
 // The config struct provides additional config options per file as described
-// above. The config struct must be allocated while the file is open, and the
+// above. The config struct must remain allocated while the file is open, and
-// config struct must be zeroed for defaults and backwards compatibility.
+// the config struct must be zeroed for defaults and backwards compatibility.
 //
 // Returns a negative error code on failure.
 int lfs2_file_opencfg(lfs2_t *lfs2, lfs2_file_t *file,
@@ -659,6 +689,12 @@ int lfs2_dir_rewind(lfs2_t *lfs2, lfs2_dir_t *dir);
 
 /// Filesystem-level filesystem operations
 
+// Find on-disk info about the filesystem
+//
+// Fills out the fsinfo structure based on the filesystem found on-disk.
+// Returns a negative error code on failure.
+int lfs2_fs_stat(lfs2_t *lfs2, struct lfs2_fsinfo *fsinfo);
+
 // Finds the current size of the filesystem
 //
 // Note: Result is best effort. If files share COW structures, the returned
@@ -676,6 +712,40 @@ lfs2_ssize_t lfs2_fs_size(lfs2_t *lfs2);
 // Returns a negative error code on failure.
 int lfs2_fs_traverse(lfs2_t *lfs2, int (*cb)(void*, lfs2_block_t), void *data);
 
+// Attempt to proactively find free blocks
+//
+// Calling this function is not required, but may allowing the offloading of
+// the expensive block allocation scan to a less time-critical code path.
+//
+// Note: littlefs currently does not persist any found free blocks to disk.
+// This may change in the future.
+//
+// Returns a negative error code on failure. Finding no free blocks is
+// not an error.
+int lfs2_fs_gc(lfs2_t *lfs2);
+
+#ifndef LFS2_READONLY
+// Attempt to make the filesystem consistent and ready for writing
+//
+// Calling this function is not required, consistency will be implicitly
+// enforced on the first operation that writes to the filesystem, but this
+// function allows the work to be performed earlier and without other
+// filesystem changes.
+//
+// Returns a negative error code on failure.
+int lfs2_fs_mkconsistent(lfs2_t *lfs2);
+#endif
+
+#ifndef LFS2_READONLY
+// Grows the filesystem to a new size, updating the superblock with the new
+// block count.
+//
+// Note: This is irreversible.
+//
+// Returns a negative error code on failure.
+int lfs2_fs_grow(lfs2_t *lfs2, lfs2_size_t block_count);
+#endif
+
 #ifndef LFS2_READONLY
 #ifdef LFS2_MIGRATE
 // Attempts to migrate a previous version of littlefs

lib/littlefs/lfs2_util.h

@@ -167,10 +167,9 @@ static inline int lfs2_scmp(uint32_t a, uint32_t b) {
 
 // Convert between 32-bit little-endian and native order
 static inline uint32_t lfs2_fromle32(uint32_t a) {
-#if !defined(LFS2_NO_INTRINSICS) && ( \
+#if (defined(  BYTE_ORDER  ) && defined(  ORDER_LITTLE_ENDIAN  ) &&   BYTE_ORDER   ==   ORDER_LITTLE_ENDIAN  ) || \
-    (defined(  BYTE_ORDER  ) && defined(  ORDER_LITTLE_ENDIAN  ) &&   BYTE_ORDER   ==   ORDER_LITTLE_ENDIAN  ) || \
     (defined(__BYTE_ORDER  ) && defined(__ORDER_LITTLE_ENDIAN  ) && __BYTE_ORDER   == __ORDER_LITTLE_ENDIAN  ) || \
-    (defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__))
+    (defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
     return a;
 #elif !defined(LFS2_NO_INTRINSICS) && ( \
     (defined(  BYTE_ORDER  ) && defined(  ORDER_BIG_ENDIAN  ) &&   BYTE_ORDER   ==   ORDER_BIG_ENDIAN  ) || \
@@ -196,10 +195,9 @@ static inline uint32_t lfs2_frombe32(uint32_t a) {
     (defined(__BYTE_ORDER  ) && defined(__ORDER_LITTLE_ENDIAN  ) && __BYTE_ORDER   == __ORDER_LITTLE_ENDIAN  ) || \
     (defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__))
     return __builtin_bswap32(a);
-#elif !defined(LFS2_NO_INTRINSICS) && ( \
+#elif (defined(  BYTE_ORDER  ) && defined(  ORDER_BIG_ENDIAN  ) &&   BYTE_ORDER   ==   ORDER_BIG_ENDIAN  ) || \
-    (defined(  BYTE_ORDER  ) && defined(  ORDER_BIG_ENDIAN  ) &&   BYTE_ORDER   ==   ORDER_BIG_ENDIAN  ) || \
     (defined(__BYTE_ORDER  ) && defined(__ORDER_BIG_ENDIAN  ) && __BYTE_ORDER   == __ORDER_BIG_ENDIAN  ) || \
-    (defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__))
+    (defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
     return a;
 #else
     return (((uint8_t*)&a)[0] << 24) |

lib/mbedtls_errors/esp32_mbedtls_errors.c

@@ -45,10 +45,6 @@
 #include "mbedtls/aes.h"
 #endif
 
-#if defined(MBEDTLS_ARC4_C)
-#include "mbedtls/arc4.h"
-#endif
-
 #if defined(MBEDTLS_ARIA_C)
 #include "mbedtls/aria.h"
 #endif
@@ -65,10 +61,6 @@
 #include "mbedtls/bignum.h"
 #endif
 
-#if defined(MBEDTLS_BLOWFISH_C)
-#include "mbedtls/blowfish.h"
-#endif
-
 #if defined(MBEDTLS_CAMELLIA_C)
 #include "mbedtls/camellia.h"
 #endif
@@ -89,10 +81,6 @@
 #include "mbedtls/cipher.h"
 #endif
 
-#if defined(MBEDTLS_CMAC_C)
-#include "mbedtls/cmac.h"
-#endif
-
 #if defined(MBEDTLS_CTR_DRBG_C)
 #include "mbedtls/ctr_drbg.h"
 #endif
@@ -113,6 +101,14 @@
 #include "mbedtls/entropy.h"
 #endif
 
+#if defined(MBEDTLS_ERROR_C)
+#include "mbedtls/error.h"
+#endif
+
+#if defined(MBEDTLS_PLATFORM_C)
+#include "mbedtls/platform.h"
+#endif
+
 #if defined(MBEDTLS_GCM_C)
 #include "mbedtls/gcm.h"
 #endif
@@ -125,22 +121,14 @@
 #include "mbedtls/hmac_drbg.h"
 #endif
 
+#if defined(MBEDTLS_LMS_C)
+#include "mbedtls/lms.h"
+#endif
+
 #if defined(MBEDTLS_MD_C)
 #include "mbedtls/md.h"
 #endif
 
-#if defined(MBEDTLS_MD2_C)
-#include "mbedtls/md2.h"
-#endif
-
-#if defined(MBEDTLS_MD4_C)
-#include "mbedtls/md4.h"
-#endif
-
-#if defined(MBEDTLS_MD5_C)
-#include "mbedtls/md5.h"
-#endif
-
 #if defined(MBEDTLS_NET_C)
 #include "mbedtls/net_sockets.h"
 #endif
@@ -149,10 +137,6 @@
 #include "mbedtls/oid.h"
 #endif
 
-#if defined(MBEDTLS_PADLOCK_C)
-#include "mbedtls/padlock.h"
-#endif
-
 #if defined(MBEDTLS_PEM_PARSE_C) || defined(MBEDTLS_PEM_WRITE_C)
 #include "mbedtls/pem.h"
 #endif
@@ -169,18 +153,14 @@
 #include "mbedtls/pkcs5.h"
 #endif
 
-#if defined(MBEDTLS_PLATFORM_C)
+#if defined(MBEDTLS_PKCS7_C)
-#include "mbedtls/platform.h"
+#include "mbedtls/pkcs7.h"
 #endif
 
 #if defined(MBEDTLS_POLY1305_C)
 #include "mbedtls/poly1305.h"
 #endif
 
-#if defined(MBEDTLS_RIPEMD160_C)
-#include "mbedtls/ripemd160.h"
-#endif
-
 #if defined(MBEDTLS_RSA_C)
 #include "mbedtls/rsa.h"
 #endif
@@ -209,10 +189,6 @@
 #include "mbedtls/x509.h"
 #endif
 
-#if defined(MBEDTLS_XTEA_C)
-#include "mbedtls/xtea.h"
-#endif
-
 
 // Error code table type
 struct ssl_errs {
@@ -231,7 +207,6 @@ static const struct ssl_errs mbedtls_high_level_error_tab[] = {
         { -(MBEDTLS_ERR_CIPHER_FULL_BLOCK_EXPECTED), "CIPHER_FULL_BLOCK_EXPECTED" },
         { -(MBEDTLS_ERR_CIPHER_AUTH_FAILED), "CIPHER_AUTH_FAILED" },
         { -(MBEDTLS_ERR_CIPHER_INVALID_CONTEXT), "CIPHER_INVALID_CONTEXT" },
-        { -(MBEDTLS_ERR_CIPHER_HW_ACCEL_FAILED), "CIPHER_HW_ACCEL_FAILED" },
 #endif /* MBEDTLS_CIPHER_C */
 
 #if defined(MBEDTLS_DHM_C)
@@ -244,7 +219,6 @@ static const struct ssl_errs mbedtls_high_level_error_tab[] = {
         { -(MBEDTLS_ERR_DHM_INVALID_FORMAT), "DHM_INVALID_FORMAT" },
         { -(MBEDTLS_ERR_DHM_ALLOC_FAILED), "DHM_ALLOC_FAILED" },
         { -(MBEDTLS_ERR_DHM_FILE_IO_ERROR), "DHM_FILE_IO_ERROR" },
-        { -(MBEDTLS_ERR_DHM_HW_ACCEL_FAILED), "DHM_HW_ACCEL_FAILED" },
         { -(MBEDTLS_ERR_DHM_SET_GROUP_FAILED), "DHM_SET_GROUP_FAILED" },
 #endif /* MBEDTLS_DHM_C */
 
@@ -257,7 +231,6 @@ static const struct ssl_errs mbedtls_high_level_error_tab[] = {
         { -(MBEDTLS_ERR_ECP_RANDOM_FAILED), "ECP_RANDOM_FAILED" },
         { -(MBEDTLS_ERR_ECP_INVALID_KEY), "ECP_INVALID_KEY" },
         { -(MBEDTLS_ERR_ECP_SIG_LEN_MISMATCH), "ECP_SIG_LEN_MISMATCH" },
-        { -(MBEDTLS_ERR_ECP_HW_ACCEL_FAILED), "ECP_HW_ACCEL_FAILED" },
         { -(MBEDTLS_ERR_ECP_IN_PROGRESS), "ECP_IN_PROGRESS" },
 #endif /* MBEDTLS_ECP_C */
 
@@ -266,7 +239,6 @@ static const struct ssl_errs mbedtls_high_level_error_tab[] = {
         { -(MBEDTLS_ERR_MD_BAD_INPUT_DATA), "MD_BAD_INPUT_DATA" },
         { -(MBEDTLS_ERR_MD_ALLOC_FAILED), "MD_ALLOC_FAILED" },
         { -(MBEDTLS_ERR_MD_FILE_IO_ERROR), "MD_FILE_IO_ERROR" },
-        { -(MBEDTLS_ERR_MD_HW_ACCEL_FAILED), "MD_HW_ACCEL_FAILED" },
 #endif /* MBEDTLS_MD_C */
 
 #if defined(MBEDTLS_PEM_PARSE_C) || defined(MBEDTLS_PEM_WRITE_C)
@@ -296,7 +268,7 @@ static const struct ssl_errs mbedtls_high_level_error_tab[] = {
         { -(MBEDTLS_ERR_PK_UNKNOWN_NAMED_CURVE), "PK_UNKNOWN_NAMED_CURVE" },
         { -(MBEDTLS_ERR_PK_FEATURE_UNAVAILABLE), "PK_FEATURE_UNAVAILABLE" },
         { -(MBEDTLS_ERR_PK_SIG_LEN_MISMATCH), "PK_SIG_LEN_MISMATCH" },
-        { -(MBEDTLS_ERR_PK_HW_ACCEL_FAILED), "PK_HW_ACCEL_FAILED" },
+        { -(MBEDTLS_ERR_PK_BUFFER_TOO_SMALL), "PK_BUFFER_TOO_SMALL" },
 #endif /* MBEDTLS_PK_C */
 
 #if defined(MBEDTLS_PKCS12_C)
@@ -313,6 +285,21 @@ static const struct ssl_errs mbedtls_high_level_error_tab[] = {
         { -(MBEDTLS_ERR_PKCS5_PASSWORD_MISMATCH), "PKCS5_PASSWORD_MISMATCH" },
 #endif /* MBEDTLS_PKCS5_C */
 
+#if defined(MBEDTLS_PKCS7_C)
+        { -(MBEDTLS_ERR_PKCS7_INVALID_FORMAT), "PKCS7_INVALID_FORMAT" },
+        { -(MBEDTLS_ERR_PKCS7_FEATURE_UNAVAILABLE), "PKCS7_FEATURE_UNAVAILABLE" },
+        { -(MBEDTLS_ERR_PKCS7_INVALID_VERSION), "PKCS7_INVALID_VERSION" },
+        { -(MBEDTLS_ERR_PKCS7_INVALID_CONTENT_INFO), "PKCS7_INVALID_CONTENT_INFO" },
+        { -(MBEDTLS_ERR_PKCS7_INVALID_ALG), "PKCS7_INVALID_ALG" },
+        { -(MBEDTLS_ERR_PKCS7_INVALID_CERT), "PKCS7_INVALID_CERT" },
+        { -(MBEDTLS_ERR_PKCS7_INVALID_SIGNATURE), "PKCS7_INVALID_SIGNATURE" },
+        { -(MBEDTLS_ERR_PKCS7_INVALID_SIGNER_INFO), "PKCS7_INVALID_SIGNER_INFO" },
+        { -(MBEDTLS_ERR_PKCS7_BAD_INPUT_DATA), "PKCS7_BAD_INPUT_DATA" },
+        { -(MBEDTLS_ERR_PKCS7_ALLOC_FAILED), "PKCS7_ALLOC_FAILED" },
+        { -(MBEDTLS_ERR_PKCS7_VERIFY_FAIL), "PKCS7_VERIFY_FAIL" },
+        { -(MBEDTLS_ERR_PKCS7_CERT_DATE_INVALID), "PKCS7_CERT_DATE_INVALID" },
+#endif /* MBEDTLS_PKCS7_C */
+
 #if defined(MBEDTLS_RSA_C)
         { -(MBEDTLS_ERR_RSA_BAD_INPUT_DATA), "RSA_BAD_INPUT_DATA" },
         { -(MBEDTLS_ERR_RSA_INVALID_PADDING), "RSA_INVALID_PADDING" },
@@ -323,45 +310,34 @@ static const struct ssl_errs mbedtls_high_level_error_tab[] = {
         { -(MBEDTLS_ERR_RSA_VERIFY_FAILED), "RSA_VERIFY_FAILED" },
         { -(MBEDTLS_ERR_RSA_OUTPUT_TOO_LARGE), "RSA_OUTPUT_TOO_LARGE" },
         { -(MBEDTLS_ERR_RSA_RNG_FAILED), "RSA_RNG_FAILED" },
-        { -(MBEDTLS_ERR_RSA_UNSUPPORTED_OPERATION), "RSA_UNSUPPORTED_OPERATION" },
-        { -(MBEDTLS_ERR_RSA_HW_ACCEL_FAILED), "RSA_HW_ACCEL_FAILED" },
 #endif /* MBEDTLS_RSA_C */
 
 #if defined(MBEDTLS_SSL_TLS_C)
+        { -(MBEDTLS_ERR_SSL_CRYPTO_IN_PROGRESS), "SSL_CRYPTO_IN_PROGRESS" },
         { -(MBEDTLS_ERR_SSL_FEATURE_UNAVAILABLE), "SSL_FEATURE_UNAVAILABLE" },
         { -(MBEDTLS_ERR_SSL_BAD_INPUT_DATA), "SSL_BAD_INPUT_DATA" },
         { -(MBEDTLS_ERR_SSL_INVALID_MAC), "SSL_INVALID_MAC" },
         { -(MBEDTLS_ERR_SSL_INVALID_RECORD), "SSL_INVALID_RECORD" },
         { -(MBEDTLS_ERR_SSL_CONN_EOF), "SSL_CONN_EOF" },
-        { -(MBEDTLS_ERR_SSL_UNKNOWN_CIPHER), "SSL_UNKNOWN_CIPHER" },
+        { -(MBEDTLS_ERR_SSL_DECODE_ERROR), "SSL_DECODE_ERROR" },
-        { -(MBEDTLS_ERR_SSL_NO_CIPHER_CHOSEN), "SSL_NO_CIPHER_CHOSEN" },
         { -(MBEDTLS_ERR_SSL_NO_RNG), "SSL_NO_RNG" },
         { -(MBEDTLS_ERR_SSL_NO_CLIENT_CERTIFICATE), "SSL_NO_CLIENT_CERTIFICATE" },
-        { -(MBEDTLS_ERR_SSL_CERTIFICATE_TOO_LARGE), "SSL_CERTIFICATE_TOO_LARGE" },
+        { -(MBEDTLS_ERR_SSL_UNSUPPORTED_EXTENSION), "SSL_UNSUPPORTED_EXTENSION" },
-        { -(MBEDTLS_ERR_SSL_CERTIFICATE_REQUIRED), "SSL_CERTIFICATE_REQUIRED" },
+        { -(MBEDTLS_ERR_SSL_NO_APPLICATION_PROTOCOL), "SSL_NO_APPLICATION_PROTOCOL" },
         { -(MBEDTLS_ERR_SSL_PRIVATE_KEY_REQUIRED), "SSL_PRIVATE_KEY_REQUIRED" },
         { -(MBEDTLS_ERR_SSL_CA_CHAIN_REQUIRED), "SSL_CA_CHAIN_REQUIRED" },
         { -(MBEDTLS_ERR_SSL_UNEXPECTED_MESSAGE), "SSL_UNEXPECTED_MESSAGE" },
-        { -(MBEDTLS_ERR_SSL_PEER_VERIFY_FAILED), "SSL_PEER_VERIFY_FAILED" },
+        { -(MBEDTLS_ERR_SSL_UNRECOGNIZED_NAME), "SSL_UNRECOGNIZED_NAME" },
         { -(MBEDTLS_ERR_SSL_PEER_CLOSE_NOTIFY), "SSL_PEER_CLOSE_NOTIFY" },
-        { -(MBEDTLS_ERR_SSL_BAD_HS_CLIENT_HELLO), "SSL_BAD_HS_CLIENT_HELLO" },
+        { -(MBEDTLS_ERR_SSL_BAD_CERTIFICATE), "SSL_BAD_CERTIFICATE" },
-        { -(MBEDTLS_ERR_SSL_BAD_HS_SERVER_HELLO), "SSL_BAD_HS_SERVER_HELLO" },
+        { -(MBEDTLS_ERR_SSL_RECEIVED_NEW_SESSION_TICKET), "SSL_RECEIVED_NEW_SESSION_TICKET" },
-        { -(MBEDTLS_ERR_SSL_BAD_HS_CERTIFICATE), "SSL_BAD_HS_CERTIFICATE" },
+        { -(MBEDTLS_ERR_SSL_CANNOT_READ_EARLY_DATA), "SSL_CANNOT_READ_EARLY_DATA" },
-        { -(MBEDTLS_ERR_SSL_BAD_HS_CERTIFICATE_REQUEST), "SSL_BAD_HS_CERTIFICATE_REQUEST" },
+        { -(MBEDTLS_ERR_SSL_CANNOT_WRITE_EARLY_DATA), "SSL_CANNOT_WRITE_EARLY_DATA" },
-        { -(MBEDTLS_ERR_SSL_BAD_HS_SERVER_KEY_EXCHANGE), "SSL_BAD_HS_SERVER_KEY_EXCHANGE" },
-        { -(MBEDTLS_ERR_SSL_BAD_HS_SERVER_HELLO_DONE), "SSL_BAD_HS_SERVER_HELLO_DONE" },
-        { -(MBEDTLS_ERR_SSL_BAD_HS_CLIENT_KEY_EXCHANGE), "SSL_BAD_HS_CLIENT_KEY_EXCHANGE" },
-        { -(MBEDTLS_ERR_SSL_BAD_HS_CLIENT_KEY_EXCHANGE_RP), "SSL_BAD_HS_CLIENT_KEY_EXCHANGE_RP" },
-        { -(MBEDTLS_ERR_SSL_BAD_HS_CLIENT_KEY_EXCHANGE_CS), "SSL_BAD_HS_CLIENT_KEY_EXCHANGE_CS" },
-        { -(MBEDTLS_ERR_SSL_BAD_HS_CERTIFICATE_VERIFY), "SSL_BAD_HS_CERTIFICATE_VERIFY" },
-        { -(MBEDTLS_ERR_SSL_BAD_HS_CHANGE_CIPHER_SPEC), "SSL_BAD_HS_CHANGE_CIPHER_SPEC" },
-        { -(MBEDTLS_ERR_SSL_BAD_HS_FINISHED), "SSL_BAD_HS_FINISHED" },
         { -(MBEDTLS_ERR_SSL_ALLOC_FAILED), "SSL_ALLOC_FAILED" },
         { -(MBEDTLS_ERR_SSL_HW_ACCEL_FAILED), "SSL_HW_ACCEL_FAILED" },
         { -(MBEDTLS_ERR_SSL_HW_ACCEL_FALLTHROUGH), "SSL_HW_ACCEL_FALLTHROUGH" },
-        { -(MBEDTLS_ERR_SSL_COMPRESSION_FAILED), "SSL_COMPRESSION_FAILED" },
+        { -(MBEDTLS_ERR_SSL_BAD_PROTOCOL_VERSION), "SSL_BAD_PROTOCOL_VERSION" },
-        { -(MBEDTLS_ERR_SSL_BAD_HS_PROTOCOL_VERSION), "SSL_BAD_HS_PROTOCOL_VERSION" },
+        { -(MBEDTLS_ERR_SSL_HANDSHAKE_FAILURE), "SSL_HANDSHAKE_FAILURE" },
-        { -(MBEDTLS_ERR_SSL_BAD_HS_NEW_SESSION_TICKET), "SSL_BAD_HS_NEW_SESSION_TICKET" },
         { -(MBEDTLS_ERR_SSL_SESSION_TICKET_EXPIRED), "SSL_SESSION_TICKET_EXPIRED" },
         { -(MBEDTLS_ERR_SSL_PK_TYPE_MISMATCH), "SSL_PK_TYPE_MISMATCH" },
         { -(MBEDTLS_ERR_SSL_UNKNOWN_IDENTITY), "SSL_UNKNOWN_IDENTITY" },
@@ -370,18 +346,18 @@ static const struct ssl_errs mbedtls_high_level_error_tab[] = {
         { -(MBEDTLS_ERR_SSL_WAITING_SERVER_HELLO_RENEGO), "SSL_WAITING_SERVER_HELLO_RENEGO" },
         { -(MBEDTLS_ERR_SSL_HELLO_VERIFY_REQUIRED), "SSL_HELLO_VERIFY_REQUIRED" },
         { -(MBEDTLS_ERR_SSL_BUFFER_TOO_SMALL), "SSL_BUFFER_TOO_SMALL" },
-        { -(MBEDTLS_ERR_SSL_NO_USABLE_CIPHERSUITE), "SSL_NO_USABLE_CIPHERSUITE" },
         { -(MBEDTLS_ERR_SSL_WANT_READ), "SSL_WANT_READ" },
         { -(MBEDTLS_ERR_SSL_WANT_WRITE), "SSL_WANT_WRITE" },
         { -(MBEDTLS_ERR_SSL_TIMEOUT), "SSL_TIMEOUT" },
         { -(MBEDTLS_ERR_SSL_CLIENT_RECONNECT), "SSL_CLIENT_RECONNECT" },
         { -(MBEDTLS_ERR_SSL_UNEXPECTED_RECORD), "SSL_UNEXPECTED_RECORD" },
         { -(MBEDTLS_ERR_SSL_NON_FATAL), "SSL_NON_FATAL" },
-        { -(MBEDTLS_ERR_SSL_INVALID_VERIFY_HASH), "SSL_INVALID_VERIFY_HASH" },
+        { -(MBEDTLS_ERR_SSL_ILLEGAL_PARAMETER), "SSL_ILLEGAL_PARAMETER" },
         { -(MBEDTLS_ERR_SSL_CONTINUE_PROCESSING), "SSL_CONTINUE_PROCESSING" },
         { -(MBEDTLS_ERR_SSL_ASYNC_IN_PROGRESS), "SSL_ASYNC_IN_PROGRESS" },
         { -(MBEDTLS_ERR_SSL_EARLY_MESSAGE), "SSL_EARLY_MESSAGE" },
-        { -(MBEDTLS_ERR_SSL_CRYPTO_IN_PROGRESS), "SSL_CRYPTO_IN_PROGRESS" },
+        { -(MBEDTLS_ERR_SSL_UNEXPECTED_CID), "SSL_UNEXPECTED_CID" },
+        { -(MBEDTLS_ERR_SSL_VERSION_MISMATCH), "SSL_VERSION_MISMATCH" },
         { -(MBEDTLS_ERR_SSL_BAD_CONFIG), "SSL_BAD_CONFIG" },
 #endif /* MBEDTLS_SSL_TLS_C */
 
@@ -418,19 +394,11 @@ static const struct ssl_errs mbedtls_low_level_error_tab[] = {
     { -(MBEDTLS_ERR_AES_INVALID_KEY_LENGTH), "AES_INVALID_KEY_LENGTH" },
     { -(MBEDTLS_ERR_AES_INVALID_INPUT_LENGTH), "AES_INVALID_INPUT_LENGTH" },
     { -(MBEDTLS_ERR_AES_BAD_INPUT_DATA), "AES_BAD_INPUT_DATA" },
-    { -(MBEDTLS_ERR_AES_FEATURE_UNAVAILABLE), "AES_FEATURE_UNAVAILABLE" },
-    { -(MBEDTLS_ERR_AES_HW_ACCEL_FAILED), "AES_HW_ACCEL_FAILED" },
 #endif /* MBEDTLS_AES_C */
 
-#if defined(MBEDTLS_ARC4_C)
-    { -(MBEDTLS_ERR_ARC4_HW_ACCEL_FAILED), "ARC4_HW_ACCEL_FAILED" },
-#endif /* MBEDTLS_ARC4_C */
-
 #if defined(MBEDTLS_ARIA_C)
     { -(MBEDTLS_ERR_ARIA_BAD_INPUT_DATA), "ARIA_BAD_INPUT_DATA" },
     { -(MBEDTLS_ERR_ARIA_INVALID_INPUT_LENGTH), "ARIA_INVALID_INPUT_LENGTH" },
-    { -(MBEDTLS_ERR_ARIA_FEATURE_UNAVAILABLE), "ARIA_FEATURE_UNAVAILABLE" },
-    { -(MBEDTLS_ERR_ARIA_HW_ACCEL_FAILED), "ARIA_HW_ACCEL_FAILED" },
 #endif /* MBEDTLS_ARIA_C */
 
 #if defined(MBEDTLS_ASN1_PARSE_C)
@@ -459,28 +427,18 @@ static const struct ssl_errs mbedtls_low_level_error_tab[] = {
     { -(MBEDTLS_ERR_MPI_ALLOC_FAILED), "MPI_ALLOC_FAILED" },
 #endif /* MBEDTLS_BIGNUM_C */
 
-#if defined(MBEDTLS_BLOWFISH_C)
-    { -(MBEDTLS_ERR_BLOWFISH_BAD_INPUT_DATA), "BLOWFISH_BAD_INPUT_DATA" },
-    { -(MBEDTLS_ERR_BLOWFISH_INVALID_INPUT_LENGTH), "BLOWFISH_INVALID_INPUT_LENGTH" },
-    { -(MBEDTLS_ERR_BLOWFISH_HW_ACCEL_FAILED), "BLOWFISH_HW_ACCEL_FAILED" },
-#endif /* MBEDTLS_BLOWFISH_C */
-
 #if defined(MBEDTLS_CAMELLIA_C)
     { -(MBEDTLS_ERR_CAMELLIA_BAD_INPUT_DATA), "CAMELLIA_BAD_INPUT_DATA" },
     { -(MBEDTLS_ERR_CAMELLIA_INVALID_INPUT_LENGTH), "CAMELLIA_INVALID_INPUT_LENGTH" },
-    { -(MBEDTLS_ERR_CAMELLIA_HW_ACCEL_FAILED), "CAMELLIA_HW_ACCEL_FAILED" },
 #endif /* MBEDTLS_CAMELLIA_C */
 
 #if defined(MBEDTLS_CCM_C)
     { -(MBEDTLS_ERR_CCM_BAD_INPUT), "CCM_BAD_INPUT" },
     { -(MBEDTLS_ERR_CCM_AUTH_FAILED), "CCM_AUTH_FAILED" },
-    { -(MBEDTLS_ERR_CCM_HW_ACCEL_FAILED), "CCM_HW_ACCEL_FAILED" },
 #endif /* MBEDTLS_CCM_C */
 
 #if defined(MBEDTLS_CHACHA20_C)
     { -(MBEDTLS_ERR_CHACHA20_BAD_INPUT_DATA), "CHACHA20_BAD_INPUT_DATA" },
-    { -(MBEDTLS_ERR_CHACHA20_FEATURE_UNAVAILABLE), "CHACHA20_FEATURE_UNAVAILABLE" },
-    { -(MBEDTLS_ERR_CHACHA20_HW_ACCEL_FAILED), "CHACHA20_HW_ACCEL_FAILED" },
 #endif /* MBEDTLS_CHACHA20_C */
 
 #if defined(MBEDTLS_CHACHAPOLY_C)
@@ -488,10 +446,6 @@ static const struct ssl_errs mbedtls_low_level_error_tab[] = {
     { -(MBEDTLS_ERR_CHACHAPOLY_AUTH_FAILED), "CHACHAPOLY_AUTH_FAILED" },
 #endif /* MBEDTLS_CHACHAPOLY_C */
 
-#if defined(MBEDTLS_CMAC_C)
-    { -(MBEDTLS_ERR_CMAC_HW_ACCEL_FAILED), "CMAC_HW_ACCEL_FAILED" },
-#endif /* MBEDTLS_CMAC_C */
-
 #if defined(MBEDTLS_CTR_DRBG_C)
     { -(MBEDTLS_ERR_CTR_DRBG_ENTROPY_SOURCE_FAILED), "CTR_DRBG_ENTROPY_SOURCE_FAILED" },
     { -(MBEDTLS_ERR_CTR_DRBG_REQUEST_TOO_BIG), "CTR_DRBG_REQUEST_TOO_BIG" },
@@ -501,7 +455,6 @@ static const struct ssl_errs mbedtls_low_level_error_tab[] = {
 
 #if defined(MBEDTLS_DES_C)
     { -(MBEDTLS_ERR_DES_INVALID_INPUT_LENGTH), "DES_INVALID_INPUT_LENGTH" },
-    { -(MBEDTLS_ERR_DES_HW_ACCEL_FAILED), "DES_HW_ACCEL_FAILED" },
 #endif /* MBEDTLS_DES_C */
 
 #if defined(MBEDTLS_ENTROPY_C)
@@ -512,10 +465,20 @@ static const struct ssl_errs mbedtls_low_level_error_tab[] = {
     { -(MBEDTLS_ERR_ENTROPY_FILE_IO_ERROR), "ENTROPY_FILE_IO_ERROR" },
 #endif /* MBEDTLS_ENTROPY_C */
 
+#if defined(MBEDTLS_ERROR_C)
+    { -(MBEDTLS_ERR_ERROR_GENERIC_ERROR), "ERROR_GENERIC_ERROR" },
+    { -(MBEDTLS_ERR_ERROR_CORRUPTION_DETECTED), "ERROR_CORRUPTION_DETECTED" },
+#endif /* MBEDTLS_ERROR_C */
+
+#if defined(MBEDTLS_PLATFORM_C)
+    { -(MBEDTLS_ERR_PLATFORM_HW_ACCEL_FAILED), "PLATFORM_HW_ACCEL_FAILED" },
+    { -(MBEDTLS_ERR_PLATFORM_FEATURE_UNSUPPORTED), "PLATFORM_FEATURE_UNSUPPORTED" },
+#endif /* MBEDTLS_PLATFORM_C */
+
 #if defined(MBEDTLS_GCM_C)
     { -(MBEDTLS_ERR_GCM_AUTH_FAILED), "GCM_AUTH_FAILED" },
-    { -(MBEDTLS_ERR_GCM_HW_ACCEL_FAILED), "GCM_HW_ACCEL_FAILED" },
     { -(MBEDTLS_ERR_GCM_BAD_INPUT), "GCM_BAD_INPUT" },
+    { -(MBEDTLS_ERR_GCM_BUFFER_TOO_SMALL), "GCM_BUFFER_TOO_SMALL" },
 #endif /* MBEDTLS_GCM_C */
 
 #if defined(MBEDTLS_HKDF_C)
@@ -529,17 +492,13 @@ static const struct ssl_errs mbedtls_low_level_error_tab[] = {
     { -(MBEDTLS_ERR_HMAC_DRBG_ENTROPY_SOURCE_FAILED), "HMAC_DRBG_ENTROPY_SOURCE_FAILED" },
 #endif /* MBEDTLS_HMAC_DRBG_C */
 
-#if defined(MBEDTLS_MD2_C)
+#if defined(MBEDTLS_LMS_C)
-    { -(MBEDTLS_ERR_MD2_HW_ACCEL_FAILED), "MD2_HW_ACCEL_FAILED" },
+    { -(MBEDTLS_ERR_LMS_BAD_INPUT_DATA), "LMS_BAD_INPUT_DATA" },
-#endif /* MBEDTLS_MD2_C */
+    { -(MBEDTLS_ERR_LMS_OUT_OF_PRIVATE_KEYS), "LMS_OUT_OF_PRIVATE_KEYS" },
-
+    { -(MBEDTLS_ERR_LMS_VERIFY_FAILED), "LMS_VERIFY_FAILED" },
-#if defined(MBEDTLS_MD4_C)
+    { -(MBEDTLS_ERR_LMS_ALLOC_FAILED), "LMS_ALLOC_FAILED" },
-    { -(MBEDTLS_ERR_MD4_HW_ACCEL_FAILED), "MD4_HW_ACCEL_FAILED" },
+    { -(MBEDTLS_ERR_LMS_BUFFER_TOO_SMALL), "LMS_BUFFER_TOO_SMALL" },
-#endif /* MBEDTLS_MD4_C */
+#endif /* MBEDTLS_LMS_C */
-
-#if defined(MBEDTLS_MD5_C)
-    { -(MBEDTLS_ERR_MD5_HW_ACCEL_FAILED), "MD5_HW_ACCEL_FAILED" },
-#endif /* MBEDTLS_MD5_C */
 
 #if defined(MBEDTLS_NET_C)
     { -(MBEDTLS_ERR_NET_SOCKET_FAILED), "NET_SOCKET_FAILED" },
@@ -562,50 +521,26 @@ static const struct ssl_errs mbedtls_low_level_error_tab[] = {
     { -(MBEDTLS_ERR_OID_BUF_TOO_SMALL), "OID_BUF_TOO_SMALL" },
 #endif /* MBEDTLS_OID_C */
 
-#if defined(MBEDTLS_PADLOCK_C)
-    { -(MBEDTLS_ERR_PADLOCK_DATA_MISALIGNED), "PADLOCK_DATA_MISALIGNED" },
-#endif /* MBEDTLS_PADLOCK_C */
-
-#if defined(MBEDTLS_PLATFORM_C)
-    { -(MBEDTLS_ERR_PLATFORM_HW_ACCEL_FAILED), "PLATFORM_HW_ACCEL_FAILED" },
-    { -(MBEDTLS_ERR_PLATFORM_FEATURE_UNSUPPORTED), "PLATFORM_FEATURE_UNSUPPORTED" },
-#endif /* MBEDTLS_PLATFORM_C */
-
 #if defined(MBEDTLS_POLY1305_C)
     { -(MBEDTLS_ERR_POLY1305_BAD_INPUT_DATA), "POLY1305_BAD_INPUT_DATA" },
-    { -(MBEDTLS_ERR_POLY1305_FEATURE_UNAVAILABLE), "POLY1305_FEATURE_UNAVAILABLE" },
-    { -(MBEDTLS_ERR_POLY1305_HW_ACCEL_FAILED), "POLY1305_HW_ACCEL_FAILED" },
 #endif /* MBEDTLS_POLY1305_C */
 
-#if defined(MBEDTLS_RIPEMD160_C)
-    { -(MBEDTLS_ERR_RIPEMD160_HW_ACCEL_FAILED), "RIPEMD160_HW_ACCEL_FAILED" },
-#endif /* MBEDTLS_RIPEMD160_C */
-
 #if defined(MBEDTLS_SHA1_C)
-    { -(MBEDTLS_ERR_SHA1_HW_ACCEL_FAILED), "SHA1_HW_ACCEL_FAILED" },
     { -(MBEDTLS_ERR_SHA1_BAD_INPUT_DATA), "SHA1_BAD_INPUT_DATA" },
 #endif /* MBEDTLS_SHA1_C */
 
 #if defined(MBEDTLS_SHA256_C)
-    { -(MBEDTLS_ERR_SHA256_HW_ACCEL_FAILED), "SHA256_HW_ACCEL_FAILED" },
     { -(MBEDTLS_ERR_SHA256_BAD_INPUT_DATA), "SHA256_BAD_INPUT_DATA" },
 #endif /* MBEDTLS_SHA256_C */
 
 #if defined(MBEDTLS_SHA512_C)
-    { -(MBEDTLS_ERR_SHA512_HW_ACCEL_FAILED), "SHA512_HW_ACCEL_FAILED" },
     { -(MBEDTLS_ERR_SHA512_BAD_INPUT_DATA), "SHA512_BAD_INPUT_DATA" },
 #endif /* MBEDTLS_SHA512_C */
 
 #if defined(MBEDTLS_THREADING_C)
-    { -(MBEDTLS_ERR_THREADING_FEATURE_UNAVAILABLE), "THREADING_FEATURE_UNAVAILABLE" },
     { -(MBEDTLS_ERR_THREADING_BAD_INPUT_DATA), "THREADING_BAD_INPUT_DATA" },
     { -(MBEDTLS_ERR_THREADING_MUTEX_ERROR), "THREADING_MUTEX_ERROR" },
 #endif /* MBEDTLS_THREADING_C */
-
-#if defined(MBEDTLS_XTEA_C)
-    { -(MBEDTLS_ERR_XTEA_INVALID_INPUT_LENGTH), "XTEA_INVALID_INPUT_LENGTH" },
-    { -(MBEDTLS_ERR_XTEA_HW_ACCEL_FAILED), "XTEA_HW_ACCEL_FAILED" },
-#endif /* MBEDTLS_XTEA_C */
 // END generated code
 };
 
@@ -655,7 +590,7 @@ void mbedtls_strerror(int ret, char *buf, size_t buflen) {
     if (got_hl) {
         use_ret = ret & 0xFF80;
 
-        // special case
+        // special case, don't try to translate low level code
 #if defined(MBEDTLS_SSL_TLS_C)
         if (use_ret == -(MBEDTLS_ERR_SSL_FATAL_ALERT_MESSAGE)) {
             strncpy(buf, "MBEDTLS_ERR_SSL_FATAL_ALERT_MESSAGE", buflen);

lib/micropython-lib

@@ -1 +1 @@
-Subproject commit e025c843b60e93689f0f991d753010bb5bd6a722
+Subproject commit 7cdf70881519c73667efbc4a61a04d9c1a49babb

lib/uzlib/lz77.c

@@ -0,0 +1,91 @@
+/*
+ * Simple LZ77 streaming compressor.
+ *
+ * The scheme implemented here doesn't use a hash table and instead does a brute
+ * force search in the history for a previous string.  It is relatively slow
+ * (but still O(N)) but gives good compression and minimal memory usage.  For a
+ * small history window (eg 256 bytes) it's not too slow and compresses well.
+ *
+ * MIT license; Copyright (c) 2021 Damien P. George
+ */
+
+#include "uzlib.h"
+
+#include "defl_static.c"
+
+#define MATCH_LEN_MIN (3)
+#define MATCH_LEN_MAX (258)
+
+// hist should be a preallocated buffer of hist_max size bytes.
+// hist_max should be greater than 0 a power of 2 (ie 1, 2, 4, 8, ...).
+// It's possible to pass in hist=NULL, and then the history window will be taken from the
+// src passed in to uzlib_lz77_compress (this is useful when not doing streaming compression).
+void uzlib_lz77_init(uzlib_lz77_state_t *state, uint8_t *hist, size_t hist_max) {
+    memset(state, 0, sizeof(uzlib_lz77_state_t));
+    state->hist_buf = hist;
+    state->hist_max = hist_max;
+    state->hist_start = 0;
+    state->hist_len = 0;
+}
+
+// Search back in the history for the maximum match of the given src data,
+// with support for searching beyond the end of the history and into the src buffer
+// (effectively the history and src buffer are concatenated).
+static size_t uzlib_lz77_search_max_match(uzlib_lz77_state_t *state, const uint8_t *src, size_t len, size_t *longest_offset) {
+    size_t longest_len = 0;
+    for (size_t hist_search = 0; hist_search < state->hist_len; ++hist_search) {
+        // Search for a match.
+        size_t match_len;
+        for (match_len = 0; match_len <= MATCH_LEN_MAX && match_len < len; ++match_len) {
+            uint8_t hist;
+            if (hist_search + match_len < state->hist_len) {
+                hist = state->hist_buf[(state->hist_start + hist_search + match_len) & (state->hist_max - 1)];
+            } else {
+                hist = src[hist_search + match_len - state->hist_len];
+            }
+            if (src[match_len] != hist) {
+                break;
+            }
+        }
+
+        // Take this match if its length is at least the minimum, and larger than previous matches.
+        // If the length is the same as the previous longest then take this match as well, because
+        // this match will be closer (more recent in the history) and take less bits to encode.
+        if (match_len >= MATCH_LEN_MIN && match_len >= longest_len) {
+            longest_len = match_len;
+            *longest_offset = state->hist_len - hist_search;
+        }
+    }
+
+    return longest_len;
+}
+
+// Compress the given chunk of data.
+void uzlib_lz77_compress(uzlib_lz77_state_t *state, const uint8_t *src, unsigned len) {
+    const uint8_t *top = src + len;
+    while (src < top) {
+        // Look for a match in the history window.
+        size_t match_offset = 0;
+        size_t match_len = uzlib_lz77_search_max_match(state, src, top - src, &match_offset);
+
+        // Encode the literal byte or the match.
+        if (match_len == 0) {
+            uzlib_literal(state, *src);
+            match_len = 1;
+        } else {
+            uzlib_match(state, match_offset, match_len);
+        }
+
+        // Push the bytes into the history buffer.
+        size_t mask = state->hist_max - 1;
+        while (match_len--) {
+            uint8_t b = *src++;
+            state->hist_buf[(state->hist_start + state->hist_len) & mask] = b;
+            if (state->hist_len == state->hist_max) {
+                state->hist_start = (state->hist_start + 1) & mask;
+            } else {
+                ++state->hist_len;
+            }
+        }
+    }
+}

mpy-cross/Makefile

@@ -1,6 +1,7 @@
 include ../py/mkenv.mk
 
 # define main target
+// CIRCUITPY-CHANGE
 PROG ?= mpy-cross
 
 # qstr definitions (must come before including py.mk)
@@ -21,6 +22,7 @@ CWARN = -Wall -Werror
 CWARN += -Wextra -Wno-unused-parameter -Wpointer-arith
 CFLAGS += $(INC) $(CWARN) -std=gnu99 $(COPT) $(CFLAGS_EXTRA)
 CFLAGS += -fdata-sections -ffunction-sections -fno-asynchronous-unwind-tables
+// CIRCUITPY-CHANGE
 CFLAGS += -DCIRCUITPY
 
 # Debugging/Optimization
@@ -46,6 +48,7 @@ endif
 LDFLAGS += $(LDFLAGS_MOD) $(LDFLAGS_ARCH) -lm $(LDFLAGS_EXTRA)
 
 # source files
+// CIRCUITPY-CHANGE: extra files
 SRC_C = \
 	main.c \
 	gccollect.c \
@@ -57,7 +60,7 @@ SRC_C = \
 # Add fmode when compiling with mingw gcc
 COMPILER_TARGET := $(shell $(CC) -dumpmachine)
 ifneq (,$(findstring mingw,$(COMPILER_TARGET)))
-	SRC_C += windows-fmode.c
+	SRC_C += ports/windows/fmode.c
 endif
 
 OBJ = $(PY_CORE_O)

mpy-cross/README.md

@@ -1,9 +1,3 @@
-<!--
-SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors)
-
-SPDX-License-Identifier: MIT
--->
-
 MicroPython cross compiler
 ==========================
 

mpy-cross/main.c

@@ -37,7 +37,7 @@
 #include "py/stackctrl.h"
 #include "genhdr/mpversion.h"
 #ifdef _WIN32
-#include "fmode.h"
+#include "ports/windows/fmode.h"
 #endif
 
 // Command line options, with their defaults
@@ -71,7 +71,7 @@ STATIC int compile_and_save(const char *file, const char *output_file, const cha
         if (strcmp(file, "-") == 0) {
             lex = mp_lexer_new_from_fd(MP_QSTR__lt_stdin_gt_, STDIN_FILENO, false);
         } else {
-            lex = mp_lexer_new_from_file(file);
+            lex = mp_lexer_new_from_file(qstr_from_str(file));
         }
 
         qstr source_name;
@@ -104,7 +104,7 @@ STATIC int compile_and_save(const char *file, const char *output_file, const cha
                 vstr_add_str(&vstr, output_file);
             }
 
-            mp_raw_code_save_file(&cm, vstr_null_terminated_str(&vstr));
+            mp_raw_code_save_file(&cm, qstr_from_strn(vstr.buf, vstr.len));
             vstr_clear(&vstr);
         }
 
@@ -247,6 +247,7 @@ MP_NOINLINE int main_(int argc, char **argv) {
             if (strcmp(argv[a], "-X") == 0) {
                 a += 1;
             } else if (strcmp(argv[a], "--version") == 0) {
+                // CIRCUITPY-CHANGE
                 printf("CircuitPython " MICROPY_GIT_TAG " on " MICROPY_BUILD_DATE
                     "; mpy-cross emitting mpy v" MP_STRINGIFY(MPY_VERSION) "." MP_STRINGIFY(MPY_SUB_VERSION) "\n");
                 return 0;

mpy-cross/micropython.rc

@@ -0,0 +1 @@
+app     ICON    "../../logo/vector-logo-2.ico"

mpy-cross/mpconfigport.h

@@ -42,8 +42,6 @@
 #define MICROPY_EMIT_X86            (1)
 #define MICROPY_EMIT_THUMB          (1)
 #define MICROPY_EMIT_INLINE_THUMB   (1)
-#define MICROPY_EMIT_INLINE_THUMB_ARMV7M (1)
-#define MICROPY_EMIT_INLINE_THUMB_FLOAT (1)
 #define MICROPY_EMIT_ARM            (1)
 #define MICROPY_EMIT_XTENSA         (1)
 #define MICROPY_EMIT_INLINE_XTENSA  (1)
@@ -72,6 +70,7 @@
 
 #define MICROPY_FLOAT_IMPL          (MICROPY_FLOAT_IMPL_DOUBLE)
 #define MICROPY_CPYTHON_COMPAT      (1)
+// CIRCUITPY-CHANGE
 #define MICROPY_PY_ASYNC_AWAIT      (1)
 #define MICROPY_USE_INTERNAL_PRINTF (0)
 
@@ -93,50 +92,6 @@
 #define MICROPY_PY_IO               (0)
 #define MICROPY_PY_SYS              (0)
 
-// MINGW only handles these errno names.
-#ifdef __MINGW32__
-#define MICROPY_PY_ERRNO_LIST \
-    X(EPERM) \
-    X(ENOENT) \
-    X(ESRCH) \
-    X(EINTR) \
-    X(EIO) \
-    X(ENXIO) \
-    X(E2BIG) \
-    X(ENOEXEC) \
-    X(EBADF) \
-    X(ECHILD) \
-    X(EAGAIN) \
-    X(ENOMEM) \
-    X(EACCES) \
-    X(EFAULT) \
-    X(EBUSY) \
-    X(EEXIST) \
-    X(EXDEV) \
-    X(ENODEV) \
-    X(ENOTDIR) \
-    X(EISDIR) \
-    X(EINVAL) \
-    X(ENFILE) \
-    X(EMFILE) \
-    X(ENOTTY) \
-    X(EFBIG) \
-    X(ENOSPC) \
-    X(ESPIPE) \
-    X(EROFS) \
-    X(EMLINK) \
-    X(EPIPE) \
-    X(EDOM) \
-    X(ERANGE) \
-    X(EDEADLOCK) \
-    X(EDEADLK) \
-    X(ENAMETOOLONG) \
-    X(ENOLCK) \
-    X(ENOSYS) \
-    X(ENOTEMPTY) \
-    X(EILSEQ)
-#endif
-
 // type definitions for the specific machine
 
 #ifdef __LP64__

mpy-cross/mphalport.h

@@ -1,6 +1,2 @@
-// SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors)
-//
-// SPDX-License-Identifier: MIT
-
 // prevent including extmod/virtpin.h
 #define mp_hal_pin_obj_t

mpy-cross/qstrdefsport.h

@@ -1,5 +1 @@
-// SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors)
-//
-// SPDX-License-Identifier: MIT
-
 // qstrs specific to this port

ports/espressif/common-hal/_bleio/ble_events.c

@@ -64,7 +64,7 @@ void ble_event_add_handler(ble_gap_event_fn *func, void *param) {
     }
 
     // Add a new handler to the front of the list
-    ble_event_handler_entry_t *handler = m_new_ll(ble_event_handler_entry_t, 1);
+    ble_event_handler_entry_t *handler = m_new(ble_event_handler_entry_t, 1);
     ble_event_add_handler_entry(handler, func, param);
 }
 

ports/mimxrt10xx/linking/common.ld

@@ -96,7 +96,6 @@ SECTIONS
 
         /* Less critical portions of the runtime. */
         *runtime.o(.text.mp_import* .text.mp_resume* .text.mp_make_raise*  .text.mp_init)
-        *gc.o(.text.gc_never_free .text.gc_make_long_lived)
 
         /* Anything marked cold/unlikely should be in flash. */
         *(.text.unlikely.*)

ports/raspberrypi/common-hal/socketpool/Socket.c

@@ -818,7 +818,7 @@ socketpool_socket_obj_t *common_hal_socketpool_socket_accept(socketpool_socket_o
     uint8_t *ip, uint32_t *port) {
     // Create new socket object, do it here because we must not raise an out-of-memory
     // exception when the LWIP concurrency lock is held
-    socketpool_socket_obj_t *accepted = m_new_ll_obj_with_finaliser(socketpool_socket_obj_t);
+    socketpool_socket_obj_t *accepted = m_new_obj_with_finaliser(socketpool_socket_obj_t);
     socketpool_socket_reset(accepted);
 
     int ret = socketpool_socket_accept(socket, ip, port, accepted);

ports/unix/Makefile

@@ -193,6 +193,7 @@ ifeq ($(MICROPY_PY_JNI),1)
 CFLAGS += -I/usr/lib/jvm/java-7-openjdk-amd64/include -DMICROPY_PY_JNI=1
 endif
 
+# CIRCUITPY-CHANGE: CircuitPython-specific files.
 # source files
 SRC_C += \
 	main.c \
@@ -230,11 +231,9 @@ OBJ += $(addprefix $(BUILD)/, $(SHARED_SRC_C:.c=.o))
 
 # List of sources for qstr extraction
 SRC_QSTR += $(SRC_C) $(SRC_CXX) $(SHARED_SRC_C)
-# Append any auto-generated sources that are needed by sources listed in
-# SRC_QSTR
-SRC_QSTR_AUTO_DEPS +=
 
 ifneq ($(FROZEN_MANIFEST),)
+# CIRCUITPY-CHANGE
 # To use frozen code create a manifest.py file with a description of files to
 # freeze, then invoke make with FROZEN_MANIFEST=manifest.py (be sure to build from scratch).
 CFLAGS += -DMICROPY_QSTR_EXTRA_POOL=mp_qstr_frozen_const_pool

ports/unix/coverage.c

@@ -356,19 +356,20 @@ STATIC mp_obj_t extra_coverage(void) {
         mp_printf(&mp_plat_print, "# repl\n");
 
         const char *str;
-        size_t len = mp_repl_autocomplete("__n", 3, &mp_plat_print, &str);
+        size_t len = mp_repl_autocomplete("__n", 3, &mp_plat_print, &str); // expect "ame__"
         mp_printf(&mp_plat_print, "%.*s\n", (int)len, str);
 
-        len = mp_repl_autocomplete("im", 2,  &mp_plat_print, &str);
+        len = mp_repl_autocomplete("im", 2,  &mp_plat_print, &str); // expect "port"
         mp_printf(&mp_plat_print, "%.*s\n", (int)len, str);
-        mp_repl_autocomplete("import ", 7,  &mp_plat_print, &str);
+        mp_repl_autocomplete("import ", 7,  &mp_plat_print, &str); // expect the list of builtins
-        len = mp_repl_autocomplete("import ti", 9,  &mp_plat_print, &str);
+        len = mp_repl_autocomplete("import ti", 9,  &mp_plat_print, &str); // expect "me"
         mp_printf(&mp_plat_print, "%.*s\n", (int)len, str);
+        // CIRCUITPY-CHANGE
         mp_repl_autocomplete("import ra", 9,  &mp_plat_print, &str);
 
         mp_store_global(MP_QSTR_sys, mp_import_name(MP_QSTR_sys, mp_const_none, MP_OBJ_NEW_SMALL_INT(0)));
-        mp_repl_autocomplete("sys.", 4, &mp_plat_print, &str);
+        mp_repl_autocomplete("sys.", 4, &mp_plat_print, &str); // expect dir(sys)
-        len = mp_repl_autocomplete("sys.impl", 8, &mp_plat_print, &str);
+        len = mp_repl_autocomplete("sys.impl", 8, &mp_plat_print, &str); // expect "ementation"
         mp_printf(&mp_plat_print, "%.*s\n", (int)len, str);
     }
 
@@ -545,7 +546,7 @@ STATIC mp_obj_t extra_coverage(void) {
         fun_bc.context = &context;
         fun_bc.child_table = NULL;
         fun_bc.bytecode = (const byte *)"\x01"; // just needed for n_state
-        mp_code_state_t *code_state = m_new_obj_var(mp_code_state_t, mp_obj_t, 1);
+        mp_code_state_t *code_state = m_new_obj_var(mp_code_state_t, state, mp_obj_t, 1);
         code_state->fun_bc = &fun_bc;
         code_state->ip = (const byte *)"\x00"; // just needed for an invalid opcode
         code_state->sp = &code_state->state[0];
@@ -578,9 +579,10 @@ STATIC mp_obj_t extra_coverage(void) {
         mp_sched_unlock();
         mp_printf(&mp_plat_print, "unlocked\n");
 
-        // drain pending callbacks
+        // drain pending callbacks, and test mp_event_wait_indefinite(), mp_event_wait_ms()
+        mp_event_wait_indefinite(); // the unix port only waits 500us in this call
         while (mp_sched_num_pending()) {
-            mp_handle_pending(true);
+            mp_event_wait_ms(1);
         }
 
         // setting the keyboard interrupt and raising it during mp_handle_pending
@@ -610,6 +612,7 @@ STATIC mp_obj_t extra_coverage(void) {
         mp_handle_pending(true);
     }
 
+    // CIRCUITPY-CHANGE: ringbuf is different
     // ringbuf
     {
         #define RINGBUF_SIZE 99

ports/unix/main.c

@@ -54,7 +54,8 @@
 #include "genhdr/mpversion.h"
 #include "input.h"
 
-#if defined(MICROPY_UNIX_COVERAGE) // CIRCUITPY-CHANGE
+// CIRCUITPY-CHANGE
+#if defined(MICROPY_UNIX_COVERAGE)
 #include "py/objstr.h"
 typedef int os_getenv_err_t;
 mp_obj_t common_hal_os_getenv(const char *key, mp_obj_t default_);
@@ -109,6 +110,7 @@ STATIC void stderr_print_strn(void *env, const char *str, size_t len) {
     (void)env;
     ssize_t ret;
     MP_HAL_RETRY_SYSCALL(ret, write(STDERR_FILENO, str, len), {});
+    // CIRCUITPY-CHANGE: This should have been conditionalized.
     #if MICROPY_PY_OS_DUPTERM
     mp_os_dupterm_tx_strn(str, len);
     #endif
@@ -159,7 +161,8 @@ STATIC int execute_from_lexer(int source_kind, const void *source, mp_parse_inpu
             const vstr_t *vstr = source;
             lex = mp_lexer_new_from_str_len(MP_QSTR__lt_stdin_gt_, vstr->buf, vstr->len, false);
         } else if (source_kind == LEX_SRC_FILENAME) {
-            lex = mp_lexer_new_from_file((const char *)source);
+            const char *filename = (const char *)source;
+            lex = mp_lexer_new_from_file(qstr_from_str(filename));
         } else { // LEX_SRC_STDIN
             lex = mp_lexer_new_from_fd(MP_QSTR__lt_stdin_gt_, 0, false);
         }
@@ -326,10 +329,10 @@ STATIC int do_repl(void) {
         }
 
         int ret = execute_from_lexer(LEX_SRC_STR, line, MP_PARSE_SINGLE_INPUT, true);
+        free(line);
         if (ret & FORCED_EXIT) {
             return ret;
         }
-        free(line);
     }
 
     #endif

ports/unix/modmachine.c

@@ -3,7 +3,7 @@
  *
  * The MIT License (MIT)
  *
- * Copyright (c) 2013, 2014 Damien P. George
+ * Copyright (c) 2013-2023 Damien P. George
  * Copyright (c) 2015 Paul Sokolovsky
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
@@ -25,16 +25,8 @@
  * THE SOFTWARE.
  */
 
-#include <stdio.h>
+// This file is never compiled standalone, it's included directly from
-#include <stdint.h>
+// extmod/modmachine.c via MICROPY_PY_MACHINE_INCLUDEFILE.
-
-#include "py/runtime.h"
-#include "py/obj.h"
-
-#include "extmod/machine_mem.h"
-#include "extmod/machine_pinbase.h"
-#include "extmod/machine_signal.h"
-#include "extmod/machine_pulse.h"
 
 #if MICROPY_PLAT_DEV_MEM
 #include <errno.h>
@@ -44,7 +36,8 @@
 #define MICROPY_PAGE_MASK (MICROPY_PAGE_SIZE - 1)
 #endif
 
-#if MICROPY_PY_MACHINE
+// This variable is needed for machine.soft_reset(), but the variable is otherwise unused.
+int pyexec_system_exit = 0;
 
 uintptr_t mod_machine_mem_get_addr(mp_obj_t addr_o, uint align) {
     uintptr_t addr = mp_obj_get_int_truncated(addr_o);
@@ -77,39 +70,10 @@ uintptr_t mod_machine_mem_get_addr(mp_obj_t addr_o, uint align) {
     return addr;
 }
 
-#ifdef MICROPY_UNIX_MACHINE_IDLE
+STATIC void mp_machine_idle(void) {
-STATIC mp_obj_t machine_idle(void) {
-    MICROPY_UNIX_MACHINE_IDLE
-    return mp_const_none;
-}
-MP_DEFINE_CONST_FUN_OBJ_0(machine_idle_obj, machine_idle);
-#endif
-
-STATIC const mp_rom_map_elem_t machine_module_globals_table[] = {
-    { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_machine) },
-
-    { MP_ROM_QSTR(MP_QSTR_mem8), MP_ROM_PTR(&machine_mem8_obj) },
-    { MP_ROM_QSTR(MP_QSTR_mem16), MP_ROM_PTR(&machine_mem16_obj) },
-    { MP_ROM_QSTR(MP_QSTR_mem32), MP_ROM_PTR(&machine_mem32_obj) },
-
     #ifdef MICROPY_UNIX_MACHINE_IDLE
-    { MP_ROM_QSTR(MP_QSTR_idle), MP_ROM_PTR(&machine_idle_obj) },
+    MICROPY_UNIX_MACHINE_IDLE
+    #else
+    // Do nothing.
     #endif
-
+}
-    { MP_ROM_QSTR(MP_QSTR_PinBase), MP_ROM_PTR(&machine_pinbase_type) },
-    { MP_ROM_QSTR(MP_QSTR_Signal), MP_ROM_PTR(&machine_signal_type) },
-    #if MICROPY_PY_MACHINE_PULSE
-    { MP_ROM_QSTR(MP_QSTR_time_pulse_us), MP_ROM_PTR(&machine_time_pulse_us_obj) },
-    #endif
-};
-
-STATIC MP_DEFINE_CONST_DICT(machine_module_globals, machine_module_globals_table);
-
-const mp_obj_module_t mp_module_machine = {
-    .base = { &mp_type_module },
-    .globals = (mp_obj_dict_t *)&machine_module_globals,
-};
-
-MP_REGISTER_EXTENSIBLE_MODULE(MP_QSTR_machine, mp_module_machine);
-
-#endif // MICROPY_PY_MACHINE

ports/unix/modos.c

@@ -32,6 +32,7 @@
 #include "py/runtime.h"
 #include "py/mphal.h"
 
+// CIRCUITPY-CHANGE: enhanced getenv
 #if defined(MICROPY_UNIX_COVERAGE)
 #include "py/objstr.h"
 typedef int os_getenv_err_t;

ports/unix/mpconfigport.h

@@ -24,6 +24,7 @@
  * THE SOFTWARE.
  */
 
+// CIRCUITPY-CHANGE
 #pragma once
 
 // Options to control how MicroPython is built for this port, overriding
@@ -140,6 +141,9 @@ typedef long mp_off_t;
 #define MICROPY_STACKLESS_STRICT    (0)
 #endif
 
+// Implementation of the machine module.
+#define MICROPY_PY_MACHINE_INCLUDEFILE "ports/unix/modmachine.c"
+
 // Unix-specific configuration of machine.mem*.
 #define MICROPY_MACHINE_MEM_GET_READ_ADDR   mod_machine_mem_get_addr
 #define MICROPY_MACHINE_MEM_GET_WRITE_ADDR  mod_machine_mem_get_addr
@@ -148,8 +152,6 @@ typedef long mp_off_t;
 #define MICROPY_FATFS_RPATH            (2)
 #define MICROPY_FATFS_MAX_SS           (4096)
 #define MICROPY_FATFS_LFN_CODE_PAGE    437 /* 1=SFN/ANSI 437=LFN/U.S.(OEM) */
-#define MICROPY_FATFS_MKFS_FAT32       (1)
-#define MICROPY_FATFS_USE_LABEL (1)
 
 #define MICROPY_ALLOC_PATH_MAX      (PATH_MAX)
 
@@ -226,21 +228,11 @@ static inline unsigned long mp_random_seed_init(void) {
 #include <stdio.h>
 #endif
 
-// If threading is enabled, configure the atomic section.
-#if MICROPY_PY_THREAD
-#define MICROPY_BEGIN_ATOMIC_SECTION() (mp_thread_unix_begin_atomic_section(), 0xffffffff)
-#define MICROPY_END_ATOMIC_SECTION(x) (void)x; mp_thread_unix_end_atomic_section()
-#endif
-
 // In lieu of a WFI(), slow down polling from being a tight loop.
-#ifndef MICROPY_EVENT_POLL_HOOK
+//
-#define MICROPY_EVENT_POLL_HOOK \
+// Note that we don't delay for the full TIMEOUT_MS, as execution
-    do { \
+// can't be woken from the delay.
-        extern void mp_handle_pending(bool); \
+#define MICROPY_INTERNAL_WFE(TIMEOUT_MS) mp_hal_delay_us(500)
-        mp_handle_pending(true); \
-        usleep(500); /* equivalent to mp_hal_delay_us(500) */ \
-    } while (0);
-#endif
 
 // Configure the implementation of machine.idle().
 #include <sched.h>

ports/unix/mphalport.h

@@ -25,12 +25,19 @@
  */
 #include <errno.h>
 #include <unistd.h>
+// CIRCUITPY-CHANGE: extra include
 #include <stdbool.h>
 
 #ifndef CHAR_CTRL_C
 #define CHAR_CTRL_C (3)
 #endif
 
+// If threading is enabled, configure the atomic section.
+#if MICROPY_PY_THREAD
+#define MICROPY_BEGIN_ATOMIC_SECTION() (mp_thread_unix_begin_atomic_section(), 0xffffffff)
+#define MICROPY_END_ATOMIC_SECTION(x) (void)x; mp_thread_unix_end_atomic_section()
+#endif
+
 // CIRCUITPY-CHANGE: mp_hal_set_interrupt_char(int) instead of char
 void mp_hal_set_interrupt_char(int c);
 bool mp_hal_is_interrupted(void);

ports/unix/unix_mphal.c

@@ -190,13 +190,17 @@ main_term:;
     return c;
 }
 
-void mp_hal_stdout_tx_strn(const char *str, size_t len) {
+mp_uint_t mp_hal_stdout_tx_strn(const char *str, size_t len) {
     ssize_t ret;
     MP_HAL_RETRY_SYSCALL(ret, write(STDOUT_FILENO, str, len), {});
+    mp_uint_t written = ret < 0 ? 0 : ret;
     // CIRCUITPY-CHANGE: need to conditionalize MICROPY_PY_OS_DUPTERM
-    #if MICROPY_PY_OS_DUPTERM
+    #if MICROPY_PY_OS_DUPTERM    int dupterm_res = mp_os_dupterm_tx_strn(str, len);
-    mp_os_dupterm_tx_strn(str, len);
+    if (dupterm_res >= 0) {
+        written = MIN((mp_uint_t)dupterm_res, written);
+    }
     #endif
+    return written;
 }
 
 // cooked is same as uncooked because the terminal does some postprocessing
@@ -246,17 +250,10 @@ uint64_t mp_hal_time_ns(void) {
 
 #ifndef mp_hal_delay_ms
 void mp_hal_delay_ms(mp_uint_t ms) {
-    #ifdef MICROPY_EVENT_POLL_HOOK
     mp_uint_t start = mp_hal_ticks_ms();
     while (mp_hal_ticks_ms() - start < ms) {
-        // MICROPY_EVENT_POLL_HOOK does usleep(500).
+        mp_event_wait_ms(1);
-        MICROPY_EVENT_POLL_HOOK
     }
-    #else
-    // TODO: POSIX et al. define usleep() as guaranteedly capable only of 1s sleep:
-    // "The useconds argument shall be less than one million."
-    usleep(ms * 1000);
-    #endif
 }
 #endif
 

ports/unix/variants/coverage/mpconfigvariant.mk

@@ -1,6 +1,7 @@
 # Disable optimisations and enable assert() on coverage builds.
 DEBUG ?= 1
 
+# CIRCUITPY-CHANGE: add exception chaining
 CFLAGS += \
 	-fprofile-arcs -ftest-coverage \
 	-Wformat -Wmissing-declarations -Wmissing-prototypes \
@@ -13,6 +14,7 @@ LDFLAGS += -fprofile-arcs -ftest-coverage
 FROZEN_MANIFEST ?= $(VARIANT_DIR)/manifest.py
 USER_C_MODULES = $(TOP)/examples/usercmodule
 
+# CIRCUITPY-CHANGE: use CircuitPython bindings and implementations
 SRC_QRIO := $(patsubst ../../%,%,$(wildcard ../../shared-bindings/qrio/*.c ../../shared-module/qrio/*.c ../../lib/quirc/lib/*.c))
 SRC_C += $(SRC_QRIO)
 

ports/unix/variants/minimal/mpconfigvariant.h

@@ -47,6 +47,7 @@
 #define MICROPY_COMP_CONST_LITERAL (1)
 #define MICROPY_COMP_CONST_TUPLE (1)
 #define MICROPY_COMP_DOUBLE_TUPLE_ASSIGN (1)
+#define MICROPY_ENABLE_COMPILER (1)
 #define MICROPY_ENABLE_EXTERNAL_IMPORT (1)
 #define MICROPY_FULL_CHECKS (1)
 #define MICROPY_HELPER_REPL (1)

ports/unix/variants/mpconfigvariant_common.h

@@ -116,3 +116,4 @@
 // Enable the "machine" module, mostly for machine.mem*.
 #define MICROPY_PY_MACHINE             (1)
 #define MICROPY_PY_MACHINE_PULSE       (1)
+#define MICROPY_PY_MACHINE_PIN_BASE    (1)

ports/unix/variants/standard/manifest.py

@@ -1 +1,3 @@
 include("$(PORT_DIR)/variants/manifest.py")
+
+# CIRCUITPY-CHANGE: Do not include extmod/aysncio

py/argcheck.c

@@ -29,6 +29,7 @@
 
 #include "py/runtime.h"
 
+// CIRCUITPY-CHANGE: PLACE_IN_ITCM
 void PLACE_IN_ITCM(mp_arg_check_num_sig)(size_t n_args, size_t n_kw, uint32_t sig) {
     // TODO maybe take the function name as an argument so we can print nicer error messages
 
@@ -50,6 +51,7 @@ void PLACE_IN_ITCM(mp_arg_check_num_sig)(size_t n_args, size_t n_kw, uint32_t si
             #if MICROPY_ERROR_REPORTING <= MICROPY_ERROR_REPORTING_TERSE
             mp_arg_error_terse_mismatch();
             #else
+            // CIRCUITPY-CHANGE: specific mp_raise routine
             mp_raise_TypeError_varg(MP_ERROR_TEXT("function takes %d positional arguments but %d were given"),
                 n_args_min, n_args);
             #endif
@@ -59,6 +61,7 @@ void PLACE_IN_ITCM(mp_arg_check_num_sig)(size_t n_args, size_t n_kw, uint32_t si
             #if MICROPY_ERROR_REPORTING <= MICROPY_ERROR_REPORTING_TERSE
             mp_arg_error_terse_mismatch();
             #else
+            // CIRCUITPY-CHANGE: specific mp_raise routine
             mp_raise_TypeError_varg(
                 MP_ERROR_TEXT("function missing %d required positional arguments"),
                 n_args_min - n_args);
@@ -67,6 +70,7 @@ void PLACE_IN_ITCM(mp_arg_check_num_sig)(size_t n_args, size_t n_kw, uint32_t si
             #if MICROPY_ERROR_REPORTING <= MICROPY_ERROR_REPORTING_TERSE
             mp_arg_error_terse_mismatch();
             #else
+            // CIRCUITPY-CHANGE: specific mp_raise routine
             mp_raise_TypeError_varg(
                 MP_ERROR_TEXT("function expected at most %d arguments, got %d"),
                 n_args_max, n_args);
@@ -75,6 +79,7 @@ void PLACE_IN_ITCM(mp_arg_check_num_sig)(size_t n_args, size_t n_kw, uint32_t si
     }
 }
 
+// CIRCUITPY-CHANGE: better keyword arg checking in next two routines
 inline void mp_arg_check_num_kw_array(size_t n_args, size_t n_kw, size_t n_args_min, size_t n_args_max, bool takes_kw) {
     mp_arg_check_num_sig(n_args, n_kw, MP_OBJ_FUN_MAKE_SIG(n_args_min, n_args_max, takes_kw));
 }
@@ -176,7 +181,7 @@ NORETURN void mp_arg_error_unimpl_kw(void) {
 }
 #endif
 
-
+// CIRCUITPY-CHANGE: more specific mp_raise routines
 mp_int_t mp_arg_validate_int(mp_int_t i, mp_int_t required_i, qstr arg_name) {
     if (i != required_i) {
         mp_raise_ValueError_varg(MP_ERROR_TEXT("%q must be %d"), arg_name, required_i);

py/asmarm.h

@@ -132,6 +132,7 @@ void asm_arm_bx_reg(asm_arm_t *as, uint reg_src);
 // Holds a pointer to mp_fun_table
 #define ASM_ARM_REG_FUN_TABLE ASM_ARM_REG_R7
 
+// CIRCUITPY-CHANGE: prevent #if warning
 #if defined(GENERIC_ASM_API) && GENERIC_ASM_API
 
 // The following macros provide a (mostly) arch-independent API to
@@ -183,8 +184,6 @@ void asm_arm_bx_reg(asm_arm_t *as, uint reg_src);
 
 #define ASM_MOV_LOCAL_REG(as, local_num, reg_src) asm_arm_mov_local_reg((as), (local_num), (reg_src))
 #define ASM_MOV_REG_IMM(as, reg_dest, imm) asm_arm_mov_reg_i32_optimised((as), (reg_dest), (imm))
-#define ASM_MOV_REG_IMM_FIX_U16(as, reg_dest, imm) asm_arm_mov_reg_i32((as), (reg_dest), (imm))
-#define ASM_MOV_REG_IMM_FIX_WORD(as, reg_dest, imm) asm_arm_mov_reg_i32((as), (reg_dest), (imm))
 #define ASM_MOV_REG_LOCAL(as, reg_dest, local_num) asm_arm_mov_reg_local((as), (reg_dest), (local_num))
 #define ASM_MOV_REG_REG(as, reg_dest, reg_src) asm_arm_mov_reg_reg((as), (reg_dest), (reg_src))
 #define ASM_MOV_REG_LOCAL_ADDR(as, reg_dest, local_num) asm_arm_mov_reg_local_addr((as), (reg_dest), (local_num))

py/asmthumb.h

@@ -350,6 +350,7 @@ void asm_thumb_b_rel12(asm_thumb_t *as, int rel);
 // Holds a pointer to mp_fun_table
 #define ASM_THUMB_REG_FUN_TABLE ASM_THUMB_REG_R7
 
+// CIRCUITPY-CHANGE: prevent #if warning
 #if defined(GENERIC_ASM_API) && GENERIC_ASM_API
 
 // The following macros provide a (mostly) arch-independent API to
@@ -401,7 +402,6 @@ void asm_thumb_b_rel12(asm_thumb_t *as, int rel);
 
 #define ASM_MOV_LOCAL_REG(as, local_num, reg) asm_thumb_mov_local_reg((as), (local_num), (reg))
 #define ASM_MOV_REG_IMM(as, reg_dest, imm) asm_thumb_mov_reg_i32_optimised((as), (reg_dest), (imm))
-#define ASM_MOV_REG_IMM_FIX_WORD(as, reg_dest, imm) asm_thumb_mov_reg_i32((as), (reg_dest), (imm))
 #define ASM_MOV_REG_LOCAL(as, reg_dest, local_num) asm_thumb_mov_reg_local((as), (reg_dest), (local_num))
 #define ASM_MOV_REG_REG(as, reg_dest, reg_src) asm_thumb_mov_reg_reg((as), (reg_dest), (reg_src))
 #define ASM_MOV_REG_LOCAL_ADDR(as, reg_dest, local_num) asm_thumb_mov_reg_local_addr((as), (reg_dest), (local_num))

py/asmx64.h

@@ -124,7 +124,8 @@ void asm_x64_call_ind(asm_x64_t *as, size_t fun_id, int temp_r32);
 // Holds a pointer to mp_fun_table
 #define ASM_X64_REG_FUN_TABLE ASM_X64_REG_RBP
 
-#if defined(GENERIC_ASM_API) && GENERIC_ASM_API
+#// CIRCUITPY-CHANGE: prevent #if warning
+if defined(GENERIC_ASM_API) && GENERIC_ASM_API
 
 // The following macros provide a (mostly) arch-independent API to
 // generate native code, and are used by the native emitter.
@@ -186,8 +187,6 @@ void asm_x64_call_ind(asm_x64_t *as, size_t fun_id, int temp_r32);
 
 #define ASM_MOV_LOCAL_REG(as, local_num, reg_src) asm_x64_mov_r64_to_local((as), (reg_src), (local_num))
 #define ASM_MOV_REG_IMM(as, reg_dest, imm) asm_x64_mov_i64_to_r64_optimised((as), (imm), (reg_dest))
-#define ASM_MOV_REG_IMM_FIX_U16(as, reg_dest, imm) asm_x64_mov_i32_to_r64((as), (imm), (reg_dest))
-#define ASM_MOV_REG_IMM_FIX_WORD(as, reg_dest, imm) asm_x64_mov_i32_to_r64((as), (imm), (reg_dest))
 #define ASM_MOV_REG_LOCAL(as, reg_dest, local_num) asm_x64_mov_local_to_r64((as), (local_num), (reg_dest))
 #define ASM_MOV_REG_REG(as, reg_dest, reg_src) asm_x64_mov_r64_r64((as), (reg_dest), (reg_src))
 #define ASM_MOV_REG_LOCAL_ADDR(as, reg_dest, local_num) asm_x64_mov_local_addr_to_r64((as), (local_num), (reg_dest))

py/asmx86.h

@@ -120,6 +120,7 @@ void asm_x86_call_ind(asm_x86_t *as, size_t fun_id, mp_uint_t n_args, int temp_r
 // Holds a pointer to mp_fun_table
 #define ASM_X86_REG_FUN_TABLE ASM_X86_REG_EBP
 
+// CIRCUITPY-CHANGE: prevent #if warning
 #if defined(GENERIC_ASM_API) && GENERIC_ASM_API
 
 // The following macros provide a (mostly) arch-independent API to
@@ -181,8 +182,6 @@ void asm_x86_call_ind(asm_x86_t *as, size_t fun_id, mp_uint_t n_args, int temp_r
 
 #define ASM_MOV_LOCAL_REG(as, local_num, reg_src) asm_x86_mov_r32_to_local((as), (reg_src), (local_num))
 #define ASM_MOV_REG_IMM(as, reg_dest, imm) asm_x86_mov_i32_to_r32((as), (imm), (reg_dest))
-#define ASM_MOV_REG_IMM_FIX_U16(as, reg_dest, imm) asm_x86_mov_i32_to_r32((as), (imm), (reg_dest))
-#define ASM_MOV_REG_IMM_FIX_WORD(as, reg_dest, imm) asm_x86_mov_i32_to_r32((as), (imm), (reg_dest))
 #define ASM_MOV_REG_LOCAL(as, reg_dest, local_num) asm_x86_mov_local_to_r32((as), (local_num), (reg_dest))
 #define ASM_MOV_REG_REG(as, reg_dest, reg_src) asm_x86_mov_r32_r32((as), (reg_dest), (reg_src))
 #define ASM_MOV_REG_LOCAL_ADDR(as, reg_dest, local_num) asm_x86_mov_local_addr_to_r32((as), (local_num), (reg_dest))

py/asmxtensa.h

@@ -287,6 +287,7 @@ void asm_xtensa_call_ind_win(asm_xtensa_t *as, uint idx);
 #define ASM_XTENSA_REG_FUN_TABLE ASM_XTENSA_REG_A15
 #define ASM_XTENSA_REG_FUN_TABLE_WIN ASM_XTENSA_REG_A7
 
+// CIRCUITPY-CHANGE: prevent #if warning
 #if defined(GENERIC_ASM_API) && GENERIC_ASM_API
 
 // The following macros provide a (mostly) arch-independent API to
@@ -366,8 +367,6 @@ void asm_xtensa_call_ind_win(asm_xtensa_t *as, uint idx);
 
 #define ASM_MOV_LOCAL_REG(as, local_num, reg_src) asm_xtensa_mov_local_reg((as), ASM_NUM_REGS_SAVED + (local_num), (reg_src))
 #define ASM_MOV_REG_IMM(as, reg_dest, imm) asm_xtensa_mov_reg_i32_optimised((as), (reg_dest), (imm))
-#define ASM_MOV_REG_IMM_FIX_U16(as, reg_dest, imm) asm_xtensa_mov_reg_i32((as), (reg_dest), (imm))
-#define ASM_MOV_REG_IMM_FIX_WORD(as, reg_dest, imm) asm_xtensa_mov_reg_i32((as), (reg_dest), (imm))
 #define ASM_MOV_REG_LOCAL(as, reg_dest, local_num) asm_xtensa_mov_reg_local((as), (reg_dest), ASM_NUM_REGS_SAVED + (local_num))
 #define ASM_MOV_REG_REG(as, reg_dest, reg_src) asm_xtensa_op_mov_n((as), (reg_dest), (reg_src))
 #define ASM_MOV_REG_LOCAL_ADDR(as, reg_dest, local_num) asm_xtensa_mov_reg_local_addr((as), (reg_dest), ASM_NUM_REGS_SAVED + (local_num))

py/bc.c

@@ -97,9 +97,11 @@ STATIC NORETURN void fun_pos_args_mismatch(mp_obj_fun_bc_t *f, size_t expected,
     mp_arg_error_terse_mismatch();
     #elif MICROPY_ERROR_REPORTING == MICROPY_ERROR_REPORTING_NORMAL
     (void)f;
+    // CIRCUITPY-CHANGE: more specifc mp_raise routine
     mp_raise_TypeError_varg(
         MP_ERROR_TEXT("function takes %d positional arguments but %d were given"), expected, given);
     #elif MICROPY_ERROR_REPORTING == MICROPY_ERROR_REPORTING_DETAILED
+    // CIRCUITPY-CHANGE: more specifc mp_raise routine
     mp_raise_TypeError_varg(
         MP_ERROR_TEXT("%q() takes %d positional arguments but %d were given"),
         mp_obj_fun_get_name(MP_OBJ_FROM_PTR(f)), expected, given);
@@ -281,6 +283,7 @@ STATIC void mp_setup_code_state_helper(mp_code_state_t *code_state, size_t n_arg
                 if (elem != NULL) {
                     code_state_state[n_state - 1 - n_pos_args - i] = elem->value;
                 } else {
+                    // CIRCUITPY-CHANGE: more specifc mp_raise routine
                     mp_raise_TypeError_varg(
                         MP_ERROR_TEXT("function missing required keyword argument '%q'"),
                         MP_OBJ_QSTR_VALUE(arg_names[n_pos_args + i]));

py/bc.h

@@ -101,6 +101,7 @@
         out_byte(out_env, z);                                       \
     } while (0)
 
+// CIRCUITPY-CHANGE: avoid warnings
 #define MP_BC_PRELUDE_SIG_DECODE_INTO(ip, S, E, F, A, K, D)     \
     do {                                                            \
         uint8_t z = *(ip)++;                                        \

py/binary.c

@@ -51,6 +51,7 @@ size_t mp_binary_get_size(char struct_type, char val_type, size_t *palign) {
             switch (val_type) {
                 case 'b':
                 case 'B':
+                // CIRCUITPY-CHANGE: x code: padding                                  
                 case 'x':
                     size = 1;
                     break;
@@ -70,6 +71,7 @@ size_t mp_binary_get_size(char struct_type, char val_type, size_t *palign) {
                 case 'Q':
                     size = 8;
                     break;
+                // CIRCUITPY-CHANGE: non-standard typecodes can be turned off                                  
                 #if MICROPY_NONSTANDARD_TYPECODES
                 case 'P':
                 case 'O':
@@ -97,6 +99,7 @@ size_t mp_binary_get_size(char struct_type, char val_type, size_t *palign) {
                 case BYTEARRAY_TYPECODE:
                 case 'b':
                 case 'B':
+                // CIRCUITPY-CHANGE: x code: padding                                  
                 case 'x':
                     align = size = 1;
                     break;
@@ -120,6 +123,7 @@ size_t mp_binary_get_size(char struct_type, char val_type, size_t *palign) {
                     align = alignof(long long);
                     size = sizeof(long long);
                     break;
+                // CIRCUITPY-CHANGE: non-standard typecodes can be turned off                                  
                 #if MICROPY_NONSTANDARD_TYPECODES
                 case 'P':
                 case 'O':
@@ -186,6 +190,7 @@ mp_obj_t mp_binary_get_val_array(char typecode, void *p, size_t index) {
         case 'd':
             return mp_obj_new_float_from_d(((double *)p)[index]);
         #endif
+            // CIRCUITPY-CHANGE: non-standard typecodes can be turned off                                  
             #if MICROPY_NONSTANDARD_TYPECODES
         // Extension to CPython: array of objects
         case 'O':
@@ -243,6 +248,7 @@ mp_obj_t mp_binary_get_val(char struct_type, char val_type, byte *p_base, byte *
 
     long long val = mp_binary_get_int(size, is_signed(val_type), (struct_type == '>'), p);
 
+    // CIRCUITPY-CHANGE: non-standard typecodes can be turned off                                  
     if (MICROPY_NONSTANDARD_TYPECODES && (val_type == 'O')) {
         return (mp_obj_t)(mp_uint_t)val;
     #if MICROPY_NONSTANDARD_TYPECODES
@@ -316,6 +322,7 @@ void mp_binary_set_val(char struct_type, char val_type, mp_obj_t val_in, byte *p
 
     mp_uint_t val;
     switch (val_type) {
+        // CIRCUITPY-CHANGE: non-standard typecodes can be turned off                                  
         #if MICROPY_NONSTANDARD_TYPECODES
         case 'O':
             val = (mp_uint_t)val_in;
@@ -350,6 +357,7 @@ void mp_binary_set_val(char struct_type, char val_type, mp_obj_t val_in, byte *p
         }
         #endif
         default: {
+            // CIRCUITPY-CHANGE: add overflow checks
             bool signed_type = is_signed(val_type);
             #if MICROPY_LONGINT_IMPL != MICROPY_LONGINT_IMPL_NONE
             if (mp_obj_is_exact_type(val_in, &mp_type_int)) {
@@ -389,6 +397,7 @@ void mp_binary_set_val_array(char typecode, void *p, size_t index, mp_obj_t val_
             ((double *)p)[index] = mp_obj_get_float_to_d(val_in);
             break;
         #endif
+        // CIRCUITPY-CHANGE: non-standard typecodes can be turned off                                  
         #if MICROPY_NONSTANDARD_TYPECODES
         // Extension to CPython: array of objects
         case 'O':
@@ -396,6 +405,7 @@ void mp_binary_set_val_array(char typecode, void *p, size_t index, mp_obj_t val_
             break;
         #endif
         default: {
+            // CIRCUITPY-CHANGE: add overflow checks
             size_t size = mp_binary_get_size('@', typecode, NULL);
             bool signed_type = is_signed(typecode);
 
@@ -459,6 +469,7 @@ void mp_binary_set_val_array_from_int(char typecode, void *p, size_t index, mp_i
             ((double *)p)[index] = (double)val;
             break;
         #endif
+            // CIRCUITPY-CHANGE: non-standard typecodes can be turned off                                  
             #if MICROPY_NONSTANDARD_TYPECODES
         // Extension to CPython: array of pointers
         case 'P':

py/builtinevex.c

@@ -136,17 +136,18 @@ STATIC mp_obj_t eval_exec_helper(size_t n_args, const mp_obj_t *args, mp_parse_i
     }
     #endif
 
-    // Extract the source code.
-    mp_buffer_info_t bufinfo;
-    mp_get_buffer_raise(args[0], &bufinfo, MP_BUFFER_READ);
 
     // create the lexer
     // MP_PARSE_SINGLE_INPUT is used to indicate a file input
     mp_lexer_t *lex;
     if (MICROPY_PY_BUILTINS_EXECFILE && parse_input_kind == MP_PARSE_SINGLE_INPUT) {
-        lex = mp_lexer_new_from_file(bufinfo.buf);
+        lex = mp_lexer_new_from_file(mp_obj_str_get_qstr(args[0]));
         parse_input_kind = MP_PARSE_FILE_INPUT;
     } else {
+        // Extract the source code.
+        mp_buffer_info_t bufinfo;
+        mp_get_buffer_raise(args[0], &bufinfo, MP_BUFFER_READ);
+
         lex = mp_lexer_new_from_str_len(MP_QSTR__lt_string_gt_, bufinfo.buf, bufinfo.len, 0);
     }
 

py/builtinhelp.c

@@ -77,6 +77,7 @@ STATIC void mp_help_add_from_names(mp_obj_t list, const char *name) {
 }
 #endif
 
+// CIRCUITPY-CHANGE: move extern to top level to prevent warnings 
 #if MICROPY_MODULE_FROZEN
 extern const char mp_frozen_names[];
 #endif
@@ -122,6 +123,7 @@ STATIC void mp_help_print_modules(void) {
 
     #if MICROPY_ENABLE_EXTERNAL_IMPORT
     // let the user know there may be other modules available from the filesystem
+    // CIRCUITPY-CHANGE: make translatable
     serial_write_compressed(MP_ERROR_TEXT("Plus any modules on the filesystem\n"));
     #endif
 }
@@ -138,9 +140,11 @@ STATIC void mp_help_print_obj(const mp_obj_t obj) {
     const mp_obj_type_t *type = mp_obj_get_type(obj);
 
     // try to print something sensible about the given object
+    // CIRCUITPY-CHANGE: make translatable
     mp_cprintf(MP_PYTHON_PRINTER, MP_ERROR_TEXT("object "));
     mp_obj_print(obj, PRINT_STR);
 
+    // CIRCUITPY-CHANGE: make translatable
     mp_cprintf(MP_PYTHON_PRINTER, MP_ERROR_TEXT(" is of type %q\n"), type->name);
 
     mp_map_t *map = NULL;
@@ -166,6 +170,7 @@ STATIC void mp_help_print_obj(const mp_obj_t obj) {
 
 STATIC mp_obj_t mp_builtin_help(size_t n_args, const mp_obj_t *args) {
     if (n_args == 0) {
+        // CIRCUITPY-CHANGE: make translatable
         // print a general help message. Translate only works on single strings on one line.
         mp_cprintf(MP_PYTHON_PRINTER,
             MP_ERROR_TEXT("Welcome to Adafruit CircuitPython %s!\n\nVisit circuitpython.org for more information.\n\nTo list built-in modules type `help(\"modules\")`.\n"),

py/builtinimport.c

@@ -166,11 +166,11 @@ STATIC void do_load_from_lexer(mp_module_context_t *context, mp_lexer_t *lex) {
 #endif
 
 #if (MICROPY_HAS_FILE_READER && MICROPY_PERSISTENT_CODE_LOAD) || MICROPY_MODULE_FROZEN_MPY
-STATIC void do_execute_raw_code(const mp_module_context_t *context, const mp_raw_code_t *rc, const char *source_name) {
+STATIC void do_execute_raw_code(const mp_module_context_t *context, const mp_raw_code_t *rc, qstr source_name) {
-    (void)source_name;
-
     #if MICROPY_PY___FILE__
-    mp_store_attr(MP_OBJ_FROM_PTR(&context->module), MP_QSTR___file__, MP_OBJ_NEW_QSTR(qstr_from_str(source_name)));
+    mp_store_attr(MP_OBJ_FROM_PTR(&context->module), MP_QSTR___file__, MP_OBJ_NEW_QSTR(source_name));
+    #else
+    (void)source_name;
     #endif
 
     // execute the module in its context
@@ -226,7 +226,12 @@ STATIC void do_load(mp_module_context_t *module_obj, vstr_t *file) {
         if (frozen_type == MP_FROZEN_MPY) {
             const mp_frozen_module_t *frozen = modref;
             module_obj->constants = frozen->constants;
-            do_execute_raw_code(module_obj, frozen->rc, file_str + frozen_path_prefix_len);
+            #if MICROPY_PY___FILE__
+            qstr frozen_file_qstr = qstr_from_str(file_str + frozen_path_prefix_len);
+            #else
+            qstr frozen_file_qstr = MP_QSTRnull;
+            #endif
+            do_execute_raw_code(module_obj, frozen->rc, frozen_file_qstr);
             return;
         }
         #endif
@@ -234,14 +239,16 @@ STATIC void do_load(mp_module_context_t *module_obj, vstr_t *file) {
 
     #endif // MICROPY_MODULE_FROZEN
 
+    qstr file_qstr = qstr_from_str(file_str);
+
     // If we support loading .mpy files then check if the file extension is of
     // the correct format and, if so, load and execute the file.
     #if MICROPY_HAS_FILE_READER && MICROPY_PERSISTENT_CODE_LOAD
     if (file_str[file->len - 3] == 'm') {
         mp_compiled_module_t cm;
         cm.context = module_obj;
-        mp_raw_code_load_file(file_str, &cm);
+        mp_raw_code_load_file(file_qstr, &cm);
-        do_execute_raw_code(cm.context, cm.rc, file_str);
+        do_execute_raw_code(cm.context, cm.rc, file_qstr);
         return;
     }
     #endif
@@ -249,12 +256,13 @@ STATIC void do_load(mp_module_context_t *module_obj, vstr_t *file) {
     // If we can compile scripts then load the file and compile and execute it.
     #if MICROPY_ENABLE_COMPILER
     {
-        mp_lexer_t *lex = mp_lexer_new_from_file(file_str);
+        mp_lexer_t *lex = mp_lexer_new_from_file(file_qstr);
         do_load_from_lexer(module_obj, lex);
         return;
     }
     #else
     // If we get here then the file was not frozen and we can't compile scripts.
+    // CIRCUITPY-CHANGE: use more specific mp_raise
     mp_raise_ImportError(MP_ERROR_TEXT("script compilation not supported"));
     #endif
 }

py/compile.c

@@ -923,6 +923,7 @@ STATIC void compile_decorated(compiler_t *comp, mp_parse_node_struct_t *pns) {
         mp_parse_node_struct_t *pns0 = (mp_parse_node_struct_t *)pns_body->nodes[0];
         body_name = compile_funcdef_helper(comp, pns0, emit_options);
         scope_t *fscope = (scope_t *)pns0->nodes[4];
+        // CIRCUITPY-CHANGE: distinguish generators and async routines
         fscope->scope_flags |= MP_SCOPE_FLAG_GENERATOR | MP_SCOPE_FLAG_ASYNC;
     #endif
     } else {
@@ -1650,9 +1651,11 @@ STATIC void compile_try_except(compiler_t *comp, mp_parse_node_t pn_body, int n_
         if (qstr_exception_local != 0) {
             EMIT_ARG(load_const_tok, MP_TOKEN_KW_NONE);
             EMIT_ARG(label_assign, l3);
+            EMIT_ARG(adjust_stack_size, 1); // stack adjust for possible return value
             EMIT_ARG(load_const_tok, MP_TOKEN_KW_NONE);
             compile_store_id(comp, qstr_exception_local);
             compile_delete_id(comp, qstr_exception_local);
+            EMIT_ARG(adjust_stack_size, -1);
             compile_decrease_except_level(comp);
         }
 
@@ -1682,9 +1685,18 @@ STATIC void compile_try_finally(compiler_t *comp, mp_parse_node_t pn_body, int n
     } else {
         compile_try_except(comp, pn_body, n_except, pn_except, pn_else);
     }
+
+    // If the code reaches this point then the try part of the try-finally exited normally.
+    // This is indicated to the runtime by None sitting on the stack.
     EMIT_ARG(load_const_tok, MP_TOKEN_KW_NONE);
+
+    // Compile the finally block.
+    // The stack needs to be adjusted by 1 to account for the possibility that the finally is
+    // being executed as part of a return, and the return value is on the top of the stack.
     EMIT_ARG(label_assign, l_finally_block);
+    EMIT_ARG(adjust_stack_size, 1);
     compile_node(comp, pn_finally);
+    EMIT_ARG(adjust_stack_size, -1);
 
     compile_decrease_except_level(comp);
 }
@@ -1963,6 +1975,7 @@ STATIC void compile_async_stmt(compiler_t *comp, mp_parse_node_struct_t *pns) {
         // async def
         compile_funcdef(comp, pns0);
         scope_t *fscope = (scope_t *)pns0->nodes[4];
+        // CIRCUITPY-CHANGE: distinguish generators and async routines
         fscope->scope_flags |= MP_SCOPE_FLAG_GENERATOR | MP_SCOPE_FLAG_ASYNC;
     } else {
         // async for/with; first verify the scope is a generator

py/dynruntime.h

@@ -125,7 +125,8 @@ static inline void *m_realloc_dyn(void *ptr, size_t new_num_bytes) {
 #define mp_obj_get_int_truncated(o)         (mp_fun_table.native_from_obj(o, MP_NATIVE_TYPE_UINT))
 #define mp_obj_str_get_str(s)               (mp_obj_str_get_data_dyn((s), NULL))
 #define mp_obj_str_get_data(o, len)         (mp_obj_str_get_data_dyn((o), (len)))
-#define mp_get_buffer_raise(o, bufinfo, fl) (mp_fun_table.get_buffer_raise((o), (bufinfo), (fl)))
+#define mp_get_buffer(o, bufinfo, fl)       (mp_fun_table.get_buffer((o), (bufinfo), (fl)))
+#define mp_get_buffer_raise(o, bufinfo, fl) (mp_fun_table.get_buffer((o), (bufinfo), (fl) | MP_BUFFER_RAISE_IF_UNSUPPORTED))
 #define mp_get_stream_raise(s, flags)       (mp_fun_table.get_stream_raise((s), (flags)))
 #define mp_obj_is_true(o)                   (mp_fun_table.native_from_obj(o, MP_NATIVE_TYPE_BOOL))
 
@@ -137,6 +138,7 @@ static inline void *m_realloc_dyn(void *ptr, size_t new_num_bytes) {
 
 #define mp_obj_malloc_helper(n, t)          (mp_obj_malloc_helper_dyn(n, t))
 
+// CIRCUITPY-CHANGE: new routine
 #define mp_obj_assert_native_inited(o)      (mp_fun_table.assert_native_inited((o)))
 
 static inline mp_obj_t mp_obj_new_str_of_type_dyn(const mp_obj_type_t *type, const byte *data, size_t len) {
@@ -235,6 +237,7 @@ static inline void *mp_obj_malloc_helper_dyn(size_t num_bytes, const mp_obj_type
 
 #define nlr_raise(o)                            (mp_raise_dyn(o))
 #define mp_raise_type_arg(type, arg)            (mp_raise_dyn(mp_obj_new_exception_arg1_dyn((type), (arg))))
+// CIRCUITPY-CHANGE: use str
 #define mp_raise_msg(type, msg)                 (mp_fun_table.raise_msg_str((type), (msg)))
 #define mp_raise_OSError(er)                    (mp_raise_OSError_dyn(er))
 #define mp_raise_NotImplementedError(msg)       (mp_raise_msg(&mp_type_NotImplementedError, (msg)))
@@ -252,6 +255,7 @@ static NORETURN inline void mp_raise_dyn(mp_obj_t o) {
     }
 }
 
+// CIRCUITPY-CHANGE: new routine
 static NORETURN inline void mp_raise_arg1(const mp_obj_type_t *exc_type, mp_obj_t arg) {
     mp_fun_table.raise(mp_obj_new_exception_arg1_dyn(exc_type, arg));
     for (;;) {

py/emitglue.c

@@ -43,6 +43,7 @@
 #define DEBUG_printf DEBUG_printf
 #define DEBUG_OP_printf(...) DEBUG_printf(__VA_ARGS__)
 #else // don't print debugging info
+// CIRCUITPY-CHANGE: prevent warnings
 #define DEBUG_PRINT (0)
 #define DEBUG_printf(...) (void)0
 #define DEBUG_OP_printf(...) (void)0
@@ -88,6 +89,7 @@ void mp_emit_glue_assign_bytecode(mp_raw_code_t *rc, const byte *code,
     mp_prof_extract_prelude(code, prelude);
     #endif
 
+    // CIRCUITPY-CHANGE: prevent warning
     #if defined(DEBUG_PRINT) && DEBUG_PRINT
     #if !(MICROPY_PERSISTENT_CODE_SAVE || MICROPY_DEBUG_PRINTERS)
     const size_t len = 0;
@@ -111,6 +113,7 @@ void mp_emit_glue_assign_native(mp_raw_code_t *rc, mp_raw_code_kind_t kind, void
     // so that the generated native code which was created in data RAM will
     // be available for execution from instruction RAM.
     #if MICROPY_EMIT_THUMB || MICROPY_EMIT_INLINE_THUMB
+    // CIRCUITPY-CHANGE: prevent warning
     #if defined(__ICACHE_PRESENT) && __ICACHE_PRESENT == 1
     // Flush D-cache, so the code emitted is stored in RAM.
     MP_HAL_CLEAN_DCACHE(fun_data, fun_len);
@@ -189,6 +192,7 @@ mp_obj_t mp_make_function_from_raw_code(const mp_raw_code_t *rc, const mp_module
         case MP_CODE_NATIVE_VIPER:
             fun = mp_obj_new_fun_native(def_args, rc->fun_data, context, rc->children);
             // Check for a generator function, and if so change the type of the object
+            // CIRCUITPY-CHANGE: distinguish generators and async
             if ((rc->scope_flags & MP_SCOPE_FLAG_ASYNC) != 0) {
                 ((mp_obj_base_t *)MP_OBJ_TO_PTR(fun))->type = &mp_type_native_coro_wrap;
             } else if ((rc->scope_flags & MP_SCOPE_FLAG_GENERATOR) != 0) {
@@ -206,6 +210,7 @@ mp_obj_t mp_make_function_from_raw_code(const mp_raw_code_t *rc, const mp_module
             assert(rc->kind == MP_CODE_BYTECODE);
             fun = mp_obj_new_fun_bc(def_args, rc->fun_data, context, rc->children);
             // check for generator functions and if so change the type of the object
+            // CIRCUITPY-CHANGE: distinguish generators and async
             // A generator is MP_SCOPE_FLAG_ASYNC | MP_SCOPE_FLAG_GENERATOR,
             // so check for ASYNC first.
             #if MICROPY_PY_ASYNC_AWAIT

py/formatfloat.c

@@ -80,6 +80,7 @@ static inline int fp_isless1(float x) {
 
 #elif MICROPY_FLOAT_IMPL == MICROPY_FLOAT_IMPL_DOUBLE
 
+// CIRCUITPY-CHNAGE: prevent warnings
 #pragma GCC diagnostic ignored "-Wfloat-equal"
 #define FPTYPE double
 #define FPCONST(x) x

py/gc.c

@@ -305,14 +305,28 @@ STATIC bool gc_try_add_heap(size_t failed_alloc) {
     // - If the new heap won't fit in the available free space, add the largest
     //   new heap that will fit (this may lead to failed system heap allocations
     //   elsewhere, but some allocation will likely fail in this circumstance!)
-    size_t total_heap = 0;
+
+    // Compute total number of blocks in the current heap.
+    size_t total_blocks = 0;
     for (mp_state_mem_area_t *area = &MP_STATE_MEM(area);
          area != NULL;
          area = NEXT_AREA(area)) {
-        total_heap += area->gc_pool_end - area->gc_alloc_table_start;
+        total_blocks += area->gc_alloc_table_byte_len * BLOCKS_PER_ATB;
-        total_heap += ALLOC_TABLE_GAP_BYTE + sizeof(mp_state_mem_area_t);
     }
 
+    // Compute bytes needed to build a heap with total_blocks blocks.
+    size_t total_heap =
+        total_blocks / BLOCKS_PER_ATB
+        #if MICROPY_ENABLE_FINALISER
+        + total_blocks / BLOCKS_PER_FTB
+        #endif
+        + total_blocks * BYTES_PER_BLOCK
+        + ALLOC_TABLE_GAP_BYTE
+        + sizeof(mp_state_mem_area_t);
+
+    // Round up size to the nearest multiple of BYTES_PER_BLOCK.
+    total_heap = (total_heap + BYTES_PER_BLOCK - 1) & (~(BYTES_PER_BLOCK - 1));
+
     DEBUG_printf("total_heap " UINT_FMT " bytes\n", total_heap);
 
     size_t to_alloc = MIN(avail, MAX(total_heap, needed));
@@ -851,6 +865,7 @@ void *gc_alloc(size_t n_bytes, unsigned int alloc_flags) {
             }
             #endif
 
+            // CIRCUITPY-CHANGE
             #if CIRCUITPY_DEBUG
             gc_dump_alloc_table(&mp_plat_print);
             #endif
@@ -936,6 +951,7 @@ found:
     gc_dump_alloc_table(&mp_plat_print);
     #endif
 
+    // CIRCUITPY-CHANGE
     #if CIRCUITPY_MEMORYMONITOR
     memorymonitor_track_allocation(end_block - start_block + 1);
     #endif

py/gc.h

@@ -30,6 +30,7 @@
 #include <stddef.h>
 #include "py/mpprint.h"
 
+// CIRCUITPY-CHANGE
 #include "py/mpconfig.h"
 #include "py/mpstate.h"
 #include "py/misc.h"
@@ -77,14 +78,8 @@ enum {
 void *gc_alloc(size_t n_bytes, unsigned int alloc_flags);
 void gc_free(void *ptr); // does not call finaliser
 size_t gc_nbytes(const void *ptr);
-bool gc_has_finaliser(const void *ptr);
 void *gc_realloc(void *ptr, size_t n_bytes, bool allow_move);
 
-// CIRCUITPY-CHANGE
-// Prevents a pointer from ever being freed because it establishes a permanent reference to it. Use
-// very sparingly because it can leak memory.
-bool gc_never_free(void *ptr);
-
 // CIRCUITPY-CHANGE
 // True if the pointer is on the MP heap. Doesn't require that it is the start
 // of a block.

py/lexer.c

@@ -527,14 +527,14 @@ STATIC void parse_string_literal(mp_lexer_t *lex, bool is_raw, bool is_fstring)
     vstr_cut_tail_bytes(&lex->vstr, n_closing);
 }
 
+// This function returns whether it has crossed a newline or not.
+// It therefore always return true if stop_at_newline is true
 STATIC bool skip_whitespace(mp_lexer_t *lex, bool stop_at_newline) {
-    bool had_physical_newline = false;
     while (!is_end(lex)) {
         if (is_physical_newline(lex)) {
             if (stop_at_newline && lex->nested_bracket_level == 0) {
-                break;
+                return true;
             }
-            had_physical_newline = true;
             next_char(lex);
         } else if (is_whitespace(lex)) {
             next_char(lex);
@@ -543,16 +543,16 @@ STATIC bool skip_whitespace(mp_lexer_t *lex, bool stop_at_newline) {
             while (!is_end(lex) && !is_physical_newline(lex)) {
                 next_char(lex);
             }
-            // had_physical_newline will be set on next loop
+            // will return true on next loop
         } else if (is_char_and(lex, '\\', '\n')) {
-            // line-continuation, so don't set had_physical_newline
+            // line-continuation, so don't return true
             next_char(lex);
             next_char(lex);
         } else {
             break;
         }
     }
-    return had_physical_newline;
+    return false;
 }
 
 void mp_lexer_to_next(mp_lexer_t *lex) {
@@ -577,7 +577,10 @@ void mp_lexer_to_next(mp_lexer_t *lex) {
     vstr_reset(&lex->vstr);
 
     // skip white space and comments
-    bool had_physical_newline = skip_whitespace(lex, false);
+    // set the newline tokens at the line and column of the preceding line:
+    // only advance on the pointer until a new line is crossed, save the
+    // line and column, and then readvance it
+    bool had_physical_newline = skip_whitespace(lex, true);
 
     // set token source information
     lex->tok_line = lex->line;
@@ -591,7 +594,12 @@ void mp_lexer_to_next(mp_lexer_t *lex) {
         lex->tok_kind = MP_TOKEN_INDENT;
         lex->emit_dent -= 1;
 
-    } else if (had_physical_newline && lex->nested_bracket_level == 0) {
+    } else if (had_physical_newline) {
+        // The cursor is at the end of the previous line, pointing to a
+        // physical newline. Skip any remaining whitespace, comments, and
+        // newlines.
+        skip_whitespace(lex, false);
+
         lex->tok_kind = MP_TOKEN_NEWLINE;
 
         size_t num_spaces = lex->column - 1;
@@ -862,9 +870,10 @@ mp_lexer_t *mp_lexer_new(qstr src_name, mp_reader_t reader) {
     // preload first token
     mp_lexer_to_next(lex);
 
-    // Check that the first token is in the first column.  If it's not then we
+    // Check that the first token is in the first column unless it is a
-    // convert the token kind to INDENT so that the parser gives a syntax error.
+    // newline. Otherwise we convert the token kind to INDENT so that
-    if (lex->tok_column != 1) {
+    // the parser gives a syntax error.
+    if (lex->tok_column != 1 && lex->tok_kind != MP_TOKEN_NEWLINE) {
         lex->tok_kind = MP_TOKEN_INDENT;
     }
 
@@ -879,10 +888,10 @@ mp_lexer_t *mp_lexer_new_from_str_len(qstr src_name, const char *str, size_t len
 
 #if MICROPY_READER_POSIX || MICROPY_READER_VFS
 
-mp_lexer_t *mp_lexer_new_from_file(const char *filename) {
+mp_lexer_t *mp_lexer_new_from_file(qstr filename) {
     mp_reader_t reader;
     mp_reader_new_file(&reader, filename);
-    return mp_lexer_new(qstr_from_str(filename), reader);
+    return mp_lexer_new(filename, reader);
 }
 
 #if MICROPY_HELPER_LEXER_UNIX

py/lexer.h

@@ -191,7 +191,7 @@ mp_lexer_t *mp_lexer_new_from_str_len(qstr src_name, const char *str, size_t len
 
 // If MICROPY_READER_POSIX or MICROPY_READER_VFS aren't enabled then
 // this function must be implemented by the port.
-mp_lexer_t *mp_lexer_new_from_file(const char *filename);
+mp_lexer_t *mp_lexer_new_from_file(qstr filename);
 
 #if MICROPY_HELPER_LEXER_UNIX
 mp_lexer_t *mp_lexer_new_from_fd(qstr filename, int fd, bool close_fd);

py/makeqstrdata.py

@@ -10,21 +10,15 @@ supervisor/shared/translate/translate.h
 
 from __future__ import print_function
 
-import bisect
+
 import re
 import sys
 
-import collections
+# CIRCUITPY-CHANGE
-import gettext
-import os.path
-
 if hasattr(sys.stdout, "reconfigure"):
     sys.stdout.reconfigure(encoding="utf-8")
     sys.stderr.reconfigure(errors="backslashreplace")
 
-py = os.path.dirname(sys.argv[0])
-top = os.path.dirname(py)
-
 # Python 2/3 compatibility:
 #   - iterating through bytes is different
 #   - codepoint2name lives in a different module
@@ -68,20 +62,9 @@ codepoint2name[ord("^")] = "caret"
 codepoint2name[ord("|")] = "pipe"
 codepoint2name[ord("~")] = "tilde"
 
-C_ESCAPES = {
+# static qstrs, these must maintain a specific order for .mpy compatibility
-    "\a": "\\a",
+# See QSTR_LAST_STATIC at the top of py/persistentcode.c
-    "\b": "\\b",
-    "\f": "\\f",
-    "\n": "\\n",
-    "\r": "\\r",
-    "\t": "\\t",
-    "\v": "\\v",
-    "'": "\\'",
-    '"': '\\"',
-}
 
-# static qstrs, should be sorted
-# These are qstrs that are always included and always have the same number. It allows mpy files to omit them.
 static_qstr_list = [
     "",
     "__dir__",  # Put __dir__ after empty qstr for builtin dir() to work
@@ -250,74 +233,70 @@ static_qstr_list = [
     "zip",
 ]
 
-# CIRCUITPY-CHANGE
+# Additional QSTRs that must have index <255 because they are stored in
-# When taking the next merge from Micropython, prefer upstream's way of ensuring these appear in the "QSTR0" pool.
+# `mp_binary_op_method_name` and `mp_unary_op_method_name` (see py/objtype.c).
-# These qstrs have to be sorted early (preferably right after static_qstr_list) because they are required to fit in 8-bit values
+# These are not part of the .mpy compatibility list, but we place them in the
-# however they should never be *forced* to appear
+# fixed unsorted pool (i.e. QDEF0) to ensure their indices are small.
-# repeats len, hash, int from the static qstr list, but this doesn't hurt anything.
+operator_qstr_list = {
-eightbit_qstr_list = [
-    "__abs__",
-    "__add__",
-    "__and__",
     "__bool__",
-    "__complex__",
-    "__contains__",
-    "__delete__",
-    "__divmod__",
-    "__eq__",
-    "__float__",
-    "__floordiv__",
-    "__ge__",
-    "__get__",
-    "__gt__",
-    "__hash__",
-    "__iadd__",
-    "__iand__",
-    "__ifloordiv__",
-    "__ilshift__",
-    "__imatmul__",
-    "__imod__",
-    "__imul__",
-    "__int__",
-    "__invert__",
-    "__ior__",
-    "__ipow__",
-    "__irshift__",
-    "__isub__",
-    "__itruediv__",
-    "__ixor__",
-    "__le__",
-    "__len__",
-    "__lshift__",
-    "__lt__",
-    "__matmul__",
-    "__mod__",
-    "__mul__",
-    "__ne__",
-    "__neg__",
-    "__or__",
     "__pos__",
-    "__pow__",
+    "__neg__",
-    "__radd__",
+    "__invert__",
-    "__rand__",
+    "__abs__",
-    "__rfloordiv__",
+    "__float__",
-    "__rlshift__",
+    "__complex__",
-    "__rmatmul__",
-    "__rmod__",
-    "__rmul__",
-    "__ror__",
-    "__rpow__",
-    "__rrshift__",
-    "__rshift__",
-    "__rsub__",
-    "__rtruediv__",
-    "__rxor__",
-    "__set__",
     "__sizeof__",
+    "__lt__",
+    "__gt__",
+    "__eq__",
+    "__le__",
+    "__ge__",
+    "__ne__",
+    "__contains__",
+    "__iadd__",
+    "__isub__",
+    "__imul__",
+    "__imatmul__",
+    "__ifloordiv__",
+    "__itruediv__",
+    "__imod__",
+    "__ipow__",
+    "__ior__",
+    "__ixor__",
+    "__iand__",
+    "__ilshift__",
+    "__irshift__",
+    "__add__",
     "__sub__",
+    "__mul__",
+    "__matmul__",
+    "__floordiv__",
     "__truediv__",
+    "__mod__",
+    "__divmod__",
+    "__pow__",
+    "__or__",
     "__xor__",
-]
+    "__and__",
+    "__lshift__",
+    "__rshift__",
+    "__radd__",
+    "__rsub__",
+    "__rmul__",
+    "__rmatmul__",
+    "__rfloordiv__",
+    "__rtruediv__",
+    "__rmod__",
+    "__rpow__",
+    "__ror__",
+    "__rxor__",
+    "__rand__",
+    "__rlshift__",
+    "__rrshift__",
+    "__get__",
+    "__set__",
+    "__delete__",
+}
 
 
 # this must match the equivalent function in qstr.c
@@ -341,24 +320,15 @@ def qstr_escape(qst):
     return re.sub(r"[^A-Za-z0-9_]", esc_char, qst)
 
 
-# CIRCUITPY-CHANGE: add translations handling
+static_qstr_list_ident = list(map(qstr_escape, static_qstr_list))
-def parse_input_headers_with_translations(infiles):
+
+
+def parse_input_headers(infiles):
     qcfgs = {}
     qstrs = {}
+    # CIRCUITPY-CHANGE: add translations
     translations = set()
 
-    # add static qstrs
-    for qstr in static_qstr_list:
-        # work out the corresponding qstr name
-        ident = qstr_escape(qstr)
-
-        # don't add duplicates
-        assert ident not in qstrs
-
-        # add the qstr to the list, with order number to retain original order in file
-        order = len(qstrs) - 300000
-        qstrs[ident] = (order, ident, qstr)
-
     # read the qstrs in from the input files
     for infile in infiles:
         with open(infile, "rt") as f:
@@ -399,21 +369,18 @@ def parse_input_headers_with_translations(infiles):
                 ident = qstr_escape(qstr)
 
                 # don't add duplicates
+                if ident in static_qstr_list_ident:
+                    continue
                 if ident in qstrs:
                     continue
 
-                # add the qstr to the list, with order number to retain original order in file
+                qstrs[ident] = (ident, qstr)
-                order = len(qstrs)
-                # but put special method names like __add__ at the top of list, so
-                # that their id's fit into a byte
-                if ident in eightbit_qstr_list:
-                    order -= 100000
-                qstrs[ident] = (order, ident, qstr)
 
-    if not qcfgs and qstrs:
+    if not qcfgs:
         sys.stderr.write("ERROR: Empty preprocessor output - check for errors above\n")
         sys.exit(1)
 
+    # CIRCUITPY-CHANGE
     return qcfgs, qstrs, translations
 
 
@@ -454,16 +421,21 @@ def print_qstr_data(qcfgs, qstrs, translations):
     print("")
 
     # add NULL qstr with no hash or data
-    print('QDEF(MP_QSTRnull, 0, 0, "")')
+    print('QDEF0(MP_QSTRnull, 0, 0, "")')
 
-    total_qstr_size = 0
+    # add static qstrs to the first unsorted pool
-    # go through each qstr and print it out
+    for qstr in static_qstr_list:
-    for order, ident, qstr in sorted(qstrs.values(), key=lambda x: x[0]):
         qbytes = make_bytes(cfg_bytes_len, cfg_bytes_hash, qstr)
-        print("QDEF(MP_QSTR_%s, %s)" % (ident, qbytes))
+        print("QDEF0(MP_QSTR_%s, %s)" % (qstr_escape(qstr), qbytes))
 
-        total_qstr_size += len(qstr)
+    # add remaining qstrs to the sorted (by value) pool (unless they're in
+    # operator_qstr_list, in which case add them to the unsorted pool)
+    for ident, qstr in sorted(qstrs.values(), key=lambda x: x[1]):
+        qbytes = make_bytes(cfg_bytes_len, cfg_bytes_hash, qstr)
+        pool = 0 if qstr in operator_qstr_list else 1
+        print("QDEF%d(MP_QSTR_%s, %s)" % (pool, ident, qbytes))
 
+    # CIRCUITPY-CHANGE: translations
     print(
         "// Enumerate translated texts but don't actually include translations. Instead, the linker will link them in."
     )
@@ -474,21 +446,12 @@ def print_qstr_data(qcfgs, qstrs, translations):
     print("// {} bytes worth of qstr".format(total_qstr_size))
 
 
+
 def do_work(infiles):
+    # CIRCUITPY-CHANGE: include translations
     qcfgs, qstrs, translations = parse_input_headers_with_translations(infiles)
     print_qstr_data(qcfgs, qstrs, translations)
 
 
 if __name__ == "__main__":
-    import argparse
+    do_work(sys.argv[1:])
-
-    parser = argparse.ArgumentParser(
-        description="Process QSTR definitions into headers for compilation"
-    )
-    parser.add_argument(
-        "infiles", metavar="N", type=str, nargs="+", help="an integer for the accumulator"
-    )
-
-    args = parser.parse_args()
-
-    do_work(args.infiles)

py/makeqstrdefs.py

@@ -15,6 +15,7 @@ import sys
 import multiprocessing, multiprocessing.dummy
 
 
+# CIRCUITPY-CHANGE
 from html.entities import name2codepoint
 
 # add some custom names to map characters that aren't in HTML
@@ -64,6 +65,10 @@ _MODE_MODULE = "module"
 _MODE_ROOT_POINTER = "root_pointer"
 
 
+class PreprocessorError(Exception):
+    pass
+
+
 def is_c_source(fname):
     return os.path.splitext(fname)[1] in [".c"]
 
@@ -93,10 +98,10 @@ def preprocess():
 
     def pp(flags):
         def run(files):
-            completed = subprocess.run(args.pp + flags + files, stdout=subprocess.PIPE)
+            try:
-            if completed.returncode != 0:
+                return subprocess.check_output(args.pp + flags + files)
-                raise RuntimeError(" ".join(args.pp + flags + files))
+            except subprocess.CalledProcessError as er:
-            return completed.stdout
+                raise PreprocessorError(str(er))
 
         return run
 
@@ -124,6 +129,7 @@ def write_out(fname, output):
             f.write("\n".join(output) + "\n")
 
 
+# CIRCUITPY-CHANGE: added
 def qstr_unescape(qstr):
     for name in name2codepoint:
         if "__" + name + "__" in qstr:
@@ -146,7 +152,8 @@ def process_file(f):
         )
     elif args.mode == _MODE_ROOT_POINTER:
         re_match = re.compile(r"MP_REGISTER_ROOT_POINTER\(.*?\);")
-    re_translate = re.compile(r"MP_COMPRESSED_ROM_TEXT\(\"((?:(?=(\\?))\2.)*?)\"\)")
+   # CIRCUITPY-CHANGE: added
+   re_translate = re.compile(r"MP_COMPRESSED_ROM_TEXT\(\"((?:(?=(\\?))\2.)*?)\"\)")
     output = []
     last_fname = None
     for line in f:
@@ -170,6 +177,7 @@ def process_file(f):
             elif args.mode in (_MODE_COMPRESS, _MODE_MODULE, _MODE_ROOT_POINTER):
                 output.append(match)
 
+        # CIRCUITPY-CHANGE: added
         for match in re_translate.findall(line):
             output.append('TRANSLATE("' + match[0] + '")')
 
@@ -184,6 +192,7 @@ def cat_together():
 
     hasher = hashlib.md5()
     all_lines = []
+    # CIRCUITPY-CHANGE: added
     outf = open(args.output_dir + "/out", "wb")
     for fname in glob.glob(args.output_dir + "/*." + args.mode):
         with open(fname, "rb") as f:
@@ -191,6 +200,7 @@ def cat_together():
             all_lines += lines
     all_lines.sort()
     all_lines = b"\n".join(all_lines)
+    # CIRCUITPY-CHANGE: added
     outf.write(all_lines)
     outf.close()
     hasher.update(all_lines)
@@ -209,6 +219,7 @@ def cat_together():
         mode_full = "Module registrations"
     elif args.mode == _MODE_ROOT_POINTER:
         mode_full = "Root pointer registrations"
+    # CIRCUITPY-CHANGE
     if old_hash != new_hash:
         print(mode_full, "updated")
         try:
@@ -261,7 +272,12 @@ if __name__ == "__main__":
         for k, v in named_args.items():
             setattr(args, k, v)
 
-        preprocess()
+        try:
+            preprocess()
+        except PreprocessorError as er:
+            print(er)
+            sys.exit(1)
+
         sys.exit(0)
 
     args.mode = sys.argv[2]

py/makeversionhdr.py

@@ -1,6 +1,7 @@
 """
 Generate header file with macros defining MicroPython version info.
 
+# CIRCUITPY-CHANGE: This script is thoroughly reword for use with CircuitPython.
 This script works with Python 3.7 and newer
 """
 

py/malloc.c

@@ -267,6 +267,7 @@ void m_tracked_free(void *ptr_in) {
     if (ptr_in == NULL) {
         return;
     }
+    // CIRCUITPY-CHANGE: cast to avoid compiler warning
     m_tracked_node_t *node = (m_tracked_node_t *)(void *)((uint8_t *)ptr_in - sizeof(m_tracked_node_t));
     #if MICROPY_DEBUG_VERBOSE
     size_t data_bytes;

py/misc.h

@@ -34,8 +34,6 @@
 #include <stdint.h>
 #include <stddef.h>
 
-#include "mpconfig.h"
-
 typedef unsigned char byte;
 typedef unsigned int uint;
 
@@ -54,10 +52,15 @@ typedef unsigned int uint;
 
 // Static assertion macro
 #define MP_STATIC_ASSERT(cond) ((void)sizeof(char[1 - 2 * !(cond)]))
-#if defined(_MSC_VER)
+// In C++ things like comparing extern const pointers are not constant-expressions so cannot be used
-#define MP_STATIC_ASSERT_NOT_MSC(cond) (1)
+// in MP_STATIC_ASSERT. Note that not all possible compiler versions will reject this. Some gcc versions
+// do, others only with -Werror=vla, msvc always does.
+// The (void) is needed to avoid "left operand of comma operator has no effect [-Werror=unused-value]"
+// when using this macro on the left-hand side of a comma.
+#if defined(_MSC_VER) || defined(__cplusplus)
+#define MP_STATIC_ASSERT_NONCONSTEXPR(cond) ((void)1)
 #else
-#define MP_STATIC_ASSERT_NOT_MSC(cond) MP_STATIC_ASSERT(cond)
+#define MP_STATIC_ASSERT_NONCONSTEXPR(cond) MP_STATIC_ASSERT(cond)
 #endif
 
 // Round-up integer division
@@ -69,32 +72,30 @@ typedef unsigned int uint;
 // TODO make a lazy m_renew that can increase by a smaller amount than requested (but by at least 1 more element)
 
 #define m_new(type, num) ((type *)(m_malloc(sizeof(type) * (num))))
-#define m_new_ll(type, num) m_new(type, num) // CIRCUITPY-CHANGE: clue to long-lived allocator
 #define m_new_maybe(type, num) ((type *)(m_malloc_maybe(sizeof(type) * (num))))
 #define m_new0(type, num) ((type *)(m_malloc0(sizeof(type) * (num))))
 #define m_new_obj(type) (m_new(type, 1))
 #define m_new_obj_maybe(type) (m_new_maybe(type, 1))
-#define m_new_obj_var(obj_type, var_type, var_num) ((obj_type *)m_malloc(sizeof(obj_type) + sizeof(var_type) * (var_num)))
+#define m_new_obj_var(obj_type, var_field, var_type, var_num) ((obj_type *)m_malloc(offsetof(obj_type, var_field) + sizeof(var_type) * (var_num)))
-#define m_new_obj_var0(obj_type, var_type, var_num) ((obj_type *)m_malloc0(sizeof(obj_type) + sizeof(var_type) * (var_num)))
+#define m_new_obj_var0(obj_type, var_field, var_type, var_num) ((obj_type *)m_malloc0(offsetof(obj_type, var_field) + sizeof(var_type) * (var_num)))
-#define m_new_obj_var_maybe(obj_type, var_type, var_num) ((obj_type *)m_malloc_maybe(sizeof(obj_type) + sizeof(var_type) * (var_num)))
+#define m_new_obj_var_maybe(obj_type, var_field, var_type, var_num) ((obj_type *)m_malloc_maybe(offsetof(obj_type, var_field) + sizeof(var_type) * (var_num)))
 #if MICROPY_ENABLE_FINALISER
 #define m_new_obj_with_finaliser(type) ((type *)(m_malloc_with_finaliser(sizeof(type))))
-#define m_new_ll_obj_with_finaliser(type) m_new_obj_with_finaliser(type) // CIRCUITPY-CHANGE: clue to long-lived allocator
+#define m_new_obj_var_with_finaliser(type, var_field, var_type, var_num) ((type *)m_malloc_with_finaliser(offsetof(type, var_field) + sizeof(var_type) * (var_num)))
-#define m_new_obj_var_with_finaliser(type, var_type, var_num) ((type *)m_malloc_with_finaliser(sizeof(type) + sizeof(var_type) * (var_num)))
 #else
 #define m_new_obj_with_finaliser(type) m_new_obj(type)
-#define m_new_obj_var_with_finaliser(type, var_type, var_num) m_new_obj_var(type, var_type, var_num)
+#define m_new_obj_var_with_finaliser(type, var_field, var_type, var_num) m_new_obj_var(type, var_field, var_type, var_num)
 #endif
 #if MICROPY_MALLOC_USES_ALLOCATED_SIZE
 #define m_renew(type, ptr, old_num, new_num) ((type *)(m_realloc((ptr), sizeof(type) * (old_num), sizeof(type) * (new_num))))
 #define m_renew_maybe(type, ptr, old_num, new_num, allow_move) ((type *)(m_realloc_maybe((ptr), sizeof(type) * (old_num), sizeof(type) * (new_num), (allow_move))))
 #define m_del(type, ptr, num) m_free(ptr, sizeof(type) * (num))
-#define m_del_var(obj_type, var_type, var_num, ptr) (m_free(ptr, sizeof(obj_type) + sizeof(var_type) * (var_num)))
+#define m_del_var(obj_type, var_field, var_type, var_num, ptr) (m_free(ptr, offsetof(obj_type, var_field) + sizeof(var_type) * (var_num)))
 #else
 #define m_renew(type, ptr, old_num, new_num) ((type *)(m_realloc((ptr), sizeof(type) * (new_num))))
 #define m_renew_maybe(type, ptr, old_num, new_num, allow_move) ((type *)(m_realloc_maybe((ptr), sizeof(type) * (new_num), (allow_move))))
 #define m_del(type, ptr, num) ((void)(num), m_free(ptr))
-#define m_del_var(obj_type, var_type, var_num, ptr) ((void)(var_num), m_free(ptr))
+#define m_del_var(obj_type, var_field, var_type, var_num, ptr) ((void)(var_num), m_free(ptr))
 #endif
 #define m_del_obj(type, ptr) (m_del(type, ptr, 1))
 
@@ -134,9 +135,6 @@ size_t m_get_peak_bytes_allocated(void);
 // align ptr to the nearest multiple of "alignment"
 #define MP_ALIGN(ptr, alignment) (void *)(((uintptr_t)(ptr) + ((alignment) - 1)) & ~((alignment) - 1))
 
-// CIRCUITPY-CHANGE
-#define sizeof_field(TYPE, MEMBER) sizeof((((TYPE *)0)->MEMBER))
-
 /** unichar / UTF-8 *********************************************/
 
 #if MICROPY_PY_BUILTINS_STR_UNICODE
@@ -297,6 +295,7 @@ typedef union _mp_float_union_t {
 // So leave MP_COMPRESSED_ROM_TEXT in place for makeqstrdefs.py / makecompresseddata.py to find them.
 
 #else
+
 // Compression enabled and doing a regular build.
 // Map MP_COMPRESSED_ROM_TEXT to the compressed strings.
 
@@ -326,8 +325,10 @@ inline MP_ALWAYSINLINE const char *MP_COMPRESSED_ROM_TEXT(const char *msg) {
 
     return msg;
 }
+
 #endif
 
+// CIRCUITPY-CHANGE
 #elif defined(CIRCUITPY)
 #include "supervisor/shared/translate/translate.h"
 #else

py/mkenv.mk

@@ -37,6 +37,7 @@ else
 Q = @
 endif
 
+# CIRCUITPY-CHANGE
 ifneq ($(filter rules,$(BUILD_VERBOSE)),)
 # This clever shell redefinition will print out the makefile line that is causing an action.
 # Note that -j can cause the order to be confusing.
@@ -70,6 +71,7 @@ OBJCOPY = $(CROSS_COMPILE)objcopy
 SIZE = $(CROSS_COMPILE)size
 STRIP = $(CROSS_COMPILE)strip
 AR = $(CROSS_COMPILE)ar
+WINDRES = $(CROSS_COMPILE)windres
 
 MAKE_MANIFEST = $(PYTHON) $(TOP)/tools/makemanifest.py
 MAKE_FROZEN = $(PYTHON) $(TOP)/tools/make-frozen.py

py/mkrules.cmake

@@ -15,6 +15,10 @@ set(MICROPY_ROOT_POINTERS_SPLIT "${MICROPY_GENHDR_DIR}/root_pointers.split")
 set(MICROPY_ROOT_POINTERS_COLLECTED "${MICROPY_GENHDR_DIR}/root_pointers.collected")
 set(MICROPY_ROOT_POINTERS "${MICROPY_GENHDR_DIR}/root_pointers.h")
 
+if(NOT MICROPY_PREVIEW_VERSION_2)
+    set(MICROPY_PREVIEW_VERSION_2 0)
+endif()
+
 # Need to do this before extracting MICROPY_CPP_DEF below. Rest of frozen
 # manifest handling is at the end of this file.
 if(MICROPY_FROZEN_MANIFEST)
@@ -24,6 +28,12 @@ if(MICROPY_FROZEN_MANIFEST)
     )
 endif()
 
+if(MICROPY_PREVIEW_VERSION_2)
+    target_compile_definitions(${MICROPY_TARGET} PUBLIC
+        MICROPY_PREVIEW_VERSION_2=\(1\)
+    )
+endif()
+
 # Provide defaults for preprocessor flags if not already defined
 if(NOT MICROPY_CPP_FLAGS)
     get_target_property(MICROPY_CPP_INC ${MICROPY_TARGET} INCLUDE_DIRECTORIES)
@@ -89,6 +99,7 @@ add_custom_command(
 add_custom_command(
     OUTPUT ${MICROPY_QSTRDEFS_COLLECTED}
     COMMAND ${Python3_EXECUTABLE} ${MICROPY_PY_DIR}/makeqstrdefs.py cat qstr _ ${MICROPY_GENHDR_DIR}/qstr ${MICROPY_QSTRDEFS_COLLECTED}
+    BYPRODUCTS "${MICROPY_QSTRDEFS_COLLECTED}.hash"
     DEPENDS ${MICROPY_QSTRDEFS_SPLIT}
     VERBATIM
     COMMAND_EXPAND_LISTS
@@ -126,6 +137,7 @@ add_custom_command(
 add_custom_command(
     OUTPUT ${MICROPY_MODULEDEFS_COLLECTED}
     COMMAND ${Python3_EXECUTABLE} ${MICROPY_PY_DIR}/makeqstrdefs.py cat module _ ${MICROPY_GENHDR_DIR}/module ${MICROPY_MODULEDEFS_COLLECTED}
+    BYPRODUCTS "${MICROPY_MODULEDEFS_COLLECTED}.hash"
     DEPENDS ${MICROPY_MODULEDEFS_SPLIT}
     VERBATIM
     COMMAND_EXPAND_LISTS
@@ -151,6 +163,7 @@ add_custom_command(
 add_custom_command(
     OUTPUT ${MICROPY_ROOT_POINTERS_COLLECTED}
     COMMAND ${Python3_EXECUTABLE} ${MICROPY_PY_DIR}/makeqstrdefs.py cat root_pointer _ ${MICROPY_GENHDR_DIR}/root_pointer ${MICROPY_ROOT_POINTERS_COLLECTED}
+    BYPRODUCTS "${MICROPY_ROOT_POINTERS_COLLECTED}.hash"
     DEPENDS ${MICROPY_ROOT_POINTERS_SPLIT}
     VERBATIM
     COMMAND_EXPAND_LISTS
@@ -202,10 +215,33 @@ if(MICROPY_FROZEN_MANIFEST)
         )
     endif()
 
+    if(NOT MICROPY_CROSS_FLAGS)
+        set(MICROPY_CROSS_FLAGS "")
+    else()
+        set(MICROPY_CROSS_FLAGS "-f${MICROPY_CROSS_FLAGS}")
+    endif()
+
+    # Set default path variables to be passed to makemanifest.py. These will
+    # be available in path substitutions. Additional variables can be set
+    # per-board in mpconfigboard.cmake.
+    set(MICROPY_MANIFEST_PORT_DIR ${MICROPY_PORT_DIR})
+    set(MICROPY_MANIFEST_BOARD_DIR ${MICROPY_BOARD_DIR})
+    set(MICROPY_MANIFEST_MPY_DIR ${MICROPY_DIR})
+    set(MICROPY_MANIFEST_MPY_LIB_DIR ${MICROPY_LIB_DIR})
+
+    # Find all MICROPY_MANIFEST_* variables and turn them into command line arguments.
+    get_cmake_property(_manifest_vars VARIABLES)
+    list(FILTER _manifest_vars INCLUDE REGEX "MICROPY_MANIFEST_.*")
+    foreach(_manifest_var IN LISTS _manifest_vars)
+        list(APPEND _manifest_var_args "-v")
+        string(REGEX REPLACE "MICROPY_MANIFEST_(.*)" "\\1" _manifest_var_name ${_manifest_var})
+        list(APPEND _manifest_var_args "${_manifest_var_name}=${${_manifest_var}}")
+    endforeach()
+
     add_custom_target(
         BUILD_FROZEN_CONTENT ALL
         BYPRODUCTS ${MICROPY_FROZEN_CONTENT}
-        COMMAND ${Python3_EXECUTABLE} ${MICROPY_DIR}/tools/makemanifest.py -o ${MICROPY_FROZEN_CONTENT} -v "MPY_DIR=${MICROPY_DIR}" -v "MPY_LIB_DIR=${MICROPY_LIB_DIR}" -v "PORT_DIR=${MICROPY_PORT_DIR}" -v "BOARD_DIR=${MICROPY_BOARD_DIR}" -b "${CMAKE_BINARY_DIR}" -f${MICROPY_CROSS_FLAGS} --mpy-tool-flags=${MICROPY_MPY_TOOL_FLAGS} ${MICROPY_FROZEN_MANIFEST}
+        COMMAND ${Python3_EXECUTABLE} ${MICROPY_DIR}/tools/makemanifest.py -o ${MICROPY_FROZEN_CONTENT} ${_manifest_var_args} -b "${CMAKE_BINARY_DIR}" ${MICROPY_CROSS_FLAGS} --mpy-tool-flags=${MICROPY_MPY_TOOL_FLAGS} ${MICROPY_FROZEN_MANIFEST}
         DEPENDS
             ${MICROPY_QSTRDEFS_GENERATED}
             ${MICROPY_ROOT_POINTERS}

py/mkrules.mk

@@ -4,6 +4,13 @@ THIS_MAKEFILE = $(lastword $(MAKEFILE_LIST))
 include $(dir $(THIS_MAKEFILE))mkenv.mk
 endif
 
+# Enable in-progress/breaking changes that are slated for MicroPython 2.x.
+MICROPY_PREVIEW_VERSION_2 ?= 0
+
+ifeq ($(MICROPY_PREVIEW_VERSION_2),1)
+CFLAGS += -DMICROPY_PREVIEW_VERSION_2=1
+endif
+
 HELP_BUILD_ERROR ?= "See \033[1;31mhttps://github.com/micropython/micropython/wiki/Build-Troubleshooting\033[0m"
 HELP_MPY_LIB_SUBMODULE ?= "\033[1;31mError: micropython-lib submodule is not initialized.\033[0m Run 'make submodules'"
 
@@ -44,7 +51,7 @@ QSTR_GEN_CXXFLAGS += $(QSTR_GEN_FLAGS)
 # can be located. By following this scheme, it allows a single build rule
 # to be used to compile all .c files.
 
-# CIRCUITPY-CHANGE: adds STEPECHO
+# CIRCUITPY-CHANGE: use STEPECHO
 vpath %.S . $(TOP) $(USER_C_MODULES)
 $(BUILD)/%.o: %.S
 	$(STEPECHO) "CC $<"
@@ -53,11 +60,12 @@ $(BUILD)/%.o: %.S
 vpath %.s . $(TOP) $(USER_C_MODULES)
 $(BUILD)/%.o: %.s
 	$(STEPECHO) "AS $<"
-	$(Q)$(AS) -o $@ $<
+	$(Q)$(AS) $(AFLAGS) -o $@ $<
 
+# CIRCUITPY-CHANGE: use STEPECHO
 define compile_c
 $(STEPECHO) "CC $<"
-$(Q)$(CC) $(CFLAGS) -c -MD -o $@ $<
+$(Q)$(CC) $(CFLAGS) -c -MD -MF $(@:.o=.d) -o $@ $< || (echo -e $(HELP_BUILD_ERROR); false)
 @# The following fixes the dependency file.
 @# See http://make.paulandlesley.org/autodep.html for details.
 @# Regex adjusted from the above to play better with Windows paths, etc.
@@ -69,7 +77,7 @@ endef
 
 define compile_cxx
 $(ECHO) "CXX $<"
-$(Q)$(CXX) $(CXXFLAGS) -c -MD -o $@ $< || (echo -e $(HELP_BUILD_ERROR); false)
+$(Q)$(CXX) $(CXXFLAGS) -c -MD -MF $(@:.o=.d) -o $@ $< || (echo -e $(HELP_BUILD_ERROR); false)
 @# The following fixes the dependency file.
 @# See http://make.paulandlesley.org/autodep.html for details.
 @# Regex adjusted from the above to play better with Windows paths, etc.
@@ -88,6 +96,7 @@ vpath %.cpp . $(TOP) $(USER_C_MODULES)
 $(BUILD)/%.o: %.cpp
 	$(call compile_cxx)
 
+# CIRCUITPY-CHANGE: use STEPECHO
 $(BUILD)/%.pp: %.c
 	$(STEPECHO) "PreProcess $<"
 	$(Q)$(CPP) $(CFLAGS) -Wp,-C,-dD,-dI -o $@ $<
@@ -108,6 +117,7 @@ $(OBJ): | $(HEADER_BUILD)/qstrdefs.generated.h $(HEADER_BUILD)/mpversion.h $(OBJ
 # - else, if list of newer prerequisites ($?) is not empty, then process just these ($?)
 # - else, process all source files ($^) [this covers "make -B" which can set $? to empty]
 # See more information about this process in docs/develop/qstr.rst.
+# CIRCUITPY-CHANGE: use STEPECHO in multiple rules below
 $(HEADER_BUILD)/qstr.i.last: $(SRC_QSTR) $(QSTR_GLOBAL_DEPENDENCIES) | $(QSTR_GLOBAL_REQUIREMENTS)
 	$(STEPECHO) "GEN $@"
 	$(Q)$(PYTHON) $(PY_SRC)/makeqstrdefs.py pp $(CPP) output $(HEADER_BUILD)/qstr.i.last cflags $(QSTR_GEN_CFLAGS) cxxflags $(QSTR_GEN_CXXFLAGS) sources $^ dependencies $(QSTR_GLOBAL_DEPENDENCIES) changed_sources $?
@@ -158,6 +168,7 @@ $(HEADER_BUILD)/compressed.collected: $(HEADER_BUILD)/compressed.split
 # will be created if they don't exist.
 OBJ_DIRS = $(sort $(dir $(OBJ)))
 $(OBJ): | $(OBJ_DIRS)
+// CIRCUITPY-CHANGE: use $(Q)
 $(OBJ_DIRS):
 	$(Q)$(MKDIR) -p $@
 
@@ -190,8 +201,8 @@ CFLAGS += -DMICROPY_QSTR_EXTRA_POOL=mp_qstr_frozen_const_pool
 CFLAGS += -DMICROPY_MODULE_FROZEN_MPY
 CFLAGS += -DMICROPY_MODULE_FROZEN_STR
 
-# to build frozen_content.c from a manifest
 # CIRCUITPY-CHANGE: FROZEN_MANIFEST is constructed at build time
+# to build frozen_content.c from a manifest
 $(BUILD)/frozen_content.c: FORCE $(BUILD)/genhdr/qstrdefs.generated.h $(BUILD)/genhdr/root_pointers.h $(FROZEN_MANIFEST) | $(MICROPY_MPYCROSS_DEPENDENCY)
 	$(Q)test -e "$(MPY_LIB_DIR)/README.md" || (echo -e $(HELP_MPY_LIB_SUBMODULE); false)
 	$(Q)$(MAKE_MANIFEST) -o $@ -v "MPY_DIR=$(TOP)" -v "MPY_LIB_DIR=$(MPY_LIB_DIR)" -v "PORT_DIR=$(shell pwd)" -v "BOARD_DIR=$(BOARD_DIR)" -b "$(BUILD)" $(if $(MPY_CROSS_FLAGS),-f"$(MPY_CROSS_FLAGS)",) --mpy-tool-flags="$(MPY_TOOL_FLAGS)" $(FROZEN_MANIFEST)

py/modbuiltins.c

@@ -170,7 +170,7 @@ STATIC mp_obj_t mp_builtin_dir(size_t n_args, const mp_obj_t *args) {
         // Implemented by probing all possible qstrs with mp_load_method_maybe
         size_t nqstr = QSTR_TOTAL();
         for (size_t i = MP_QSTR_ + 1; i < nqstr; ++i) {
-            // CIRCUITPY-CHANGE: changes PR #6539
+            // CIRCUITPY-CHANGE: changes PR #6539: catch dir() exceptions
             mp_obj_t dest[2] = {};
             mp_load_method_protected(args[0], i, dest, true);
             if (dest[0] != MP_OBJ_NULL) {
@@ -740,6 +740,7 @@ STATIC const mp_rom_map_elem_t mp_module_builtins_globals_table[] = {
     { MP_ROM_QSTR(MP_QSTR_IndentationError), MP_ROM_PTR(&mp_type_IndentationError) },
     { MP_ROM_QSTR(MP_QSTR_IndexError), MP_ROM_PTR(&mp_type_IndexError) },
     { MP_ROM_QSTR(MP_QSTR_KeyboardInterrupt), MP_ROM_PTR(&mp_type_KeyboardInterrupt) },
+    // CIRCUITPY-CHANGE: add ReloadException     
     { MP_ROM_QSTR(MP_QSTR_ReloadException), MP_ROM_PTR(&mp_type_ReloadException) },
     { MP_ROM_QSTR(MP_QSTR_KeyError), MP_ROM_PTR(&mp_type_KeyError) },
     { MP_ROM_QSTR(MP_QSTR_LookupError), MP_ROM_PTR(&mp_type_LookupError) },
@@ -747,6 +748,7 @@ STATIC const mp_rom_map_elem_t mp_module_builtins_globals_table[] = {
     { MP_ROM_QSTR(MP_QSTR_NameError), MP_ROM_PTR(&mp_type_NameError) },
     { MP_ROM_QSTR(MP_QSTR_NotImplementedError), MP_ROM_PTR(&mp_type_NotImplementedError) },
     { MP_ROM_QSTR(MP_QSTR_OSError), MP_ROM_PTR(&mp_type_OSError) },
+    // CIRCUITPY-CHANGE: add TimeoutEror ConnectionError, BrokenPipeError     
     { MP_ROM_QSTR(MP_QSTR_TimeoutError), MP_ROM_PTR(&mp_type_TimeoutError) },
     { MP_ROM_QSTR(MP_QSTR_ConnectionError), MP_ROM_PTR(&mp_type_ConnectionError) },
     { MP_ROM_QSTR(MP_QSTR_BrokenPipeError), MP_ROM_PTR(&mp_type_BrokenPipeError) },
@@ -767,6 +769,7 @@ STATIC const mp_rom_map_elem_t mp_module_builtins_globals_table[] = {
     { MP_ROM_QSTR(MP_QSTR_ViperTypeError), MP_ROM_PTR(&mp_type_ViperTypeError) },
     #endif
     { MP_ROM_QSTR(MP_QSTR_ZeroDivisionError), MP_ROM_PTR(&mp_type_ZeroDivisionError) },
+    // CIRCUITYPY-CHANGE
     #if CIRCUITPY_WARNINGS
     { MP_ROM_QSTR(MP_QSTR_Warning), MP_ROM_PTR(&mp_type_Warning) },
     { MP_ROM_QSTR(MP_QSTR_FutureWarning), MP_ROM_PTR(&mp_type_FutureWarning) },

py/moderrno.c

@@ -121,6 +121,7 @@ qstr mp_errno_to_str(mp_obj_t errno_val) {
     #endif
 }
 
+// CIRCUITPY-CHANGE
 // For commonly encountered errors, return human readable strings, otherwise try errno name
 const char *mp_common_errno_to_str(mp_obj_t errno_val, char *buf, size_t len) {
     if (!mp_obj_is_small_int(errno_val)) {

py/modmicropython.c

@@ -37,6 +37,7 @@
 // Various builtins specific to MicroPython runtime,
 // living in micropython module
 
+// CIRCUITPY-CHANGE: avoid warning
 #if CIRCUITPY_MICROPYTHON_ADVANCED && MICROPY_ENABLE_COMPILER
 STATIC mp_obj_t mp_micropython_opt_level(size_t n_args, const mp_obj_t *args) {
     if (n_args == 0) {
@@ -49,6 +50,7 @@ STATIC mp_obj_t mp_micropython_opt_level(size_t n_args, const mp_obj_t *args) {
 STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_micropython_opt_level_obj, 0, 1, mp_micropython_opt_level);
 #endif
 
+// CIRCUITPY-CHANGE: avoid warning
 #if CIRCUITPY_MICROPYTHON_ADVANCED && MICROPY_PY_MICROPYTHON_MEM_INFO
 
 #if MICROPY_MEM_STATS
@@ -109,6 +111,7 @@ STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_micropython_qstr_info_obj, 0, 1, m
 
 #endif // MICROPY_PY_MICROPYTHON_MEM_INFO
 
+// CIRCUITPY-CHANGE: avoid warning
 #if CIRCUITPY_MICROPYTHON_ADVANCED && MICROPY_PY_MICROPYTHON_STACK_USE
 STATIC mp_obj_t mp_micropython_stack_use(void) {
     return MP_OBJ_NEW_SMALL_INT(mp_stack_usage());
@@ -116,6 +119,7 @@ STATIC mp_obj_t mp_micropython_stack_use(void) {
 STATIC MP_DEFINE_CONST_FUN_OBJ_0(mp_micropython_stack_use_obj, mp_micropython_stack_use);
 #endif
 
+// CIRCUITPY-CHANGE: avoid warning
 #if CIRCUITPY_MICROPYTHON_ADVANCED && MICROPY_ENABLE_PYSTACK
 STATIC mp_obj_t mp_micropython_pystack_use(void) {
     return MP_OBJ_NEW_SMALL_INT(mp_pystack_usage());
@@ -123,6 +127,7 @@ STATIC mp_obj_t mp_micropython_pystack_use(void) {
 STATIC MP_DEFINE_CONST_FUN_OBJ_0(mp_micropython_pystack_use_obj, mp_micropython_pystack_use);
 #endif
 
+// CIRCUITPY-CHANGE: avoid warning
 #if CIRCUITPY_MICROPYTHON_ADVANCED && MICROPY_ENABLE_GC
 STATIC mp_obj_t mp_micropython_heap_lock(void) {
     gc_lock();
@@ -144,10 +149,12 @@ STATIC MP_DEFINE_CONST_FUN_OBJ_0(mp_micropython_heap_locked_obj, mp_micropython_
 #endif
 #endif
 
+// CIRCUITPY-CHANGE: avoid warning
 #if CIRCUITPY_MICROPYTHON_ADVANCED && MICROPY_ENABLE_EMERGENCY_EXCEPTION_BUF && (MICROPY_EMERGENCY_EXCEPTION_BUF_SIZE == 0)
 STATIC MP_DEFINE_CONST_FUN_OBJ_1(mp_alloc_emergency_exception_buf_obj, mp_alloc_emergency_exception_buf);
 #endif
 
+// CIRCUITPY-CHANGE: avoid warning
 #if CIRCUITPY_MICROPYTHON_ADVANCED && MICROPY_KBD_EXCEPTION
 STATIC mp_obj_t mp_micropython_kbd_intr(mp_obj_t int_chr_in) {
     mp_hal_set_interrupt_char(mp_obj_get_int(int_chr_in));
@@ -169,9 +176,11 @@ STATIC MP_DEFINE_CONST_FUN_OBJ_2(mp_micropython_schedule_obj, mp_micropython_sch
 STATIC const mp_rom_map_elem_t mp_module_micropython_globals_table[] = {
     { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_micropython) },
     { MP_ROM_QSTR(MP_QSTR_const), MP_ROM_PTR(&mp_identity_obj) },
+    // CIRCUITPY-CHANGE: avoid warning
     #if CIRCUITPY_MICROPYTHON_ADVANCED && MICROPY_ENABLE_COMPILER
     { MP_ROM_QSTR(MP_QSTR_opt_level), MP_ROM_PTR(&mp_micropython_opt_level_obj) },
     #endif
+    // CIRCUITPY-CHANGE: avoid warning
     #if CIRCUITPY_MICROPYTHON_ADVANCED && MICROPY_PY_MICROPYTHON_MEM_INFO
     #if MICROPY_MEM_STATS
     { MP_ROM_QSTR(MP_QSTR_mem_total), MP_ROM_PTR(&mp_micropython_mem_total_obj) },
@@ -181,15 +190,19 @@ STATIC const mp_rom_map_elem_t mp_module_micropython_globals_table[] = {
     { MP_ROM_QSTR(MP_QSTR_mem_info), MP_ROM_PTR(&mp_micropython_mem_info_obj) },
     { MP_ROM_QSTR(MP_QSTR_qstr_info), MP_ROM_PTR(&mp_micropython_qstr_info_obj) },
     #endif
+    // CIRCUITPY-CHANGE: avoid warning
     #if CIRCUITPY_MICROPYTHON_ADVANCED && MICROPY_PY_MICROPYTHON_STACK_USE
     { MP_ROM_QSTR(MP_QSTR_stack_use), MP_ROM_PTR(&mp_micropython_stack_use_obj) },
     #endif
+    // CIRCUITPY-CHANGE: avoid warning
     #if CIRCUITPY_MICROPYTHON_ADVANCED && MICROPY_ENABLE_EMERGENCY_EXCEPTION_BUF && (MICROPY_EMERGENCY_EXCEPTION_BUF_SIZE == 0)
     { MP_ROM_QSTR(MP_QSTR_alloc_emergency_exception_buf), MP_ROM_PTR(&mp_alloc_emergency_exception_buf_obj) },
     #endif
+    // CIRCUITPY-CHANGE: avoid warning
     #if CIRCUITPY_MICROPYTHON_ADVANCED && MICROPY_ENABLE_PYSTACK
     { MP_ROM_QSTR(MP_QSTR_pystack_use), MP_ROM_PTR(&mp_micropython_pystack_use_obj) },
     #endif
+    // CIRCUITPY-CHANGE: avoid warning
     #if CIRCUITPY_MICROPYTHON_ADVANCED && MICROPY_ENABLE_GC
     { MP_ROM_QSTR(MP_QSTR_heap_lock), MP_ROM_PTR(&mp_micropython_heap_lock_obj) },
     { MP_ROM_QSTR(MP_QSTR_heap_unlock), MP_ROM_PTR(&mp_micropython_heap_unlock_obj) },
@@ -197,6 +210,7 @@ STATIC const mp_rom_map_elem_t mp_module_micropython_globals_table[] = {
     { MP_ROM_QSTR(MP_QSTR_heap_locked), MP_ROM_PTR(&mp_micropython_heap_locked_obj) },
     #endif
     #endif
+    // CIRCUITPY-CHANGE: avoid warning
     #if CIRCUITPY_MICROPYTHON_ADVANCED && MICROPY_KBD_EXCEPTION
     { MP_ROM_QSTR(MP_QSTR_kbd_intr), MP_ROM_PTR(&mp_micropython_kbd_intr_obj) },
     #endif

py/modsys.c

@@ -46,6 +46,7 @@
 
 #if MICROPY_PY_SYS
 
+// CIRCUITPY-CHANGE
 #include "genhdr/mpversion.h"
 
 // defined per port; type of these is irrelevant, just need pointer
@@ -61,31 +62,55 @@ const mp_print_t mp_sys_stdout_print = {&mp_sys_stdout_obj, mp_stream_write_adap
 STATIC const MP_DEFINE_STR_OBJ(mp_sys_version_obj, "3.4.0; " MICROPY_BANNER_NAME_AND_VERSION);
 
 // version_info - Python language version that this implementation conforms to, as a tuple of ints
-#define I(n) MP_OBJ_NEW_SMALL_INT(n)
+// TODO: CPython is now at 5-element array (major, minor, micro, releaselevel, serial), but save 2 els so far...
-// TODO: CPython is now at 5-element array, but save 2 els so far...
+STATIC const mp_rom_obj_tuple_t mp_sys_version_info_obj = {{&mp_type_tuple}, 3, {MP_ROM_INT(3), MP_ROM_INT(4), MP_ROM_INT(0)}};
-STATIC const mp_obj_tuple_t mp_sys_version_info_obj = {{&mp_type_tuple}, 3, {I(3), I(4), I(0)}};
 
 // sys.implementation object
 // this holds the MicroPython version
-STATIC const mp_obj_tuple_t mp_sys_implementation_version_info_obj = {
+STATIC const mp_rom_obj_tuple_t mp_sys_implementation_version_info_obj = {
     {&mp_type_tuple},
-    3,
+    4,
-    { I(MICROPY_VERSION_MAJOR), I(MICROPY_VERSION_MINOR), I(MICROPY_VERSION_MICRO) }
+    {
+        MP_ROM_INT(MICROPY_VERSION_MAJOR),
+        MP_ROM_INT(MICROPY_VERSION_MINOR),
+        MP_ROM_INT(MICROPY_VERSION_MICRO),
+        #if MICROPY_VERSION_PRERELEASE
+        MP_ROM_QSTR(MP_QSTR_preview),
+        #else
+        MP_ROM_QSTR(MP_QSTR_),
+        #endif
+    }
 };
 STATIC const MP_DEFINE_STR_OBJ(mp_sys_implementation_machine_obj, MICROPY_BANNER_MACHINE);
-#if MICROPY_PERSISTENT_CODE_LOAD
+#define SYS_IMPLEMENTATION_ELEMS_BASE \
-#define SYS_IMPLEMENTATION_ELEMS \
-    MP_ROM_QSTR(MP_QSTR_circuitpython), \
-    MP_ROM_PTR(&mp_sys_implementation_version_info_obj), \
-    MP_ROM_PTR(&mp_sys_implementation_machine_obj), \
-    MP_ROM_INT(MPY_FILE_HEADER_INT)
-#else
-#define SYS_IMPLEMENTATION_ELEMS \
     MP_ROM_QSTR(MP_QSTR_circuitpython), \
     MP_ROM_PTR(&mp_sys_implementation_version_info_obj), \
     MP_ROM_PTR(&mp_sys_implementation_machine_obj)
+
+#if MICROPY_PERSISTENT_CODE_LOAD
+#define SYS_IMPLEMENTATION_ELEMS__MPY \
+    , MP_ROM_INT(MPY_FILE_HEADER_INT)
+#else
+#define SYS_IMPLEMENTATION_ELEMS \
+    MP_ROM_QSTR(MP_QSTR_micropython), \
+    MP_ROM_PTR(&mp_sys_implementation_version_info_obj), \
+    MP_ROM_PTR(&mp_sys_implementation_machine_obj)
+
+#if MICROPY_PERSISTENT_CODE_LOAD
+#define SYS_IMPLEMENTATION_ELEMS__MPY \
+    , MP_ROM_INT(MPY_FILE_HEADER_INT)
+#else
+#define SYS_IMPLEMENTATION_ELEMS__MPY
 #endif
+
 #if MICROPY_PY_ATTRTUPLE
+#if MICROPY_PREVIEW_VERSION_2
+#define SYS_IMPLEMENTATION_ELEMS__V2 \
+    , MP_ROM_TRUE
+#else
+#define SYS_IMPLEMENTATION_ELEMS__V2
+#endif
+
 STATIC const qstr impl_fields[] = {
     MP_QSTR_name,
     MP_QSTR_version,
@@ -93,19 +118,30 @@ STATIC const qstr impl_fields[] = {
     #if MICROPY_PERSISTENT_CODE_LOAD
     MP_QSTR__mpy,
     #endif
+    #if MICROPY_PREVIEW_VERSION_2
+    MP_QSTR__v2,
+    #endif
 };
 STATIC MP_DEFINE_ATTRTUPLE(
     mp_sys_implementation_obj,
     impl_fields,
-    3 + MICROPY_PERSISTENT_CODE_LOAD,
+    3 + MICROPY_PERSISTENT_CODE_LOAD + MICROPY_PREVIEW_VERSION_2,
-    SYS_IMPLEMENTATION_ELEMS
+    SYS_IMPLEMENTATION_ELEMS_BASE
+    SYS_IMPLEMENTATION_ELEMS__MPY
+    SYS_IMPLEMENTATION_ELEMS__V2
     );
 #else
 STATIC const mp_rom_obj_tuple_t mp_sys_implementation_obj = {
     {&mp_type_tuple},
     3 + MICROPY_PERSISTENT_CODE_LOAD,
+    // Do not include SYS_IMPLEMENTATION_ELEMS__V2 because
+    // SYS_IMPLEMENTATION_ELEMS__MPY may be empty if
+    // MICROPY_PERSISTENT_CODE_LOAD is disabled, which means they'll share
+    // the same index. Cannot query _v2 if MICROPY_PY_ATTRTUPLE is
+    // disabled.
     {
-        SYS_IMPLEMENTATION_ELEMS
+        SYS_IMPLEMENTATION_ELEMS_BASE
+                                SYS_IMPLEMENTATION_ELEMS__MPY
     }
 };
 #endif
@@ -123,6 +159,10 @@ STATIC const MP_DEFINE_STR_OBJ(mp_sys_platform_obj, MICROPY_PY_SYS_PLATFORM);
 MP_DEFINE_STR_OBJ(mp_sys_executable_obj, "");
 #endif
 
+#if MICROPY_PY_SYS_INTERN
+MP_DEFINE_CONST_FUN_OBJ_1(mp_sys_intern_obj, mp_obj_str_intern_checked);
+#endif
+
 // exit([retval]): raise SystemExit, with optional argument given to the exception
 STATIC mp_obj_t mp_sys_exit(size_t n_args, const mp_obj_t *args) {
     if (n_args == 0) {
@@ -270,6 +310,10 @@ STATIC const mp_rom_map_elem_t mp_module_sys_globals_table[] = {
     #endif
     #endif
 
+    #if MICROPY_PY_SYS_INTERN
+    { MP_ROM_QSTR(MP_QSTR_intern), MP_ROM_PTR(&mp_sys_intern_obj) },
+    #endif
+
     #if MICROPY_PY_SYS_EXIT
     { MP_ROM_QSTR(MP_QSTR_exit), MP_ROM_PTR(&mp_sys_exit_obj) },
     #endif

py/modthread.c

@@ -175,6 +175,7 @@ STATIC void *thread_entry(void *args_in) {
     // The GC starts off unlocked on this thread.
     ts.gc_lock_depth = 0;
 
+    ts.nlr_jump_callback_top = NULL;
     ts.mp_pending_exception = MP_OBJ_NULL;
 
     // set locals and globals from the calling context
@@ -235,7 +236,7 @@ STATIC mp_obj_t mod_thread_start_new_thread(size_t n_args, const mp_obj_t *args)
     // check for keyword arguments
     if (n_args == 2) {
         // just position arguments
-        th_args = m_new_obj_var(thread_entry_args_t, mp_obj_t, pos_args_len);
+        th_args = m_new_obj_var(thread_entry_args_t, args, mp_obj_t, pos_args_len);
         th_args->n_kw = 0;
     } else {
         // positional and keyword arguments
@@ -243,7 +244,7 @@ STATIC mp_obj_t mod_thread_start_new_thread(size_t n_args, const mp_obj_t *args)
             mp_raise_TypeError(MP_ERROR_TEXT("expecting a dict for keyword args"));
         }
         mp_map_t *map = &((mp_obj_dict_t *)MP_OBJ_TO_PTR(args[2]))->map;
-        th_args = m_new_obj_var(thread_entry_args_t, mp_obj_t, pos_args_len + 2 * map->used);
+        th_args = m_new_obj_var(thread_entry_args_t, args, mp_obj_t, pos_args_len + 2 * map->used);
         th_args->n_kw = map->used;
         // copy across the keyword arguments
         for (size_t i = 0, n = pos_args_len; i < map->alloc; ++i) {

py/mpconfig.h

@@ -26,6 +26,7 @@
 #ifndef MICROPY_INCLUDED_PY_MPCONFIG_H
 #define MICROPY_INCLUDED_PY_MPCONFIG_H
 
+// CIRCUITPY-CHANGE
 // Is this a CircuitPython build?
 #ifndef CIRCUITPY
 #define CIRCUITPY 0
@@ -37,22 +38,34 @@
 #else
 // Current version of MicroPython
 #define MICROPY_VERSION_MAJOR 1
-#define MICROPY_VERSION_MINOR 21
+#define MICROPY_VERSION_MINOR 22
-#define MICROPY_VERSION_MICRO 0
+#define MICROPY_VERSION_MICRO 2
+#define MICROPY_VERSION_PRERELEASE 0
 
-// Combined version as a 32-bit number for convenience
+// Combined version as a 32-bit number for convenience to allow version
-#define MICROPY_VERSION ( \
+// comparison. Doesn't include prerelease state.
-    MICROPY_VERSION_MAJOR << 16 \
+// e.g. #if MICROPY_VERSION < MICROPY_MAKE_VERSION(1, 22, 0)
-        | MICROPY_VERSION_MINOR << 8 \
+#define MICROPY_MAKE_VERSION(major, minor, patch) (major << 16 | minor << 8 | patch)
-        | MICROPY_VERSION_MICRO)
+#define MICROPY_VERSION MICROPY_MAKE_VERSION(MICROPY_VERSION_MAJOR, MICROPY_VERSION_MINOR, MICROPY_VERSION_MICRO)
 
-// String version
+// String version. This is only used directly for platform.platform and
-#define MICROPY_VERSION_STRING \
+// os.uname().release. All other version info available in the firmware (e.g.
+// the REPL banner) comes from MICROPY_GIT_TAG.
+#define MICROPY_VERSION_STRING_BASE \
     MP_STRINGIFY(MICROPY_VERSION_MAJOR) "." \
     MP_STRINGIFY(MICROPY_VERSION_MINOR) "." \
     MP_STRINGIFY(MICROPY_VERSION_MICRO)
+#if MICROPY_VERSION_PRERELEASE
+#define MICROPY_VERSION_STRING MICROPY_VERSION_STRING_BASE "-preview"
+#else
+#define MICROPY_VERSION_STRING MICROPY_VERSION_STRING_BASE
 #endif
 
+// If this is enabled, then in-progress/breaking changes slated for the 2.x
+// release will be enabled.
+#ifndef MICROPY_PREVIEW_VERSION_2
+#define MICROPY_PREVIEW_VERSION_2 (0)
+#endif
 
 // This file contains default configuration settings for MicroPython.
 // You can override any of the options below using mpconfigport.h file
@@ -1324,11 +1337,13 @@ typedef double mp_float_t;
 #define MICROPY_PY_COLLECTIONS_DEQUE (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES)
 #endif
 
+// CIRCUITPY-CHANGE
 // Whether "collections.deque" supports iteration
 #ifndef MICROPY_PY_COLLECTIONS_DEQUE_ITER
 #define MICROPY_PY_COLLECTIONS_DEQUE_ITER (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES)
 #endif
 
+// CIRCUITPY-CHANGE
 // Whether "collections.deque" supports subscription
 #ifndef MICROPY_PY_COLLECTIONS_DEQUE_SUBSCR
 #define MICROPY_PY_COLLECTIONS_DEQUE_SUBSCR (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES)
@@ -1473,6 +1488,11 @@ typedef double mp_float_t;
 #define MICROPY_PY_SYS_EXECUTABLE (0)
 #endif
 
+// Whether to provide "sys.intern"
+#ifndef MICROPY_PY_SYS_INTERN
+#define MICROPY_PY_SYS_INTERN (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EVERYTHING)
+#endif
+
 // Whether to provide "sys.exit" function
 #ifndef MICROPY_PY_SYS_EXIT
 #define MICROPY_PY_SYS_EXIT (1)
@@ -1879,8 +1899,12 @@ typedef double mp_float_t;
 
 // String used for the banner, and sys.version additional information
 #ifndef MICROPY_BANNER_NAME_AND_VERSION
+#if MICROPY_PREVIEW_VERSION_2
+#define MICROPY_BANNER_NAME_AND_VERSION "MicroPython (with v2.0 preview) " MICROPY_GIT_TAG " on " MICROPY_BUILD_DATE
+#else
 #define MICROPY_BANNER_NAME_AND_VERSION "CircuitPython " MICROPY_GIT_TAG " on " MICROPY_BUILD_DATE
 #endif
+#endif
 
 // String used for the second part of the banner, and sys.implementation._machine
 #ifndef MICROPY_BANNER_MACHINE
@@ -1891,14 +1915,6 @@ typedef double mp_float_t;
 #endif
 #endif
 
-// On embedded platforms, these will typically enable/disable irqs.
-#ifndef MICROPY_BEGIN_ATOMIC_SECTION
-#define MICROPY_BEGIN_ATOMIC_SECTION() (0)
-#endif
-#ifndef MICROPY_END_ATOMIC_SECTION
-#define MICROPY_END_ATOMIC_SECTION(state) (void)(state)
-#endif
-
 // Allow to override static modifier for global objects, e.g. to use with
 // object code analysis tools which don't support static symbols.
 #ifndef STATIC

py/mperrno.h

@@ -27,6 +27,7 @@
 #define MICROPY_INCLUDED_PY_MPERRNO_H
 
 #include "py/mpconfig.h"
+// CIRCUITPY-CHANGE
 #include "py/obj.h"
 
 #if MICROPY_USE_INTERNAL_ERRNO

py/mphal.h

@@ -35,6 +35,14 @@
 #include <mphalport.h>
 #endif
 
+// On embedded platforms, these will typically enable/disable irqs.
+#ifndef MICROPY_BEGIN_ATOMIC_SECTION
+#define MICROPY_BEGIN_ATOMIC_SECTION() (0)
+#endif
+#ifndef MICROPY_END_ATOMIC_SECTION
+#define MICROPY_END_ATOMIC_SECTION(state) (void)(state)
+#endif
+
 #ifndef mp_hal_stdio_poll
 uintptr_t mp_hal_stdio_poll(uintptr_t poll_flags);
 #endif
@@ -48,7 +56,7 @@ void mp_hal_stdout_tx_str(const char *str);
 #endif
 
 #ifndef mp_hal_stdout_tx_strn
-void mp_hal_stdout_tx_strn(const char *str, size_t len);
+mp_uint_t mp_hal_stdout_tx_strn(const char *str, size_t len);
 #endif
 
 #ifndef mp_hal_stdout_tx_strn_cooked
@@ -90,4 +98,17 @@ uint64_t mp_hal_time_ns(void);
 #include "extmod/virtpin.h"
 #endif
 
+// Event handling and wait-for-event functions.
+
+#ifndef MICROPY_INTERNAL_WFE
+// Fallback definition for ports that don't need to suspend the CPU.
+#define MICROPY_INTERNAL_WFE(TIMEOUT_MS) (void)0
+#endif
+
+#ifndef MICROPY_INTERNAL_EVENT_HOOK
+// Fallback definition for ports that don't need any port-specific
+// non-blocking event processing.
+#define MICROPY_INTERNAL_EVENT_HOOK (void)0
+#endif
+
 #endif // MICROPY_INCLUDED_PY_MPHAL_H