Merge pull request #8720 from jepler/codeop-compile_command

Add codeop.compile_command

.github/actions/deps/external/action.yml

@@ -25,6 +25,7 @@ runs:
         inputs.port != 'espressif'
       uses: carlosperate/arm-none-eabi-gcc-action@v1
       with:
+        # When changing this update what Windows grabs too!
         release: '13.2.Rel1'
 
     # espressif

.github/workflows/build.yml

@@ -249,9 +249,9 @@ jobs:
         python3 -c "import sys, locale; print(sys.getdefaultencoding(), locale.getpreferredencoding(False))"
     - name: Install dependencies
       run: |
-        wget --no-verbose -O gcc-arm.zip https://developer.arm.com/-/media/Files/downloads/gnu-rm/10-2020q4/gcc-arm-none-eabi-10-2020-q4-major-win32.zip
+        wget --no-verbose -O gcc-arm.zip https://developer.arm.com/-/media/Files/downloads/gnu/13.2.rel1/binrel/arm-gnu-toolchain-13.2.rel1-mingw-w64-i686-arm-none-eabi.zip
         unzip -q -d /tmp gcc-arm.zip
-        tar -C /tmp/gcc-arm-none-* -cf - . | tar -C /usr/local -xf -
+        tar -C /tmp/arm-gnu-toolchain* -cf - . | tar -C /usr/local -xf -
         pip install wheel
         # requirements_dev.txt doesn't install on windows. (with msys2 python)
         # instead, pick a subset for what we want to do

ports/unix/variants/coverage/mpconfigvariant.mk

@@ -35,6 +35,7 @@ SRC_BITMAP := \
 	shared-bindings/audiomixer/Mixer.c \
 	shared-bindings/audiomixer/MixerVoice.c \
 	shared-bindings/bitmaptools/__init__.c \
+	shared-bindings/codeop/__init__.c \
 	shared-bindings/displayio/Bitmap.c \
 	shared-bindings/jpegio/__init__.c \
 	shared-bindings/jpegio/JpegDecoder.c \
@@ -87,6 +88,7 @@ CFLAGS += \
 	-DCIRCUITPY_AUDIOMIXER=1 \
 	-DCIRCUITPY_AUDIOCORE_DEBUG=1 \
 	-DCIRCUITPY_BITMAPTOOLS=1 \
+	-DCIRCUITPY_CODEOP=1 \
 	-DCIRCUITPY_DISPLAYIO_UNIX=1 \
 	-DCIRCUITPY_FUTURE=1 \
 	-DCIRCUITPY_GIFIO=1 \

py/circuitpy_defns.mk

@@ -168,6 +168,9 @@ endif
 ifeq ($(CIRCUITPY_CANIO),1)
 SRC_PATTERNS += canio/%
 endif
+ifeq ($(CIRCUITPY_CODEOP),1)
+SRC_PATTERNS += codeop/%
+endif
 ifeq ($(CIRCUITPY_COUNTIO),1)
 SRC_PATTERNS += countio/%
 endif
@@ -547,6 +550,7 @@ $(filter $(SRC_PATTERNS), \
 	__future__/__init__.c \
 	camera/ImageFormat.c \
 	canio/Match.c \
+	codeop/__init__.c \
 	countio/Edge.c \
 	digitalio/Direction.c \
 	digitalio/DriveMode.c \

py/circuitpy_mpconfig.mk

@@ -182,6 +182,9 @@ CFLAGS += -DCIRCUITPY_CAMERA=$(CIRCUITPY_CAMERA)
 CIRCUITPY_CANIO ?= 0
 CFLAGS += -DCIRCUITPY_CANIO=$(CIRCUITPY_CANIO)
 
+CIRCUITPY_CODEOP ?= $(CIRCUITPY_FULL_BUILD)
+CFLAGS += -DCIRCUITPY_CODEOP=$(CIRCUITPY_CODEOP)
+
 CIRCUITPY_COLLECTIONS ?= 1
 CFLAGS += -DCIRCUITPY_COLLECTIONS=$(CIRCUITPY_COLLECTIONS)
 

py/makeqstrdata.py

@@ -250,6 +250,73 @@ static_qstr_list = [
     "zip",
 ]
 
+# These qstrs have to be sorted early (preferably right after static_qstr_list) because they are required to fit in 8-bit values
+# however they should never be *forced* to appear
+# repeats len, hash, int from the static qstr list, but this doesn't hurt anything.
+eightbit_qstr_list = [
+    "__abs__",
+    "__add__",
+    "__and__",
+    "__bool__",
+    "__complex__",
+    "__contains__",
+    "__delete__",
+    "__divmod__",
+    "__eq__",
+    "__float__",
+    "__floordiv__",
+    "__ge__",
+    "__get__",
+    "__gt__",
+    "__hash__",
+    "__iadd__",
+    "__iand__",
+    "__ifloordiv__",
+    "__ilshift__",
+    "__imatmul__",
+    "__imod__",
+    "__imul__",
+    "__int__",
+    "__invert__",
+    "__ior__",
+    "__ipow__",
+    "__irshift__",
+    "__isub__",
+    "__itruediv__",
+    "__ixor__",
+    "__le__",
+    "__len__",
+    "__lshift__",
+    "__lt__",
+    "__matmul__",
+    "__mod__",
+    "__mul__",
+    "__ne__",
+    "__neg__",
+    "__or__",
+    "__pos__",
+    "__pow__",
+    "__radd__",
+    "__rand__",
+    "__rfloordiv__",
+    "__rlshift__",
+    "__rmatmul__",
+    "__rmod__",
+    "__rmul__",
+    "__ror__",
+    "__rpow__",
+    "__rrshift__",
+    "__rshift__",
+    "__rsub__",
+    "__rtruediv__",
+    "__rxor__",
+    "__set__",
+    "__sizeof__",
+    "__sub__",
+    "__truediv__",
+    "__xor__",
+]
+
 
 # this must match the equivalent function in qstr.c
 def compute_hash(qstr, bytes_hash):
@@ -337,18 +404,7 @@ def parse_input_headers_with_translations(infiles):
                 order = len(qstrs)
                 # but put special method names like __add__ at the top of list, so
                 # that their id's fit into a byte
-                if ident == "":
-                    # Sort empty qstr above all still
-                    order = -200000
-                elif ident == "__dir__":
-                    # Put __dir__ after empty qstr for builtin dir() to work
-                    order = -190000
-                elif ident.startswith("__"):
-                    order -= 100000
-                # CIRCUITPY-CHANGE
-                elif ident.startswith("_lt"):
-                    order -= 100000
-                elif ident.startswith("_gt"):
+                if ident in eightbit_qstr_list:
                     order -= 100000
                 qstrs[ident] = (order, ident, qstr)
 

shared-bindings/codeop/__init__.c

@@ -0,0 +1,79 @@
+/*
+ * This file is part of the MicroPython project, http://micropython.org/
+ *
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2023 Jeff Epler for Adafruit Industries
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#include <string.h>
+
+#include "py/builtin.h"
+#include "py/obj.h"
+#include "py/runtime.h"
+#include "py/repl.h"
+
+STATIC const char *get_arg_str(mp_obj_t arg, qstr name) {
+    return mp_obj_str_get_str(mp_arg_validate_type_string(arg, name));
+}
+
+//| """Utilities to compile possibly incomplete Python source code."""
+//|
+
+//| def compile_command(source: str, filename: str = "<input>", symbol: str = "single"):
+//|     """Compile a command and determine whether it is incomplete
+//|
+//|     The 'completeness' determination is slightly different than in standard Python
+//|     (it's whatever the internal function ``mp_repl_continue_with_input`` does).
+//|     In particular, it's important that the code not end with a newline character
+//|     or it is likely to be treated as a complete command."""
+//|
+STATIC mp_obj_t compile_command(mp_uint_t n_args, const mp_obj_t *pos_args, mp_map_t *kw_args) {
+    enum { ARG_source, ARG_filename, ARG_symbol };
+    static const mp_arg_t allowed_args[] = {
+        { MP_QSTR_source, MP_ARG_REQUIRED | MP_ARG_OBJ, { .u_obj = mp_const_none } },
+        { MP_QSTR_filename, MP_ARG_OBJ, { .u_obj = MP_ROM_QSTR(MP_QSTR__lt_input_gt_) } },
+        { MP_QSTR_symbol, MP_ARG_OBJ, { .u_obj = MP_ROM_QSTR(MP_QSTR_single) } },
+    };
+    mp_arg_val_t args[MP_ARRAY_SIZE(allowed_args)];
+    mp_arg_parse_all(n_args, pos_args, kw_args, MP_ARRAY_SIZE(allowed_args), allowed_args, args);
+
+    const char *source = get_arg_str(args[ARG_source].u_obj, MP_QSTR_source);
+    if (mp_repl_continue_with_input(source)) {
+        return mp_const_none;
+    }
+
+    return mp_call_function_n_kw((mp_obj_t)&mp_builtin_compile_obj, 3, 0, &args[0].u_obj);
+}
+MP_DEFINE_CONST_FUN_OBJ_KW(compile_command_obj, 1, compile_command);
+
+STATIC const mp_rom_map_elem_t codeop_module_globals_table[] = {
+    { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_codeop) },
+    { MP_ROM_QSTR(MP_QSTR_compile_command),  MP_ROM_PTR(&compile_command_obj) },
+};
+
+STATIC MP_DEFINE_CONST_DICT(codeop_module_globals, codeop_module_globals_table);
+
+const mp_obj_module_t codeop_module = {
+    .base = { &mp_type_module },
+    .globals = (mp_obj_dict_t *)&codeop_module_globals,
+};
+
+MP_REGISTER_MODULE(MP_QSTR_codeop, codeop_module);

tests/circuitpython/codeop_compile.py

@@ -0,0 +1,26 @@
+try:
+    from codeop import compile_command
+except ImportError:
+    print("SKIP")
+    raise SystemExit
+
+
+def test(snippet):
+    result = compile_command(snippet)
+    print("None" if result is None else "<code>")
+
+
+# Complete command
+test("3+3")
+
+# Complete command
+test("if 1:\n    print('hi mom')\n")
+
+# Incomplete command
+test("if 1:")
+
+# Incomplete multiline string
+test("'''I'm sure it's OK")
+
+# Incomplete parenthesized expression
+test("[1, 2")

tests/unix/extra_coverage.py.exp

@@ -53,14 +53,14 @@ port
 builtins        micropython     __future__      _asyncio
 _thread         aesio           array           audiocore
 audiomixer      binascii        bitmaptools     cexample
-cmath           collections     cppexample      displayio
-errno           example_package                 gc
-hashlib         heapq           io              jpegio
-json            locale          math            os
-platform        qrio            rainbowio       random
-re              select          struct          synthio
-sys             time            traceback       uctypes
-ulab            zlib
+cmath           codeop          collections     cppexample
+displayio       errno           example_package
+gc              hashlib         heapq           io
+jpegio          json            locale          math
+os              platform        qrio            rainbowio
+random          re              select          struct
+synthio         sys             time            traceback
+uctypes         ulab            zlib
 me
 
 rainbowio       random