jepler/circuitpython
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

extmod/modmarshal: Add new marshal module.

Browse source 
This commit implements a small subset of the CPython `marshal` module.  It
implements `marshal.dumps()` and `marshal.loads()`, but only supports
(un)marshalling code objects at this stage.  The semantics match CPython,
except that the actual marshalled bytes is not compatible with CPython's
marshalled bytes.

The module is enabled at the everything level (only on the unix coverage
build at this stage).

Signed-off-by: Damien George <damien@micropython.org>
This commit is contained in:
Damien George 2025-01-20 22:24:10 +11:00
parent a11ba7775e
commit c3a18d74eb
9 changed files with 283 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
extmod/extmod.cmake
View file

@ -24,6 +24,7 @@ set(MICROPY_SOURCE_EXTMOD
${MICROPY_EXTMOD_DIR}/modframebuf.c
${MICROPY_EXTMOD_DIR}/modlwip.c
${MICROPY_EXTMOD_DIR}/modmachine.c
${MICROPY_EXTMOD_DIR}/modmarshal.c
${MICROPY_EXTMOD_DIR}/modnetwork.c
${MICROPY_EXTMOD_DIR}/modonewire.c
${MICROPY_EXTMOD_DIR}/modasyncio.c

1
extmod/extmod.mk
View file

@ -29,6 +29,7 @@ SRC_EXTMOD_C += \
extmod/modjson.c \
extmod/modlwip.c \
extmod/modmachine.c \
extmod/modmarshal.c \
extmod/modnetwork.c \
extmod/modonewire.c \
extmod/modopenamp.c \

88
extmod/modmarshal.c Normal file
View file

@ -0,0 +1,88 @@
/*
* This file is part of the MicroPython project, http://micropython.org/
*
* The MIT License (MIT)
*
* Copyright (c) 2025 Damien P. George
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
*
*/
#include "py/objcode.h"
#include "py/objfun.h"
#include "py/persistentcode.h"
#include "py/runtime.h"
#if MICROPY_PY_MARSHAL
static mp_obj_t marshal_dumps(mp_obj_t value_in) {
if (mp_obj_is_type(value_in, &mp_type_code)) {
mp_obj_code_t *code = MP_OBJ_TO_PTR(value_in);
const void *proto_fun = mp_code_get_proto_fun(code);
const uint8_t *bytecode;
if (mp_proto_fun_is_bytecode(proto_fun)) {
bytecode = proto_fun;
} else {
const mp_raw_code_t *rc = proto_fun;
if (!(rc->kind == MP_CODE_BYTECODE && rc->children == NULL)) {
mp_raise_ValueError(MP_ERROR_TEXT("function must be bytecode with no children"));
}
bytecode = rc->fun_data;
}
return mp_raw_code_save_fun_to_bytes(mp_code_get_constants(code), bytecode);
} else {
mp_raise_ValueError(MP_ERROR_TEXT("unmarshallable object"));
}
}
static MP_DEFINE_CONST_FUN_OBJ_1(marshal_dumps_obj, marshal_dumps);
static mp_obj_t marshal_loads(mp_obj_t data_in) {
mp_buffer_info_t bufinfo;
mp_get_buffer_raise(data_in, &bufinfo, MP_BUFFER_READ);
mp_module_context_t ctx;
ctx.module.globals = mp_globals_get();
mp_compiled_module_t cm = { .context = &ctx };
mp_raw_code_load_mem(bufinfo.buf, bufinfo.len, &cm);
#if MICROPY_PY_BUILTINS_CODE <= MICROPY_PY_BUILTINS_CODE_BASIC
return mp_obj_new_code(ctx.constants, cm.rc);
#else
mp_module_context_t *ctx_ptr = m_new_obj(mp_module_context_t);
*ctx_ptr = ctx;
return mp_obj_new_code(ctx_ptr, cm.rc, true);
#endif
}
static MP_DEFINE_CONST_FUN_OBJ_1(marshal_loads_obj, marshal_loads);
static const mp_rom_map_elem_t mod_marshal_globals_table[] = {
{ MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_marshal) },
{ MP_ROM_QSTR(MP_QSTR_dumps), MP_ROM_PTR(&marshal_dumps_obj) },
{ MP_ROM_QSTR(MP_QSTR_loads), MP_ROM_PTR(&marshal_loads_obj) },
};
static MP_DEFINE_CONST_DICT(mod_marshal_globals, mod_marshal_globals_table);
const mp_obj_module_t mp_module_marshal = {
.base = { &mp_type_module },
.globals = (mp_obj_dict_t *)&mod_marshal_globals,
};
MP_REGISTER_MODULE(MP_QSTR_marshal, mp_module_marshal);
#endif // MICROPY_PY_MARSHAL

1
ports/windows/msvc/sources.props
View file

@ -15,6 +15,7 @@
<PyExtModSource Include="$(PyBaseDir)extmod\modheapq.c" />
<PyExtModSource Include="$(PyBaseDir)extmod\modjson.c" />
<PyExtModSource Include="$(PyBaseDir)extmod\modmachine.c" />
<PyExtModSource Include="$(PyBaseDir)extmod\modmarshal.c" />
<PyExtModSource Include="$(PyBaseDir)extmod\modos.c" />
<PyExtModSource Include="$(PyBaseDir)extmod\modrandom.c" />
<PyExtModSource Include="$(PyBaseDir)extmod\modre.c" />

7
py/mpconfig.h
View file

@ -344,7 +344,7 @@
// Whether to support converting functions to persistent code (bytes)
#ifndef MICROPY_PERSISTENT_CODE_SAVE_FUN
#define MICROPY_PERSISTENT_CODE_SAVE_FUN (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EVERYTHING)
#define MICROPY_PERSISTENT_CODE_SAVE_FUN (MICROPY_PY_MARSHAL)
#endif
// Whether generated code can persist independently of the VM/runtime instance
@ -1382,6 +1382,11 @@ typedef double mp_float_t;
#define MICROPY_PY_COLLECTIONS_NAMEDTUPLE__ASDICT (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EVERYTHING)
#endif
// Whether to provide "marshal" module
#ifndef MICROPY_PY_MARSHAL
#define MICROPY_PY_MARSHAL (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EVERYTHING)
#endif
// Whether to provide "math" module
#ifndef MICROPY_PY_MATH
#define MICROPY_PY_MATH (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_CORE_FEATURES)

38
tests/extmod/marshal_basic.py Normal file
View file

@ -0,0 +1,38 @@
# Test the marshal module, basic functionality.
try:
import marshal
(lambda: 0).__code__
except (AttributeError, ImportError):
print("SKIP")
raise SystemExit
ftype = type(lambda: 0)
# Test basic dumps and loads.
print(ftype(marshal.loads(marshal.dumps((lambda: a).__code__)), {"a": 4})())
# Test dumps of a result from compile().
ftype(marshal.loads(marshal.dumps(compile("print(a)", "", "exec"))), {"print": print, "a": 5})()
# Test marshalling a function with arguments.
print(ftype(marshal.loads(marshal.dumps((lambda x, y: x + y).__code__)), {})(1, 2))
# Test marshalling a function with default arguments.
print(ftype(marshal.loads(marshal.dumps((lambda x=0: x).__code__)), {})("arg"))
# Test marshalling a function containing constant objects (a tuple).
print(ftype(marshal.loads(marshal.dumps((lambda: (None, ...)).__code__)), {})())
# Test instantiating multiple code's with different globals dicts.
code = marshal.loads(marshal.dumps((lambda: a).__code__))
f1 = ftype(code, {"a": 1})
f2 = ftype(code, {"a": 2})
print(f1(), f2())
# Test unmarshallable object.
try:
marshal.dumps(type)
except ValueError:
print("ValueError")

21
tests/extmod/marshal_micropython.py Normal file
View file

@ -0,0 +1,21 @@
# Test the marshal module, MicroPython-specific functionality.
try:
import marshal
except ImportError:
print("SKIP")
raise SystemExit
import unittest
class Test(unittest.TestCase):
def test_function_with_children(self):
# Can't marshal a function with children (in this case the module has a child function f).
code = compile("def f(): pass", "", "exec")
with self.assertRaises(ValueError):
marshal.dumps(code)
if __name__ == "__main__":
unittest.main()

122
tests/extmod/marshal_stress.py Normal file
View file

@ -0,0 +1,122 @@
# Test the marshal module, stressing edge cases.
try:
import marshal
(lambda: 0).__code__
except (AttributeError, ImportError):
print("SKIP")
raise SystemExit
ftype = type(lambda: 0)
# Test a large function.
def large_function(arg0, arg1, arg2, arg3):
# Arguments.
print(arg0, arg1, arg2, arg3)
# Positive medium-sized integer (still a small-int though).
print(1234)
# Negative small-ish integer.
print(-20)
# More than 64 constant objects.
x = (0,)
x = (1,)
x = (2,)
x = (3,)
x = (4,)
x = (5,)
x = (6,)
x = (7,)
x = (8,)
x = (9,)
x = (10,)
x = (11,)
x = (12,)
x = (13,)
x = (14,)
x = (15,)
x = (16,)
x = (17,)
x = (18,)
x = (19,)
x = (20,)
x = (21,)
x = (22,)
x = (23,)
x = (24,)
x = (25,)
x = (26,)
x = (27,)
x = (28,)
x = (29,)
x = (30,)
x = (31,)
x = (32,)
x = (33,)
x = (34,)
x = (35,)
x = (36,)
x = (37,)
x = (38,)
x = (39,)
x = (40,)
x = (41,)
x = (42,)
x = (43,)
x = (44,)
x = (45,)
x = (46,)
x = (47,)
x = (48,)
x = (49,)
x = (50,)
x = (51,)
x = (52,)
x = (53,)
x = (54,)
x = (55,)
x = (56,)
x = (57,)
x = (58,)
x = (59,)
x = (60,)
x = (61,)
x = (62,)
x = (63,)
x = (64,)
# Small jump.
x = 0
while x < 2:
print("loop", x)
x += 1
# Large jump.
x = 0
while x < 2:
try:
try:
try:
print
except Exception as e:
print
finally:
print
except Exception as e:
print
finally:
print
except Exception as e:
print
finally:
print("loop", x)
x += 1
code = marshal.dumps(large_function.__code__)
ftype(marshal.loads(code), {"print": print})(0, 1, 2, 3)

10
tests/ports/unix/extra_coverage.py.exp
View file

@ -56,13 +56,13 @@ cmath collections cppexample cryptolib
deflate errno example_package
ffi framebuf gc hashlib
heapq io json machine
math os platform random
re select socket struct
sys termios time tls
uctypes vfs websocket
marshal math os platform
random re select socket
struct sys termios time
tls uctypes vfs websocket
me
micropython machine math
micropython machine marshal math
argv atexit byteorder exc_info
executable exit getsizeof implementation