From 513253bc3f22e5afa67155f7b4ca29a770929a3e Mon Sep 17 00:00:00 2001 From: Bernhard Boser Date: Mon, 9 Nov 2020 18:09:56 -0800 Subject: [PATCH] moved logic to shared-module and added documentation --- ports/nrf/mpconfigport.mk | 2 - py/circuitpy_defns.mk | 1 + py/circuitpy_mpconfig.h | 2 +- py/circuitpy_mpconfig.mk | 2 +- shared-bindings/msgpack/__init__.c | 367 +++------------------------- shared-module/msgpack/__init__.c | 372 +++++++++++++++++++++++++++++ shared-module/msgpack/__init__.h | 34 +++ 7 files changed, 437 insertions(+), 343 deletions(-) create mode 100644 shared-module/msgpack/__init__.c create mode 100644 shared-module/msgpack/__init__.h diff --git a/ports/nrf/mpconfigport.mk b/ports/nrf/mpconfigport.mk index 2ee182b7d5..9560064fbc 100644 --- a/ports/nrf/mpconfigport.mk +++ b/ports/nrf/mpconfigport.mk @@ -52,8 +52,6 @@ MCU_SUB_VARIANT = nrf52840 # Fits on nrf52840 but space is tight on nrf52833. CIRCUITPY_AESIO ?= 1 -CIRCUITPY_MSGPACK ?= 1 - SD ?= s140 SOFTDEV_VERSION ?= 6.1.0 diff --git a/py/circuitpy_defns.mk b/py/circuitpy_defns.mk index 144168c79e..8c452c2b03 100644 --- a/py/circuitpy_defns.mk +++ b/py/circuitpy_defns.mk @@ -465,6 +465,7 @@ SRC_SHARED_MODULE_ALL = \ memorymonitor/AllocationAlarm.c \ memorymonitor/AllocationSize.c \ network/__init__.c \ + msgpack/__init__.c \ os/__init__.c \ random/__init__.c \ rgbmatrix/RGBMatrix.c \ diff --git a/py/circuitpy_mpconfig.h b/py/circuitpy_mpconfig.h index 3c4e5c2909..64fedd506c 100644 --- a/py/circuitpy_mpconfig.h +++ b/py/circuitpy_mpconfig.h @@ -815,6 +815,7 @@ extern const struct _mp_obj_module_t msgpack_module; _EVE_MODULE \ MEMORYMONITOR_MODULE \ MICROCONTROLLER_MODULE \ + MSGPACK_MODULE \ NEOPIXEL_WRITE_MODULE \ NETWORK_MODULE \ SOCKET_MODULE \ @@ -846,7 +847,6 @@ extern const struct _mp_obj_module_t msgpack_module; USTACK_MODULE \ WATCHDOG_MODULE \ WIFI_MODULE \ - MSGPACK_MODULE \ // If weak links are enabled, just include strong links in the main list of modules, // and also include the underscore alternate names. diff --git a/py/circuitpy_mpconfig.mk b/py/circuitpy_mpconfig.mk index 4b6f4eb966..348a22cd4b 100644 --- a/py/circuitpy_mpconfig.mk +++ b/py/circuitpy_mpconfig.mk @@ -297,5 +297,5 @@ CFLAGS += -DCIRCUITPY_WIFI=$(CIRCUITPY_WIFI) CIRCUITPY_ENABLE_MPY_NATIVE ?= 0 CFLAGS += -DCIRCUITPY_ENABLE_MPY_NATIVE=$(CIRCUITPY_ENABLE_MPY_NATIVE) -CIRCUITPY_MSGPACK ?= 0 +CIRCUITPY_MSGPACK ?= $(CIRCUITPY_FULL_BUILD) CFLAGS += -DCIRCUITPY_MSGPACK=$(CIRCUITPY_MSGPACK) diff --git a/shared-bindings/msgpack/__init__.c b/shared-bindings/msgpack/__init__.c index 74ab644702..6329c50269 100644 --- a/shared-bindings/msgpack/__init__.c +++ b/shared-bindings/msgpack/__init__.c @@ -24,359 +24,48 @@ * THE SOFTWARE. */ -#include +#include "shared-bindings/msgpack/__init__.h" +#include "shared-module/msgpack/__init__.h" -#include "py/binary.h" -#include "py/objarray.h" -#include "py/objlist.h" -#include "py/objstringio.h" -#include "py/parsenum.h" -#include "py/runtime.h" -#include "py/stream.h" +//| """Pack object in msgpack format +//| +//| The msgpack format is similar to json, except that the encoded data is binary. +//| See https://msgpack.org for details. +//| +//| Example: +//| import msgpack +//| from io import StringIO +//| +//| s = StringIO() +//| msgpack.pack({'list': [True, False, None, 1, 'abc'], 'str': 'blah'}, s) +//| s.seek(0) +//| print(msgpack.unpack(s))""" +//| -#include "supervisor/shared/translate.h" +//| def pack(obj, stream): +//| """Pack obj to stream.""" +//| ... +//| -//////////////////////////////////////////////////////////////// -// stream management - -typedef struct _msgpack_stream_t { - mp_obj_t stream_obj; - mp_uint_t (*read)(mp_obj_t obj, void *buf, mp_uint_t size, int *errcode); - mp_uint_t (*write)(mp_obj_t obj, const void *buf, mp_uint_t size, int *errcode); - int errcode; -} msgpack_stream_t; - -STATIC msgpack_stream_t get_stream(mp_obj_t stream_obj, int flags) { - const mp_stream_p_t *stream_p = mp_get_stream_raise(stream_obj, flags); - msgpack_stream_t s = {stream_obj, stream_p->read, stream_p->write, 0}; - return s; -} - -//////////////////////////////////////////////////////////////// -// readers - -STATIC void read(msgpack_stream_t *s, void *buf, mp_uint_t size) { - if (size == 0) return; - mp_uint_t ret = s->read(s->stream_obj, buf, size, &s->errcode); - if (s->errcode != 0) { - mp_raise_OSError(s->errcode); - } - if (ret == 0) { - mp_raise_msg(&mp_type_EOFError, NULL); - } -} - -STATIC uint32_t read_bytes(msgpack_stream_t *s, mp_uint_t n_bytes) { - uint32_t res = 0; - read(s, &res, n_bytes); - return res; -} - -size_t read_size(msgpack_stream_t *s, uint8_t len_index) { - size_t n_bytes = 4; - switch (len_index) { - case 0: n_bytes = 1; break; - case 1: n_bytes = 2; break; - case 2: n_bytes = 4; break; - } - size_t res = 0; - read(s, &res, n_bytes); - return res; -} - -//////////////////////////////////////////////////////////////// -// writers - -STATIC void write(msgpack_stream_t *s, const void *buf, mp_uint_t size) { - mp_uint_t ret = s->write(s->stream_obj, buf, size, &s->errcode); - if (s->errcode != 0) { - mp_raise_OSError(s->errcode); - } - if (ret == 0) { - mp_raise_msg(&mp_type_EOFError, NULL); - } -} - -STATIC void write_bytes(msgpack_stream_t *s, mp_uint_t n_bytes, uint32_t obj) { - write(s, &obj, n_bytes); -} - -void write_size(msgpack_stream_t *s, uint8_t code, size_t size) { - if ((uint8_t)size == size) { - write_bytes(s, 1, code); - write_bytes(s, 1, size); - } else if ((uint16_t)size == size) { - write_bytes(s, 1, code+1); - write_bytes(s, 2, size); - } else { - write_bytes(s, 1, code+2); - write_bytes(s, 4, size); - } -} - -//////////////////////////////////////////////////////////////// -// packers - -// This is a helper function to iterate through a dictionary. The state of -// the iteration is held in *cur and should be initialised with zero for the -// first call. Will return NULL when no more elements are available. -STATIC mp_map_elem_t *dict_iter_next(mp_obj_dict_t *dict, size_t *cur) { - size_t max = dict->map.alloc; - mp_map_t *map = &dict->map; - - for (size_t i = *cur; i < max; i++) { - if (MP_MAP_SLOT_IS_FILLED(map, i)) { - *cur = i + 1; - return &(map->table[i]); - } - } - - return NULL; -} - - -STATIC void pack_int(msgpack_stream_t *s, int32_t x) { - if (x > -32 && x < 128) { - write_bytes(s, 1, x); - } else if ((int8_t)x == x) { - write_bytes(s, 1, 0xd0); - write_bytes(s, 1, x); - } else if ((int16_t)x == x) { - write_bytes(s, 1, 0xd1); - write_bytes(s, 2, x); - } else { - write_bytes(s, 1, 0xd2); - write_bytes(s, 4, x); - } -} - -void pack_bin(msgpack_stream_t *s, const uint8_t* data, size_t len) { - write_size(s, 0xc4, len); - for (size_t i=0; ilen); - for (size_t i=0; ilen; i++) { - _pack(self->items[i], s); - } - } else if (MP_OBJ_IS_TYPE(obj, &mp_type_list)) { - // list (layout differs from tuple) - mp_obj_list_t *self = MP_OBJ_TO_PTR(obj); - pack_array(s, self->len); - for (size_t i=0; ilen; i++) { - _pack(self->items[i], s); - } - } else if (MP_OBJ_IS_TYPE(obj, &mp_type_dict)) { - // dict - mp_obj_dict_t *self = MP_OBJ_TO_PTR(obj); - pack_dict(s, self->map.used); - size_t cur = 0; - mp_map_elem_t *next = NULL; - while ((next = dict_iter_next(self, &cur)) != NULL) { - _pack(next->key, s); - _pack(next->value, s); - } - } else if (mp_obj_is_float(obj)) { - mp_float_t f = mp_obj_float_get(obj); - write_bytes(s, 1, 0xca); - write(s, &f, 4); - } else if (obj == mp_const_none) { - write_bytes(s, 1, 0xc0); - } else if (obj == mp_const_false) { - write_bytes(s, 1, 0xc2); - } else if (obj == mp_const_true) { - write_bytes(s, 1, 0xc3); - } else { - mp_raise_ValueError(translate("no packer")); - } -} - -// Write obj to stream in msgpack format STATIC mp_obj_t mod_msgpack_pack(mp_obj_t obj, mp_obj_t stream_obj) { - msgpack_stream_t s = get_stream(stream_obj, MP_STREAM_OP_WRITE); - _pack(obj, &s); + common_hal_msgpack_pack(obj, stream_obj); return mp_const_none; } STATIC MP_DEFINE_CONST_FUN_OBJ_2(mod_msgpack_pack_obj, mod_msgpack_pack); -//////////////////////////////////////////////////////////////// -// un-packers -STATIC mp_obj_t _unpack(msgpack_stream_t *s) { - uint8_t code = read_bytes(s, 1); - if (((code & 0b10000000) == 0) || ((code & 0b11100000) == 0b11100000)) { - // int - return MP_OBJ_NEW_SMALL_INT((int8_t)code); - } - if ((code & 0b11100000) == 0b10100000) { - // str - size_t len = code & 0b11111; - // allocate on stack; len < 32 - char str[len]; - read(s, &str, len); - return mp_obj_new_str(str, len); - } - if ((code & 0b11110000) == 0b10010000) { - // array (tuple) - size_t len = code & 0b1111; - mp_obj_tuple_t *t = MP_OBJ_TO_PTR(mp_obj_new_tuple(len, NULL)); - for (size_t i=0; iitems[i] = _unpack(s); - } - return MP_OBJ_FROM_PTR(t); - } - if ((code & 0b11110000) == 0b10000000) { - // map (dict) - size_t len = code & 0b1111; - mp_obj_dict_t *d = MP_OBJ_TO_PTR(mp_obj_new_dict(len)); - for (size_t i=0; iitems[i] = _unpack(s); - } - return MP_OBJ_FROM_PTR(t); - } - case 0xc1: // never used - case 0xc7: // ext 8 - case 0xc8: // ext 16 - case 0xc9: // ext 32 - case 0xcb: // float 64 - case 0xcc: // uint 8 - case 0xcd: // uint 16 - case 0xce: // uint 32 - case 0xcf: // uint 64 - case 0xd3: // int 64 - case 0xd4: // fixenxt 1 - case 0xd5: // fixenxt 2 - case 0xd6: // fixenxt 4 - case 0xd7: // fixenxt 8 - case 0xd8: // fixenxt 16 - default: - mp_raise_ValueError(translate("no unpacker found")); - } -} +//| def pack(obj, stream) -> obj: +//| """Unpack and return one object (in msgpack format) from stream. +//| Call repeatedly to read multiple objects from the stream.""" +//| ... +//| -// Read msgpack encoded object from stream STATIC mp_obj_t mod_msgpack_unpack(mp_obj_t stream_obj) { - msgpack_stream_t s = get_stream(stream_obj, MP_STREAM_OP_READ); - return _unpack(&s); + return common_hal_msgpack_unpack(stream_obj); } STATIC MP_DEFINE_CONST_FUN_OBJ_1(mod_msgpack_unpack_obj, mod_msgpack_unpack); + STATIC const mp_rom_map_elem_t msgpack_module_globals_table[] = { { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_msgpack) }, { MP_ROM_QSTR(MP_QSTR_pack), MP_ROM_PTR(&mod_msgpack_pack_obj) }, diff --git a/shared-module/msgpack/__init__.c b/shared-module/msgpack/__init__.c new file mode 100644 index 0000000000..4a969901d1 --- /dev/null +++ b/shared-module/msgpack/__init__.c @@ -0,0 +1,372 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2019 Bernhard Boser + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include + +#include "py/binary.h" +#include "py/objarray.h" +#include "py/objlist.h" +#include "py/objstringio.h" +#include "py/parsenum.h" +#include "py/runtime.h" +#include "py/stream.h" + +#include "supervisor/shared/translate.h" + +//////////////////////////////////////////////////////////////// +// stream management + +typedef struct _msgpack_stream_t { + mp_obj_t stream_obj; + mp_uint_t (*read)(mp_obj_t obj, void *buf, mp_uint_t size, int *errcode); + mp_uint_t (*write)(mp_obj_t obj, const void *buf, mp_uint_t size, int *errcode); + int errcode; +} msgpack_stream_t; + +STATIC msgpack_stream_t get_stream(mp_obj_t stream_obj, int flags) { + const mp_stream_p_t *stream_p = mp_get_stream_raise(stream_obj, flags); + msgpack_stream_t s = {stream_obj, stream_p->read, stream_p->write, 0}; + return s; +} + +//////////////////////////////////////////////////////////////// +// readers + +STATIC void read(msgpack_stream_t *s, void *buf, mp_uint_t size) { + if (size == 0) return; + mp_uint_t ret = s->read(s->stream_obj, buf, size, &s->errcode); + if (s->errcode != 0) { + mp_raise_OSError(s->errcode); + } + if (ret == 0) { + mp_raise_msg(&mp_type_EOFError, NULL); + } +} + +STATIC uint32_t read_bytes(msgpack_stream_t *s, mp_uint_t n_bytes) { + uint32_t res = 0; + read(s, &res, n_bytes); + return res; +} + +size_t read_size(msgpack_stream_t *s, uint8_t len_index) { + size_t n_bytes = 4; + switch (len_index) { + case 0: n_bytes = 1; break; + case 1: n_bytes = 2; break; + case 2: n_bytes = 4; break; + } + size_t res = 0; + read(s, &res, n_bytes); + return res; +} + +//////////////////////////////////////////////////////////////// +// writers + +STATIC void write(msgpack_stream_t *s, const void *buf, mp_uint_t size) { + mp_uint_t ret = s->write(s->stream_obj, buf, size, &s->errcode); + if (s->errcode != 0) { + mp_raise_OSError(s->errcode); + } + if (ret == 0) { + mp_raise_msg(&mp_type_EOFError, NULL); + } +} + +STATIC void write_bytes(msgpack_stream_t *s, mp_uint_t n_bytes, uint32_t obj) { + write(s, &obj, n_bytes); +} + +void write_size(msgpack_stream_t *s, uint8_t code, size_t size) { + if ((uint8_t)size == size) { + write_bytes(s, 1, code); + write_bytes(s, 1, size); + } else if ((uint16_t)size == size) { + write_bytes(s, 1, code+1); + write_bytes(s, 2, size); + } else { + write_bytes(s, 1, code+2); + write_bytes(s, 4, size); + } +} + +//////////////////////////////////////////////////////////////// +// packers + +// This is a helper function to iterate through a dictionary. The state of +// the iteration is held in *cur and should be initialised with zero for the +// first call. Will return NULL when no more elements are available. +STATIC mp_map_elem_t *dict_iter_next(mp_obj_dict_t *dict, size_t *cur) { + size_t max = dict->map.alloc; + mp_map_t *map = &dict->map; + + for (size_t i = *cur; i < max; i++) { + if (MP_MAP_SLOT_IS_FILLED(map, i)) { + *cur = i + 1; + return &(map->table[i]); + } + } + + return NULL; +} + +STATIC void pack_int(msgpack_stream_t *s, int32_t x) { + if (x > -32 && x < 128) { + write_bytes(s, 1, x); + } else if ((int8_t)x == x) { + write_bytes(s, 1, 0xd0); + write_bytes(s, 1, x); + } else if ((int16_t)x == x) { + write_bytes(s, 1, 0xd1); + write_bytes(s, 2, x); + } else { + write_bytes(s, 1, 0xd2); + write_bytes(s, 4, x); + } +} + +void pack_bin(msgpack_stream_t *s, const uint8_t* data, size_t len) { + write_size(s, 0xc4, len); + for (size_t i=0; ilen); + for (size_t i=0; ilen; i++) { + pack(self->items[i], s); + } + } else if (MP_OBJ_IS_TYPE(obj, &mp_type_list)) { + // list (layout differs from tuple) + mp_obj_list_t *self = MP_OBJ_TO_PTR(obj); + pack_array(s, self->len); + for (size_t i=0; ilen; i++) { + pack(self->items[i], s); + } + } else if (MP_OBJ_IS_TYPE(obj, &mp_type_dict)) { + // dict + mp_obj_dict_t *self = MP_OBJ_TO_PTR(obj); + pack_dict(s, self->map.used); + size_t cur = 0; + mp_map_elem_t *next = NULL; + while ((next = dict_iter_next(self, &cur)) != NULL) { + pack(next->key, s); + pack(next->value, s); + } + } else if (mp_obj_is_float(obj)) { + mp_float_t f = mp_obj_float_get(obj); + write_bytes(s, 1, 0xca); + write(s, &f, 4); + } else if (obj == mp_const_none) { + write_bytes(s, 1, 0xc0); + } else if (obj == mp_const_false) { + write_bytes(s, 1, 0xc2); + } else if (obj == mp_const_true) { + write_bytes(s, 1, 0xc3); + } else { + mp_raise_ValueError(translate("no packer")); + } +} + +//////////////////////////////////////////////////////////////// +// unpacker + +mp_obj_t unpack(msgpack_stream_t *s) { + uint8_t code = read_bytes(s, 1); + if (((code & 0b10000000) == 0) || ((code & 0b11100000) == 0b11100000)) { + // int + return MP_OBJ_NEW_SMALL_INT((int8_t)code); + } + if ((code & 0b11100000) == 0b10100000) { + // str + size_t len = code & 0b11111; + // allocate on stack; len < 32 + char str[len]; + read(s, &str, len); + return mp_obj_new_str(str, len); + } + if ((code & 0b11110000) == 0b10010000) { + // array (tuple) + size_t len = code & 0b1111; + mp_obj_tuple_t *t = MP_OBJ_TO_PTR(mp_obj_new_tuple(len, NULL)); + for (size_t i=0; iitems[i] = unpack(s); + } + return MP_OBJ_FROM_PTR(t); + } + if ((code & 0b11110000) == 0b10000000) { + // map (dict) + size_t len = code & 0b1111; + mp_obj_dict_t *d = MP_OBJ_TO_PTR(mp_obj_new_dict(len)); + for (size_t i=0; iitems[i] = unpack(s); + } + return MP_OBJ_FROM_PTR(t); + } + case 0xc1: // never used + case 0xc7: // ext 8 + case 0xc8: // ext 16 + case 0xc9: // ext 32 + case 0xcb: // float 64 + case 0xcc: // uint 8 + case 0xcd: // uint 16 + case 0xce: // uint 32 + case 0xcf: // uint 64 + case 0xd3: // int 64 + case 0xd4: // fixenxt 1 + case 0xd5: // fixenxt 2 + case 0xd6: // fixenxt 4 + case 0xd7: // fixenxt 8 + case 0xd8: // fixenxt 16 + default: + mp_raise_ValueError(translate("no unpacker found")); + } +} + +void common_hal_msgpack_pack(mp_obj_t obj, mp_obj_t stream_obj) { + msgpack_stream_t stream = get_stream(stream_obj, MP_STREAM_OP_WRITE); + pack(obj, &stream); +} + +mp_obj_t common_hal_msgpack_unpack(mp_obj_t stream_obj) { + msgpack_stream_t stream = get_stream(stream_obj, MP_STREAM_OP_WRITE); + return unpack(&stream); +} + diff --git a/shared-module/msgpack/__init__.h b/shared-module/msgpack/__init__.h new file mode 100644 index 0000000000..0a5e7852ad --- /dev/null +++ b/shared-module/msgpack/__init__.h @@ -0,0 +1,34 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2017 Scott Shawcroft for Adafruit Industries + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +#ifndef MICROPY_INCLUDED_SHARED_MODULE_MSGPACK___INIT___H +#define MICROPY_INCLUDED_SHARED_MODULE_MSGPACK___INIT___H + +#include "py/stream.h" + +void common_hal_msgpack_pack(mp_obj_t obj, mp_obj_t stream_obj); +mp_obj_t common_hal_msgpack_unpack(mp_obj_t stream_obj); + +#endif