py/runtime: Add support for using __all__ in star import.

When the symbol `__all__` is defined in a module, `mp_import_all()` should
import all listed symbols into the global environment, rather than relying
on the underscore-is-private default.  This is the standard in CPython.

Each item is loaded in the same way as if it would be an explicit import
statement, and will invoke the module's `__getattr__` function if needed.
This provides a straightforward solution for fixing star import of modules
using a dynamic loader, such as `extmod/asyncio` (see issue #7266).

This improvement has been enabled at BASIC_FEATURES level, to avoid
impacting devices with limited ressources, for which star import is of
little use anyway.

Additionally, detailled reporting of errors during `__all__` import has
been implemented to match CPython, but this is only enabled when
ERROR_REPORTING is set to MICROPY_ERROR_REPORTING_DETAILED.

Signed-off-by: Yoctopuce dev <dev@yoctopuce.com>

py/mpconfig.h

@@ -1323,6 +1323,11 @@ typedef double mp_float_t;
 #define MICROPY_PY___FILE__ (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_CORE_FEATURES)
 #endif
 
+// Whether to process __all__ when importing all public symbols from module
+#ifndef MICROPY_MODULE___ALL__
+#define MICROPY_MODULE___ALL__ (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_BASIC_FEATURES)
+#endif
+
 // Whether to provide mem-info related functions in micropython module
 #ifndef MICROPY_PY_MICROPYTHON_MEM_INFO
 #define MICROPY_PY_MICROPYTHON_MEM_INFO (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES)

py/runtime.c

@@ -1247,6 +1247,19 @@ void mp_load_method(mp_obj_t base, qstr attr, mp_obj_t *dest) {
             mp_raise_msg_varg(&mp_type_AttributeError,
                 MP_ERROR_TEXT("type object '%q' has no attribute '%q'"),
                 ((mp_obj_type_t *)MP_OBJ_TO_PTR(base))->name, attr);
+        #if MICROPY_MODULE___ALL__ && MICROPY_ERROR_REPORTING >= MICROPY_ERROR_REPORTING_DETAILED
+        } else if (mp_obj_is_type(base, &mp_type_module)) {
+            // report errors in __all__ as done by CPython
+            mp_obj_t dest_name[2];
+            qstr module_name = MP_QSTR_;
+            mp_load_method_maybe(base, MP_QSTR___name__, dest_name);
+            if (mp_obj_is_qstr(dest_name[0])) {
+                module_name = mp_obj_str_get_qstr(dest_name[0]);
+            }
+            mp_raise_msg_varg(&mp_type_AttributeError,
+                MP_ERROR_TEXT("module '%q' has no attribute '%q'"),
+                module_name, attr);
+        #endif
         } else {
             mp_raise_msg_varg(&mp_type_AttributeError,
                 MP_ERROR_TEXT("'%s' object has no attribute '%q'"),
@@ -1593,8 +1606,28 @@ mp_obj_t mp_import_from(mp_obj_t module, qstr name) {
 void mp_import_all(mp_obj_t module) {
     DEBUG_printf("import all %p\n", module);
 
-    // TODO: Support __all__
     mp_map_t *map = &mp_obj_module_get_globals(module)->map;
+
+    #if MICROPY_MODULE___ALL__
+    mp_map_elem_t *elem = mp_map_lookup(map, MP_OBJ_NEW_QSTR(MP_QSTR___all__), MP_MAP_LOOKUP);
+    if (elem != NULL) {
+        // When __all__ is defined, we must explicitly load all specified
+        // symbols, possibly invoking the module __getattr__ function
+        size_t len;
+        mp_obj_t *items;
+        mp_obj_get_array(elem->value, &len, &items);
+        for (size_t i = 0; i < len; i++) {
+            qstr qname = mp_obj_str_get_qstr(items[i]);
+            mp_obj_t dest[2];
+            mp_load_method(module, qname, dest);
+            mp_store_name(qname, dest[0]);
+        }
+        return;
+    }
+    #endif
+
+    // By default, the set of public names includes all names found in the module's
+    // namespace which do not begin with an underscore character ('_')
     for (size_t i = 0; i < map->alloc; i++) {
         if (mp_map_slot_is_filled(map, i)) {
             // Entry in module global scope may be generated programmatically

tests/cpydiff/core_import_all.py

@@ -1,10 +0,0 @@
-"""
-categories: Core,import
-description: __all__ is unsupported in __init__.py in MicroPython.
-cause: Not implemented.
-workaround: Manually import the sub-modules directly in __init__.py using ``from . import foo, bar``.
-"""
-
-from modules3 import *
-
-foo.hello()

tests/cpydiff/modules3/__init__.py

@@ -1 +0,0 @@
-__all__ = ["foo"]

tests/cpydiff/modules3/foo.py

@@ -1,2 +0,0 @@
-def hello():
-    print("hello")

tests/import/import_star.py

@@ -0,0 +1,59 @@
+# test `from package import *` conventions, including __all__ support
+#
+# This test requires MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_BASIC_FEATURES
+
+try:
+    next(iter([]), 42)
+except TypeError:
+    # 2-argument version of next() not supported
+    # we are probably not at MICROPY_CONFIG_ROM_LEVEL_BASIC_FEATURES
+    print('SKIP')
+    raise SystemExit
+
+# 1. test default visibility
+from pkgstar_default import *
+
+print('visibleFun' in globals())
+print('VisibleClass' in globals())
+print('_hiddenFun' in globals())
+print('_HiddenClass' in globals())
+print(visibleFun())
+
+# 2. test explicit visibility as defined by __all__ (as an array)
+from pkgstar_all_array import *
+
+print('publicFun' in globals())
+print('PublicClass' in globals())
+print('unlistedFun' in globals())
+print('UnlistedClass' in globals())
+print('_privateFun' in globals())
+print('_PrivateClass' in globals())
+print(publicFun())
+# test dynamic import as used in asyncio
+print('dynamicFun' in globals())
+print(dynamicFun())
+
+# 3. test explicit visibility as defined by __all__ (as an tuple)
+from pkgstar_all_tuple import *
+
+print('publicFun2' in globals())
+print('PublicClass2' in globals())
+print('unlistedFun2' in globals())
+print('UnlistedClass2' in globals())
+print(publicFun2())
+
+# 4. test reporting of missing entries in __all__
+try:
+    from pkgstar_all_miss import *
+
+    print("missed detection of incorrect __all__ definition")
+except AttributeError as er:
+    print("AttributeError triggered for bad __all__ definition")
+
+# 5. test reporting of invalid __all__ definition
+try:
+    from pkgstar_all_inval import *
+
+    print("missed detection of incorrect __all__ definition")
+except TypeError as er:
+    print("TypeError triggered for bad __all__ definition")

tests/import/pkgstar_all_array/__init__.py

@@ -0,0 +1,49 @@
+__all__ = ['publicFun', 'PublicClass', 'dynamicFun']
+
+
+# Definitions below should always be imported by a star import
+def publicFun():
+    return 1
+
+
+class PublicClass:
+    def __init__(self):
+        self._val = 1
+
+
+# If __all__ support is enabled, definitions below
+# should not be imported by a star import
+def unlistedFun():
+    return 0
+
+
+class UnlistedClass:
+    def __init__(self):
+        self._val = 0
+
+
+# Definitions below should be not be imported by a star import
+# (they start with an underscore, and are not listed in __all__)
+def _privateFun():
+    return -1
+
+
+class _PrivateClass:
+    def __init__(self):
+        self._val = -1
+
+
+# Test lazy loaded function, as used by extmod/asyncio:
+# Works with a star import only if __all__ support is enabled
+_attrs = {
+    "dynamicFun": "funcs",
+}
+
+
+def __getattr__(attr):
+    mod = _attrs.get(attr, None)
+    if mod is None:
+        raise AttributeError(attr)
+    value = getattr(__import__(mod, globals(), locals(), True, 1), attr)
+    globals()[attr] = value
+    return value

tests/import/pkgstar_all_array/funcs.py

@@ -0,0 +1,2 @@
+def dynamicFun():
+    return 777

tests/import/pkgstar_all_inval/__init__.py

@@ -0,0 +1 @@
+__all__ = 42

tests/import/pkgstar_all_miss/__init__.py

@@ -0,0 +1,8 @@
+__all__ = ('existingFun', 'missingFun')
+
+
+def existingFun():
+    return None
+
+
+# missingFun is not defined, should raise an error on import

tests/import/pkgstar_all_tuple/__init__.py

@@ -0,0 +1,22 @@
+__all__ = ('publicFun2', 'PublicClass2')
+
+
+# Definitions below should always be imported by a star import
+def publicFun2():
+    return 2
+
+
+class PublicClass2:
+    def __init__(self):
+        self._val = 2
+
+
+# If __all__ support is enabled, definitions below
+# should not be imported by a star import
+def unlistedFun2():
+    return 0
+
+
+class UnlistedClass2:
+    def __init__(self):
+        self._val = 0

tests/import/pkgstar_default/__init__.py

@@ -0,0 +1,20 @@
+# When __all__ is undefined, star import should only
+# show objects that do not start with an underscore
+
+
+def visibleFun():
+    return 42
+
+
+class VisibleClass:
+    def __init__(self):
+        self._val = 42
+
+
+def _hiddenFun():
+    return -1
+
+
+class _HiddenClass:
+    def __init__(self):
+        self._val = -1