Add working_directory for subsequent code file

This allows subfolders to be treated similar to / for multiple
apps within different folders.

Also, fix up the internal current working directory so it doesn't
depend on volumes.

Fixes #9045 and fixes #8409.

.gitmodules

@@ -173,7 +173,7 @@
 [submodule "ports/raspberrypi/sdk"]
 	path = ports/raspberrypi/sdk
 	url = https://github.com/adafruit/pico-sdk.git
-	branch = force_inline_critical_section
+	branch = force_inline_critical_section_2.1.1
 [submodule "data/nvm.toml"]
 	path = data/nvm.toml
 	url = https://github.com/adafruit/nvm.toml.git

extmod/vfs_fat.c

@@ -38,6 +38,7 @@
 
 // CIRCUITPY-CHANGE: extra includes
 #include <string.h>
+#include "py/gc.h"
 #include "py/obj.h"
 #include "py/objproperty.h"
 #include "py/runtime.h"
@@ -342,7 +343,10 @@ static mp_obj_t fat_vfs_stat(mp_obj_t vfs_in, mp_obj_t path_in) {
         FRESULT res = f_stat(&self->fatfs, path, &fno);
         if (res != FR_OK) {
             // CIRCUITPY-CHANGE
-            mp_raise_OSError_fresult(res);
+            if (gc_alloc_possible()) {
+                mp_raise_OSError_fresult(res);
+            }
+            return mp_const_none;
         }
     }
 

main.c

@@ -457,14 +457,19 @@ static bool __attribute__((noinline)) run_code_py(safe_mode_t safe_mode, bool *s
         usb_setup_with_vm();
         #endif
 
+        // Always return to root before trying to run files.
+        common_hal_os_chdir("/");
         // Check if a different run file has been allocated
         if (next_code_configuration != NULL) {
             next_code_configuration->options &= ~SUPERVISOR_NEXT_CODE_OPT_NEWLY_SET;
             next_code_options = next_code_configuration->options;
             if (next_code_configuration->filename[0] != '\0') {
+                if (next_code_configuration->working_directory != NULL) {
+                    common_hal_os_chdir(next_code_configuration->working_directory);
+                }
                 // This is where the user's python code is actually executed:
                 const char *const filenames[] = { next_code_configuration->filename };
-                found_main = maybe_run_list(filenames, MP_ARRAY_SIZE(filenames));
+                found_main = maybe_run_list(filenames, 1);
                 if (!found_main) {
                     serial_write(next_code_configuration->filename);
                     serial_write_compressed(MP_ERROR_TEXT(" not found.\n"));
@@ -1105,9 +1110,6 @@ int __attribute__((used)) main(void) {
             }
             simulate_reset = false;
 
-            // Always return to root before trying to run files.
-            common_hal_os_chdir("/");
-
             if (pyexec_mode_kind == PYEXEC_MODE_FRIENDLY_REPL) {
                 // If code.py did a fake deep sleep, pretend that we
                 // are running code.py for the first time after a hard

ports/atmel-samd/boards/circuitbrains_basic_m0/mpconfigboard.mk

@@ -10,4 +10,5 @@ SPI_FLASH_FILESYSTEM = 1
 EXTERNAL_FLASH_DEVICES = "W25Q32JVxQ"
 LONGINT_IMPL = MPZ
 
+CIRCUITPY_CODEOP = 0
 CIRCUITPY_JPEGIO = 0

ports/atmel-samd/boards/feather_m0_supersized/mpconfigboard.mk

@@ -9,3 +9,5 @@ CHIP_FAMILY = samd21
 SPI_FLASH_FILESYSTEM = 1
 EXTERNAL_FLASH_DEVICES = "S25FL064L"
 LONGINT_IMPL = MPZ
+
+CIRCUITPY_CODEOP = 0

shared-bindings/os/__init__.h

@@ -10,6 +10,7 @@
 #include <stdbool.h>
 
 #include "py/objtuple.h"
+#include "shared-module/os/__init__.h"
 
 void common_hal_os_chdir(const char *path);
 mp_obj_t common_hal_os_getcwd(void);

shared-bindings/supervisor/__init__.c

@@ -3,6 +3,7 @@
 // SPDX-FileCopyrightText: Copyright (c) 2016-2017 Scott Shawcroft for Adafruit Industries
 //
 // SPDX-License-Identifier: MIT
+#include <stddef.h>
 #include <string.h>
 
 #include "py/obj.h"
@@ -25,6 +26,7 @@
 #include "shared-bindings/time/__init__.h"
 #include "shared-bindings/supervisor/Runtime.h"
 #include "shared-bindings/supervisor/StatusBar.h"
+#include "shared-bindings/util.h"
 
 //| """Supervisor settings"""
 //|
@@ -57,6 +59,7 @@ MP_DEFINE_CONST_FUN_OBJ_0(supervisor_reload_obj, supervisor_reload);
 //| def set_next_code_file(
 //|     filename: Optional[str],
 //|     *,
+//|     working_directory: Optional[str] = None,
 //|     reload_on_success: bool = False,
 //|     reload_on_error: bool = False,
 //|     sticky_on_success: bool = False,
@@ -99,6 +102,7 @@ MP_DEFINE_CONST_FUN_OBJ_0(supervisor_reload_obj, supervisor_reload);
 static mp_obj_t supervisor_set_next_code_file(size_t n_args, const mp_obj_t *pos_args, mp_map_t *kw_args) {
     static const mp_arg_t allowed_args[] = {
         { MP_QSTR_filename, MP_ARG_REQUIRED | MP_ARG_OBJ, {.u_rom_obj = mp_const_none} },
+        { MP_QSTR_working_directory, MP_ARG_KW_ONLY | MP_ARG_OBJ, {.u_rom_obj = mp_const_none} },
         { MP_QSTR_reload_on_success, MP_ARG_KW_ONLY | MP_ARG_BOOL, {.u_bool = false} },
         { MP_QSTR_reload_on_error, MP_ARG_KW_ONLY | MP_ARG_BOOL, {.u_bool = false} },
         { MP_QSTR_sticky_on_success, MP_ARG_KW_ONLY | MP_ARG_BOOL, {.u_bool = false} },
@@ -107,6 +111,7 @@ static mp_obj_t supervisor_set_next_code_file(size_t n_args, const mp_obj_t *pos
     };
     struct {
         mp_arg_val_t filename;
+        mp_arg_val_t working_directory;
         mp_arg_val_t reload_on_success;
         mp_arg_val_t reload_on_error;
         mp_arg_val_t sticky_on_success;
@@ -118,6 +123,11 @@ static mp_obj_t supervisor_set_next_code_file(size_t n_args, const mp_obj_t *pos
     if (!mp_obj_is_str_or_bytes(filename_obj) && filename_obj != mp_const_none) {
         mp_raise_TypeError_varg(MP_ERROR_TEXT("%q must be of type %q or %q, not %q"), MP_QSTR_filename, MP_QSTR_str, MP_QSTR_None, mp_obj_get_type(filename_obj)->name);
     }
+
+    mp_obj_t working_directory_obj = args.working_directory.u_obj;
+    if (!mp_obj_is_str_or_bytes(working_directory_obj) && working_directory_obj != mp_const_none) {
+        mp_raise_TypeError_varg(MP_ERROR_TEXT("%q must be of type %q or %q, not %q"), MP_QSTR_working_directory, MP_QSTR_str, MP_QSTR_None, mp_obj_get_type(working_directory_obj)->name);
+    }
     if (filename_obj == mp_const_none) {
         filename_obj = mp_const_empty_bytes;
     }
@@ -139,18 +149,50 @@ static mp_obj_t supervisor_set_next_code_file(size_t n_args, const mp_obj_t *pos
     }
     size_t len;
     const char *filename = mp_obj_str_get_data(filename_obj, &len);
+    if (!path_exists(filename)) {
+        mp_raise_ValueError(MP_ERROR_TEXT("File not found"));
+    }
+
+    size_t working_directory_len = 0;
+    const char *working_directory = NULL;
+    if (working_directory_obj != mp_const_none) {
+        working_directory = mp_obj_str_get_data(working_directory_obj, &working_directory_len);
+        if (!path_exists(working_directory)) {
+            mp_raise_ValueError_varg(MP_ERROR_TEXT("Invalid %q"), MP_QSTR_working_directory);
+        }
+    }
     if (next_code_configuration != NULL) {
         port_free(next_code_configuration);
         next_code_configuration = NULL;
     }
     if (options != 0 || len != 0) {
-        next_code_configuration = port_malloc(sizeof(supervisor_next_code_info_t) + len + 1, false);
-        if (next_code_configuration == NULL) {
-            m_malloc_fail(sizeof(supervisor_next_code_info_t) + len + 1);
+
+        size_t next_code_size = sizeof(supervisor_next_code_info_t) + len + 1;
+        if (working_directory_len > 0) {
+            next_code_size += working_directory_len + 1;
         }
+        next_code_configuration = port_malloc(next_code_size, false);
+        if (next_code_configuration == NULL) {
+            m_malloc_fail(next_code_size);
+        }
+        char *filename_ptr = (char *)next_code_configuration + sizeof(supervisor_next_code_info_t);
+
+        // Copy filename
+        memcpy(filename_ptr, filename, len);
+        filename_ptr[len] = '\0';
+
+        char *working_directory_ptr = NULL;
+        // Copy working directory after filename if present
+        if (working_directory_len > 0) {
+            working_directory_ptr = filename_ptr + len + 1;
+            memcpy(working_directory_ptr, working_directory, working_directory_len);
+            working_directory_ptr[working_directory_len] = '\0';
+        }
+        // Set everything up last. We may have raised an exception early and we
+        // don't want to free the memory if we failed.
+        next_code_configuration->filename = filename_ptr;
+        next_code_configuration->working_directory = working_directory_ptr;
         next_code_configuration->options = options | SUPERVISOR_NEXT_CODE_OPT_NEWLY_SET;
-        memcpy(&next_code_configuration->filename, filename, len);
-        next_code_configuration->filename[len] = '\0';
     }
     return mp_const_none;
 }

shared-bindings/supervisor/__init__.h

@@ -6,6 +6,8 @@
 
 #pragma once
 
+#include <stddef.h>
+#include <stdint.h>
 // #include "py/mpconfig.h"
 #include "py/obj.h"
 
@@ -18,7 +20,8 @@
 
 typedef struct {
     uint8_t options;
-    char filename[];
+    const char *working_directory;
+    const char *filename;
 } supervisor_next_code_info_t;
 
 extern const super_runtime_obj_t common_hal_supervisor_runtime_obj;

shared-bindings/util.c

@@ -8,6 +8,8 @@
 
 #include "shared-bindings/util.h"
 
+#include "shared-bindings/os/__init__.h"
+
 // If so, deinit() has already been called on the object, so complain.
 void raise_deinited_error(void) {
     mp_raise_ValueError(MP_ERROR_TEXT("Object has been deinitialized and can no longer be used. Create a new object."));
@@ -33,3 +35,16 @@ void properties_construct_helper(mp_obj_t self_in, const mp_arg_t *args, const m
         }
     }
 }
+
+bool path_exists(const char *path) {
+    // Use common_hal_os_stat to check if path exists
+    nlr_buf_t nlr;
+    if (nlr_push(&nlr) == 0) {
+        common_hal_os_stat(path);
+        nlr_pop();
+        return true;
+    } else {
+        // Path doesn't exist
+        return false;
+    }
+}

shared-bindings/util.h

@@ -12,3 +12,4 @@
 NORETURN void raise_deinited_error(void);
 void properties_print_helper(const mp_print_t *print, mp_obj_t self_in, const mp_arg_t *properties, size_t n_properties);
 void properties_construct_helper(mp_obj_t self_in, const mp_arg_t *args, const mp_arg_val_t *vals, size_t n_properties);
+bool path_exists(const char *path);

shared-module/os/__init__.c

@@ -6,6 +6,7 @@
 //
 // SPDX-License-Identifier: MIT
 
+#include <stddef.h>
 #include <string.h>
 
 #include "extmod/vfs.h"
@@ -63,9 +64,80 @@ static mp_obj_t mp_vfs_proxy_call(mp_vfs_mount_t *vfs, qstr meth_name, size_t n_
     return mp_call_method_n_kw(n_args, 0, meth);
 }
 
+const char *common_hal_os_path_abspath(const char *path) {
+    const char *cwd;
+    if (path[0] == '/') {
+        cwd = "";
+    } else {
+        cwd = MP_STATE_VM(cwd_path);
+        if (cwd == NULL) {
+            char *new_cwd = m_malloc(2);
+            strcpy(new_cwd, "/");
+            MP_STATE_VM(cwd_path) = new_cwd;
+            cwd = new_cwd;
+        }
+    }
+
+    // Store the current output length for previous components so we can rewind to before them.
+    char *full_path = m_malloc(strlen(cwd) + strlen(path) + 2);
+    size_t full_path_len = 0;
+    memcpy(full_path, cwd, strlen(cwd));
+    full_path_len += strlen(cwd);
+    if (full_path_len > 0 && full_path[full_path_len - 1] != '/') {
+        full_path[full_path_len++] = '/';
+    }
+    memcpy(full_path + full_path_len, path, strlen(path) + 1);
+
+    // Scan to see if the path has any `..` in it and return the same string if it doesn't
+    bool found_dot_dot = false;
+    size_t slash_count = 0;
+    for (size_t i = 0; i < strlen(full_path); i++) {
+        if (full_path[i] == '/') {
+            slash_count++;
+        }
+        if (i + 2 < strlen(full_path) && full_path[i] == '/' && full_path[i + 1] == '.' && full_path[i + 2] == '.' && (i + 3 == strlen(full_path) || full_path[i + 3] == '/')) {
+            found_dot_dot = true;
+        }
+    }
+    if (!found_dot_dot) {
+        return full_path;
+    }
+
+    size_t slashes[slash_count];
+    size_t output_len = 0;
+    size_t component_len = 0;
+    slash_count = 0;
+
+    // Remove `..` and `.`
+    size_t original_len = strlen(full_path);
+    for (size_t i = 0; i <= original_len; i++) {
+        full_path[output_len++] = full_path[i];
+        // Treat the final nul character as a slash.
+        if (full_path[i] == '/' || full_path[i] == '\0') {
+            if (component_len == 1 && full_path[i - 1] == '.') {
+                // Remove the dot
+                output_len = slashes[slash_count - 1];
+            } else if (component_len == 2 && full_path[i - 1] == '.' && full_path[i - 2] == '.') {
+                // Remove the double dot and the previous component if it exists
+                slash_count--;
+                output_len = slashes[slash_count - 1];
+            } else {
+                slashes[slash_count] = output_len;
+                slash_count++;
+            }
+            component_len = 0;
+        } else {
+            component_len++;
+        }
+    }
+    full_path[output_len] = '\0';
+    return full_path;
+}
+
 void common_hal_os_chdir(const char *path) {
+    MP_STATE_VM(cwd_path) = common_hal_os_path_abspath(path);
     mp_obj_t path_out;
-    mp_vfs_mount_t *vfs = lookup_dir_path(path, &path_out);
+    mp_vfs_mount_t *vfs = lookup_dir_path(MP_STATE_VM(cwd_path), &path_out);
     MP_STATE_VM(vfs_cur) = vfs;
     if (vfs == MP_VFS_ROOT) {
         // If we change to the root dir and a VFS is mounted at the root then
@@ -84,12 +156,17 @@ void common_hal_os_chdir(const char *path) {
 }
 
 mp_obj_t common_hal_os_getcwd(void) {
-    return mp_vfs_getcwd();
+    const char *cwd = MP_STATE_VM(cwd_path);
+    if (cwd == NULL) {
+        return MP_OBJ_NEW_QSTR(MP_QSTR__slash_);
+    }
+    return mp_obj_new_str_of_type(&mp_type_str, (const byte *)cwd, strlen(cwd));
 }
 
 mp_obj_t common_hal_os_listdir(const char *path) {
+    const char *abspath = common_hal_os_path_abspath(path);
     mp_obj_t path_out;
-    mp_vfs_mount_t *vfs = lookup_dir_path(path, &path_out);
+    mp_vfs_mount_t *vfs = lookup_dir_path(abspath, &path_out);
     if (vfs == MP_VFS_ROOT) {
         vfs = MP_STATE_VM(vfs_mount_table);
         while (vfs != NULL) {
@@ -114,8 +191,9 @@ mp_obj_t common_hal_os_listdir(const char *path) {
 }
 
 void common_hal_os_mkdir(const char *path) {
+    const char *abspath = common_hal_os_path_abspath(path);
     mp_obj_t path_out;
-    mp_vfs_mount_t *vfs = lookup_dir_path(path, &path_out);
+    mp_vfs_mount_t *vfs = lookup_dir_path(abspath, &path_out);
     if (vfs == MP_VFS_ROOT || (vfs != MP_VFS_NONE && !strcmp(mp_obj_str_get_str(path_out), "/"))) {
         mp_raise_OSError(MP_EEXIST);
     }
@@ -123,8 +201,9 @@ void common_hal_os_mkdir(const char *path) {
 }
 
 void common_hal_os_remove(const char *path) {
+    const char *abspath = common_hal_os_path_abspath(path);
     mp_obj_t path_out;
-    mp_vfs_mount_t *vfs = lookup_path(path, &path_out);
+    mp_vfs_mount_t *vfs = lookup_path(abspath, &path_out);
     mp_vfs_proxy_call(vfs, MP_QSTR_remove, 1, &path_out);
 }
 
@@ -140,14 +219,16 @@ void common_hal_os_rename(const char *old_path, const char *new_path) {
 }
 
 void common_hal_os_rmdir(const char *path) {
+    const char *abspath = common_hal_os_path_abspath(path);
     mp_obj_t path_out;
-    mp_vfs_mount_t *vfs = lookup_dir_path(path, &path_out);
+    mp_vfs_mount_t *vfs = lookup_dir_path(abspath, &path_out);
     mp_vfs_proxy_call(vfs, MP_QSTR_rmdir, 1, &path_out);
 }
 
 mp_obj_t common_hal_os_stat(const char *path) {
+    const char *abspath = common_hal_os_path_abspath(path);
     mp_obj_t path_out;
-    mp_vfs_mount_t *vfs = lookup_path(path, &path_out);
+    mp_vfs_mount_t *vfs = lookup_path(abspath, &path_out);
     if (vfs == MP_VFS_ROOT) {
         mp_obj_tuple_t *t = MP_OBJ_TO_PTR(mp_obj_new_tuple(10, NULL));
         t->items[0] = MP_OBJ_NEW_SMALL_INT(MP_S_IFDIR); // st_mode
@@ -160,8 +241,9 @@ mp_obj_t common_hal_os_stat(const char *path) {
 }
 
 mp_obj_t common_hal_os_statvfs(const char *path) {
+    const char *abspath = common_hal_os_path_abspath(path);
     mp_obj_t path_out;
-    mp_vfs_mount_t *vfs = lookup_path(path, &path_out);
+    mp_vfs_mount_t *vfs = lookup_path(abspath, &path_out);
     if (vfs == MP_VFS_ROOT) {
         // statvfs called on the root directory, see if there's anything mounted there
         for (vfs = MP_STATE_VM(vfs_mount_table); vfs != NULL; vfs = vfs->next) {
@@ -192,8 +274,11 @@ mp_obj_t common_hal_os_statvfs(const char *path) {
 }
 
 void common_hal_os_utime(const char *path, mp_obj_t times) {
+    const char *abspath = common_hal_os_path_abspath(path);
     mp_obj_t args[2];
-    mp_vfs_mount_t *vfs = lookup_path(path, &args[0]);
+    mp_vfs_mount_t *vfs = lookup_path(abspath, &args[0]);
     args[1] = times;
     mp_vfs_proxy_call(vfs, MP_QSTR_utime, 2, args);
 }
+
+MP_REGISTER_ROOT_POINTER(const char *cwd_path);

shared-module/os/__init__.h

@@ -26,3 +26,6 @@ os_getenv_err_t common_hal_os_getenv_str(const char *key, char *value, size_t va
 // If any error code is returned, value is guaranteed not modified
 // An error that is not 'open' or 'not found' is printed on the repl.
 os_getenv_err_t common_hal_os_getenv_int(const char *key, mp_int_t *value);
+
+// Not made available to the VM but used by other modules to normalize paths.
+const char *common_hal_os_path_abspath(const char *path);

shared-module/storage/__init__.c

@@ -82,10 +82,11 @@ static mp_obj_t mp_vfs_proxy_call(mp_vfs_mount_t *vfs, qstr meth_name, size_t n_
 }
 
 void common_hal_storage_mount(mp_obj_t vfs_obj, const char *mount_path, bool readonly) {
+    const char *abs_mount_path = common_hal_os_path_abspath(mount_path);
     // create new object
     mp_vfs_mount_t *vfs = m_new_obj(mp_vfs_mount_t);
-    vfs->str = mount_path;
-    vfs->len = strlen(mount_path);
+    vfs->str = abs_mount_path;
+    vfs->len = strlen(abs_mount_path);
     vfs->obj = vfs_obj;
     vfs->next = NULL;
 
@@ -98,7 +99,7 @@ void common_hal_storage_mount(mp_obj_t vfs_obj, const char *mount_path, bool rea
     if (strcmp(vfs->str, "/") != 0) {
         nlr_buf_t nlr;
         if (nlr_push(&nlr) == 0) {
-            mp_obj_t mount_point_stat = common_hal_os_stat(mount_path);
+            mp_obj_t mount_point_stat = common_hal_os_stat(abs_mount_path);
             nlr_pop();
             mp_obj_tuple_t *t = MP_OBJ_TO_PTR(mount_point_stat);
             if ((MP_OBJ_SMALL_INT_VALUE(t->items[0]) & MP_S_IFDIR) == 0) {
@@ -112,7 +113,7 @@ void common_hal_storage_mount(mp_obj_t vfs_obj, const char *mount_path, bool rea
 
     // check that the destination mount point is unused
     const char *path_out;
-    mp_vfs_mount_t *existing_mount = mp_vfs_lookup_path(mount_path, &path_out);
+    mp_vfs_mount_t *existing_mount = mp_vfs_lookup_path(abs_mount_path, &path_out);
     if (existing_mount != MP_VFS_NONE && existing_mount != MP_VFS_ROOT) {
         if (vfs->len != 1 && existing_mount->len == 1) {
             // if root dir is mounted, still allow to mount something within a subdir of root
@@ -157,8 +158,9 @@ void common_hal_storage_umount_object(mp_obj_t vfs_obj) {
 }
 
 static mp_obj_t storage_object_from_path(const char *mount_path) {
+    const char *abs_mount_path = common_hal_os_path_abspath(mount_path);
     for (mp_vfs_mount_t **vfsp = &MP_STATE_VM(vfs_mount_table); *vfsp != NULL; vfsp = &(*vfsp)->next) {
-        if (strcmp(mount_path, (*vfsp)->str) == 0) {
+        if (strcmp(abs_mount_path, (*vfsp)->str) == 0) {
             return (*vfsp)->obj;
         }
     }
@@ -175,8 +177,9 @@ mp_obj_t common_hal_storage_getmount(const char *mount_path) {
 
 void common_hal_storage_remount(const char *mount_path, bool readonly, bool disable_concurrent_write_protection) {
     const char *path_under_mount;
-    fs_user_mount_t *fs_usermount = filesystem_for_path(mount_path, &path_under_mount);
-    if (path_under_mount[0] != 0 && strcmp(mount_path, "/") != 0) {
+    const char *abs_mount_path = common_hal_os_path_abspath(mount_path);
+    fs_user_mount_t *fs_usermount = filesystem_for_path(abs_mount_path, &path_under_mount);
+    if (path_under_mount[0] != 0 && strcmp(abs_mount_path, "/") != 0) {
         mp_raise_OSError(MP_EINVAL);
     }