py/modsys: Allow sys.path to be assigned to.

Previously sys.path could be modified by append/pop or slice assignment.

This allows `sys.path = [...]`, which can be simpler in many cases, but
also improves CPython compatibility.

It also allows sys.path to be set to a tuple which means that you can
clear sys.path (e.g. temporarily) with no allocations.

This also makes sys.path (and sys.argv for consistency) able to be disabled
via mpconfig. The unix port (and upytesthelper) require them, so they
explicitly verify that they're enabled.

This work was funded through GitHub Sponsors.

Signed-off-by: Jim Mussared <jim.mussared@gmail.com>

ports/unix/main.c

@@ -69,6 +69,14 @@ long heap_size = 1024 * 1024 * (sizeof(mp_uint_t) / 4);
 #define MICROPY_GC_SPLIT_HEAP_N_HEAPS (1)
 #endif
 
+#if !MICROPY_PY_SYS_PATH
+#error "The unix port requires MICROPY_PY_SYS_PATH=1"
+#endif
+
+#if !MICROPY_PY_SYS_ARGV
+#error "The unix port requires MICROPY_PY_SYS_ARGV=1"
+#endif
+
 STATIC void stderr_print_strn(void *env, const char *str, size_t len) {
     (void)env;
     ssize_t ret;
@@ -538,44 +546,40 @@ MP_NOINLINE int main_(int argc, char **argv) {
     }
     #endif
 
-    char *home = getenv("HOME");
-    char *path = getenv("MICROPYPATH");
-    if (path == NULL) {
-        path = MICROPY_PY_SYS_PATH_DEFAULT;
-    }
-    size_t path_num = 1; // [0] is for current dir (or base dir of the script)
-    if (*path == PATHLIST_SEP_CHAR) {
-        path_num++;
-    }
-    for (char *p = path; p != NULL; p = strchr(p, PATHLIST_SEP_CHAR)) {
-        path_num++;
-        if (p != NULL) {
-            p++;
-        }
-    }
-    mp_obj_list_init(MP_OBJ_TO_PTR(mp_sys_path), path_num);
-    mp_obj_t *path_items;
-    mp_obj_list_get(mp_sys_path, &path_num, &path_items);
-    path_items[0] = MP_OBJ_NEW_QSTR(MP_QSTR_);
     {
-        char *p = path;
-        for (mp_uint_t i = 1; i < path_num; i++) {
-            char *p1 = strchr(p, PATHLIST_SEP_CHAR);
-            if (p1 == NULL) {
-                p1 = p + strlen(p);
+        // sys.path starts as [""]
+        mp_sys_path = mp_obj_new_list(0, NULL);
+        mp_obj_list_append(mp_sys_path, MP_OBJ_NEW_QSTR(MP_QSTR_));
+
+        // Add colon-separated entries from MICROPYPATH.
+        char *home = getenv("HOME");
+        char *path = getenv("MICROPYPATH");
+        if (path == NULL) {
+            path = MICROPY_PY_SYS_PATH_DEFAULT;
+        }
+        if (*path == PATHLIST_SEP_CHAR) {
+            // First entry is empty. We've already added an empty entry to sys.path, so skip it.
+            ++path;
+        }
+        bool path_remaining = *path;
+        while (path_remaining) {
+            char *path_entry_end = strchr(path, PATHLIST_SEP_CHAR);
+            if (path_entry_end == NULL) {
+                path_entry_end = path + strlen(path);
+                path_remaining = false;
             }
-            if (p[0] == '~' && p[1] == '/' && home != NULL) {
+            if (path[0] == '~' && path[1] == '/' && home != NULL) {
                 // Expand standalone ~ to $HOME
                 int home_l = strlen(home);
                 vstr_t vstr;
-                vstr_init(&vstr, home_l + (p1 - p - 1) + 1);
+                vstr_init(&vstr, home_l + (path_entry_end - path - 1) + 1);
                 vstr_add_strn(&vstr, home, home_l);
-                vstr_add_strn(&vstr, p + 1, p1 - p - 1);
-                path_items[i] = mp_obj_new_str_from_vstr(&vstr);
+                vstr_add_strn(&vstr, path + 1, path_entry_end - path - 1);
+                mp_obj_list_append(mp_sys_path, mp_obj_new_str_from_vstr(&vstr));
             } else {
-                path_items[i] = mp_obj_new_str_via_qstr(p, p1 - p);
+                mp_obj_list_append(mp_sys_path, mp_obj_new_str_via_qstr(path, path_entry_end - path));
             }
-            p = p1 + 1;
+            path = path_entry_end + 1;
         }
     }
 
@@ -710,7 +714,7 @@ MP_NOINLINE int main_(int argc, char **argv) {
 
             // Set base dir of the script as first entry in sys.path.
             char *p = strrchr(basedir, '/');
-            path_items[0] = mp_obj_new_str_via_qstr(basedir, p - basedir);
+            mp_obj_list_store(mp_sys_path, MP_OBJ_NEW_SMALL_INT(0), mp_obj_new_str_via_qstr(basedir, p - basedir));
             free(pathbuf);
 
             set_sys_argv(argv, argc, a);

ports/unix/mpconfigport.h

@@ -154,7 +154,7 @@ typedef long mp_off_t;
 // Ensure builtinimport.c works with -m.
 #define MICROPY_MODULE_OVERRIDE_MAIN_IMPORT (1)
 
-// Don't default sys.argv because we do that in main.
+// Don't default sys.argv and sys.path because we do that in main.
 #define MICROPY_PY_SYS_PATH_ARGV_DEFAULTS (0)
 
 // Enable sys.executable.

py/builtinimport.c

@@ -118,7 +118,7 @@ STATIC mp_import_stat_t stat_top_level(qstr mod_name, vstr_t *dest) {
     #if MICROPY_PY_SYS
     size_t path_num;
     mp_obj_t *path_items;
-    mp_obj_list_get(mp_sys_path, &path_num, &path_items);
+    mp_obj_get_array(mp_sys_path, &path_num, &path_items);
 
     // go through each sys.path entry, trying to import "<entry>/<mod_name>".
     for (size_t i = 0; i < path_num; i++) {
@@ -365,7 +365,7 @@ STATIC mp_obj_t process_import_at_level(qstr full_mod_name, qstr level_mod_name,
     // which may have come from the filesystem.
     size_t path_num;
     mp_obj_t *path_items;
-    mp_obj_list_get(mp_sys_path, &path_num, &path_items);
+    mp_obj_get_array(mp_sys_path, &path_num, &path_items);
     if (path_num)
     #endif
     {

py/modsys.c

@@ -195,6 +195,9 @@ STATIC mp_obj_t mp_sys_settrace(mp_obj_t obj) {
 MP_DEFINE_CONST_FUN_OBJ_1(mp_sys_settrace_obj, mp_sys_settrace);
 #endif // MICROPY_PY_SYS_SETTRACE
 
+#if MICROPY_PY_SYS_PATH && !MICROPY_PY_SYS_ATTR_DELEGATION
+#error "MICROPY_PY_SYS_PATH requires MICROPY_PY_SYS_ATTR_DELEGATION"
+#endif
 
 #if MICROPY_PY_SYS_PS1_PS2 && !MICROPY_PY_SYS_ATTR_DELEGATION
 #error "MICROPY_PY_SYS_PS1_PS2 requires MICROPY_PY_SYS_ATTR_DELEGATION"
@@ -211,6 +214,11 @@ MP_DEFINE_CONST_FUN_OBJ_1(mp_sys_settrace_obj, mp_sys_settrace);
 #if MICROPY_PY_SYS_ATTR_DELEGATION
 // Must be kept in sync with the enum at the top of mpstate.h.
 STATIC const uint16_t sys_mutable_keys[] = {
+    #if MICROPY_PY_SYS_PATH
+    // Code should access this (as an mp_obj_t) for use with e.g.
+    // mp_obj_list_append by using the `mp_sys_path` macro defined in runtime.h.
+    MP_QSTR_path,
+    #endif
     #if MICROPY_PY_SYS_PS1_PS2
     MP_QSTR_ps1,
     MP_QSTR_ps2,
@@ -231,8 +239,9 @@ void mp_module_sys_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) {
 STATIC const mp_rom_map_elem_t mp_module_sys_globals_table[] = {
     { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_sys) },
 
-    { MP_ROM_QSTR(MP_QSTR_path), MP_ROM_PTR(&MP_STATE_VM(mp_sys_path_obj)) },
+    #if MICROPY_PY_SYS_ARGV
     { MP_ROM_QSTR(MP_QSTR_argv), MP_ROM_PTR(&MP_STATE_VM(mp_sys_argv_obj)) },
+    #endif
     { MP_ROM_QSTR(MP_QSTR_version), MP_ROM_PTR(&mp_sys_version_obj) },
     { MP_ROM_QSTR(MP_QSTR_version_info), MP_ROM_PTR(&mp_sys_version_info_obj) },
     { MP_ROM_QSTR(MP_QSTR_implementation), MP_ROM_PTR(&mp_sys_implementation_obj) },
@@ -308,10 +317,11 @@ const mp_obj_module_t mp_module_sys = {
 // available.
 MP_REGISTER_MODULE(MP_QSTR_sys, mp_module_sys);
 
-// If MICROPY_PY_SYS_PATH_ARGV_DEFAULTS is not enabled then these two lists
-// must be initialised after the call to mp_init.
-MP_REGISTER_ROOT_POINTER(mp_obj_list_t mp_sys_path_obj);
+#if MICROPY_PY_SYS_ARGV
+// Code should access this (as an mp_obj_t) for use with e.g.
+// mp_obj_list_append by using the `mp_sys_argv` macro defined in runtime.h.
 MP_REGISTER_ROOT_POINTER(mp_obj_list_t mp_sys_argv_obj);
+#endif
 
 #if MICROPY_PY_SYS_EXC_INFO
 // current exception being handled, for sys.exc_info()

py/mpconfig.h

@@ -1421,6 +1421,23 @@ typedef double mp_float_t;
 #define MICROPY_PY_SYS_ATEXIT (0)
 #endif
 
+// Whether to provide the "sys.path" attribute (which forces module delegation
+// and mutable sys attributes to be enabled).
+// If MICROPY_PY_SYS_PATH_ARGV_DEFAULTS is enabled, this is initialised in
+// mp_init to an empty list. Otherwise the port must initialise it using
+// `mp_sys_path = mp_obj_new_list(...)`.
+#ifndef MICROPY_PY_SYS_PATH
+#define MICROPY_PY_SYS_PATH (1)
+#endif
+
+// Whether to provide the "sys.argv" attribute.
+// If MICROPY_PY_SYS_PATH_ARGV_DEFAULTS is enabled, this is initialised in
+// mp_init to an empty list. Otherwise the port must initialise it using
+// `mp_obj_list_init(MP_OBJ_TO_PTR(mp_sys_argv), ...);`
+#ifndef MICROPY_PY_SYS_ARGV
+#define MICROPY_PY_SYS_ARGV (1)
+#endif
+
 // Whether to provide sys.{ps1,ps2} mutable attributes, to control REPL prompts
 #ifndef MICROPY_PY_SYS_PS1_PS2
 #define MICROPY_PY_SYS_PS1_PS2 (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES)
@@ -1455,7 +1472,7 @@ typedef double mp_float_t;
 // Whether the sys module supports attribute delegation
 // This is enabled automatically when needed by other features
 #ifndef MICROPY_PY_SYS_ATTR_DELEGATION
-#define MICROPY_PY_SYS_ATTR_DELEGATION (MICROPY_PY_SYS_PS1_PS2 || MICROPY_PY_SYS_TRACEBACKLIMIT)
+#define MICROPY_PY_SYS_ATTR_DELEGATION (MICROPY_PY_SYS_PATH || MICROPY_PY_SYS_PS1_PS2 || MICROPY_PY_SYS_TRACEBACKLIMIT)
 #endif
 
 // Whether to provide "errno" module

py/mpstate.h

@@ -43,6 +43,9 @@
 #if MICROPY_PY_SYS_ATTR_DELEGATION
 // Must be kept in sync with sys_mutable_keys in modsys.c.
 enum {
+    #if MICROPY_PY_SYS_PATH
+    MP_SYS_MUTABLE_PATH,
+    #endif
     #if MICROPY_PY_SYS_PS1_PS2
     MP_SYS_MUTABLE_PS1,
     MP_SYS_MUTABLE_PS2,

py/runtime.c

@@ -136,13 +136,17 @@ void mp_init(void) {
     #endif
 
     #if MICROPY_PY_SYS_PATH_ARGV_DEFAULTS
-    mp_obj_list_init(MP_OBJ_TO_PTR(mp_sys_path), 0);
+    #if MICROPY_PY_SYS_PATH
+    mp_sys_path = mp_obj_new_list(0, NULL);
     mp_obj_list_append(mp_sys_path, MP_OBJ_NEW_QSTR(MP_QSTR_)); // current dir (or base dir of the script)
     #if MICROPY_MODULE_FROZEN
     mp_obj_list_append(mp_sys_path, MP_OBJ_NEW_QSTR(MP_QSTR__dot_frozen));
     #endif
+    #endif
+    #if MICROPY_PY_SYS_ARGV
     mp_obj_list_init(MP_OBJ_TO_PTR(mp_sys_argv), 0);
     #endif
+    #endif // MICROPY_PY_SYS_PATH_ARGV_DEFAULTS
 
     #if MICROPY_PY_SYS_ATEXIT
     MP_STATE_VM(sys_exitfunc) = mp_const_none;

py/runtime.h

@@ -227,8 +227,13 @@ int mp_native_type_from_qstr(qstr qst);
 mp_uint_t mp_native_from_obj(mp_obj_t obj, mp_uint_t type);
 mp_obj_t mp_native_to_obj(mp_uint_t val, mp_uint_t type);
 
-#define mp_sys_path (MP_OBJ_FROM_PTR(&MP_STATE_VM(mp_sys_path_obj)))
+#if MICROPY_PY_SYS_PATH
+#define mp_sys_path (MP_STATE_VM(sys_mutable[MP_SYS_MUTABLE_PATH]))
+#endif
+
+#if MICROPY_PY_SYS_ARGV
 #define mp_sys_argv (MP_OBJ_FROM_PTR(&MP_STATE_VM(mp_sys_argv_obj)))
+#endif
 
 #if MICROPY_WARNINGS
 #ifndef mp_warning

shared/upytesthelper/upytesthelper.c

@@ -31,6 +31,14 @@
 #include "py/compile.h"
 #include "upytesthelper.h"
 
+#if !MICROPY_PY_SYS_PATH
+#error "upytesthelper requires MICROPY_PY_SYS_PATH=1"
+#endif
+
+#if !MICROPY_PY_SYS_ARGV
+#error "upytesthelper requires MICROPY_PY_SYS_ARGV=1"
+#endif
+
 static const char *test_exp_output;
 static int test_exp_output_len, test_rem_output_len;
 static int test_failed;
@@ -93,7 +101,7 @@ void upytest_execute_test(const char *src) {
     // reinitialized before running each.
     gc_init(heap_start, heap_end);
     mp_init();
-    mp_obj_list_init(mp_sys_path, 0);
+    mp_sys_path = mp_obj_new_list(0, NULL);
     #if MICROPY_MODULE_FROZEN
     mp_obj_list_append(mp_sys_path, MP_OBJ_NEW_QSTR(MP_QSTR__dot_frozen));
     #endif