Merge pull request #262 from pfalcon/sys-path

Implement sys.path support

py/builtinimport.c

@@ -19,6 +19,8 @@
 #include "map.h"
 #include "builtin.h"
 
+mp_obj_t sys_path;
+
 mp_obj_t mp_builtin___import__(int n_args, mp_obj_t *args) {
     /*
     printf("import:\n");
@@ -37,10 +39,43 @@ mp_obj_t mp_builtin___import__(int n_args, mp_obj_t *args) {
     }
 
     // find the file to import
-    mp_lexer_t *lex = mp_import_open_file(mod_name);
+    uint mod_name_len;
+    const byte* mod_name_p = qstr_data(mod_name, &mod_name_len);
+    mp_lexer_t *lex = NULL;
+
+    uint path_num = 0;
+    mp_obj_t *path_items;
+    if (sys_path != MP_OBJ_NULL) {
+        mp_obj_list_get(sys_path, &path_num, &path_items);
+    }
+
+    if (path_num == 0) {
+        CHECKBUF(fname, PATH_MAX);
+        CHECKBUF_APPEND(fname, mod_name_p, mod_name_len);
+        CHECKBUF_APPEND(fname, ".py", sizeof(".py") - 1);
+        CHECKBUF_APPEND_0(fname);
+        lex = mp_lexer_new_from_file(fname);
+    } else {
+        for (int i = 0; i < path_num; i++) {
+            CHECKBUF(fname, PATH_MAX);
+            uint p_len;
+            const byte *p = mp_obj_str_get_data(path_items[i], &p_len);
+            if (p_len > 0) {
+                CHECKBUF_APPEND(fname, p, p_len);
+                CHECKBUF_APPEND(fname, "/", 1);
+            }
+            CHECKBUF_APPEND(fname, mod_name_p, mod_name_len);
+            CHECKBUF_APPEND(fname, ".py", sizeof(".py") - 1);
+            CHECKBUF_APPEND_0(fname);
+            lex = mp_lexer_new_from_file(fname);
+            if (lex != NULL) {
+                break;
+            }
+        }
+    }
+
     if (lex == NULL) {
-        // TODO handle lexer error correctly
-        return mp_const_none;
+        nlr_jump(mp_obj_new_exception_msg_varg(MP_QSTR_ImportError, "ImportError: No module named '%s'", mod_name_p));
     }
     qstr source_name = mp_lexer_source_name(lex);
 

py/lexerunix.c

@@ -14,7 +14,6 @@
 mp_lexer_t *mp_lexer_new_from_file(const char *filename) {
     int fd = open(filename, O_RDONLY);
     if (fd < 0) {
-        printf("cannot open file %s\n", filename);
         return NULL;
     }
     uint size = lseek(fd, 0, SEEK_END);
@@ -31,24 +30,4 @@ mp_lexer_t *mp_lexer_new_from_file(const char *filename) {
     return mp_lexer_new_from_str_len(qstr_from_str(filename), data, size, size);
 }
 
-/******************************************************************************/
-/* unix implementation of import                                              */
-
-// TODO properly!
-
-static const char *import_base_dir = NULL;
-
-void mp_import_set_directory(const char *dir) {
-    import_base_dir = dir;
-}
-
-mp_lexer_t *mp_import_open_file(qstr mod_name) {
-    vstr_t *vstr = vstr_new();
-    if (import_base_dir != NULL) {
-        vstr_printf(vstr, "%s/", import_base_dir);
-    }
-    vstr_printf(vstr, "%s.py", qstr_str(mod_name));
-    return mp_lexer_new_from_file(vstr_str(vstr)); // TODO does lexer need to copy the string? can we free it here?
-}
-
 #endif // MICROPY_ENABLE_LEXER_UNIX

py/misc.h

@@ -10,6 +10,11 @@
 typedef unsigned char byte;
 typedef unsigned int uint;
 
+/** generic ops *************************************************/
+
+#define MIN(x, y) ((x) < (y) ? (x) : (y))
+#define MAX(x, y) ((x) > (y) ? (x) : (y))
+
 /** memomry allocation ******************************************/
 
 // TODO make a lazy m_renew that can increase by a smaller amount than requested (but by at least 1 more element)
@@ -85,6 +90,17 @@ void vstr_add_strn(vstr_t *vstr, const char *str, int len);
 void vstr_cut_tail(vstr_t *vstr, int len);
 void vstr_printf(vstr_t *vstr, const char *fmt, ...);
 
+/** non-dynamic size-bounded variable buffer/string *************/
+
+#define CHECKBUF(buf, max_size) char buf[max_size + 1]; uint buf##_len = max_size; char *buf##_p = buf;
+#define CHECKBUF_APPEND(buf, src, src_len) \
+        { int l = MIN(src_len, buf##_len); \
+        memcpy(buf##_p, src, l); \
+        buf##_len -= l; \
+        buf##_p += l; }
+#define CHECKBUF_APPEND_0(buf) { *buf##_p = 0; }
+#define CHECKBUF_LEN(buf) (buf##_p - buf)
+
 #ifdef va_start
 void vstr_vprintf(vstr_t *vstr, const char *fmt, va_list ap);
 #endif

py/qstrdefs.h

@@ -73,6 +73,7 @@ Q(max)
 Q(min)
 Q(next)
 Q(ord)
+Q(path)
 Q(pow)
 Q(print)
 Q(range)

py/runtime.c

@@ -169,8 +169,14 @@ void rt_init(void) {
 
 #if MICROPY_CPYTHON_COMPAT
     // Precreate sys module, so "import sys" didn't throw exceptions.
-    mp_obj_new_module(MP_QSTR_sys);
+    mp_obj_t m_sys = mp_obj_new_module(MP_QSTR_sys);
+    // Avoid warning of unused var
+    (void)m_sys;
 #endif
+    // init sys.path
+    // for efficiency, left to platform-specific startup code
+    //sys_path = mp_obj_new_list(0, NULL);
+    //rt_store_attr(m_sys, MP_QSTR_path, sys_path);
 
     mp_module_micropython_init();
 

py/runtime.h

@@ -46,3 +46,4 @@ void rt_locals_set(struct _mp_map_t *m);
 struct _mp_map_t *rt_globals_get(void);
 void rt_globals_set(struct _mp_map_t *m);
 struct _mp_map_t *rt_loaded_modules_get(void);
+extern mp_obj_t sys_path;

stm/lexerfatfs.c

@@ -51,36 +51,3 @@ mp_lexer_t *mp_lexer_new_from_file(const char *filename) {
     fb->pos = 0;
     return mp_lexer_new(qstr_from_str(filename), fb, (mp_lexer_stream_next_char_t)file_buf_next_char, (mp_lexer_stream_close_t)file_buf_close);
 }
-
-/******************************************************************************/
-// implementation of import
-
-#include "ff.h"
-
-mp_lexer_t *mp_import_open_file(qstr mod_name) {
-    vstr_t *vstr = vstr_new();
-    FRESULT res;
-
-    // look for module in src/
-    vstr_printf(vstr, "0:/src/%s.py", qstr_str(mod_name));
-    res = f_stat(vstr_str(vstr), NULL);
-    if (res == FR_OK) {
-        // found file
-        return mp_lexer_new_from_file(vstr_str(vstr)); // TODO does lexer need to copy the string? can we free it here?
-    }
-
-    // look for module in /
-    vstr_reset(vstr);
-    vstr_printf(vstr, "0:/%s.py", qstr_str(mod_name));
-    res = f_stat(vstr_str(vstr), NULL);
-    if (res == FR_OK) {
-        // found file
-        return mp_lexer_new_from_file(vstr_str(vstr)); // TODO does lexer need to copy the string? can we free it here?
-    }
-
-    // could not find file
-    vstr_free(vstr);
-    printf("import %s: could not find file in src/ or /\n", qstr_str(mod_name));
-
-    return NULL;
-}

unix/main.c

@@ -147,18 +147,6 @@ static void do_repl(void) {
 }
 
 static void do_file(const char *file) {
-    // hack: set dir for import based on where this file is
-    {
-        const char * s = strrchr(file, '/');
-        if (s != NULL) {
-            int len = s - file;
-            char *dir = m_new(char, len + 1);
-            memcpy(dir, file, len);
-            dir[len] = '\0';
-            mp_import_set_directory(dir);
-        }
-    }
-
     mp_lexer_t *lex = mp_lexer_new_from_file(file);
     execute_from_lexer(lex, MP_PARSE_FILE_INPUT, false);
 }
@@ -233,7 +221,42 @@ int main(int argc, char **argv) {
     qstr_init();
     rt_init();
 
+    char *home = getenv("HOME");
+    char *path = getenv("MICROPYPATH");
+    if (path == NULL) {
+        path = "~/.micropython/lib:/usr/lib/micropython";
+    }
+    uint path_num = 1; // [0] is for current dir (or base dir of the script)
+    for (char *p = path; p != NULL; p = strchr(p, ':')) {
+        path_num++;
+        if (p != NULL) {
+            p++;
+        }
+    }
+    sys_path = mp_obj_new_list(path_num, NULL);
+    mp_obj_t *path_items;
+    mp_obj_list_get(sys_path, &path_num, &path_items);
+    path_items[0] = MP_OBJ_NEW_QSTR(MP_QSTR_);
+    char *p = path;
+    for (int i = 1; i < path_num; i++) {
+        char *p1 = strchr(p, ':');
+        if (p1 == NULL) {
+            p1 = p + strlen(p);
+        }
+        if (p[0] == '~' && p[1] == '/' && home != NULL) {
+            // Expand standalone ~ to $HOME
+            CHECKBUF(buf, PATH_MAX);
+            CHECKBUF_APPEND(buf, home, strlen(home));
+            CHECKBUF_APPEND(buf, p + 1, p1 - p - 1);
+            path_items[i] = MP_OBJ_NEW_QSTR(qstr_from_strn(buf, CHECKBUF_LEN(buf)));
+        } else {
+            path_items[i] = MP_OBJ_NEW_QSTR(qstr_from_strn(p, p1 - p));
+        }
+        p = p1 + 1;
+    }
+
     mp_obj_t m_sys = mp_obj_new_module(MP_QSTR_sys);
+    rt_store_attr(m_sys, MP_QSTR_path, sys_path);
     mp_obj_t py_argv = mp_obj_new_list(0, NULL);
     rt_store_attr(m_sys, MP_QSTR_argv, py_argv);
 
@@ -284,6 +307,13 @@ int main(int argc, char **argv) {
                     return usage();
                 }
             } else {
+                // Set base dir of the script as first entry in sys.path
+                char *basedir = realpath(argv[a], NULL);
+                if (basedir != NULL) {
+                    char *p = strrchr(basedir, '/');
+                    path_items[0] = MP_OBJ_NEW_QSTR(qstr_from_strn(basedir, p - basedir));
+                    free(basedir);
+                }
                 for (int i = a; i < argc; i++) {
                     rt_list_append(py_argv, MP_OBJ_NEW_QSTR(qstr_from_str(argv[i])));
                 }