Add dotenv read support

os.getenv() will use it (when available) to load variables from
/.env

This will also be useful when we need secrets or config for
CircuitPython outside of the VM (like WiFi credentials.)

Fixes #4212

py/circuitpy_defns.mk

@@ -158,6 +158,9 @@ endif
 ifeq ($(CIRCUITPY_DISPLAYIO),1)
 SRC_PATTERNS += displayio/%
 endif
+ifeq ($(CIRCUITPY_DOTENV),1)
+SRC_PATTERNS += dotenv/%
+endif
 ifeq ($(CIRCUITPY_PARALLELDISPLAY),1)
 SRC_PATTERNS += paralleldisplay/%
 endif
@@ -548,6 +551,7 @@ SRC_SHARED_MODULE_ALL = \
 	displayio/TileGrid.c \
 	displayio/area.c \
 	displayio/__init__.c \
+	dotenv/__init__.c \
 	floppyio/__init__.c \
 	fontio/BuiltinFont.c \
 	fontio/__init__.c \

py/circuitpy_mpconfig.mk

@@ -199,6 +199,9 @@ CFLAGS += -DCIRCUITPY_BITMAPTOOLS=$(CIRCUITPY_BITMAPTOOLS)
 CFLAGS += -DCIRCUITPY_FRAMEBUFFERIO=$(CIRCUITPY_FRAMEBUFFERIO)
 CFLAGS += -DCIRCUITPY_VECTORIO=$(CIRCUITPY_VECTORIO)
 
+CIRCUITPY_DOTENV ?= $(CIRCUITPY_FULL_BUILD)
+CFLAGS += -DCIRCUITPY_DOTENV=$(CIRCUITPY_DOTENV)
+
 CIRCUITPY_DUALBANK ?= 0
 CFLAGS += -DCIRCUITPY_DUALBANK=$(CIRCUITPY_DUALBANK)
 

shared-bindings/dotenv/__init__.c

@@ -0,0 +1,105 @@
+/*
+ * This file is part of the Micro Python project, http://micropython.org/
+ *
+ * The MIT License (MIT)
+ *
+ * SPDX-FileCopyrightText: Copyright (c) 2022 Scott Shawcroft for Adafruit Industries
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include <string.h>
+
+#include "extmod/vfs.h"
+#include "lib/oofatfs/ff.h"
+#include "lib/oofatfs/diskio.h"
+#include "py/mpstate.h"
+#include "py/obj.h"
+#include "py/objstr.h"
+#include "py/runtime.h"
+#include "shared-bindings/dotenv/__init__.h"
+
+//| """Functions to manage environment variables from a .env file.
+//|
+//|    A subset of the CPython `dotenv library <https://saurabh-kumar.com/python-dotenv/>`_. It does
+//|    not support variables or double quotes.
+//|
+//|    The simplest way to define keys and values is to put them in single quotes. \ and ' are
+//|    escaped by \ in single quotes. Newlines can occur in quotes for multiline values. Comments
+//|    start with # and apply for the rest of the line.
+//|
+//|    File format example:
+//|
+//|    .. code-block::
+//|
+//|        key=value
+//|          key2 = value2
+//|        'key3' = 'value with spaces'
+//|        # comment
+//|        key4 = value3 # comment 2
+//|        'key5'=value4
+//|        key=value5 # overrides the first one
+//|        multiline = 'hello
+//|        world
+//|        how are you?'
+//|
+//| """
+//|
+//| import typing
+
+//| def get_key(dotenv_path: str, key_to_get: str) -> Optional[str]:
+//|     """Get the value for the given key from the given .env file. If the key occurs multiple
+//|        times in the file, then the last value will be returned.
+//|
+//|        Returns None if the key isn't found or doesn't have a value."""
+//|     ...
+//|
+STATIC mp_obj_t _dotenv_get_key(mp_obj_t path_in, mp_obj_t key_to_get_in) {
+    return common_hal_dotenv_get_key(mp_obj_str_get_str(path_in),
+        mp_obj_str_get_str(key_to_get_in));
+}
+MP_DEFINE_CONST_FUN_OBJ_2(dotenv_get_key_obj, _dotenv_get_key);
+
+//| def load_dotenv() -> None:
+//|     """Does nothing in CircuitPython because os.getenv will automatically read .env when
+//|        available.
+//|
+//|        Present in CircuitPython so CPython-compatible code can use it without error."""
+//|     ...
+//|
+STATIC mp_obj_t dotenv_load_dotenv(void) {
+    return mp_const_none;
+}
+MP_DEFINE_CONST_FUN_OBJ_0(dotenv_load_dotenv_obj, dotenv_load_dotenv);
+
+STATIC const mp_rom_map_elem_t dotenv_module_globals_table[] = {
+    { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_dotenv) },
+
+    { MP_ROM_QSTR(MP_QSTR_get_key), MP_ROM_PTR(&dotenv_get_key_obj) },
+    { MP_ROM_QSTR(MP_QSTR_load_dotenv), MP_ROM_PTR(&dotenv_load_dotenv_obj) },
+};
+
+STATIC MP_DEFINE_CONST_DICT(dotenv_module_globals, dotenv_module_globals_table);
+
+const mp_obj_module_t dotenv_module = {
+    .base = { &mp_type_module },
+    .globals = (mp_obj_dict_t *)&dotenv_module_globals,
+};
+
+MP_REGISTER_MODULE(MP_QSTR_dotenv, dotenv_module, CIRCUITPY_DOTENV);

shared-bindings/dotenv/__init__.h

@@ -0,0 +1,39 @@
+/*
+ * This file is part of the MicroPython project, http://micropython.org/
+ *
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2022 Scott Shawcroft for Adafruit Industries
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#ifndef MICROPY_INCLUDED_SHARED_BINDINGS_DOTENV___INIT___H
+#define MICROPY_INCLUDED_SHARED_BINDINGS_DOTENV___INIT___H
+
+#include <stdint.h>
+#include <stdbool.h>
+
+#include "py/objtuple.h"
+
+#include "shared-module/dotenv/__init__.h"
+
+mp_obj_t common_hal_dotenv_get_key(const char *path, const char *key);
+
+#endif  // MICROPY_INCLUDED_SHARED_BINDINGS_DOTENV___INIT___H

shared-bindings/os/__init__.c

@@ -83,6 +83,25 @@ STATIC mp_obj_t os_getcwd(void) {
 }
 MP_DEFINE_CONST_FUN_OBJ_0(os_getcwd_obj, os_getcwd);
 
+//| def getenv(key: str, default: Optional[str] = None) -> Optional[str]:
+//|     """Get the environment variable value for the given key or return ``default``.
+//|
+//|        This may load values from disk so cache the result instead of calling this often."""
+//|     ...
+//|
+STATIC mp_obj_t os_getenv(size_t n_args, const mp_obj_t *pos_args, mp_map_t *kw_args) {
+    enum { ARG_key, ARG_default };
+    static const mp_arg_t allowed_args[] = {
+        { MP_QSTR_key, MP_ARG_REQUIRED | MP_ARG_OBJ },
+        { MP_QSTR_default,  MP_ARG_OBJ, {.u_obj = mp_const_none} },
+    };
+    mp_arg_val_t args[MP_ARRAY_SIZE(allowed_args)];
+    mp_arg_parse_all(n_args, pos_args, kw_args, MP_ARRAY_SIZE(allowed_args), allowed_args, args);
+
+    return common_hal_os_getenv(mp_obj_str_get_str(args[ARG_key].u_obj), args[ARG_default].u_obj);
+}
+STATIC MP_DEFINE_CONST_FUN_OBJ_KW(os_getenv_obj, 1, os_getenv);
+
 //| def listdir(dir: str) -> str:
 //|     """With no argument, list the current directory.  Otherwise list the given directory."""
 //|     ...
@@ -220,6 +239,7 @@ STATIC const mp_rom_map_elem_t os_module_globals_table[] = {
 
     { MP_ROM_QSTR(MP_QSTR_chdir), MP_ROM_PTR(&os_chdir_obj) },
     { MP_ROM_QSTR(MP_QSTR_getcwd), MP_ROM_PTR(&os_getcwd_obj) },
+    { MP_ROM_QSTR(MP_QSTR_getenv), MP_ROM_PTR(&os_getenv_obj) },
     { MP_ROM_QSTR(MP_QSTR_listdir), MP_ROM_PTR(&os_listdir_obj) },
     { MP_ROM_QSTR(MP_QSTR_mkdir), MP_ROM_PTR(&os_mkdir_obj) },
     { MP_ROM_QSTR(MP_QSTR_remove), MP_ROM_PTR(&os_remove_obj) },

shared-bindings/os/__init__.h

@@ -37,6 +37,7 @@ extern const mp_rom_obj_tuple_t common_hal_os_uname_info_obj;
 mp_obj_t common_hal_os_uname(void);
 void common_hal_os_chdir(const char *path);
 mp_obj_t common_hal_os_getcwd(void);
+mp_obj_t common_hal_os_getenv(const char *key, mp_obj_t default_);
 mp_obj_t common_hal_os_listdir(const char *path);
 void common_hal_os_mkdir(const char *path);
 void common_hal_os_remove(const char *path);

shared-module/dotenv/__init__.c

@@ -0,0 +1,221 @@
+/*
+ * This file is part of the Micro Python project, http://micropython.org/
+ *
+ * The MIT License (MIT)
+ *
+ * SPDX-FileCopyrightText: Copyright (c) 2022 Scott Shawcroft for Adafruit Industries
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include <string.h>
+
+#include "shared-bindings/dotenv/__init__.h"
+
+#include "extmod/vfs.h"
+#include "extmod/vfs_fat.h"
+#include "py/mpstate.h"
+#include "py/objstr.h"
+
+STATIC uint8_t consume_spaces(FIL *active_file) {
+    uint8_t character = ' ';
+    size_t quantity_read = 1;
+    while (unichar_isspace(character) && quantity_read > 0) {
+        f_read(active_file, &character, 1, &quantity_read);
+    }
+    return character;
+}
+
+// Starting at the start of a new line, determines if the key matches the given
+// key. File pointer is left after the = after the key.
+STATIC bool key_matches(FIL *active_file, const char *key) {
+    uint8_t character = ' ';
+    size_t quantity_read = 1;
+    character = consume_spaces(active_file);
+    bool quoted = false;
+    if (character == '\'') {
+        quoted = true;
+        f_read(active_file, &character, 1, &quantity_read);
+    }
+    size_t key_pos = 0;
+    bool escaped = false;
+    bool matches = true;
+    size_t key_len = strlen(key);
+    while (quantity_read > 0) {
+        if (character == '\\' && !escaped && quoted) {
+            escaped = true;
+        } else if (!escaped && quoted && character == '\'') {
+            quoted = false;
+            // Move past the quoted before breaking so we can check the validity of data past it.
+            f_read(active_file, &character, 1, &quantity_read);
+            break;
+        } else if (!quoted && (unichar_isspace(character) || character == '=' || character == '\n' || character == '#')) {
+            break;
+        } else {
+            matches = matches && key[key_pos] == character;
+            escaped = false;
+            key_pos++;
+        }
+
+        f_read(active_file, &character, 1, &quantity_read);
+    }
+    if (unichar_isspace(character)) {
+        character = consume_spaces(active_file);
+    }
+    if (character == '=' || character == '\n' || character == '#') {
+        // Rewind one so the value can find it.
+        f_lseek(active_file, f_tell(active_file) - 1);
+    } else {
+        // We're followed by something else that is invalid syntax.
+        matches = false;
+    }
+    return matches && key_pos == key_len;
+}
+
+STATIC bool next_line(FIL *active_file) {
+    uint8_t character = ' ';
+    size_t quantity_read = 1;
+    bool quoted = false;
+    bool escaped = false;
+    // Track comments because they last until the end of the line.
+    bool comment = false;
+    FRESULT result = FR_OK;
+    // Consume all characters while quoted or others up to \n.
+    while (result == FR_OK && quantity_read > 0 && (quoted || character != '\n')) {
+        if (character == '#' || comment) {
+            // Comments consume any escaping.
+            comment = true;
+        } else if (!escaped) {
+            if (character == '\'') {
+                quoted = !quoted;
+            } else if (character == '\\') {
+                escaped = true;
+            }
+        } else {
+            escaped = false;
+        }
+        result = f_read(active_file, &character, 1, &quantity_read);
+    }
+    return result == FR_OK && quantity_read > 0;
+}
+
+STATIC mp_int_t read_value(FIL *active_file, char *value, size_t value_len) {
+    uint8_t character = ' ';
+    size_t quantity_read = 1;
+    // Consume spaces before =
+    character = consume_spaces(active_file);
+    if (character != '=') {
+        if (character == '#' || character == '\n') {
+            // Keys without an = after them are valid with the value None.
+            return 0;
+        }
+        // All other characters are invalid.
+        return -1;
+    }
+    character = ' ';
+    // Consume space after =
+    while (unichar_isspace(character) && quantity_read > 0) {
+        f_read(active_file, &character, 1, &quantity_read);
+    }
+    bool quoted = false;
+    if (character == '\'') {
+        quoted = true;
+        f_read(active_file, &character, 1, &quantity_read);
+    }
+    if (character == '"') {
+        // We don't support double quoted values.
+        return -1;
+    }
+    // Copy the value over.
+    size_t value_pos = 0;
+    bool escaped = false;
+    // Count trailing spaces so we can ignore them at the end of unquoted
+    // values.
+    size_t trailing_spaces = 0;
+    while (quantity_read > 0) {
+        // Consume the first \ if the value is quoted.
+        if (quoted && character == '\\' && !escaped) {
+            escaped = true;
+            // Drop this slash by short circuiting the rest of the loop.
+            f_read(active_file, &character, 1, &quantity_read);
+            continue;
+        }
+        if (quoted && !escaped && character == '\'') {
+            // trailing ' means the value is done.
+            break;
+        }
+        // Unquoted values are ended by a newline or comment.
+        if (!quoted && (character == '\n' || character == '#')) {
+            break;
+        }
+        if (!quoted && unichar_isspace(character)) {
+            trailing_spaces += 1;
+        } else {
+            trailing_spaces = 0;
+        }
+        escaped = false;
+        // Only copy the value over if we have space. Otherwise, we'll just
+        // count the overall length.
+        if (value_pos < value_len) {
+            value[value_pos] = character;
+        }
+        value_pos++;
+        f_read(active_file, &character, 1, &quantity_read);
+    }
+
+    return value_pos - trailing_spaces;
+}
+
+mp_int_t dotenv_get_key(const char *path, const char *key, char *value, mp_int_t value_len) {
+    FIL active_file;
+    FATFS *fs = &((fs_user_mount_t *)MP_STATE_VM(vfs_mount_table)->obj)->fatfs;
+    FRESULT result = f_open(fs, &active_file, path, FA_READ);
+    if (result != FR_OK) {
+        return -1;
+    }
+
+    mp_int_t actual_value_len = -1;
+    bool read_ok = true;
+    while (read_ok) {
+        if (key_matches(&active_file, key)) {
+            actual_value_len = read_value(&active_file, value, value_len);
+        }
+
+        read_ok = next_line(&active_file);
+    }
+    f_close(&active_file);
+    return actual_value_len;
+}
+
+mp_obj_t common_hal_dotenv_get_key(const char *path, const char *key) {
+    // Use the stack for short values. Longer values will require a heap allocation after we know
+    // the length.
+    char value[64];
+    mp_int_t actual_len = dotenv_get_key(path, key, value, sizeof(value));
+    if (actual_len <= 0) {
+        return mp_const_none;
+    }
+    if ((size_t)actual_len >= sizeof(value)) {
+        mp_obj_str_t *str = MP_OBJ_TO_PTR(mp_obj_new_str_copy(&mp_type_str, NULL, actual_len + 1));
+        dotenv_get_key(path, key, (char *)str->data, actual_len + 1);
+        str->hash = qstr_compute_hash(str->data, actual_len);
+        return MP_OBJ_FROM_PTR(str);
+    }
+    return mp_obj_new_str(value, actual_len);
+}

shared-module/dotenv/__init__.h

@@ -0,0 +1,28 @@
+/*
+ * This file is part of the MicroPython project, http://micropython.org/
+ *
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2022 Scott Shawcroft for Adafruit Industries
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+// Allocation free version that returns the full length of the value.
+mp_int_t dotenv_get_key(const char *path, const char *key, char *value, mp_int_t value_len);

shared-module/os/__init__.c

@@ -36,6 +36,10 @@
 #include "py/runtime.h"
 #include "shared-bindings/os/__init__.h"
 
+#if CIRCUITPY_DOTENV
+#include "shared-bindings/dotenv/__init__.h"
+#endif
+
 // This provides all VFS related OS functions so that ports can share the code
 // as needed. It does not provide uname.
 
@@ -107,6 +111,16 @@ mp_obj_t common_hal_os_getcwd(void) {
     return mp_vfs_getcwd();
 }
 
+mp_obj_t common_hal_os_getenv(const char *key, mp_obj_t default_) {
+    #if CIRCUITPY_DOTENV
+    mp_obj_t env_obj = common_hal_dotenv_get_key("/.env", key);
+    if (env_obj != mp_const_none) {
+        return env_obj;
+    }
+    #endif
+    return default_;
+}
+
 mp_obj_t common_hal_os_listdir(const char *path) {
     mp_obj_t path_out;
     mp_vfs_mount_t *vfs = lookup_dir_path(path, &path_out);