Add dotenv read support

os.getenv() will use it (when available) to load variables from
/.env

This will also be useful when we need secrets or config for
CircuitPython outside of the VM (like WiFi credentials.)

Fixes #4212
This commit is contained in:
Scott Shawcroft 2022-05-18 14:01:35 -07:00
parent 1dce806f77
commit a30de85d57
No known key found for this signature in database
GPG Key ID: 0DFD512649C052DA
9 changed files with 435 additions and 0 deletions

View File

@ -158,6 +158,9 @@ endif
ifeq ($(CIRCUITPY_DISPLAYIO),1)
SRC_PATTERNS += displayio/%
endif
ifeq ($(CIRCUITPY_DOTENV),1)
SRC_PATTERNS += dotenv/%
endif
ifeq ($(CIRCUITPY_PARALLELDISPLAY),1)
SRC_PATTERNS += paralleldisplay/%
endif
@ -548,6 +551,7 @@ SRC_SHARED_MODULE_ALL = \
displayio/TileGrid.c \
displayio/area.c \
displayio/__init__.c \
dotenv/__init__.c \
floppyio/__init__.c \
fontio/BuiltinFont.c \
fontio/__init__.c \

View File

@ -199,6 +199,9 @@ CFLAGS += -DCIRCUITPY_BITMAPTOOLS=$(CIRCUITPY_BITMAPTOOLS)
CFLAGS += -DCIRCUITPY_FRAMEBUFFERIO=$(CIRCUITPY_FRAMEBUFFERIO)
CFLAGS += -DCIRCUITPY_VECTORIO=$(CIRCUITPY_VECTORIO)
CIRCUITPY_DOTENV ?= $(CIRCUITPY_FULL_BUILD)
CFLAGS += -DCIRCUITPY_DOTENV=$(CIRCUITPY_DOTENV)
CIRCUITPY_DUALBANK ?= 0
CFLAGS += -DCIRCUITPY_DUALBANK=$(CIRCUITPY_DUALBANK)

View File

@ -0,0 +1,105 @@
/*
* This file is part of the Micro Python project, http://micropython.org/
*
* The MIT License (MIT)
*
* SPDX-FileCopyrightText: Copyright (c) 2022 Scott Shawcroft for Adafruit Industries
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
*/
#include <string.h>
#include "extmod/vfs.h"
#include "lib/oofatfs/ff.h"
#include "lib/oofatfs/diskio.h"
#include "py/mpstate.h"
#include "py/obj.h"
#include "py/objstr.h"
#include "py/runtime.h"
#include "shared-bindings/dotenv/__init__.h"
//| """Functions to manage environment variables from a .env file.
//|
//| A subset of the CPython `dotenv library <https://saurabh-kumar.com/python-dotenv/>`_. It does
//| not support variables or double quotes.
//|
//| The simplest way to define keys and values is to put them in single quotes. \ and ' are
//| escaped by \ in single quotes. Newlines can occur in quotes for multiline values. Comments
//| start with # and apply for the rest of the line.
//|
//| File format example:
//|
//| .. code-block::
//|
//| key=value
//| key2 = value2
//| 'key3' = 'value with spaces'
//| # comment
//| key4 = value3 # comment 2
//| 'key5'=value4
//| key=value5 # overrides the first one
//| multiline = 'hello
//| world
//| how are you?'
//|
//| """
//|
//| import typing
//| def get_key(dotenv_path: str, key_to_get: str) -> Optional[str]:
//| """Get the value for the given key from the given .env file. If the key occurs multiple
//| times in the file, then the last value will be returned.
//|
//| Returns None if the key isn't found or doesn't have a value."""
//| ...
//|
STATIC mp_obj_t _dotenv_get_key(mp_obj_t path_in, mp_obj_t key_to_get_in) {
return common_hal_dotenv_get_key(mp_obj_str_get_str(path_in),
mp_obj_str_get_str(key_to_get_in));
}
MP_DEFINE_CONST_FUN_OBJ_2(dotenv_get_key_obj, _dotenv_get_key);
//| def load_dotenv() -> None:
//| """Does nothing in CircuitPython because os.getenv will automatically read .env when
//| available.
//|
//| Present in CircuitPython so CPython-compatible code can use it without error."""
//| ...
//|
STATIC mp_obj_t dotenv_load_dotenv(void) {
return mp_const_none;
}
MP_DEFINE_CONST_FUN_OBJ_0(dotenv_load_dotenv_obj, dotenv_load_dotenv);
STATIC const mp_rom_map_elem_t dotenv_module_globals_table[] = {
{ MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_dotenv) },
{ MP_ROM_QSTR(MP_QSTR_get_key), MP_ROM_PTR(&dotenv_get_key_obj) },
{ MP_ROM_QSTR(MP_QSTR_load_dotenv), MP_ROM_PTR(&dotenv_load_dotenv_obj) },
};
STATIC MP_DEFINE_CONST_DICT(dotenv_module_globals, dotenv_module_globals_table);
const mp_obj_module_t dotenv_module = {
.base = { &mp_type_module },
.globals = (mp_obj_dict_t *)&dotenv_module_globals,
};
MP_REGISTER_MODULE(MP_QSTR_dotenv, dotenv_module, CIRCUITPY_DOTENV);

View File

@ -0,0 +1,39 @@
/*
* This file is part of the MicroPython project, http://micropython.org/
*
* The MIT License (MIT)
*
* Copyright (c) 2022 Scott Shawcroft for Adafruit Industries
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
*/
#ifndef MICROPY_INCLUDED_SHARED_BINDINGS_DOTENV___INIT___H
#define MICROPY_INCLUDED_SHARED_BINDINGS_DOTENV___INIT___H
#include <stdint.h>
#include <stdbool.h>
#include "py/objtuple.h"
#include "shared-module/dotenv/__init__.h"
mp_obj_t common_hal_dotenv_get_key(const char *path, const char *key);
#endif // MICROPY_INCLUDED_SHARED_BINDINGS_DOTENV___INIT___H

View File

@ -83,6 +83,25 @@ STATIC mp_obj_t os_getcwd(void) {
}
MP_DEFINE_CONST_FUN_OBJ_0(os_getcwd_obj, os_getcwd);
//| def getenv(key: str, default: Optional[str] = None) -> Optional[str]:
//| """Get the environment variable value for the given key or return ``default``.
//|
//| This may load values from disk so cache the result instead of calling this often."""
//| ...
//|
STATIC mp_obj_t os_getenv(size_t n_args, const mp_obj_t *pos_args, mp_map_t *kw_args) {
enum { ARG_key, ARG_default };
static const mp_arg_t allowed_args[] = {
{ MP_QSTR_key, MP_ARG_REQUIRED | MP_ARG_OBJ },
{ MP_QSTR_default, MP_ARG_OBJ, {.u_obj = mp_const_none} },
};
mp_arg_val_t args[MP_ARRAY_SIZE(allowed_args)];
mp_arg_parse_all(n_args, pos_args, kw_args, MP_ARRAY_SIZE(allowed_args), allowed_args, args);
return common_hal_os_getenv(mp_obj_str_get_str(args[ARG_key].u_obj), args[ARG_default].u_obj);
}
STATIC MP_DEFINE_CONST_FUN_OBJ_KW(os_getenv_obj, 1, os_getenv);
//| def listdir(dir: str) -> str:
//| """With no argument, list the current directory. Otherwise list the given directory."""
//| ...
@ -220,6 +239,7 @@ STATIC const mp_rom_map_elem_t os_module_globals_table[] = {
{ MP_ROM_QSTR(MP_QSTR_chdir), MP_ROM_PTR(&os_chdir_obj) },
{ MP_ROM_QSTR(MP_QSTR_getcwd), MP_ROM_PTR(&os_getcwd_obj) },
{ MP_ROM_QSTR(MP_QSTR_getenv), MP_ROM_PTR(&os_getenv_obj) },
{ MP_ROM_QSTR(MP_QSTR_listdir), MP_ROM_PTR(&os_listdir_obj) },
{ MP_ROM_QSTR(MP_QSTR_mkdir), MP_ROM_PTR(&os_mkdir_obj) },
{ MP_ROM_QSTR(MP_QSTR_remove), MP_ROM_PTR(&os_remove_obj) },

View File

@ -37,6 +37,7 @@ extern const mp_rom_obj_tuple_t common_hal_os_uname_info_obj;
mp_obj_t common_hal_os_uname(void);
void common_hal_os_chdir(const char *path);
mp_obj_t common_hal_os_getcwd(void);
mp_obj_t common_hal_os_getenv(const char *key, mp_obj_t default_);
mp_obj_t common_hal_os_listdir(const char *path);
void common_hal_os_mkdir(const char *path);
void common_hal_os_remove(const char *path);

View File

@ -0,0 +1,221 @@
/*
* This file is part of the Micro Python project, http://micropython.org/
*
* The MIT License (MIT)
*
* SPDX-FileCopyrightText: Copyright (c) 2022 Scott Shawcroft for Adafruit Industries
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
*/
#include <string.h>
#include "shared-bindings/dotenv/__init__.h"
#include "extmod/vfs.h"
#include "extmod/vfs_fat.h"
#include "py/mpstate.h"
#include "py/objstr.h"
STATIC uint8_t consume_spaces(FIL *active_file) {
uint8_t character = ' ';
size_t quantity_read = 1;
while (unichar_isspace(character) && quantity_read > 0) {
f_read(active_file, &character, 1, &quantity_read);
}
return character;
}
// Starting at the start of a new line, determines if the key matches the given
// key. File pointer is left after the = after the key.
STATIC bool key_matches(FIL *active_file, const char *key) {
uint8_t character = ' ';
size_t quantity_read = 1;
character = consume_spaces(active_file);
bool quoted = false;
if (character == '\'') {
quoted = true;
f_read(active_file, &character, 1, &quantity_read);
}
size_t key_pos = 0;
bool escaped = false;
bool matches = true;
size_t key_len = strlen(key);
while (quantity_read > 0) {
if (character == '\\' && !escaped && quoted) {
escaped = true;
} else if (!escaped && quoted && character == '\'') {
quoted = false;
// Move past the quoted before breaking so we can check the validity of data past it.
f_read(active_file, &character, 1, &quantity_read);
break;
} else if (!quoted && (unichar_isspace(character) || character == '=' || character == '\n' || character == '#')) {
break;
} else {
matches = matches && key[key_pos] == character;
escaped = false;
key_pos++;
}
f_read(active_file, &character, 1, &quantity_read);
}
if (unichar_isspace(character)) {
character = consume_spaces(active_file);
}
if (character == '=' || character == '\n' || character == '#') {
// Rewind one so the value can find it.
f_lseek(active_file, f_tell(active_file) - 1);
} else {
// We're followed by something else that is invalid syntax.
matches = false;
}
return matches && key_pos == key_len;
}
STATIC bool next_line(FIL *active_file) {
uint8_t character = ' ';
size_t quantity_read = 1;
bool quoted = false;
bool escaped = false;
// Track comments because they last until the end of the line.
bool comment = false;
FRESULT result = FR_OK;
// Consume all characters while quoted or others up to \n.
while (result == FR_OK && quantity_read > 0 && (quoted || character != '\n')) {
if (character == '#' || comment) {
// Comments consume any escaping.
comment = true;
} else if (!escaped) {
if (character == '\'') {
quoted = !quoted;
} else if (character == '\\') {
escaped = true;
}
} else {
escaped = false;
}
result = f_read(active_file, &character, 1, &quantity_read);
}
return result == FR_OK && quantity_read > 0;
}
STATIC mp_int_t read_value(FIL *active_file, char *value, size_t value_len) {
uint8_t character = ' ';
size_t quantity_read = 1;
// Consume spaces before =
character = consume_spaces(active_file);
if (character != '=') {
if (character == '#' || character == '\n') {
// Keys without an = after them are valid with the value None.
return 0;
}
// All other characters are invalid.
return -1;
}
character = ' ';
// Consume space after =
while (unichar_isspace(character) && quantity_read > 0) {
f_read(active_file, &character, 1, &quantity_read);
}
bool quoted = false;
if (character == '\'') {
quoted = true;
f_read(active_file, &character, 1, &quantity_read);
}
if (character == '"') {
// We don't support double quoted values.
return -1;
}
// Copy the value over.
size_t value_pos = 0;
bool escaped = false;
// Count trailing spaces so we can ignore them at the end of unquoted
// values.
size_t trailing_spaces = 0;
while (quantity_read > 0) {
// Consume the first \ if the value is quoted.
if (quoted && character == '\\' && !escaped) {
escaped = true;
// Drop this slash by short circuiting the rest of the loop.
f_read(active_file, &character, 1, &quantity_read);
continue;
}
if (quoted && !escaped && character == '\'') {
// trailing ' means the value is done.
break;
}
// Unquoted values are ended by a newline or comment.
if (!quoted && (character == '\n' || character == '#')) {
break;
}
if (!quoted && unichar_isspace(character)) {
trailing_spaces += 1;
} else {
trailing_spaces = 0;
}
escaped = false;
// Only copy the value over if we have space. Otherwise, we'll just
// count the overall length.
if (value_pos < value_len) {
value[value_pos] = character;
}
value_pos++;
f_read(active_file, &character, 1, &quantity_read);
}
return value_pos - trailing_spaces;
}
mp_int_t dotenv_get_key(const char *path, const char *key, char *value, mp_int_t value_len) {
FIL active_file;
FATFS *fs = &((fs_user_mount_t *)MP_STATE_VM(vfs_mount_table)->obj)->fatfs;
FRESULT result = f_open(fs, &active_file, path, FA_READ);
if (result != FR_OK) {
return -1;
}
mp_int_t actual_value_len = -1;
bool read_ok = true;
while (read_ok) {
if (key_matches(&active_file, key)) {
actual_value_len = read_value(&active_file, value, value_len);
}
read_ok = next_line(&active_file);
}
f_close(&active_file);
return actual_value_len;
}
mp_obj_t common_hal_dotenv_get_key(const char *path, const char *key) {
// Use the stack for short values. Longer values will require a heap allocation after we know
// the length.
char value[64];
mp_int_t actual_len = dotenv_get_key(path, key, value, sizeof(value));
if (actual_len <= 0) {
return mp_const_none;
}
if ((size_t)actual_len >= sizeof(value)) {
mp_obj_str_t *str = MP_OBJ_TO_PTR(mp_obj_new_str_copy(&mp_type_str, NULL, actual_len + 1));
dotenv_get_key(path, key, (char *)str->data, actual_len + 1);
str->hash = qstr_compute_hash(str->data, actual_len);
return MP_OBJ_FROM_PTR(str);
}
return mp_obj_new_str(value, actual_len);
}

View File

@ -0,0 +1,28 @@
/*
* This file is part of the MicroPython project, http://micropython.org/
*
* The MIT License (MIT)
*
* Copyright (c) 2022 Scott Shawcroft for Adafruit Industries
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
*/
// Allocation free version that returns the full length of the value.
mp_int_t dotenv_get_key(const char *path, const char *key, char *value, mp_int_t value_len);

View File

@ -36,6 +36,10 @@
#include "py/runtime.h"
#include "shared-bindings/os/__init__.h"
#if CIRCUITPY_DOTENV
#include "shared-bindings/dotenv/__init__.h"
#endif
// This provides all VFS related OS functions so that ports can share the code
// as needed. It does not provide uname.
@ -107,6 +111,16 @@ mp_obj_t common_hal_os_getcwd(void) {
return mp_vfs_getcwd();
}
mp_obj_t common_hal_os_getenv(const char *key, mp_obj_t default_) {
#if CIRCUITPY_DOTENV
mp_obj_t env_obj = common_hal_dotenv_get_key("/.env", key);
if (env_obj != mp_const_none) {
return env_obj;
}
#endif
return default_;
}
mp_obj_t common_hal_os_listdir(const char *path) {
mp_obj_t path_out;
mp_vfs_mount_t *vfs = lookup_dir_path(path, &path_out);