From 777542c716e07e78439c3c6bbf442722ebaaecc6 Mon Sep 17 00:00:00 2001 From: Scott Shawcroft Date: Thu, 24 May 2018 13:12:18 -0700 Subject: [PATCH] Add basic memory allocation outside Python runtime This allows for the heap to fill all space but the stack. It also allows us to designate space for memory outside the runtime for things such as USB descriptors, flash cache and main filename. Fixes #754 --- main.c | 79 ++++++++------ ports/atmel-samd/Makefile | 2 +- .../external_flash/external_flash.c | 29 ++++- ports/atmel-samd/supervisor/filesystem.c | 1 + ports/atmel-samd/supervisor/memory.c | 100 ++++++++++++++++++ shared-bindings/supervisor/__init__.c | 13 ++- supervisor/memory.h | 43 ++++++++ supervisor/supervisor.mk | 1 + 8 files changed, 233 insertions(+), 35 deletions(-) mode change 100644 => 100755 ports/atmel-samd/external_flash/external_flash.c mode change 100644 => 100755 ports/atmel-samd/supervisor/filesystem.c create mode 100755 ports/atmel-samd/supervisor/memory.c mode change 100644 => 100755 shared-bindings/supervisor/__init__.c create mode 100755 supervisor/memory.h mode change 100644 => 100755 supervisor/supervisor.mk diff --git a/main.c b/main.c index 91d8200892..73bc5a0873 100755 --- a/main.c +++ b/main.c @@ -45,6 +45,7 @@ #include "mpconfigboard.h" #include "supervisor/cpu.h" +#include "supervisor/memory.h" #include "supervisor/port.h" #include "supervisor/filesystem.h" // TODO(tannewt): Figure out how to choose language at compile time. @@ -73,12 +74,21 @@ void do_str(const char *src, mp_parse_input_kind_t input_kind) { } } -static char heap[PORT_HEAP_SIZE]; - -void reset_mp(void) { +void start_mp(supervisor_allocation* heap) { reset_status_led(); autoreload_stop(); + // Stack limit should be less than real stack size, so we have a chance + // to recover from limit hit. (Limit is measured in bytes.) + mp_stack_ctrl_init(); + mp_stack_set_limit((char*)&_estack - (char*)&_ebss - 1024); + +#if MICROPY_MAX_STACK_USAGE + // _ezero (same as _ebss) is an int, so start 4 bytes above it. + mp_stack_set_bottom(&_ezero + 1); + mp_stack_fill_with_sentinel(); +#endif + // Sync the file systems in case any used RAM from the GC to cache. As soon // as we re-init the GC all bets are off on the cache. filesystem_flush(); @@ -87,7 +97,7 @@ void reset_mp(void) { readline_init0(); #if MICROPY_ENABLE_GC - gc_init(heap, heap + sizeof(heap)); + gc_init(heap->ptr, heap->ptr + heap->length); #endif mp_init(); mp_obj_list_init(mp_sys_path, 0); @@ -99,6 +109,11 @@ void reset_mp(void) { mp_obj_list_init(mp_sys_argv, 0); } + +void stop_mp(void) { + +} + #define STRING_LIST(...) {__VA_ARGS__, ""} // Look for the first file that exists in the list of filenames, using mp_import_stat(). @@ -124,7 +139,7 @@ bool maybe_run_list(const char ** filenames, pyexec_result_t* exec_result) { return true; } -bool start_mp(safe_mode_t safe_mode) { +bool run_code_py(safe_mode_t safe_mode) { bool serial_connected_at_start = serial_connected(); #ifdef CIRCUITPY_AUTORELOAD_DELAY_MS if (serial_connected_at_start) { @@ -155,7 +170,9 @@ bool start_mp(safe_mode_t safe_mode) { const char *supported_filenames[] = STRING_LIST("code.txt", "code.py", "main.py", "main.txt"); const char *double_extension_filenames[] = STRING_LIST("code.txt.py", "code.py.txt", "code.txt.txt","code.py.py", "main.txt.py", "main.py.txt", "main.txt.txt","main.py.py"); - reset_mp(); + + supervisor_allocation* heap = allocate_remaining_memory(); + start_mp(heap); found_main = maybe_run_list(supported_filenames, &result); if (!found_main){ found_main = maybe_run_list(double_extension_filenames, &result); @@ -163,6 +180,8 @@ bool start_mp(safe_mode_t safe_mode) { serial_write(MSG_DOUBLE_FILE_EXTENSION); } } + stop_mp(); + free_memory(heap); reset_port(); reset_board(); @@ -306,27 +325,36 @@ void __attribute__ ((noinline)) run_boot_py(safe_mode_t safe_mode) { // Reset to remove any state that boot.py setup. It should only be used to // change internal state that's not in the heap. reset_port(); - reset_mp(); } } +int run_repl(void) { + int exit_code = PYEXEC_FORCED_EXIT; + supervisor_allocation* heap = allocate_remaining_memory(); + start_mp(heap); + autoreload_suspend(); + new_status_color(REPL_RUNNING); + if (pyexec_mode_kind == PYEXEC_MODE_RAW_REPL) { + exit_code = pyexec_raw_repl(); + } else { + exit_code = pyexec_friendly_repl(); + } + reset_port(); + reset_board(); + stop_mp(); + free_memory(heap); + autoreload_resume(); + return exit_code; +} + int __attribute__((used)) main(void) { + memory_init(); + // initialise the cpu and peripherals safe_mode_t safe_mode = port_init(); rgb_led_status_init(); - // Stack limit should be less than real stack size, so we have a chance - // to recover from limit hit. (Limit is measured in bytes.) - mp_stack_set_top((char*)&_estack); - mp_stack_set_limit((char*)&_estack - (char*)&_ebss - 1024); - -#if MICROPY_MAX_STACK_USAGE - // _ezero (same as _ebss) is an int, so start 4 bytes above it. - mp_stack_set_bottom(&_ezero + 1); - mp_stack_fill_with_sentinel(); -#endif - // Create a new filesystem only if we're not in a safe mode. // A power brownout here could make it appear as if there's // no SPI flash filesystem, and we might erase the existing one. @@ -335,7 +363,6 @@ int __attribute__((used)) main(void) { // Reset everything and prep MicroPython to run boot.py. reset_port(); reset_board(); - reset_mp(); // Turn on autoreload by default but before boot.py in case it wants to change it. autoreload_enable(); @@ -355,24 +382,14 @@ int __attribute__((used)) main(void) { bool first_run = true; for (;;) { if (!skip_repl) { - reset_mp(); - autoreload_suspend(); - new_status_color(REPL_RUNNING); - if (pyexec_mode_kind == PYEXEC_MODE_RAW_REPL) { - exit_code = pyexec_raw_repl(); - } else { - exit_code = pyexec_friendly_repl(); - } - autoreload_resume(); - reset_port(); - reset_board(); + exit_code = run_repl(); } if (exit_code == PYEXEC_FORCED_EXIT) { if (!first_run) { serial_write(MSG_SOFT_REBOOT MSG_NEWLINE); } first_run = false; - skip_repl = start_mp(safe_mode); + skip_repl = run_code_py(safe_mode); } else if (exit_code != 0) { break; } diff --git a/ports/atmel-samd/Makefile b/ports/atmel-samd/Makefile index 249f003f9b..008efb1483 100755 --- a/ports/atmel-samd/Makefile +++ b/ports/atmel-samd/Makefile @@ -106,7 +106,7 @@ ifeq ($(DEBUG), 1) # Turn on Python modules useful for debugging (e.g. uheap, ustack). CFLAGS += -ggdb # You may want to disable -flto if it interferes with debugging. - CFLAGS += -flto + #CFLAGS += -flto # You may want to enable these flags to make setting breakpoints easier. ## CFLAGS += -fno-inline -fno-ipa-sra ifeq ($(CHIP_FAMILY), samd21) diff --git a/ports/atmel-samd/external_flash/external_flash.c b/ports/atmel-samd/external_flash/external_flash.c old mode 100644 new mode 100755 index e6c407e73e..b4a637be43 --- a/ports/atmel-samd/external_flash/external_flash.c +++ b/ports/atmel-samd/external_flash/external_flash.c @@ -37,6 +37,7 @@ #include "py/runtime.h" #include "lib/oofatfs/ff.h" #include "shared-bindings/microcontroller/__init__.h" +#include "supervisor/memory.h" #include "supervisor/shared/rgb_led_status.h" #include "hal_gpio.h" @@ -59,6 +60,8 @@ static const external_flash_device* flash_device = NULL; // cache. static uint32_t dirty_mask; +static supervisor_allocation* supervisor_cache = NULL; + // Wait until both the write enable and write in progress bits have cleared. static bool wait_for_flash_ready(void) { uint8_t read_status_response[1] = {0x00}; @@ -308,6 +311,23 @@ static bool flush_scratch_flash(void) { static bool allocate_ram_cache(void) { uint8_t blocks_per_sector = SPI_FLASH_ERASE_SIZE / FILESYSTEM_BLOCK_SIZE; uint8_t pages_per_block = FILESYSTEM_BLOCK_SIZE / SPI_FLASH_PAGE_SIZE; + + uint32_t table_size = blocks_per_sector * pages_per_block * sizeof(uint32_t); + // Attempt to allocate outside the heap first. + supervisor_cache = allocate_memory(table_size + SPI_FLASH_ERASE_SIZE, false); + if (supervisor_cache != NULL) { + MP_STATE_VM(flash_ram_cache) = (uint8_t **) supervisor_cache->ptr; + uint8_t* page_start = (uint8_t *) supervisor_cache->ptr + table_size; + + for (uint8_t i = 0; i < blocks_per_sector; i++) { + for (uint8_t j = 0; j < pages_per_block; j++) { + uint32_t offset = i * pages_per_block + j; + MP_STATE_VM(flash_ram_cache)[offset] = page_start + offset * SPI_FLASH_PAGE_SIZE; + } + } + return true; + } + MP_STATE_VM(flash_ram_cache) = m_malloc_maybe(blocks_per_sector * pages_per_block * sizeof(uint32_t), false); if (MP_STATE_VM(flash_ram_cache) == NULL) { return false; @@ -383,14 +403,19 @@ static bool flush_ram_cache(bool keep_cache) { write_flash(current_sector + (i * pages_per_block + j) * SPI_FLASH_PAGE_SIZE, MP_STATE_VM(flash_ram_cache)[i * pages_per_block + j], SPI_FLASH_PAGE_SIZE); - if (!keep_cache) { + if (!keep_cache && supervisor_cache == NULL) { m_free(MP_STATE_VM(flash_ram_cache)[i * pages_per_block + j]); } } } // We're done with the cache for now so give it back. if (!keep_cache) { - m_free(MP_STATE_VM(flash_ram_cache)); + if (supervisor_cache != NULL) { + free_memory(supervisor_cache); + supervisor_cache = NULL; + } else { + m_free(MP_STATE_VM(flash_ram_cache)); + } MP_STATE_VM(flash_ram_cache) = NULL; } return true; diff --git a/ports/atmel-samd/supervisor/filesystem.c b/ports/atmel-samd/supervisor/filesystem.c old mode 100644 new mode 100755 index 8a6010951f..8f4fef64af --- a/ports/atmel-samd/supervisor/filesystem.c +++ b/ports/atmel-samd/supervisor/filesystem.c @@ -51,6 +51,7 @@ void filesystem_init(bool create_allowed, bool force_create) { // try to mount the flash FRESULT res = f_mount(&vfs_fat->fatfs); + force_create = true; if ((res == FR_NO_FILESYSTEM && create_allowed) || force_create) { // No filesystem so create a fresh one, or reformat has been requested. diff --git a/ports/atmel-samd/supervisor/memory.c b/ports/atmel-samd/supervisor/memory.c new file mode 100755 index 0000000000..4aad30ecba --- /dev/null +++ b/ports/atmel-samd/supervisor/memory.c @@ -0,0 +1,100 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2018 Scott Shawcroft for Adafruit Industries + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include "supervisor/memory.h" + +#include + +#define CIRCUITPY_SUPERVISOR_ALLOC_COUNT 8 + +static supervisor_allocation allocations[CIRCUITPY_SUPERVISOR_ALLOC_COUNT]; +// We use uint32_t* to ensure word (4 byte) alignment. +uint32_t* low_address; +uint32_t* high_address; +extern uint32_t _ebss; +extern uint32_t _estack; + +void memory_init(void) { + low_address = &_ebss; + high_address = &_estack; + + // snag 3k for stack + allocate_memory(1024*4, true); +} + +void free_memory(supervisor_allocation* allocation) { + uint8_t index = 0; + bool found = false; + for (index = 0; index < CIRCUITPY_SUPERVISOR_ALLOC_COUNT; index++) { + found = allocation == &allocations[index]; + if (found) { + break; + } + } + if (allocation->ptr == high_address) { + high_address += allocation->length / 4; + } else if (allocation->ptr + allocation->length / 4 == low_address) { + low_address = allocation->ptr; + } + allocation->ptr = NULL; +} + +supervisor_allocation* allocate_remaining_memory(void) { + if (low_address == high_address) { + return NULL; + } + return allocate_memory((high_address - low_address) * 4, false); +} + +supervisor_allocation* allocate_memory(uint32_t length, bool high) { + if ((high_address - low_address) * 4 < (int32_t) length) { + return NULL; + } + uint8_t index = 0; + int8_t direction = 1; + if (high) { + index = CIRCUITPY_SUPERVISOR_ALLOC_COUNT - 1; + direction = -1; + } + for (; index < CIRCUITPY_SUPERVISOR_ALLOC_COUNT; index += direction) { + if (allocations[index].ptr == NULL) { + break; + } + } + if (index >= CIRCUITPY_SUPERVISOR_ALLOC_COUNT) { + return NULL; + } + supervisor_allocation* alloc = &allocations[index]; + if (high) { + high_address -= length / 4; + alloc->ptr = high_address; + } else { + alloc->ptr = low_address; + low_address += length / 4; + } + alloc->length = length; + return alloc; +} diff --git a/shared-bindings/supervisor/__init__.c b/shared-bindings/supervisor/__init__.c old mode 100644 new mode 100755 index e6429be320..619af5a35d --- a/shared-bindings/supervisor/__init__.c +++ b/shared-bindings/supervisor/__init__.c @@ -31,7 +31,7 @@ #include "supervisor/shared/autoreload.h" #include "supervisor/shared/rgb_led_status.h" - + #include "shared-bindings/supervisor/__init__.h" #include "shared-bindings/supervisor/Runtime.h" @@ -107,6 +107,15 @@ STATIC mp_obj_t supervisor_reload(void) { } MP_DEFINE_CONST_FUN_OBJ_0(supervisor_reload_obj, supervisor_reload); +//| .. method:: set_main_file(filename) +//| +//| Set which file to run after a reload. +//| +STATIC mp_obj_t supervisor_set_main_file(void) { + + return mp_const_none; +} +MP_DEFINE_CONST_FUN_OBJ_0(supervisor_set_main_file_obj, supervisor_set_main_file); STATIC const mp_rom_map_elem_t supervisor_module_globals_table[] = { { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_supervisor) }, @@ -115,6 +124,8 @@ STATIC const mp_rom_map_elem_t supervisor_module_globals_table[] = { { MP_OBJ_NEW_QSTR(MP_QSTR_set_rgb_status_brightness), MP_ROM_PTR(&supervisor_set_rgb_status_brightness_obj) }, { MP_ROM_QSTR(MP_QSTR_runtime), MP_ROM_PTR(&common_hal_supervisor_runtime_obj) }, { MP_ROM_QSTR(MP_QSTR_reload), MP_ROM_PTR(&supervisor_reload_obj) }, + { MP_ROM_QSTR(MP_QSTR_set_main_file), MP_ROM_PTR(&supervisor_set_main_file_obj) }, + }; STATIC MP_DEFINE_CONST_DICT(supervisor_module_globals, supervisor_module_globals_table); diff --git a/supervisor/memory.h b/supervisor/memory.h new file mode 100755 index 0000000000..1d64c2cde5 --- /dev/null +++ b/supervisor/memory.h @@ -0,0 +1,43 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2018 Scott Shawcroft for Adafruit Industries + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#ifndef MICROPY_INCLUDED_SUPERVISOR_MEMORY_H +#define MICROPY_INCLUDED_SUPERVISOR_MEMORY_H + +#include +#include + +typedef struct { + uint32_t* ptr; + uint32_t length; // in bytes +} supervisor_allocation; + +void memory_init(void); +void free_memory(supervisor_allocation* allocation); +supervisor_allocation* allocate_remaining_memory(void); +supervisor_allocation* allocate_memory(uint32_t length, bool high_address); + +#endif // MICROPY_INCLUDED_SUPERVISOR_MEMORY_H diff --git a/supervisor/supervisor.mk b/supervisor/supervisor.mk old mode 100644 new mode 100755 index c56557197c..ea6afae299 --- a/supervisor/supervisor.mk +++ b/supervisor/supervisor.mk @@ -1,5 +1,6 @@ SRC_SUPERVISOR = \ main.c \ + supervisor/memory.c \ supervisor/port.c \ supervisor/shared/autoreload.c \ supervisor/shared/rgb_led_status.c