Add basic memory allocation outside Python runtime

This allows for the heap to fill all space but the stack. It also
allows us to designate space for memory outside the runtime for
things such as USB descriptors, flash cache and main filename.

Fixes #754

main.c

@@ -45,6 +45,7 @@
 
 #include "mpconfigboard.h"
 #include "supervisor/cpu.h"
+#include "supervisor/memory.h"
 #include "supervisor/port.h"
 #include "supervisor/filesystem.h"
 // TODO(tannewt): Figure out how to choose language at compile time.
@@ -73,12 +74,21 @@ void do_str(const char *src, mp_parse_input_kind_t input_kind) {
     }
 }
 
-static char heap[PORT_HEAP_SIZE];
-
-void reset_mp(void) {
+void start_mp(supervisor_allocation* heap) {
     reset_status_led();
     autoreload_stop();
 
+    // Stack limit should be less than real stack size, so we have a chance
+    // to recover from limit hit.  (Limit is measured in bytes.)
+    mp_stack_ctrl_init();
+    mp_stack_set_limit((char*)&_estack - (char*)&_ebss - 1024);
+
+#if MICROPY_MAX_STACK_USAGE
+    // _ezero (same as _ebss) is an int, so start 4 bytes above it.
+    mp_stack_set_bottom(&_ezero + 1);
+    mp_stack_fill_with_sentinel();
+#endif
+
     // Sync the file systems in case any used RAM from the GC to cache. As soon
     // as we re-init the GC all bets are off on the cache.
     filesystem_flush();
@@ -87,7 +97,7 @@ void reset_mp(void) {
     readline_init0();
 
     #if MICROPY_ENABLE_GC
-    gc_init(heap, heap + sizeof(heap));
+    gc_init(heap->ptr, heap->ptr + heap->length);
     #endif
     mp_init();
     mp_obj_list_init(mp_sys_path, 0);
@@ -99,6 +109,11 @@ void reset_mp(void) {
 
     mp_obj_list_init(mp_sys_argv, 0);
 }
+
+void stop_mp(void) {
+
+}
+
 #define STRING_LIST(...) {__VA_ARGS__, ""}
 
 // Look for the first file that exists in the list of filenames, using mp_import_stat().
@@ -124,7 +139,7 @@ bool maybe_run_list(const char ** filenames, pyexec_result_t* exec_result) {
     return true;
 }
 
-bool start_mp(safe_mode_t safe_mode) {
+bool run_code_py(safe_mode_t safe_mode) {
     bool serial_connected_at_start = serial_connected();
     #ifdef CIRCUITPY_AUTORELOAD_DELAY_MS
     if (serial_connected_at_start) {
@@ -155,7 +170,9 @@ bool start_mp(safe_mode_t safe_mode) {
         const char *supported_filenames[] = STRING_LIST("code.txt", "code.py", "main.py", "main.txt");
         const char *double_extension_filenames[] = STRING_LIST("code.txt.py", "code.py.txt", "code.txt.txt","code.py.py",
                                                     "main.txt.py", "main.py.txt", "main.txt.txt","main.py.py");
-        reset_mp();
+
+        supervisor_allocation* heap = allocate_remaining_memory();
+        start_mp(heap);
         found_main = maybe_run_list(supported_filenames, &result);
         if (!found_main){
             found_main = maybe_run_list(double_extension_filenames, &result);
@@ -163,6 +180,8 @@ bool start_mp(safe_mode_t safe_mode) {
                 serial_write(MSG_DOUBLE_FILE_EXTENSION);
             }
         }
+        stop_mp();
+        free_memory(heap);
 
         reset_port();
         reset_board();
@@ -306,27 +325,36 @@ void __attribute__ ((noinline)) run_boot_py(safe_mode_t safe_mode) {
         // Reset to remove any state that boot.py setup. It should only be used to
         // change internal state that's not in the heap.
         reset_port();
-        reset_mp();
     }
 }
 
+int run_repl(void) {
+    int exit_code = PYEXEC_FORCED_EXIT;
+    supervisor_allocation* heap = allocate_remaining_memory();
+    start_mp(heap);
+    autoreload_suspend();
+    new_status_color(REPL_RUNNING);
+    if (pyexec_mode_kind == PYEXEC_MODE_RAW_REPL) {
+        exit_code = pyexec_raw_repl();
+    } else {
+        exit_code = pyexec_friendly_repl();
+    }
+    reset_port();
+    reset_board();
+    stop_mp();
+    free_memory(heap);
+    autoreload_resume();
+    return exit_code;
+}
+
 int __attribute__((used)) main(void) {
+    memory_init();
+    
     // initialise the cpu and peripherals
     safe_mode_t safe_mode = port_init();
 
     rgb_led_status_init();
 
-    // Stack limit should be less than real stack size, so we have a chance
-    // to recover from limit hit.  (Limit is measured in bytes.)
-    mp_stack_set_top((char*)&_estack);
-    mp_stack_set_limit((char*)&_estack - (char*)&_ebss - 1024);
-
-#if MICROPY_MAX_STACK_USAGE
-    // _ezero (same as _ebss) is an int, so start 4 bytes above it.
-    mp_stack_set_bottom(&_ezero + 1);
-    mp_stack_fill_with_sentinel();
-#endif
-
     // Create a new filesystem only if we're not in a safe mode.
     // A power brownout here could make it appear as if there's
     // no SPI flash filesystem, and we might erase the existing one.
@@ -335,7 +363,6 @@ int __attribute__((used)) main(void) {
     // Reset everything and prep MicroPython to run boot.py.
     reset_port();
     reset_board();
-    reset_mp();
 
     // Turn on autoreload by default but before boot.py in case it wants to change it.
     autoreload_enable();
@@ -355,24 +382,14 @@ int __attribute__((used)) main(void) {
     bool first_run = true;
     for (;;) {
         if (!skip_repl) {
-            reset_mp();
-            autoreload_suspend();
-            new_status_color(REPL_RUNNING);
-            if (pyexec_mode_kind == PYEXEC_MODE_RAW_REPL) {
-                exit_code = pyexec_raw_repl();
-            } else {
-                exit_code = pyexec_friendly_repl();
-            }
-            autoreload_resume();
-            reset_port();
-            reset_board();
+            exit_code = run_repl();
         }
         if (exit_code == PYEXEC_FORCED_EXIT) {
             if (!first_run) {
                 serial_write(MSG_SOFT_REBOOT MSG_NEWLINE);
             }
             first_run = false;
-            skip_repl = start_mp(safe_mode);
+            skip_repl = run_code_py(safe_mode);
         } else if (exit_code != 0) {
             break;
         }

ports/atmel-samd/Makefile

@@ -106,7 +106,7 @@ ifeq ($(DEBUG), 1)
   # Turn on Python modules useful for debugging (e.g. uheap, ustack).
   CFLAGS += -ggdb
   # You may want to disable -flto if it interferes with debugging.
-  CFLAGS += -flto
+  #CFLAGS += -flto
   # You may want to enable these flags to make setting breakpoints easier.
 ##   CFLAGS += -fno-inline -fno-ipa-sra
   ifeq ($(CHIP_FAMILY), samd21)

ports/atmel-samd/external_flash/external_flash.c

@@ -37,6 +37,7 @@
 #include "py/runtime.h"
 #include "lib/oofatfs/ff.h"
 #include "shared-bindings/microcontroller/__init__.h"
+#include "supervisor/memory.h"
 #include "supervisor/shared/rgb_led_status.h"
 
 #include "hal_gpio.h"
@@ -59,6 +60,8 @@ static const external_flash_device* flash_device = NULL;
 // cache.
 static uint32_t dirty_mask;
 
+static supervisor_allocation* supervisor_cache = NULL;
+
 // Wait until both the write enable and write in progress bits have cleared.
 static bool wait_for_flash_ready(void) {
     uint8_t read_status_response[1] = {0x00};
@@ -308,6 +311,23 @@ static bool flush_scratch_flash(void) {
 static bool allocate_ram_cache(void) {
     uint8_t blocks_per_sector = SPI_FLASH_ERASE_SIZE / FILESYSTEM_BLOCK_SIZE;
     uint8_t pages_per_block = FILESYSTEM_BLOCK_SIZE / SPI_FLASH_PAGE_SIZE;
+
+    uint32_t table_size = blocks_per_sector * pages_per_block * sizeof(uint32_t);
+    // Attempt to allocate outside the heap first.
+    supervisor_cache = allocate_memory(table_size + SPI_FLASH_ERASE_SIZE, false);
+    if (supervisor_cache != NULL) {
+        MP_STATE_VM(flash_ram_cache) = (uint8_t **) supervisor_cache->ptr;
+        uint8_t* page_start = (uint8_t *) supervisor_cache->ptr + table_size;
+
+        for (uint8_t i = 0; i < blocks_per_sector; i++) {
+            for (uint8_t j = 0; j < pages_per_block; j++) {
+                uint32_t offset = i * pages_per_block + j;
+                MP_STATE_VM(flash_ram_cache)[offset] = page_start + offset * SPI_FLASH_PAGE_SIZE;
+            }
+        }
+        return true;
+    }
+
     MP_STATE_VM(flash_ram_cache) = m_malloc_maybe(blocks_per_sector * pages_per_block * sizeof(uint32_t), false);
     if (MP_STATE_VM(flash_ram_cache) == NULL) {
         return false;
@@ -383,14 +403,19 @@ static bool flush_ram_cache(bool keep_cache) {
             write_flash(current_sector + (i * pages_per_block + j) * SPI_FLASH_PAGE_SIZE,
                         MP_STATE_VM(flash_ram_cache)[i * pages_per_block + j],
                         SPI_FLASH_PAGE_SIZE);
-            if (!keep_cache) {
+            if (!keep_cache && supervisor_cache == NULL) {
                 m_free(MP_STATE_VM(flash_ram_cache)[i * pages_per_block + j]);
             }
         }
     }
     // We're done with the cache for now so give it back.
     if (!keep_cache) {
+        if (supervisor_cache != NULL) {
+            free_memory(supervisor_cache);
+            supervisor_cache = NULL;
+        } else {
             m_free(MP_STATE_VM(flash_ram_cache));
+        }
         MP_STATE_VM(flash_ram_cache) = NULL;
     }
     return true;

ports/atmel-samd/supervisor/filesystem.c

@@ -51,6 +51,7 @@ void filesystem_init(bool create_allowed, bool force_create) {
 
     // try to mount the flash
     FRESULT res = f_mount(&vfs_fat->fatfs);
+    force_create = true;
 
     if ((res == FR_NO_FILESYSTEM && create_allowed) || force_create) {
         // No filesystem so create a fresh one, or reformat has been requested.

ports/atmel-samd/supervisor/memory.c

@@ -0,0 +1,100 @@
+/*
+ * This file is part of the MicroPython project, http://micropython.org/
+ *
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2018 Scott Shawcroft for Adafruit Industries
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "supervisor/memory.h"
+
+#include <stddef.h>
+
+#define CIRCUITPY_SUPERVISOR_ALLOC_COUNT 8
+
+static supervisor_allocation allocations[CIRCUITPY_SUPERVISOR_ALLOC_COUNT];
+// We use uint32_t* to ensure word (4 byte) alignment.
+uint32_t* low_address;
+uint32_t* high_address;
+extern uint32_t _ebss;
+extern uint32_t _estack;
+
+void memory_init(void) {
+    low_address = &_ebss;
+    high_address = &_estack;
+
+    // snag 3k for stack
+    allocate_memory(1024*4, true);
+}
+
+void free_memory(supervisor_allocation* allocation) {
+    uint8_t index = 0;
+    bool found = false;
+    for (index = 0; index < CIRCUITPY_SUPERVISOR_ALLOC_COUNT; index++) {
+        found = allocation == &allocations[index];
+        if (found) {
+            break;
+        }
+    }
+    if (allocation->ptr == high_address) {
+        high_address += allocation->length / 4;
+    } else if (allocation->ptr + allocation->length / 4 == low_address) {
+        low_address = allocation->ptr;
+    }
+    allocation->ptr = NULL;
+}
+
+supervisor_allocation* allocate_remaining_memory(void) {
+    if (low_address == high_address) {
+        return NULL;
+    }
+    return allocate_memory((high_address - low_address) * 4, false);
+}
+
+supervisor_allocation* allocate_memory(uint32_t length, bool high) {
+    if ((high_address - low_address) * 4 < (int32_t) length) {
+        return NULL;
+    }
+    uint8_t index = 0;
+    int8_t direction = 1;
+    if (high) {
+        index = CIRCUITPY_SUPERVISOR_ALLOC_COUNT - 1;
+        direction = -1;
+    }
+    for (; index < CIRCUITPY_SUPERVISOR_ALLOC_COUNT; index += direction) {
+        if (allocations[index].ptr == NULL) {
+            break;
+        }
+    }
+    if (index >= CIRCUITPY_SUPERVISOR_ALLOC_COUNT) {
+        return NULL;
+    }
+    supervisor_allocation* alloc = &allocations[index];
+    if (high) {
+        high_address -= length / 4;
+        alloc->ptr = high_address;
+    } else {
+        alloc->ptr = low_address;
+        low_address += length / 4;
+    }
+    alloc->length = length;
+    return alloc;
+}

shared-bindings/supervisor/__init__.c

@@ -107,6 +107,15 @@ STATIC mp_obj_t supervisor_reload(void) {
 }
 MP_DEFINE_CONST_FUN_OBJ_0(supervisor_reload_obj, supervisor_reload);
 
+//| .. method:: set_main_file(filename)
+//|
+//|   Set which file to run after a reload.
+//|
+STATIC mp_obj_t supervisor_set_main_file(void) {
+    
+    return mp_const_none;
+}
+MP_DEFINE_CONST_FUN_OBJ_0(supervisor_set_main_file_obj, supervisor_set_main_file);
 
 STATIC const mp_rom_map_elem_t supervisor_module_globals_table[] = {
     { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_supervisor) },
@@ -115,6 +124,8 @@ STATIC const mp_rom_map_elem_t supervisor_module_globals_table[] = {
     { MP_OBJ_NEW_QSTR(MP_QSTR_set_rgb_status_brightness),  MP_ROM_PTR(&supervisor_set_rgb_status_brightness_obj) },
     { MP_ROM_QSTR(MP_QSTR_runtime),  MP_ROM_PTR(&common_hal_supervisor_runtime_obj) },
     { MP_ROM_QSTR(MP_QSTR_reload),  MP_ROM_PTR(&supervisor_reload_obj) },
+    { MP_ROM_QSTR(MP_QSTR_set_main_file),  MP_ROM_PTR(&supervisor_set_main_file_obj) },
+
 };
 
 STATIC MP_DEFINE_CONST_DICT(supervisor_module_globals, supervisor_module_globals_table);

supervisor/memory.h

@@ -0,0 +1,43 @@
+/*
+ * This file is part of the MicroPython project, http://micropython.org/
+ *
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2018 Scott Shawcroft for Adafruit Industries
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#ifndef MICROPY_INCLUDED_SUPERVISOR_MEMORY_H
+#define MICROPY_INCLUDED_SUPERVISOR_MEMORY_H
+
+#include <stdbool.h>
+#include <stdint.h>
+
+typedef struct {
+    uint32_t* ptr;
+    uint32_t length; // in bytes
+} supervisor_allocation;
+
+void memory_init(void);
+void free_memory(supervisor_allocation* allocation);
+supervisor_allocation* allocate_remaining_memory(void);
+supervisor_allocation* allocate_memory(uint32_t length, bool high_address);
+
+#endif  // MICROPY_INCLUDED_SUPERVISOR_MEMORY_H

supervisor/supervisor.mk

@@ -1,5 +1,6 @@
 SRC_SUPERVISOR = \
 	main.c \
+	supervisor/memory.c \
 	supervisor/port.c \
 	supervisor/shared/autoreload.c \
 	supervisor/shared/rgb_led_status.c