Allow use of CONFIG_SPIRAM_USE_CAPS_ALLOC

.. and switch makerfabs tft7 over to it as a test.

We have our existing way of "reserving" PSRAM for esp-idf (we actually
control it all but add back the "reserved" part). However, this does
not work with off the shelf esp_lcd, which only will allocate a
framebuffer in PSRAM if CONFIG_SPIRAM_USE_CAPS_ALLOC (or CONFIG_SPIRAM_USE_ALLOC)
is defined, not if CONFIG_SPIRAM_USE_MEMMAP is.

This new way is possibly compatible with more esp-idf code, but it complicates
CircuitPython's initial startup since nothing until port_heap_init is
permitted to use the CP heap or supervisor allocator. In practice this
seems to be OK today.

Right now this doesn't change the setting across all boards with PSRAM and so
it does not revert esp-idf to its prior state. Instead, what I'm thinking is
that we can do it during or just after the IDF5 update when sdkconfig files
will be getting an overhaul anyway.

main.c

@@ -1042,6 +1042,10 @@ int __attribute__((used)) main(void) {
         set_safe_mode(SAFE_MODE_NO_CIRCUITPY);
     }
 
+    // We maybe can't initialize the heap until here, because on espressif port we need to be able to check for reserved psram in settings.toml
+    // (but it's OK if this is a no-op due to the heap beinig initialized in port_init())
+    set_safe_mode(port_heap_init(get_safe_mode()));
+
     #if CIRCUITPY_ALARM
     // Record which alarm woke us up, if any.
     // common_hal_alarm_record_wake_alarm() should return a static, non-heap object
@@ -1171,6 +1175,13 @@ void gc_collect(void) {
 MP_WEAK void port_gc_collect() {
 }
 
+// A port may initialize the heap in port_init but if it cannot (for instance
+// in espressif it must be done after CIRCUITPY is mounted) then it must provde
+// an implementation of this function.
+MP_WEAK safe_mode_t port_heap_init(safe_mode_t safe_mode_in) {
+    return safe_mode_in;
+}
+
 void NORETURN nlr_jump_fail(void *val) {
     reset_into_safe_mode(SAFE_MODE_NLR_JUMP_FAIL);
     while (true) {

ports/espressif/boards/makerfabs_tft7/sdkconfig

@@ -21,8 +21,8 @@ CONFIG_SPIRAM_SPEED_80M=y
 CONFIG_SPIRAM=y
 CONFIG_SPIRAM_BOOT_INIT=y
 # CONFIG_SPIRAM_IGNORE_NOTFOUND is not set
-CONFIG_SPIRAM_USE_MEMMAP=y
+#CONFIG_SPIRAM_USE_MEMMAP=y
-# CONFIG_SPIRAM_USE_CAPS_ALLOC is not set
+CONFIG_SPIRAM_USE_CAPS_ALLOC=y
 # CONFIG_SPIRAM_USE_MALLOC is not set
 CONFIG_SPIRAM_MEMTEST=y
 #

ports/espressif/common-hal/espidf/__init__.c

@@ -79,7 +79,7 @@ bool common_hal_espidf_set_reserved_psram(size_t amount) {
 supervisor_allocation *psram_for_idf;
 
 void common_hal_espidf_reserve_psram(void) {
-    #ifdef CONFIG_SPIRAM
+    #ifdef CONFIG_SPIRAM_USE_MEMMAP
     if (!psram_for_idf) {
         ESP_LOGI(TAG, "Reserving %d bytes of psram", reserved_psram);
         if (reserved_psram == 0) {

ports/espressif/supervisor/port.c

@@ -299,33 +299,8 @@ safe_mode_t port_init(void) {
     #endif
     #endif
 
-    #ifdef CONFIG_SPIRAM
-    {
-        intptr_t heap_start = common_hal_espidf_get_psram_start();
-        intptr_t heap_end = common_hal_espidf_get_psram_end();
-        size_t spiram_size = heap_end - heap_start;
-        if (spiram_size > 0) {
-            heap = (uint32_t *)heap_start;
-            heap_size = (heap_end - heap_start) / sizeof(uint32_t);
-        } else {
-            ESP_LOGE(TAG, "CONFIG_SPIRAM enabled but no spiram heap available");
-        }
-    }
-    #endif
-
     _never_reset_spi_ram_flash();
 
-    if (heap == NULL) {
-        size_t heap_total = heap_caps_get_total_size(MALLOC_CAP_8BIT);
-        heap_size = MIN(heap_caps_get_largest_free_block(MALLOC_CAP_8BIT), heap_total / 2);
-        heap = malloc(heap_size);
-        heap_size = heap_size / sizeof(uint32_t);
-    }
-    if (heap == NULL) {
-        heap_size = 0;
-        return SAFE_MODE_NO_HEAP;
-    }
-
     esp_reset_reason_t reason = esp_reset_reason();
     switch (reason) {
         case ESP_RST_BROWNOUT:
@@ -344,6 +319,57 @@ safe_mode_t port_init(void) {
     return SAFE_MODE_NONE;
 }
 
+safe_mode_t port_heap_init(safe_mode_t sm) {
+    mp_int_t reserved = 0;
+    if (filesystem_present() && common_hal_os_getenv_int("CIRCUITPY_RESERVED_PSRAM", &reserved) == GETENV_OK) {
+        common_hal_espidf_set_reserved_psram(reserved);
+    }
+
+    #ifdef CONFIG_SPIRAM_USE_MEMMAP
+    {
+        intptr_t heap_start = common_hal_espidf_get_psram_start();
+        intptr_t heap_end = common_hal_espidf_get_psram_end();
+        size_t spiram_size = heap_end - heap_start;
+        if (spiram_size > 0) {
+            heap = (uint32_t *)heap_start;
+            heap_size = (heap_end - heap_start) / sizeof(uint32_t);
+            common_hal_espidf_reserve_psram();
+        } else {
+            ESP_LOGE(TAG, "CONFIG_SPIRAM_USE_MMAP enabled but no spiram heap available");
+        }
+    }
+    #elif CONFIG_SPIRAM_USE_CAPS_ALLOC
+    {
+        intptr_t psram_start = common_hal_espidf_get_psram_start();
+        intptr_t psram_end = common_hal_espidf_get_psram_end();
+        size_t psram_amount = psram_end - psram_start;
+        size_t biggest_block = heap_caps_get_largest_free_block(MALLOC_CAP_SPIRAM);
+        size_t try_alloc = MIN(biggest_block, psram_amount - common_hal_espidf_get_reserved_psram());
+        heap = heap_caps_malloc(try_alloc, MALLOC_CAP_SPIRAM);
+
+        if (heap) {
+            heap_size = try_alloc;
+        } else {
+            ESP_LOGE(TAG, "CONFIG_SPIRAM_USE_CAPS_ALLOC but no spiram heap available");
+        }
+    }
+    #endif
+
+    if (heap == NULL) {
+        size_t heap_total = heap_caps_get_total_size(MALLOC_CAP_8BIT);
+        heap_size = MIN(heap_caps_get_largest_free_block(MALLOC_CAP_8BIT), heap_total / 2);
+        heap = malloc(heap_size);
+        heap_size = heap_size / sizeof(uint32_t);
+    }
+    if (heap == NULL) {
+        heap_size = 0;
+        return SAFE_MODE_NO_HEAP;
+    }
+
+    return sm;
+
+}
+
 void reset_port(void) {
     // TODO deinit for esp32-camera
     #if CIRCUITPY_ESPCAMERA
@@ -533,11 +559,6 @@ void port_idle_until_interrupt(void) {
 
 void port_post_boot_py(bool heap_valid) {
     if (!heap_valid && filesystem_present()) {
-        mp_int_t reserved;
-        if (common_hal_os_getenv_int("CIRCUITPY_RESERVED_PSRAM", &reserved) == GETENV_OK) {
-            common_hal_espidf_set_reserved_psram(reserved);
-        }
-        common_hal_espidf_reserve_psram();
     }
 }
 

supervisor/port.h

@@ -43,6 +43,11 @@ extern uint32_t _ebss;
 
 safe_mode_t port_init(void);
 
+// If the port does not initialize the heap during port_init(), it must provide
+// this function which is called after CIRCUITPY is mounted.
+// If not required, a default (weak) implementation that does nothing is used.
+safe_mode_t port_heap_init(safe_mode_t);
+
 // Reset the microcontroller completely.
 void reset_cpu(void) NORETURN;