From d4aab422eefd85a7faa9e475b88e9093b2eb19e3 Mon Sep 17 00:00:00 2001
From: Kamil Tomaszewski <kamil.tomaszewski@sony.com>
Date: Sat, 14 Jan 2023 20:31:06 +0100
Subject: [PATCH 01/35] spresense: update SDK to 2.6.0

---
 ports/cxd56/README.md                       | 2 +-
 ports/cxd56/configs/circuitpython/defconfig | 1 +
 ports/cxd56/spresense-exported-sdk          | 2 +-
 ports/cxd56/supervisor/port.c               | 4 ++--
 4 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/ports/cxd56/README.md b/ports/cxd56/README.md
index 0656ad1f1a..4266c61c0c 100644
--- a/ports/cxd56/README.md
+++ b/ports/cxd56/README.md
@@ -75,7 +75,7 @@ Bootloader information:
 
 * You have to accept the End User License Agreement to be able to download and use the Spresense bootloader binary.
 
-Download the spresense binaries zip archive from: [Spresense firmware v2-3-000](https://developer.sony.com/file/download/download-spresense-firmware-v2-3-000)
+Download the spresense binaries zip archive from: [Spresense firmware v2-4-000](https://developer.sony.com/file/download/download-spresense-firmware-v2-4-000)
 
 Extract spresense binaries in your PC to ports/spresense/spresense-exported-sdk/firmware/
 
diff --git a/ports/cxd56/configs/circuitpython/defconfig b/ports/cxd56/configs/circuitpython/defconfig
index 6a7c39cba8..92e282649b 100644
--- a/ports/cxd56/configs/circuitpython/defconfig
+++ b/ports/cxd56/configs/circuitpython/defconfig
@@ -113,4 +113,5 @@ CONFIG_USEC_PER_TICK=1000
 CONFIG_USERMAIN_STACKSIZE=8192
 CONFIG_USER_ENTRYPOINT="spresense_main"
 CONFIG_VIDEO_ISX012=y
+CONFIG_VIDEO_ISX019=y
 CONFIG_VIDEO_STREAM=y
diff --git a/ports/cxd56/spresense-exported-sdk b/ports/cxd56/spresense-exported-sdk
index 6a148be849..4f902ca3ff 160000
--- a/ports/cxd56/spresense-exported-sdk
+++ b/ports/cxd56/spresense-exported-sdk
@@ -1 +1 @@
-Subproject commit 6a148be8497704d4afb5d14c175a12a592813fac
+Subproject commit 4f902ca3ffeb327e6c325940ef5133eda588c2e4
diff --git a/ports/cxd56/supervisor/port.c b/ports/cxd56/supervisor/port.c
index 10fa4112e8..dfd3cd6646 100644
--- a/ports/cxd56/supervisor/port.c
+++ b/ports/cxd56/supervisor/port.c
@@ -111,13 +111,13 @@ bool port_has_fixed_stack(void) {
 uint32_t *port_stack_get_limit(void) {
     struct tcb_s *rtcb = this_task();
 
-    return rtcb->adj_stack_ptr - (uint32_t)rtcb->adj_stack_size;
+    return rtcb->stack_base_ptr;
 }
 
 uint32_t *port_stack_get_top(void) {
     struct tcb_s *rtcb = this_task();
 
-    return rtcb->adj_stack_ptr;
+    return rtcb->stack_base_ptr + (uint32_t)rtcb->adj_stack_size;
 }
 
 uint32_t *port_heap_get_bottom(void) {

From 3fdebd4f608a0973b3de6303a1367fbe12cd1575 Mon Sep 17 00:00:00 2001
From: Kamil Tomaszewski <kamil.tomaszewski@sony.com>
Date: Wed, 1 Feb 2023 12:37:29 +0100
Subject: [PATCH 02/35] spresense: Add HDR camera support

---
 ports/cxd56/common-hal/camera/Camera.c | 36 +++++++++++++++++++++++---
 1 file changed, 32 insertions(+), 4 deletions(-)

diff --git a/ports/cxd56/common-hal/camera/Camera.c b/ports/cxd56/common-hal/camera/Camera.c
index f304ed8f74..606ad0f1e6 100644
--- a/ports/cxd56/common-hal/camera/Camera.c
+++ b/ports/cxd56/common-hal/camera/Camera.c
@@ -47,7 +47,7 @@ typedef struct {
     uint16_t height;
 } image_size_t;
 
-STATIC const image_size_t image_size_table[] = {
+STATIC const image_size_t isx012_image_size_table[] = {
     { VIDEO_HSIZE_QVGA, VIDEO_VSIZE_QVGA },
     { VIDEO_HSIZE_VGA, VIDEO_VSIZE_VGA },
     { VIDEO_HSIZE_HD, VIDEO_VSIZE_HD },
@@ -57,12 +57,40 @@ STATIC const image_size_t image_size_table[] = {
     { VIDEO_HSIZE_5M, VIDEO_VSIZE_5M },
 };
 
+STATIC const image_size_t isx019_image_size_table[] = {
+    { VIDEO_HSIZE_QVGA, VIDEO_VSIZE_QVGA },
+    { VIDEO_HSIZE_VGA, VIDEO_VSIZE_VGA },
+    { VIDEO_HSIZE_HD, VIDEO_VSIZE_HD },
+    { VIDEO_HSIZE_QUADVGA, VIDEO_VSIZE_QUADVGA },
+};
+
+static const char *get_imgsensor_name() {
+    static struct v4l2_capability cap;
+
+    ioctl(camera_dev.fd, VIDIOC_QUERYCAP, (unsigned long)&cap);
+
+    return (const char *)cap.driver;
+}
+
 static bool camera_check_width_and_height(uint16_t width, uint16_t height) {
-    for (int i = 0; i < MP_ARRAY_SIZE(image_size_table); i++) {
-        if (image_size_table[i].width == width && image_size_table[i].height == height) {
-            return true;
+    const char *sensor;
+
+    sensor = get_imgsensor_name();
+
+    if (strncmp(sensor, "ISX012", strlen("ISX012")) == 0) {
+        for (int i = 0; i < MP_ARRAY_SIZE(isx012_image_size_table); i++) {
+            if (isx012_image_size_table[i].width == width && isx012_image_size_table[i].height == height) {
+                return true;
+            }
+        }
+    } else if (strncmp(sensor, "ISX019", strlen("ISX019"))) {
+        for (int i = 0; i < MP_ARRAY_SIZE(isx019_image_size_table); i++) {
+            if (isx019_image_size_table[i].width == width && isx019_image_size_table[i].height == height) {
+                return true;
+            }
         }
     }
+
     return false;
 }
 

From 07e8506b79b22f04cad5520716748e41d18e3f64 Mon Sep 17 00:00:00 2001
From: Alex Sirota <67526318+ajs256@users.noreply.github.com>
Date: Fri, 3 Feb 2023 20:19:11 -0800
Subject: [PATCH 03/35] Add board hack_club_sprig

---
 .../boards/hack_club_sprig/board.c            | 128 ++++++++++++++++++
 .../boards/hack_club_sprig/mpconfigboard.h    |   4 +
 .../boards/hack_club_sprig/mpconfigboard.mk   |  11 ++
 .../hack_club_sprig/pico-sdk-configboard.h    |   1 +
 .../raspberrypi/boards/hack_club_sprig/pins.c |  85 ++++++++++++
 5 files changed, 229 insertions(+)
 create mode 100644 ports/raspberrypi/boards/hack_club_sprig/board.c
 create mode 100644 ports/raspberrypi/boards/hack_club_sprig/mpconfigboard.h
 create mode 100644 ports/raspberrypi/boards/hack_club_sprig/mpconfigboard.mk
 create mode 100644 ports/raspberrypi/boards/hack_club_sprig/pico-sdk-configboard.h
 create mode 100644 ports/raspberrypi/boards/hack_club_sprig/pins.c

diff --git a/ports/raspberrypi/boards/hack_club_sprig/board.c b/ports/raspberrypi/boards/hack_club_sprig/board.c
new file mode 100644
index 0000000000..0169e85e0e
--- /dev/null
+++ b/ports/raspberrypi/boards/hack_club_sprig/board.c
@@ -0,0 +1,128 @@
+/*
+ * This file is part of the MicroPython project, http://micropython.org/
+ *
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2021 Scott Shawcroft for Adafruit Industries
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "supervisor/board.h"
+#include "mpconfigboard.h"
+#include "shared-bindings/busio/SPI.h"
+#include "shared-bindings/displayio/FourWire.h"
+#include "shared-module/displayio/__init__.h"
+#include "shared-module/displayio/mipi_constants.h"
+#include "supervisor/shared/board.h"
+
+displayio_fourwire_obj_t board_display_obj;
+
+// display init sequence from CircuitPython library https://github.com/adafruit/Adafruit_CircuitPython_ST7735R/blob/dfae353330cf051d1f31db9e4b681c8d70900cc5/adafruit_st7735r.py
+uint8_t display_init_sequence[] = {
+    // sw reset
+    0x01, 0x80, 0x96,
+    // sleep out and delay
+    0x11, 0x80, 0xFF,
+    // _FRMCTR1
+    0xB1, 0x03, 0x01, 0x2C, 0x2D,
+    // _FRMCTR2
+    0xB2, 0x03, 0x01, 0x2C, 0x2D,
+    // _FRMCTR3
+    0xB3, 0x06, 0x01, 0x2C, 0x2D, 0x01, 0x2C, 0x2D,
+    // _INVCTR line inversion
+    0xB4, 0x01, 0x07,
+    // _PWCTR1 GVDD = 4.7V, 1.0uA
+    0xC0, 0x03, 0xA2, 0x02, 0x84,
+    // _PWCTR2 VGH=14.7V, VGL=-7.35V
+    0xC1, 0x01, 0xC5,
+    // _PWCTR3 Opamp current small, Boost frequency
+    0xC2, 0x02, 0x0A, 0x00,
+    0xC3, 0x02, 0x8A, 0x2A,
+    0xC4, 0x02, 0x8A, 0xEE,
+    // _VMCTR1 VCOMH = 4V, VOML = -1.1V
+    0xC5, 0x01, 0x0E,
+    // _INVOFF
+    0x20, 0x00,
+    // _MADCTL bottom to top refresh
+    0x36, 0x01, 0x18,
+    // COLMOD - 16 bit color
+    0x3A, 0x01, 0x05,
+    // _GMCTRP1 Gamma
+    0xE0, 0x10, 0x02, 0x1C, 0x07, 0x12, 0x37, 0x32, 0x29, 0x2D, 0x29, 0x25, 0x2B, 0x39, 0x00, 0x01, 0x03, 0x10,
+    // _GMCTRN1
+    0xE1, 0x10, 0x03, 0x1d, 0x07, 0x06, 0x2E, 0x2C, 0x29, 0x2D, 0x2E, 0x2E, 0x37, 0x3F, 0x00, 0x00, 0x02, 0x10,
+    // _NORON
+    0x13, 0x80, 0x0A,
+    // _DISPON
+    0x29, 0x80, 0x64,
+    // _MADCTL Default rotation + BGR encoding
+    0x36, 0x01, 0xC0,
+};
+
+
+void board_init(void) {
+    busio_spi_obj_t *spi = &displays[0].fourwire_bus.inline_bus;
+    common_hal_busio_spi_construct(spi, &pin_GPIO18, &pin_GPIO19, &pin_GPIO16, false);
+    common_hal_busio_spi_never_reset(spi);
+
+    displayio_fourwire_obj_t *bus = &displays[0].fourwire_bus;
+    bus->base.type = &displayio_fourwire_type;
+    common_hal_displayio_fourwire_construct(bus,
+        spi,
+        &pin_GPIO22, // DC
+        &pin_GPIO20, // CS
+        &pin_GPIO26, // RST
+        30000000,
+        0,
+        0);
+
+    displayio_display_obj_t *display = &displays[0].display;
+    display->base.type = &displayio_display_type;
+    common_hal_displayio_display_construct(display,
+        bus,
+        160, // Width
+        128, // Height
+        0, // column start
+        0, // row start
+        270, // rotation
+        16, // Color depth
+        false, // Grayscale
+        false, // pixels in a byte share a row. Only valid for depths < 8
+        1, // bytes per cell. Only valid for depths < 8
+        false, // reverse_pixels_in_byte. Only valid for depths < 8
+        true, // reverse_bytes_in_word
+        MIPI_COMMAND_SET_COLUMN_ADDRESS, // Set column command
+        MIPI_COMMAND_SET_PAGE_ADDRESS, // Set row command
+        MIPI_COMMAND_WRITE_MEMORY_START, // Write memory command
+        display_init_sequence,
+        sizeof(display_init_sequence),
+        &pin_GPIO17,  // backlight pin
+        NO_BRIGHTNESS_COMMAND,
+        1.0f, // brightness
+        false, // single_byte_bounds
+        false, // data_as_commands
+        true, // auto_refresh
+        60, // native_frames_per_second
+        true, // backlight_on_high
+        false, // SH1107_addressing
+        50000); // backlight pwm frequency
+}
+
+// Use the MP_WEAK supervisor/shared/board.c versions of routines not defined here.
diff --git a/ports/raspberrypi/boards/hack_club_sprig/mpconfigboard.h b/ports/raspberrypi/boards/hack_club_sprig/mpconfigboard.h
new file mode 100644
index 0000000000..442e455c5e
--- /dev/null
+++ b/ports/raspberrypi/boards/hack_club_sprig/mpconfigboard.h
@@ -0,0 +1,4 @@
+#define MICROPY_HW_BOARD_NAME "Hack Club Sprig"
+#define MICROPY_HW_MCU_NAME "rp2040"
+
+#define MICROPY_HW_LED_STATUS (&pin_GPIO4)
diff --git a/ports/raspberrypi/boards/hack_club_sprig/mpconfigboard.mk b/ports/raspberrypi/boards/hack_club_sprig/mpconfigboard.mk
new file mode 100644
index 0000000000..1bde636031
--- /dev/null
+++ b/ports/raspberrypi/boards/hack_club_sprig/mpconfigboard.mk
@@ -0,0 +1,11 @@
+USB_VID = 0x1209
+USB_PID = 0x9000
+USB_PRODUCT = "Sprig"
+USB_MANUFACTURER = "Hack Club"
+
+CHIP_VARIANT = RP2040
+CHIP_FAMILY = rp2
+
+EXTERNAL_FLASH_DEVICES = "W25Q16JVxQ"
+
+CIRCUITPY__EVE = 1
diff --git a/ports/raspberrypi/boards/hack_club_sprig/pico-sdk-configboard.h b/ports/raspberrypi/boards/hack_club_sprig/pico-sdk-configboard.h
new file mode 100644
index 0000000000..36da55d457
--- /dev/null
+++ b/ports/raspberrypi/boards/hack_club_sprig/pico-sdk-configboard.h
@@ -0,0 +1 @@
+// Put board-specific pico-sdk definitions here. This file must exist.
diff --git a/ports/raspberrypi/boards/hack_club_sprig/pins.c b/ports/raspberrypi/boards/hack_club_sprig/pins.c
new file mode 100644
index 0000000000..f82b8fafc5
--- /dev/null
+++ b/ports/raspberrypi/boards/hack_club_sprig/pins.c
@@ -0,0 +1,85 @@
+#include "shared-bindings/board/__init__.h"
+
+STATIC const mp_rom_map_elem_t board_module_globals_table[] = {
+    CIRCUITPYTHON_BOARD_DICT_STANDARD_ITEMS
+
+    { MP_ROM_QSTR(MP_QSTR_GP0), MP_ROM_PTR(&pin_GPIO0) },
+    { MP_ROM_QSTR(MP_QSTR_GP1), MP_ROM_PTR(&pin_GPIO1) },
+    { MP_ROM_QSTR(MP_QSTR_GP2), MP_ROM_PTR(&pin_GPIO2) },
+    { MP_ROM_QSTR(MP_QSTR_GP3), MP_ROM_PTR(&pin_GPIO3) },
+    { MP_ROM_QSTR(MP_QSTR_GP4), MP_ROM_PTR(&pin_GPIO4) },
+    { MP_ROM_QSTR(MP_QSTR_GP5), MP_ROM_PTR(&pin_GPIO5) },
+    { MP_ROM_QSTR(MP_QSTR_GP6), MP_ROM_PTR(&pin_GPIO6) },
+    { MP_ROM_QSTR(MP_QSTR_GP7), MP_ROM_PTR(&pin_GPIO7) },
+    { MP_ROM_QSTR(MP_QSTR_GP8), MP_ROM_PTR(&pin_GPIO8) },
+    { MP_ROM_QSTR(MP_QSTR_GP9), MP_ROM_PTR(&pin_GPIO9) },
+    { MP_ROM_QSTR(MP_QSTR_GP10), MP_ROM_PTR(&pin_GPIO10) },
+    { MP_ROM_QSTR(MP_QSTR_GP11), MP_ROM_PTR(&pin_GPIO11) },
+    { MP_ROM_QSTR(MP_QSTR_GP12), MP_ROM_PTR(&pin_GPIO12) },
+    { MP_ROM_QSTR(MP_QSTR_GP13), MP_ROM_PTR(&pin_GPIO13) },
+    { MP_ROM_QSTR(MP_QSTR_GP14), MP_ROM_PTR(&pin_GPIO14) },
+    { MP_ROM_QSTR(MP_QSTR_GP15), MP_ROM_PTR(&pin_GPIO15) },
+    { MP_ROM_QSTR(MP_QSTR_GP16), MP_ROM_PTR(&pin_GPIO16) },
+    { MP_ROM_QSTR(MP_QSTR_GP17), MP_ROM_PTR(&pin_GPIO17) },
+    { MP_ROM_QSTR(MP_QSTR_GP18), MP_ROM_PTR(&pin_GPIO18) },
+    { MP_ROM_QSTR(MP_QSTR_GP19), MP_ROM_PTR(&pin_GPIO19) },
+    { MP_ROM_QSTR(MP_QSTR_GP20), MP_ROM_PTR(&pin_GPIO20) },
+    { MP_ROM_QSTR(MP_QSTR_GP21), MP_ROM_PTR(&pin_GPIO21) },
+    { MP_ROM_QSTR(MP_QSTR_GP22), MP_ROM_PTR(&pin_GPIO22) },
+
+    { MP_ROM_QSTR(MP_QSTR_SMPS_MODE), MP_ROM_PTR(&pin_GPIO23) },
+    { MP_ROM_QSTR(MP_QSTR_GP23), MP_ROM_PTR(&pin_GPIO23) },
+
+    { MP_ROM_QSTR(MP_QSTR_VBUS_SENSE), MP_ROM_PTR(&pin_GPIO24) },
+    { MP_ROM_QSTR(MP_QSTR_GP24), MP_ROM_PTR(&pin_GPIO24) },
+
+    { MP_ROM_QSTR(MP_QSTR_LED), MP_ROM_PTR(&pin_GPIO25) },
+    { MP_ROM_QSTR(MP_QSTR_GP25), MP_ROM_PTR(&pin_GPIO25) },
+
+    { MP_ROM_QSTR(MP_QSTR_GP26_A0), MP_ROM_PTR(&pin_GPIO26) },
+    { MP_ROM_QSTR(MP_QSTR_GP26), MP_ROM_PTR(&pin_GPIO26) },
+    { MP_ROM_QSTR(MP_QSTR_A0), MP_ROM_PTR(&pin_GPIO26) },
+
+    { MP_ROM_QSTR(MP_QSTR_GP27_A1), MP_ROM_PTR(&pin_GPIO27) },
+    { MP_ROM_QSTR(MP_QSTR_GP27), MP_ROM_PTR(&pin_GPIO27) },
+    { MP_ROM_QSTR(MP_QSTR_A1), MP_ROM_PTR(&pin_GPIO27) },
+
+    { MP_ROM_QSTR(MP_QSTR_GP28_A2), MP_ROM_PTR(&pin_GPIO28) },
+    { MP_ROM_QSTR(MP_QSTR_GP28), MP_ROM_PTR(&pin_GPIO28) },
+    { MP_ROM_QSTR(MP_QSTR_A2), MP_ROM_PTR(&pin_GPIO28) },
+
+    { MP_ROM_QSTR(MP_QSTR_A3), MP_ROM_PTR(&pin_GPIO29) },
+    { MP_ROM_QSTR(MP_QSTR_VOLTAGE_MONITOR), MP_ROM_PTR(&pin_GPIO29) },
+
+
+    // Start Sprig-specific definitions
+
+    { MP_ROM_QSTR(MP_QSTR_BLUE_LED), MP_ROM_PTR(&pin_GPIO4) },
+
+    { MP_ROM_QSTR(MP_QSTR_BUTTON_W), MP_ROM_PTR(&pin_GPIO5) },
+    { MP_ROM_QSTR(MP_QSTR_BUTTON_A), MP_ROM_PTR(&pin_GPIO6) },
+    { MP_ROM_QSTR(MP_QSTR_BUTTON_S), MP_ROM_PTR(&pin_GPIO7) },
+    { MP_ROM_QSTR(MP_QSTR_BUTTON_D), MP_ROM_PTR(&pin_GPIO8) },
+
+    { MP_ROM_QSTR(MP_QSTR_AUDIO_DIN), MP_ROM_PTR(&pin_GPIO9) },
+    { MP_ROM_QSTR(MP_QSTR_AUDIO_BCLK), MP_ROM_PTR(&pin_GPIO10) },
+    { MP_ROM_QSTR(MP_QSTR_AUDIO_LRCLK), MP_ROM_PTR(&pin_GPIO11) },
+
+    { MP_ROM_QSTR(MP_QSTR_BUTTON_I), MP_ROM_PTR(&pin_GPIO12) },
+    { MP_ROM_QSTR(MP_QSTR_BUTTON_J), MP_ROM_PTR(&pin_GPIO13) },
+    { MP_ROM_QSTR(MP_QSTR_BUTTON_K), MP_ROM_PTR(&pin_GPIO14) },
+    { MP_ROM_QSTR(MP_QSTR_BUTTON_L), MP_ROM_PTR(&pin_GPIO15) },
+
+    { MP_ROM_QSTR(MP_QSTR_MISO), MP_ROM_PTR(&pin_GPIO16) },
+    { MP_ROM_QSTR(MP_QSTR_TFT_LITE), MP_ROM_PTR(&pin_GPIO17) },
+    { MP_ROM_QSTR(MP_QSTR_SCK), MP_ROM_PTR(&pin_GPIO18) },
+    { MP_ROM_QSTR(MP_QSTR_MOSI), MP_ROM_PTR(&pin_GPIO19) },
+    { MP_ROM_QSTR(MP_QSTR_TFT_CS), MP_ROM_PTR(&pin_GPIO20) },
+    { MP_ROM_QSTR(MP_QSTR_CARD_CS), MP_ROM_PTR(&pin_GPIO21) },
+    { MP_ROM_QSTR(MP_QSTR_TFT_DC), MP_ROM_PTR(&pin_GPIO22) },
+    { MP_ROM_QSTR(MP_QSTR_TFT_RESET), MP_ROM_PTR(&pin_GPIO23) },
+
+    { MP_ROM_QSTR(MP_QSTR_WHITE_LED), MP_ROM_PTR(&pin_GPIO28) },
+
+};
+MP_DEFINE_CONST_DICT(board_module_globals, board_module_globals_table);

From e78143a2f50bb3b15e44193965a9d8184caa9f55 Mon Sep 17 00:00:00 2001
From: Alex <67526318+ajs256@users.noreply.github.com>
Date: Sun, 5 Feb 2023 11:18:23 -0800
Subject: [PATCH 04/35] fix copyright header

---
 ports/raspberrypi/boards/hack_club_sprig/board.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ports/raspberrypi/boards/hack_club_sprig/board.c b/ports/raspberrypi/boards/hack_club_sprig/board.c
index 0169e85e0e..775e6ca5df 100644
--- a/ports/raspberrypi/boards/hack_club_sprig/board.c
+++ b/ports/raspberrypi/boards/hack_club_sprig/board.c
@@ -3,7 +3,7 @@
  *
  * The MIT License (MIT)
  *
- * Copyright (c) 2021 Scott Shawcroft for Adafruit Industries
+ * Copyright (c) 2023 ajs256
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal

From b9f689adf407a0062c2d5d0e27883f2e978716b1 Mon Sep 17 00:00:00 2001
From: Jeff Epler <jepler@gmail.com>
Date: Wed, 8 Feb 2023 16:18:45 -0600
Subject: [PATCH 05/35] Use lowest drive level for PIO

---
 ports/raspberrypi/common-hal/rp2pio/StateMachine.c | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/ports/raspberrypi/common-hal/rp2pio/StateMachine.c b/ports/raspberrypi/common-hal/rp2pio/StateMachine.c
index ecaee90cc2..e9cb3d34bb 100644
--- a/ports/raspberrypi/common-hal/rp2pio/StateMachine.c
+++ b/ports/raspberrypi/common-hal/rp2pio/StateMachine.c
@@ -289,6 +289,11 @@ bool rp2pio_statemachine_construct(rp2pio_statemachine_obj_t *self,
             }
             pio_gpio_init(self->pio, pin_number);
         }
+
+        // Use lowest drive level for all State Machine outputs. (#7515
+        // workaround). Remove if/when Pin objects get a drive_strength
+        // property and use that value instead.
+        gpio_set_drive_strength(pin_number, GPIO_DRIVE_STRENGTH_2MA);
     }
 
     pio_sm_config c = {0, 0, 0};

From 17751ad2830984b0940915af019f93edde8994c6 Mon Sep 17 00:00:00 2001
From: Neradoc <neraOnGit@ri1.fr>
Date: Fri, 10 Feb 2023 04:58:37 +0100
Subject: [PATCH 06/35] Update build_board_info.py to sh module 2.0.0

---
 tools/build_board_info.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/tools/build_board_info.py b/tools/build_board_info.py
index f810d942a7..5209337b95 100755
--- a/tools/build_board_info.py
+++ b/tools/build_board_info.py
@@ -10,6 +10,7 @@ import subprocess
 import sys
 import sh
 import base64
+from io import StringIO
 from datetime import date
 from sh.contrib import git
 
@@ -58,9 +59,13 @@ def get_languages(list_all=False):
 
 def get_version_info():
     version = None
-    sha = git("rev-parse", "--short", "HEAD").stdout.decode("utf-8")
+    buffer = StringIO()
+    git("rev-parse", "--short", "HEAD", _out=buffer)
+    sha = buffer.getvalue().strip()
     try:
-        version = git("describe", "--tags", "--exact-match").stdout.decode("utf-8").strip()
+        buffer = StringIO()
+        git("describe", "--tags", "--exact-match", _out=buffer)
+        version = buffer.getvalue().strip()
     except sh.ErrorReturnCode_128:
         # No exact match
         pass

From baaa2362c23aa20166113be6a53e6e9a54a614fc Mon Sep 17 00:00:00 2001
From: MicroDev <70126934+MicroDev1@users.noreply.github.com>
Date: Fri, 10 Feb 2023 16:38:25 +0530
Subject: [PATCH 07/35] use already built mpy-cross

---
 .github/workflows/build.yml | 83 ++++++++++++++++++++++++-------------
 requirements-dev.txt        |  6 +--
 tools/ci_set_matrix.py      |  4 +-
 3 files changed, 60 insertions(+), 33 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index f7d57041e6..8cae4c3e68 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -21,11 +21,15 @@ jobs:
     runs-on: ubuntu-20.04
     outputs:
       build-doc: ${{ steps.set-matrix.outputs.build-doc }}
+      build-boards: ${{ steps.set-matrix.outputs.build-boards }}
       boards-aarch: ${{ steps.set-matrix.outputs.boards-aarch }}
       boards-arm: ${{ steps.set-matrix.outputs.boards-arm }}
       boards-espressif: ${{ steps.set-matrix.outputs.boards-espressif }}
       boards-riscv: ${{ steps.set-matrix.outputs.boards-riscv }}
       cp-version: ${{ steps.set-up-submodules.outputs.version }}
+    env:
+      MICROPY_CPYTHON3: python3.8
+      MICROPY_MICROPYTHON: ${{ github.workspace }}/ports/unix/micropython-coverage
     steps:
     - name: Dump GitHub context
       run: echo "$GITHUB_CONTEXT"
@@ -39,14 +43,14 @@ jobs:
     - name: Set up python
       uses: actions/setup-python@v4
       with:
-        python-version: "3.x"
+        python-version: 3.x
     - name: Duplicate USB VID/PID check
       run: python3 -u -m tools.ci_check_duplicate_usb_vid_pid
     - name: Set up submodules
       id: set-up-submodules
       uses: ./.github/actions/fetch_submodules
       with:
-        cache: "cache"
+        cache: cache
         version: true
     - name: Install dependencies
       run: |
@@ -60,24 +64,27 @@ jobs:
         python3 --version
     - name: Build mpy-cross
       run: make -C mpy-cross -j2
+    - uses: actions/upload-artifact@v3
+      with:
+        name: mpy-cross
+        path: mpy-cross/mpy-cross
     - name: Build unix port
-      run: |
-        make -C ports/unix VARIANT=coverage -j2
+      run: make -C ports/unix VARIANT=coverage -j2
     - name: Test all
-      run: MICROPY_CPYTHON3=python3.8 MICROPY_MICROPYTHON=../ports/unix/micropython-coverage ./run-tests.py -j1
+      run:  ./run-tests.py -j2
       working-directory: tests
     - name: Print failure info
-      run: MICROPY_CPYTHON3=python3.8 MICROPY_MICROPYTHON=../ports/unix/micropython-coverage ./run-tests.py -j1 --print-failures
+      run: ./run-tests.py -j2 --print-failures
       if: failure()
       working-directory: tests
     - name: Native Tests
-      run: MICROPY_CPYTHON3=python3.8 MICROPY_MICROPYTHON=../ports/unix/micropython-coverage ./run-tests.py -j1 --emit native
+      run: ./run-tests.py -j2 --emit native
       working-directory: tests
     - name: mpy Tests
-      run: MICROPY_CPYTHON3=python3.8 MICROPY_MICROPYTHON=../ports/unix/micropython-coverage ./run-tests.py -j1 --via-mpy -d basics float micropython
+      run: ./run-tests.py -j2 --via-mpy -d basics float micropython
       working-directory: tests
     - name: Native mpy Tests
-      run: MICROPY_CPYTHON3=python3.8 MICROPY_MICROPYTHON=../ports/unix/micropython-coverage ./run-tests.py -j1 --via-mpy --emit native -d basics float micropython
+      run: ./run-tests.py -j2 --via-mpy --emit native -d basics float micropython
       working-directory: tests
     - name: Build native modules
       run: |
@@ -90,7 +97,7 @@ jobs:
         make -C examples/natmod/ure
         make -C examples/natmod/uzlib
     - name: Test native modules
-      run: MICROPY_CPYTHON3=python3.8 MICROPY_MICROPYTHON=../ports/unix/micropython-coverage ./run-natmodtests.py extmod/{btree*,framebuf*,uheapq*,ure*,uzlib*}.py
+      run: ./run-natmodtests.py extmod/{btree*,framebuf*,uheapq*,ure*,uzlib*}.py
       working-directory: tests
     - name: Build mpy-cross.static-aarch64
       run: make -C mpy-cross -j2 -f Makefile.static-aarch64
@@ -166,11 +173,7 @@ jobs:
   mpy-cross-mac:
     runs-on: macos-11
     needs: test
-    if: >-
-      needs.test.outputs.boards-aarch != '[]' ||
-      needs.test.outputs.boards-arm != '[]' ||
-      needs.test.outputs.boards-espressif != '[]' ||
-      needs.test.outputs.boards-riscv != '[]'
+    if: ${{ needs.test.outputs.build-boards == 'True' }}
     env:
       CP_VERSION: ${{ needs.test.outputs.cp-version }}
     steps:
@@ -182,7 +185,7 @@ jobs:
     - name: Set up python
       uses: actions/setup-python@v4
       with:
-        python-version: "3.x"
+        python-version: 3.x
     - name: Set up submodules
       uses: ./.github/actions/fetch_submodules
     - name: Versions
@@ -235,7 +238,7 @@ jobs:
     - name: Set up python
       uses: actions/setup-python@v4
       with:
-        python-version: "3.x"
+        python-version: 3.x
     - name: Set up submodules
       uses: ./.github/actions/fetch_submodules
     - name: Install dependencies
@@ -305,7 +308,7 @@ jobs:
     - name: Set up python
       uses: actions/setup-python@v4
       with:
-        python-version: "3.x"
+        python-version: 3.x
     - name: Set up submodules
       id: set-up-submodules
       uses: ./.github/actions/fetch_submodules
@@ -334,9 +337,15 @@ jobs:
         arm-none-eabi-gcc --version
         python3 --version
         mkfs.fat --version || true
-    - name: Build mpy-cross
+    - name: Download mpy-cross
       if: ${{ steps.set-up-submodules.outputs.frozen == 'True' }}
-      run: make -C mpy-cross -j2
+      uses: actions/download-artifact@v3
+      with:
+        name: mpy-cross
+        path: mpy-cross
+    - name: Make mpy-cross executable
+      if: ${{ steps.set-up-submodules.outputs.frozen == 'True' }}
+      run: sudo chmod +x mpy-cross/mpy-cross
     - name: Setup build failure matcher
       run: echo "::add-matcher::$GITHUB_WORKSPACE/.github/workflows/match-build-fail.json"
     - name: Build
@@ -380,7 +389,7 @@ jobs:
     - name: Set up python
       uses: actions/setup-python@v4
       with:
-        python-version: "3.x"
+        python-version: 3.x
     - name: Set up submodules
       id: set-up-submodules
       uses: ./.github/actions/fetch_submodules
@@ -396,9 +405,15 @@ jobs:
         gcc --version
         arm-none-eabi-gcc --version
         python3 --version
-    - name: Build mpy-cross
+    - name: Download mpy-cross
       if: ${{ steps.set-up-submodules.outputs.frozen == 'True' }}
-      run: make -C mpy-cross -j2
+      uses: actions/download-artifact@v3
+      with:
+        name: mpy-cross
+        path: mpy-cross
+    - name: Make mpy-cross executable
+      if: ${{ steps.set-up-submodules.outputs.frozen == 'True' }}
+      run: sudo chmod +x mpy-cross/mpy-cross
     - name: Setup build failure matcher
       run: echo "::add-matcher::$GITHUB_WORKSPACE/.github/workflows/match-build-fail.json"
     - name: Build
@@ -489,9 +504,15 @@ jobs:
         python3 --version
         ninja --version
         cmake --version
-    - name: Build mpy-cross
+    - name: Download mpy-cross
       if: ${{ steps.set-up-submodules.outputs.frozen == 'True' }}
-      run: make -C mpy-cross -j2
+      uses: actions/download-artifact@v3
+      with:
+        name: mpy-cross
+        path: mpy-cross
+    - name: Make mpy-cross executable
+      if: ${{ steps.set-up-submodules.outputs.frozen == 'True' }}
+      run: sudo chmod +x mpy-cross/mpy-cross
     - name: Setup build failure matcher
       run: echo "::add-matcher::$GITHUB_WORKSPACE/.github/workflows/match-build-fail.json"
     - name: Build
@@ -537,7 +558,7 @@ jobs:
     - name: Set up python
       uses: actions/setup-python@v4
       with:
-        python-version: "3.x"
+        python-version: 3.x
     - name: Set up submodules
       id: set-up-submodules
       uses: ./.github/actions/fetch_submodules
@@ -552,9 +573,15 @@ jobs:
         gcc --version
         riscv64-unknown-elf-gcc --version
         python3 --version
-    - name: Build mpy-cross
+    - name: Download mpy-cross
       if: ${{ steps.set-up-submodules.outputs.frozen == 'True' }}
-      run: make -C mpy-cross -j2
+      uses: actions/download-artifact@v3
+      with:
+        name: mpy-cross
+        path: mpy-cross
+    - name: Make mpy-cross executable
+      if: ${{ steps.set-up-submodules.outputs.frozen == 'True' }}
+      run: sudo chmod +x mpy-cross/mpy-cross
     - name: Setup build failure matcher
       run: echo "::add-matcher::$GITHUB_WORKSPACE/.github/workflows/match-build-fail.json"
     - name: Build
diff --git a/requirements-dev.txt b/requirements-dev.txt
index 0dcfad3956..5e8fe3d3f6 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -6,6 +6,7 @@ typer
 sh
 click
 cpp-coveralls
+
 requests
 requests-cache
 
@@ -13,8 +14,8 @@ requests-cache
 polib
 
 # For pre-commit
-pyyaml
 black
+pyyaml
 pre-commit
 
 # for combining the Nordic SoftDevice with CircuitPython
@@ -23,9 +24,6 @@ intelhex
 # for building & testing natmods
 pyelftools
 
-# for stubs and annotations
-adafruit-circuitpython-typing
-
 # for mbedtls certificate store
 cryptography
 
diff --git a/tools/ci_set_matrix.py b/tools/ci_set_matrix.py
index bf899127ae..354ad51d71 100755
--- a/tools/ci_set_matrix.py
+++ b/tools/ci_set_matrix.py
@@ -204,7 +204,9 @@ def set_boards_to_build(build_all: bool):
             break
 
     # Split boards by architecture.
-    print("Building boards:")
+    build_boards = bool(boards_to_build)
+    print("Building boards:", build_boards)
+    set_output("build-boards", build_boards)
     arch_to_boards = {"aarch": [], "arm": [], "riscv": [], "espressif": []}
     for board in sorted(boards_to_build):
         print(" ", board)

From 23bb17c240745c655986e65a3f288a02364baf27 Mon Sep 17 00:00:00 2001
From: MicroDev <70126934+MicroDev1@users.noreply.github.com>
Date: Fri, 10 Feb 2023 17:10:56 +0530
Subject: [PATCH 08/35] use composite actions and reusable workflow

---
 .github/actions/external_deps/action.yml |  81 ++++++
 .github/actions/port_deps/action.yml     |  51 ++++
 .github/actions/upload_aws/action.yml    |  27 ++
 .github/workflows/build-boards.yml       |  99 +++++++
 .github/workflows/build.yml              | 346 +++--------------------
 5 files changed, 293 insertions(+), 311 deletions(-)
 create mode 100644 .github/actions/external_deps/action.yml
 create mode 100644 .github/actions/port_deps/action.yml
 create mode 100644 .github/actions/upload_aws/action.yml
 create mode 100644 .github/workflows/build-boards.yml

diff --git a/.github/actions/external_deps/action.yml b/.github/actions/external_deps/action.yml
new file mode 100644
index 0000000000..c9766eadc2
--- /dev/null
+++ b/.github/actions/external_deps/action.yml
@@ -0,0 +1,81 @@
+name: Fetch external deps
+
+inputs:
+  platform:
+    required: false
+    default: none
+    type: choice
+    options:
+    - arm
+    - aarch
+    - espressif
+    - riscv
+    - none
+
+runs:
+  using: "composite"
+  steps:
+    # aarch
+    - name: Get aarch toolchain
+      if: inputs.platform == 'aarch'
+      run: |
+        wget --no-verbose https://adafruit-circuit-python.s3.amazonaws.com/gcc-arm-10.3-2021.07-x86_64-aarch64-none-elf.tar.xz
+        sudo tar -C /usr --strip-components=1 -xaf gcc-arm-10.3-2021.07-x86_64-aarch64-none-elf.tar.xz
+        sudo apt-get install -y mtools
+      shell: bash
+    - name: Install mkfs.fat
+      if: inputs.platform == 'aarch'
+      run: |
+        wget https://github.com/dosfstools/dosfstools/releases/download/v4.2/dosfstools-4.2.tar.gz
+        tar -xaf dosfstools-4.2.tar.gz
+        cd dosfstools-4.2
+        ./configure
+        make -j 2
+        cd src
+        echo >> $GITHUB_PATH $(pwd)
+      shell: bash
+
+    # arm
+    - name: Get arm toolchain
+      if: inputs.platform == 'aarch' || inputs.platform == 'arm'
+      uses: carlosperate/arm-none-eabi-gcc-action@v1
+      with:
+        release: '10-2020-q4'
+
+    # espressif
+    - name: Get espressif toolchain
+      if: inputs.platform == 'espressif'
+      run: sudo apt-get install -y ninja-build
+      shell: bash
+    - name: Install IDF tools
+      if: inputs.platform == 'espressif'
+      run: |
+        echo "Installing ESP-IDF tools"
+        $IDF_PATH/tools/idf_tools.py --non-interactive install required
+        $IDF_PATH/tools/idf_tools.py --non-interactive install cmake
+        echo "Installing Python environment and packages"
+        $IDF_PATH/tools/idf_tools.py --non-interactive install-python-env
+        rm -rf $IDF_TOOLS_PATH/dist
+      shell: bash
+    - name: Set environment
+      if: inputs.platform == 'espressif'
+      run: |
+        source $IDF_PATH/export.sh
+        echo >> $GITHUB_ENV "IDF_PYTHON_ENV_PATH=$IDF_PYTHON_ENV_PATH"
+        echo >> $GITHUB_PATH "$PATH"
+      shell: bash
+
+    # riscv
+    - name: Get riscv toolchain
+      if: inputs.platform == 'riscv'
+      run: |
+        wget https://static.dev.sifive.com/dev-tools/riscv64-unknown-elf-gcc-8.3.0-2019.08.0-x86_64-linux-centos6.tar.gz
+        sudo tar -C /usr --strip-components=1 -xaf riscv64-unknown-elf-gcc-8.3.0-2019.08.0-x86_64-linux-centos6.tar.gz
+      shell: bash
+
+    # common
+    - name: Install dependencies
+      run: |
+        sudo apt-get install -y gettext
+        pip install -r requirements-dev.txt
+      shell: bash
diff --git a/.github/actions/port_deps/action.yml b/.github/actions/port_deps/action.yml
new file mode 100644
index 0000000000..c507983406
--- /dev/null
+++ b/.github/actions/port_deps/action.yml
@@ -0,0 +1,51 @@
+name: Fetch port deps
+
+inputs:
+  platform:
+    required: false
+    default: none
+    type: choice
+    options:
+    - espressif
+    - none
+
+runs:
+  using: composite
+  steps:
+    # espressif
+    - name: Set IDF env
+      if: inputs.platform == 'espressif'
+      run: |
+        echo >> $GITHUB_ENV "IDF_PATH=$GITHUB_WORKSPACE/ports/espressif/esp-idf"
+        echo >> $GITHUB_ENV "IDF_TOOLS_PATH=$GITHUB_WORKSPACE/.idf_tools"
+      shell: bash
+
+    - name: Get IDF commit
+      if: inputs.platform == 'espressif'
+      id: idf-commit
+      run: |
+        COMMIT=$(git submodule status ports/espressif/esp-idf | grep -o -P '(?<=^-).*(?= )')
+        echo "$COMMIT"
+        echo "commit=$COMMIT" >> $GITHUB_OUTPUT
+      shell: bash
+
+    - name: Cache IDF submodules
+      if: inputs.platform == 'espressif'
+      uses: actions/cache@v3
+      with:
+        path: |
+          .git/modules/ports/espressif/esp-idf
+          ports/espressif/esp-idf
+        key: submodules-idf-${{ steps.idf-commit.outputs.commit }}
+
+    - name: Cache IDF tools
+      if: inputs.platform == 'espressif'
+      uses: actions/cache@v3
+      with:
+        path: ${{ env.IDF_TOOLS_PATH }}
+        key: tools-idf-${{ steps.idf-commit.outputs.commit }}-${{ runner.os }}-${{ env.pythonLocation }}
+
+    - name: Initialize IDF submodules
+      if: inputs.platform == 'espressif'
+      run: git submodule update --init --depth=1 --recursive $IDF_PATH
+      shell: bash
diff --git a/.github/actions/upload_aws/action.yml b/.github/actions/upload_aws/action.yml
new file mode 100644
index 0000000000..a2d43f8de7
--- /dev/null
+++ b/.github/actions/upload_aws/action.yml
@@ -0,0 +1,27 @@
+name: Upload to AWS S3
+
+
+inputs:
+  source:
+    required: true
+    type: string
+
+  destination:
+    required: true
+    type: string
+
+
+runs:
+  using: composite
+  steps:
+    - name: Upload to S3
+      if: >-
+        (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') ||
+        (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))
+      run: |
+        pip install awscli
+        [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp bin/ s3://adafruit-circuit-python/bin/ --recursive --no-progress --region us-east-1
+      env:
+        AWS_PAGER: ''
+        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
diff --git a/.github/workflows/build-boards.yml b/.github/workflows/build-boards.yml
new file mode 100644
index 0000000000..10b572065a
--- /dev/null
+++ b/.github/workflows/build-boards.yml
@@ -0,0 +1,99 @@
+name: Build boards
+
+on:
+  workflow_call:
+    inputs:
+      platform:
+        required: true
+        type: string
+
+      boards:
+        required: true
+        type: string
+
+      cp-version:
+        required: true
+        type: string
+
+jobs:
+  build:
+    runs-on: ubuntu-22.04
+    env:
+      CP_VERSION: ${{ inputs.cp-version }}
+    strategy:
+      fail-fast: false
+      matrix:
+        board: ${{ fromJSON(inputs.boards) }}
+    steps:
+    - name: Dump GitHub context
+      run: echo "$GITHUB_CONTEXT"
+      env:
+        GITHUB_CONTEXT: ${{ toJson(github) }}
+    - name: Set up repository
+      uses: actions/checkout@v3
+      with:
+        submodules: false
+        fetch-depth: 1
+    - name: Set up python
+      uses: actions/setup-python@v4
+      with:
+        python-version: 3.x
+    - name: Set up port
+      uses: ./.github/actions/port_deps
+      with:
+        platform: ${{ inputs.platform }}
+    - name: Set up submodules
+      id: set-up-submodules
+      uses: ./.github/actions/fetch_submodules
+    - name: Set up external
+      uses: ./.github/actions/external_deps
+      with:
+        platform: ${{ inputs.platform }}
+
+    - name: Versions
+      run: |
+        gcc --version
+        python3 --version
+        cmake --version || true
+        ninja --version || true
+        aarch64-none-elf-gcc --version || true
+        arm-none-eabi-gcc --version || true
+        xtensa-esp32-elf-gcc --version || true
+        riscv32-esp-elf-gcc --version || true
+        riscv64-unknown-elf-gcc --version || true
+        mkfs.fat --version || true
+
+    - name: Download mpy-cross
+      if: ${{ steps.set-up-submodules.outputs.frozen == 'True' }}
+      uses: actions/download-artifact@v3
+      with:
+        name: mpy-cross
+        path: mpy-cross
+    - name: Make mpy-cross executable
+      if: ${{ steps.set-up-submodules.outputs.frozen == 'True' }}
+      run: sudo chmod +x mpy-cross/mpy-cross
+
+    - name: Set up build failure matcher
+      run: echo "::add-matcher::$GITHUB_WORKSPACE/.github/workflows/match-build-fail.json"
+    - name: Build
+      run: python3 -u build_release_files.py
+      working-directory: tools
+      env:
+        BOARDS: ${{ matrix.board }}
+
+    - name: Upload artifact
+      uses: actions/upload-artifact@v3
+      with:
+        name: ${{ matrix.board }}
+        path: bin/${{ matrix.board }}
+    - name: Upload to S3
+      if: >-
+        (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') ||
+        (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))
+      run: |
+        pip install awscli
+        [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp bin/ s3://adafruit-circuit-python/bin/ --recursive --no-progress --region us-east-1
+      env:
+        AWS_PAGER: ''
+        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 8cae4c3e68..b01e520fa9 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -137,16 +137,17 @@ jobs:
         [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross/mpy-cross.static-raspbian s3://adafruit-circuit-python/bin/mpy-cross/mpy-cross.static-raspbian-${{ env.CP_VERSION }} --no-progress --region us-east-1
         [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross/mpy-cross.static s3://adafruit-circuit-python/bin/mpy-cross/mpy-cross.static-amd64-linux-${{ env.CP_VERSION }} --no-progress --region us-east-1
         [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross/mpy-cross.static.exe s3://adafruit-circuit-python/bin/mpy-cross/mpy-cross.static-x64-windows-${{ env.CP_VERSION }}.exe --no-progress --region us-east-1
-    - name: Get last commit with checks
-      id: get-last-commit-with-checks
-      if: github.event_name == 'pull_request'
-      working-directory: tools
-      env:
-        REPO: ${{ github.repository }}
-        PULL: ${{ github.event.number }}
-        GITHUB_TOKEN: ${{ github.token }}
-        EXCLUDE_COMMIT: ${{ github.event.after }}
-      run: python3 -u ci_changes_per_commit.py
+    # Disabled: Needs to be updated
+    # - name: Get last commit with checks
+    #   id: get-last-commit-with-checks
+    #   if: github.event_name == 'pull_request'
+    #   working-directory: tools
+    #   run: python3 -u ci_changes_per_commit.py
+    #   env:
+    #     REPO: ${{ github.repository }}
+    #     PULL: ${{ github.event.number }}
+    #     GITHUB_TOKEN: ${{ github.token }}
+    #     EXCLUDE_COMMIT: ${{ github.event.after }}
     - name: Set head sha
       if: github.event_name == 'pull_request'
       run: echo "HEAD_SHA=$(git show -s --format=%s $GITHUB_SHA | grep -o -P "(?<=Merge ).*(?= into)")" >> $GITHUB_ENV
@@ -289,318 +290,41 @@ jobs:
         [ -z "$TWINE_USERNAME" ] || twine upload circuitpython-stubs/dist/*
 
 
-  build-aarch:
-    runs-on: ubuntu-22.04
+  aarch:
     needs: test
     if: ${{ needs.test.outputs.boards-aarch != '[]' }}
-    env:
-      CP_VERSION: ${{ needs.test.outputs.cp-version }}
-    strategy:
-      fail-fast: false
-      matrix:
-        board: ${{ fromJSON(needs.test.outputs.boards-aarch) }}
-    steps:
-    - name: Set up repository
-      uses: actions/checkout@v3
-      with:
-        submodules: false
-        fetch-depth: 1
-    - name: Set up python
-      uses: actions/setup-python@v4
-      with:
-        python-version: 3.x
-    - name: Set up submodules
-      id: set-up-submodules
-      uses: ./.github/actions/fetch_submodules
-    - name: Install dependencies
-      run: |
-        sudo apt-get install -y gettext mtools
-        pip install -r requirements-dev.txt
-        wget --no-verbose https://adafruit-circuit-python.s3.amazonaws.com/gcc-arm-10.3-2021.07-x86_64-aarch64-none-elf.tar.xz
-        sudo tar -C /usr --strip-components=1 -xaf gcc-arm-10.3-2021.07-x86_64-aarch64-none-elf.tar.xz
-    - uses: carlosperate/arm-none-eabi-gcc-action@v1
-      with:
-        release: '10-2020-q4'
-    - name: Install mkfs.fat
-      run: |
-        wget https://github.com/dosfstools/dosfstools/releases/download/v4.2/dosfstools-4.2.tar.gz
-        tar -xaf dosfstools-4.2.tar.gz
-        cd dosfstools-4.2
-        ./configure
-        make -j 2
-        cd src
-        echo >>$GITHUB_PATH $(pwd)
-    - name: Versions
-      run: |
-        gcc --version
-        aarch64-none-elf-gcc --version
-        arm-none-eabi-gcc --version
-        python3 --version
-        mkfs.fat --version || true
-    - name: Download mpy-cross
-      if: ${{ steps.set-up-submodules.outputs.frozen == 'True' }}
-      uses: actions/download-artifact@v3
-      with:
-        name: mpy-cross
-        path: mpy-cross
-    - name: Make mpy-cross executable
-      if: ${{ steps.set-up-submodules.outputs.frozen == 'True' }}
-      run: sudo chmod +x mpy-cross/mpy-cross
-    - name: Setup build failure matcher
-      run: echo "::add-matcher::$GITHUB_WORKSPACE/.github/workflows/match-build-fail.json"
-    - name: Build
-      run: python3 -u build_release_files.py
-      working-directory: tools
-      env:
-        BOARDS: ${{ matrix.board }}
-    - uses: actions/upload-artifact@v3
-      with:
-        name: ${{ matrix.board }}
-        path: bin/${{ matrix.board }}
-    - name: Upload to S3
-      if: >-
-        (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') ||
-        (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))
-      run: |
-        pip install awscli
-        [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp bin/ s3://adafruit-circuit-python/bin/ --recursive --no-progress --region us-east-1
-      env:
-        AWS_PAGER: ''
-        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+    uses: ./.github/workflows/build-boards.yml
+    with:
+      platform: aarch
+      boards: ${{ needs.test.outputs.boards-aarch }}
+      cp-version: ${{ needs.test.outputs.cp-version }}
 
 
-  build-arm:
-    runs-on: ubuntu-22.04
+  arm:
     needs: test
     if: ${{ needs.test.outputs.boards-arm != '[]' }}
-    env:
-      CP_VERSION: ${{ needs.test.outputs.cp-version }}
-    strategy:
-      fail-fast: false
-      matrix:
-        board: ${{ fromJSON(needs.test.outputs.boards-arm) }}
-    steps:
-    - name: Set up repository
-      uses: actions/checkout@v3
-      with:
-        submodules: false
-        fetch-depth: 1
-    - name: Set up python
-      uses: actions/setup-python@v4
-      with:
-        python-version: 3.x
-    - name: Set up submodules
-      id: set-up-submodules
-      uses: ./.github/actions/fetch_submodules
-    - uses: carlosperate/arm-none-eabi-gcc-action@v1
-      with:
-        release: '10-2020-q4'
-    - name: Install dependencies
-      run: |
-        sudo apt-get install -y gettext
-        pip install -r requirements-dev.txt
-    - name: Versions
-      run: |
-        gcc --version
-        arm-none-eabi-gcc --version
-        python3 --version
-    - name: Download mpy-cross
-      if: ${{ steps.set-up-submodules.outputs.frozen == 'True' }}
-      uses: actions/download-artifact@v3
-      with:
-        name: mpy-cross
-        path: mpy-cross
-    - name: Make mpy-cross executable
-      if: ${{ steps.set-up-submodules.outputs.frozen == 'True' }}
-      run: sudo chmod +x mpy-cross/mpy-cross
-    - name: Setup build failure matcher
-      run: echo "::add-matcher::$GITHUB_WORKSPACE/.github/workflows/match-build-fail.json"
-    - name: Build
-      run: python3 -u build_release_files.py
-      working-directory: tools
-      env:
-        BOARDS: ${{ matrix.board }}
-    - uses: actions/upload-artifact@v3
-      with:
-        name: ${{ matrix.board }}
-        path: bin/${{ matrix.board }}
-    - name: Upload to S3
-      if: >-
-        (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') ||
-        (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))
-      run: |
-        pip install awscli
-        [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp bin/ s3://adafruit-circuit-python/bin/ --recursive --no-progress --region us-east-1
-      env:
-        AWS_PAGER: ''
-        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+    uses: ./.github/workflows/build-boards.yml
+    with:
+      platform: arm
+      boards: ${{ needs.test.outputs.boards-arm }}
+      cp-version: ${{ needs.test.outputs.cp-version }}
 
 
-  build-espressif:
-    runs-on: ubuntu-22.04
+  espressif:
     needs: test
     if: ${{ needs.test.outputs.boards-espressif != '[]' }}
-    env:
-      CP_VERSION: ${{ needs.test.outputs.cp-version }}
-      IDF_PATH: ${{ github.workspace }}/ports/espressif/esp-idf
-      IDF_TOOLS_PATH: ${{ github.workspace }}/.idf_tools
-    strategy:
-      fail-fast: false
-      matrix:
-        board: ${{ fromJSON(needs.test.outputs.boards-espressif) }}
-    steps:
-    - name: Set up repository
-      uses: actions/checkout@v3
-      with:
-        submodules: false
-        fetch-depth: 1
-    - name: Set up python
-      id: setup-python
-      uses: actions/setup-python@v4
-      with:
-        python-version: "3.10"
-    - name: Get IDF commit
-      id: idf-commit
-      run: |
-        COMMIT=$(git submodule status ports/espressif/esp-idf | grep -o -P '(?<=^-).*(?= )')
-        echo "$COMMIT"
-        echo "commit=$COMMIT" >> $GITHUB_OUTPUT
-    - name: Cache IDF submodules
-      uses: actions/cache@v3
-      with:
-        path: |
-          .git/modules/ports/espressif/esp-idf
-          ports/espressif/esp-idf
-        key: submodules-idf-${{ steps.idf-commit.outputs.commit }}
-    - name: Cache IDF tools
-      uses: actions/cache@v3
-      with:
-        path: ${{ env.IDF_TOOLS_PATH }}
-        key: ${{ runner.os }}-Python-${{ steps.setup-python.outputs.python-version }}-tools-idf-${{ steps.idf-commit.outputs.commit }}
-    - name: Initialize IDF submodules
-      run: git submodule update --init --depth=1 --recursive $IDF_PATH
-    - name: Install IDF tools
-      run: |
-        echo "Installing ESP-IDF tools"
-        $IDF_PATH/tools/idf_tools.py --non-interactive install required
-        $IDF_PATH/tools/idf_tools.py --non-interactive install cmake
-        echo "Installing Python environment and packages"
-        $IDF_PATH/tools/idf_tools.py --non-interactive install-python-env
-        rm -rf $IDF_TOOLS_PATH/dist
-    - name: Set up submodules
-      id: set-up-submodules
-      uses: ./.github/actions/fetch_submodules
-    - name: Install dependencies
-      run: |
-        source $IDF_PATH/export.sh
-        sudo apt-get install -y gettext ninja-build
-        pip install -r requirements-dev.txt
-    - name: Versions
-      run: |
-        source $IDF_PATH/export.sh
-        gcc --version
-        python3 --version
-        ninja --version
-        cmake --version
-    - name: Download mpy-cross
-      if: ${{ steps.set-up-submodules.outputs.frozen == 'True' }}
-      uses: actions/download-artifact@v3
-      with:
-        name: mpy-cross
-        path: mpy-cross
-    - name: Make mpy-cross executable
-      if: ${{ steps.set-up-submodules.outputs.frozen == 'True' }}
-      run: sudo chmod +x mpy-cross/mpy-cross
-    - name: Setup build failure matcher
-      run: echo "::add-matcher::$GITHUB_WORKSPACE/.github/workflows/match-build-fail.json"
-    - name: Build
-      run: |
-        source $IDF_PATH/export.sh
-        python3 -u build_release_files.py
-      working-directory: tools
-      env:
-        BOARDS: ${{ matrix.board }}
-    - uses: actions/upload-artifact@v3
-      with:
-        name: ${{ matrix.board }}
-        path: bin/${{ matrix.board }}
-    - name: Upload to S3
-      if: >-
-        (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') ||
-        (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))
-      run: |
-        pip install awscli
-        [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp bin/ s3://adafruit-circuit-python/bin/ --recursive --no-progress --region us-east-1
-      env:
-        AWS_PAGER: ''
-        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+    uses: ./.github/workflows/build-boards.yml
+    with:
+      platform: espressif
+      boards: ${{ needs.test.outputs.boards-espressif }}
+      cp-version: ${{ needs.test.outputs.cp-version }}
 
 
-  build-riscv:
-    runs-on: ubuntu-22.04
+  riscv:
     needs: test
     if: ${{ needs.test.outputs.boards-riscv != '[]' }}
-    env:
-      CP_VERSION: ${{ needs.test.outputs.cp-version }}
-    strategy:
-      fail-fast: false
-      matrix:
-        board: ${{ fromJSON(needs.test.outputs.boards-riscv) }}
-    steps:
-    - name: Set up repository
-      uses: actions/checkout@v3
-      with:
-        submodules: false
-        fetch-depth: 1
-    - name: Set up python
-      uses: actions/setup-python@v4
-      with:
-        python-version: 3.x
-    - name: Set up submodules
-      id: set-up-submodules
-      uses: ./.github/actions/fetch_submodules
-    - name: Install dependencies
-      run: |
-        sudo apt-get install -y gettext
-        pip install -r requirements-dev.txt
-        wget https://static.dev.sifive.com/dev-tools/riscv64-unknown-elf-gcc-8.3.0-2019.08.0-x86_64-linux-centos6.tar.gz
-        sudo tar -C /usr --strip-components=1 -xaf riscv64-unknown-elf-gcc-8.3.0-2019.08.0-x86_64-linux-centos6.tar.gz
-    - name: Versions
-      run: |
-        gcc --version
-        riscv64-unknown-elf-gcc --version
-        python3 --version
-    - name: Download mpy-cross
-      if: ${{ steps.set-up-submodules.outputs.frozen == 'True' }}
-      uses: actions/download-artifact@v3
-      with:
-        name: mpy-cross
-        path: mpy-cross
-    - name: Make mpy-cross executable
-      if: ${{ steps.set-up-submodules.outputs.frozen == 'True' }}
-      run: sudo chmod +x mpy-cross/mpy-cross
-    - name: Setup build failure matcher
-      run: echo "::add-matcher::$GITHUB_WORKSPACE/.github/workflows/match-build-fail.json"
-    - name: Build
-      run: python3 -u build_release_files.py
-      working-directory: tools
-      env:
-        BOARDS: ${{ matrix.board }}
-    - uses: actions/upload-artifact@v3
-      with:
-        name: ${{ matrix.board }}
-        path: bin/${{ matrix.board }}
-    - name: Upload to S3
-      if: >-
-        (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') ||
-        (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))
-      run: |
-        pip install awscli
-        [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp bin/ s3://adafruit-circuit-python/bin/ --recursive --no-progress --region us-east-1
-      env:
-        AWS_PAGER: ''
-        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+    uses: ./.github/workflows/build-boards.yml
+    with:
+      platform: riscv
+      boards: ${{ needs.test.outputs.boards-riscv }}
+      cp-version: ${{ needs.test.outputs.cp-version }}

From 0d957fe15cbdca39d3d670129e0d891ca0f55b23 Mon Sep 17 00:00:00 2001
From: Jeff Epler <jepler@gmail.com>
Date: Fri, 10 Feb 2023 08:44:58 -0600
Subject: [PATCH 09/35] Fix several places where an exception could be chained
 wrongly

If an exception's chain or context can refer to a pointer from a different
VM, a crash would typically result.

This couldn't turn up on UNIX testing because the VM is never torn
down and rebuilt like it is on hardware.

Because in the 'static' case the GeneratorObject is now fully initialized
whenever it's raised, the initialization can be dropped, which reduces
the flash size slightly.
Closes: #7565
---
 py/objexcept.c    | 12 ++++++++++++
 py/objexcept.h    |  1 +
 py/objgenerator.c |  7 +++----
 py/runtime.c      |  8 ++------
 4 files changed, 18 insertions(+), 10 deletions(-)

diff --git a/py/objexcept.c b/py/objexcept.c
index 976535754b..24fb564e23 100644
--- a/py/objexcept.c
+++ b/py/objexcept.c
@@ -155,6 +155,12 @@ void mp_obj_exception_print(const mp_print_t *print, mp_obj_t o_in, mp_print_kin
     mp_obj_tuple_print(print, MP_OBJ_FROM_PTR(o->args), kind);
 }
 
+void mp_obj_exception_initialize0(mp_obj_exception_t *o_exc, const mp_obj_type_t *type) {
+    o_exc->base.type = type;
+    o_exc->args = (mp_obj_tuple_t *)&mp_const_empty_tuple_obj;
+    mp_obj_exception_clear_traceback(o_exc);
+}
+
 mp_obj_t mp_obj_exception_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args) {
     mp_arg_check_num(n_args, n_kw, 0, MP_OBJ_FUN_ARGS_MAX, false);
 
@@ -583,6 +589,12 @@ void mp_obj_exception_clear_traceback(mp_obj_t self_in) {
     // just set the traceback to the empty traceback object
     // we don't want to call any memory management functions here
     self->traceback = (mp_obj_traceback_t *)&mp_const_empty_traceback_obj;
+    #if MICROPY_CPYTHON_EXCEPTION_CHAIN
+    self->cause = 0;
+    self->context = 0;
+    self->suppress_context = false;
+    self->marked = false;
+    #endif
 }
 
 void mp_obj_exception_add_traceback(mp_obj_t self_in, qstr file, size_t line, qstr block) {
diff --git a/py/objexcept.h b/py/objexcept.h
index 77b338e951..1230fdf18a 100644
--- a/py/objexcept.h
+++ b/py/objexcept.h
@@ -43,6 +43,7 @@ typedef struct _mp_obj_exception_t {
 
 void mp_obj_exception_print(const mp_print_t *print, mp_obj_t o_in, mp_print_kind_t kind);
 void mp_obj_exception_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest);
+void mp_obj_exception_initialize0(mp_obj_exception_t *o_exc, const mp_obj_type_t *type);
 mp_obj_exception_t *mp_obj_exception_get_native(mp_obj_t self_in);
 
 #define MP_DEFINE_EXCEPTION(exc_name, base_name) \
diff --git a/py/objgenerator.c b/py/objgenerator.c
index 2256911e23..7c3ec99307 100644
--- a/py/objgenerator.c
+++ b/py/objgenerator.c
@@ -40,10 +40,11 @@
 // Instance of GeneratorExit exception - needed by generator.close()
 #if MICROPY_CONST_GENERATOREXIT_OBJ
 const
+mp_obj_exception_t mp_static_GeneratorExit_obj = {{&mp_type_GeneratorExit}, (mp_obj_tuple_t *)&mp_const_empty_tuple_obj, (mp_obj_traceback_t *)&mp_const_empty_traceback_obj};
 #else
 static
+mp_obj_exception_t mp_static_GeneratorExit_obj;
 #endif
-mp_obj_exception_t mp_static_GeneratorExit_obj = {{&mp_type_GeneratorExit}, (mp_obj_tuple_t *)&mp_const_empty_tuple_obj, (mp_obj_traceback_t *)&mp_const_empty_traceback_obj};
 
 /******************************************************************************/
 /* generator wrapper                                                          */
@@ -370,9 +371,7 @@ STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(gen_instance_throw_obj, 2, 4, gen_ins
 static mp_obj_t generatorexit(void) {
     #if MICROPY_CPYTHON_EXCEPTION_CHAIN
     MP_STATIC_ASSERT(!MICROPY_CONST_GENERATOREXIT_OBJ);
-    mp_static_GeneratorExit_obj.context = NULL;
-    mp_static_GeneratorExit_obj.cause = NULL;
-    mp_static_GeneratorExit_obj.suppress_context = false;
+    mp_obj_exception_initialize0(&mp_static_GeneratorExit_obj, &mp_type_GeneratorExit);
     #endif
     return MP_OBJ_FROM_PTR(&mp_static_GeneratorExit_obj);
 }
diff --git a/py/runtime.c b/py/runtime.c
index 9779456d0c..9227594d83 100644
--- a/py/runtime.c
+++ b/py/runtime.c
@@ -78,14 +78,10 @@ void mp_init(void) {
 
     #if MICROPY_KBD_EXCEPTION
     // initialise the exception object for raising KeyboardInterrupt
-    MP_STATE_VM(mp_kbd_exception).base.type = &mp_type_KeyboardInterrupt;
-    MP_STATE_VM(mp_kbd_exception).args = (mp_obj_tuple_t *)&mp_const_empty_tuple_obj;
-    MP_STATE_VM(mp_kbd_exception).traceback = (mp_obj_traceback_t *)&mp_const_empty_traceback_obj;
+    mp_obj_exception_initialize0(&MP_STATE_VM(mp_kbd_exception), &mp_type_KeyboardInterrupt);
     #endif
 
-    MP_STATE_VM(mp_reload_exception).base.type = &mp_type_ReloadException;
-    MP_STATE_VM(mp_reload_exception).args = (mp_obj_tuple_t *)&mp_const_empty_tuple_obj;
-    MP_STATE_VM(mp_reload_exception).traceback = (mp_obj_traceback_t *)&mp_const_empty_traceback_obj;
+    mp_obj_exception_initialize0(&MP_STATE_VM(mp_reload_exception), &mp_type_ReloadException);
 
     // call port specific initialization if any
     #ifdef MICROPY_PORT_INIT_FUNC

From cec36b62f1843901be0574fecc84e83ef5fa246a Mon Sep 17 00:00:00 2001
From: Neradoc <neraOnGit@ri1.fr>
Date: Fri, 10 Feb 2023 04:58:37 +0100
Subject: [PATCH 10/35] Update build_board_info.py to sh module 2.0.0

---
 tools/build_board_info.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/tools/build_board_info.py b/tools/build_board_info.py
index f810d942a7..5209337b95 100755
--- a/tools/build_board_info.py
+++ b/tools/build_board_info.py
@@ -10,6 +10,7 @@ import subprocess
 import sys
 import sh
 import base64
+from io import StringIO
 from datetime import date
 from sh.contrib import git
 
@@ -58,9 +59,13 @@ def get_languages(list_all=False):
 
 def get_version_info():
     version = None
-    sha = git("rev-parse", "--short", "HEAD").stdout.decode("utf-8")
+    buffer = StringIO()
+    git("rev-parse", "--short", "HEAD", _out=buffer)
+    sha = buffer.getvalue().strip()
     try:
-        version = git("describe", "--tags", "--exact-match").stdout.decode("utf-8").strip()
+        buffer = StringIO()
+        git("describe", "--tags", "--exact-match", _out=buffer)
+        version = buffer.getvalue().strip()
     except sh.ErrorReturnCode_128:
         # No exact match
         pass

From 17e03c47682f2a345f32cd48840886fa9a38199b Mon Sep 17 00:00:00 2001
From: MicroDev <70126934+MicroDev1@users.noreply.github.com>
Date: Fri, 10 Feb 2023 19:47:51 +0530
Subject: [PATCH 11/35] ci max parallelism

---
 .github/actions/fetch_submodules/action.yml |   7 +-
 .github/actions/mpy_cross/action.yml        |  30 ++++
 .github/workflows/build-boards.yml          |  17 +--
 .github/workflows/build-mpy-cross.yml       |  75 +++++++++
 .github/workflows/build.yml                 | 161 ++++++--------------
 .github/workflows/run-tests.yml             |  76 +++++++++
 tools/ci_fetch_deps.py                      |   7 +-
 7 files changed, 244 insertions(+), 129 deletions(-)
 create mode 100644 .github/actions/mpy_cross/action.yml
 create mode 100644 .github/workflows/build-mpy-cross.yml
 create mode 100644 .github/workflows/run-tests.yml

diff --git a/.github/actions/fetch_submodules/action.yml b/.github/actions/fetch_submodules/action.yml
index 407259967c..c8c6c9b1a0 100644
--- a/.github/actions/fetch_submodules/action.yml
+++ b/.github/actions/fetch_submodules/action.yml
@@ -1,6 +1,11 @@
 name: 'Fetch Submodules'
 
 inputs:
+  target:
+    description: 'The target for ci_fetch_deps'
+    required: false
+    type: string
+
   submodules:
     description: 'The submodules to cache'
     required: false
@@ -63,7 +68,7 @@ runs:
 
     - name: CircuitPython dependencies
       id: cp-deps
-      run: python tools/ci_fetch_deps.py ${{ matrix.board || github.job }}
+      run: python tools/ci_fetch_deps.py ${{ inputs.target || matrix.board || github.job }}
       shell: bash
 
     - name: CircuitPython version
diff --git a/.github/actions/mpy_cross/action.yml b/.github/actions/mpy_cross/action.yml
new file mode 100644
index 0000000000..ea3466e5fb
--- /dev/null
+++ b/.github/actions/mpy_cross/action.yml
@@ -0,0 +1,30 @@
+name: Set up mpy-cross
+
+runs:
+  using: composite
+  steps:
+    - name: Download mpy-cross
+      id: download-mpy-cross
+      continue-on-error: true
+      uses: actions/download-artifact@v3
+      with:
+        name: mpy-cross
+        path: mpy-cross
+
+    - name: Make mpy-cross executable
+      if: steps.download-mpy-cross.outcome == 'success'
+      run: sudo chmod +x mpy-cross/mpy-cross
+      shell: bash
+
+    - name: Build mpy-cross
+      if: steps.download-mpy-cross.outcome == 'failure'
+      run: make -C mpy-cross -j2
+      shell: bash
+
+    - name: Upload mpy-cross
+      continue-on-error: true
+      if: steps.download-mpy-cross.outcome == 'failure'
+      uses: actions/upload-artifact@v3
+      with:
+        name: mpy-cross
+        path: mpy-cross/mpy-cross
diff --git a/.github/workflows/build-boards.yml b/.github/workflows/build-boards.yml
index 10b572065a..9397b6f70a 100644
--- a/.github/workflows/build-boards.yml
+++ b/.github/workflows/build-boards.yml
@@ -25,10 +25,6 @@ jobs:
       matrix:
         board: ${{ fromJSON(inputs.boards) }}
     steps:
-    - name: Dump GitHub context
-      run: echo "$GITHUB_CONTEXT"
-      env:
-        GITHUB_CONTEXT: ${{ toJson(github) }}
     - name: Set up repository
       uses: actions/checkout@v3
       with:
@@ -49,6 +45,9 @@ jobs:
       uses: ./.github/actions/external_deps
       with:
         platform: ${{ inputs.platform }}
+    - name: Set up mpy-cross
+      if: steps.set-up-submodules.outputs.frozen == 'True'
+      uses: ./.github/actions/mpy_cross
 
     - name: Versions
       run: |
@@ -63,16 +62,6 @@ jobs:
         riscv64-unknown-elf-gcc --version || true
         mkfs.fat --version || true
 
-    - name: Download mpy-cross
-      if: ${{ steps.set-up-submodules.outputs.frozen == 'True' }}
-      uses: actions/download-artifact@v3
-      with:
-        name: mpy-cross
-        path: mpy-cross
-    - name: Make mpy-cross executable
-      if: ${{ steps.set-up-submodules.outputs.frozen == 'True' }}
-      run: sudo chmod +x mpy-cross/mpy-cross
-
     - name: Set up build failure matcher
       run: echo "::add-matcher::$GITHUB_WORKSPACE/.github/workflows/match-build-fail.json"
     - name: Build
diff --git a/.github/workflows/build-mpy-cross.yml b/.github/workflows/build-mpy-cross.yml
new file mode 100644
index 0000000000..cab3a533ae
--- /dev/null
+++ b/.github/workflows/build-mpy-cross.yml
@@ -0,0 +1,75 @@
+name: Build mpy-cross
+
+on:
+  workflow_call:
+    inputs:
+      cp-version:
+        required: true
+        type: string
+
+jobs:
+  build:
+    runs-on: ubuntu-22.04
+    strategy:
+      fail-fast: false
+      matrix:
+        mpy-cross: ["static", "static-aarch64", "static-mingw", "static-raspbian"]
+    env:
+      CP_VERSION: ${{ inputs.cp-version }}
+      EX_static-mingw: static.exe
+      OS_static: linux-amd64
+      OS_static-aarch64: linux-aarch64
+      OS_static-mingw: windows
+      OS_static-raspbian: linux-raspbian
+    steps:
+    - name: Set up repository
+      uses: actions/checkout@v3
+      with:
+        submodules: false
+        fetch-depth: 1
+
+    - name: Set up python
+      uses: actions/setup-python@v4
+      with:
+        python-version: 3.x
+
+    - name: Set up submodules
+      uses: ./.github/actions/fetch_submodules
+      with:
+        target: mpy-cross
+
+    - name: Install dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y gettext
+
+    - name: Install toolchain (aarch64)
+      if: matrix.mpy-cross == 'static-aarch64'
+      run: sudo apt-get install -y gcc-aarch64-linux-gnu
+
+    - name: Install toolchain (mingw)
+      if: matrix.mpy-cross == 'static-mingw'
+      run: sudo apt-get install -y mingw-w64
+
+    - name: Build mpy-cross.${{ matrix.mpy-cross }}
+      run: make -C mpy-cross -j2 -f Makefile.${{ matrix.mpy-cross }}
+
+    - name: Upload artifact
+      uses: actions/upload-artifact@v3
+      with:
+        name: mpy-cross.${{ env[format('EX_{0}', matrix.mpy-cross)] || matrix.mpy-cross }}
+        path: mpy-cross/mpy-cross.${{ env[format('EX_{0}', matrix.mpy-cross)] || matrix.mpy-cross }}
+
+    - name: Upload to S3
+      if: >-
+        (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') ||
+        (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))
+      env:
+        AWS_PAGER: ''
+        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      run: >-
+        [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp
+        mpy-cross/mpy-cross.${{ env[format('EX_{0}', matrix.mpy-cross)] || matrix.mpy-cross }}
+        s3://adafruit-circuit-python/bin/mpy-cross/$OS_${{ matrix.mpy-cross }}/mpy-cross-$CP_VERSION.${{ env[format('EX_{0}', matrix.mpy-cross)] || matrix.mpy-cross }}
+        --no-progress --region us-east-1
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index b01e520fa9..93c06022ce 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -17,7 +17,7 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  test:
+  scheduler:
     runs-on: ubuntu-20.04
     outputs:
       build-doc: ${{ steps.set-matrix.outputs.build-doc }}
@@ -54,89 +54,8 @@ jobs:
         version: true
     - name: Install dependencies
       run: |
-        sudo apt-get update
-        sudo apt-get install -y eatmydata
-        sudo eatmydata apt-get install -y gettext gcc-aarch64-linux-gnu mingw-w64
-        pip install -r requirements-dev.txt
-    - name: Versions
-      run: |
-        gcc --version
-        python3 --version
-    - name: Build mpy-cross
-      run: make -C mpy-cross -j2
-    - uses: actions/upload-artifact@v3
-      with:
-        name: mpy-cross
-        path: mpy-cross/mpy-cross
-    - name: Build unix port
-      run: make -C ports/unix VARIANT=coverage -j2
-    - name: Test all
-      run:  ./run-tests.py -j2
-      working-directory: tests
-    - name: Print failure info
-      run: ./run-tests.py -j2 --print-failures
-      if: failure()
-      working-directory: tests
-    - name: Native Tests
-      run: ./run-tests.py -j2 --emit native
-      working-directory: tests
-    - name: mpy Tests
-      run: ./run-tests.py -j2 --via-mpy -d basics float micropython
-      working-directory: tests
-    - name: Native mpy Tests
-      run: ./run-tests.py -j2 --via-mpy --emit native -d basics float micropython
-      working-directory: tests
-    - name: Build native modules
-      run: |
-        make -C examples/natmod/features1
-        make -C examples/natmod/features2
-        make -C examples/natmod/btree
-        make -C examples/natmod/framebuf
-        make -C examples/natmod/uheapq
-        make -C examples/natmod/urandom
-        make -C examples/natmod/ure
-        make -C examples/natmod/uzlib
-    - name: Test native modules
-      run: ./run-natmodtests.py extmod/{btree*,framebuf*,uheapq*,ure*,uzlib*}.py
-      working-directory: tests
-    - name: Build mpy-cross.static-aarch64
-      run: make -C mpy-cross -j2 -f Makefile.static-aarch64
-    - uses: actions/upload-artifact@v3
-      with:
-        name: mpy-cross.static-aarch64
-        path: mpy-cross/mpy-cross.static-aarch64
-    - name: Build mpy-cross.static-raspbian
-      run: make -C mpy-cross -j2 -f Makefile.static-raspbian
-    - uses: actions/upload-artifact@v3
-      with:
-        name: mpy-cross.static-raspbian
-        path: mpy-cross/mpy-cross.static-raspbian
-    - name: Build mpy-cross.static
-      run: make -C mpy-cross -j2 -f Makefile.static
-    - uses: actions/upload-artifact@v3
-      with:
-        name: mpy-cross.static-amd64-linux
-        path: mpy-cross/mpy-cross.static
-    - name: Build mpy-cross.static-mingw
-      run: make -C mpy-cross -j2 -f Makefile.static-mingw
-    - uses: actions/upload-artifact@v3
-      with:
-        name: mpy-cross.static-x64-windows
-        path: mpy-cross/mpy-cross.static.exe
-    - name: Upload to S3
-      if: >-
-        (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') ||
-        (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))
-      env:
-        AWS_PAGER: ''
-        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-      run: |
-        pip install awscli
-        [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross/mpy-cross.static-aarch64 s3://adafruit-circuit-python/bin/mpy-cross/mpy-cross.static-aarch64-${{ env.CP_VERSION }} --no-progress --region us-east-1
-        [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross/mpy-cross.static-raspbian s3://adafruit-circuit-python/bin/mpy-cross/mpy-cross.static-raspbian-${{ env.CP_VERSION }} --no-progress --region us-east-1
-        [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross/mpy-cross.static s3://adafruit-circuit-python/bin/mpy-cross/mpy-cross.static-amd64-linux-${{ env.CP_VERSION }} --no-progress --region us-east-1
-        [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross/mpy-cross.static.exe s3://adafruit-circuit-python/bin/mpy-cross/mpy-cross.static-x64-windows-${{ env.CP_VERSION }}.exe --no-progress --region us-east-1
+        sudo apt-get install -y gettext
+        pip install requests requests-cache sh
     # Disabled: Needs to be updated
     # - name: Get last commit with checks
     #   id: get-last-commit-with-checks
@@ -148,6 +67,8 @@ jobs:
     #     PULL: ${{ github.event.number }}
     #     GITHUB_TOKEN: ${{ github.token }}
     #     EXCLUDE_COMMIT: ${{ github.event.after }}
+    - name: Set up mpy-cross
+      uses: ./.github/actions/mpy_cross
     - name: Set head sha
       if: github.event_name == 'pull_request'
       run: echo "HEAD_SHA=$(git show -s --format=%s $GITHUB_SHA | grep -o -P "(?<=Merge ).*(?= into)")" >> $GITHUB_ENV
@@ -170,13 +91,28 @@ jobs:
         CHANGED_FILES: ${{ steps.get-changes.outputs.changed_files }}
         LAST_FAILED_JOBS: ${{ steps.get-last-commit-with-checks.outputs.check_runs }}
 
+  tests:
+    needs: scheduler
+    if: needs.scheduler.outputs.build-boards == 'True'
+    uses: ./.github/workflows/run-tests.yml
+    with:
+      cp-version: ${{ needs.scheduler.outputs.cp-version }}
+
+
+  mpy-cross:
+    needs: scheduler
+    if: needs.scheduler.outputs.build-boards == 'True'
+    uses: ./.github/workflows/build-mpy-cross.yml
+    with:
+      cp-version: ${{ needs.scheduler.outputs.cp-version }}
+
 
   mpy-cross-mac:
     runs-on: macos-11
-    needs: test
-    if: ${{ needs.test.outputs.build-boards == 'True' }}
+    needs: scheduler
+    if: ${{ needs.scheduler.outputs.build-boards == 'True' }}
     env:
-      CP_VERSION: ${{ needs.test.outputs.cp-version }}
+      CP_VERSION: ${{ needs.scheduler.outputs.cp-version }}
     steps:
     - name: Set up repository
       uses: actions/checkout@v3
@@ -213,23 +149,25 @@ jobs:
         name: mpy-cross-macos-11-universal
         path: mpy-cross-macos-universal
     - name: Upload mpy-cross build to S3
-      run: |
-        [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross-macos-universal s3://adafruit-circuit-python/bin/mpy-cross/mpy-cross-macos-11-${{ env.CP_VERSION }}-universal --no-progress --region us-east-1
-        [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross/mpy-cross-arm64 s3://adafruit-circuit-python/bin/mpy-cross/mpy-cross-macos-11-${{ env.CP_VERSION }}-arm64 --no-progress --region us-east-1
-        [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross/mpy-cross s3://adafruit-circuit-python/bin/mpy-cross/mpy-cross-macos-11-${{ env.CP_VERSION }}-x64 --no-progress --region us-east-1
+      if: >-
+          (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') ||
+          (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))
       env:
         AWS_PAGER: ''
         AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
         AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-      if: (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') || (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))
+      run: |
+        [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross-macos-universal s3://adafruit-circuit-python/bin/mpy-cross/mpy-cross-macos-11-${{ env.CP_VERSION }}-universal --no-progress --region us-east-1
+        [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross/mpy-cross-arm64 s3://adafruit-circuit-python/bin/mpy-cross/mpy-cross-macos-11-${{ env.CP_VERSION }}-arm64 --no-progress --region us-east-1
+        [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross/mpy-cross s3://adafruit-circuit-python/bin/mpy-cross/mpy-cross-macos-11-${{ env.CP_VERSION }}-x64 --no-progress --region us-east-1
 
 
   build-doc:
     runs-on: ubuntu-22.04
-    needs: test
-    if: ${{ needs.test.outputs.build-doc == 'True' }}
+    needs: scheduler
+    if: ${{ needs.scheduler.outputs.build-doc == 'True' }}
     env:
-      CP_VERSION: ${{ needs.test.outputs.cp-version }}
+      CP_VERSION: ${{ needs.scheduler.outputs.cp-version }}
     steps:
     - name: Set up repository
       uses: actions/checkout@v3
@@ -276,7 +214,6 @@ jobs:
         AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
         AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
       run: |
-        pip install awscli
         zip -9r circuitpython-stubs.zip circuitpython-stubs
         [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp circuitpython-stubs/dist/*.tar.gz s3://adafruit-circuit-python/bin/stubs/circuitpython-stubs-${{ env.CP_VERSION }}.zip --no-progress --region us-east-1
     - name: Upload stubs to PyPi
@@ -291,40 +228,40 @@ jobs:
 
 
   aarch:
-    needs: test
-    if: ${{ needs.test.outputs.boards-aarch != '[]' }}
+    needs: [scheduler, mpy-cross, tests]
+    if: ${{ needs.scheduler.outputs.boards-aarch != '[]' }}
     uses: ./.github/workflows/build-boards.yml
     with:
       platform: aarch
-      boards: ${{ needs.test.outputs.boards-aarch }}
-      cp-version: ${{ needs.test.outputs.cp-version }}
+      boards: ${{ needs.scheduler.outputs.boards-aarch }}
+      cp-version: ${{ needs.scheduler.outputs.cp-version }}
 
 
   arm:
-    needs: test
-    if: ${{ needs.test.outputs.boards-arm != '[]' }}
+    needs: [scheduler, mpy-cross, tests]
+    if: ${{ needs.scheduler.outputs.boards-arm != '[]' }}
     uses: ./.github/workflows/build-boards.yml
     with:
       platform: arm
-      boards: ${{ needs.test.outputs.boards-arm }}
-      cp-version: ${{ needs.test.outputs.cp-version }}
+      boards: ${{ needs.scheduler.outputs.boards-arm }}
+      cp-version: ${{ needs.scheduler.outputs.cp-version }}
 
 
   espressif:
-    needs: test
-    if: ${{ needs.test.outputs.boards-espressif != '[]' }}
+    needs: [scheduler, mpy-cross, tests]
+    if: ${{ needs.scheduler.outputs.boards-espressif != '[]' }}
     uses: ./.github/workflows/build-boards.yml
     with:
       platform: espressif
-      boards: ${{ needs.test.outputs.boards-espressif }}
-      cp-version: ${{ needs.test.outputs.cp-version }}
+      boards: ${{ needs.scheduler.outputs.boards-espressif }}
+      cp-version: ${{ needs.scheduler.outputs.cp-version }}
 
 
   riscv:
-    needs: test
-    if: ${{ needs.test.outputs.boards-riscv != '[]' }}
+    needs: [scheduler, mpy-cross, tests]
+    if: ${{ needs.scheduler.outputs.boards-riscv != '[]' }}
     uses: ./.github/workflows/build-boards.yml
     with:
       platform: riscv
-      boards: ${{ needs.test.outputs.boards-riscv }}
-      cp-version: ${{ needs.test.outputs.cp-version }}
+      boards: ${{ needs.scheduler.outputs.boards-riscv }}
+      cp-version: ${{ needs.scheduler.outputs.cp-version }}
diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
new file mode 100644
index 0000000000..15fc8489a4
--- /dev/null
+++ b/.github/workflows/run-tests.yml
@@ -0,0 +1,76 @@
+name: Run tests
+
+on:
+  workflow_call:
+    inputs:
+      cp-version:
+        required: true
+        type: string
+
+jobs:
+  run:
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        test: [all, mpy, native, native_mpy]
+    env:
+      CP_VERSION: ${{ inputs.cp-version }}
+      MICROPY_CPYTHON3: python3.8
+      MICROPY_MICROPYTHON: ../ports/unix/micropython-coverage
+      TEST_all:
+      TEST_mpy: --via-mpy -d basics float micropython
+      TEST_native: --emit native
+      TEST_native_mpy: --via-mpy --emit native -d basics float micropython
+    steps:
+    - name: Set up repository
+      uses: actions/checkout@v3
+      with:
+        submodules: false
+        fetch-depth: 1
+    - name: Set up python
+      uses: actions/setup-python@v4
+      with:
+        python-version: 3.x
+    - name: Set up submodules
+      uses: ./.github/actions/fetch_submodules
+      with:
+        target: tests
+
+    - name: Install dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y gettext
+
+    - name: Set up mpy-cross
+      uses: ./.github/actions/mpy_cross
+
+    - name: Build unix port
+      run: make -C ports/unix VARIANT=coverage -j2
+
+    - name: Test all
+      run: ./run-tests.py -j2 $TEST_${{ matrix.test }}
+      working-directory: tests
+    - name: Print failure info
+      run: ./run-tests.py -j2 --print-failures
+      if: failure()
+      working-directory: tests
+
+    - name: Set up native modules
+      if: matrix.test == 'all'
+      run: pip install -r requirements-dev.txt
+    - name: Build native modules
+      if: matrix.test == 'all'
+      run: |
+        make -C examples/natmod/features1
+        make -C examples/natmod/features2
+        make -C examples/natmod/btree
+        make -C examples/natmod/framebuf
+        make -C examples/natmod/uheapq
+        make -C examples/natmod/urandom
+        make -C examples/natmod/ure
+        make -C examples/natmod/uzlib
+    - name: Test native modules
+      if: matrix.test == 'all'
+      run: ./run-natmodtests.py extmod/{btree*,framebuf*,uheapq*,ure*,uzlib*}.py
+      working-directory: tests
diff --git a/tools/ci_fetch_deps.py b/tools/ci_fetch_deps.py
index 5cf2279314..bdf2967af8 100644
--- a/tools/ci_fetch_deps.py
+++ b/tools/ci_fetch_deps.py
@@ -73,12 +73,15 @@ def main():
 
     print("Target:", TARGET)
 
-    if TARGET == "test":
+    if TARGET == "scheduler":
+        # submodules = ["tools/"]
+        submodules = ["extmod/ulab", "lib/", "tools/"]
+    elif TARGET == "tests":
         submodules = ["extmod/ulab", "lib/", "tools/"]
     elif TARGET == "build-doc":
         # used in .readthedocs.yml to generate RTD
         submodules = ["extmod/ulab", "frozen/"]
-    elif TARGET == "mpy-cross-mac":
+    elif TARGET == "mpy-cross" or TARGET == "mpy-cross-mac":
         submodules = ["tools/"]  # for huffman
     elif TARGET == "windows":
         # This builds one board from a number of ports so fill out a bunch of submodules

From 70a4efc08ae6495810704ceb2830406e4439d6df Mon Sep 17 00:00:00 2001
From: MicroDev <70126934+MicroDev1@users.noreply.github.com>
Date: Fri, 10 Feb 2023 23:03:13 +0530
Subject: [PATCH 12/35] use composite action for aws s3 upload

---
 .github/actions/upload_aws/action.yml | 19 ++++++++++++-------
 .github/workflows/build-boards.yml    | 11 +++--------
 .github/workflows/build-mpy-cross.yml | 14 ++++----------
 .github/workflows/build.yml           | 26 +++++++++++---------------
 4 files changed, 30 insertions(+), 40 deletions(-)

diff --git a/.github/actions/upload_aws/action.yml b/.github/actions/upload_aws/action.yml
index a2d43f8de7..7aee8c7e78 100644
--- a/.github/actions/upload_aws/action.yml
+++ b/.github/actions/upload_aws/action.yml
@@ -1,15 +1,19 @@
 name: Upload to AWS S3
 
-
 inputs:
   source:
     required: true
     type: string
 
   destination:
-    required: true
+    required: false
     type: string
 
+  AWS_ACCESS_KEY_ID:
+    required: true
+
+  AWS_SECRET_ACCESS_KEY:
+    required: true
 
 runs:
   using: composite
@@ -18,10 +22,11 @@ runs:
       if: >-
         (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') ||
         (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))
-      run: |
-        pip install awscli
-        [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp bin/ s3://adafruit-circuit-python/bin/ --recursive --no-progress --region us-east-1
+      run: >-
+        [ -z "$AWS_ACCESS_KEY_ID" ] ||
+        aws s3 cp ${{ inputs.source }} s3://adafruit-circuit-python/bin/${{ inputs.destination }} --recursive --no-progress --region us-east-1
       env:
         AWS_PAGER: ''
-        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        AWS_ACCESS_KEY_ID: ${{ inputs.AWS_ACCESS_KEY_ID }}
+        AWS_SECRET_ACCESS_KEY: ${{ inputs.AWS_SECRET_ACCESS_KEY }}
+      shell: bash
diff --git a/.github/workflows/build-boards.yml b/.github/workflows/build-boards.yml
index 9397b6f70a..9b53615fe7 100644
--- a/.github/workflows/build-boards.yml
+++ b/.github/workflows/build-boards.yml
@@ -76,13 +76,8 @@ jobs:
         name: ${{ matrix.board }}
         path: bin/${{ matrix.board }}
     - name: Upload to S3
-      if: >-
-        (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') ||
-        (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))
-      run: |
-        pip install awscli
-        [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp bin/ s3://adafruit-circuit-python/bin/ --recursive --no-progress --region us-east-1
-      env:
-        AWS_PAGER: ''
+      uses: ./.github/actions/upload_aws
+      with:
+        source: bin/
         AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
         AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
diff --git a/.github/workflows/build-mpy-cross.yml b/.github/workflows/build-mpy-cross.yml
index cab3a533ae..3d557df8ee 100644
--- a/.github/workflows/build-mpy-cross.yml
+++ b/.github/workflows/build-mpy-cross.yml
@@ -61,15 +61,9 @@ jobs:
         path: mpy-cross/mpy-cross.${{ env[format('EX_{0}', matrix.mpy-cross)] || matrix.mpy-cross }}
 
     - name: Upload to S3
-      if: >-
-        (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') ||
-        (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))
-      env:
-        AWS_PAGER: ''
+      uses: ./.github/actions/upload_aws
+      with:
+        source: mpy-cross/mpy-cross.${{ env[format('EX_{0}', matrix.mpy-cross)] || matrix.mpy-cross }}
+        destination: mpy-cross/${{ env[format('OS_{0}', matrix.mpy-cross)] }}/mpy-cross-${{ env.CP_VERSION }}.${{ env[format('EX_{0}', matrix.mpy-cross)] || matrix.mpy-cross }}
         AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
         AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-      run: >-
-        [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp
-        mpy-cross/mpy-cross.${{ env[format('EX_{0}', matrix.mpy-cross)] || matrix.mpy-cross }}
-        s3://adafruit-circuit-python/bin/mpy-cross/$OS_${{ matrix.mpy-cross }}/mpy-cross-$CP_VERSION.${{ env[format('EX_{0}', matrix.mpy-cross)] || matrix.mpy-cross }}
-        --no-progress --region us-east-1
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 93c06022ce..37bb4133cd 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -144,23 +144,23 @@ jobs:
         path: mpy-cross/mpy-cross-arm64
     - name: Make universal binary
       run: lipo -create -output mpy-cross-macos-universal mpy-cross/mpy-cross mpy-cross/mpy-cross-arm64
-    - uses: actions/upload-artifact@v3
+    - name: Upload artifact
+      uses: actions/upload-artifact@v3
       with:
         name: mpy-cross-macos-11-universal
         path: mpy-cross-macos-universal
-    - name: Upload mpy-cross build to S3
+    - name: Upload to S3
       if: >-
           (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') ||
           (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))
-      env:
-        AWS_PAGER: ''
-        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
       run: |
         [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross-macos-universal s3://adafruit-circuit-python/bin/mpy-cross/mpy-cross-macos-11-${{ env.CP_VERSION }}-universal --no-progress --region us-east-1
         [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross/mpy-cross-arm64 s3://adafruit-circuit-python/bin/mpy-cross/mpy-cross-macos-11-${{ env.CP_VERSION }}-arm64 --no-progress --region us-east-1
         [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross/mpy-cross s3://adafruit-circuit-python/bin/mpy-cross/mpy-cross-macos-11-${{ env.CP_VERSION }}-x64 --no-progress --region us-east-1
-
+      env:
+        AWS_PAGER: ''
+        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
 
   build-doc:
     runs-on: ubuntu-22.04
@@ -206,16 +206,12 @@ jobs:
         name: docs
         path: _build/latex
     - name: Upload to S3
-      if: >-
-        (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') ||
-        (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))
-      env:
-        AWS_PAGER: ''
+      uses: ./.github/actions/upload_aws
+      with:
+        source: circuitpython-stubs/dist/*.tar.gz
+        destination: stubs/circuitpython-stubs-${{ env.CP_VERSION }}.zip
         AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
         AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-      run: |
-        zip -9r circuitpython-stubs.zip circuitpython-stubs
-        [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp circuitpython-stubs/dist/*.tar.gz s3://adafruit-circuit-python/bin/stubs/circuitpython-stubs-${{ env.CP_VERSION }}.zip --no-progress --region us-east-1
     - name: Upload stubs to PyPi
       if: github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested')
       env:

From 3c93594563443a8e219f3df6c149f2169ec5bd6b Mon Sep 17 00:00:00 2001
From: Neradoc <neraOnGit@ri1.fr>
Date: Fri, 10 Feb 2023 19:54:10 +0100
Subject: [PATCH 13/35] Deinit the reset pin when displayio.I2CDisplay raises
 an exception

---
 shared-module/displayio/I2CDisplay.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/shared-module/displayio/I2CDisplay.c b/shared-module/displayio/I2CDisplay.c
index 8fae5d31fd..f38e4d6296 100644
--- a/shared-module/displayio/I2CDisplay.c
+++ b/shared-module/displayio/I2CDisplay.c
@@ -54,6 +54,7 @@ void common_hal_displayio_i2cdisplay_construct(displayio_i2cdisplay_obj_t *self,
     // Probe the bus to see if a device acknowledges the given address.
     if (!common_hal_busio_i2c_probe(i2c, device_address)) {
         self->base.type = &mp_type_NoneType;
+        common_hal_displayio_i2cdisplay_deinit(self);
         mp_raise_ValueError_varg(translate("Unable to find I2C Display at %x"), device_address);
     }
 

From d959ef1e5b9c2f9b42e0169142dab1bdbd48b0fe Mon Sep 17 00:00:00 2001
From: MicroDev <70126934+MicroDev1@users.noreply.github.com>
Date: Sat, 11 Feb 2023 12:00:27 +0530
Subject: [PATCH 14/35] use venv to cache python deps

---
 .../external}/action.yml                      | 28 ++++++++++++---
 .../{port_deps => deps/port}/action.yml       |  2 +-
 .github/actions/deps/python/action.yml        | 35 +++++++++++++++++++
 .../submodules}/action.yml                    |  6 ++--
 .github/actions/mpy_cross/action.yml          | 13 +++++--
 .github/workflows/build-boards.yml            |  8 ++---
 .github/workflows/build-mpy-cross.yml         | 28 +++++++--------
 .github/workflows/build.yml                   | 19 +++++-----
 .github/workflows/create_website_pr.yml       | 10 +++---
 .github/workflows/ports_windows.yml           |  2 +-
 .github/workflows/pre-commit.yml              | 12 +++----
 .github/workflows/run-tests.yml               | 20 +++--------
 12 files changed, 116 insertions(+), 67 deletions(-)
 rename .github/actions/{external_deps => deps/external}/action.yml (79%)
 rename .github/actions/{port_deps => deps/port}/action.yml (92%)
 create mode 100644 .github/actions/deps/python/action.yml
 rename .github/actions/{fetch_submodules => deps/submodules}/action.yml (96%)

diff --git a/.github/actions/external_deps/action.yml b/.github/actions/deps/external/action.yml
similarity index 79%
rename from .github/actions/external_deps/action.yml
rename to .github/actions/deps/external/action.yml
index c9766eadc2..408a2e8555 100644
--- a/.github/actions/external_deps/action.yml
+++ b/.github/actions/deps/external/action.yml
@@ -12,8 +12,18 @@ inputs:
     - riscv
     - none
 
+  apt:
+    required: false
+    default: true
+    type: boolean
+
+  python:
+    required: false
+    default: true
+    type: boolean
+
 runs:
-  using: "composite"
+  using: composite
   steps:
     # aarch
     - name: Get aarch toolchain
@@ -74,8 +84,16 @@ runs:
       shell: bash
 
     # common
-    - name: Install dependencies
-      run: |
-        sudo apt-get install -y gettext
-        pip install -r requirements-dev.txt
+    - name: Cache python dependencies
+      if: inputs.python == 'true' && inputs.platform != 'espressif'
+      uses: ./.github/actions/deps/python
+      with:
+        action: ${{ fromJSON('["restore", "cache"]')[github.job == 'scheduler'] }}
+    - name: Install python dependencies
+      if: inputs.python == 'true'
+      run: pip install -r requirements-dev.txt
+      shell: bash
+    - name: Install dependencies
+      if: inputs.apt == 'true'
+      run: sudo apt-get install -y gettext
       shell: bash
diff --git a/.github/actions/port_deps/action.yml b/.github/actions/deps/port/action.yml
similarity index 92%
rename from .github/actions/port_deps/action.yml
rename to .github/actions/deps/port/action.yml
index c507983406..36b67c6081 100644
--- a/.github/actions/port_deps/action.yml
+++ b/.github/actions/deps/port/action.yml
@@ -43,7 +43,7 @@ runs:
       uses: actions/cache@v3
       with:
         path: ${{ env.IDF_TOOLS_PATH }}
-        key: tools-idf-${{ steps.idf-commit.outputs.commit }}-${{ runner.os }}-${{ env.pythonLocation }}
+        key: ${{ runner.os }}-${{ env.pythonLocation }}-tools-idf-${{ steps.idf-commit.outputs.commit }}
 
     - name: Initialize IDF submodules
       if: inputs.platform == 'espressif'
diff --git a/.github/actions/deps/python/action.yml b/.github/actions/deps/python/action.yml
new file mode 100644
index 0000000000..fb978ef063
--- /dev/null
+++ b/.github/actions/deps/python/action.yml
@@ -0,0 +1,35 @@
+name: Fetch python deps
+
+inputs:
+  action:
+    description: The cache action to use
+    required: false
+    default: restore
+    type: choice
+    options:
+    - cache
+    - restore
+
+runs:
+  using: composite
+  steps:
+    - name: Cache python dependencies
+      if: ${{ inputs.action == 'cache' }}
+      uses: actions/cache@v3
+      with:
+        path: .cp_tools
+        key: ${{ runner.os }}-${{ env.pythonLocation }}-tools-cp-${{ hashFiles('requirements-dev.txt') }}
+
+    - name: Restore python dependencies
+      if: ${{ inputs.action == 'restore' }}
+      uses: actions/cache/restore@v3
+      with:
+        path: .cp_tools
+        key: ${{ runner.os }}-${{ env.pythonLocation }}-tools-cp-${{ hashFiles('requirements-dev.txt') }}
+
+    - name: Set up venv
+      run: |
+        python -m venv .cp_tools
+        source .cp_tools/bin/activate
+        echo >> $GITHUB_PATH "$PATH"
+      shell: bash
diff --git a/.github/actions/fetch_submodules/action.yml b/.github/actions/deps/submodules/action.yml
similarity index 96%
rename from .github/actions/fetch_submodules/action.yml
rename to .github/actions/deps/submodules/action.yml
index c8c6c9b1a0..bca76b204c 100644
--- a/.github/actions/fetch_submodules/action.yml
+++ b/.github/actions/deps/submodules/action.yml
@@ -12,7 +12,7 @@ inputs:
     default: '["extmod/ulab", "lib/", "tools/"]'
     type: string
 
-  cache:
+  action:
     description: 'The cache action to use'
     required: false
     default: 'restore'
@@ -47,7 +47,7 @@ runs:
       shell: bash
 
     - name: Cache submodules
-      if: ${{ inputs.cache == 'cache' }}
+      if: ${{ inputs.action == 'cache' }}
       uses: actions/cache@v3
       with:
         path: ".git/modules/\n${{ join(fromJSON(steps.create-submodule-status.outputs.submodules), '\n') }}"
@@ -55,7 +55,7 @@ runs:
         enableCrossOsArchive: true
 
     - name: Restore submodules
-      if: ${{ inputs.cache == 'restore' }}
+      if: ${{ inputs.action == 'restore' }}
       uses: actions/cache/restore@v3
       with:
         path: ".git/modules/\n${{ join(fromJSON(steps.create-submodule-status.outputs.submodules), '\n') }}"
diff --git a/.github/actions/mpy_cross/action.yml b/.github/actions/mpy_cross/action.yml
index ea3466e5fb..d9fe54fdc7 100644
--- a/.github/actions/mpy_cross/action.yml
+++ b/.github/actions/mpy_cross/action.yml
@@ -1,10 +1,17 @@
 name: Set up mpy-cross
 
+inputs:
+  download:
+    required: false
+    default: true
+    type: boolean
+
 runs:
   using: composite
   steps:
     - name: Download mpy-cross
       id: download-mpy-cross
+      if: inputs.download == 'true'
       continue-on-error: true
       uses: actions/download-artifact@v3
       with:
@@ -12,18 +19,18 @@ runs:
         path: mpy-cross
 
     - name: Make mpy-cross executable
-      if: steps.download-mpy-cross.outcome == 'success'
+      if: inputs.download == 'true' && steps.download-mpy-cross.outcome == 'success'
       run: sudo chmod +x mpy-cross/mpy-cross
       shell: bash
 
     - name: Build mpy-cross
-      if: steps.download-mpy-cross.outcome == 'failure'
+      if: inputs.download == 'false' || steps.download-mpy-cross.outcome == 'failure'
       run: make -C mpy-cross -j2
       shell: bash
 
     - name: Upload mpy-cross
+      if: inputs.download == 'false' || steps.download-mpy-cross.outcome == 'failure'
       continue-on-error: true
-      if: steps.download-mpy-cross.outcome == 'failure'
       uses: actions/upload-artifact@v3
       with:
         name: mpy-cross
diff --git a/.github/workflows/build-boards.yml b/.github/workflows/build-boards.yml
index 9b53615fe7..c302be7c9f 100644
--- a/.github/workflows/build-boards.yml
+++ b/.github/workflows/build-boards.yml
@@ -35,14 +35,14 @@ jobs:
       with:
         python-version: 3.x
     - name: Set up port
-      uses: ./.github/actions/port_deps
+      uses: ./.github/actions/deps/port
       with:
         platform: ${{ inputs.platform }}
     - name: Set up submodules
       id: set-up-submodules
-      uses: ./.github/actions/fetch_submodules
+      uses: ./.github/actions/deps/submodules
     - name: Set up external
-      uses: ./.github/actions/external_deps
+      uses: ./.github/actions/deps/external
       with:
         platform: ${{ inputs.platform }}
     - name: Set up mpy-cross
@@ -64,7 +64,7 @@ jobs:
 
     - name: Set up build failure matcher
       run: echo "::add-matcher::$GITHUB_WORKSPACE/.github/workflows/match-build-fail.json"
-    - name: Build
+    - name: Build board
       run: python3 -u build_release_files.py
       working-directory: tools
       env:
diff --git a/.github/workflows/build-mpy-cross.yml b/.github/workflows/build-mpy-cross.yml
index 3d557df8ee..a2c0c64289 100644
--- a/.github/workflows/build-mpy-cross.yml
+++ b/.github/workflows/build-mpy-cross.yml
@@ -27,26 +27,22 @@ jobs:
       with:
         submodules: false
         fetch-depth: 1
-
     - name: Set up python
       uses: actions/setup-python@v4
       with:
         python-version: 3.x
-
     - name: Set up submodules
-      uses: ./.github/actions/fetch_submodules
+      uses: ./.github/actions/deps/submodules
       with:
         target: mpy-cross
-
-    - name: Install dependencies
-      run: |
-        sudo apt-get update
-        sudo apt-get install -y gettext
+    - name: Set up external
+      uses: ./.github/actions/deps/external
+      with:
+        python: false
 
     - name: Install toolchain (aarch64)
       if: matrix.mpy-cross == 'static-aarch64'
       run: sudo apt-get install -y gcc-aarch64-linux-gnu
-
     - name: Install toolchain (mingw)
       if: matrix.mpy-cross == 'static-mingw'
       run: sudo apt-get install -y mingw-w64
@@ -54,16 +50,20 @@ jobs:
     - name: Build mpy-cross.${{ matrix.mpy-cross }}
       run: make -C mpy-cross -j2 -f Makefile.${{ matrix.mpy-cross }}
 
+    - name: Set output
+      run: |
+        echo >> $GITHUB_ENV "EX=${{ env[format('EX_{0}', matrix.mpy-cross)] || matrix.mpy-cross }}"
+        echo >> $GITHUB_ENV "OS=${{ env[format('OS_{0}', matrix.mpy-cross)] }}"
+
     - name: Upload artifact
       uses: actions/upload-artifact@v3
       with:
-        name: mpy-cross.${{ env[format('EX_{0}', matrix.mpy-cross)] || matrix.mpy-cross }}
-        path: mpy-cross/mpy-cross.${{ env[format('EX_{0}', matrix.mpy-cross)] || matrix.mpy-cross }}
-
+        name: mpy-cross.${{ env.EX }}
+        path: mpy-cross/mpy-cross.${{ env.EX }}
     - name: Upload to S3
       uses: ./.github/actions/upload_aws
       with:
-        source: mpy-cross/mpy-cross.${{ env[format('EX_{0}', matrix.mpy-cross)] || matrix.mpy-cross }}
-        destination: mpy-cross/${{ env[format('OS_{0}', matrix.mpy-cross)] }}/mpy-cross-${{ env.CP_VERSION }}.${{ env[format('EX_{0}', matrix.mpy-cross)] || matrix.mpy-cross }}
+        source: mpy-cross/mpy-cross.${{ env.EX }}
+        destination: mpy-cross/${{ env.OS }}/mpy-cross-${{ env.OS }}-${{ env.CP_VERSION }}.${{ env.EX }}
         AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
         AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 37bb4133cd..8c40acd6d3 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -27,9 +27,6 @@ jobs:
       boards-espressif: ${{ steps.set-matrix.outputs.boards-espressif }}
       boards-riscv: ${{ steps.set-matrix.outputs.boards-riscv }}
       cp-version: ${{ steps.set-up-submodules.outputs.version }}
-    env:
-      MICROPY_CPYTHON3: python3.8
-      MICROPY_MICROPYTHON: ${{ github.workspace }}/ports/unix/micropython-coverage
     steps:
     - name: Dump GitHub context
       run: echo "$GITHUB_CONTEXT"
@@ -48,14 +45,12 @@ jobs:
       run: python3 -u -m tools.ci_check_duplicate_usb_vid_pid
     - name: Set up submodules
       id: set-up-submodules
-      uses: ./.github/actions/fetch_submodules
+      uses: ./.github/actions/deps/submodules
       with:
-        cache: cache
+        action: cache
         version: true
-    - name: Install dependencies
-      run: |
-        sudo apt-get install -y gettext
-        pip install requests requests-cache sh
+    - name: Set up external
+      uses: ./.github/actions/deps/external
     # Disabled: Needs to be updated
     # - name: Get last commit with checks
     #   id: get-last-commit-with-checks
@@ -69,6 +64,8 @@ jobs:
     #     EXCLUDE_COMMIT: ${{ github.event.after }}
     - name: Set up mpy-cross
       uses: ./.github/actions/mpy_cross
+      with:
+        download: false
     - name: Set head sha
       if: github.event_name == 'pull_request'
       run: echo "HEAD_SHA=$(git show -s --format=%s $GITHUB_SHA | grep -o -P "(?<=Merge ).*(?= into)")" >> $GITHUB_ENV
@@ -124,7 +121,7 @@ jobs:
       with:
         python-version: 3.x
     - name: Set up submodules
-      uses: ./.github/actions/fetch_submodules
+      uses: ./.github/actions/deps/submodules
     - name: Versions
       run: |
         gcc --version
@@ -179,7 +176,7 @@ jobs:
       with:
         python-version: 3.x
     - name: Set up submodules
-      uses: ./.github/actions/fetch_submodules
+      uses: ./.github/actions/deps/submodules
     - name: Install dependencies
       run: |
         sudo apt-get update
diff --git a/.github/workflows/create_website_pr.yml b/.github/workflows/create_website_pr.yml
index 19004d9bea..9f89490161 100644
--- a/.github/workflows/create_website_pr.yml
+++ b/.github/workflows/create_website_pr.yml
@@ -24,13 +24,15 @@ jobs:
     - name: Set up python
       uses: actions/setup-python@v4
       with:
-        python-version: "3.x"
+        python-version: 3.x
     - name: Set up submodules
-      uses: ./.github/actions/fetch_submodules
+      uses: ./.github/actions/deps/submodules
       with:
         version: true
-    - name: Install dependencies
-      run: pip install -r requirements-dev.txt
+    - name: Set up external
+      uses: ./.github/actions/deps/external
+      with:
+        apt: false
     - name: Versions
       run: |
         gcc --version
diff --git a/.github/workflows/ports_windows.yml b/.github/workflows/ports_windows.yml
index 66d09c2418..ba2042660a 100644
--- a/.github/workflows/ports_windows.yml
+++ b/.github/workflows/ports_windows.yml
@@ -77,7 +77,7 @@ jobs:
         fetch-depth: 1
 
     - name: Set up submodules
-      uses: ./.github/actions/fetch_submodules
+      uses: ./.github/actions/deps/submodules
       with:
         version: true
 
diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml
index 02aed6e9b4..fc36f2a505 100644
--- a/.github/workflows/pre-commit.yml
+++ b/.github/workflows/pre-commit.yml
@@ -24,13 +24,13 @@ jobs:
     - name: Set up python
       uses: actions/setup-python@v4
       with:
-        python-version: "3.x"
-    - name: CircuitPython dependencies
-      run: python tools/ci_fetch_deps.py ${{ github.job }}
+        python-version: 3.x
+    - name: Set up submodules
+      uses: ./.github/actions/deps/submodules
+    - name: Set up external
+      uses: ./.github/actions/deps/external
     - name: Install dependencies
-      run: |
-        sudo apt-get install -y gettext uncrustify
-        pip3 install black polib pyyaml
+      run: sudo apt-get install -y uncrustify
     - name: Run pre-commit
       uses: pre-commit/action@v3.0.0
     - name: Make patch
diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index 15fc8489a4..1ab0f6eeab 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -33,32 +33,22 @@ jobs:
       with:
         python-version: 3.x
     - name: Set up submodules
-      uses: ./.github/actions/fetch_submodules
+      uses: ./.github/actions/deps/submodules
       with:
         target: tests
-
-    - name: Install dependencies
-      run: |
-        sudo apt-get update
-        sudo apt-get install -y gettext
-
+    - name: Set up external
+      uses: ./.github/actions/deps/external
     - name: Set up mpy-cross
       uses: ./.github/actions/mpy_cross
-
     - name: Build unix port
       run: make -C ports/unix VARIANT=coverage -j2
-
-    - name: Test all
-      run: ./run-tests.py -j2 $TEST_${{ matrix.test }}
+    - name: Run tests
+      run: ./run-tests.py -j2 ${{ env[format('TEST_{0}', matrix.test)] }}
       working-directory: tests
     - name: Print failure info
       run: ./run-tests.py -j2 --print-failures
       if: failure()
       working-directory: tests
-
-    - name: Set up native modules
-      if: matrix.test == 'all'
-      run: pip install -r requirements-dev.txt
     - name: Build native modules
       if: matrix.test == 'all'
       run: |

From a2bbca1428efc334243e1ac474244d667d5854bd Mon Sep 17 00:00:00 2001
From: RetiredWizard <github@retiredwizard.com>
Date: Thu, 9 Feb 2023 01:14:01 -0500
Subject: [PATCH 15/35] Broadcom Raspberry Pi Zero2W neopixel timing fix

These changes result in working neopixel functionality. I've tested on
both the zero2w and the pi4b (The 4b didn't exhibit the original issue)
and the boards now behave properly with 1 to 30 pixels and the board hanging
no longer occurs.

Remove mod that didn't help during testing
Restoring back to original structure
Replace 2 microsecond delay w/deterministic loop
Remove unneded check for empty queue

Put transmit delay outside loop so Queue is used

Make sure last transmission is complete
---
 .../common-hal/neopixel_write/__init__.c      | 40 +++++++++++++++----
 1 file changed, 32 insertions(+), 8 deletions(-)

diff --git a/ports/broadcom/common-hal/neopixel_write/__init__.c b/ports/broadcom/common-hal/neopixel_write/__init__.c
index 0cd76ebca9..822c61e36e 100644
--- a/ports/broadcom/common-hal/neopixel_write/__init__.c
+++ b/ports/broadcom/common-hal/neopixel_write/__init__.c
@@ -45,7 +45,11 @@ void common_hal_neopixel_write(const digitalio_digitalinout_obj_t *digitalinout,
     uint8_t *pixels, uint32_t num_bytes) {
     // Wait to make sure we don't append onto the last transmission. This should only be a tick or
     // two.
-    while (port_get_raw_ticks(NULL) < next_start_raw_ticks) {
+    int icnt;
+    while ((port_get_raw_ticks(NULL) < next_start_raw_ticks) & 
+        (next_start_raw_ticks-port_get_raw_ticks(NULL) < 100)) {
+        
+        RUN_BACKGROUND_TASKS;
     }
 
     BP_Function_Enum alt_function = GPIO_FUNCTION_OUTPUT;
@@ -92,7 +96,8 @@ void common_hal_neopixel_write(const digitalio_digitalinout_obj_t *digitalinout,
 
         // Wait for the clock to start up.
         COMPLETE_MEMORY_READS;
-        while (CM_PWM->CS_b.BUSY == 0) {
+        icnt = 0;
+        while ((CM_PWM->CS_b.BUSY == 0) & (icnt++ < 1000)) {
         }
     }
 
@@ -134,22 +139,41 @@ void common_hal_neopixel_write(const digitalio_digitalinout_obj_t *digitalinout,
                 expanded |= 0x80000000;
             }
         }
-        while (pwm->STA_b.FULL1 == 1) {
-            RUN_BACKGROUND_TASKS;
-        }
         if (channel == 1) {
+            icnt=0;
+            while ((pwm->STA_b.FULL1 == 1) & (icnt++ < 150)) {
+                RUN_BACKGROUND_TASKS;
+            }
             // Dummy value for the first channel.
             pwm->FIF1 = 0x000000;
         }
+        icnt=0;
+        while ((pwm->STA_b.FULL1 == 1) & (icnt++ < 150)) {
+            RUN_BACKGROUND_TASKS;
+        }
         pwm->FIF1 = expanded;
         if (channel == 0) {
+            icnt=0;
+            while ((pwm->STA_b.FULL1 == 1) & (icnt++ < 150)) {
+                RUN_BACKGROUND_TASKS;
+            }
             // Dummy value for the second channel.
             pwm->FIF1 = 0x000000;
         }
     }
-    // Wait just a little bit so that transmission can start.
-    common_hal_mcu_delay_us(2);
-    while (pwm->STA_b.STA1 == 1) {
+
+    icnt = 0;
+    while ((pwm->STA_b.EMPT1 == 0) & (icnt++ < 2500)) {
+        RUN_BACKGROUND_TASKS;
+    }
+    // Wait for transmission to start.
+    icnt = 0;
+    while (((pwm->STA_b.STA1 ==0) & (pwm->STA_b.STA2 == 0)) & (icnt++ < 150)) {
+        RUN_BACKGROUND_TASKS;
+    }
+    // Wait for transmission to complete.
+    icnt = 0;
+    while (((pwm->STA_b.STA1 == 1) | (pwm->STA_b.STA2 == 1)) & (icnt++ < 150)) {
         RUN_BACKGROUND_TASKS;
     }
 

From a462a316bddb3860c3bc5e352084a03d31d08650 Mon Sep 17 00:00:00 2001
From: RetiredWizard <github@retiredwizard.com>
Date: Sat, 11 Feb 2023 12:10:03 -0500
Subject: [PATCH 16/35] Fix pre-commit formatting

---
 ports/broadcom/common-hal/neopixel_write/__init__.c | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/ports/broadcom/common-hal/neopixel_write/__init__.c b/ports/broadcom/common-hal/neopixel_write/__init__.c
index 822c61e36e..60b288eb28 100644
--- a/ports/broadcom/common-hal/neopixel_write/__init__.c
+++ b/ports/broadcom/common-hal/neopixel_write/__init__.c
@@ -47,7 +47,7 @@ void common_hal_neopixel_write(const digitalio_digitalinout_obj_t *digitalinout,
     // two.
     int icnt;
     while ((port_get_raw_ticks(NULL) < next_start_raw_ticks) & 
-        (next_start_raw_ticks-port_get_raw_ticks(NULL) < 100)) {
+           (next_start_raw_ticks - port_get_raw_ticks(NULL) < 100)) {
         
         RUN_BACKGROUND_TASKS;
     }
@@ -140,20 +140,20 @@ void common_hal_neopixel_write(const digitalio_digitalinout_obj_t *digitalinout,
             }
         }
         if (channel == 1) {
-            icnt=0;
+            icnt = 0;
             while ((pwm->STA_b.FULL1 == 1) & (icnt++ < 150)) {
                 RUN_BACKGROUND_TASKS;
             }
             // Dummy value for the first channel.
             pwm->FIF1 = 0x000000;
         }
-        icnt=0;
+        icnt = 0;
         while ((pwm->STA_b.FULL1 == 1) & (icnt++ < 150)) {
             RUN_BACKGROUND_TASKS;
         }
         pwm->FIF1 = expanded;
         if (channel == 0) {
-            icnt=0;
+            icnt = 0;
             while ((pwm->STA_b.FULL1 == 1) & (icnt++ < 150)) {
                 RUN_BACKGROUND_TASKS;
             }
@@ -168,7 +168,7 @@ void common_hal_neopixel_write(const digitalio_digitalinout_obj_t *digitalinout,
     }
     // Wait for transmission to start.
     icnt = 0;
-    while (((pwm->STA_b.STA1 ==0) & (pwm->STA_b.STA2 == 0)) & (icnt++ < 150)) {
+    while (((pwm->STA_b.STA1 == 0) & (pwm->STA_b.STA2 == 0)) & (icnt++ < 150)) {
         RUN_BACKGROUND_TASKS;
     }
     // Wait for transmission to complete.

From f29cd4a836eb97cb589818932b55b002468e964e Mon Sep 17 00:00:00 2001
From: RetiredWizard <github@retiredwizard.com>
Date: Sat, 11 Feb 2023 12:13:34 -0500
Subject: [PATCH 17/35] Pre-commit fix trim trailing whitespace

---
 ports/broadcom/common-hal/neopixel_write/__init__.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ports/broadcom/common-hal/neopixel_write/__init__.c b/ports/broadcom/common-hal/neopixel_write/__init__.c
index 60b288eb28..023feb995f 100644
--- a/ports/broadcom/common-hal/neopixel_write/__init__.c
+++ b/ports/broadcom/common-hal/neopixel_write/__init__.c
@@ -46,7 +46,7 @@ void common_hal_neopixel_write(const digitalio_digitalinout_obj_t *digitalinout,
     // Wait to make sure we don't append onto the last transmission. This should only be a tick or
     // two.
     int icnt;
-    while ((port_get_raw_ticks(NULL) < next_start_raw_ticks) & 
+    while ((port_get_raw_ticks(NULL) < next_start_raw_ticks) &
            (next_start_raw_ticks - port_get_raw_ticks(NULL) < 100)) {
         
         RUN_BACKGROUND_TASKS;

From 2104708c58c881a7ecf0b972245cc3ffcb48901d Mon Sep 17 00:00:00 2001
From: RetiredWizard <github@retiredwizard.com>
Date: Sat, 11 Feb 2023 12:17:55 -0500
Subject: [PATCH 18/35] missed a trailing space

---
 ports/broadcom/common-hal/neopixel_write/__init__.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ports/broadcom/common-hal/neopixel_write/__init__.c b/ports/broadcom/common-hal/neopixel_write/__init__.c
index 023feb995f..38519a0232 100644
--- a/ports/broadcom/common-hal/neopixel_write/__init__.c
+++ b/ports/broadcom/common-hal/neopixel_write/__init__.c
@@ -48,7 +48,7 @@ void common_hal_neopixel_write(const digitalio_digitalinout_obj_t *digitalinout,
     int icnt;
     while ((port_get_raw_ticks(NULL) < next_start_raw_ticks) &
            (next_start_raw_ticks - port_get_raw_ticks(NULL) < 100)) {
-        
+
         RUN_BACKGROUND_TASKS;
     }
 

From 359a27e166617260d75dc6ef6769114ebac03d19 Mon Sep 17 00:00:00 2001
From: RetiredWizard <github@retiredwizard.com>
Date: Sat, 11 Feb 2023 12:19:21 -0500
Subject: [PATCH 19/35] Pre-commit is stubborn

---
 ports/broadcom/common-hal/neopixel_write/__init__.c | 1 -
 1 file changed, 1 deletion(-)

diff --git a/ports/broadcom/common-hal/neopixel_write/__init__.c b/ports/broadcom/common-hal/neopixel_write/__init__.c
index 38519a0232..5629c97425 100644
--- a/ports/broadcom/common-hal/neopixel_write/__init__.c
+++ b/ports/broadcom/common-hal/neopixel_write/__init__.c
@@ -48,7 +48,6 @@ void common_hal_neopixel_write(const digitalio_digitalinout_obj_t *digitalinout,
     int icnt;
     while ((port_get_raw_ticks(NULL) < next_start_raw_ticks) &
            (next_start_raw_ticks - port_get_raw_ticks(NULL) < 100)) {
-
         RUN_BACKGROUND_TASKS;
     }
 

From 1679790481a80270e6d79bb7fe069cd71fe46488 Mon Sep 17 00:00:00 2001
From: RetiredWizard <github@retiredwizard.com>
Date: Sat, 11 Feb 2023 20:21:25 -0500
Subject: [PATCH 20/35] Tweaked to run without delays on zero w

---
 ports/broadcom/common-hal/neopixel_write/__init__.c | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/ports/broadcom/common-hal/neopixel_write/__init__.c b/ports/broadcom/common-hal/neopixel_write/__init__.c
index 5629c97425..d685ac53ce 100644
--- a/ports/broadcom/common-hal/neopixel_write/__init__.c
+++ b/ports/broadcom/common-hal/neopixel_write/__init__.c
@@ -97,6 +97,7 @@ void common_hal_neopixel_write(const digitalio_digitalinout_obj_t *digitalinout,
         COMPLETE_MEMORY_READS;
         icnt = 0;
         while ((CM_PWM->CS_b.BUSY == 0) & (icnt++ < 1000)) {
+            COMPLETE_MEMORY_READS;
         }
     }
 
@@ -142,6 +143,7 @@ void common_hal_neopixel_write(const digitalio_digitalinout_obj_t *digitalinout,
             icnt = 0;
             while ((pwm->STA_b.FULL1 == 1) & (icnt++ < 150)) {
                 RUN_BACKGROUND_TASKS;
+                COMPLETE_MEMORY_READS;
             }
             // Dummy value for the first channel.
             pwm->FIF1 = 0x000000;
@@ -149,12 +151,14 @@ void common_hal_neopixel_write(const digitalio_digitalinout_obj_t *digitalinout,
         icnt = 0;
         while ((pwm->STA_b.FULL1 == 1) & (icnt++ < 150)) {
             RUN_BACKGROUND_TASKS;
+            COMPLETE_MEMORY_READS;
         }
         pwm->FIF1 = expanded;
         if (channel == 0) {
             icnt = 0;
             while ((pwm->STA_b.FULL1 == 1) & (icnt++ < 150)) {
                 RUN_BACKGROUND_TASKS;
+                COMPLETE_MEMORY_READS;
             }
             // Dummy value for the second channel.
             pwm->FIF1 = 0x000000;
@@ -164,17 +168,23 @@ void common_hal_neopixel_write(const digitalio_digitalinout_obj_t *digitalinout,
     icnt = 0;
     while ((pwm->STA_b.EMPT1 == 0) & (icnt++ < 2500)) {
         RUN_BACKGROUND_TASKS;
+        COMPLETE_MEMORY_READS;
     }
     // Wait for transmission to start.
     icnt = 0;
     while (((pwm->STA_b.STA1 == 0) & (pwm->STA_b.STA2 == 0)) & (icnt++ < 150)) {
         RUN_BACKGROUND_TASKS;
+        COMPLETE_MEMORY_READS;
     }
     // Wait for transmission to complete.
     icnt = 0;
     while (((pwm->STA_b.STA1 == 1) | (pwm->STA_b.STA2 == 1)) & (icnt++ < 150)) {
         RUN_BACKGROUND_TASKS;
+        COMPLETE_MEMORY_READS;
     }
+    // Shouldn't be anything left in queue but clear it so the clock doesn't crash if there is
+    pwm->CTL = PWM0_CTL_CLRF1_Msk;
+    COMPLETE_MEMORY_READS;
 
     gpio_set_function(digitalinout->pin->number, GPIO_FUNCTION_OUTPUT);
 

From 65e9d937ac2e0df60d35414274ea5f4a30412147 Mon Sep 17 00:00:00 2001
From: MicroDev <70126934+MicroDev1@users.noreply.github.com>
Date: Mon, 13 Feb 2023 10:42:24 +0530
Subject: [PATCH 21/35] use python based msgfmt

---
 .github/actions/deps/external/action.yml |  17 +-
 .github/actions/deps/python/action.yml   |  13 +-
 .github/workflows/build-mpy-cross.yml    |   4 -
 .github/workflows/build.yml              |   4 +-
 .github/workflows/create_website_pr.yml  |   2 -
 .github/workflows/pre-commit.yml         |   2 +-
 .github/workflows/run-tests.yml          |   1 +
 py/py.mk                                 |   2 +-
 tools/msgfmt.py                          | 244 +++++++++++++++++++++++
 9 files changed, 259 insertions(+), 30 deletions(-)
 create mode 100644 tools/msgfmt.py

diff --git a/.github/actions/deps/external/action.yml b/.github/actions/deps/external/action.yml
index 408a2e8555..9f5c3bd7c9 100644
--- a/.github/actions/deps/external/action.yml
+++ b/.github/actions/deps/external/action.yml
@@ -12,16 +12,6 @@ inputs:
     - riscv
     - none
 
-  apt:
-    required: false
-    default: true
-    type: boolean
-
-  python:
-    required: false
-    default: true
-    type: boolean
-
 runs:
   using: composite
   steps:
@@ -85,15 +75,10 @@ runs:
 
     # common
     - name: Cache python dependencies
-      if: inputs.python == 'true' && inputs.platform != 'espressif'
+      if: inputs.platform != 'espressif'
       uses: ./.github/actions/deps/python
       with:
         action: ${{ fromJSON('["restore", "cache"]')[github.job == 'scheduler'] }}
     - name: Install python dependencies
-      if: inputs.python == 'true'
       run: pip install -r requirements-dev.txt
       shell: bash
-    - name: Install dependencies
-      if: inputs.apt == 'true'
-      run: sudo apt-get install -y gettext
-      shell: bash
diff --git a/.github/actions/deps/python/action.yml b/.github/actions/deps/python/action.yml
index fb978ef063..9b3732c9e0 100644
--- a/.github/actions/deps/python/action.yml
+++ b/.github/actions/deps/python/action.yml
@@ -14,22 +14,29 @@ runs:
   using: composite
   steps:
     - name: Cache python dependencies
-      if: ${{ inputs.action == 'cache' }}
+      id: cache-python-deps
+      if: inputs.action == 'cache'
       uses: actions/cache@v3
       with:
         path: .cp_tools
         key: ${{ runner.os }}-${{ env.pythonLocation }}-tools-cp-${{ hashFiles('requirements-dev.txt') }}
 
     - name: Restore python dependencies
-      if: ${{ inputs.action == 'restore' }}
+      id: restore-python-deps
+      if: inputs.action == 'restore'
       uses: actions/cache/restore@v3
       with:
         path: .cp_tools
         key: ${{ runner.os }}-${{ env.pythonLocation }}-tools-cp-${{ hashFiles('requirements-dev.txt') }}
 
     - name: Set up venv
+      if: inputs.action == 'cache' && !steps.cache-python-deps.outputs.cache-hit
+      run: python -m venv .cp_tools
+      shell: bash
+
+    - name: Activate venv
+      if: inputs.action == 'cache' || (inputs.action == 'restore' && steps.restore-python-deps.outputs.cache-hit)
       run: |
-        python -m venv .cp_tools
         source .cp_tools/bin/activate
         echo >> $GITHUB_PATH "$PATH"
       shell: bash
diff --git a/.github/workflows/build-mpy-cross.yml b/.github/workflows/build-mpy-cross.yml
index a2c0c64289..c17c9da1cb 100644
--- a/.github/workflows/build-mpy-cross.yml
+++ b/.github/workflows/build-mpy-cross.yml
@@ -35,10 +35,6 @@ jobs:
       uses: ./.github/actions/deps/submodules
       with:
         target: mpy-cross
-    - name: Set up external
-      uses: ./.github/actions/deps/external
-      with:
-        python: false
 
     - name: Install toolchain (aarch64)
       if: matrix.mpy-cross == 'static-aarch64'
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 8c40acd6d3..cdf9462289 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -179,9 +179,7 @@ jobs:
       uses: ./.github/actions/deps/submodules
     - name: Install dependencies
       run: |
-        sudo apt-get update
-        sudo apt-get install -y eatmydata
-        sudo eatmydata apt-get install -y latexmk librsvg2-bin texlive-fonts-recommended texlive-latex-recommended texlive-latex-extra
+        sudo apt-get install -y latexmk librsvg2-bin texlive-fonts-recommended texlive-latex-recommended texlive-latex-extra
         pip install -r requirements-doc.txt
     - name: Build and Validate Stubs
       run: make check-stubs -j2
diff --git a/.github/workflows/create_website_pr.yml b/.github/workflows/create_website_pr.yml
index 9f89490161..eeae8c8299 100644
--- a/.github/workflows/create_website_pr.yml
+++ b/.github/workflows/create_website_pr.yml
@@ -31,8 +31,6 @@ jobs:
         version: true
     - name: Set up external
       uses: ./.github/actions/deps/external
-      with:
-        apt: false
     - name: Versions
       run: |
         gcc --version
diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml
index fc36f2a505..c0dff33564 100644
--- a/.github/workflows/pre-commit.yml
+++ b/.github/workflows/pre-commit.yml
@@ -30,7 +30,7 @@ jobs:
     - name: Set up external
       uses: ./.github/actions/deps/external
     - name: Install dependencies
-      run: sudo apt-get install -y uncrustify
+      run: sudo apt-get install -y gettext uncrustify
     - name: Run pre-commit
       uses: pre-commit/action@v3.0.0
     - name: Make patch
diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index 1ab0f6eeab..12e8d2ef74 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -37,6 +37,7 @@ jobs:
       with:
         target: tests
     - name: Set up external
+      if: matrix.test == 'all'
       uses: ./.github/actions/deps/external
     - name: Set up mpy-cross
       uses: ./.github/actions/mpy_cross
diff --git a/py/py.mk b/py/py.mk
index 0cbfd252f2..c598f44cbd 100644
--- a/py/py.mk
+++ b/py/py.mk
@@ -243,7 +243,7 @@ $(HEADER_BUILD)/mpversion.h: FORCE | $(HEADER_BUILD)
 MPCONFIGPORT_MK = $(wildcard mpconfigport.mk)
 
 $(HEADER_BUILD)/$(TRANSLATION).mo: $(TOP)/locale/$(TRANSLATION).po | $(HEADER_BUILD)
-	$(Q)msgfmt -o $@ $^
+	$(Q)$(PYTHON) $(TOP)/tools/msgfmt.py -o $@ $^
 
 $(HEADER_BUILD)/qstrdefs.preprocessed.h: $(PY_QSTR_DEFS) $(QSTR_DEFS) $(QSTR_DEFS_COLLECTED) mpconfigport.h $(MPCONFIGPORT_MK) $(PY_SRC)/mpconfig.h | $(HEADER_BUILD)
 	$(STEPECHO) "GEN $@"
diff --git a/tools/msgfmt.py b/tools/msgfmt.py
new file mode 100644
index 0000000000..1ed594b84f
--- /dev/null
+++ b/tools/msgfmt.py
@@ -0,0 +1,244 @@
+#! /usr/bin/env python3
+# Written by Martin v. Löwis <loewis@informatik.hu-berlin.de>
+
+"""Generate binary message catalog from textual translation description.
+This program converts a textual Uniforum-style message catalog (.po file) into
+a binary GNU catalog (.mo file).  This is essentially the same function as the
+GNU msgfmt program, however, it is a simpler implementation.  Currently it
+does not handle plural forms but it does handle message contexts.
+Usage: msgfmt.py [OPTIONS] filename.po
+Options:
+    -o file
+    --output-file=file
+        Specify the output file to write to.  If omitted, output will go to a
+        file named filename.mo (based off the input file name).
+    -h
+    --help
+        Print this message and exit.
+    -V
+    --version
+        Display version information and exit.
+"""
+
+import os
+import sys
+import ast
+import getopt
+import struct
+import array
+from email.parser import HeaderParser
+
+__version__ = "1.2"
+
+MESSAGES = {}
+
+
+def usage(code, msg=""):
+    print(__doc__, file=sys.stderr)
+    if msg:
+        print(msg, file=sys.stderr)
+    sys.exit(code)
+
+
+def add(ctxt, id, str, fuzzy):
+    "Add a non-fuzzy translation to the dictionary."
+    global MESSAGES
+    if not fuzzy and str:
+        if ctxt is None:
+            MESSAGES[id] = str
+        else:
+            MESSAGES[b"%b\x04%b" % (ctxt, id)] = str
+
+
+def generate():
+    "Return the generated output."
+    global MESSAGES
+    # the keys are sorted in the .mo file
+    keys = sorted(MESSAGES.keys())
+    offsets = []
+    ids = strs = b""
+    for id in keys:
+        # For each string, we need size and file offset.  Each string is NUL
+        # terminated; the NUL does not count into the size.
+        offsets.append((len(ids), len(id), len(strs), len(MESSAGES[id])))
+        ids += id + b"\0"
+        strs += MESSAGES[id] + b"\0"
+    output = ""
+    # The header is 7 32-bit unsigned integers.  We don't use hash tables, so
+    # the keys start right after the index tables.
+    # translated string.
+    keystart = 7 * 4 + 16 * len(keys)
+    # and the values start after the keys
+    valuestart = keystart + len(ids)
+    koffsets = []
+    voffsets = []
+    # The string table first has the list of keys, then the list of values.
+    # Each entry has first the size of the string, then the file offset.
+    for o1, l1, o2, l2 in offsets:
+        koffsets += [l1, o1 + keystart]
+        voffsets += [l2, o2 + valuestart]
+    offsets = koffsets + voffsets
+    output = struct.pack(
+        "Iiiiiii",
+        0x950412DE,  # Magic
+        0,  # Version
+        len(keys),  # # of entries
+        7 * 4,  # start of key index
+        7 * 4 + len(keys) * 8,  # start of value index
+        0,
+        0,
+    )  # size and offset of hash table
+    output += array.array("i", offsets).tobytes()
+    output += ids
+    output += strs
+    return output
+
+
+def make(filename, outfile):
+    ID = 1
+    STR = 2
+    CTXT = 3
+
+    # Compute .mo name from .po name and arguments
+    if filename.endswith(".po"):
+        infile = filename
+    else:
+        infile = filename + ".po"
+    if outfile is None:
+        outfile = os.path.splitext(infile)[0] + ".mo"
+
+    try:
+        with open(infile, "rb") as f:
+            lines = f.readlines()
+    except IOError as msg:
+        print(msg, file=sys.stderr)
+        sys.exit(1)
+
+    section = msgctxt = None
+    fuzzy = 0
+
+    # Start off assuming Latin-1, so everything decodes without failure,
+    # until we know the exact encoding
+    encoding = "latin-1"
+
+    # Parse the catalog
+    lno = 0
+    for l in lines:
+        l = l.decode(encoding)
+        lno += 1
+        # If we get a comment line after a msgstr, this is a new entry
+        if l[0] == "#" and section == STR:
+            add(msgctxt, msgid, msgstr, fuzzy)
+            section = msgctxt = None
+            fuzzy = 0
+        # Record a fuzzy mark
+        if l[:2] == "#," and "fuzzy" in l:
+            fuzzy = 1
+        # Skip comments
+        if l[0] == "#":
+            continue
+        # Now we are in a msgid or msgctxt section, output previous section
+        if l.startswith("msgctxt"):
+            if section == STR:
+                add(msgctxt, msgid, msgstr, fuzzy)
+            section = CTXT
+            l = l[7:]
+            msgctxt = b""
+        elif l.startswith("msgid") and not l.startswith("msgid_plural"):
+            if section == STR:
+                add(msgctxt, msgid, msgstr, fuzzy)
+                if not msgid:
+                    # See whether there is an encoding declaration
+                    p = HeaderParser()
+                    charset = p.parsestr(msgstr.decode(encoding)).get_content_charset()
+                    if charset:
+                        encoding = charset
+            section = ID
+            l = l[5:]
+            msgid = msgstr = b""
+            is_plural = False
+        # This is a message with plural forms
+        elif l.startswith("msgid_plural"):
+            if section != ID:
+                print(
+                    "msgid_plural not preceded by msgid on %s:%d" % (infile, lno), file=sys.stderr
+                )
+                sys.exit(1)
+            l = l[12:]
+            msgid += b"\0"  # separator of singular and plural
+            is_plural = True
+        # Now we are in a msgstr section
+        elif l.startswith("msgstr"):
+            section = STR
+            if l.startswith("msgstr["):
+                if not is_plural:
+                    print("plural without msgid_plural on %s:%d" % (infile, lno), file=sys.stderr)
+                    sys.exit(1)
+                l = l.split("]", 1)[1]
+                if msgstr:
+                    msgstr += b"\0"  # Separator of the various plural forms
+            else:
+                if is_plural:
+                    print(
+                        "indexed msgstr required for plural on  %s:%d" % (infile, lno),
+                        file=sys.stderr,
+                    )
+                    sys.exit(1)
+                l = l[6:]
+        # Skip empty lines
+        l = l.strip()
+        if not l:
+            continue
+        l = ast.literal_eval(l)
+        if section == CTXT:
+            msgctxt += l.encode(encoding)
+        elif section == ID:
+            msgid += l.encode(encoding)
+        elif section == STR:
+            msgstr += l.encode(encoding)
+        else:
+            print("Syntax error on %s:%d" % (infile, lno), "before:", file=sys.stderr)
+            print(l, file=sys.stderr)
+            sys.exit(1)
+    # Add last entry
+    if section == STR:
+        add(msgctxt, msgid, msgstr, fuzzy)
+
+    # Compute output
+    output = generate()
+
+    try:
+        with open(outfile, "wb") as f:
+            f.write(output)
+    except IOError as msg:
+        print(msg, file=sys.stderr)
+
+
+def main():
+    try:
+        opts, args = getopt.getopt(sys.argv[1:], "hVo:", ["help", "version", "output-file="])
+    except getopt.error as msg:
+        usage(1, msg)
+
+    outfile = None
+    # parse options
+    for opt, arg in opts:
+        if opt in ("-h", "--help"):
+            usage(0)
+        elif opt in ("-V", "--version"):
+            print("msgfmt.py", __version__)
+            sys.exit(0)
+        elif opt in ("-o", "--output-file"):
+            outfile = arg
+    # do it
+    if not args:
+        print("No input file given", file=sys.stderr)
+        print("Try `msgfmt --help' for more information.", file=sys.stderr)
+        return
+
+    for filename in args:
+        make(filename, outfile)
+
+
+if __name__ == "__main__":
+    main()

From 676cbb308db2b8a8da758ecab7f6f4f8ea0fcfa6 Mon Sep 17 00:00:00 2001
From: Pierre Constantineau <jpconstantineau@gmail.com>
Date: Mon, 13 Feb 2023 09:52:18 -0600
Subject: [PATCH 22/35] minor update to boards.c to fix mismatched number of
 LEDs causing LEDs not to be reset

---
 ports/raspberrypi/boards/jpconstantineau_pykey18/board.c | 2 +-
 ports/raspberrypi/boards/jpconstantineau_pykey44/board.c | 2 +-
 ports/raspberrypi/boards/jpconstantineau_pykey87/board.c | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/ports/raspberrypi/boards/jpconstantineau_pykey18/board.c b/ports/raspberrypi/boards/jpconstantineau_pykey18/board.c
index 4785f742ec..5b1190727f 100644
--- a/ports/raspberrypi/boards/jpconstantineau_pykey18/board.c
+++ b/ports/raspberrypi/boards/jpconstantineau_pykey18/board.c
@@ -31,7 +31,7 @@
 
 void reset_board(void) {
     // turn off any left over LED
-    board_reset_user_neopixels(&pin_GPIO29, 62);
+    board_reset_user_neopixels(&pin_GPIO29, 19);
 }
 
 // Use the MP_WEAK supervisor/shared/board.c versions of routines not defined here.
diff --git a/ports/raspberrypi/boards/jpconstantineau_pykey44/board.c b/ports/raspberrypi/boards/jpconstantineau_pykey44/board.c
index 4785f742ec..4e0aa74450 100644
--- a/ports/raspberrypi/boards/jpconstantineau_pykey44/board.c
+++ b/ports/raspberrypi/boards/jpconstantineau_pykey44/board.c
@@ -31,7 +31,7 @@
 
 void reset_board(void) {
     // turn off any left over LED
-    board_reset_user_neopixels(&pin_GPIO29, 62);
+    board_reset_user_neopixels(&pin_GPIO29, 45);
 }
 
 // Use the MP_WEAK supervisor/shared/board.c versions of routines not defined here.
diff --git a/ports/raspberrypi/boards/jpconstantineau_pykey87/board.c b/ports/raspberrypi/boards/jpconstantineau_pykey87/board.c
index 4785f742ec..a77563c663 100644
--- a/ports/raspberrypi/boards/jpconstantineau_pykey87/board.c
+++ b/ports/raspberrypi/boards/jpconstantineau_pykey87/board.c
@@ -31,7 +31,7 @@
 
 void reset_board(void) {
     // turn off any left over LED
-    board_reset_user_neopixels(&pin_GPIO29, 62);
+    board_reset_user_neopixels(&pin_GPIO29, 88);
 }
 
 // Use the MP_WEAK supervisor/shared/board.c versions of routines not defined here.

From 575f177dd09f0f518f7e2c8514b20e9b12c19af8 Mon Sep 17 00:00:00 2001
From: RetiredWizard <github@retiredwizard.com>
Date: Mon, 13 Feb 2023 14:44:46 -0500
Subject: [PATCH 23/35] Update
 ports/broadcom/common-hal/neopixel_write/__init__.c

Co-authored-by: Scott Shawcroft <scott@tannewt.org>
---
 ports/broadcom/common-hal/neopixel_write/__init__.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ports/broadcom/common-hal/neopixel_write/__init__.c b/ports/broadcom/common-hal/neopixel_write/__init__.c
index d685ac53ce..38b1028bb3 100644
--- a/ports/broadcom/common-hal/neopixel_write/__init__.c
+++ b/ports/broadcom/common-hal/neopixel_write/__init__.c
@@ -96,7 +96,7 @@ void common_hal_neopixel_write(const digitalio_digitalinout_obj_t *digitalinout,
         // Wait for the clock to start up.
         COMPLETE_MEMORY_READS;
         icnt = 0;
-        while ((CM_PWM->CS_b.BUSY == 0) & (icnt++ < 1000)) {
+        while ((CM_PWM->CS_b.BUSY == 0) && (icnt++ < 1000)) {
             COMPLETE_MEMORY_READS;
         }
     }

From 791aefd3880d8c0f58baa2e9cd7cc40815d05059 Mon Sep 17 00:00:00 2001
From: RetiredWizard <github@retiredwizard.com>
Date: Mon, 13 Feb 2023 14:44:54 -0500
Subject: [PATCH 24/35] Update
 ports/broadcom/common-hal/neopixel_write/__init__.c

Co-authored-by: Scott Shawcroft <scott@tannewt.org>
---
 ports/broadcom/common-hal/neopixel_write/__init__.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ports/broadcom/common-hal/neopixel_write/__init__.c b/ports/broadcom/common-hal/neopixel_write/__init__.c
index 38b1028bb3..b631953c50 100644
--- a/ports/broadcom/common-hal/neopixel_write/__init__.c
+++ b/ports/broadcom/common-hal/neopixel_write/__init__.c
@@ -46,7 +46,7 @@ void common_hal_neopixel_write(const digitalio_digitalinout_obj_t *digitalinout,
     // Wait to make sure we don't append onto the last transmission. This should only be a tick or
     // two.
     int icnt;
-    while ((port_get_raw_ticks(NULL) < next_start_raw_ticks) &
+    while ((port_get_raw_ticks(NULL) < next_start_raw_ticks) &&
            (next_start_raw_ticks - port_get_raw_ticks(NULL) < 100)) {
         RUN_BACKGROUND_TASKS;
     }

From dcb6955fa5a8619e222109a334bdd31f377fd337 Mon Sep 17 00:00:00 2001
From: RetiredWizard <github@retiredwizard.com>
Date: Mon, 13 Feb 2023 14:58:37 -0500
Subject: [PATCH 25/35] use boolean and symbols

---
 ports/broadcom/common-hal/neopixel_write/__init__.c | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/ports/broadcom/common-hal/neopixel_write/__init__.c b/ports/broadcom/common-hal/neopixel_write/__init__.c
index b631953c50..b42bfb9de8 100644
--- a/ports/broadcom/common-hal/neopixel_write/__init__.c
+++ b/ports/broadcom/common-hal/neopixel_write/__init__.c
@@ -141,7 +141,7 @@ void common_hal_neopixel_write(const digitalio_digitalinout_obj_t *digitalinout,
         }
         if (channel == 1) {
             icnt = 0;
-            while ((pwm->STA_b.FULL1 == 1) & (icnt++ < 150)) {
+            while ((pwm->STA_b.FULL1 == 1) && (icnt++ < 150)) {
                 RUN_BACKGROUND_TASKS;
                 COMPLETE_MEMORY_READS;
             }
@@ -149,14 +149,14 @@ void common_hal_neopixel_write(const digitalio_digitalinout_obj_t *digitalinout,
             pwm->FIF1 = 0x000000;
         }
         icnt = 0;
-        while ((pwm->STA_b.FULL1 == 1) & (icnt++ < 150)) {
+        while ((pwm->STA_b.FULL1 == 1) && (icnt++ < 150)) {
             RUN_BACKGROUND_TASKS;
             COMPLETE_MEMORY_READS;
         }
         pwm->FIF1 = expanded;
         if (channel == 0) {
             icnt = 0;
-            while ((pwm->STA_b.FULL1 == 1) & (icnt++ < 150)) {
+            while ((pwm->STA_b.FULL1 == 1) && (icnt++ < 150)) {
                 RUN_BACKGROUND_TASKS;
                 COMPLETE_MEMORY_READS;
             }
@@ -166,19 +166,19 @@ void common_hal_neopixel_write(const digitalio_digitalinout_obj_t *digitalinout,
     }
 
     icnt = 0;
-    while ((pwm->STA_b.EMPT1 == 0) & (icnt++ < 2500)) {
+    while ((pwm->STA_b.EMPT1 == 0) && (icnt++ < 2500)) {
         RUN_BACKGROUND_TASKS;
         COMPLETE_MEMORY_READS;
     }
     // Wait for transmission to start.
     icnt = 0;
-    while (((pwm->STA_b.STA1 == 0) & (pwm->STA_b.STA2 == 0)) & (icnt++ < 150)) {
+    while (((pwm->STA_b.STA1 == 0) && (pwm->STA_b.STA2 == 0)) && (icnt++ < 150)) {
         RUN_BACKGROUND_TASKS;
         COMPLETE_MEMORY_READS;
     }
     // Wait for transmission to complete.
     icnt = 0;
-    while (((pwm->STA_b.STA1 == 1) | (pwm->STA_b.STA2 == 1)) & (icnt++ < 150)) {
+    while (((pwm->STA_b.STA1 == 1) | (pwm->STA_b.STA2 == 1)) && (icnt++ < 150)) {
         RUN_BACKGROUND_TASKS;
         COMPLETE_MEMORY_READS;
     }

From 39d3d97ea4a0805dd12294728f16bca4a6301fbd Mon Sep 17 00:00:00 2001
From: RetiredWizard <github@retiredwizard.com>
Date: Mon, 13 Feb 2023 19:46:35 -0500
Subject: [PATCH 26/35] Remove unnecessary memory barriers

---
 ports/broadcom/common-hal/neopixel_write/__init__.c | 11 +----------
 1 file changed, 1 insertion(+), 10 deletions(-)

diff --git a/ports/broadcom/common-hal/neopixel_write/__init__.c b/ports/broadcom/common-hal/neopixel_write/__init__.c
index b42bfb9de8..f5c3c245f6 100644
--- a/ports/broadcom/common-hal/neopixel_write/__init__.c
+++ b/ports/broadcom/common-hal/neopixel_write/__init__.c
@@ -96,9 +96,7 @@ void common_hal_neopixel_write(const digitalio_digitalinout_obj_t *digitalinout,
         // Wait for the clock to start up.
         COMPLETE_MEMORY_READS;
         icnt = 0;
-        while ((CM_PWM->CS_b.BUSY == 0) && (icnt++ < 1000)) {
-            COMPLETE_MEMORY_READS;
-        }
+        while ((CM_PWM->CS_b.BUSY == 0) && (icnt++ < 1000)) {}
     }
 
     PWM0_Type *pwm = PWM0;
@@ -143,7 +141,6 @@ void common_hal_neopixel_write(const digitalio_digitalinout_obj_t *digitalinout,
             icnt = 0;
             while ((pwm->STA_b.FULL1 == 1) && (icnt++ < 150)) {
                 RUN_BACKGROUND_TASKS;
-                COMPLETE_MEMORY_READS;
             }
             // Dummy value for the first channel.
             pwm->FIF1 = 0x000000;
@@ -151,14 +148,12 @@ void common_hal_neopixel_write(const digitalio_digitalinout_obj_t *digitalinout,
         icnt = 0;
         while ((pwm->STA_b.FULL1 == 1) && (icnt++ < 150)) {
             RUN_BACKGROUND_TASKS;
-            COMPLETE_MEMORY_READS;
         }
         pwm->FIF1 = expanded;
         if (channel == 0) {
             icnt = 0;
             while ((pwm->STA_b.FULL1 == 1) && (icnt++ < 150)) {
                 RUN_BACKGROUND_TASKS;
-                COMPLETE_MEMORY_READS;
             }
             // Dummy value for the second channel.
             pwm->FIF1 = 0x000000;
@@ -168,23 +163,19 @@ void common_hal_neopixel_write(const digitalio_digitalinout_obj_t *digitalinout,
     icnt = 0;
     while ((pwm->STA_b.EMPT1 == 0) && (icnt++ < 2500)) {
         RUN_BACKGROUND_TASKS;
-        COMPLETE_MEMORY_READS;
     }
     // Wait for transmission to start.
     icnt = 0;
     while (((pwm->STA_b.STA1 == 0) && (pwm->STA_b.STA2 == 0)) && (icnt++ < 150)) {
         RUN_BACKGROUND_TASKS;
-        COMPLETE_MEMORY_READS;
     }
     // Wait for transmission to complete.
     icnt = 0;
     while (((pwm->STA_b.STA1 == 1) | (pwm->STA_b.STA2 == 1)) && (icnt++ < 150)) {
         RUN_BACKGROUND_TASKS;
-        COMPLETE_MEMORY_READS;
     }
     // Shouldn't be anything left in queue but clear it so the clock doesn't crash if there is
     pwm->CTL = PWM0_CTL_CLRF1_Msk;
-    COMPLETE_MEMORY_READS;
 
     gpio_set_function(digitalinout->pin->number, GPIO_FUNCTION_OUTPUT);
 

From 09ccf2988d78a387f19ff0a9db7413fa8b62a926 Mon Sep 17 00:00:00 2001
From: RetiredWizard <github@retiredwizard.com>
Date: Mon, 13 Feb 2023 21:04:11 -0500
Subject: [PATCH 27/35] Replace bitwise or with boolean or

---
 ports/broadcom/common-hal/neopixel_write/__init__.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ports/broadcom/common-hal/neopixel_write/__init__.c b/ports/broadcom/common-hal/neopixel_write/__init__.c
index f5c3c245f6..aa256c7413 100644
--- a/ports/broadcom/common-hal/neopixel_write/__init__.c
+++ b/ports/broadcom/common-hal/neopixel_write/__init__.c
@@ -171,7 +171,7 @@ void common_hal_neopixel_write(const digitalio_digitalinout_obj_t *digitalinout,
     }
     // Wait for transmission to complete.
     icnt = 0;
-    while (((pwm->STA_b.STA1 == 1) | (pwm->STA_b.STA2 == 1)) && (icnt++ < 150)) {
+    while (((pwm->STA_b.STA1 == 1) || (pwm->STA_b.STA2 == 1)) && (icnt++ < 150)) {
         RUN_BACKGROUND_TASKS;
     }
     // Shouldn't be anything left in queue but clear it so the clock doesn't crash if there is

From 6ebb911a4dde0801c9fd5f56da7ee43c51375fd5 Mon Sep 17 00:00:00 2001
From: RetiredWizard <github@retiredwizard.com>
Date: Mon, 13 Feb 2023 21:06:47 -0500
Subject: [PATCH 28/35] pre-commit formatting fix

---
 ports/broadcom/common-hal/neopixel_write/__init__.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/ports/broadcom/common-hal/neopixel_write/__init__.c b/ports/broadcom/common-hal/neopixel_write/__init__.c
index aa256c7413..245244614f 100644
--- a/ports/broadcom/common-hal/neopixel_write/__init__.c
+++ b/ports/broadcom/common-hal/neopixel_write/__init__.c
@@ -96,7 +96,8 @@ void common_hal_neopixel_write(const digitalio_digitalinout_obj_t *digitalinout,
         // Wait for the clock to start up.
         COMPLETE_MEMORY_READS;
         icnt = 0;
-        while ((CM_PWM->CS_b.BUSY == 0) && (icnt++ < 1000)) {}
+        while ((CM_PWM->CS_b.BUSY == 0) && (icnt++ < 1000)) {
+        }
     }
 
     PWM0_Type *pwm = PWM0;

From 50c52fca88d3e8098bc9857f8223c1d961494391 Mon Sep 17 00:00:00 2001
From: MicroDev <70126934+MicroDev1@users.noreply.github.com>
Date: Tue, 14 Feb 2023 20:33:24 +0530
Subject: [PATCH 29/35] split raspberrypi and refactor espressif

---
 .github/actions/deps/external/action.yml      | 14 ++++----
 .../deps/{port => ports/espressif}/action.yml | 17 +---------
 .github/workflows/build-boards.yml            |  5 ++-
 .github/workflows/build.yml                   | 20 +++++++++---
 tools/ci_set_matrix.py                        | 32 ++++++++++---------
 5 files changed, 42 insertions(+), 46 deletions(-)
 rename .github/actions/deps/{port => ports/espressif}/action.yml (74%)

diff --git a/.github/actions/deps/external/action.yml b/.github/actions/deps/external/action.yml
index 9f5c3bd7c9..3503e5b194 100644
--- a/.github/actions/deps/external/action.yml
+++ b/.github/actions/deps/external/action.yml
@@ -8,7 +8,7 @@ inputs:
     options:
     - arm
     - aarch
-    - espressif
+    - esp
     - riscv
     - none
 
@@ -42,13 +42,13 @@ runs:
       with:
         release: '10-2020-q4'
 
-    # espressif
-    - name: Get espressif toolchain
-      if: inputs.platform == 'espressif'
+    # esp
+    - name: Get esp toolchain
+      if: inputs.platform == 'esp'
       run: sudo apt-get install -y ninja-build
       shell: bash
     - name: Install IDF tools
-      if: inputs.platform == 'espressif'
+      if: inputs.platform == 'esp'
       run: |
         echo "Installing ESP-IDF tools"
         $IDF_PATH/tools/idf_tools.py --non-interactive install required
@@ -58,7 +58,7 @@ runs:
         rm -rf $IDF_TOOLS_PATH/dist
       shell: bash
     - name: Set environment
-      if: inputs.platform == 'espressif'
+      if: inputs.platform == 'esp'
       run: |
         source $IDF_PATH/export.sh
         echo >> $GITHUB_ENV "IDF_PYTHON_ENV_PATH=$IDF_PYTHON_ENV_PATH"
@@ -75,7 +75,7 @@ runs:
 
     # common
     - name: Cache python dependencies
-      if: inputs.platform != 'espressif'
+      if: inputs.platform != 'esp'
       uses: ./.github/actions/deps/python
       with:
         action: ${{ fromJSON('["restore", "cache"]')[github.job == 'scheduler'] }}
diff --git a/.github/actions/deps/port/action.yml b/.github/actions/deps/ports/espressif/action.yml
similarity index 74%
rename from .github/actions/deps/port/action.yml
rename to .github/actions/deps/ports/espressif/action.yml
index 36b67c6081..aa1e98f8a0 100644
--- a/.github/actions/deps/port/action.yml
+++ b/.github/actions/deps/ports/espressif/action.yml
@@ -1,27 +1,15 @@
-name: Fetch port deps
-
-inputs:
-  platform:
-    required: false
-    default: none
-    type: choice
-    options:
-    - espressif
-    - none
+name: Fetch espressif port deps
 
 runs:
   using: composite
   steps:
-    # espressif
     - name: Set IDF env
-      if: inputs.platform == 'espressif'
       run: |
         echo >> $GITHUB_ENV "IDF_PATH=$GITHUB_WORKSPACE/ports/espressif/esp-idf"
         echo >> $GITHUB_ENV "IDF_TOOLS_PATH=$GITHUB_WORKSPACE/.idf_tools"
       shell: bash
 
     - name: Get IDF commit
-      if: inputs.platform == 'espressif'
       id: idf-commit
       run: |
         COMMIT=$(git submodule status ports/espressif/esp-idf | grep -o -P '(?<=^-).*(?= )')
@@ -30,7 +18,6 @@ runs:
       shell: bash
 
     - name: Cache IDF submodules
-      if: inputs.platform == 'espressif'
       uses: actions/cache@v3
       with:
         path: |
@@ -39,13 +26,11 @@ runs:
         key: submodules-idf-${{ steps.idf-commit.outputs.commit }}
 
     - name: Cache IDF tools
-      if: inputs.platform == 'espressif'
       uses: actions/cache@v3
       with:
         path: ${{ env.IDF_TOOLS_PATH }}
         key: ${{ runner.os }}-${{ env.pythonLocation }}-tools-idf-${{ steps.idf-commit.outputs.commit }}
 
     - name: Initialize IDF submodules
-      if: inputs.platform == 'espressif'
       run: git submodule update --init --depth=1 --recursive $IDF_PATH
       shell: bash
diff --git a/.github/workflows/build-boards.yml b/.github/workflows/build-boards.yml
index c302be7c9f..b9e6577a19 100644
--- a/.github/workflows/build-boards.yml
+++ b/.github/workflows/build-boards.yml
@@ -35,9 +35,8 @@ jobs:
       with:
         python-version: 3.x
     - name: Set up port
-      uses: ./.github/actions/deps/port
-      with:
-        platform: ${{ inputs.platform }}
+      if: inputs.platform == 'esp'
+      uses: ./.github/actions/deps/ports/espressif
     - name: Set up submodules
       id: set-up-submodules
       uses: ./.github/actions/deps/submodules
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index cdf9462289..dae1ffc271 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -24,8 +24,9 @@ jobs:
       build-boards: ${{ steps.set-matrix.outputs.build-boards }}
       boards-aarch: ${{ steps.set-matrix.outputs.boards-aarch }}
       boards-arm: ${{ steps.set-matrix.outputs.boards-arm }}
-      boards-espressif: ${{ steps.set-matrix.outputs.boards-espressif }}
+      boards-esp: ${{ steps.set-matrix.outputs.boards-esp }}
       boards-riscv: ${{ steps.set-matrix.outputs.boards-riscv }}
+      boards-rpi: ${{ steps.set-matrix.outputs.boards-rpi }}
       cp-version: ${{ steps.set-up-submodules.outputs.version }}
     steps:
     - name: Dump GitHub context
@@ -238,13 +239,13 @@ jobs:
       cp-version: ${{ needs.scheduler.outputs.cp-version }}
 
 
-  espressif:
+  esp:
     needs: [scheduler, mpy-cross, tests]
-    if: ${{ needs.scheduler.outputs.boards-espressif != '[]' }}
+    if: ${{ needs.scheduler.outputs.boards-esp != '[]' }}
     uses: ./.github/workflows/build-boards.yml
     with:
-      platform: espressif
-      boards: ${{ needs.scheduler.outputs.boards-espressif }}
+      platform: esp
+      boards: ${{ needs.scheduler.outputs.boards-esp }}
       cp-version: ${{ needs.scheduler.outputs.cp-version }}
 
 
@@ -256,3 +257,12 @@ jobs:
       platform: riscv
       boards: ${{ needs.scheduler.outputs.boards-riscv }}
       cp-version: ${{ needs.scheduler.outputs.cp-version }}
+
+  rpi:
+    needs: [scheduler, mpy-cross, tests]
+    if: ${{ needs.scheduler.outputs.boards-rpi != '[]' }}
+    uses: ./.github/workflows/build-boards.yml
+    with:
+      platform: arm
+      boards: ${{ needs.scheduler.outputs.boards-rpi }}
+      cp-version: ${{ needs.scheduler.outputs.cp-version }}
diff --git a/tools/ci_set_matrix.py b/tools/ci_set_matrix.py
index 354ad51d71..6e7a4f1229 100755
--- a/tools/ci_set_matrix.py
+++ b/tools/ci_set_matrix.py
@@ -46,11 +46,11 @@ PORT_TO_ARCH = {
     "atmel-samd": "arm",
     "broadcom": "aarch",
     "cxd56": "arm",
-    "espressif": "espressif",
+    "espressif": "esp",
     "litex": "riscv",
     "mimxrt10xx": "arm",
     "nrf": "arm",
-    "raspberrypi": "arm",
+    "raspberrypi": "rpi",
     "stm": "arm",
 }
 
@@ -204,31 +204,33 @@ def set_boards_to_build(build_all: bool):
             break
 
     # Split boards by architecture.
+    arch_to_boards = {"aarch": [], "arm": [], "esp": [], "riscv": [], "rpi": []}
+
+    # Append previously failed boards
+    for arch in arch_to_boards:
+        arch_to_job = f"build-{arch}"
+        if arch_to_job in last_failed_jobs:
+            for board in last_failed_jobs[arch_to_job]:
+                if not board in boards_to_build:
+                    boards_to_build.append(board)
+
     build_boards = bool(boards_to_build)
     print("Building boards:", build_boards)
     set_output("build-boards", build_boards)
-    arch_to_boards = {"aarch": [], "arm": [], "riscv": [], "espressif": []}
+
+    # Append boards according to arch
     for board in sorted(boards_to_build):
-        print(" ", board)
         port = board_to_port.get(board)
         # A board can appear due to its _deletion_ (rare)
         # if this happens it's not in `board_to_port`.
         if not port:
             continue
-        arch = PORT_TO_ARCH[port]
-        arch_to_boards[arch].append(board)
+        arch_to_boards[PORT_TO_ARCH[port]].append(board)
+        print(" ", board)
 
     # Set the step outputs for each architecture
     for arch in arch_to_boards:
-        # Append previous failed jobs
-        if f"build-{arch}" in last_failed_jobs:
-            failed_boards = last_failed_jobs[f"build-{arch}"]
-            for board in failed_boards:
-                if not board in arch_to_boards[arch]:
-                    print(" ", board)
-                    arch_to_boards[arch].append(board)
-        # Set Output
-        set_output(f"boards-{arch}", json.dumps(sorted(arch_to_boards[arch])))
+        set_output(f"boards-{arch}", json.dumps(arch_to_boards[arch]))
 
 
 def set_docs_to_build(build_doc: bool):

From 73840f840db1e7f2374ccc57f236e1185aa3c744 Mon Sep 17 00:00:00 2001
From: Jeff Epler <jepler@gmail.com>
Date: Tue, 14 Feb 2023 10:46:20 -0600
Subject: [PATCH 30/35] Improve boot_out.txt truncation

 * write any partial message
 * instead of "..." show a sensible (translatable) message

This does slightly lower the amount of data that can be printed,
and makes the exact amount dependent on the language. However, if boot.py
intentionally needs to produce larger amounts of output, it can
deliberately mount the filesystem in RW mode and perform any writes needed.
In that case it's up to the boot.py to choose an appropriate way to
limit the number of writes if needed for the application.
---
 locale/circuitpython.pot        |  4 ++++
 supervisor/shared/micropython.c | 11 +++++++++--
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/locale/circuitpython.pot b/locale/circuitpython.pot
index 72ee60cc0c..4e7abcb0b8 100644
--- a/locale/circuitpython.pot
+++ b/locale/circuitpython.pot
@@ -2410,6 +2410,10 @@ msgid ""
 "You pressed the reset button during boot. Press again to exit safe mode."
 msgstr ""
 
+#: supervisor/shared/micropython.c
+msgid "[truncated due to length]"
+msgstr ""
+
 #: py/objtype.c
 msgid "__init__() should return None"
 msgstr ""
diff --git a/supervisor/shared/micropython.c b/supervisor/shared/micropython.c
index ebc0aef2d1..5ee0061544 100644
--- a/supervisor/shared/micropython.c
+++ b/supervisor/shared/micropython.c
@@ -62,8 +62,15 @@ void mp_hal_stdout_tx_strn(const char *str, size_t len) {
     #ifdef CIRCUITPY_BOOT_OUTPUT_FILE
     if (boot_output != NULL) {
         // Ensure boot_out.txt is capped at 1 filesystem block and ends with a newline
-        if (len + boot_output->len > 508) {
-            vstr_add_str(boot_output, "...\n");
+        #define TRUNCATED translate("[truncated due to length]")
+        size_t truncated_message_len = decompress_length(TRUNCATED);
+        size_t maxlen = 512 - truncated_message_len; // includes trailing '\0' so we do not need to account for trailing newline '\n' in vstr_add_byte
+        if (len + boot_output->len > maxlen) {
+            size_t remaining_len = maxlen - boot_output->len;
+            vstr_add_strn(boot_output, str, remaining_len);
+            char buf[truncated_message_len];
+            vstr_add_str(boot_output, decompress(TRUNCATED, buf));
+            vstr_add_byte(boot_output, '\n');
             boot_output = NULL;
         } else {
             vstr_add_strn(boot_output, str, len);

From 98b61279e4c691e750b905dc7357f0081cbf91d8 Mon Sep 17 00:00:00 2001
From: Gregory Neverov <42853258+gneverov@users.noreply.github.com>
Date: Tue, 14 Feb 2023 09:05:24 -0800
Subject: [PATCH 31/35] Add function common_hal_busio_uart_never_reset for rp2

---
 ports/raspberrypi/common-hal/busio/UART.c | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/ports/raspberrypi/common-hal/busio/UART.c b/ports/raspberrypi/common-hal/busio/UART.c
index 4212249ec9..8c33226707 100644
--- a/ports/raspberrypi/common-hal/busio/UART.c
+++ b/ports/raspberrypi/common-hal/busio/UART.c
@@ -339,3 +339,18 @@ bool common_hal_busio_uart_ready_to_tx(busio_uart_obj_t *self) {
     }
     return uart_is_writable(self->uart);
 }
+
+STATIC void pin_never_reset(uint8_t pin) {
+    if (pin != NO_PIN) {
+        never_reset_pin_number(pin);
+    }
+}
+
+void common_hal_busio_uart_never_reset(busio_uart_obj_t *self) {
+    never_reset_uart(self->uart_id);
+    pin_never_reset(self->tx_pin);
+    pin_never_reset(self->rx_pin);
+    pin_never_reset(self->cts_pin);
+    pin_never_reset(self->rs485_dir_pin);
+    pin_never_reset(self->rts_pin);
+}

From 5168e96d0c478b4de0e064d1a91e27775ced731f Mon Sep 17 00:00:00 2001
From: MicroDev <70126934+MicroDev1@users.noreply.github.com>
Date: Wed, 15 Feb 2023 09:53:56 +0530
Subject: [PATCH 32/35] fix docs upload to aws

---
 .github/workflows/build.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index dae1ffc271..0af395a620 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -201,6 +201,11 @@ jobs:
       with:
         name: docs
         path: _build/latex
+    - name: Zip stubs
+      if: >-
+        (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') ||
+        (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))
+      run: zip -9r circuitpython-stubs.zip circuitpython-stubs
     - name: Upload to S3
       uses: ./.github/actions/upload_aws
       with:

From cfea7c1d6995a79d954a0253f24cc3228f83abf5 Mon Sep 17 00:00:00 2001
From: MicroDev <70126934+MicroDev1@users.noreply.github.com>
Date: Wed, 15 Feb 2023 10:56:54 +0530
Subject: [PATCH 33/35] refactor mpy-cross-mac aws path

---
 .github/workflows/build.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 0af395a620..ad488152f1 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -152,9 +152,9 @@ jobs:
           (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') ||
           (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))
       run: |
-        [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross-macos-universal s3://adafruit-circuit-python/bin/mpy-cross/mpy-cross-macos-11-${{ env.CP_VERSION }}-universal --no-progress --region us-east-1
-        [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross/mpy-cross-arm64 s3://adafruit-circuit-python/bin/mpy-cross/mpy-cross-macos-11-${{ env.CP_VERSION }}-arm64 --no-progress --region us-east-1
-        [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross/mpy-cross s3://adafruit-circuit-python/bin/mpy-cross/mpy-cross-macos-11-${{ env.CP_VERSION }}-x64 --no-progress --region us-east-1
+        [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross-macos-universal s3://adafruit-circuit-python/bin/mpy-cross/macos-11/mpy-cross-macos-11-${{ env.CP_VERSION }}-universal --no-progress --region us-east-1
+        [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross/mpy-cross-arm64 s3://adafruit-circuit-python/bin/mpy-cross/macos-11/mpy-cross-macos-11-${{ env.CP_VERSION }}-arm64 --no-progress --region us-east-1
+        [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross/mpy-cross s3://adafruit-circuit-python/bin/mpy-cross/macos-11/mpy-cross-macos-11-${{ env.CP_VERSION }}-x64 --no-progress --region us-east-1
       env:
         AWS_PAGER: ''
         AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}

From 9de039b62555e71c9b4bc5c849d20cd8c3399d47 Mon Sep 17 00:00:00 2001
From: MicroDev <70126934+MicroDev1@users.noreply.github.com>
Date: Wed, 15 Feb 2023 21:25:56 +0530
Subject: [PATCH 34/35] pass secrets to reusable workflows

---
 .github/workflows/build-boards.yml    |  7 +++++--
 .github/workflows/build-mpy-cross.yml |  5 +++++
 .github/workflows/build.yml           | 12 ++++++------
 3 files changed, 16 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/build-boards.yml b/.github/workflows/build-boards.yml
index b9e6577a19..457fce5b98 100644
--- a/.github/workflows/build-boards.yml
+++ b/.github/workflows/build-boards.yml
@@ -6,14 +6,17 @@ on:
       platform:
         required: true
         type: string
-
       boards:
         required: true
         type: string
-
       cp-version:
         required: true
         type: string
+    secrets:
+      AWS_ACCESS_KEY_ID:
+        required: false
+      AWS_SECRET_ACCESS_KEY:
+        required: false
 
 jobs:
   build:
diff --git a/.github/workflows/build-mpy-cross.yml b/.github/workflows/build-mpy-cross.yml
index c17c9da1cb..10c4498bc8 100644
--- a/.github/workflows/build-mpy-cross.yml
+++ b/.github/workflows/build-mpy-cross.yml
@@ -6,6 +6,11 @@ on:
       cp-version:
         required: true
         type: string
+    secrets:
+      AWS_ACCESS_KEY_ID:
+        required: false
+      AWS_SECRET_ACCESS_KEY:
+        required: false
 
 jobs:
   build:
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index ad488152f1..6159f7291e 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -96,15 +96,14 @@ jobs:
     with:
       cp-version: ${{ needs.scheduler.outputs.cp-version }}
 
-
   mpy-cross:
     needs: scheduler
     if: needs.scheduler.outputs.build-boards == 'True'
     uses: ./.github/workflows/build-mpy-cross.yml
+    secrets: inherit
     with:
       cp-version: ${{ needs.scheduler.outputs.cp-version }}
 
-
   mpy-cross-mac:
     runs-on: macos-11
     needs: scheduler
@@ -223,41 +222,41 @@ jobs:
         [ -z "$TWINE_USERNAME" ] || echo "Uploading dev release to PyPi"
         [ -z "$TWINE_USERNAME" ] || twine upload circuitpython-stubs/dist/*
 
-
   aarch:
     needs: [scheduler, mpy-cross, tests]
     if: ${{ needs.scheduler.outputs.boards-aarch != '[]' }}
     uses: ./.github/workflows/build-boards.yml
+    secrets: inherit
     with:
       platform: aarch
       boards: ${{ needs.scheduler.outputs.boards-aarch }}
       cp-version: ${{ needs.scheduler.outputs.cp-version }}
 
-
   arm:
     needs: [scheduler, mpy-cross, tests]
     if: ${{ needs.scheduler.outputs.boards-arm != '[]' }}
     uses: ./.github/workflows/build-boards.yml
+    secrets: inherit
     with:
       platform: arm
       boards: ${{ needs.scheduler.outputs.boards-arm }}
       cp-version: ${{ needs.scheduler.outputs.cp-version }}
 
-
   esp:
     needs: [scheduler, mpy-cross, tests]
     if: ${{ needs.scheduler.outputs.boards-esp != '[]' }}
     uses: ./.github/workflows/build-boards.yml
+    secrets: inherit
     with:
       platform: esp
       boards: ${{ needs.scheduler.outputs.boards-esp }}
       cp-version: ${{ needs.scheduler.outputs.cp-version }}
 
-
   riscv:
     needs: [scheduler, mpy-cross, tests]
     if: ${{ needs.scheduler.outputs.boards-riscv != '[]' }}
     uses: ./.github/workflows/build-boards.yml
+    secrets: inherit
     with:
       platform: riscv
       boards: ${{ needs.scheduler.outputs.boards-riscv }}
@@ -267,6 +266,7 @@ jobs:
     needs: [scheduler, mpy-cross, tests]
     if: ${{ needs.scheduler.outputs.boards-rpi != '[]' }}
     uses: ./.github/workflows/build-boards.yml
+    secrets: inherit
     with:
       platform: arm
       boards: ${{ needs.scheduler.outputs.boards-rpi }}

From ffbb7550b40a2de6b8945af90904f9428b6bce10 Mon Sep 17 00:00:00 2001
From: Gregory Neverov <42853258+gneverov@users.noreply.github.com>
Date: Wed, 15 Feb 2023 11:08:09 -0800
Subject: [PATCH 35/35] Move serial init to earlier in boot process

---
 main.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/main.c b/main.c
index d70e718b6d..6e953a6233 100644
--- a/main.c
+++ b/main.c
@@ -925,6 +925,9 @@ int __attribute__((used)) main(void) {
     common_hal_nvm_bytearray_set_bytes(&common_hal_mcu_nvm_obj,0,&value_out,1);
     #endif
 
+    // Start the debug serial
+    serial_early_init();
+
     // Wait briefly to give a reset window where we'll enter safe mode after the reset.
     if (safe_mode == NO_SAFE_MODE) {
         safe_mode = wait_for_safe_mode_reset();
@@ -941,9 +944,6 @@ int __attribute__((used)) main(void) {
     supervisor_bluetooth_init();
     #endif
 
-    // Start the debug serial
-    serial_early_init();
-
     #if !INTERNAL_FLASH_FILESYSTEM
     // Set up anything that might need to get done before we try to use SPI flash
     // This is needed for some boards where flash relies on GPIO setup to work