Merge branch 'adafruit:main' into settings-toml-pystack

.github/actions/deps/external/action.yml

@@ -0,0 +1,84 @@
+name: Fetch external deps
+
+inputs:
+  platform:
+    required: false
+    default: none
+    type: choice
+    options:
+    - arm
+    - aarch
+    - esp
+    - riscv
+    - none
+
+runs:
+  using: composite
+  steps:
+    # aarch
+    - name: Get aarch toolchain
+      if: inputs.platform == 'aarch'
+      run: |
+        wget --no-verbose https://adafruit-circuit-python.s3.amazonaws.com/gcc-arm-10.3-2021.07-x86_64-aarch64-none-elf.tar.xz
+        sudo tar -C /usr --strip-components=1 -xaf gcc-arm-10.3-2021.07-x86_64-aarch64-none-elf.tar.xz
+        sudo apt-get install -y mtools
+      shell: bash
+    - name: Install mkfs.fat
+      if: inputs.platform == 'aarch'
+      run: |
+        wget https://github.com/dosfstools/dosfstools/releases/download/v4.2/dosfstools-4.2.tar.gz
+        tar -xaf dosfstools-4.2.tar.gz
+        cd dosfstools-4.2
+        ./configure
+        make -j 2
+        cd src
+        echo >> $GITHUB_PATH $(pwd)
+      shell: bash
+
+    # arm
+    - name: Get arm toolchain
+      if: inputs.platform == 'aarch' || inputs.platform == 'arm'
+      uses: carlosperate/arm-none-eabi-gcc-action@v1
+      with:
+        release: '10-2020-q4'
+
+    # esp
+    - name: Get esp toolchain
+      if: inputs.platform == 'esp'
+      run: sudo apt-get install -y ninja-build
+      shell: bash
+    - name: Install IDF tools
+      if: inputs.platform == 'esp'
+      run: |
+        echo "Installing ESP-IDF tools"
+        $IDF_PATH/tools/idf_tools.py --non-interactive install required
+        $IDF_PATH/tools/idf_tools.py --non-interactive install cmake
+        echo "Installing Python environment and packages"
+        $IDF_PATH/tools/idf_tools.py --non-interactive install-python-env
+        rm -rf $IDF_TOOLS_PATH/dist
+      shell: bash
+    - name: Set environment
+      if: inputs.platform == 'esp'
+      run: |
+        source $IDF_PATH/export.sh
+        echo >> $GITHUB_ENV "IDF_PYTHON_ENV_PATH=$IDF_PYTHON_ENV_PATH"
+        echo >> $GITHUB_PATH "$PATH"
+      shell: bash
+
+    # riscv
+    - name: Get riscv toolchain
+      if: inputs.platform == 'riscv'
+      run: |
+        wget https://static.dev.sifive.com/dev-tools/riscv64-unknown-elf-gcc-8.3.0-2019.08.0-x86_64-linux-centos6.tar.gz
+        sudo tar -C /usr --strip-components=1 -xaf riscv64-unknown-elf-gcc-8.3.0-2019.08.0-x86_64-linux-centos6.tar.gz
+      shell: bash
+
+    # common
+    - name: Cache python dependencies
+      if: inputs.platform != 'esp'
+      uses: ./.github/actions/deps/python
+      with:
+        action: ${{ fromJSON('["restore", "cache"]')[github.job == 'scheduler'] }}
+    - name: Install python dependencies
+      run: pip install -r requirements-dev.txt
+      shell: bash

.github/actions/deps/ports/espressif/action.yml

@@ -0,0 +1,36 @@
+name: Fetch espressif port deps
+
+runs:
+  using: composite
+  steps:
+    - name: Set IDF env
+      run: |
+        echo >> $GITHUB_ENV "IDF_PATH=$GITHUB_WORKSPACE/ports/espressif/esp-idf"
+        echo >> $GITHUB_ENV "IDF_TOOLS_PATH=$GITHUB_WORKSPACE/.idf_tools"
+      shell: bash
+
+    - name: Get IDF commit
+      id: idf-commit
+      run: |
+        COMMIT=$(git submodule status ports/espressif/esp-idf | grep -o -P '(?<=^-).*(?= )')
+        echo "$COMMIT"
+        echo "commit=$COMMIT" >> $GITHUB_OUTPUT
+      shell: bash
+
+    - name: Cache IDF submodules
+      uses: actions/cache@v3
+      with:
+        path: |
+          .git/modules/ports/espressif/esp-idf
+          ports/espressif/esp-idf
+        key: submodules-idf-${{ steps.idf-commit.outputs.commit }}
+
+    - name: Cache IDF tools
+      uses: actions/cache@v3
+      with:
+        path: ${{ env.IDF_TOOLS_PATH }}
+        key: ${{ runner.os }}-${{ env.pythonLocation }}-tools-idf-${{ steps.idf-commit.outputs.commit }}
+
+    - name: Initialize IDF submodules
+      run: git submodule update --init --depth=1 --recursive $IDF_PATH
+      shell: bash

.github/actions/deps/python/action.yml

@@ -0,0 +1,42 @@
+name: Fetch python deps
+
+inputs:
+  action:
+    description: The cache action to use
+    required: false
+    default: restore
+    type: choice
+    options:
+    - cache
+    - restore
+
+runs:
+  using: composite
+  steps:
+    - name: Cache python dependencies
+      id: cache-python-deps
+      if: inputs.action == 'cache'
+      uses: actions/cache@v3
+      with:
+        path: .cp_tools
+        key: ${{ runner.os }}-${{ env.pythonLocation }}-tools-cp-${{ hashFiles('requirements-dev.txt') }}
+
+    - name: Restore python dependencies
+      id: restore-python-deps
+      if: inputs.action == 'restore'
+      uses: actions/cache/restore@v3
+      with:
+        path: .cp_tools
+        key: ${{ runner.os }}-${{ env.pythonLocation }}-tools-cp-${{ hashFiles('requirements-dev.txt') }}
+
+    - name: Set up venv
+      if: inputs.action == 'cache' && !steps.cache-python-deps.outputs.cache-hit
+      run: python -m venv .cp_tools
+      shell: bash
+
+    - name: Activate venv
+      if: inputs.action == 'cache' || (inputs.action == 'restore' && steps.restore-python-deps.outputs.cache-hit)
+      run: |
+        source .cp_tools/bin/activate
+        echo >> $GITHUB_PATH "$PATH"
+      shell: bash

.github/actions/deps/submodules/action.yml

@@ -1,13 +1,18 @@
 name: 'Fetch Submodules'
 
 inputs:
+  target:
+    description: 'The target for ci_fetch_deps'
+    required: false
+    type: string
+
   submodules:
     description: 'The submodules to cache'
     required: false
     default: '["extmod/ulab", "lib/", "tools/"]'
     type: string
 
-  cache:
+  action:
     description: 'The cache action to use'
     required: false
     default: 'restore'
@@ -42,7 +47,7 @@ runs:
       shell: bash
 
     - name: Cache submodules
-      if: ${{ inputs.cache == 'cache' }}
+      if: ${{ inputs.action == 'cache' }}
       uses: actions/cache@v3
       with:
         path: ".git/modules/\n${{ join(fromJSON(steps.create-submodule-status.outputs.submodules), '\n') }}"
@@ -50,7 +55,7 @@ runs:
         enableCrossOsArchive: true
 
     - name: Restore submodules
-      if: ${{ inputs.cache == 'restore' }}
+      if: ${{ inputs.action == 'restore' }}
       uses: actions/cache/restore@v3
       with:
         path: ".git/modules/\n${{ join(fromJSON(steps.create-submodule-status.outputs.submodules), '\n') }}"
@@ -63,7 +68,7 @@ runs:
 
     - name: CircuitPython dependencies
       id: cp-deps
-      run: python tools/ci_fetch_deps.py ${{ matrix.board || github.job }}
+      run: python tools/ci_fetch_deps.py ${{ inputs.target || matrix.board || github.job }}
       shell: bash
 
     - name: CircuitPython version

.github/actions/mpy_cross/action.yml

@@ -0,0 +1,37 @@
+name: Set up mpy-cross
+
+inputs:
+  download:
+    required: false
+    default: true
+    type: boolean
+
+runs:
+  using: composite
+  steps:
+    - name: Download mpy-cross
+      id: download-mpy-cross
+      if: inputs.download == 'true'
+      continue-on-error: true
+      uses: actions/download-artifact@v3
+      with:
+        name: mpy-cross
+        path: mpy-cross
+
+    - name: Make mpy-cross executable
+      if: inputs.download == 'true' && steps.download-mpy-cross.outcome == 'success'
+      run: sudo chmod +x mpy-cross/mpy-cross
+      shell: bash
+
+    - name: Build mpy-cross
+      if: inputs.download == 'false' || steps.download-mpy-cross.outcome == 'failure'
+      run: make -C mpy-cross -j2
+      shell: bash
+
+    - name: Upload mpy-cross
+      if: inputs.download == 'false' || steps.download-mpy-cross.outcome == 'failure'
+      continue-on-error: true
+      uses: actions/upload-artifact@v3
+      with:
+        name: mpy-cross
+        path: mpy-cross/mpy-cross

.github/actions/upload_aws/action.yml

@@ -0,0 +1,32 @@
+name: Upload to AWS S3
+
+inputs:
+  source:
+    required: true
+    type: string
+
+  destination:
+    required: false
+    type: string
+
+  AWS_ACCESS_KEY_ID:
+    required: true
+
+  AWS_SECRET_ACCESS_KEY:
+    required: true
+
+runs:
+  using: composite
+  steps:
+    - name: Upload to S3
+      if: >-
+        (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') ||
+        (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))
+      run: >-
+        [ -z "$AWS_ACCESS_KEY_ID" ] ||
+        aws s3 cp ${{ inputs.source }} s3://adafruit-circuit-python/bin/${{ inputs.destination }} --recursive --no-progress --region us-east-1
+      env:
+        AWS_PAGER: ''
+        AWS_ACCESS_KEY_ID: ${{ inputs.AWS_ACCESS_KEY_ID }}
+        AWS_SECRET_ACCESS_KEY: ${{ inputs.AWS_SECRET_ACCESS_KEY }}
+      shell: bash

.github/workflows/build-boards.yml

@@ -0,0 +1,85 @@
+name: Build boards
+
+on:
+  workflow_call:
+    inputs:
+      platform:
+        required: true
+        type: string
+      boards:
+        required: true
+        type: string
+      cp-version:
+        required: true
+        type: string
+    secrets:
+      AWS_ACCESS_KEY_ID:
+        required: false
+      AWS_SECRET_ACCESS_KEY:
+        required: false
+
+jobs:
+  build:
+    runs-on: ubuntu-22.04
+    env:
+      CP_VERSION: ${{ inputs.cp-version }}
+    strategy:
+      fail-fast: false
+      matrix:
+        board: ${{ fromJSON(inputs.boards) }}
+    steps:
+    - name: Set up repository
+      uses: actions/checkout@v3
+      with:
+        submodules: false
+        fetch-depth: 1
+    - name: Set up python
+      uses: actions/setup-python@v4
+      with:
+        python-version: 3.x
+    - name: Set up port
+      if: inputs.platform == 'esp'
+      uses: ./.github/actions/deps/ports/espressif
+    - name: Set up submodules
+      id: set-up-submodules
+      uses: ./.github/actions/deps/submodules
+    - name: Set up external
+      uses: ./.github/actions/deps/external
+      with:
+        platform: ${{ inputs.platform }}
+    - name: Set up mpy-cross
+      if: steps.set-up-submodules.outputs.frozen == 'True'
+      uses: ./.github/actions/mpy_cross
+
+    - name: Versions
+      run: |
+        gcc --version
+        python3 --version
+        cmake --version || true
+        ninja --version || true
+        aarch64-none-elf-gcc --version || true
+        arm-none-eabi-gcc --version || true
+        xtensa-esp32-elf-gcc --version || true
+        riscv32-esp-elf-gcc --version || true
+        riscv64-unknown-elf-gcc --version || true
+        mkfs.fat --version || true
+
+    - name: Set up build failure matcher
+      run: echo "::add-matcher::$GITHUB_WORKSPACE/.github/workflows/match-build-fail.json"
+    - name: Build board
+      run: python3 -u build_release_files.py
+      working-directory: tools
+      env:
+        BOARDS: ${{ matrix.board }}
+
+    - name: Upload artifact
+      uses: actions/upload-artifact@v3
+      with:
+        name: ${{ matrix.board }}
+        path: bin/${{ matrix.board }}
+    - name: Upload to S3
+      uses: ./.github/actions/upload_aws
+      with:
+        source: bin/
+        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}

.github/workflows/build-mpy-cross.yml

@@ -0,0 +1,70 @@
+name: Build mpy-cross
+
+on:
+  workflow_call:
+    inputs:
+      cp-version:
+        required: true
+        type: string
+    secrets:
+      AWS_ACCESS_KEY_ID:
+        required: false
+      AWS_SECRET_ACCESS_KEY:
+        required: false
+
+jobs:
+  build:
+    runs-on: ubuntu-22.04
+    strategy:
+      fail-fast: false
+      matrix:
+        mpy-cross: ["static", "static-aarch64", "static-mingw", "static-raspbian"]
+    env:
+      CP_VERSION: ${{ inputs.cp-version }}
+      EX_static-mingw: static.exe
+      OS_static: linux-amd64
+      OS_static-aarch64: linux-aarch64
+      OS_static-mingw: windows
+      OS_static-raspbian: linux-raspbian
+    steps:
+    - name: Set up repository
+      uses: actions/checkout@v3
+      with:
+        submodules: false
+        fetch-depth: 1
+    - name: Set up python
+      uses: actions/setup-python@v4
+      with:
+        python-version: 3.x
+    - name: Set up submodules
+      uses: ./.github/actions/deps/submodules
+      with:
+        target: mpy-cross
+
+    - name: Install toolchain (aarch64)
+      if: matrix.mpy-cross == 'static-aarch64'
+      run: sudo apt-get install -y gcc-aarch64-linux-gnu
+    - name: Install toolchain (mingw)
+      if: matrix.mpy-cross == 'static-mingw'
+      run: sudo apt-get install -y mingw-w64
+
+    - name: Build mpy-cross.${{ matrix.mpy-cross }}
+      run: make -C mpy-cross -j2 -f Makefile.${{ matrix.mpy-cross }}
+
+    - name: Set output
+      run: |
+        echo >> $GITHUB_ENV "EX=${{ env[format('EX_{0}', matrix.mpy-cross)] || matrix.mpy-cross }}"
+        echo >> $GITHUB_ENV "OS=${{ env[format('OS_{0}', matrix.mpy-cross)] }}"
+
+    - name: Upload artifact
+      uses: actions/upload-artifact@v3
+      with:
+        name: mpy-cross.${{ env.EX }}
+        path: mpy-cross/mpy-cross.${{ env.EX }}
+    - name: Upload to S3
+      uses: ./.github/actions/upload_aws
+      with:
+        source: mpy-cross/mpy-cross.${{ env.EX }}
+        destination: mpy-cross/${{ env.OS }}/mpy-cross-${{ env.OS }}-${{ env.CP_VERSION }}.${{ env.EX }}
+        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}

.github/workflows/build.yml

@@ -17,14 +17,16 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  test:
+  scheduler:
     runs-on: ubuntu-20.04
     outputs:
       build-doc: ${{ steps.set-matrix.outputs.build-doc }}
+      build-boards: ${{ steps.set-matrix.outputs.build-boards }}
       boards-aarch: ${{ steps.set-matrix.outputs.boards-aarch }}
       boards-arm: ${{ steps.set-matrix.outputs.boards-arm }}
-      boards-espressif: ${{ steps.set-matrix.outputs.boards-espressif }}
+      boards-esp: ${{ steps.set-matrix.outputs.boards-esp }}
       boards-riscv: ${{ steps.set-matrix.outputs.boards-riscv }}
+      boards-rpi: ${{ steps.set-matrix.outputs.boards-rpi }}
       cp-version: ${{ steps.set-up-submodules.outputs.version }}
     steps:
     - name: Dump GitHub context
@@ -39,107 +41,32 @@ jobs:
     - name: Set up python
       uses: actions/setup-python@v4
       with:
-        python-version: "3.x"
+        python-version: 3.x
     - name: Duplicate USB VID/PID check
       run: python3 -u -m tools.ci_check_duplicate_usb_vid_pid
     - name: Set up submodules
       id: set-up-submodules
-      uses: ./.github/actions/fetch_submodules
+      uses: ./.github/actions/deps/submodules
       with:
-        cache: "cache"
+        action: cache
         version: true
-    - name: Install dependencies
-      run: |
-        sudo apt-get update
-        sudo apt-get install -y eatmydata
-        sudo eatmydata apt-get install -y gettext gcc-aarch64-linux-gnu mingw-w64
-        pip install -r requirements-dev.txt
-    - name: Versions
-      run: |
-        gcc --version
-        python3 --version
-    - name: Build mpy-cross
-      run: make -C mpy-cross -j2
-    - name: Build unix port
-      run: |
-        make -C ports/unix VARIANT=coverage -j2
-    - name: Test all
-      run: MICROPY_CPYTHON3=python3.8 MICROPY_MICROPYTHON=../ports/unix/micropython-coverage ./run-tests.py -j1
-      working-directory: tests
-    - name: Print failure info
-      run: MICROPY_CPYTHON3=python3.8 MICROPY_MICROPYTHON=../ports/unix/micropython-coverage ./run-tests.py -j1 --print-failures
-      if: failure()
-      working-directory: tests
-    - name: Native Tests
-      run: MICROPY_CPYTHON3=python3.8 MICROPY_MICROPYTHON=../ports/unix/micropython-coverage ./run-tests.py -j1 --emit native
-      working-directory: tests
-    - name: mpy Tests
-      run: MICROPY_CPYTHON3=python3.8 MICROPY_MICROPYTHON=../ports/unix/micropython-coverage ./run-tests.py -j1 --via-mpy -d basics float micropython
-      working-directory: tests
-    - name: Native mpy Tests
-      run: MICROPY_CPYTHON3=python3.8 MICROPY_MICROPYTHON=../ports/unix/micropython-coverage ./run-tests.py -j1 --via-mpy --emit native -d basics float micropython
-      working-directory: tests
-    - name: Build native modules
-      run: |
-        make -C examples/natmod/features1
-        make -C examples/natmod/features2
-        make -C examples/natmod/btree
-        make -C examples/natmod/framebuf
-        make -C examples/natmod/uheapq
-        make -C examples/natmod/urandom
-        make -C examples/natmod/ure
-        make -C examples/natmod/uzlib
-    - name: Test native modules
-      run: MICROPY_CPYTHON3=python3.8 MICROPY_MICROPYTHON=../ports/unix/micropython-coverage ./run-natmodtests.py extmod/{btree*,framebuf*,uheapq*,ure*,uzlib*}.py
-      working-directory: tests
-    - name: Build mpy-cross.static-aarch64
-      run: make -C mpy-cross -j2 -f Makefile.static-aarch64
-    - uses: actions/upload-artifact@v3
+    - name: Set up external
+      uses: ./.github/actions/deps/external
+    # Disabled: Needs to be updated
+    # - name: Get last commit with checks
+    #   id: get-last-commit-with-checks
+    #   if: github.event_name == 'pull_request'
+    #   working-directory: tools
+    #   run: python3 -u ci_changes_per_commit.py
+    #   env:
+    #     REPO: ${{ github.repository }}
+    #     PULL: ${{ github.event.number }}
+    #     GITHUB_TOKEN: ${{ github.token }}
+    #     EXCLUDE_COMMIT: ${{ github.event.after }}
+    - name: Set up mpy-cross
+      uses: ./.github/actions/mpy_cross
       with:
-        name: mpy-cross.static-aarch64
-        path: mpy-cross/mpy-cross.static-aarch64
-    - name: Build mpy-cross.static-raspbian
-      run: make -C mpy-cross -j2 -f Makefile.static-raspbian
-    - uses: actions/upload-artifact@v3
-      with:
-        name: mpy-cross.static-raspbian
-        path: mpy-cross/mpy-cross.static-raspbian
-    - name: Build mpy-cross.static
-      run: make -C mpy-cross -j2 -f Makefile.static
-    - uses: actions/upload-artifact@v3
-      with:
-        name: mpy-cross.static-amd64-linux
-        path: mpy-cross/mpy-cross.static
-    - name: Build mpy-cross.static-mingw
-      run: make -C mpy-cross -j2 -f Makefile.static-mingw
-    - uses: actions/upload-artifact@v3
-      with:
-        name: mpy-cross.static-x64-windows
-        path: mpy-cross/mpy-cross.static.exe
-    - name: Upload to S3
-      if: >-
-        (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') ||
-        (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))
-      env:
-        AWS_PAGER: ''
-        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-      run: |
-        pip install awscli
-        [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross/mpy-cross.static-aarch64 s3://adafruit-circuit-python/bin/mpy-cross/mpy-cross.static-aarch64-${{ env.CP_VERSION }} --no-progress --region us-east-1
-        [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross/mpy-cross.static-raspbian s3://adafruit-circuit-python/bin/mpy-cross/mpy-cross.static-raspbian-${{ env.CP_VERSION }} --no-progress --region us-east-1
-        [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross/mpy-cross.static s3://adafruit-circuit-python/bin/mpy-cross/mpy-cross.static-amd64-linux-${{ env.CP_VERSION }} --no-progress --region us-east-1
-        [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross/mpy-cross.static.exe s3://adafruit-circuit-python/bin/mpy-cross/mpy-cross.static-x64-windows-${{ env.CP_VERSION }}.exe --no-progress --region us-east-1
-    - name: Get last commit with checks
-      id: get-last-commit-with-checks
-      if: github.event_name == 'pull_request'
-      working-directory: tools
-      env:
-        REPO: ${{ github.repository }}
-        PULL: ${{ github.event.number }}
-        GITHUB_TOKEN: ${{ github.token }}
-        EXCLUDE_COMMIT: ${{ github.event.after }}
-      run: python3 -u ci_changes_per_commit.py
+        download: false
     - name: Set head sha
       if: github.event_name == 'pull_request'
       run: echo "HEAD_SHA=$(git show -s --format=%s $GITHUB_SHA | grep -o -P "(?<=Merge ).*(?= into)")" >> $GITHUB_ENV
@@ -162,17 +89,27 @@ jobs:
         CHANGED_FILES: ${{ steps.get-changes.outputs.changed_files }}
         LAST_FAILED_JOBS: ${{ steps.get-last-commit-with-checks.outputs.check_runs }}
 
+  tests:
+    needs: scheduler
+    if: needs.scheduler.outputs.build-boards == 'True'
+    uses: ./.github/workflows/run-tests.yml
+    with:
+      cp-version: ${{ needs.scheduler.outputs.cp-version }}
+
+  mpy-cross:
+    needs: scheduler
+    if: needs.scheduler.outputs.build-boards == 'True'
+    uses: ./.github/workflows/build-mpy-cross.yml
+    secrets: inherit
+    with:
+      cp-version: ${{ needs.scheduler.outputs.cp-version }}
 
   mpy-cross-mac:
     runs-on: macos-11
-    needs: test
-    if: >-
-      needs.test.outputs.boards-aarch != '[]' ||
-      needs.test.outputs.boards-arm != '[]' ||
-      needs.test.outputs.boards-espressif != '[]' ||
-      needs.test.outputs.boards-riscv != '[]'
+    needs: scheduler
+    if: ${{ needs.scheduler.outputs.build-boards == 'True' }}
     env:
-      CP_VERSION: ${{ needs.test.outputs.cp-version }}
+      CP_VERSION: ${{ needs.scheduler.outputs.cp-version }}
     steps:
     - name: Set up repository
       uses: actions/checkout@v3
@@ -182,9 +119,9 @@ jobs:
     - name: Set up python
       uses: actions/setup-python@v4
       with:
-        python-version: "3.x"
+        python-version: 3.x
     - name: Set up submodules
-      uses: ./.github/actions/fetch_submodules
+      uses: ./.github/actions/deps/submodules
     - name: Versions
       run: |
         gcc --version
@@ -204,28 +141,30 @@ jobs:
         path: mpy-cross/mpy-cross-arm64
     - name: Make universal binary
       run: lipo -create -output mpy-cross-macos-universal mpy-cross/mpy-cross mpy-cross/mpy-cross-arm64
-    - uses: actions/upload-artifact@v3
+    - name: Upload artifact
+      uses: actions/upload-artifact@v3
       with:
         name: mpy-cross-macos-11-universal
         path: mpy-cross-macos-universal
-    - name: Upload mpy-cross build to S3
+    - name: Upload to S3
+      if: >-
+          (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') ||
+          (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))
       run: |
-        [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross-macos-universal s3://adafruit-circuit-python/bin/mpy-cross/mpy-cross-macos-11-${{ env.CP_VERSION }}-universal --no-progress --region us-east-1
-        [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross/mpy-cross-arm64 s3://adafruit-circuit-python/bin/mpy-cross/mpy-cross-macos-11-${{ env.CP_VERSION }}-arm64 --no-progress --region us-east-1
-        [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross/mpy-cross s3://adafruit-circuit-python/bin/mpy-cross/mpy-cross-macos-11-${{ env.CP_VERSION }}-x64 --no-progress --region us-east-1
+        [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross-macos-universal s3://adafruit-circuit-python/bin/mpy-cross/macos-11/mpy-cross-macos-11-${{ env.CP_VERSION }}-universal --no-progress --region us-east-1
+        [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross/mpy-cross-arm64 s3://adafruit-circuit-python/bin/mpy-cross/macos-11/mpy-cross-macos-11-${{ env.CP_VERSION }}-arm64 --no-progress --region us-east-1
+        [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross/mpy-cross s3://adafruit-circuit-python/bin/mpy-cross/macos-11/mpy-cross-macos-11-${{ env.CP_VERSION }}-x64 --no-progress --region us-east-1
       env:
         AWS_PAGER: ''
         AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
         AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-      if: (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') || (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))
-
 
   build-doc:
     runs-on: ubuntu-22.04
-    needs: test
-    if: ${{ needs.test.outputs.build-doc == 'True' }}
+    needs: scheduler
+    if: ${{ needs.scheduler.outputs.build-doc == 'True' }}
     env:
-      CP_VERSION: ${{ needs.test.outputs.cp-version }}
+      CP_VERSION: ${{ needs.scheduler.outputs.cp-version }}
     steps:
     - name: Set up repository
       uses: actions/checkout@v3
@@ -235,14 +174,12 @@ jobs:
     - name: Set up python
       uses: actions/setup-python@v4
       with:
-        python-version: "3.x"
+        python-version: 3.x
     - name: Set up submodules
-      uses: ./.github/actions/fetch_submodules
+      uses: ./.github/actions/deps/submodules
     - name: Install dependencies
       run: |
-        sudo apt-get update
-        sudo apt-get install -y eatmydata
-        sudo eatmydata apt-get install -y latexmk librsvg2-bin texlive-fonts-recommended texlive-latex-recommended texlive-latex-extra
+        sudo apt-get install -y latexmk librsvg2-bin texlive-fonts-recommended texlive-latex-recommended texlive-latex-extra
         pip install -r requirements-doc.txt
     - name: Build and Validate Stubs
       run: make check-stubs -j2
@@ -263,18 +200,18 @@ jobs:
       with:
         name: docs
         path: _build/latex
-    - name: Upload to S3
+    - name: Zip stubs
       if: >-
         (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') ||
         (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))
-      env:
-        AWS_PAGER: ''
+      run: zip -9r circuitpython-stubs.zip circuitpython-stubs
+    - name: Upload to S3
+      uses: ./.github/actions/upload_aws
+      with:
+        source: circuitpython-stubs/dist/*.tar.gz
+        destination: stubs/circuitpython-stubs-${{ env.CP_VERSION }}.zip
         AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
         AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-      run: |
-        pip install awscli
-        zip -9r circuitpython-stubs.zip circuitpython-stubs
-        [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp circuitpython-stubs/dist/*.tar.gz s3://adafruit-circuit-python/bin/stubs/circuitpython-stubs-${{ env.CP_VERSION }}.zip --no-progress --region us-east-1
     - name: Upload stubs to PyPi
       if: github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested')
       env:
@@ -285,295 +222,52 @@ jobs:
         [ -z "$TWINE_USERNAME" ] || echo "Uploading dev release to PyPi"
         [ -z "$TWINE_USERNAME" ] || twine upload circuitpython-stubs/dist/*
 
+  aarch:
+    needs: [scheduler, mpy-cross, tests]
+    if: ${{ needs.scheduler.outputs.boards-aarch != '[]' }}
+    uses: ./.github/workflows/build-boards.yml
+    secrets: inherit
+    with:
+      platform: aarch
+      boards: ${{ needs.scheduler.outputs.boards-aarch }}
+      cp-version: ${{ needs.scheduler.outputs.cp-version }}
 
-  build-aarch:
-    runs-on: ubuntu-22.04
-    needs: test
-    if: ${{ needs.test.outputs.boards-aarch != '[]' }}
-    env:
-      CP_VERSION: ${{ needs.test.outputs.cp-version }}
-    strategy:
-      fail-fast: false
-      matrix:
-        board: ${{ fromJSON(needs.test.outputs.boards-aarch) }}
-    steps:
-    - name: Set up repository
-      uses: actions/checkout@v3
+  arm:
+    needs: [scheduler, mpy-cross, tests]
+    if: ${{ needs.scheduler.outputs.boards-arm != '[]' }}
+    uses: ./.github/workflows/build-boards.yml
+    secrets: inherit
     with:
-        submodules: false
-        fetch-depth: 1
-    - name: Set up python
-      uses: actions/setup-python@v4
-      with:
-        python-version: "3.x"
-    - name: Set up submodules
-      id: set-up-submodules
-      uses: ./.github/actions/fetch_submodules
-    - name: Install dependencies
-      run: |
-        sudo apt-get install -y gettext mtools
-        pip install -r requirements-dev.txt
-        wget --no-verbose https://adafruit-circuit-python.s3.amazonaws.com/gcc-arm-10.3-2021.07-x86_64-aarch64-none-elf.tar.xz
-        sudo tar -C /usr --strip-components=1 -xaf gcc-arm-10.3-2021.07-x86_64-aarch64-none-elf.tar.xz
-    - uses: carlosperate/arm-none-eabi-gcc-action@v1
-      with:
-        release: '10-2020-q4'
-    - name: Install mkfs.fat
-      run: |
-        wget https://github.com/dosfstools/dosfstools/releases/download/v4.2/dosfstools-4.2.tar.gz
-        tar -xaf dosfstools-4.2.tar.gz
-        cd dosfstools-4.2
-        ./configure
-        make -j 2
-        cd src
-        echo >>$GITHUB_PATH $(pwd)
-    - name: Versions
-      run: |
-        gcc --version
-        aarch64-none-elf-gcc --version
-        arm-none-eabi-gcc --version
-        python3 --version
-        mkfs.fat --version || true
-    - name: Build mpy-cross
-      if: ${{ steps.set-up-submodules.outputs.frozen == 'True' }}
-      run: make -C mpy-cross -j2
-    - name: Setup build failure matcher
-      run: echo "::add-matcher::$GITHUB_WORKSPACE/.github/workflows/match-build-fail.json"
-    - name: Build
-      run: python3 -u build_release_files.py
-      working-directory: tools
-      env:
-        BOARDS: ${{ matrix.board }}
-    - uses: actions/upload-artifact@v3
-      with:
-        name: ${{ matrix.board }}
-        path: bin/${{ matrix.board }}
-    - name: Upload to S3
-      if: >-
-        (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') ||
-        (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))
-      run: |
-        pip install awscli
-        [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp bin/ s3://adafruit-circuit-python/bin/ --recursive --no-progress --region us-east-1
-      env:
-        AWS_PAGER: ''
-        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      platform: arm
+      boards: ${{ needs.scheduler.outputs.boards-arm }}
+      cp-version: ${{ needs.scheduler.outputs.cp-version }}
 
+  esp:
+    needs: [scheduler, mpy-cross, tests]
+    if: ${{ needs.scheduler.outputs.boards-esp != '[]' }}
+    uses: ./.github/workflows/build-boards.yml
+    secrets: inherit
+    with:
+      platform: esp
+      boards: ${{ needs.scheduler.outputs.boards-esp }}
+      cp-version: ${{ needs.scheduler.outputs.cp-version }}
 
-  build-arm:
-    runs-on: ubuntu-22.04
-    needs: test
-    if: ${{ needs.test.outputs.boards-arm != '[]' }}
-    env:
-      CP_VERSION: ${{ needs.test.outputs.cp-version }}
-    strategy:
-      fail-fast: false
-      matrix:
-        board: ${{ fromJSON(needs.test.outputs.boards-arm) }}
-    steps:
-    - name: Set up repository
-      uses: actions/checkout@v3
+  riscv:
+    needs: [scheduler, mpy-cross, tests]
+    if: ${{ needs.scheduler.outputs.boards-riscv != '[]' }}
+    uses: ./.github/workflows/build-boards.yml
+    secrets: inherit
     with:
-        submodules: false
-        fetch-depth: 1
-    - name: Set up python
-      uses: actions/setup-python@v4
-      with:
-        python-version: "3.x"
-    - name: Set up submodules
-      id: set-up-submodules
-      uses: ./.github/actions/fetch_submodules
-    - uses: carlosperate/arm-none-eabi-gcc-action@v1
-      with:
-        release: '10-2020-q4'
-    - name: Install dependencies
-      run: |
-        sudo apt-get install -y gettext
-        pip install -r requirements-dev.txt
-    - name: Versions
-      run: |
-        gcc --version
-        arm-none-eabi-gcc --version
-        python3 --version
-    - name: Build mpy-cross
-      if: ${{ steps.set-up-submodules.outputs.frozen == 'True' }}
-      run: make -C mpy-cross -j2
-    - name: Setup build failure matcher
-      run: echo "::add-matcher::$GITHUB_WORKSPACE/.github/workflows/match-build-fail.json"
-    - name: Build
-      run: python3 -u build_release_files.py
-      working-directory: tools
-      env:
-        BOARDS: ${{ matrix.board }}
-    - uses: actions/upload-artifact@v3
-      with:
-        name: ${{ matrix.board }}
-        path: bin/${{ matrix.board }}
-    - name: Upload to S3
-      if: >-
-        (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') ||
-        (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))
-      run: |
-        pip install awscli
-        [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp bin/ s3://adafruit-circuit-python/bin/ --recursive --no-progress --region us-east-1
-      env:
-        AWS_PAGER: ''
-        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      platform: riscv
+      boards: ${{ needs.scheduler.outputs.boards-riscv }}
+      cp-version: ${{ needs.scheduler.outputs.cp-version }}
 
-
-  build-espressif:
-    runs-on: ubuntu-22.04
-    needs: test
-    if: ${{ needs.test.outputs.boards-espressif != '[]' }}
-    env:
-      CP_VERSION: ${{ needs.test.outputs.cp-version }}
-      IDF_PATH: ${{ github.workspace }}/ports/espressif/esp-idf
-      IDF_TOOLS_PATH: ${{ github.workspace }}/.idf_tools
-    strategy:
-      fail-fast: false
-      matrix:
-        board: ${{ fromJSON(needs.test.outputs.boards-espressif) }}
-    steps:
-    - name: Set up repository
-      uses: actions/checkout@v3
+  rpi:
+    needs: [scheduler, mpy-cross, tests]
+    if: ${{ needs.scheduler.outputs.boards-rpi != '[]' }}
+    uses: ./.github/workflows/build-boards.yml
+    secrets: inherit
     with:
-        submodules: false
-        fetch-depth: 1
-    - name: Set up python
-      id: setup-python
-      uses: actions/setup-python@v4
-      with:
-        python-version: "3.10"
-    - name: Get IDF commit
-      id: idf-commit
-      run: |
-        COMMIT=$(git submodule status ports/espressif/esp-idf | grep -o -P '(?<=^-).*(?= )')
-        echo "$COMMIT"
-        echo "commit=$COMMIT" >> $GITHUB_OUTPUT
-    - name: Cache IDF submodules
-      uses: actions/cache@v3
-      with:
-        path: |
-          .git/modules/ports/espressif/esp-idf
-          ports/espressif/esp-idf
-        key: submodules-idf-${{ steps.idf-commit.outputs.commit }}
-    - name: Cache IDF tools
-      uses: actions/cache@v3
-      with:
-        path: ${{ env.IDF_TOOLS_PATH }}
-        key: ${{ runner.os }}-Python-${{ steps.setup-python.outputs.python-version }}-tools-idf-${{ steps.idf-commit.outputs.commit }}
-    - name: Initialize IDF submodules
-      run: git submodule update --init --depth=1 --recursive $IDF_PATH
-    - name: Install IDF tools
-      run: |
-        echo "Installing ESP-IDF tools"
-        $IDF_PATH/tools/idf_tools.py --non-interactive install required
-        $IDF_PATH/tools/idf_tools.py --non-interactive install cmake
-        echo "Installing Python environment and packages"
-        $IDF_PATH/tools/idf_tools.py --non-interactive install-python-env
-        rm -rf $IDF_TOOLS_PATH/dist
-    - name: Set up submodules
-      id: set-up-submodules
-      uses: ./.github/actions/fetch_submodules
-    - name: Install dependencies
-      run: |
-        source $IDF_PATH/export.sh
-        sudo apt-get install -y gettext ninja-build
-        pip install -r requirements-dev.txt
-    - name: Versions
-      run: |
-        source $IDF_PATH/export.sh
-        gcc --version
-        python3 --version
-        ninja --version
-        cmake --version
-    - name: Build mpy-cross
-      if: ${{ steps.set-up-submodules.outputs.frozen == 'True' }}
-      run: make -C mpy-cross -j2
-    - name: Setup build failure matcher
-      run: echo "::add-matcher::$GITHUB_WORKSPACE/.github/workflows/match-build-fail.json"
-    - name: Build
-      run: |
-        source $IDF_PATH/export.sh
-        python3 -u build_release_files.py
-      working-directory: tools
-      env:
-        BOARDS: ${{ matrix.board }}
-    - uses: actions/upload-artifact@v3
-      with:
-        name: ${{ matrix.board }}
-        path: bin/${{ matrix.board }}
-    - name: Upload to S3
-      if: >-
-        (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') ||
-        (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))
-      run: |
-        pip install awscli
-        [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp bin/ s3://adafruit-circuit-python/bin/ --recursive --no-progress --region us-east-1
-      env:
-        AWS_PAGER: ''
-        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-
-
-  build-riscv:
-    runs-on: ubuntu-22.04
-    needs: test
-    if: ${{ needs.test.outputs.boards-riscv != '[]' }}
-    env:
-      CP_VERSION: ${{ needs.test.outputs.cp-version }}
-    strategy:
-      fail-fast: false
-      matrix:
-        board: ${{ fromJSON(needs.test.outputs.boards-riscv) }}
-    steps:
-    - name: Set up repository
-      uses: actions/checkout@v3
-      with:
-        submodules: false
-        fetch-depth: 1
-    - name: Set up python
-      uses: actions/setup-python@v4
-      with:
-        python-version: "3.x"
-    - name: Set up submodules
-      id: set-up-submodules
-      uses: ./.github/actions/fetch_submodules
-    - name: Install dependencies
-      run: |
-        sudo apt-get install -y gettext
-        pip install -r requirements-dev.txt
-        wget https://static.dev.sifive.com/dev-tools/riscv64-unknown-elf-gcc-8.3.0-2019.08.0-x86_64-linux-centos6.tar.gz
-        sudo tar -C /usr --strip-components=1 -xaf riscv64-unknown-elf-gcc-8.3.0-2019.08.0-x86_64-linux-centos6.tar.gz
-    - name: Versions
-      run: |
-        gcc --version
-        riscv64-unknown-elf-gcc --version
-        python3 --version
-    - name: Build mpy-cross
-      if: ${{ steps.set-up-submodules.outputs.frozen == 'True' }}
-      run: make -C mpy-cross -j2
-    - name: Setup build failure matcher
-      run: echo "::add-matcher::$GITHUB_WORKSPACE/.github/workflows/match-build-fail.json"
-    - name: Build
-      run: python3 -u build_release_files.py
-      working-directory: tools
-      env:
-        BOARDS: ${{ matrix.board }}
-    - uses: actions/upload-artifact@v3
-      with:
-        name: ${{ matrix.board }}
-        path: bin/${{ matrix.board }}
-    - name: Upload to S3
-      if: >-
-        (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') ||
-        (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))
-      run: |
-        pip install awscli
-        [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp bin/ s3://adafruit-circuit-python/bin/ --recursive --no-progress --region us-east-1
-      env:
-        AWS_PAGER: ''
-        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      platform: arm
+      boards: ${{ needs.scheduler.outputs.boards-rpi }}
+      cp-version: ${{ needs.scheduler.outputs.cp-version }}

.github/workflows/create_website_pr.yml

@@ -24,13 +24,13 @@ jobs:
     - name: Set up python
       uses: actions/setup-python@v4
       with:
-        python-version: "3.x"
+        python-version: 3.x
     - name: Set up submodules
-      uses: ./.github/actions/fetch_submodules
+      uses: ./.github/actions/deps/submodules
       with:
         version: true
-    - name: Install dependencies
-      run: pip install -r requirements-dev.txt
+    - name: Set up external
+      uses: ./.github/actions/deps/external
     - name: Versions
       run: |
         gcc --version

.github/workflows/ports_windows.yml

@@ -77,7 +77,7 @@ jobs:
         fetch-depth: 1
 
     - name: Set up submodules
-      uses: ./.github/actions/fetch_submodules
+      uses: ./.github/actions/deps/submodules
       with:
         version: true
 

.github/workflows/pre-commit.yml

@@ -24,13 +24,13 @@ jobs:
     - name: Set up python
       uses: actions/setup-python@v4
       with:
-        python-version: "3.x"
-    - name: CircuitPython dependencies
-      run: python tools/ci_fetch_deps.py ${{ github.job }}
+        python-version: 3.x
+    - name: Set up submodules
+      uses: ./.github/actions/deps/submodules
+    - name: Set up external
+      uses: ./.github/actions/deps/external
     - name: Install dependencies
-      run: |
-        sudo apt-get install -y gettext uncrustify
-        pip3 install black polib pyyaml
+      run: sudo apt-get install -y gettext uncrustify
     - name: Run pre-commit
       uses: pre-commit/action@v3.0.0
     - name: Make patch

.github/workflows/run-tests.yml

@@ -0,0 +1,67 @@
+name: Run tests
+
+on:
+  workflow_call:
+    inputs:
+      cp-version:
+        required: true
+        type: string
+
+jobs:
+  run:
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        test: [all, mpy, native, native_mpy]
+    env:
+      CP_VERSION: ${{ inputs.cp-version }}
+      MICROPY_CPYTHON3: python3.8
+      MICROPY_MICROPYTHON: ../ports/unix/micropython-coverage
+      TEST_all:
+      TEST_mpy: --via-mpy -d basics float micropython
+      TEST_native: --emit native
+      TEST_native_mpy: --via-mpy --emit native -d basics float micropython
+    steps:
+    - name: Set up repository
+      uses: actions/checkout@v3
+      with:
+        submodules: false
+        fetch-depth: 1
+    - name: Set up python
+      uses: actions/setup-python@v4
+      with:
+        python-version: 3.x
+    - name: Set up submodules
+      uses: ./.github/actions/deps/submodules
+      with:
+        target: tests
+    - name: Set up external
+      if: matrix.test == 'all'
+      uses: ./.github/actions/deps/external
+    - name: Set up mpy-cross
+      uses: ./.github/actions/mpy_cross
+    - name: Build unix port
+      run: make -C ports/unix VARIANT=coverage -j2
+    - name: Run tests
+      run: ./run-tests.py -j2 ${{ env[format('TEST_{0}', matrix.test)] }}
+      working-directory: tests
+    - name: Print failure info
+      run: ./run-tests.py -j2 --print-failures
+      if: failure()
+      working-directory: tests
+    - name: Build native modules
+      if: matrix.test == 'all'
+      run: |
+        make -C examples/natmod/features1
+        make -C examples/natmod/features2
+        make -C examples/natmod/btree
+        make -C examples/natmod/framebuf
+        make -C examples/natmod/uheapq
+        make -C examples/natmod/urandom
+        make -C examples/natmod/ure
+        make -C examples/natmod/uzlib
+    - name: Test native modules
+      if: matrix.test == 'all'
+      run: ./run-natmodtests.py extmod/{btree*,framebuf*,uheapq*,ure*,uzlib*}.py
+      working-directory: tests

locale/circuitpython.pot

@@ -2424,6 +2424,10 @@ msgid ""
 "You pressed the reset button during boot. Press again to exit safe mode."
 msgstr ""
 
+#: supervisor/shared/micropython.c
+msgid "[truncated due to length]"
+msgstr ""
+
 #: py/objtype.c
 msgid "__init__() should return None"
 msgstr ""

main.c

@@ -960,6 +960,9 @@ int __attribute__((used)) main(void) {
     common_hal_nvm_bytearray_set_bytes(&common_hal_mcu_nvm_obj,0,&value_out,1);
     #endif
 
+    // Start the debug serial
+    serial_early_init();
+
     // Wait briefly to give a reset window where we'll enter safe mode after the reset.
     if (safe_mode == NO_SAFE_MODE) {
         safe_mode = wait_for_safe_mode_reset();
@@ -976,9 +979,6 @@ int __attribute__((used)) main(void) {
     supervisor_bluetooth_init();
     #endif
 
-    // Start the debug serial
-    serial_early_init();
-
     #if !INTERNAL_FLASH_FILESYSTEM
     // Set up anything that might need to get done before we try to use SPI flash
     // This is needed for some boards where flash relies on GPIO setup to work

ports/broadcom/common-hal/neopixel_write/__init__.c

@@ -45,7 +45,10 @@ void common_hal_neopixel_write(const digitalio_digitalinout_obj_t *digitalinout,
     uint8_t *pixels, uint32_t num_bytes) {
     // Wait to make sure we don't append onto the last transmission. This should only be a tick or
     // two.
-    while (port_get_raw_ticks(NULL) < next_start_raw_ticks) {
+    int icnt;
+    while ((port_get_raw_ticks(NULL) < next_start_raw_ticks) &&
+           (next_start_raw_ticks - port_get_raw_ticks(NULL) < 100)) {
+        RUN_BACKGROUND_TASKS;
     }
 
     BP_Function_Enum alt_function = GPIO_FUNCTION_OUTPUT;
@@ -92,7 +95,8 @@ void common_hal_neopixel_write(const digitalio_digitalinout_obj_t *digitalinout,
 
         // Wait for the clock to start up.
         COMPLETE_MEMORY_READS;
-        while (CM_PWM->CS_b.BUSY == 0) {
+        icnt = 0;
+        while ((CM_PWM->CS_b.BUSY == 0) && (icnt++ < 1000)) {
         }
     }
 
@@ -134,24 +138,45 @@ void common_hal_neopixel_write(const digitalio_digitalinout_obj_t *digitalinout,
                 expanded |= 0x80000000;
             }
         }
-        while (pwm->STA_b.FULL1 == 1) {
+        if (channel == 1) {
+            icnt = 0;
+            while ((pwm->STA_b.FULL1 == 1) && (icnt++ < 150)) {
                 RUN_BACKGROUND_TASKS;
             }
-        if (channel == 1) {
             // Dummy value for the first channel.
             pwm->FIF1 = 0x000000;
         }
+        icnt = 0;
+        while ((pwm->STA_b.FULL1 == 1) && (icnt++ < 150)) {
+            RUN_BACKGROUND_TASKS;
+        }
         pwm->FIF1 = expanded;
         if (channel == 0) {
+            icnt = 0;
+            while ((pwm->STA_b.FULL1 == 1) && (icnt++ < 150)) {
+                RUN_BACKGROUND_TASKS;
+            }
             // Dummy value for the second channel.
             pwm->FIF1 = 0x000000;
         }
     }
-    // Wait just a little bit so that transmission can start.
-    common_hal_mcu_delay_us(2);
-    while (pwm->STA_b.STA1 == 1) {
+
+    icnt = 0;
+    while ((pwm->STA_b.EMPT1 == 0) && (icnt++ < 2500)) {
         RUN_BACKGROUND_TASKS;
     }
+    // Wait for transmission to start.
+    icnt = 0;
+    while (((pwm->STA_b.STA1 == 0) && (pwm->STA_b.STA2 == 0)) && (icnt++ < 150)) {
+        RUN_BACKGROUND_TASKS;
+    }
+    // Wait for transmission to complete.
+    icnt = 0;
+    while (((pwm->STA_b.STA1 == 1) || (pwm->STA_b.STA2 == 1)) && (icnt++ < 150)) {
+        RUN_BACKGROUND_TASKS;
+    }
+    // Shouldn't be anything left in queue but clear it so the clock doesn't crash if there is
+    pwm->CTL = PWM0_CTL_CLRF1_Msk;
 
     gpio_set_function(digitalinout->pin->number, GPIO_FUNCTION_OUTPUT);
 

ports/cxd56/README.md

@@ -75,7 +75,7 @@ Bootloader information:
 
 * You have to accept the End User License Agreement to be able to download and use the Spresense bootloader binary.
 
-Download the spresense binaries zip archive from: [Spresense firmware v2-3-000](https://developer.sony.com/file/download/download-spresense-firmware-v2-3-000)
+Download the spresense binaries zip archive from: [Spresense firmware v2-4-000](https://developer.sony.com/file/download/download-spresense-firmware-v2-4-000)
 
 Extract spresense binaries in your PC to ports/spresense/spresense-exported-sdk/firmware/
 

ports/cxd56/common-hal/camera/Camera.c

@@ -47,7 +47,7 @@ typedef struct {
     uint16_t height;
 } image_size_t;
 
-STATIC const image_size_t image_size_table[] = {
+STATIC const image_size_t isx012_image_size_table[] = {
     { VIDEO_HSIZE_QVGA, VIDEO_VSIZE_QVGA },
     { VIDEO_HSIZE_VGA, VIDEO_VSIZE_VGA },
     { VIDEO_HSIZE_HD, VIDEO_VSIZE_HD },
@@ -57,12 +57,40 @@ STATIC const image_size_t image_size_table[] = {
     { VIDEO_HSIZE_5M, VIDEO_VSIZE_5M },
 };
 
+STATIC const image_size_t isx019_image_size_table[] = {
+    { VIDEO_HSIZE_QVGA, VIDEO_VSIZE_QVGA },
+    { VIDEO_HSIZE_VGA, VIDEO_VSIZE_VGA },
+    { VIDEO_HSIZE_HD, VIDEO_VSIZE_HD },
+    { VIDEO_HSIZE_QUADVGA, VIDEO_VSIZE_QUADVGA },
+};
+
+static const char *get_imgsensor_name() {
+    static struct v4l2_capability cap;
+
+    ioctl(camera_dev.fd, VIDIOC_QUERYCAP, (unsigned long)&cap);
+
+    return (const char *)cap.driver;
+}
+
 static bool camera_check_width_and_height(uint16_t width, uint16_t height) {
-    for (int i = 0; i < MP_ARRAY_SIZE(image_size_table); i++) {
-        if (image_size_table[i].width == width && image_size_table[i].height == height) {
+    const char *sensor;
+
+    sensor = get_imgsensor_name();
+
+    if (strncmp(sensor, "ISX012", strlen("ISX012")) == 0) {
+        for (int i = 0; i < MP_ARRAY_SIZE(isx012_image_size_table); i++) {
+            if (isx012_image_size_table[i].width == width && isx012_image_size_table[i].height == height) {
                 return true;
             }
         }
+    } else if (strncmp(sensor, "ISX019", strlen("ISX019"))) {
+        for (int i = 0; i < MP_ARRAY_SIZE(isx019_image_size_table); i++) {
+            if (isx019_image_size_table[i].width == width && isx019_image_size_table[i].height == height) {
+                return true;
+            }
+        }
+    }
+
     return false;
 }
 

ports/cxd56/configs/circuitpython/defconfig

@@ -113,4 +113,5 @@ CONFIG_USEC_PER_TICK=1000
 CONFIG_USERMAIN_STACKSIZE=8192
 CONFIG_USER_ENTRYPOINT="spresense_main"
 CONFIG_VIDEO_ISX012=y
+CONFIG_VIDEO_ISX019=y
 CONFIG_VIDEO_STREAM=y

ports/cxd56/spresense-exported-sdk

@@ -1 +1 @@
-Subproject commit 6a148be8497704d4afb5d14c175a12a592813fac
+Subproject commit 4f902ca3ffeb327e6c325940ef5133eda588c2e4

ports/cxd56/supervisor/port.c

@@ -111,13 +111,13 @@ bool port_has_fixed_stack(void) {
 uint32_t *port_stack_get_limit(void) {
     struct tcb_s *rtcb = this_task();
 
-    return rtcb->adj_stack_ptr - (uint32_t)rtcb->adj_stack_size;
+    return rtcb->stack_base_ptr;
 }
 
 uint32_t *port_stack_get_top(void) {
     struct tcb_s *rtcb = this_task();
 
-    return rtcb->adj_stack_ptr;
+    return rtcb->stack_base_ptr + (uint32_t)rtcb->adj_stack_size;
 }
 
 uint32_t *port_heap_get_bottom(void) {

ports/raspberrypi/boards/hack_club_sprig/board.c

@@ -0,0 +1,128 @@
+/*
+ * This file is part of the MicroPython project, http://micropython.org/
+ *
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2023 ajs256
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "supervisor/board.h"
+#include "mpconfigboard.h"
+#include "shared-bindings/busio/SPI.h"
+#include "shared-bindings/displayio/FourWire.h"
+#include "shared-module/displayio/__init__.h"
+#include "shared-module/displayio/mipi_constants.h"
+#include "supervisor/shared/board.h"
+
+displayio_fourwire_obj_t board_display_obj;
+
+// display init sequence from CircuitPython library https://github.com/adafruit/Adafruit_CircuitPython_ST7735R/blob/dfae353330cf051d1f31db9e4b681c8d70900cc5/adafruit_st7735r.py
+uint8_t display_init_sequence[] = {
+    // sw reset
+    0x01, 0x80, 0x96,
+    // sleep out and delay
+    0x11, 0x80, 0xFF,
+    // _FRMCTR1
+    0xB1, 0x03, 0x01, 0x2C, 0x2D,
+    // _FRMCTR2
+    0xB2, 0x03, 0x01, 0x2C, 0x2D,
+    // _FRMCTR3
+    0xB3, 0x06, 0x01, 0x2C, 0x2D, 0x01, 0x2C, 0x2D,
+    // _INVCTR line inversion
+    0xB4, 0x01, 0x07,
+    // _PWCTR1 GVDD = 4.7V, 1.0uA
+    0xC0, 0x03, 0xA2, 0x02, 0x84,
+    // _PWCTR2 VGH=14.7V, VGL=-7.35V
+    0xC1, 0x01, 0xC5,
+    // _PWCTR3 Opamp current small, Boost frequency
+    0xC2, 0x02, 0x0A, 0x00,
+    0xC3, 0x02, 0x8A, 0x2A,
+    0xC4, 0x02, 0x8A, 0xEE,
+    // _VMCTR1 VCOMH = 4V, VOML = -1.1V
+    0xC5, 0x01, 0x0E,
+    // _INVOFF
+    0x20, 0x00,
+    // _MADCTL bottom to top refresh
+    0x36, 0x01, 0x18,
+    // COLMOD - 16 bit color
+    0x3A, 0x01, 0x05,
+    // _GMCTRP1 Gamma
+    0xE0, 0x10, 0x02, 0x1C, 0x07, 0x12, 0x37, 0x32, 0x29, 0x2D, 0x29, 0x25, 0x2B, 0x39, 0x00, 0x01, 0x03, 0x10,
+    // _GMCTRN1
+    0xE1, 0x10, 0x03, 0x1d, 0x07, 0x06, 0x2E, 0x2C, 0x29, 0x2D, 0x2E, 0x2E, 0x37, 0x3F, 0x00, 0x00, 0x02, 0x10,
+    // _NORON
+    0x13, 0x80, 0x0A,
+    // _DISPON
+    0x29, 0x80, 0x64,
+    // _MADCTL Default rotation + BGR encoding
+    0x36, 0x01, 0xC0,
+};
+
+
+void board_init(void) {
+    busio_spi_obj_t *spi = &displays[0].fourwire_bus.inline_bus;
+    common_hal_busio_spi_construct(spi, &pin_GPIO18, &pin_GPIO19, &pin_GPIO16, false);
+    common_hal_busio_spi_never_reset(spi);
+
+    displayio_fourwire_obj_t *bus = &displays[0].fourwire_bus;
+    bus->base.type = &displayio_fourwire_type;
+    common_hal_displayio_fourwire_construct(bus,
+        spi,
+        &pin_GPIO22, // DC
+        &pin_GPIO20, // CS
+        &pin_GPIO26, // RST
+        30000000,
+        0,
+        0);
+
+    displayio_display_obj_t *display = &displays[0].display;
+    display->base.type = &displayio_display_type;
+    common_hal_displayio_display_construct(display,
+        bus,
+        160, // Width
+        128, // Height
+        0, // column start
+        0, // row start
+        270, // rotation
+        16, // Color depth
+        false, // Grayscale
+        false, // pixels in a byte share a row. Only valid for depths < 8
+        1, // bytes per cell. Only valid for depths < 8
+        false, // reverse_pixels_in_byte. Only valid for depths < 8
+        true, // reverse_bytes_in_word
+        MIPI_COMMAND_SET_COLUMN_ADDRESS, // Set column command
+        MIPI_COMMAND_SET_PAGE_ADDRESS, // Set row command
+        MIPI_COMMAND_WRITE_MEMORY_START, // Write memory command
+        display_init_sequence,
+        sizeof(display_init_sequence),
+        &pin_GPIO17,  // backlight pin
+        NO_BRIGHTNESS_COMMAND,
+        1.0f, // brightness
+        false, // single_byte_bounds
+        false, // data_as_commands
+        true, // auto_refresh
+        60, // native_frames_per_second
+        true, // backlight_on_high
+        false, // SH1107_addressing
+        50000); // backlight pwm frequency
+}
+
+// Use the MP_WEAK supervisor/shared/board.c versions of routines not defined here.

ports/raspberrypi/boards/hack_club_sprig/mpconfigboard.h

@@ -0,0 +1,4 @@
+#define MICROPY_HW_BOARD_NAME "Hack Club Sprig"
+#define MICROPY_HW_MCU_NAME "rp2040"
+
+#define MICROPY_HW_LED_STATUS (&pin_GPIO4)

ports/raspberrypi/boards/hack_club_sprig/mpconfigboard.mk

@@ -0,0 +1,11 @@
+USB_VID = 0x1209
+USB_PID = 0x9000
+USB_PRODUCT = "Sprig"
+USB_MANUFACTURER = "Hack Club"
+
+CHIP_VARIANT = RP2040
+CHIP_FAMILY = rp2
+
+EXTERNAL_FLASH_DEVICES = "W25Q16JVxQ"
+
+CIRCUITPY__EVE = 1

ports/raspberrypi/boards/hack_club_sprig/pico-sdk-configboard.h

@@ -0,0 +1 @@
+// Put board-specific pico-sdk definitions here. This file must exist.

ports/raspberrypi/boards/hack_club_sprig/pins.c

@@ -0,0 +1,85 @@
+#include "shared-bindings/board/__init__.h"
+
+STATIC const mp_rom_map_elem_t board_module_globals_table[] = {
+    CIRCUITPYTHON_BOARD_DICT_STANDARD_ITEMS
+
+    { MP_ROM_QSTR(MP_QSTR_GP0), MP_ROM_PTR(&pin_GPIO0) },
+    { MP_ROM_QSTR(MP_QSTR_GP1), MP_ROM_PTR(&pin_GPIO1) },
+    { MP_ROM_QSTR(MP_QSTR_GP2), MP_ROM_PTR(&pin_GPIO2) },
+    { MP_ROM_QSTR(MP_QSTR_GP3), MP_ROM_PTR(&pin_GPIO3) },
+    { MP_ROM_QSTR(MP_QSTR_GP4), MP_ROM_PTR(&pin_GPIO4) },
+    { MP_ROM_QSTR(MP_QSTR_GP5), MP_ROM_PTR(&pin_GPIO5) },
+    { MP_ROM_QSTR(MP_QSTR_GP6), MP_ROM_PTR(&pin_GPIO6) },
+    { MP_ROM_QSTR(MP_QSTR_GP7), MP_ROM_PTR(&pin_GPIO7) },
+    { MP_ROM_QSTR(MP_QSTR_GP8), MP_ROM_PTR(&pin_GPIO8) },
+    { MP_ROM_QSTR(MP_QSTR_GP9), MP_ROM_PTR(&pin_GPIO9) },
+    { MP_ROM_QSTR(MP_QSTR_GP10), MP_ROM_PTR(&pin_GPIO10) },
+    { MP_ROM_QSTR(MP_QSTR_GP11), MP_ROM_PTR(&pin_GPIO11) },
+    { MP_ROM_QSTR(MP_QSTR_GP12), MP_ROM_PTR(&pin_GPIO12) },
+    { MP_ROM_QSTR(MP_QSTR_GP13), MP_ROM_PTR(&pin_GPIO13) },
+    { MP_ROM_QSTR(MP_QSTR_GP14), MP_ROM_PTR(&pin_GPIO14) },
+    { MP_ROM_QSTR(MP_QSTR_GP15), MP_ROM_PTR(&pin_GPIO15) },
+    { MP_ROM_QSTR(MP_QSTR_GP16), MP_ROM_PTR(&pin_GPIO16) },
+    { MP_ROM_QSTR(MP_QSTR_GP17), MP_ROM_PTR(&pin_GPIO17) },
+    { MP_ROM_QSTR(MP_QSTR_GP18), MP_ROM_PTR(&pin_GPIO18) },
+    { MP_ROM_QSTR(MP_QSTR_GP19), MP_ROM_PTR(&pin_GPIO19) },
+    { MP_ROM_QSTR(MP_QSTR_GP20), MP_ROM_PTR(&pin_GPIO20) },
+    { MP_ROM_QSTR(MP_QSTR_GP21), MP_ROM_PTR(&pin_GPIO21) },
+    { MP_ROM_QSTR(MP_QSTR_GP22), MP_ROM_PTR(&pin_GPIO22) },
+
+    { MP_ROM_QSTR(MP_QSTR_SMPS_MODE), MP_ROM_PTR(&pin_GPIO23) },
+    { MP_ROM_QSTR(MP_QSTR_GP23), MP_ROM_PTR(&pin_GPIO23) },
+
+    { MP_ROM_QSTR(MP_QSTR_VBUS_SENSE), MP_ROM_PTR(&pin_GPIO24) },
+    { MP_ROM_QSTR(MP_QSTR_GP24), MP_ROM_PTR(&pin_GPIO24) },
+
+    { MP_ROM_QSTR(MP_QSTR_LED), MP_ROM_PTR(&pin_GPIO25) },
+    { MP_ROM_QSTR(MP_QSTR_GP25), MP_ROM_PTR(&pin_GPIO25) },
+
+    { MP_ROM_QSTR(MP_QSTR_GP26_A0), MP_ROM_PTR(&pin_GPIO26) },
+    { MP_ROM_QSTR(MP_QSTR_GP26), MP_ROM_PTR(&pin_GPIO26) },
+    { MP_ROM_QSTR(MP_QSTR_A0), MP_ROM_PTR(&pin_GPIO26) },
+
+    { MP_ROM_QSTR(MP_QSTR_GP27_A1), MP_ROM_PTR(&pin_GPIO27) },
+    { MP_ROM_QSTR(MP_QSTR_GP27), MP_ROM_PTR(&pin_GPIO27) },
+    { MP_ROM_QSTR(MP_QSTR_A1), MP_ROM_PTR(&pin_GPIO27) },
+
+    { MP_ROM_QSTR(MP_QSTR_GP28_A2), MP_ROM_PTR(&pin_GPIO28) },
+    { MP_ROM_QSTR(MP_QSTR_GP28), MP_ROM_PTR(&pin_GPIO28) },
+    { MP_ROM_QSTR(MP_QSTR_A2), MP_ROM_PTR(&pin_GPIO28) },
+
+    { MP_ROM_QSTR(MP_QSTR_A3), MP_ROM_PTR(&pin_GPIO29) },
+    { MP_ROM_QSTR(MP_QSTR_VOLTAGE_MONITOR), MP_ROM_PTR(&pin_GPIO29) },
+
+
+    // Start Sprig-specific definitions
+
+    { MP_ROM_QSTR(MP_QSTR_BLUE_LED), MP_ROM_PTR(&pin_GPIO4) },
+
+    { MP_ROM_QSTR(MP_QSTR_BUTTON_W), MP_ROM_PTR(&pin_GPIO5) },
+    { MP_ROM_QSTR(MP_QSTR_BUTTON_A), MP_ROM_PTR(&pin_GPIO6) },
+    { MP_ROM_QSTR(MP_QSTR_BUTTON_S), MP_ROM_PTR(&pin_GPIO7) },
+    { MP_ROM_QSTR(MP_QSTR_BUTTON_D), MP_ROM_PTR(&pin_GPIO8) },
+
+    { MP_ROM_QSTR(MP_QSTR_AUDIO_DIN), MP_ROM_PTR(&pin_GPIO9) },
+    { MP_ROM_QSTR(MP_QSTR_AUDIO_BCLK), MP_ROM_PTR(&pin_GPIO10) },
+    { MP_ROM_QSTR(MP_QSTR_AUDIO_LRCLK), MP_ROM_PTR(&pin_GPIO11) },
+
+    { MP_ROM_QSTR(MP_QSTR_BUTTON_I), MP_ROM_PTR(&pin_GPIO12) },
+    { MP_ROM_QSTR(MP_QSTR_BUTTON_J), MP_ROM_PTR(&pin_GPIO13) },
+    { MP_ROM_QSTR(MP_QSTR_BUTTON_K), MP_ROM_PTR(&pin_GPIO14) },
+    { MP_ROM_QSTR(MP_QSTR_BUTTON_L), MP_ROM_PTR(&pin_GPIO15) },
+
+    { MP_ROM_QSTR(MP_QSTR_MISO), MP_ROM_PTR(&pin_GPIO16) },
+    { MP_ROM_QSTR(MP_QSTR_TFT_LITE), MP_ROM_PTR(&pin_GPIO17) },
+    { MP_ROM_QSTR(MP_QSTR_SCK), MP_ROM_PTR(&pin_GPIO18) },
+    { MP_ROM_QSTR(MP_QSTR_MOSI), MP_ROM_PTR(&pin_GPIO19) },
+    { MP_ROM_QSTR(MP_QSTR_TFT_CS), MP_ROM_PTR(&pin_GPIO20) },
+    { MP_ROM_QSTR(MP_QSTR_CARD_CS), MP_ROM_PTR(&pin_GPIO21) },
+    { MP_ROM_QSTR(MP_QSTR_TFT_DC), MP_ROM_PTR(&pin_GPIO22) },
+    { MP_ROM_QSTR(MP_QSTR_TFT_RESET), MP_ROM_PTR(&pin_GPIO23) },
+
+    { MP_ROM_QSTR(MP_QSTR_WHITE_LED), MP_ROM_PTR(&pin_GPIO28) },
+
+};
+MP_DEFINE_CONST_DICT(board_module_globals, board_module_globals_table);

ports/raspberrypi/boards/jpconstantineau_pykey18/board.c

@@ -31,7 +31,7 @@
 
 void reset_board(void) {
     // turn off any left over LED
-    board_reset_user_neopixels(&pin_GPIO29, 62);
+    board_reset_user_neopixels(&pin_GPIO29, 19);
 }
 
 // Use the MP_WEAK supervisor/shared/board.c versions of routines not defined here.

ports/raspberrypi/boards/jpconstantineau_pykey44/board.c

@@ -31,7 +31,7 @@
 
 void reset_board(void) {
     // turn off any left over LED
-    board_reset_user_neopixels(&pin_GPIO29, 62);
+    board_reset_user_neopixels(&pin_GPIO29, 45);
 }
 
 // Use the MP_WEAK supervisor/shared/board.c versions of routines not defined here.

ports/raspberrypi/boards/jpconstantineau_pykey87/board.c

@@ -31,7 +31,7 @@
 
 void reset_board(void) {
     // turn off any left over LED
-    board_reset_user_neopixels(&pin_GPIO29, 62);
+    board_reset_user_neopixels(&pin_GPIO29, 88);
 }
 
 // Use the MP_WEAK supervisor/shared/board.c versions of routines not defined here.

ports/raspberrypi/common-hal/busio/UART.c

@@ -339,3 +339,18 @@ bool common_hal_busio_uart_ready_to_tx(busio_uart_obj_t *self) {
     }
     return uart_is_writable(self->uart);
 }
+
+STATIC void pin_never_reset(uint8_t pin) {
+    if (pin != NO_PIN) {
+        never_reset_pin_number(pin);
+    }
+}
+
+void common_hal_busio_uart_never_reset(busio_uart_obj_t *self) {
+    never_reset_uart(self->uart_id);
+    pin_never_reset(self->tx_pin);
+    pin_never_reset(self->rx_pin);
+    pin_never_reset(self->cts_pin);
+    pin_never_reset(self->rs485_dir_pin);
+    pin_never_reset(self->rts_pin);
+}

ports/raspberrypi/common-hal/rp2pio/StateMachine.c

@@ -289,6 +289,11 @@ bool rp2pio_statemachine_construct(rp2pio_statemachine_obj_t *self,
             }
             pio_gpio_init(self->pio, pin_number);
         }
+
+        // Use lowest drive level for all State Machine outputs. (#7515
+        // workaround). Remove if/when Pin objects get a drive_strength
+        // property and use that value instead.
+        gpio_set_drive_strength(pin_number, GPIO_DRIVE_STRENGTH_2MA);
     }
 
     pio_sm_config c = {0, 0, 0};

py/objexcept.c

@@ -155,6 +155,12 @@ void mp_obj_exception_print(const mp_print_t *print, mp_obj_t o_in, mp_print_kin
     mp_obj_tuple_print(print, MP_OBJ_FROM_PTR(o->args), kind);
 }
 
+void mp_obj_exception_initialize0(mp_obj_exception_t *o_exc, const mp_obj_type_t *type) {
+    o_exc->base.type = type;
+    o_exc->args = (mp_obj_tuple_t *)&mp_const_empty_tuple_obj;
+    mp_obj_exception_clear_traceback(o_exc);
+}
+
 mp_obj_t mp_obj_exception_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args) {
     mp_arg_check_num(n_args, n_kw, 0, MP_OBJ_FUN_ARGS_MAX, false);
 
@@ -583,6 +589,12 @@ void mp_obj_exception_clear_traceback(mp_obj_t self_in) {
     // just set the traceback to the empty traceback object
     // we don't want to call any memory management functions here
     self->traceback = (mp_obj_traceback_t *)&mp_const_empty_traceback_obj;
+    #if MICROPY_CPYTHON_EXCEPTION_CHAIN
+    self->cause = 0;
+    self->context = 0;
+    self->suppress_context = false;
+    self->marked = false;
+    #endif
 }
 
 void mp_obj_exception_add_traceback(mp_obj_t self_in, qstr file, size_t line, qstr block) {

py/objexcept.h

@@ -43,6 +43,7 @@ typedef struct _mp_obj_exception_t {
 
 void mp_obj_exception_print(const mp_print_t *print, mp_obj_t o_in, mp_print_kind_t kind);
 void mp_obj_exception_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest);
+void mp_obj_exception_initialize0(mp_obj_exception_t *o_exc, const mp_obj_type_t *type);
 mp_obj_exception_t *mp_obj_exception_get_native(mp_obj_t self_in);
 
 #define MP_DEFINE_EXCEPTION(exc_name, base_name) \

py/objgenerator.c

@@ -40,10 +40,11 @@
 // Instance of GeneratorExit exception - needed by generator.close()
 #if MICROPY_CONST_GENERATOREXIT_OBJ
 const
+mp_obj_exception_t mp_static_GeneratorExit_obj = {{&mp_type_GeneratorExit}, (mp_obj_tuple_t *)&mp_const_empty_tuple_obj, (mp_obj_traceback_t *)&mp_const_empty_traceback_obj};
 #else
 static
+mp_obj_exception_t mp_static_GeneratorExit_obj;
 #endif
-mp_obj_exception_t mp_static_GeneratorExit_obj = {{&mp_type_GeneratorExit}, (mp_obj_tuple_t *)&mp_const_empty_tuple_obj, (mp_obj_traceback_t *)&mp_const_empty_traceback_obj};
 
 /******************************************************************************/
 /* generator wrapper                                                          */
@@ -370,9 +371,7 @@ STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(gen_instance_throw_obj, 2, 4, gen_ins
 static mp_obj_t generatorexit(void) {
     #if MICROPY_CPYTHON_EXCEPTION_CHAIN
     MP_STATIC_ASSERT(!MICROPY_CONST_GENERATOREXIT_OBJ);
-    mp_static_GeneratorExit_obj.context = NULL;
-    mp_static_GeneratorExit_obj.cause = NULL;
-    mp_static_GeneratorExit_obj.suppress_context = false;
+    mp_obj_exception_initialize0(&mp_static_GeneratorExit_obj, &mp_type_GeneratorExit);
     #endif
     return MP_OBJ_FROM_PTR(&mp_static_GeneratorExit_obj);
 }

py/py.mk

@@ -243,7 +243,7 @@ $(HEADER_BUILD)/mpversion.h: FORCE | $(HEADER_BUILD)
 MPCONFIGPORT_MK = $(wildcard mpconfigport.mk)
 
 $(HEADER_BUILD)/$(TRANSLATION).mo: $(TOP)/locale/$(TRANSLATION).po | $(HEADER_BUILD)
-	$(Q)msgfmt -o $@ $^
+	$(Q)$(PYTHON) $(TOP)/tools/msgfmt.py -o $@ $^
 
 $(HEADER_BUILD)/qstrdefs.preprocessed.h: $(PY_QSTR_DEFS) $(QSTR_DEFS) $(QSTR_DEFS_COLLECTED) mpconfigport.h $(MPCONFIGPORT_MK) $(PY_SRC)/mpconfig.h | $(HEADER_BUILD)
 	$(STEPECHO) "GEN $@"

py/runtime.c

@@ -78,14 +78,10 @@ void mp_init(void) {
 
     #if MICROPY_KBD_EXCEPTION
     // initialise the exception object for raising KeyboardInterrupt
-    MP_STATE_VM(mp_kbd_exception).base.type = &mp_type_KeyboardInterrupt;
-    MP_STATE_VM(mp_kbd_exception).args = (mp_obj_tuple_t *)&mp_const_empty_tuple_obj;
-    MP_STATE_VM(mp_kbd_exception).traceback = (mp_obj_traceback_t *)&mp_const_empty_traceback_obj;
+    mp_obj_exception_initialize0(&MP_STATE_VM(mp_kbd_exception), &mp_type_KeyboardInterrupt);
     #endif
 
-    MP_STATE_VM(mp_reload_exception).base.type = &mp_type_ReloadException;
-    MP_STATE_VM(mp_reload_exception).args = (mp_obj_tuple_t *)&mp_const_empty_tuple_obj;
-    MP_STATE_VM(mp_reload_exception).traceback = (mp_obj_traceback_t *)&mp_const_empty_traceback_obj;
+    mp_obj_exception_initialize0(&MP_STATE_VM(mp_reload_exception), &mp_type_ReloadException);
 
     // call port specific initialization if any
     #ifdef MICROPY_PORT_INIT_FUNC

requirements-dev.txt

@@ -6,6 +6,7 @@ typer
 sh
 click
 cpp-coveralls
+
 requests
 requests-cache
 
@@ -13,8 +14,8 @@ requests-cache
 polib
 
 # For pre-commit
-pyyaml
 black
+pyyaml
 pre-commit
 
 # for combining the Nordic SoftDevice with CircuitPython
@@ -23,9 +24,6 @@ intelhex
 # for building & testing natmods
 pyelftools
 
-# for stubs and annotations
-adafruit-circuitpython-typing
-
 # for mbedtls certificate store
 cryptography
 

shared-module/displayio/I2CDisplay.c

@@ -54,6 +54,7 @@ void common_hal_displayio_i2cdisplay_construct(displayio_i2cdisplay_obj_t *self,
     // Probe the bus to see if a device acknowledges the given address.
     if (!common_hal_busio_i2c_probe(i2c, device_address)) {
         self->base.type = &mp_type_NoneType;
+        common_hal_displayio_i2cdisplay_deinit(self);
         mp_raise_ValueError_varg(translate("Unable to find I2C Display at %x"), device_address);
     }
 

supervisor/shared/micropython.c

@@ -62,8 +62,15 @@ void mp_hal_stdout_tx_strn(const char *str, size_t len) {
     #ifdef CIRCUITPY_BOOT_OUTPUT_FILE
     if (boot_output != NULL) {
         // Ensure boot_out.txt is capped at 1 filesystem block and ends with a newline
-        if (len + boot_output->len > 508) {
-            vstr_add_str(boot_output, "...\n");
+        #define TRUNCATED translate("[truncated due to length]")
+        size_t truncated_message_len = decompress_length(TRUNCATED);
+        size_t maxlen = 512 - truncated_message_len; // includes trailing '\0' so we do not need to account for trailing newline '\n' in vstr_add_byte
+        if (len + boot_output->len > maxlen) {
+            size_t remaining_len = maxlen - boot_output->len;
+            vstr_add_strn(boot_output, str, remaining_len);
+            char buf[truncated_message_len];
+            vstr_add_str(boot_output, decompress(TRUNCATED, buf));
+            vstr_add_byte(boot_output, '\n');
             boot_output = NULL;
         } else {
             vstr_add_strn(boot_output, str, len);

tools/build_board_info.py

@@ -10,6 +10,7 @@ import subprocess
 import sys
 import sh
 import base64
+from io import StringIO
 from datetime import date
 from sh.contrib import git
 
@@ -58,9 +59,13 @@ def get_languages(list_all=False):
 
 def get_version_info():
     version = None
-    sha = git("rev-parse", "--short", "HEAD").stdout.decode("utf-8")
+    buffer = StringIO()
+    git("rev-parse", "--short", "HEAD", _out=buffer)
+    sha = buffer.getvalue().strip()
     try:
-        version = git("describe", "--tags", "--exact-match").stdout.decode("utf-8").strip()
+        buffer = StringIO()
+        git("describe", "--tags", "--exact-match", _out=buffer)
+        version = buffer.getvalue().strip()
     except sh.ErrorReturnCode_128:
         # No exact match
         pass

tools/ci_fetch_deps.py

@@ -73,12 +73,15 @@ def main():
 
     print("Target:", TARGET)
 
-    if TARGET == "test":
+    if TARGET == "scheduler":
+        # submodules = ["tools/"]
+        submodules = ["extmod/ulab", "lib/", "tools/"]
+    elif TARGET == "tests":
         submodules = ["extmod/ulab", "lib/", "tools/"]
     elif TARGET == "build-doc":
         # used in .readthedocs.yml to generate RTD
         submodules = ["extmod/ulab", "frozen/"]
-    elif TARGET == "mpy-cross-mac":
+    elif TARGET == "mpy-cross" or TARGET == "mpy-cross-mac":
         submodules = ["tools/"]  # for huffman
     elif TARGET == "windows":
         # This builds one board from a number of ports so fill out a bunch of submodules

tools/ci_set_matrix.py

@@ -46,11 +46,11 @@ PORT_TO_ARCH = {
     "atmel-samd": "arm",
     "broadcom": "aarch",
     "cxd56": "arm",
-    "espressif": "espressif",
+    "espressif": "esp",
     "litex": "riscv",
     "mimxrt10xx": "arm",
     "nrf": "arm",
-    "raspberrypi": "arm",
+    "raspberrypi": "rpi",
     "stm": "arm",
 }
 
@@ -204,29 +204,33 @@ def set_boards_to_build(build_all: bool):
             break
 
     # Split boards by architecture.
-    print("Building boards:")
-    arch_to_boards = {"aarch": [], "arm": [], "riscv": [], "espressif": []}
+    arch_to_boards = {"aarch": [], "arm": [], "esp": [], "riscv": [], "rpi": []}
+
+    # Append previously failed boards
+    for arch in arch_to_boards:
+        arch_to_job = f"build-{arch}"
+        if arch_to_job in last_failed_jobs:
+            for board in last_failed_jobs[arch_to_job]:
+                if not board in boards_to_build:
+                    boards_to_build.append(board)
+
+    build_boards = bool(boards_to_build)
+    print("Building boards:", build_boards)
+    set_output("build-boards", build_boards)
+
+    # Append boards according to arch
     for board in sorted(boards_to_build):
-        print(" ", board)
         port = board_to_port.get(board)
         # A board can appear due to its _deletion_ (rare)
         # if this happens it's not in `board_to_port`.
         if not port:
             continue
-        arch = PORT_TO_ARCH[port]
-        arch_to_boards[arch].append(board)
+        arch_to_boards[PORT_TO_ARCH[port]].append(board)
+        print(" ", board)
 
     # Set the step outputs for each architecture
     for arch in arch_to_boards:
-        # Append previous failed jobs
-        if f"build-{arch}" in last_failed_jobs:
-            failed_boards = last_failed_jobs[f"build-{arch}"]
-            for board in failed_boards:
-                if not board in arch_to_boards[arch]:
-                    print(" ", board)
-                    arch_to_boards[arch].append(board)
-        # Set Output
-        set_output(f"boards-{arch}", json.dumps(sorted(arch_to_boards[arch])))
+        set_output(f"boards-{arch}", json.dumps(arch_to_boards[arch]))
 
 
 def set_docs_to_build(build_doc: bool):

tools/msgfmt.py

@@ -0,0 +1,244 @@
+#! /usr/bin/env python3
+# Written by Martin v. Löwis <loewis@informatik.hu-berlin.de>
+
+"""Generate binary message catalog from textual translation description.
+This program converts a textual Uniforum-style message catalog (.po file) into
+a binary GNU catalog (.mo file).  This is essentially the same function as the
+GNU msgfmt program, however, it is a simpler implementation.  Currently it
+does not handle plural forms but it does handle message contexts.
+Usage: msgfmt.py [OPTIONS] filename.po
+Options:
+    -o file
+    --output-file=file
+        Specify the output file to write to.  If omitted, output will go to a
+        file named filename.mo (based off the input file name).
+    -h
+    --help
+        Print this message and exit.
+    -V
+    --version
+        Display version information and exit.
+"""
+
+import os
+import sys
+import ast
+import getopt
+import struct
+import array
+from email.parser import HeaderParser
+
+__version__ = "1.2"
+
+MESSAGES = {}
+
+
+def usage(code, msg=""):
+    print(__doc__, file=sys.stderr)
+    if msg:
+        print(msg, file=sys.stderr)
+    sys.exit(code)
+
+
+def add(ctxt, id, str, fuzzy):
+    "Add a non-fuzzy translation to the dictionary."
+    global MESSAGES
+    if not fuzzy and str:
+        if ctxt is None:
+            MESSAGES[id] = str
+        else:
+            MESSAGES[b"%b\x04%b" % (ctxt, id)] = str
+
+
+def generate():
+    "Return the generated output."
+    global MESSAGES
+    # the keys are sorted in the .mo file
+    keys = sorted(MESSAGES.keys())
+    offsets = []
+    ids = strs = b""
+    for id in keys:
+        # For each string, we need size and file offset.  Each string is NUL
+        # terminated; the NUL does not count into the size.
+        offsets.append((len(ids), len(id), len(strs), len(MESSAGES[id])))
+        ids += id + b"\0"
+        strs += MESSAGES[id] + b"\0"
+    output = ""
+    # The header is 7 32-bit unsigned integers.  We don't use hash tables, so
+    # the keys start right after the index tables.
+    # translated string.
+    keystart = 7 * 4 + 16 * len(keys)
+    # and the values start after the keys
+    valuestart = keystart + len(ids)
+    koffsets = []
+    voffsets = []
+    # The string table first has the list of keys, then the list of values.
+    # Each entry has first the size of the string, then the file offset.
+    for o1, l1, o2, l2 in offsets:
+        koffsets += [l1, o1 + keystart]
+        voffsets += [l2, o2 + valuestart]
+    offsets = koffsets + voffsets
+    output = struct.pack(
+        "Iiiiiii",
+        0x950412DE,  # Magic
+        0,  # Version
+        len(keys),  # # of entries
+        7 * 4,  # start of key index
+        7 * 4 + len(keys) * 8,  # start of value index
+        0,
+        0,
+    )  # size and offset of hash table
+    output += array.array("i", offsets).tobytes()
+    output += ids
+    output += strs
+    return output
+
+
+def make(filename, outfile):
+    ID = 1
+    STR = 2
+    CTXT = 3
+
+    # Compute .mo name from .po name and arguments
+    if filename.endswith(".po"):
+        infile = filename
+    else:
+        infile = filename + ".po"
+    if outfile is None:
+        outfile = os.path.splitext(infile)[0] + ".mo"
+
+    try:
+        with open(infile, "rb") as f:
+            lines = f.readlines()
+    except IOError as msg:
+        print(msg, file=sys.stderr)
+        sys.exit(1)
+
+    section = msgctxt = None
+    fuzzy = 0
+
+    # Start off assuming Latin-1, so everything decodes without failure,
+    # until we know the exact encoding
+    encoding = "latin-1"
+
+    # Parse the catalog
+    lno = 0
+    for l in lines:
+        l = l.decode(encoding)
+        lno += 1
+        # If we get a comment line after a msgstr, this is a new entry
+        if l[0] == "#" and section == STR:
+            add(msgctxt, msgid, msgstr, fuzzy)
+            section = msgctxt = None
+            fuzzy = 0
+        # Record a fuzzy mark
+        if l[:2] == "#," and "fuzzy" in l:
+            fuzzy = 1
+        # Skip comments
+        if l[0] == "#":
+            continue
+        # Now we are in a msgid or msgctxt section, output previous section
+        if l.startswith("msgctxt"):
+            if section == STR:
+                add(msgctxt, msgid, msgstr, fuzzy)
+            section = CTXT
+            l = l[7:]
+            msgctxt = b""
+        elif l.startswith("msgid") and not l.startswith("msgid_plural"):
+            if section == STR:
+                add(msgctxt, msgid, msgstr, fuzzy)
+                if not msgid:
+                    # See whether there is an encoding declaration
+                    p = HeaderParser()
+                    charset = p.parsestr(msgstr.decode(encoding)).get_content_charset()
+                    if charset:
+                        encoding = charset
+            section = ID
+            l = l[5:]
+            msgid = msgstr = b""
+            is_plural = False
+        # This is a message with plural forms
+        elif l.startswith("msgid_plural"):
+            if section != ID:
+                print(
+                    "msgid_plural not preceded by msgid on %s:%d" % (infile, lno), file=sys.stderr
+                )
+                sys.exit(1)
+            l = l[12:]
+            msgid += b"\0"  # separator of singular and plural
+            is_plural = True
+        # Now we are in a msgstr section
+        elif l.startswith("msgstr"):
+            section = STR
+            if l.startswith("msgstr["):
+                if not is_plural:
+                    print("plural without msgid_plural on %s:%d" % (infile, lno), file=sys.stderr)
+                    sys.exit(1)
+                l = l.split("]", 1)[1]
+                if msgstr:
+                    msgstr += b"\0"  # Separator of the various plural forms
+            else:
+                if is_plural:
+                    print(
+                        "indexed msgstr required for plural on  %s:%d" % (infile, lno),
+                        file=sys.stderr,
+                    )
+                    sys.exit(1)
+                l = l[6:]
+        # Skip empty lines
+        l = l.strip()
+        if not l:
+            continue
+        l = ast.literal_eval(l)
+        if section == CTXT:
+            msgctxt += l.encode(encoding)
+        elif section == ID:
+            msgid += l.encode(encoding)
+        elif section == STR:
+            msgstr += l.encode(encoding)
+        else:
+            print("Syntax error on %s:%d" % (infile, lno), "before:", file=sys.stderr)
+            print(l, file=sys.stderr)
+            sys.exit(1)
+    # Add last entry
+    if section == STR:
+        add(msgctxt, msgid, msgstr, fuzzy)
+
+    # Compute output
+    output = generate()
+
+    try:
+        with open(outfile, "wb") as f:
+            f.write(output)
+    except IOError as msg:
+        print(msg, file=sys.stderr)
+
+
+def main():
+    try:
+        opts, args = getopt.getopt(sys.argv[1:], "hVo:", ["help", "version", "output-file="])
+    except getopt.error as msg:
+        usage(1, msg)
+
+    outfile = None
+    # parse options
+    for opt, arg in opts:
+        if opt in ("-h", "--help"):
+            usage(0)
+        elif opt in ("-V", "--version"):
+            print("msgfmt.py", __version__)
+            sys.exit(0)
+        elif opt in ("-o", "--output-file"):
+            outfile = arg
+    # do it
+    if not args:
+        print("No input file given", file=sys.stderr)
+        print("Try `msgfmt --help' for more information.", file=sys.stderr)
+        return
+
+    for filename in args:
+        make(filename, outfile)
+
+
+if __name__ == "__main__":
+    main()