Merge pull request #7567 from MicroDev1/ci

CI Enhancements & Refactoring

.github/actions/deps/external/action.yml

@@ -0,0 +1,84 @@
+name: Fetch external deps
+
+inputs:
+  platform:
+    required: false
+    default: none
+    type: choice
+    options:
+    - arm
+    - aarch
+    - esp
+    - riscv
+    - none
+
+runs:
+  using: composite
+  steps:
+    # aarch
+    - name: Get aarch toolchain
+      if: inputs.platform == 'aarch'
+      run: |
+        wget --no-verbose https://adafruit-circuit-python.s3.amazonaws.com/gcc-arm-10.3-2021.07-x86_64-aarch64-none-elf.tar.xz
+        sudo tar -C /usr --strip-components=1 -xaf gcc-arm-10.3-2021.07-x86_64-aarch64-none-elf.tar.xz
+        sudo apt-get install -y mtools
+      shell: bash
+    - name: Install mkfs.fat
+      if: inputs.platform == 'aarch'
+      run: |
+        wget https://github.com/dosfstools/dosfstools/releases/download/v4.2/dosfstools-4.2.tar.gz
+        tar -xaf dosfstools-4.2.tar.gz
+        cd dosfstools-4.2
+        ./configure
+        make -j 2
+        cd src
+        echo >> $GITHUB_PATH $(pwd)
+      shell: bash
+
+    # arm
+    - name: Get arm toolchain
+      if: inputs.platform == 'aarch' || inputs.platform == 'arm'
+      uses: carlosperate/arm-none-eabi-gcc-action@v1
+      with:
+        release: '10-2020-q4'
+
+    # esp
+    - name: Get esp toolchain
+      if: inputs.platform == 'esp'
+      run: sudo apt-get install -y ninja-build
+      shell: bash
+    - name: Install IDF tools
+      if: inputs.platform == 'esp'
+      run: |
+        echo "Installing ESP-IDF tools"
+        $IDF_PATH/tools/idf_tools.py --non-interactive install required
+        $IDF_PATH/tools/idf_tools.py --non-interactive install cmake
+        echo "Installing Python environment and packages"
+        $IDF_PATH/tools/idf_tools.py --non-interactive install-python-env
+        rm -rf $IDF_TOOLS_PATH/dist
+      shell: bash
+    - name: Set environment
+      if: inputs.platform == 'esp'
+      run: |
+        source $IDF_PATH/export.sh
+        echo >> $GITHUB_ENV "IDF_PYTHON_ENV_PATH=$IDF_PYTHON_ENV_PATH"
+        echo >> $GITHUB_PATH "$PATH"
+      shell: bash
+
+    # riscv
+    - name: Get riscv toolchain
+      if: inputs.platform == 'riscv'
+      run: |
+        wget https://static.dev.sifive.com/dev-tools/riscv64-unknown-elf-gcc-8.3.0-2019.08.0-x86_64-linux-centos6.tar.gz
+        sudo tar -C /usr --strip-components=1 -xaf riscv64-unknown-elf-gcc-8.3.0-2019.08.0-x86_64-linux-centos6.tar.gz
+      shell: bash
+
+    # common
+    - name: Cache python dependencies
+      if: inputs.platform != 'esp'
+      uses: ./.github/actions/deps/python
+      with:
+        action: ${{ fromJSON('["restore", "cache"]')[github.job == 'scheduler'] }}
+    - name: Install python dependencies
+      run: pip install -r requirements-dev.txt
+      shell: bash

.github/actions/deps/ports/espressif/action.yml

@@ -0,0 +1,36 @@
+name: Fetch espressif port deps
+
+runs:
+  using: composite
+  steps:
+    - name: Set IDF env
+      run: |
+        echo >> $GITHUB_ENV "IDF_PATH=$GITHUB_WORKSPACE/ports/espressif/esp-idf"
+        echo >> $GITHUB_ENV "IDF_TOOLS_PATH=$GITHUB_WORKSPACE/.idf_tools"
+      shell: bash
+
+    - name: Get IDF commit
+      id: idf-commit
+      run: |
+        COMMIT=$(git submodule status ports/espressif/esp-idf | grep -o -P '(?<=^-).*(?= )')
+        echo "$COMMIT"
+        echo "commit=$COMMIT" >> $GITHUB_OUTPUT
+      shell: bash
+
+    - name: Cache IDF submodules
+      uses: actions/cache@v3
+      with:
+        path: |
+          .git/modules/ports/espressif/esp-idf
+          ports/espressif/esp-idf
+        key: submodules-idf-${{ steps.idf-commit.outputs.commit }}
+
+    - name: Cache IDF tools
+      uses: actions/cache@v3
+      with:
+        path: ${{ env.IDF_TOOLS_PATH }}
+        key: ${{ runner.os }}-${{ env.pythonLocation }}-tools-idf-${{ steps.idf-commit.outputs.commit }}
+
+    - name: Initialize IDF submodules
+      run: git submodule update --init --depth=1 --recursive $IDF_PATH
+      shell: bash

.github/actions/deps/python/action.yml

@@ -0,0 +1,42 @@
+name: Fetch python deps
+
+inputs:
+  action:
+    description: The cache action to use
+    required: false
+    default: restore
+    type: choice
+    options:
+    - cache
+    - restore
+
+runs:
+  using: composite
+  steps:
+    - name: Cache python dependencies
+      id: cache-python-deps
+      if: inputs.action == 'cache'
+      uses: actions/cache@v3
+      with:
+        path: .cp_tools
+        key: ${{ runner.os }}-${{ env.pythonLocation }}-tools-cp-${{ hashFiles('requirements-dev.txt') }}
+
+    - name: Restore python dependencies
+      id: restore-python-deps
+      if: inputs.action == 'restore'
+      uses: actions/cache/restore@v3
+      with:
+        path: .cp_tools
+        key: ${{ runner.os }}-${{ env.pythonLocation }}-tools-cp-${{ hashFiles('requirements-dev.txt') }}
+
+    - name: Set up venv
+      if: inputs.action == 'cache' && !steps.cache-python-deps.outputs.cache-hit
+      run: python -m venv .cp_tools
+      shell: bash
+
+    - name: Activate venv
+      if: inputs.action == 'cache' || (inputs.action == 'restore' && steps.restore-python-deps.outputs.cache-hit)
+      run: |
+        source .cp_tools/bin/activate
+        echo >> $GITHUB_PATH "$PATH"
+      shell: bash

.github/actions/deps/submodules/action.yml

@@ -1,13 +1,18 @@
 name: 'Fetch Submodules'
 
 inputs:
+  target:
+    description: 'The target for ci_fetch_deps'
+    required: false
+    type: string
+
   submodules:
     description: 'The submodules to cache'
     required: false
     default: '["extmod/ulab", "lib/", "tools/"]'
     type: string
 
-  cache:
+  action:
     description: 'The cache action to use'
     required: false
     default: 'restore'
@@ -42,7 +47,7 @@ runs:
       shell: bash
 
     - name: Cache submodules
-      if: ${{ inputs.cache == 'cache' }}
+      if: ${{ inputs.action == 'cache' }}
       uses: actions/cache@v3
       with:
         path: ".git/modules/\n${{ join(fromJSON(steps.create-submodule-status.outputs.submodules), '\n') }}"
@@ -50,7 +55,7 @@ runs:
         enableCrossOsArchive: true
 
     - name: Restore submodules
-      if: ${{ inputs.cache == 'restore' }}
+      if: ${{ inputs.action == 'restore' }}
       uses: actions/cache/restore@v3
       with:
         path: ".git/modules/\n${{ join(fromJSON(steps.create-submodule-status.outputs.submodules), '\n') }}"
@@ -63,7 +68,7 @@ runs:
 
     - name: CircuitPython dependencies
       id: cp-deps
-      run: python tools/ci_fetch_deps.py ${{ matrix.board || github.job }}
+      run: python tools/ci_fetch_deps.py ${{ inputs.target || matrix.board || github.job }}
       shell: bash
 
     - name: CircuitPython version

.github/actions/mpy_cross/action.yml

@@ -0,0 +1,37 @@
+name: Set up mpy-cross
+
+inputs:
+  download:
+    required: false
+    default: true
+    type: boolean
+
+runs:
+  using: composite
+  steps:
+    - name: Download mpy-cross
+      id: download-mpy-cross
+      if: inputs.download == 'true'
+      continue-on-error: true
+      uses: actions/download-artifact@v3
+      with:
+        name: mpy-cross
+        path: mpy-cross
+
+    - name: Make mpy-cross executable
+      if: inputs.download == 'true' && steps.download-mpy-cross.outcome == 'success'
+      run: sudo chmod +x mpy-cross/mpy-cross
+      shell: bash
+
+    - name: Build mpy-cross
+      if: inputs.download == 'false' || steps.download-mpy-cross.outcome == 'failure'
+      run: make -C mpy-cross -j2
+      shell: bash
+
+    - name: Upload mpy-cross
+      if: inputs.download == 'false' || steps.download-mpy-cross.outcome == 'failure'
+      continue-on-error: true
+      uses: actions/upload-artifact@v3
+      with:
+        name: mpy-cross
+        path: mpy-cross/mpy-cross

.github/actions/upload_aws/action.yml

@@ -0,0 +1,32 @@
+name: Upload to AWS S3
+
+inputs:
+  source:
+    required: true
+    type: string
+
+  destination:
+    required: false
+    type: string
+
+  AWS_ACCESS_KEY_ID:
+    required: true
+
+  AWS_SECRET_ACCESS_KEY:
+    required: true
+
+runs:
+  using: composite
+  steps:
+    - name: Upload to S3
+      if: >-
+        (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') ||
+        (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))
+      run: >-
+        [ -z "$AWS_ACCESS_KEY_ID" ] ||
+        aws s3 cp ${{ inputs.source }} s3://adafruit-circuit-python/bin/${{ inputs.destination }} --recursive --no-progress --region us-east-1
+      env:
+        AWS_PAGER: ''
+        AWS_ACCESS_KEY_ID: ${{ inputs.AWS_ACCESS_KEY_ID }}
+        AWS_SECRET_ACCESS_KEY: ${{ inputs.AWS_SECRET_ACCESS_KEY }}
+      shell: bash

.github/workflows/build-boards.yml

@@ -0,0 +1,82 @@
+name: Build boards
+
+on:
+  workflow_call:
+    inputs:
+      platform:
+        required: true
+        type: string
+
+      boards:
+        required: true
+        type: string
+
+      cp-version:
+        required: true
+        type: string
+
+jobs:
+  build:
+    runs-on: ubuntu-22.04
+    env:
+      CP_VERSION: ${{ inputs.cp-version }}
+    strategy:
+      fail-fast: false
+      matrix:
+        board: ${{ fromJSON(inputs.boards) }}
+    steps:
+    - name: Set up repository
+      uses: actions/checkout@v3
+      with:
+        submodules: false
+        fetch-depth: 1
+    - name: Set up python
+      uses: actions/setup-python@v4
+      with:
+        python-version: 3.x
+    - name: Set up port
+      if: inputs.platform == 'esp'
+      uses: ./.github/actions/deps/ports/espressif
+    - name: Set up submodules
+      id: set-up-submodules
+      uses: ./.github/actions/deps/submodules
+    - name: Set up external
+      uses: ./.github/actions/deps/external
+      with:
+        platform: ${{ inputs.platform }}
+    - name: Set up mpy-cross
+      if: steps.set-up-submodules.outputs.frozen == 'True'
+      uses: ./.github/actions/mpy_cross
+
+    - name: Versions
+      run: |
+        gcc --version
+        python3 --version
+        cmake --version || true
+        ninja --version || true
+        aarch64-none-elf-gcc --version || true
+        arm-none-eabi-gcc --version || true
+        xtensa-esp32-elf-gcc --version || true
+        riscv32-esp-elf-gcc --version || true
+        riscv64-unknown-elf-gcc --version || true
+        mkfs.fat --version || true
+
+    - name: Set up build failure matcher
+      run: echo "::add-matcher::$GITHUB_WORKSPACE/.github/workflows/match-build-fail.json"
+    - name: Build board
+      run: python3 -u build_release_files.py
+      working-directory: tools
+      env:
+        BOARDS: ${{ matrix.board }}
+
+    - name: Upload artifact
+      uses: actions/upload-artifact@v3
+      with:
+        name: ${{ matrix.board }}
+        path: bin/${{ matrix.board }}
+    - name: Upload to S3
+      uses: ./.github/actions/upload_aws
+      with:
+        source: bin/
+        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}

.github/workflows/build-mpy-cross.yml

@@ -0,0 +1,65 @@
+name: Build mpy-cross
+
+on:
+  workflow_call:
+    inputs:
+      cp-version:
+        required: true
+        type: string
+
+jobs:
+  build:
+    runs-on: ubuntu-22.04
+    strategy:
+      fail-fast: false
+      matrix:
+        mpy-cross: ["static", "static-aarch64", "static-mingw", "static-raspbian"]
+    env:
+      CP_VERSION: ${{ inputs.cp-version }}
+      EX_static-mingw: static.exe
+      OS_static: linux-amd64
+      OS_static-aarch64: linux-aarch64
+      OS_static-mingw: windows
+      OS_static-raspbian: linux-raspbian
+    steps:
+    - name: Set up repository
+      uses: actions/checkout@v3
+      with:
+        submodules: false
+        fetch-depth: 1
+    - name: Set up python
+      uses: actions/setup-python@v4
+      with:
+        python-version: 3.x
+    - name: Set up submodules
+      uses: ./.github/actions/deps/submodules
+      with:
+        target: mpy-cross
+
+    - name: Install toolchain (aarch64)
+      if: matrix.mpy-cross == 'static-aarch64'
+      run: sudo apt-get install -y gcc-aarch64-linux-gnu
+    - name: Install toolchain (mingw)
+      if: matrix.mpy-cross == 'static-mingw'
+      run: sudo apt-get install -y mingw-w64
+
+    - name: Build mpy-cross.${{ matrix.mpy-cross }}
+      run: make -C mpy-cross -j2 -f Makefile.${{ matrix.mpy-cross }}
+
+    - name: Set output
+      run: |
+        echo >> $GITHUB_ENV "EX=${{ env[format('EX_{0}', matrix.mpy-cross)] || matrix.mpy-cross }}"
+        echo >> $GITHUB_ENV "OS=${{ env[format('OS_{0}', matrix.mpy-cross)] }}"
+
+    - name: Upload artifact
+      uses: actions/upload-artifact@v3
+      with:
+        name: mpy-cross.${{ env.EX }}
+        path: mpy-cross/mpy-cross.${{ env.EX }}
+    - name: Upload to S3
+      uses: ./.github/actions/upload_aws
+      with:
+        source: mpy-cross/mpy-cross.${{ env.EX }}
+        destination: mpy-cross/${{ env.OS }}/mpy-cross-${{ env.OS }}-${{ env.CP_VERSION }}.${{ env.EX }}
+        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}

.github/workflows/build.yml

@@ -17,14 +17,16 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  test:
+  scheduler:
     runs-on: ubuntu-20.04
     outputs:
       build-doc: ${{ steps.set-matrix.outputs.build-doc }}
+      build-boards: ${{ steps.set-matrix.outputs.build-boards }}
       boards-aarch: ${{ steps.set-matrix.outputs.boards-aarch }}
       boards-arm: ${{ steps.set-matrix.outputs.boards-arm }}
-      boards-espressif: ${{ steps.set-matrix.outputs.boards-espressif }}
+      boards-esp: ${{ steps.set-matrix.outputs.boards-esp }}
       boards-riscv: ${{ steps.set-matrix.outputs.boards-riscv }}
+      boards-rpi: ${{ steps.set-matrix.outputs.boards-rpi }}
       cp-version: ${{ steps.set-up-submodules.outputs.version }}
     steps:
     - name: Dump GitHub context
@@ -39,107 +41,32 @@ jobs:
     - name: Set up python
       uses: actions/setup-python@v4
       with:
-        python-version: "3.x"
+        python-version: 3.x
     - name: Duplicate USB VID/PID check
       run: python3 -u -m tools.ci_check_duplicate_usb_vid_pid
     - name: Set up submodules
       id: set-up-submodules
-      uses: ./.github/actions/fetch_submodules
+      uses: ./.github/actions/deps/submodules
       with:
-        cache: "cache"
+        action: cache
         version: true
-    - name: Install dependencies
-      run: |
-        sudo apt-get update
-        sudo apt-get install -y eatmydata
-        sudo eatmydata apt-get install -y gettext gcc-aarch64-linux-gnu mingw-w64
-        pip install -r requirements-dev.txt
-    - name: Versions
-      run: |
-        gcc --version
-        python3 --version
-    - name: Build mpy-cross
-      run: make -C mpy-cross -j2
-    - name: Build unix port
-      run: |
-        make -C ports/unix VARIANT=coverage -j2
-    - name: Test all
-      run: MICROPY_CPYTHON3=python3.8 MICROPY_MICROPYTHON=../ports/unix/micropython-coverage ./run-tests.py -j1
-      working-directory: tests
-    - name: Print failure info
-      run: MICROPY_CPYTHON3=python3.8 MICROPY_MICROPYTHON=../ports/unix/micropython-coverage ./run-tests.py -j1 --print-failures
-      if: failure()
-      working-directory: tests
-    - name: Native Tests
-      run: MICROPY_CPYTHON3=python3.8 MICROPY_MICROPYTHON=../ports/unix/micropython-coverage ./run-tests.py -j1 --emit native
-      working-directory: tests
-    - name: mpy Tests
-      run: MICROPY_CPYTHON3=python3.8 MICROPY_MICROPYTHON=../ports/unix/micropython-coverage ./run-tests.py -j1 --via-mpy -d basics float micropython
-      working-directory: tests
-    - name: Native mpy Tests
-      run: MICROPY_CPYTHON3=python3.8 MICROPY_MICROPYTHON=../ports/unix/micropython-coverage ./run-tests.py -j1 --via-mpy --emit native -d basics float micropython
-      working-directory: tests
-    - name: Build native modules
-      run: |
-        make -C examples/natmod/features1
-        make -C examples/natmod/features2
-        make -C examples/natmod/btree
-        make -C examples/natmod/framebuf
-        make -C examples/natmod/uheapq
-        make -C examples/natmod/urandom
-        make -C examples/natmod/ure
-        make -C examples/natmod/uzlib
-    - name: Test native modules
-      run: MICROPY_CPYTHON3=python3.8 MICROPY_MICROPYTHON=../ports/unix/micropython-coverage ./run-natmodtests.py extmod/{btree*,framebuf*,uheapq*,ure*,uzlib*}.py
-      working-directory: tests
-    - name: Build mpy-cross.static-aarch64
-      run: make -C mpy-cross -j2 -f Makefile.static-aarch64
-    - uses: actions/upload-artifact@v3
+    - name: Set up external
+      uses: ./.github/actions/deps/external
+    # Disabled: Needs to be updated
+    # - name: Get last commit with checks
+    #   id: get-last-commit-with-checks
+    #   if: github.event_name == 'pull_request'
+    #   working-directory: tools
+    #   run: python3 -u ci_changes_per_commit.py
+    #   env:
+    #     REPO: ${{ github.repository }}
+    #     PULL: ${{ github.event.number }}
+    #     GITHUB_TOKEN: ${{ github.token }}
+    #     EXCLUDE_COMMIT: ${{ github.event.after }}
+    - name: Set up mpy-cross
+      uses: ./.github/actions/mpy_cross
       with:
-        name: mpy-cross.static-aarch64
-        path: mpy-cross/mpy-cross.static-aarch64
-    - name: Build mpy-cross.static-raspbian
-      run: make -C mpy-cross -j2 -f Makefile.static-raspbian
-    - uses: actions/upload-artifact@v3
-      with:
-        name: mpy-cross.static-raspbian
-        path: mpy-cross/mpy-cross.static-raspbian
-    - name: Build mpy-cross.static
-      run: make -C mpy-cross -j2 -f Makefile.static
-    - uses: actions/upload-artifact@v3
-      with:
-        name: mpy-cross.static-amd64-linux
-        path: mpy-cross/mpy-cross.static
-    - name: Build mpy-cross.static-mingw
-      run: make -C mpy-cross -j2 -f Makefile.static-mingw
-    - uses: actions/upload-artifact@v3
-      with:
-        name: mpy-cross.static-x64-windows
-        path: mpy-cross/mpy-cross.static.exe
-    - name: Upload to S3
-      if: >-
-        (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') ||
-        (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))
-      env:
-        AWS_PAGER: ''
-        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-      run: |
-        pip install awscli
-        [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross/mpy-cross.static-aarch64 s3://adafruit-circuit-python/bin/mpy-cross/mpy-cross.static-aarch64-${{ env.CP_VERSION }} --no-progress --region us-east-1
-        [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross/mpy-cross.static-raspbian s3://adafruit-circuit-python/bin/mpy-cross/mpy-cross.static-raspbian-${{ env.CP_VERSION }} --no-progress --region us-east-1
-        [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross/mpy-cross.static s3://adafruit-circuit-python/bin/mpy-cross/mpy-cross.static-amd64-linux-${{ env.CP_VERSION }} --no-progress --region us-east-1
-        [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross/mpy-cross.static.exe s3://adafruit-circuit-python/bin/mpy-cross/mpy-cross.static-x64-windows-${{ env.CP_VERSION }}.exe --no-progress --region us-east-1
-    - name: Get last commit with checks
-      id: get-last-commit-with-checks
-      if: github.event_name == 'pull_request'
-      working-directory: tools
-      env:
-        REPO: ${{ github.repository }}
-        PULL: ${{ github.event.number }}
-        GITHUB_TOKEN: ${{ github.token }}
-        EXCLUDE_COMMIT: ${{ github.event.after }}
-      run: python3 -u ci_changes_per_commit.py
+        download: false
     - name: Set head sha
       if: github.event_name == 'pull_request'
       run: echo "HEAD_SHA=$(git show -s --format=%s $GITHUB_SHA | grep -o -P "(?<=Merge ).*(?= into)")" >> $GITHUB_ENV
@@ -162,17 +89,28 @@ jobs:
         CHANGED_FILES: ${{ steps.get-changes.outputs.changed_files }}
         LAST_FAILED_JOBS: ${{ steps.get-last-commit-with-checks.outputs.check_runs }}
 
+  tests:
+    needs: scheduler
+    if: needs.scheduler.outputs.build-boards == 'True'
+    uses: ./.github/workflows/run-tests.yml
+    with:
+      cp-version: ${{ needs.scheduler.outputs.cp-version }}
+
+
+  mpy-cross:
+    needs: scheduler
+    if: needs.scheduler.outputs.build-boards == 'True'
+    uses: ./.github/workflows/build-mpy-cross.yml
+    with:
+      cp-version: ${{ needs.scheduler.outputs.cp-version }}
+
 
   mpy-cross-mac:
     runs-on: macos-11
-    needs: test
-    if: >-
-      needs.test.outputs.boards-aarch != '[]' ||
-      needs.test.outputs.boards-arm != '[]' ||
-      needs.test.outputs.boards-espressif != '[]' ||
-      needs.test.outputs.boards-riscv != '[]'
+    needs: scheduler
+    if: ${{ needs.scheduler.outputs.build-boards == 'True' }}
     env:
-      CP_VERSION: ${{ needs.test.outputs.cp-version }}
+      CP_VERSION: ${{ needs.scheduler.outputs.cp-version }}
     steps:
     - name: Set up repository
       uses: actions/checkout@v3
@@ -182,9 +120,9 @@ jobs:
     - name: Set up python
       uses: actions/setup-python@v4
       with:
-        python-version: "3.x"
+        python-version: 3.x
     - name: Set up submodules
-      uses: ./.github/actions/fetch_submodules
+      uses: ./.github/actions/deps/submodules
     - name: Versions
       run: |
         gcc --version
@@ -204,11 +142,15 @@ jobs:
         path: mpy-cross/mpy-cross-arm64
     - name: Make universal binary
       run: lipo -create -output mpy-cross-macos-universal mpy-cross/mpy-cross mpy-cross/mpy-cross-arm64
-    - uses: actions/upload-artifact@v3
+    - name: Upload artifact
+      uses: actions/upload-artifact@v3
       with:
         name: mpy-cross-macos-11-universal
         path: mpy-cross-macos-universal
-    - name: Upload mpy-cross build to S3
+    - name: Upload to S3
+      if: >-
+          (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') ||
+          (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))
       run: |
         [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross-macos-universal s3://adafruit-circuit-python/bin/mpy-cross/mpy-cross-macos-11-${{ env.CP_VERSION }}-universal --no-progress --region us-east-1
         [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross/mpy-cross-arm64 s3://adafruit-circuit-python/bin/mpy-cross/mpy-cross-macos-11-${{ env.CP_VERSION }}-arm64 --no-progress --region us-east-1
@@ -217,15 +159,13 @@ jobs:
         AWS_PAGER: ''
         AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
         AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-      if: (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') || (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))
-
 
   build-doc:
     runs-on: ubuntu-22.04
-    needs: test
-    if: ${{ needs.test.outputs.build-doc == 'True' }}
+    needs: scheduler
+    if: ${{ needs.scheduler.outputs.build-doc == 'True' }}
     env:
-      CP_VERSION: ${{ needs.test.outputs.cp-version }}
+      CP_VERSION: ${{ needs.scheduler.outputs.cp-version }}
     steps:
     - name: Set up repository
       uses: actions/checkout@v3
@@ -235,14 +175,12 @@ jobs:
     - name: Set up python
       uses: actions/setup-python@v4
       with:
-        python-version: "3.x"
+        python-version: 3.x
     - name: Set up submodules
-      uses: ./.github/actions/fetch_submodules
+      uses: ./.github/actions/deps/submodules
     - name: Install dependencies
       run: |
-        sudo apt-get update
-        sudo apt-get install -y eatmydata
-        sudo eatmydata apt-get install -y latexmk librsvg2-bin texlive-fonts-recommended texlive-latex-recommended texlive-latex-extra
+        sudo apt-get install -y latexmk librsvg2-bin texlive-fonts-recommended texlive-latex-recommended texlive-latex-extra
         pip install -r requirements-doc.txt
     - name: Build and Validate Stubs
       run: make check-stubs -j2
@@ -264,17 +202,12 @@ jobs:
         name: docs
         path: _build/latex
     - name: Upload to S3
-      if: >-
-        (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') ||
-        (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))
-      env:
-        AWS_PAGER: ''
+      uses: ./.github/actions/upload_aws
+      with:
+        source: circuitpython-stubs/dist/*.tar.gz
+        destination: stubs/circuitpython-stubs-${{ env.CP_VERSION }}.zip
         AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
         AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-      run: |
-        pip install awscli
-        zip -9r circuitpython-stubs.zip circuitpython-stubs
-        [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp circuitpython-stubs/dist/*.tar.gz s3://adafruit-circuit-python/bin/stubs/circuitpython-stubs-${{ env.CP_VERSION }}.zip --no-progress --region us-east-1
     - name: Upload stubs to PyPi
       if: github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested')
       env:
@@ -286,294 +219,50 @@ jobs:
         [ -z "$TWINE_USERNAME" ] || twine upload circuitpython-stubs/dist/*
 
 
-  build-aarch:
-    runs-on: ubuntu-22.04
-    needs: test
-    if: ${{ needs.test.outputs.boards-aarch != '[]' }}
-    env:
-      CP_VERSION: ${{ needs.test.outputs.cp-version }}
-    strategy:
-      fail-fast: false
-      matrix:
-        board: ${{ fromJSON(needs.test.outputs.boards-aarch) }}
-    steps:
-    - name: Set up repository
-      uses: actions/checkout@v3
-      with:
-        submodules: false
-        fetch-depth: 1
-    - name: Set up python
-      uses: actions/setup-python@v4
-      with:
-        python-version: "3.x"
-    - name: Set up submodules
-      id: set-up-submodules
-      uses: ./.github/actions/fetch_submodules
-    - name: Install dependencies
-      run: |
-        sudo apt-get install -y gettext mtools
-        pip install -r requirements-dev.txt
-        wget --no-verbose https://adafruit-circuit-python.s3.amazonaws.com/gcc-arm-10.3-2021.07-x86_64-aarch64-none-elf.tar.xz
-        sudo tar -C /usr --strip-components=1 -xaf gcc-arm-10.3-2021.07-x86_64-aarch64-none-elf.tar.xz
-    - uses: carlosperate/arm-none-eabi-gcc-action@v1
-      with:
-        release: '10-2020-q4'
-    - name: Install mkfs.fat
-      run: |
-        wget https://github.com/dosfstools/dosfstools/releases/download/v4.2/dosfstools-4.2.tar.gz
-        tar -xaf dosfstools-4.2.tar.gz
-        cd dosfstools-4.2
-        ./configure
-        make -j 2
-        cd src
-        echo >>$GITHUB_PATH $(pwd)
-    - name: Versions
-      run: |
-        gcc --version
-        aarch64-none-elf-gcc --version
-        arm-none-eabi-gcc --version
-        python3 --version
-        mkfs.fat --version || true
-    - name: Build mpy-cross
-      if: ${{ steps.set-up-submodules.outputs.frozen == 'True' }}
-      run: make -C mpy-cross -j2
-    - name: Setup build failure matcher
-      run: echo "::add-matcher::$GITHUB_WORKSPACE/.github/workflows/match-build-fail.json"
-    - name: Build
-      run: python3 -u build_release_files.py
-      working-directory: tools
-      env:
-        BOARDS: ${{ matrix.board }}
-    - uses: actions/upload-artifact@v3
-      with:
-        name: ${{ matrix.board }}
-        path: bin/${{ matrix.board }}
-    - name: Upload to S3
-      if: >-
-        (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') ||
-        (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))
-      run: |
-        pip install awscli
-        [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp bin/ s3://adafruit-circuit-python/bin/ --recursive --no-progress --region us-east-1
-      env:
-        AWS_PAGER: ''
-        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+  aarch:
+    needs: [scheduler, mpy-cross, tests]
+    if: ${{ needs.scheduler.outputs.boards-aarch != '[]' }}
+    uses: ./.github/workflows/build-boards.yml
+    with:
+      platform: aarch
+      boards: ${{ needs.scheduler.outputs.boards-aarch }}
+      cp-version: ${{ needs.scheduler.outputs.cp-version }}
 
 
-  build-arm:
-    runs-on: ubuntu-22.04
-    needs: test
-    if: ${{ needs.test.outputs.boards-arm != '[]' }}
-    env:
-      CP_VERSION: ${{ needs.test.outputs.cp-version }}
-    strategy:
-      fail-fast: false
-      matrix:
-        board: ${{ fromJSON(needs.test.outputs.boards-arm) }}
-    steps:
-    - name: Set up repository
-      uses: actions/checkout@v3
-      with:
-        submodules: false
-        fetch-depth: 1
-    - name: Set up python
-      uses: actions/setup-python@v4
-      with:
-        python-version: "3.x"
-    - name: Set up submodules
-      id: set-up-submodules
-      uses: ./.github/actions/fetch_submodules
-    - uses: carlosperate/arm-none-eabi-gcc-action@v1
-      with:
-        release: '10-2020-q4'
-    - name: Install dependencies
-      run: |
-        sudo apt-get install -y gettext
-        pip install -r requirements-dev.txt
-    - name: Versions
-      run: |
-        gcc --version
-        arm-none-eabi-gcc --version
-        python3 --version
-    - name: Build mpy-cross
-      if: ${{ steps.set-up-submodules.outputs.frozen == 'True' }}
-      run: make -C mpy-cross -j2
-    - name: Setup build failure matcher
-      run: echo "::add-matcher::$GITHUB_WORKSPACE/.github/workflows/match-build-fail.json"
-    - name: Build
-      run: python3 -u build_release_files.py
-      working-directory: tools
-      env:
-        BOARDS: ${{ matrix.board }}
-    - uses: actions/upload-artifact@v3
-      with:
-        name: ${{ matrix.board }}
-        path: bin/${{ matrix.board }}
-    - name: Upload to S3
-      if: >-
-        (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') ||
-        (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))
-      run: |
-        pip install awscli
-        [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp bin/ s3://adafruit-circuit-python/bin/ --recursive --no-progress --region us-east-1
-      env:
-        AWS_PAGER: ''
-        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+  arm:
+    needs: [scheduler, mpy-cross, tests]
+    if: ${{ needs.scheduler.outputs.boards-arm != '[]' }}
+    uses: ./.github/workflows/build-boards.yml
+    with:
+      platform: arm
+      boards: ${{ needs.scheduler.outputs.boards-arm }}
+      cp-version: ${{ needs.scheduler.outputs.cp-version }}
 
 
-  build-espressif:
-    runs-on: ubuntu-22.04
-    needs: test
-    if: ${{ needs.test.outputs.boards-espressif != '[]' }}
-    env:
-      CP_VERSION: ${{ needs.test.outputs.cp-version }}
-      IDF_PATH: ${{ github.workspace }}/ports/espressif/esp-idf
-      IDF_TOOLS_PATH: ${{ github.workspace }}/.idf_tools
-    strategy:
-      fail-fast: false
-      matrix:
-        board: ${{ fromJSON(needs.test.outputs.boards-espressif) }}
-    steps:
-    - name: Set up repository
-      uses: actions/checkout@v3
-      with:
-        submodules: false
-        fetch-depth: 1
-    - name: Set up python
-      id: setup-python
-      uses: actions/setup-python@v4
-      with:
-        python-version: "3.10"
-    - name: Get IDF commit
-      id: idf-commit
-      run: |
-        COMMIT=$(git submodule status ports/espressif/esp-idf | grep -o -P '(?<=^-).*(?= )')
-        echo "$COMMIT"
-        echo "commit=$COMMIT" >> $GITHUB_OUTPUT
-    - name: Cache IDF submodules
-      uses: actions/cache@v3
-      with:
-        path: |
-          .git/modules/ports/espressif/esp-idf
-          ports/espressif/esp-idf
-        key: submodules-idf-${{ steps.idf-commit.outputs.commit }}
-    - name: Cache IDF tools
-      uses: actions/cache@v3
-      with:
-        path: ${{ env.IDF_TOOLS_PATH }}
-        key: ${{ runner.os }}-Python-${{ steps.setup-python.outputs.python-version }}-tools-idf-${{ steps.idf-commit.outputs.commit }}
-    - name: Initialize IDF submodules
-      run: git submodule update --init --depth=1 --recursive $IDF_PATH
-    - name: Install IDF tools
-      run: |
-        echo "Installing ESP-IDF tools"
-        $IDF_PATH/tools/idf_tools.py --non-interactive install required
-        $IDF_PATH/tools/idf_tools.py --non-interactive install cmake
-        echo "Installing Python environment and packages"
-        $IDF_PATH/tools/idf_tools.py --non-interactive install-python-env
-        rm -rf $IDF_TOOLS_PATH/dist
-    - name: Set up submodules
-      id: set-up-submodules
-      uses: ./.github/actions/fetch_submodules
-    - name: Install dependencies
-      run: |
-        source $IDF_PATH/export.sh
-        sudo apt-get install -y gettext ninja-build
-        pip install -r requirements-dev.txt
-    - name: Versions
-      run: |
-        source $IDF_PATH/export.sh
-        gcc --version
-        python3 --version
-        ninja --version
-        cmake --version
-    - name: Build mpy-cross
-      if: ${{ steps.set-up-submodules.outputs.frozen == 'True' }}
-      run: make -C mpy-cross -j2
-    - name: Setup build failure matcher
-      run: echo "::add-matcher::$GITHUB_WORKSPACE/.github/workflows/match-build-fail.json"
-    - name: Build
-      run: |
-        source $IDF_PATH/export.sh
-        python3 -u build_release_files.py
-      working-directory: tools
-      env:
-        BOARDS: ${{ matrix.board }}
-    - uses: actions/upload-artifact@v3
-      with:
-        name: ${{ matrix.board }}
-        path: bin/${{ matrix.board }}
-    - name: Upload to S3
-      if: >-
-        (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') ||
-        (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))
-      run: |
-        pip install awscli
-        [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp bin/ s3://adafruit-circuit-python/bin/ --recursive --no-progress --region us-east-1
-      env:
-        AWS_PAGER: ''
-        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+  esp:
+    needs: [scheduler, mpy-cross, tests]
+    if: ${{ needs.scheduler.outputs.boards-esp != '[]' }}
+    uses: ./.github/workflows/build-boards.yml
+    with:
+      platform: esp
+      boards: ${{ needs.scheduler.outputs.boards-esp }}
+      cp-version: ${{ needs.scheduler.outputs.cp-version }}
 
 
-  build-riscv:
-    runs-on: ubuntu-22.04
-    needs: test
-    if: ${{ needs.test.outputs.boards-riscv != '[]' }}
-    env:
-      CP_VERSION: ${{ needs.test.outputs.cp-version }}
-    strategy:
-      fail-fast: false
-      matrix:
-        board: ${{ fromJSON(needs.test.outputs.boards-riscv) }}
-    steps:
-    - name: Set up repository
-      uses: actions/checkout@v3
-      with:
-        submodules: false
-        fetch-depth: 1
-    - name: Set up python
-      uses: actions/setup-python@v4
-      with:
-        python-version: "3.x"
-    - name: Set up submodules
-      id: set-up-submodules
-      uses: ./.github/actions/fetch_submodules
-    - name: Install dependencies
-      run: |
-        sudo apt-get install -y gettext
-        pip install -r requirements-dev.txt
-        wget https://static.dev.sifive.com/dev-tools/riscv64-unknown-elf-gcc-8.3.0-2019.08.0-x86_64-linux-centos6.tar.gz
-        sudo tar -C /usr --strip-components=1 -xaf riscv64-unknown-elf-gcc-8.3.0-2019.08.0-x86_64-linux-centos6.tar.gz
-    - name: Versions
-      run: |
-        gcc --version
-        riscv64-unknown-elf-gcc --version
-        python3 --version
-    - name: Build mpy-cross
-      if: ${{ steps.set-up-submodules.outputs.frozen == 'True' }}
-      run: make -C mpy-cross -j2
-    - name: Setup build failure matcher
-      run: echo "::add-matcher::$GITHUB_WORKSPACE/.github/workflows/match-build-fail.json"
-    - name: Build
-      run: python3 -u build_release_files.py
-      working-directory: tools
-      env:
-        BOARDS: ${{ matrix.board }}
-    - uses: actions/upload-artifact@v3
-      with:
-        name: ${{ matrix.board }}
-        path: bin/${{ matrix.board }}
-    - name: Upload to S3
-      if: >-
-        (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') ||
-        (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))
-      run: |
-        pip install awscli
-        [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp bin/ s3://adafruit-circuit-python/bin/ --recursive --no-progress --region us-east-1
-      env:
-        AWS_PAGER: ''
-        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+  riscv:
+    needs: [scheduler, mpy-cross, tests]
+    if: ${{ needs.scheduler.outputs.boards-riscv != '[]' }}
+    uses: ./.github/workflows/build-boards.yml
+    with:
+      platform: riscv
+      boards: ${{ needs.scheduler.outputs.boards-riscv }}
+      cp-version: ${{ needs.scheduler.outputs.cp-version }}
+
+  rpi:
+    needs: [scheduler, mpy-cross, tests]
+    if: ${{ needs.scheduler.outputs.boards-rpi != '[]' }}
+    uses: ./.github/workflows/build-boards.yml
+    with:
+      platform: arm
+      boards: ${{ needs.scheduler.outputs.boards-rpi }}
+      cp-version: ${{ needs.scheduler.outputs.cp-version }}

.github/workflows/create_website_pr.yml

@@ -24,13 +24,13 @@ jobs:
     - name: Set up python
       uses: actions/setup-python@v4
       with:
-        python-version: "3.x"
+        python-version: 3.x
     - name: Set up submodules
-      uses: ./.github/actions/fetch_submodules
+      uses: ./.github/actions/deps/submodules
       with:
         version: true
-    - name: Install dependencies
-      run: pip install -r requirements-dev.txt
+    - name: Set up external
+      uses: ./.github/actions/deps/external
     - name: Versions
       run: |
         gcc --version

.github/workflows/ports_windows.yml

@@ -77,7 +77,7 @@ jobs:
         fetch-depth: 1
 
     - name: Set up submodules
-      uses: ./.github/actions/fetch_submodules
+      uses: ./.github/actions/deps/submodules
       with:
         version: true
 

.github/workflows/pre-commit.yml

@@ -24,13 +24,13 @@ jobs:
     - name: Set up python
       uses: actions/setup-python@v4
       with:
-        python-version: "3.x"
-    - name: CircuitPython dependencies
-      run: python tools/ci_fetch_deps.py ${{ github.job }}
+        python-version: 3.x
+    - name: Set up submodules
+      uses: ./.github/actions/deps/submodules
+    - name: Set up external
+      uses: ./.github/actions/deps/external
     - name: Install dependencies
-      run: |
-        sudo apt-get install -y gettext uncrustify
-        pip3 install black polib pyyaml
+      run: sudo apt-get install -y gettext uncrustify
     - name: Run pre-commit
       uses: pre-commit/action@v3.0.0
     - name: Make patch

.github/workflows/run-tests.yml

@@ -0,0 +1,67 @@
+name: Run tests
+
+on:
+  workflow_call:
+    inputs:
+      cp-version:
+        required: true
+        type: string
+
+jobs:
+  run:
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        test: [all, mpy, native, native_mpy]
+    env:
+      CP_VERSION: ${{ inputs.cp-version }}
+      MICROPY_CPYTHON3: python3.8
+      MICROPY_MICROPYTHON: ../ports/unix/micropython-coverage
+      TEST_all:
+      TEST_mpy: --via-mpy -d basics float micropython
+      TEST_native: --emit native
+      TEST_native_mpy: --via-mpy --emit native -d basics float micropython
+    steps:
+    - name: Set up repository
+      uses: actions/checkout@v3
+      with:
+        submodules: false
+        fetch-depth: 1
+    - name: Set up python
+      uses: actions/setup-python@v4
+      with:
+        python-version: 3.x
+    - name: Set up submodules
+      uses: ./.github/actions/deps/submodules
+      with:
+        target: tests
+    - name: Set up external
+      if: matrix.test == 'all'
+      uses: ./.github/actions/deps/external
+    - name: Set up mpy-cross
+      uses: ./.github/actions/mpy_cross
+    - name: Build unix port
+      run: make -C ports/unix VARIANT=coverage -j2
+    - name: Run tests
+      run: ./run-tests.py -j2 ${{ env[format('TEST_{0}', matrix.test)] }}
+      working-directory: tests
+    - name: Print failure info
+      run: ./run-tests.py -j2 --print-failures
+      if: failure()
+      working-directory: tests
+    - name: Build native modules
+      if: matrix.test == 'all'
+      run: |
+        make -C examples/natmod/features1
+        make -C examples/natmod/features2
+        make -C examples/natmod/btree
+        make -C examples/natmod/framebuf
+        make -C examples/natmod/uheapq
+        make -C examples/natmod/urandom
+        make -C examples/natmod/ure
+        make -C examples/natmod/uzlib
+    - name: Test native modules
+      if: matrix.test == 'all'
+      run: ./run-natmodtests.py extmod/{btree*,framebuf*,uheapq*,ure*,uzlib*}.py
+      working-directory: tests

py/py.mk

@@ -243,7 +243,7 @@ $(HEADER_BUILD)/mpversion.h: FORCE | $(HEADER_BUILD)
 MPCONFIGPORT_MK = $(wildcard mpconfigport.mk)
 
 $(HEADER_BUILD)/$(TRANSLATION).mo: $(TOP)/locale/$(TRANSLATION).po | $(HEADER_BUILD)
-	$(Q)msgfmt -o $@ $^
+	$(Q)$(PYTHON) $(TOP)/tools/msgfmt.py -o $@ $^
 
 $(HEADER_BUILD)/qstrdefs.preprocessed.h: $(PY_QSTR_DEFS) $(QSTR_DEFS) $(QSTR_DEFS_COLLECTED) mpconfigport.h $(MPCONFIGPORT_MK) $(PY_SRC)/mpconfig.h | $(HEADER_BUILD)
 	$(STEPECHO) "GEN $@"

requirements-dev.txt

@@ -6,6 +6,7 @@ typer
 sh
 click
 cpp-coveralls
+
 requests
 requests-cache
 
@@ -13,8 +14,8 @@ requests-cache
 polib
 
 # For pre-commit
-pyyaml
 black
+pyyaml
 pre-commit
 
 # for combining the Nordic SoftDevice with CircuitPython
@@ -23,9 +24,6 @@ intelhex
 # for building & testing natmods
 pyelftools
 
-# for stubs and annotations
-adafruit-circuitpython-typing
-
 # for mbedtls certificate store
 cryptography
 

tools/ci_fetch_deps.py

@@ -73,12 +73,15 @@ def main():
 
     print("Target:", TARGET)
 
-    if TARGET == "test":
+    if TARGET == "scheduler":
+        # submodules = ["tools/"]
+        submodules = ["extmod/ulab", "lib/", "tools/"]
+    elif TARGET == "tests":
         submodules = ["extmod/ulab", "lib/", "tools/"]
     elif TARGET == "build-doc":
         # used in .readthedocs.yml to generate RTD
         submodules = ["extmod/ulab", "frozen/"]
-    elif TARGET == "mpy-cross-mac":
+    elif TARGET == "mpy-cross" or TARGET == "mpy-cross-mac":
         submodules = ["tools/"]  # for huffman
     elif TARGET == "windows":
         # This builds one board from a number of ports so fill out a bunch of submodules

tools/ci_set_matrix.py

@@ -46,11 +46,11 @@ PORT_TO_ARCH = {
     "atmel-samd": "arm",
     "broadcom": "aarch",
     "cxd56": "arm",
-    "espressif": "espressif",
+    "espressif": "esp",
     "litex": "riscv",
     "mimxrt10xx": "arm",
     "nrf": "arm",
-    "raspberrypi": "arm",
+    "raspberrypi": "rpi",
     "stm": "arm",
 }
 
@@ -204,29 +204,33 @@ def set_boards_to_build(build_all: bool):
             break
 
     # Split boards by architecture.
-    print("Building boards:")
-    arch_to_boards = {"aarch": [], "arm": [], "riscv": [], "espressif": []}
+    arch_to_boards = {"aarch": [], "arm": [], "esp": [], "riscv": [], "rpi": []}
+
+    # Append previously failed boards
+    for arch in arch_to_boards:
+        arch_to_job = f"build-{arch}"
+        if arch_to_job in last_failed_jobs:
+            for board in last_failed_jobs[arch_to_job]:
+                if not board in boards_to_build:
+                    boards_to_build.append(board)
+
+    build_boards = bool(boards_to_build)
+    print("Building boards:", build_boards)
+    set_output("build-boards", build_boards)
+
+    # Append boards according to arch
     for board in sorted(boards_to_build):
-        print(" ", board)
         port = board_to_port.get(board)
         # A board can appear due to its _deletion_ (rare)
         # if this happens it's not in `board_to_port`.
         if not port:
             continue
-        arch = PORT_TO_ARCH[port]
-        arch_to_boards[arch].append(board)
+        arch_to_boards[PORT_TO_ARCH[port]].append(board)
+        print(" ", board)
 
     # Set the step outputs for each architecture
     for arch in arch_to_boards:
-        # Append previous failed jobs
-        if f"build-{arch}" in last_failed_jobs:
-            failed_boards = last_failed_jobs[f"build-{arch}"]
-            for board in failed_boards:
-                if not board in arch_to_boards[arch]:
-                    print(" ", board)
-                    arch_to_boards[arch].append(board)
-        # Set Output
-        set_output(f"boards-{arch}", json.dumps(sorted(arch_to_boards[arch])))
+        set_output(f"boards-{arch}", json.dumps(arch_to_boards[arch]))
 
 
 def set_docs_to_build(build_doc: bool):

tools/msgfmt.py

@@ -0,0 +1,244 @@
+#! /usr/bin/env python3
+# Written by Martin v. Löwis <loewis@informatik.hu-berlin.de>
+
+"""Generate binary message catalog from textual translation description.
+This program converts a textual Uniforum-style message catalog (.po file) into
+a binary GNU catalog (.mo file).  This is essentially the same function as the
+GNU msgfmt program, however, it is a simpler implementation.  Currently it
+does not handle plural forms but it does handle message contexts.
+Usage: msgfmt.py [OPTIONS] filename.po
+Options:
+    -o file
+    --output-file=file
+        Specify the output file to write to.  If omitted, output will go to a
+        file named filename.mo (based off the input file name).
+    -h
+    --help
+        Print this message and exit.
+    -V
+    --version
+        Display version information and exit.
+"""
+
+import os
+import sys
+import ast
+import getopt
+import struct
+import array
+from email.parser import HeaderParser
+
+__version__ = "1.2"
+
+MESSAGES = {}
+
+
+def usage(code, msg=""):
+    print(__doc__, file=sys.stderr)
+    if msg:
+        print(msg, file=sys.stderr)
+    sys.exit(code)
+
+
+def add(ctxt, id, str, fuzzy):
+    "Add a non-fuzzy translation to the dictionary."
+    global MESSAGES
+    if not fuzzy and str:
+        if ctxt is None:
+            MESSAGES[id] = str
+        else:
+            MESSAGES[b"%b\x04%b" % (ctxt, id)] = str
+
+
+def generate():
+    "Return the generated output."
+    global MESSAGES
+    # the keys are sorted in the .mo file
+    keys = sorted(MESSAGES.keys())
+    offsets = []
+    ids = strs = b""
+    for id in keys:
+        # For each string, we need size and file offset.  Each string is NUL
+        # terminated; the NUL does not count into the size.
+        offsets.append((len(ids), len(id), len(strs), len(MESSAGES[id])))
+        ids += id + b"\0"
+        strs += MESSAGES[id] + b"\0"
+    output = ""
+    # The header is 7 32-bit unsigned integers.  We don't use hash tables, so
+    # the keys start right after the index tables.
+    # translated string.
+    keystart = 7 * 4 + 16 * len(keys)
+    # and the values start after the keys
+    valuestart = keystart + len(ids)
+    koffsets = []
+    voffsets = []
+    # The string table first has the list of keys, then the list of values.
+    # Each entry has first the size of the string, then the file offset.
+    for o1, l1, o2, l2 in offsets:
+        koffsets += [l1, o1 + keystart]
+        voffsets += [l2, o2 + valuestart]
+    offsets = koffsets + voffsets
+    output = struct.pack(
+        "Iiiiiii",
+        0x950412DE,  # Magic
+        0,  # Version
+        len(keys),  # # of entries
+        7 * 4,  # start of key index
+        7 * 4 + len(keys) * 8,  # start of value index
+        0,
+        0,
+    )  # size and offset of hash table
+    output += array.array("i", offsets).tobytes()
+    output += ids
+    output += strs
+    return output
+
+
+def make(filename, outfile):
+    ID = 1
+    STR = 2
+    CTXT = 3
+
+    # Compute .mo name from .po name and arguments
+    if filename.endswith(".po"):
+        infile = filename
+    else:
+        infile = filename + ".po"
+    if outfile is None:
+        outfile = os.path.splitext(infile)[0] + ".mo"
+
+    try:
+        with open(infile, "rb") as f:
+            lines = f.readlines()
+    except IOError as msg:
+        print(msg, file=sys.stderr)
+        sys.exit(1)
+
+    section = msgctxt = None
+    fuzzy = 0
+
+    # Start off assuming Latin-1, so everything decodes without failure,
+    # until we know the exact encoding
+    encoding = "latin-1"
+
+    # Parse the catalog
+    lno = 0
+    for l in lines:
+        l = l.decode(encoding)
+        lno += 1
+        # If we get a comment line after a msgstr, this is a new entry
+        if l[0] == "#" and section == STR:
+            add(msgctxt, msgid, msgstr, fuzzy)
+            section = msgctxt = None
+            fuzzy = 0
+        # Record a fuzzy mark
+        if l[:2] == "#," and "fuzzy" in l:
+            fuzzy = 1
+        # Skip comments
+        if l[0] == "#":
+            continue
+        # Now we are in a msgid or msgctxt section, output previous section
+        if l.startswith("msgctxt"):
+            if section == STR:
+                add(msgctxt, msgid, msgstr, fuzzy)
+            section = CTXT
+            l = l[7:]
+            msgctxt = b""
+        elif l.startswith("msgid") and not l.startswith("msgid_plural"):
+            if section == STR:
+                add(msgctxt, msgid, msgstr, fuzzy)
+                if not msgid:
+                    # See whether there is an encoding declaration
+                    p = HeaderParser()
+                    charset = p.parsestr(msgstr.decode(encoding)).get_content_charset()
+                    if charset:
+                        encoding = charset
+            section = ID
+            l = l[5:]
+            msgid = msgstr = b""
+            is_plural = False
+        # This is a message with plural forms
+        elif l.startswith("msgid_plural"):
+            if section != ID:
+                print(
+                    "msgid_plural not preceded by msgid on %s:%d" % (infile, lno), file=sys.stderr
+                )
+                sys.exit(1)
+            l = l[12:]
+            msgid += b"\0"  # separator of singular and plural
+            is_plural = True
+        # Now we are in a msgstr section
+        elif l.startswith("msgstr"):
+            section = STR
+            if l.startswith("msgstr["):
+                if not is_plural:
+                    print("plural without msgid_plural on %s:%d" % (infile, lno), file=sys.stderr)
+                    sys.exit(1)
+                l = l.split("]", 1)[1]
+                if msgstr:
+                    msgstr += b"\0"  # Separator of the various plural forms
+            else:
+                if is_plural:
+                    print(
+                        "indexed msgstr required for plural on  %s:%d" % (infile, lno),
+                        file=sys.stderr,
+                    )
+                    sys.exit(1)
+                l = l[6:]
+        # Skip empty lines
+        l = l.strip()
+        if not l:
+            continue
+        l = ast.literal_eval(l)
+        if section == CTXT:
+            msgctxt += l.encode(encoding)
+        elif section == ID:
+            msgid += l.encode(encoding)
+        elif section == STR:
+            msgstr += l.encode(encoding)
+        else:
+            print("Syntax error on %s:%d" % (infile, lno), "before:", file=sys.stderr)
+            print(l, file=sys.stderr)
+            sys.exit(1)
+    # Add last entry
+    if section == STR:
+        add(msgctxt, msgid, msgstr, fuzzy)
+
+    # Compute output
+    output = generate()
+
+    try:
+        with open(outfile, "wb") as f:
+            f.write(output)
+    except IOError as msg:
+        print(msg, file=sys.stderr)
+
+
+def main():
+    try:
+        opts, args = getopt.getopt(sys.argv[1:], "hVo:", ["help", "version", "output-file="])
+    except getopt.error as msg:
+        usage(1, msg)
+
+    outfile = None
+    # parse options
+    for opt, arg in opts:
+        if opt in ("-h", "--help"):
+            usage(0)
+        elif opt in ("-V", "--version"):
+            print("msgfmt.py", __version__)
+            sys.exit(0)
+        elif opt in ("-o", "--output-file"):
+            outfile = arg
+    # do it
+    if not args:
+        print("No input file given", file=sys.stderr)
+        print("Try `msgfmt --help' for more information.", file=sys.stderr)
+        return
+
+    for filename in args:
+        make(filename, outfile)
+
+
+if __name__ == "__main__":
+    main()