Merge pull request #7453 from MicroDev1/ci

Multiple CI Enhancements

.github/workflows/build.yml

@@ -16,46 +16,67 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
   cancel-in-progress: true
 
+env:
+  CACHE_SUBMODULES: "['extmod/ulab', 'lib/', 'tools/']"
+
 jobs:
   test:
     runs-on: ubuntu-20.04
     outputs:
       build-doc: ${{ steps.set-matrix.outputs.build-doc }}
-      boards-arm: ${{ steps.set-matrix.outputs.boards-arm }}
-      boards-riscv: ${{ steps.set-matrix.outputs.boards-riscv }}
-      boards-espressif: ${{ steps.set-matrix.outputs.boards-espressif }}
       boards-aarch: ${{ steps.set-matrix.outputs.boards-aarch }}
+      boards-arm: ${{ steps.set-matrix.outputs.boards-arm }}
+      boards-espressif: ${{ steps.set-matrix.outputs.boards-espressif }}
+      boards-riscv: ${{ steps.set-matrix.outputs.boards-riscv }}
+      cp-version: ${{ steps.cp-version.outputs.cp-version }}
     steps:
     - name: Dump GitHub context
       run: echo "$GITHUB_CONTEXT"
       env:
         GITHUB_CONTEXT: ${{ toJson(github) }}
-    - uses: actions/checkout@v3
+    - name: Set up repository
+      uses: actions/checkout@v3
       with:
         submodules: false
         fetch-depth: 1
-    - name: Set up Python 3
+    - name: Set up python
       uses: actions/setup-python@v4
       with:
         python-version: "3.x"
-    - name: Get CP deps
-      run: python tools/ci_fetch_deps.py test ${{ github.sha }}
-    - name: CircuitPython version
+    - name: Duplicate USB VID/PID check
+      run: python3 -u -m tools.ci_check_duplicate_usb_vid_pid
+    - name: Create submodule status
+      run: git submodule status ${{ join(fromJSON(env.CACHE_SUBMODULES), ' ') }} >> submodule_status
+    - name: Cache submodules
+      uses: actions/cache@v3
+      with:
+        path: ".git/modules/\n${{ join(fromJSON(env.CACHE_SUBMODULES), '\n') }}"
+        key: submodules-common-${{ hashFiles('submodule_status') }}
+    - name: CircuitPython dependencies
       run: |
-        tools/describe || git log --parents HEAD~4..
-        echo >>$GITHUB_ENV CP_VERSION=$(tools/describe)
+        python tools/ci_fetch_deps.py ${{ github.job }}
+        echo "::group::Fetch history and tags"
+        git fetch --no-recurse-submodules --shallow-since="2021-07-01" --tags https://github.com/adafruit/circuitpython HEAD
+        git fetch --no-recurse-submodules --shallow-since="2021-07-01" origin $GITHUB_SHA
+        git repack -d
+        echo "::endgroup::"
+    - name: CircuitPython version
+      id: cp-version
+      run: |
+        CP_VERSION=$(tools/describe)
+        echo "$CP_VERSION"
+        echo "CP_VERSION=$CP_VERSION" >> $GITHUB_ENV
+        echo "cp-version=$CP_VERSION" >> $GITHUB_OUTPUT
     - name: Install dependencies
       run: |
         sudo apt-get update
         sudo apt-get install -y eatmydata
         sudo eatmydata apt-get install -y gettext gcc-aarch64-linux-gnu mingw-w64
-        pip install -r requirements-ci.txt -r requirements-dev.txt
+        pip install -r requirements-dev.txt
     - name: Versions
       run: |
         gcc --version
         python3 --version
-    - name: Duplicate USB VID/PID Check
-      run: python3 -u -m tools.ci_check_duplicate_usb_vid_pid
     - name: Build mpy-cross
       run: make -C mpy-cross -j2
     - name: Build unix port
@@ -114,13 +135,16 @@ jobs:
       with:
         name: mpy-cross.static-x64-windows
         path: mpy-cross/mpy-cross.static.exe
-    - name: Upload mpy-cross builds to S3
-      if: (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') || (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))
+    - name: Upload to S3
+      if: >-
+        (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') ||
+        (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))
       env:
         AWS_PAGER: ''
         AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
         AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
       run: |
+        pip install awscli
         [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross/mpy-cross.static-aarch64 s3://adafruit-circuit-python/bin/mpy-cross/mpy-cross.static-aarch64-${{ env.CP_VERSION }} --no-progress --region us-east-1
         [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross/mpy-cross.static-raspbian s3://adafruit-circuit-python/bin/mpy-cross/mpy-cross.static-raspbian-${{ env.CP_VERSION }} --no-progress --region us-east-1
         [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross/mpy-cross.static s3://adafruit-circuit-python/bin/mpy-cross/mpy-cross.static-amd64-linux-${{ env.CP_VERSION }} --no-progress --region us-east-1
@@ -160,29 +184,35 @@ jobs:
 
   mpy-cross-mac:
     runs-on: macos-11
-    steps:
-    - name: Dump GitHub context
+    needs: test
+    if: >-
+      needs.test.outputs.boards-aarch != '[]' ||
+      needs.test.outputs.boards-arm != '[]' ||
+      needs.test.outputs.boards-espressif != '[]' ||
+      needs.test.outputs.boards-riscv != '[]'
     env:
-        GITHUB_CONTEXT: ${{ toJson(github) }}
-      run: echo "$GITHUB_CONTEXT"
-    - uses: actions/checkout@v3
+      CP_VERSION: ${{ needs.test.outputs.cp-version }}
+    steps:
+    - name: Set up repository
+      uses: actions/checkout@v3
       with:
         submodules: false
         fetch-depth: 1
-    - name: Set up Python 3
+    - name: Set up python
       uses: actions/setup-python@v4
       with:
-        python-version: "3.10"
-    - name: Get CP deps
-      run: python tools/ci_fetch_deps.py mpy-cross-mac ${{ github.sha }}
+        python-version: "3.x"
+    - name: Create submodule status
+      run: git submodule status ${{ join(fromJSON(env.CACHE_SUBMODULES), ' ') }} >> submodule_status
+    - name: Restore submodules
+      uses: actions/cache/restore@v3
+      with:
+        path: ".git/modules/\n${{ join(fromJSON(env.CACHE_SUBMODULES), '\n') }}"
+        key: submodules-common-${{ hashFiles('submodule_status') }}
+    - name: CircuitPython dependencies
+      run: python tools/ci_fetch_deps.py ${{ github.job }}
     - name: CircuitPython version
-      run: |
-        tools/describe || git log --parents HEAD~4..
-        echo >>$GITHUB_ENV CP_VERSION=$(tools/describe)
-    - name: Install dependencies
-      run: |
-        brew install gettext
-        echo >>$GITHUB_PATH /usr/local/opt/gettext/bin
+      run: tools/describe
     - name: Versions
       run: |
         gcc --version
@@ -222,27 +252,35 @@ jobs:
     runs-on: ubuntu-20.04
     needs: test
     if: ${{ needs.test.outputs.build-doc == 'True' }}
+    env:
+      CP_VERSION: ${{ needs.test.outputs.cp-version }}
     steps:
-    - uses: actions/checkout@v3
+    - name: Set up repository
+      uses: actions/checkout@v3
       with:
         submodules: false
         fetch-depth: 1
-    - name: Get CP deps
-      run: python tools/ci_fetch_deps.py docs ${{ github.sha }}
+    - name: Create submodule status
+      run: git submodule status ${{ join(fromJSON(env.CACHE_SUBMODULES), ' ') }} >> submodule_status
+    - name: Restore submodules
+      uses: actions/cache/restore@v3
+      with:
+        path: ".git/modules/\n${{ join(fromJSON(env.CACHE_SUBMODULES), '\n') }}"
+        key: submodules-common-${{ hashFiles('submodule_status') }}
+    - name: CircuitPython dependencies
+      run: python tools/ci_fetch_deps.py ${{ github.job }}
     - name: CircuitPython version
-      run: |
-        tools/describe || git log --parents HEAD~4..
-        echo >>$GITHUB_ENV CP_VERSION=$(tools/describe)
-    - name: Set up Python 3
+      run: tools/describe
+    - name: Set up python
       uses: actions/setup-python@v4
       with:
-        python-version: "3.10"
+        python-version: "3.x"
     - name: Install dependencies
       run: |
         sudo apt-get update
         sudo apt-get install -y eatmydata
         sudo eatmydata apt-get install -y latexmk librsvg2-bin texlive-fonts-recommended texlive-latex-recommended texlive-latex-extra
-        pip install -r requirements-ci.txt -r requirements-doc.txt
+        pip install -r requirements-doc.txt
     - name: Build and Validate Stubs
       run: make check-stubs -j2
     - uses: actions/upload-artifact@v3
@@ -262,13 +300,16 @@ jobs:
       with:
         name: docs
         path: _build/latex
-    - name: Upload stubs to S3
-      if: (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') || (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))
+    - name: Upload to S3
+      if: >-
+        (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') ||
+        (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))
       env:
         AWS_PAGER: ''
         AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
         AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
       run: |
+        pip install awscli
         zip -9r circuitpython-stubs.zip circuitpython-stubs
         [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp circuitpython-stubs/dist/*.tar.gz s3://adafruit-circuit-python/bin/stubs/circuitpython-stubs-${{ env.CP_VERSION }}.zip --no-progress --region us-east-1
     - name: Upload stubs to PyPi
@@ -282,225 +323,40 @@ jobs:
         [ -z "$TWINE_USERNAME" ] || twine upload circuitpython-stubs/dist/*
 
 
-  build-arm:
-    runs-on: ubuntu-20.04
-    needs: test
-    strategy:
-      fail-fast: false
-      matrix:
-        board: ${{ fromJSON(needs.test.outputs.boards-arm) }}
-    if: ${{ needs.test.outputs.boards-arm != '[]' }}
-    steps:
-    - name: Set up Python 3
-      uses: actions/setup-python@v4
-      with:
-        python-version: "3.10"
-    - uses: actions/checkout@v3
-      with:
-        submodules: false
-        fetch-depth: 1
-    - name: Get CP deps
-      run: python tools/ci_fetch_deps.py ${{ matrix.board }} ${{ github.sha }}
-    - uses: carlosperate/arm-none-eabi-gcc-action@v1
-      with:
-        release: '10-2020-q4'
-    - name: Install dependencies
-      run: |
-        sudo apt-get install -y gettext
-        pip install -r requirements-ci.txt -r requirements-dev.txt
-    - name: Versions
-      run: |
-        gcc --version
-        arm-none-eabi-gcc --version
-        python3 --version
-    - name: mpy-cross
-      run: make -C mpy-cross -j2
-    - name: Setup build failure matcher
-      run: echo "::add-matcher::$GITHUB_WORKSPACE/.github/workflows/match-build-fail.json"
-    - name: build
-      run: python3 -u build_release_files.py
-      working-directory: tools
-      env:
-        BOARDS: ${{ matrix.board }}
-    - uses: actions/upload-artifact@v3
-      with:
-        name: ${{ matrix.board }}
-        path: bin/${{ matrix.board }}
-    - name: Upload to S3
-      run: "[ -z \"$AWS_ACCESS_KEY_ID\" ] || aws s3 cp bin/ s3://adafruit-circuit-python/bin/ --recursive --no-progress --region us-east-1"
-      env:
-        AWS_PAGER: ''
-        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-      if: (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') || (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))
-
-
-  build-riscv:
-    runs-on: ubuntu-20.04
-    needs: test
-    strategy:
-      fail-fast: false
-      matrix:
-        board: ${{ fromJSON(needs.test.outputs.boards-riscv) }}
-    if: ${{ needs.test.outputs.boards-riscv != '[]' }}
-    steps:
-    - name: Set up Python 3
-      uses: actions/setup-python@v4
-      with:
-        python-version: "3.10"
-    - uses: actions/checkout@v3
-      with:
-        submodules: false
-        fetch-depth: 1
-    - name: Get CP deps
-      run: python tools/ci_fetch_deps.py ${{ matrix.board }} ${{ github.sha }}
-    - name: Install dependencies
-      run: |
-        sudo apt-get install -y gettext
-        pip install -r requirements-ci.txt -r requirements-dev.txt
-        wget https://static.dev.sifive.com/dev-tools/riscv64-unknown-elf-gcc-8.3.0-2019.08.0-x86_64-linux-centos6.tar.gz
-        sudo tar -C /usr --strip-components=1 -xaf riscv64-unknown-elf-gcc-8.3.0-2019.08.0-x86_64-linux-centos6.tar.gz
-    - name: Versions
-      run: |
-        gcc --version
-        riscv64-unknown-elf-gcc --version
-        python3 --version
-    - name: mpy-cross
-      run: make -C mpy-cross -j2
-    - name: Setup build failure matcher
-      run: echo "::add-matcher::$GITHUB_WORKSPACE/.github/workflows/match-build-fail.json"
-    - name: build
-      run: python3 -u build_release_files.py
-      working-directory: tools
-      env:
-        BOARDS: ${{ matrix.board }}
-    - uses: actions/upload-artifact@v3
-      with:
-        name: ${{ matrix.board }}
-        path: bin/${{ matrix.board }}
-    - name: Upload to S3
-      run: "[ -z \"$AWS_ACCESS_KEY_ID\" ] || aws s3 cp bin/ s3://adafruit-circuit-python/bin/ --recursive --no-progress --region us-east-1"
-      env:
-        AWS_PAGER: ''
-        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-      if: (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') || (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))
-
-
-  build-espressif:
-    runs-on: ubuntu-20.04
-    needs: test
-    strategy:
-      fail-fast: false
-      matrix:
-        board: ${{ fromJSON(needs.test.outputs.boards-espressif) }}
-    if: ${{ needs.test.outputs.boards-espressif != '[]' }}
-    steps:
-    - name: Set up Python 3
-      id: py3
-      uses: actions/setup-python@v4
-      with:
-        python-version: "3.10"
-    - uses: actions/checkout@v3
-      with:
-        submodules: false
-        fetch-depth: 1
-    - name: Get CP deps
-      run: python tools/ci_fetch_deps.py ${{ matrix.board }} ${{ github.sha }}
-    - name: CircuitPython version
-      run: |
-        tools/describe || git log --parents HEAD~4..
-        echo >>$GITHUB_ENV CP_VERSION=$(tools/describe)
-    - uses: actions/cache@v3
-      name: Fetch IDF tool cache
-      id: idf-cache
-      with:
-        path: ${{ github.workspace }}/.idf_tools
-        key: ${{ runner.os }}-idf-tools-${{ hashFiles('.git/modules/ports/espressif/esp-idf/HEAD') }}-${{ steps.py3.outputs.python-path }}-20220404
-    - name: Clone IDF submodules
-      run: git submodule update --init $IDF_PATH
-      env:
-        IDF_PATH: ${{ github.workspace }}/ports/espressif/esp-idf
-    - name: Install IDF tools
-      run: |
-        echo "Installing ESP-IDF tools"
-        $IDF_PATH/tools/idf_tools.py --non-interactive install required
-        $IDF_PATH/tools/idf_tools.py --non-interactive install cmake
-        echo "Installing Python environment and packages"
-        $IDF_PATH/tools/idf_tools.py --non-interactive install-python-env
-        rm -rf $IDF_TOOLS_PATH/dist
-      env:
-        IDF_PATH: ${{ github.workspace }}/ports/espressif/esp-idf
-        IDF_TOOLS_PATH: ${{ github.workspace }}/.idf_tools
-    - name: Install dependencies
-      run: |
-        source $IDF_PATH/export.sh
-        sudo apt-get install -y gettext ninja-build
-        pip install -r requirements-ci.txt -r requirements-dev.txt
-      env:
-        IDF_PATH: ${{ github.workspace }}/ports/espressif/esp-idf
-        IDF_TOOLS_PATH: ${{ github.workspace }}/.idf_tools
-    - name: Versions
-      run: |
-        source $IDF_PATH/export.sh
-        gcc --version
-        python3 --version
-        ninja --version
-        cmake --version
-      shell: bash
-      env:
-        IDF_PATH: ${{ github.workspace }}/ports/espressif/esp-idf
-        IDF_TOOLS_PATH: ${{ github.workspace }}/.idf_tools
-    - name: mpy-cross
-      run: make -C mpy-cross -j2
-    - name: Setup build failure matcher
-      run: echo "::add-matcher::$GITHUB_WORKSPACE/.github/workflows/match-build-fail.json"
-    - name: build
-      run: |
-        source $IDF_PATH/export.sh
-        python3 -u build_release_files.py
-      working-directory: tools
-      shell: bash
-      env:
-        IDF_PATH: ${{ github.workspace }}/ports/espressif/esp-idf
-        IDF_TOOLS_PATH: ${{ github.workspace }}/.idf_tools
-        BOARDS: ${{ matrix.board }}
-    - uses: actions/upload-artifact@v3
-      with:
-        name: ${{ matrix.board }}
-        path: bin/${{ matrix.board }}
-    - name: Upload to S3
-      run: "[ -z \"$AWS_ACCESS_KEY_ID\" ] || aws s3 cp bin/ s3://adafruit-circuit-python/bin/ --recursive --no-progress --region us-east-1"
-      env:
-        AWS_PAGER: ''
-        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-      if: (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') || (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))
-
-
   build-aarch:
     runs-on: ubuntu-20.04
     needs: test
+    if: ${{ needs.test.outputs.boards-aarch != '[]' }}
+    env:
+      CP_VERSION: ${{ needs.test.outputs.cp-version }}
     strategy:
       fail-fast: false
       matrix:
         board: ${{ fromJSON(needs.test.outputs.boards-aarch) }}
-    if: ${{ needs.test.outputs.boards-aarch != '[]' }}
     steps:
-    - name: Set up Python 3
-      uses: actions/setup-python@v4
-      with:
-        python-version: "3.10"
-    - uses: actions/checkout@v3
+    - name: Set up repository
+      uses: actions/checkout@v3
       with:
         submodules: false
         fetch-depth: 1
-    - name: Get CP deps
-      run: python tools/ci_fetch_deps.py ${{ matrix.board }} ${{ github.sha }}
+    - name: Set up python
+      uses: actions/setup-python@v4
+      with:
+        python-version: "3.x"
+    - name: Create submodule status
+      run: git submodule status ${{ join(fromJSON(env.CACHE_SUBMODULES), ' ') }} >> submodule_status
+    - name: Restore submodules
+      uses: actions/cache/restore@v3
+      with:
+        path: ".git/modules/\n${{ join(fromJSON(env.CACHE_SUBMODULES), '\n') }}"
+        key: submodules-common-${{ hashFiles('submodule_status') }}
+    - name: CircuitPython dependencies
+      id: cp-deps
+      run: python tools/ci_fetch_deps.py ${{ matrix.board }}
     - name: Install dependencies
       run: |
         sudo apt-get install -y gettext mtools
-        pip install -r requirements-ci.txt -r requirements-dev.txt
+        pip install -r requirements-dev.txt
         wget --no-verbose https://adafruit-circuit-python.s3.amazonaws.com/gcc-arm-10.3-2021.07-x86_64-aarch64-none-elf.tar.xz
         sudo tar -C /usr --strip-components=1 -xaf gcc-arm-10.3-2021.07-x86_64-aarch64-none-elf.tar.xz
     - uses: carlosperate/arm-none-eabi-gcc-action@v1
@@ -522,11 +378,12 @@ jobs:
         arm-none-eabi-gcc --version
         python3 --version
         mkfs.fat --version || true
-    - name: mpy-cross
+    - name: Build mpy-cross
+      if: ${{ steps.cp-deps.outputs.frozen_tags == 'True' }}
       run: make -C mpy-cross -j2
     - name: Setup build failure matcher
       run: echo "::add-matcher::$GITHUB_WORKSPACE/.github/workflows/match-build-fail.json"
-    - name: build
+    - name: Build
       run: python3 -u build_release_files.py
       working-directory: tools
       env:
@@ -536,9 +393,256 @@ jobs:
         name: ${{ matrix.board }}
         path: bin/${{ matrix.board }}
     - name: Upload to S3
-      run: "[ -z \"$AWS_ACCESS_KEY_ID\" ] || aws s3 cp bin/ s3://adafruit-circuit-python/bin/ --recursive --no-progress --region us-east-1"
+      if: >-
+        (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') ||
+        (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))
+      run: |
+        pip install awscli
+        [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp bin/ s3://adafruit-circuit-python/bin/ --recursive --no-progress --region us-east-1
+      env:
+        AWS_PAGER: ''
+        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+
+
+  build-arm:
+    runs-on: ubuntu-20.04
+    needs: test
+    if: ${{ needs.test.outputs.boards-arm != '[]' }}
+    env:
+      CP_VERSION: ${{ needs.test.outputs.cp-version }}
+    strategy:
+      fail-fast: false
+      matrix:
+        board: ${{ fromJSON(needs.test.outputs.boards-arm) }}
+    steps:
+    - name: Set up repository
+      uses: actions/checkout@v3
+      with:
+        submodules: false
+        fetch-depth: 1
+    - name: Set up python
+      uses: actions/setup-python@v4
+      with:
+        python-version: "3.x"
+    - name: Create submodule status
+      run: git submodule status ${{ join(fromJSON(env.CACHE_SUBMODULES), ' ') }} >> submodule_status
+    - name: Restore submodules
+      uses: actions/cache/restore@v3
+      with:
+        path: ".git/modules/\n${{ join(fromJSON(env.CACHE_SUBMODULES), '\n') }}"
+        key: submodules-common-${{ hashFiles('submodule_status') }}
+    - name: CircuitPython dependencies
+      id: cp-deps
+      run: python tools/ci_fetch_deps.py ${{ matrix.board }}
+    - uses: carlosperate/arm-none-eabi-gcc-action@v1
+      with:
+        release: '10-2020-q4'
+    - name: Install dependencies
+      run: |
+        sudo apt-get install -y gettext
+        pip install -r requirements-dev.txt
+    - name: Versions
+      run: |
+        gcc --version
+        arm-none-eabi-gcc --version
+        python3 --version
+    - name: Build mpy-cross
+      if: ${{ steps.cp-deps.outputs.frozen_tags == 'True' }}
+      run: make -C mpy-cross -j2
+    - name: Setup build failure matcher
+      run: echo "::add-matcher::$GITHUB_WORKSPACE/.github/workflows/match-build-fail.json"
+    - name: Build
+      run: python3 -u build_release_files.py
+      working-directory: tools
+      env:
+        BOARDS: ${{ matrix.board }}
+    - uses: actions/upload-artifact@v3
+      with:
+        name: ${{ matrix.board }}
+        path: bin/${{ matrix.board }}
+    - name: Upload to S3
+      if: >-
+        (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') ||
+        (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))
+      run: |
+        pip install awscli
+        [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp bin/ s3://adafruit-circuit-python/bin/ --recursive --no-progress --region us-east-1
+      env:
+        AWS_PAGER: ''
+        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+
+
+
+  build-espressif:
+    runs-on: ubuntu-20.04
+    needs: test
+    if: ${{ needs.test.outputs.boards-espressif != '[]' }}
+    env:
+      CP_VERSION: ${{ needs.test.outputs.cp-version }}
+      IDF_PATH: ${{ github.workspace }}/ports/espressif/esp-idf
+      IDF_TOOLS_PATH: ${{ github.workspace }}/.idf_tools
+    strategy:
+      fail-fast: false
+      matrix:
+        board: ${{ fromJSON(needs.test.outputs.boards-espressif) }}
+    steps:
+    - name: Set up repository
+      uses: actions/checkout@v3
+      with:
+        submodules: false
+        fetch-depth: 1
+    - name: Set up python
+      id: setup-python
+      uses: actions/setup-python@v4
+      with:
+        python-version: "3.10"
+    - name: Create submodule status
+      run: git submodule status ${{ join(fromJSON(env.CACHE_SUBMODULES), ' ') }} >> submodule_status
+    - name: Restore submodules
+      uses: actions/cache/restore@v3
+      with:
+        path: ".git/modules/\n${{ join(fromJSON(env.CACHE_SUBMODULES), '\n') }}"
+        key: submodules-common-${{ hashFiles('submodule_status') }}
+    - name: Get IDF commit
+      id: idf-commit
+      run: |
+        COMMIT=$(git submodule status ports/espressif/esp-idf | grep -o -P '(?<=^-).*(?= )')
+        echo "$COMMIT"
+        echo "commit=$COMMIT" >> $GITHUB_OUTPUT
+    - name: Cache IDF submodules
+      uses: actions/cache@v3
+      with:
+        path: |
+          .git/modules/ports/espressif/esp-idf
+          ports/espressif/esp-idf
+        key: submodules-idf-${{ steps.idf-commit.outputs.commit }}
+    - name: CircuitPython dependencies
+      id: cp-deps
+      run: python tools/ci_fetch_deps.py ${{ matrix.board }}
+    - name: CircuitPython version
+      run: tools/describe
+    - name: Cache IDF tools
+      uses: actions/cache@v3
+      with:
+        path: ${{ env.IDF_TOOLS_PATH }}
+        key: ${{ runner.os }}-Python-${{ steps.setup-python.outputs.python-version }}-tools-idf-${{ steps.idf-commit.outputs.commit }}
+    - name: Clone IDF submodules
+      run: git submodule update --init --depth=1
+      working-directory: ${{ env.IDF_PATH }}
+    - name: Install IDF tools
+      run: |
+        echo "Installing ESP-IDF tools"
+        $IDF_PATH/tools/idf_tools.py --non-interactive install required
+        $IDF_PATH/tools/idf_tools.py --non-interactive install cmake
+        echo "Installing Python environment and packages"
+        $IDF_PATH/tools/idf_tools.py --non-interactive install-python-env
+        rm -rf $IDF_TOOLS_PATH/dist
+    - name: Install dependencies
+      run: |
+        source $IDF_PATH/export.sh
+        sudo apt-get install -y gettext ninja-build
+        pip install -r requirements-dev.txt
+    - name: Versions
+      run: |
+        source $IDF_PATH/export.sh
+        gcc --version
+        python3 --version
+        ninja --version
+        cmake --version
+    - name: Build mpy-cross
+      if: ${{ steps.cp-deps.outputs.frozen_tags == 'True' }}
+      run: make -C mpy-cross -j2
+    - name: Setup build failure matcher
+      run: echo "::add-matcher::$GITHUB_WORKSPACE/.github/workflows/match-build-fail.json"
+    - name: Build
+      run: |
+        source $IDF_PATH/export.sh
+        python3 -u build_release_files.py
+      working-directory: tools
+      env:
+        BOARDS: ${{ matrix.board }}
+    - uses: actions/upload-artifact@v3
+      with:
+        name: ${{ matrix.board }}
+        path: bin/${{ matrix.board }}
+    - name: Upload to S3
+      if: >-
+        (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') ||
+        (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))
+      run: |
+        pip install awscli
+        [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp bin/ s3://adafruit-circuit-python/bin/ --recursive --no-progress --region us-east-1
+      env:
+        AWS_PAGER: ''
+        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+
+
+  build-riscv:
+    runs-on: ubuntu-20.04
+    needs: test
+    if: ${{ needs.test.outputs.boards-riscv != '[]' }}
+    env:
+      CP_VERSION: ${{ needs.test.outputs.cp-version }}
+    strategy:
+      fail-fast: false
+      matrix:
+        board: ${{ fromJSON(needs.test.outputs.boards-riscv) }}
+    steps:
+    - name: Set up repository
+      uses: actions/checkout@v3
+      with:
+        submodules: false
+        fetch-depth: 1
+    - name: Set up python
+      uses: actions/setup-python@v4
+      with:
+        python-version: "3.x"
+    - name: Create submodule status
+      run: git submodule status ${{ join(fromJSON(env.CACHE_SUBMODULES), ' ') }} >> submodule_status
+    - name: Restore submodules
+      uses: actions/cache/restore@v3
+      with:
+        path: ".git/modules/\n${{ join(fromJSON(env.CACHE_SUBMODULES), '\n') }}"
+        key: submodules-common-${{ hashFiles('submodule_status') }}
+    - name: CircuitPython dependencies
+      id: cp-deps
+      run: python tools/ci_fetch_deps.py ${{ matrix.board }}
+    - name: Install dependencies
+      run: |
+        sudo apt-get install -y gettext
+        pip install -r requirements-dev.txt
+        wget https://static.dev.sifive.com/dev-tools/riscv64-unknown-elf-gcc-8.3.0-2019.08.0-x86_64-linux-centos6.tar.gz
+        sudo tar -C /usr --strip-components=1 -xaf riscv64-unknown-elf-gcc-8.3.0-2019.08.0-x86_64-linux-centos6.tar.gz
+    - name: Versions
+      run: |
+        gcc --version
+        riscv64-unknown-elf-gcc --version
+        python3 --version
+    - name: Build mpy-cross
+      if: ${{ steps.cp-deps.outputs.frozen_tags == 'True' }}
+      run: make -C mpy-cross -j2
+    - name: Setup build failure matcher
+      run: echo "::add-matcher::$GITHUB_WORKSPACE/.github/workflows/match-build-fail.json"
+    - name: Build
+      run: python3 -u build_release_files.py
+      working-directory: tools
+      env:
+        BOARDS: ${{ matrix.board }}
+    - uses: actions/upload-artifact@v3
+      with:
+        name: ${{ matrix.board }}
+        path: bin/${{ matrix.board }}
+    - name: Upload to S3
+      if: >-
+        (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') ||
+        (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))
+      run: |
+        pip install awscli
+        [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp bin/ s3://adafruit-circuit-python/bin/ --recursive --no-progress --region us-east-1
       env:
         AWS_PAGER: ''
         AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
         AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-      if: (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') || (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))

.github/workflows/ports_windows.yml

@@ -4,7 +4,7 @@ on:
   push:
   pull_request:
     paths:
-      - '.github/workflows/*.yml'
+      - '.github/workflows/ports_windows.yml'
       - 'extmod/**'
       - 'lib/**'
       - 'mpy-cross/**'
@@ -71,12 +71,31 @@ jobs:
         which python; python --version; python -c "import cascadetoml"
         which python3; python3 --version; python3 -c "import cascadetoml"
 
-    - uses: actions/checkout@v3
+    - name: Set up repository
+      uses: actions/checkout@v3
       with:
         submodules: false
         fetch-depth: 1
-    - name: Get CP deps
-      run: python tools/ci_fetch_deps.py windows ${{ github.sha }}
+    - name: Create submodule status
+      run: git submodule status extmod/ulab lib/ tools/ >> submodule_status
+    - name: Restore submodules
+      uses: actions/cache/restore@v3
+      with:
+        path: |
+          .git/modules/
+          extmod/ulab
+          lib/
+          tools/
+        key: submodules-common-${{ hashFiles('submodule_status') }}
+        enableCrossOsArchive: true
+    - name: CircuitPython dependencies
+      run: |
+        python tools/ci_fetch_deps.py windows
+        echo "::group::Fetch history and tags"
+        git fetch --no-recurse-submodules --shallow-since="2021-07-01" --tags https://github.com/adafruit/circuitpython HEAD
+        git fetch --no-recurse-submodules --shallow-since="2021-07-01" origin $GITHUB_SHA
+        git repack -d
+        echo "::endgroup::"
     - name: CircuitPython version
       run: |
         tools/describe || git log --parents HEAD~4..

requirements-ci.txt

@@ -1,2 +1,2 @@
 # For uploading artifacts
-awscli
+# awscli

tools/ci_fetch_deps.py

@@ -1,19 +1,16 @@
-import pathlib
-import shlex
-import subprocess
+import os
 import sys
 import time
+import shlex
+import pathlib
+import subprocess
 
-# target will be a board, "test", "docs", "mpy-cross-mac", or "windows"
-
-target = sys.argv[1]
-ref = sys.argv[2]
-
-print(target, ref)
+# Target will be a board, "test", "docs", "mpy-cross-mac", or "windows"
+TARGET = sys.argv[1]
 
 # Submodules needed by port builds outside of their ports directory.
 # Should we try and detect these?
-port_deps = {
+PORT_DEPS = {
     "atmel-samd": [
         "extmod/ulab/",
         "lib/adafruit_floppy/",
@@ -58,41 +55,40 @@ def run(title, command, check=True):
     try:
         subprocess.run(shlex.split(command), stderr=subprocess.STDOUT, check=check)
     finally:
-        print("Duration:", time.monotonic() - start, flush=True)
         print("::endgroup::", flush=True)
+        print("Duration:", time.monotonic() - start, flush=True)
 
 
-run(
-    "Fetch back to the start of 2021 to get tag history",
-    'git fetch --tags --recurse-submodules=no --shallow-since="2021-07-01" https://github.com/adafruit/circuitpython HEAD',
-)
-run(
-    "Fetch back to the start of 2021 to get commit history",
-    f'git fetch --recurse-submodules=no --shallow-since="2021-07-01" origin {ref}',
-)
-# See https://stackoverflow.com/questions/63878612/git-fatal-error-in-object-unshallow-sha-1#comment118418373_63879454
-run('Fix for bug "fatal: error in object: unshallow"', "git repack -d")
-run("Init submodules", "git submodule init")
-run("Submodule status", "git submodule status")
+def set_output(name, value):
+    if "GITHUB_OUTPUT" in os.environ:
+        with open(os.environ["GITHUB_OUTPUT"], "at") as f:
+            print(f"{name}={value}", file=f)
+    else:
+        print(f"Would set GitHub actions output {name} to '{value}'")
 
-submodules = []
 
-if target == "test":
-    submodules = ["extmod/", "lib/", "tools/", "extmod/ulab", "lib/berkeley-db-1.xx"]
-elif target == "docs":
+def main():
+    submodules = []
+    submodules_tags = []
+
+    print("Target:", TARGET)
+
+    if TARGET == "test":
+        submodules = ["extmod/ulab", "lib/", "tools/"]
+    elif TARGET == "build-doc":
         # used in .readthedocs.yml to generate RTD
-    submodules = ["extmod/ulab/", "frozen/"]
-elif target == "mpy-cross-mac":
+        submodules = ["extmod/ulab", "frozen/"]
+    elif TARGET == "mpy-cross-mac":
         submodules = ["tools/"]  # for huffman
-elif target == "windows":
+    elif TARGET == "windows":
         # This builds one board from a number of ports so fill out a bunch of submodules
-    submodules = ["extmod/", "lib/", "tools/", "ports/", "data/nvm.toml/"]
-elif target == "website":
+        submodules = ["extmod/ulab", "lib/", "tools/", "ports/", "data/nvm.toml"]
+    elif TARGET == "website":
         submodules = ["tools/adabot/", "frozen/"]
-else:
-    p = list(pathlib.Path(".").glob(f"ports/*/boards/{target}/mpconfigboard.mk"))
+    else:
+        p = list(pathlib.Path(".").glob(f"ports/*/boards/{TARGET}/mpconfigboard.mk"))
         if not p:
-        raise RuntimeError(f"Unsupported target: {target}")
+            raise RuntimeError(f"Unsupported target: {TARGET}")
 
         config = p[0]
         # Add the ports folder to init submodules
@@ -100,7 +96,7 @@ else:
         port = port_folder.name
         submodules.append(str(port_folder))
         submodules.append("tools/")  # for huffman
-    submodules.extend(port_deps[port])
+        submodules.extend(PORT_DEPS[port])
         with config.open() as f:
             for line in f.readlines():
                 prefix = "FROZEN_MPY_DIRS += $(TOP)/"
@@ -111,20 +107,30 @@ else:
                     if lib_folder.count("/") > 1:
                         lib_folder = lib_folder.split("/", maxsplit=2)
                         lib_folder = "/".join(lib_folder[:2])
-                submodules.append(lib_folder)
+                    submodules_tags.append(lib_folder)
 
-print(submodules)
-if submodules:
-    submodules = " ".join(submodules)
-    # This line will fail because the submodule's need different commits than the tip of the branch. We
-    # fix it later.
+    print("Submodule tags[Y]:", submodules_tags)
+    print("Submodule tags[N]:", submodules)
+
+    if submodules_tags:
         run(
-        "Init the submodules we'll need",
-        f"git submodule update --init -N --depth 1 {submodules}",
-        check=False,
+            "Init the submodules with tags",
+            f"git submodule update --init {' '.join(submodules_tags)}",
         )
 
+    if submodules:
         run(
-        "Fetch the submodule sha",
-        "git submodule foreach 'git fetch --tags --depth 1 origin $sha1 && git checkout -q $sha1'",
+            "Init the submodules without tags",
+            f"git submodule update --init --depth=1 {' '.join(submodules)}",
         )
+
+    for submodule in submodules_tags:
+        if submodule.startswith("frozen"):
+            set_output("frozen_tags", True)
+            break
+    else:
+        set_output("frozen_tags", False)
+
+
+if __name__ == "__main__":
+    main()

tools/describe

@@ -1,3 +1,6 @@
 #!/bin/sh
-# Add any supplied arguments.
-git describe --first-parent --dirty --tags --match "[1-9].*" "$@"
+if [ -z "$CP_VERSION" ]; then
+    git describe --first-parent --dirty --tags --match "[1-9].*" "$@"
+else
+    echo $CP_VERSION
+fi