use venv to cache python deps

.github/actions/deps/external/action.yml

@@ -12,8 +12,18 @@ inputs:
     - riscv
     - none
 
+  apt:
+    required: false
+    default: true
+    type: boolean
+
+  python:
+    required: false
+    default: true
+    type: boolean
+
 runs:
-  using: "composite"
+  using: composite
   steps:
     # aarch
     - name: Get aarch toolchain
@@ -74,8 +84,16 @@ runs:
       shell: bash
 
     # common
-    - name: Install dependencies
-      run: |
-        sudo apt-get install -y gettext
-        pip install -r requirements-dev.txt
+    - name: Cache python dependencies
+      if: inputs.python == 'true' && inputs.platform != 'espressif'
+      uses: ./.github/actions/deps/python
+      with:
+        action: ${{ fromJSON('["restore", "cache"]')[github.job == 'scheduler'] }}
+    - name: Install python dependencies
+      if: inputs.python == 'true'
+      run: pip install -r requirements-dev.txt
+      shell: bash
+    - name: Install dependencies
+      if: inputs.apt == 'true'
+      run: sudo apt-get install -y gettext
       shell: bash

.github/actions/deps/port/action.yml

@@ -43,7 +43,7 @@ runs:
       uses: actions/cache@v3
       with:
         path: ${{ env.IDF_TOOLS_PATH }}
-        key: tools-idf-${{ steps.idf-commit.outputs.commit }}-${{ runner.os }}-${{ env.pythonLocation }}
+        key: ${{ runner.os }}-${{ env.pythonLocation }}-tools-idf-${{ steps.idf-commit.outputs.commit }}
 
     - name: Initialize IDF submodules
       if: inputs.platform == 'espressif'

.github/actions/deps/python/action.yml

@@ -0,0 +1,35 @@
+name: Fetch python deps
+
+inputs:
+  action:
+    description: The cache action to use
+    required: false
+    default: restore
+    type: choice
+    options:
+    - cache
+    - restore
+
+runs:
+  using: composite
+  steps:
+    - name: Cache python dependencies
+      if: ${{ inputs.action == 'cache' }}
+      uses: actions/cache@v3
+      with:
+        path: .cp_tools
+        key: ${{ runner.os }}-${{ env.pythonLocation }}-tools-cp-${{ hashFiles('requirements-dev.txt') }}
+
+    - name: Restore python dependencies
+      if: ${{ inputs.action == 'restore' }}
+      uses: actions/cache/restore@v3
+      with:
+        path: .cp_tools
+        key: ${{ runner.os }}-${{ env.pythonLocation }}-tools-cp-${{ hashFiles('requirements-dev.txt') }}
+
+    - name: Set up venv
+      run: |
+        python -m venv .cp_tools
+        source .cp_tools/bin/activate
+        echo >> $GITHUB_PATH "$PATH"
+      shell: bash

.github/actions/deps/submodules/action.yml

@@ -12,7 +12,7 @@ inputs:
     default: '["extmod/ulab", "lib/", "tools/"]'
     type: string
 
-  cache:
+  action:
     description: 'The cache action to use'
     required: false
     default: 'restore'
@@ -47,7 +47,7 @@ runs:
       shell: bash
 
     - name: Cache submodules
-      if: ${{ inputs.cache == 'cache' }}
+      if: ${{ inputs.action == 'cache' }}
       uses: actions/cache@v3
       with:
         path: ".git/modules/\n${{ join(fromJSON(steps.create-submodule-status.outputs.submodules), '\n') }}"
@@ -55,7 +55,7 @@ runs:
         enableCrossOsArchive: true
 
     - name: Restore submodules
-      if: ${{ inputs.cache == 'restore' }}
+      if: ${{ inputs.action == 'restore' }}
       uses: actions/cache/restore@v3
       with:
         path: ".git/modules/\n${{ join(fromJSON(steps.create-submodule-status.outputs.submodules), '\n') }}"

.github/actions/mpy_cross/action.yml

@@ -1,10 +1,17 @@
 name: Set up mpy-cross
 
+inputs:
+  download:
+    required: false
+    default: true
+    type: boolean
+
 runs:
   using: composite
   steps:
     - name: Download mpy-cross
       id: download-mpy-cross
+      if: inputs.download == 'true'
       continue-on-error: true
       uses: actions/download-artifact@v3
       with:
@@ -12,18 +19,18 @@ runs:
         path: mpy-cross
 
     - name: Make mpy-cross executable
-      if: steps.download-mpy-cross.outcome == 'success'
+      if: inputs.download == 'true' && steps.download-mpy-cross.outcome == 'success'
       run: sudo chmod +x mpy-cross/mpy-cross
       shell: bash
 
     - name: Build mpy-cross
-      if: steps.download-mpy-cross.outcome == 'failure'
+      if: inputs.download == 'false' || steps.download-mpy-cross.outcome == 'failure'
       run: make -C mpy-cross -j2
       shell: bash
 
     - name: Upload mpy-cross
+      if: inputs.download == 'false' || steps.download-mpy-cross.outcome == 'failure'
       continue-on-error: true
-      if: steps.download-mpy-cross.outcome == 'failure'
       uses: actions/upload-artifact@v3
       with:
         name: mpy-cross

.github/workflows/build-boards.yml

@@ -35,14 +35,14 @@ jobs:
       with:
         python-version: 3.x
     - name: Set up port
-      uses: ./.github/actions/port_deps
+      uses: ./.github/actions/deps/port
       with:
         platform: ${{ inputs.platform }}
     - name: Set up submodules
       id: set-up-submodules
-      uses: ./.github/actions/fetch_submodules
+      uses: ./.github/actions/deps/submodules
     - name: Set up external
-      uses: ./.github/actions/external_deps
+      uses: ./.github/actions/deps/external
       with:
         platform: ${{ inputs.platform }}
     - name: Set up mpy-cross
@@ -64,7 +64,7 @@ jobs:
 
     - name: Set up build failure matcher
       run: echo "::add-matcher::$GITHUB_WORKSPACE/.github/workflows/match-build-fail.json"
-    - name: Build
+    - name: Build board
       run: python3 -u build_release_files.py
       working-directory: tools
       env:

.github/workflows/build-mpy-cross.yml

@@ -27,26 +27,22 @@ jobs:
       with:
         submodules: false
         fetch-depth: 1
-
     - name: Set up python
       uses: actions/setup-python@v4
       with:
         python-version: 3.x
-
     - name: Set up submodules
-      uses: ./.github/actions/fetch_submodules
+      uses: ./.github/actions/deps/submodules
       with:
         target: mpy-cross
-
-    - name: Install dependencies
-      run: |
-        sudo apt-get update
-        sudo apt-get install -y gettext
+    - name: Set up external
+      uses: ./.github/actions/deps/external
+      with:
+        python: false
 
     - name: Install toolchain (aarch64)
       if: matrix.mpy-cross == 'static-aarch64'
       run: sudo apt-get install -y gcc-aarch64-linux-gnu
-
     - name: Install toolchain (mingw)
       if: matrix.mpy-cross == 'static-mingw'
       run: sudo apt-get install -y mingw-w64
@@ -54,16 +50,20 @@ jobs:
     - name: Build mpy-cross.${{ matrix.mpy-cross }}
       run: make -C mpy-cross -j2 -f Makefile.${{ matrix.mpy-cross }}
 
+    - name: Set output
+      run: |
+        echo >> $GITHUB_ENV "EX=${{ env[format('EX_{0}', matrix.mpy-cross)] || matrix.mpy-cross }}"
+        echo >> $GITHUB_ENV "OS=${{ env[format('OS_{0}', matrix.mpy-cross)] }}"
+
     - name: Upload artifact
       uses: actions/upload-artifact@v3
       with:
-        name: mpy-cross.${{ env[format('EX_{0}', matrix.mpy-cross)] || matrix.mpy-cross }}
-        path: mpy-cross/mpy-cross.${{ env[format('EX_{0}', matrix.mpy-cross)] || matrix.mpy-cross }}
-
+        name: mpy-cross.${{ env.EX }}
+        path: mpy-cross/mpy-cross.${{ env.EX }}
     - name: Upload to S3
       uses: ./.github/actions/upload_aws
       with:
-        source: mpy-cross/mpy-cross.${{ env[format('EX_{0}', matrix.mpy-cross)] || matrix.mpy-cross }}
-        destination: mpy-cross/${{ env[format('OS_{0}', matrix.mpy-cross)] }}/mpy-cross-${{ env.CP_VERSION }}.${{ env[format('EX_{0}', matrix.mpy-cross)] || matrix.mpy-cross }}
+        source: mpy-cross/mpy-cross.${{ env.EX }}
+        destination: mpy-cross/${{ env.OS }}/mpy-cross-${{ env.OS }}-${{ env.CP_VERSION }}.${{ env.EX }}
         AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
         AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}

.github/workflows/build.yml

@@ -27,9 +27,6 @@ jobs:
       boards-espressif: ${{ steps.set-matrix.outputs.boards-espressif }}
       boards-riscv: ${{ steps.set-matrix.outputs.boards-riscv }}
       cp-version: ${{ steps.set-up-submodules.outputs.version }}
-    env:
-      MICROPY_CPYTHON3: python3.8
-      MICROPY_MICROPYTHON: ${{ github.workspace }}/ports/unix/micropython-coverage
     steps:
     - name: Dump GitHub context
       run: echo "$GITHUB_CONTEXT"
@@ -48,14 +45,12 @@ jobs:
       run: python3 -u -m tools.ci_check_duplicate_usb_vid_pid
     - name: Set up submodules
       id: set-up-submodules
-      uses: ./.github/actions/fetch_submodules
+      uses: ./.github/actions/deps/submodules
       with:
-        cache: cache
+        action: cache
         version: true
-    - name: Install dependencies
-      run: |
-        sudo apt-get install -y gettext
-        pip install requests requests-cache sh
+    - name: Set up external
+      uses: ./.github/actions/deps/external
     # Disabled: Needs to be updated
     # - name: Get last commit with checks
     #   id: get-last-commit-with-checks
@@ -69,6 +64,8 @@ jobs:
     #     EXCLUDE_COMMIT: ${{ github.event.after }}
     - name: Set up mpy-cross
       uses: ./.github/actions/mpy_cross
+      with:
+        download: false
     - name: Set head sha
       if: github.event_name == 'pull_request'
       run: echo "HEAD_SHA=$(git show -s --format=%s $GITHUB_SHA | grep -o -P "(?<=Merge ).*(?= into)")" >> $GITHUB_ENV
@@ -124,7 +121,7 @@ jobs:
       with:
         python-version: 3.x
     - name: Set up submodules
-      uses: ./.github/actions/fetch_submodules
+      uses: ./.github/actions/deps/submodules
     - name: Versions
       run: |
         gcc --version
@@ -179,7 +176,7 @@ jobs:
       with:
         python-version: 3.x
     - name: Set up submodules
-      uses: ./.github/actions/fetch_submodules
+      uses: ./.github/actions/deps/submodules
     - name: Install dependencies
       run: |
         sudo apt-get update

.github/workflows/create_website_pr.yml

@@ -24,13 +24,15 @@ jobs:
     - name: Set up python
       uses: actions/setup-python@v4
       with:
-        python-version: "3.x"
+        python-version: 3.x
     - name: Set up submodules
-      uses: ./.github/actions/fetch_submodules
+      uses: ./.github/actions/deps/submodules
       with:
         version: true
-    - name: Install dependencies
-      run: pip install -r requirements-dev.txt
+    - name: Set up external
+      uses: ./.github/actions/deps/external
+      with:
+        apt: false
     - name: Versions
       run: |
         gcc --version

.github/workflows/ports_windows.yml

@@ -77,7 +77,7 @@ jobs:
         fetch-depth: 1
 
     - name: Set up submodules
-      uses: ./.github/actions/fetch_submodules
+      uses: ./.github/actions/deps/submodules
       with:
         version: true
 

.github/workflows/pre-commit.yml

@@ -24,13 +24,13 @@ jobs:
     - name: Set up python
       uses: actions/setup-python@v4
       with:
-        python-version: "3.x"
-    - name: CircuitPython dependencies
-      run: python tools/ci_fetch_deps.py ${{ github.job }}
+        python-version: 3.x
+    - name: Set up submodules
+      uses: ./.github/actions/deps/submodules
+    - name: Set up external
+      uses: ./.github/actions/deps/external
     - name: Install dependencies
-      run: |
-        sudo apt-get install -y gettext uncrustify
-        pip3 install black polib pyyaml
+      run: sudo apt-get install -y uncrustify
     - name: Run pre-commit
       uses: pre-commit/action@v3.0.0
     - name: Make patch

.github/workflows/run-tests.yml

@@ -33,32 +33,22 @@ jobs:
       with:
         python-version: 3.x
     - name: Set up submodules
-      uses: ./.github/actions/fetch_submodules
+      uses: ./.github/actions/deps/submodules
       with:
         target: tests
-
-    - name: Install dependencies
-      run: |
-        sudo apt-get update
-        sudo apt-get install -y gettext
-
+    - name: Set up external
+      uses: ./.github/actions/deps/external
     - name: Set up mpy-cross
       uses: ./.github/actions/mpy_cross
-
     - name: Build unix port
       run: make -C ports/unix VARIANT=coverage -j2
-
-    - name: Test all
-      run: ./run-tests.py -j2 $TEST_${{ matrix.test }}
+    - name: Run tests
+      run: ./run-tests.py -j2 ${{ env[format('TEST_{0}', matrix.test)] }}
       working-directory: tests
     - name: Print failure info
       run: ./run-tests.py -j2 --print-failures
       if: failure()
       working-directory: tests
-
-    - name: Set up native modules
-      if: matrix.test == 'all'
-      run: pip install -r requirements-dev.txt
     - name: Build native modules
       if: matrix.test == 'all'
       run: |