From d959ef1e5b9c2f9b42e0169142dab1bdbd48b0fe Mon Sep 17 00:00:00 2001
From: MicroDev <70126934+MicroDev1@users.noreply.github.com>
Date: Sat, 11 Feb 2023 12:00:27 +0530
Subject: [PATCH] use venv to cache python deps

---
 .../external}/action.yml                      | 28 ++++++++++++---
 .../{port_deps => deps/port}/action.yml       |  2 +-
 .github/actions/deps/python/action.yml        | 35 +++++++++++++++++++
 .../submodules}/action.yml                    |  6 ++--
 .github/actions/mpy_cross/action.yml          | 13 +++++--
 .github/workflows/build-boards.yml            |  8 ++---
 .github/workflows/build-mpy-cross.yml         | 28 +++++++--------
 .github/workflows/build.yml                   | 19 +++++-----
 .github/workflows/create_website_pr.yml       | 10 +++---
 .github/workflows/ports_windows.yml           |  2 +-
 .github/workflows/pre-commit.yml              | 12 +++----
 .github/workflows/run-tests.yml               | 20 +++--------
 12 files changed, 116 insertions(+), 67 deletions(-)
 rename .github/actions/{external_deps => deps/external}/action.yml (79%)
 rename .github/actions/{port_deps => deps/port}/action.yml (92%)
 create mode 100644 .github/actions/deps/python/action.yml
 rename .github/actions/{fetch_submodules => deps/submodules}/action.yml (96%)

diff --git a/.github/actions/external_deps/action.yml b/.github/actions/deps/external/action.yml
similarity index 79%
rename from .github/actions/external_deps/action.yml
rename to .github/actions/deps/external/action.yml
index c9766eadc2..408a2e8555 100644
--- a/.github/actions/external_deps/action.yml
+++ b/.github/actions/deps/external/action.yml
@@ -12,8 +12,18 @@ inputs:
     - riscv
     - none
 
+  apt:
+    required: false
+    default: true
+    type: boolean
+
+  python:
+    required: false
+    default: true
+    type: boolean
+
 runs:
-  using: "composite"
+  using: composite
   steps:
     # aarch
     - name: Get aarch toolchain
@@ -74,8 +84,16 @@ runs:
       shell: bash
 
     # common
-    - name: Install dependencies
-      run: |
-        sudo apt-get install -y gettext
-        pip install -r requirements-dev.txt
+    - name: Cache python dependencies
+      if: inputs.python == 'true' && inputs.platform != 'espressif'
+      uses: ./.github/actions/deps/python
+      with:
+        action: ${{ fromJSON('["restore", "cache"]')[github.job == 'scheduler'] }}
+    - name: Install python dependencies
+      if: inputs.python == 'true'
+      run: pip install -r requirements-dev.txt
+      shell: bash
+    - name: Install dependencies
+      if: inputs.apt == 'true'
+      run: sudo apt-get install -y gettext
       shell: bash
diff --git a/.github/actions/port_deps/action.yml b/.github/actions/deps/port/action.yml
similarity index 92%
rename from .github/actions/port_deps/action.yml
rename to .github/actions/deps/port/action.yml
index c507983406..36b67c6081 100644
--- a/.github/actions/port_deps/action.yml
+++ b/.github/actions/deps/port/action.yml
@@ -43,7 +43,7 @@ runs:
       uses: actions/cache@v3
       with:
         path: ${{ env.IDF_TOOLS_PATH }}
-        key: tools-idf-${{ steps.idf-commit.outputs.commit }}-${{ runner.os }}-${{ env.pythonLocation }}
+        key: ${{ runner.os }}-${{ env.pythonLocation }}-tools-idf-${{ steps.idf-commit.outputs.commit }}
 
     - name: Initialize IDF submodules
       if: inputs.platform == 'espressif'
diff --git a/.github/actions/deps/python/action.yml b/.github/actions/deps/python/action.yml
new file mode 100644
index 0000000000..fb978ef063
--- /dev/null
+++ b/.github/actions/deps/python/action.yml
@@ -0,0 +1,35 @@
+name: Fetch python deps
+
+inputs:
+  action:
+    description: The cache action to use
+    required: false
+    default: restore
+    type: choice
+    options:
+    - cache
+    - restore
+
+runs:
+  using: composite
+  steps:
+    - name: Cache python dependencies
+      if: ${{ inputs.action == 'cache' }}
+      uses: actions/cache@v3
+      with:
+        path: .cp_tools
+        key: ${{ runner.os }}-${{ env.pythonLocation }}-tools-cp-${{ hashFiles('requirements-dev.txt') }}
+
+    - name: Restore python dependencies
+      if: ${{ inputs.action == 'restore' }}
+      uses: actions/cache/restore@v3
+      with:
+        path: .cp_tools
+        key: ${{ runner.os }}-${{ env.pythonLocation }}-tools-cp-${{ hashFiles('requirements-dev.txt') }}
+
+    - name: Set up venv
+      run: |
+        python -m venv .cp_tools
+        source .cp_tools/bin/activate
+        echo >> $GITHUB_PATH "$PATH"
+      shell: bash
diff --git a/.github/actions/fetch_submodules/action.yml b/.github/actions/deps/submodules/action.yml
similarity index 96%
rename from .github/actions/fetch_submodules/action.yml
rename to .github/actions/deps/submodules/action.yml
index c8c6c9b1a0..bca76b204c 100644
--- a/.github/actions/fetch_submodules/action.yml
+++ b/.github/actions/deps/submodules/action.yml
@@ -12,7 +12,7 @@ inputs:
     default: '["extmod/ulab", "lib/", "tools/"]'
     type: string
 
-  cache:
+  action:
     description: 'The cache action to use'
     required: false
     default: 'restore'
@@ -47,7 +47,7 @@ runs:
       shell: bash
 
     - name: Cache submodules
-      if: ${{ inputs.cache == 'cache' }}
+      if: ${{ inputs.action == 'cache' }}
       uses: actions/cache@v3
       with:
         path: ".git/modules/\n${{ join(fromJSON(steps.create-submodule-status.outputs.submodules), '\n') }}"
@@ -55,7 +55,7 @@ runs:
         enableCrossOsArchive: true
 
     - name: Restore submodules
-      if: ${{ inputs.cache == 'restore' }}
+      if: ${{ inputs.action == 'restore' }}
       uses: actions/cache/restore@v3
       with:
         path: ".git/modules/\n${{ join(fromJSON(steps.create-submodule-status.outputs.submodules), '\n') }}"
diff --git a/.github/actions/mpy_cross/action.yml b/.github/actions/mpy_cross/action.yml
index ea3466e5fb..d9fe54fdc7 100644
--- a/.github/actions/mpy_cross/action.yml
+++ b/.github/actions/mpy_cross/action.yml
@@ -1,10 +1,17 @@
 name: Set up mpy-cross
 
+inputs:
+  download:
+    required: false
+    default: true
+    type: boolean
+
 runs:
   using: composite
   steps:
     - name: Download mpy-cross
       id: download-mpy-cross
+      if: inputs.download == 'true'
       continue-on-error: true
       uses: actions/download-artifact@v3
       with:
@@ -12,18 +19,18 @@ runs:
         path: mpy-cross
 
     - name: Make mpy-cross executable
-      if: steps.download-mpy-cross.outcome == 'success'
+      if: inputs.download == 'true' && steps.download-mpy-cross.outcome == 'success'
       run: sudo chmod +x mpy-cross/mpy-cross
       shell: bash
 
     - name: Build mpy-cross
-      if: steps.download-mpy-cross.outcome == 'failure'
+      if: inputs.download == 'false' || steps.download-mpy-cross.outcome == 'failure'
       run: make -C mpy-cross -j2
       shell: bash
 
     - name: Upload mpy-cross
+      if: inputs.download == 'false' || steps.download-mpy-cross.outcome == 'failure'
       continue-on-error: true
-      if: steps.download-mpy-cross.outcome == 'failure'
       uses: actions/upload-artifact@v3
       with:
         name: mpy-cross
diff --git a/.github/workflows/build-boards.yml b/.github/workflows/build-boards.yml
index 9b53615fe7..c302be7c9f 100644
--- a/.github/workflows/build-boards.yml
+++ b/.github/workflows/build-boards.yml
@@ -35,14 +35,14 @@ jobs:
       with:
         python-version: 3.x
     - name: Set up port
-      uses: ./.github/actions/port_deps
+      uses: ./.github/actions/deps/port
       with:
         platform: ${{ inputs.platform }}
     - name: Set up submodules
       id: set-up-submodules
-      uses: ./.github/actions/fetch_submodules
+      uses: ./.github/actions/deps/submodules
     - name: Set up external
-      uses: ./.github/actions/external_deps
+      uses: ./.github/actions/deps/external
       with:
         platform: ${{ inputs.platform }}
     - name: Set up mpy-cross
@@ -64,7 +64,7 @@ jobs:
 
     - name: Set up build failure matcher
       run: echo "::add-matcher::$GITHUB_WORKSPACE/.github/workflows/match-build-fail.json"
-    - name: Build
+    - name: Build board
       run: python3 -u build_release_files.py
       working-directory: tools
       env:
diff --git a/.github/workflows/build-mpy-cross.yml b/.github/workflows/build-mpy-cross.yml
index 3d557df8ee..a2c0c64289 100644
--- a/.github/workflows/build-mpy-cross.yml
+++ b/.github/workflows/build-mpy-cross.yml
@@ -27,26 +27,22 @@ jobs:
       with:
         submodules: false
         fetch-depth: 1
-
     - name: Set up python
       uses: actions/setup-python@v4
       with:
         python-version: 3.x
-
     - name: Set up submodules
-      uses: ./.github/actions/fetch_submodules
+      uses: ./.github/actions/deps/submodules
       with:
         target: mpy-cross
-
-    - name: Install dependencies
-      run: |
-        sudo apt-get update
-        sudo apt-get install -y gettext
+    - name: Set up external
+      uses: ./.github/actions/deps/external
+      with:
+        python: false
 
     - name: Install toolchain (aarch64)
       if: matrix.mpy-cross == 'static-aarch64'
       run: sudo apt-get install -y gcc-aarch64-linux-gnu
-
     - name: Install toolchain (mingw)
       if: matrix.mpy-cross == 'static-mingw'
       run: sudo apt-get install -y mingw-w64
@@ -54,16 +50,20 @@ jobs:
     - name: Build mpy-cross.${{ matrix.mpy-cross }}
       run: make -C mpy-cross -j2 -f Makefile.${{ matrix.mpy-cross }}
 
+    - name: Set output
+      run: |
+        echo >> $GITHUB_ENV "EX=${{ env[format('EX_{0}', matrix.mpy-cross)] || matrix.mpy-cross }}"
+        echo >> $GITHUB_ENV "OS=${{ env[format('OS_{0}', matrix.mpy-cross)] }}"
+
     - name: Upload artifact
       uses: actions/upload-artifact@v3
       with:
-        name: mpy-cross.${{ env[format('EX_{0}', matrix.mpy-cross)] || matrix.mpy-cross }}
-        path: mpy-cross/mpy-cross.${{ env[format('EX_{0}', matrix.mpy-cross)] || matrix.mpy-cross }}
-
+        name: mpy-cross.${{ env.EX }}
+        path: mpy-cross/mpy-cross.${{ env.EX }}
     - name: Upload to S3
       uses: ./.github/actions/upload_aws
       with:
-        source: mpy-cross/mpy-cross.${{ env[format('EX_{0}', matrix.mpy-cross)] || matrix.mpy-cross }}
-        destination: mpy-cross/${{ env[format('OS_{0}', matrix.mpy-cross)] }}/mpy-cross-${{ env.CP_VERSION }}.${{ env[format('EX_{0}', matrix.mpy-cross)] || matrix.mpy-cross }}
+        source: mpy-cross/mpy-cross.${{ env.EX }}
+        destination: mpy-cross/${{ env.OS }}/mpy-cross-${{ env.OS }}-${{ env.CP_VERSION }}.${{ env.EX }}
         AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
         AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 37bb4133cd..8c40acd6d3 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -27,9 +27,6 @@ jobs:
       boards-espressif: ${{ steps.set-matrix.outputs.boards-espressif }}
       boards-riscv: ${{ steps.set-matrix.outputs.boards-riscv }}
       cp-version: ${{ steps.set-up-submodules.outputs.version }}
-    env:
-      MICROPY_CPYTHON3: python3.8
-      MICROPY_MICROPYTHON: ${{ github.workspace }}/ports/unix/micropython-coverage
     steps:
     - name: Dump GitHub context
       run: echo "$GITHUB_CONTEXT"
@@ -48,14 +45,12 @@ jobs:
       run: python3 -u -m tools.ci_check_duplicate_usb_vid_pid
     - name: Set up submodules
       id: set-up-submodules
-      uses: ./.github/actions/fetch_submodules
+      uses: ./.github/actions/deps/submodules
       with:
-        cache: cache
+        action: cache
         version: true
-    - name: Install dependencies
-      run: |
-        sudo apt-get install -y gettext
-        pip install requests requests-cache sh
+    - name: Set up external
+      uses: ./.github/actions/deps/external
     # Disabled: Needs to be updated
     # - name: Get last commit with checks
     #   id: get-last-commit-with-checks
@@ -69,6 +64,8 @@ jobs:
     #     EXCLUDE_COMMIT: ${{ github.event.after }}
     - name: Set up mpy-cross
       uses: ./.github/actions/mpy_cross
+      with:
+        download: false
     - name: Set head sha
       if: github.event_name == 'pull_request'
       run: echo "HEAD_SHA=$(git show -s --format=%s $GITHUB_SHA | grep -o -P "(?<=Merge ).*(?= into)")" >> $GITHUB_ENV
@@ -124,7 +121,7 @@ jobs:
       with:
         python-version: 3.x
     - name: Set up submodules
-      uses: ./.github/actions/fetch_submodules
+      uses: ./.github/actions/deps/submodules
     - name: Versions
       run: |
         gcc --version
@@ -179,7 +176,7 @@ jobs:
       with:
         python-version: 3.x
     - name: Set up submodules
-      uses: ./.github/actions/fetch_submodules
+      uses: ./.github/actions/deps/submodules
     - name: Install dependencies
       run: |
         sudo apt-get update
diff --git a/.github/workflows/create_website_pr.yml b/.github/workflows/create_website_pr.yml
index 19004d9bea..9f89490161 100644
--- a/.github/workflows/create_website_pr.yml
+++ b/.github/workflows/create_website_pr.yml
@@ -24,13 +24,15 @@ jobs:
     - name: Set up python
       uses: actions/setup-python@v4
       with:
-        python-version: "3.x"
+        python-version: 3.x
     - name: Set up submodules
-      uses: ./.github/actions/fetch_submodules
+      uses: ./.github/actions/deps/submodules
       with:
         version: true
-    - name: Install dependencies
-      run: pip install -r requirements-dev.txt
+    - name: Set up external
+      uses: ./.github/actions/deps/external
+      with:
+        apt: false
     - name: Versions
       run: |
         gcc --version
diff --git a/.github/workflows/ports_windows.yml b/.github/workflows/ports_windows.yml
index 66d09c2418..ba2042660a 100644
--- a/.github/workflows/ports_windows.yml
+++ b/.github/workflows/ports_windows.yml
@@ -77,7 +77,7 @@ jobs:
         fetch-depth: 1
 
     - name: Set up submodules
-      uses: ./.github/actions/fetch_submodules
+      uses: ./.github/actions/deps/submodules
       with:
         version: true
 
diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml
index 02aed6e9b4..fc36f2a505 100644
--- a/.github/workflows/pre-commit.yml
+++ b/.github/workflows/pre-commit.yml
@@ -24,13 +24,13 @@ jobs:
     - name: Set up python
       uses: actions/setup-python@v4
       with:
-        python-version: "3.x"
-    - name: CircuitPython dependencies
-      run: python tools/ci_fetch_deps.py ${{ github.job }}
+        python-version: 3.x
+    - name: Set up submodules
+      uses: ./.github/actions/deps/submodules
+    - name: Set up external
+      uses: ./.github/actions/deps/external
     - name: Install dependencies
-      run: |
-        sudo apt-get install -y gettext uncrustify
-        pip3 install black polib pyyaml
+      run: sudo apt-get install -y uncrustify
     - name: Run pre-commit
       uses: pre-commit/action@v3.0.0
     - name: Make patch
diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index 15fc8489a4..1ab0f6eeab 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -33,32 +33,22 @@ jobs:
       with:
         python-version: 3.x
     - name: Set up submodules
-      uses: ./.github/actions/fetch_submodules
+      uses: ./.github/actions/deps/submodules
       with:
         target: tests
-
-    - name: Install dependencies
-      run: |
-        sudo apt-get update
-        sudo apt-get install -y gettext
-
+    - name: Set up external
+      uses: ./.github/actions/deps/external
     - name: Set up mpy-cross
       uses: ./.github/actions/mpy_cross
-
     - name: Build unix port
       run: make -C ports/unix VARIANT=coverage -j2
-
-    - name: Test all
-      run: ./run-tests.py -j2 $TEST_${{ matrix.test }}
+    - name: Run tests
+      run: ./run-tests.py -j2 ${{ env[format('TEST_{0}', matrix.test)] }}
       working-directory: tests
     - name: Print failure info
       run: ./run-tests.py -j2 --print-failures
       if: failure()
       working-directory: tests
-
-    - name: Set up native modules
-      if: matrix.test == 'all'
-      run: pip install -r requirements-dev.txt
     - name: Build native modules
       if: matrix.test == 'all'
       run: |