cache common submodules and share cp version

.github/workflows/build.yml

@@ -21,29 +21,54 @@ jobs:
     runs-on: ubuntu-20.04
     outputs:
       build-doc: ${{ steps.set-matrix.outputs.build-doc }}
-      boards-arm: ${{ steps.set-matrix.outputs.boards-arm }}
-      boards-riscv: ${{ steps.set-matrix.outputs.boards-riscv }}
-      boards-espressif: ${{ steps.set-matrix.outputs.boards-espressif }}
       boards-aarch: ${{ steps.set-matrix.outputs.boards-aarch }}
+      boards-arm: ${{ steps.set-matrix.outputs.boards-arm }}
+      boards-espressif: ${{ steps.set-matrix.outputs.boards-espressif }}
+      boards-riscv: ${{ steps.set-matrix.outputs.boards-riscv }}
+      cp-version: ${{ steps.cp-version.outputs.cp-version }}
     steps:
     - name: Dump GitHub context
       run: echo "$GITHUB_CONTEXT"
       env:
         GITHUB_CONTEXT: ${{ toJson(github) }}
-    - uses: actions/checkout@v3
+    - name: Set up repository
+      uses: actions/checkout@v3
       with:
         submodules: false
         fetch-depth: 1
-    - name: Set up Python 3
+    - name: Set up python
       uses: actions/setup-python@v4
       with:
         python-version: "3.x"
-    - name: Get CP deps
+    - name: Duplicate USB VID/PID check
-      run: python tools/ci_fetch_deps.py test ${{ github.sha }}
+      run: python3 -u -m tools.ci_check_duplicate_usb_vid_pid
-    - name: CircuitPython version
+    - name: Create submodule status
+      run: git submodule status extmod/ulab frozen/ lib/ tools/ >> submodule_status
+    - name: Cache submodules
+      uses: actions/cache@v3
+      with:
+        path: |
+          .git/modules/
+          extmod/ulab
+          frozen/
+          lib/
+          tools/
+        key: submodules-common-${{ hashFiles('submodule_status') }}
+    - name: CircuitPython dependencies
       run: |
-        tools/describe || git log --parents HEAD~4..
+        python tools/ci_fetch_deps.py ${{ github.job }}
-        echo >>$GITHUB_ENV CP_VERSION=$(tools/describe)
+        echo "::group::Fetch history and tags"
+        git fetch --no-recurse-submodules --shallow-since="2021-07-01" --tags https://github.com/adafruit/circuitpython HEAD
+        git fetch --no-recurse-submodules --shallow-since="2021-07-01" origin $GITHUB_SHA
+        git repack -d
+        echo "::endgroup::"
+    - name: CircuitPython version
+      id: cp-version
+      run: |
+        CP_VERSION=$(tools/describe)
+        echo "$CP_VERSION"
+        echo "CP_VERSION=$CP_VERSION" >> $GITHUB_ENV
+        echo "cp-version=$CP_VERSION" >> $GITHUB_OUTPUT
     - name: Install dependencies
       run: |
         sudo apt-get update
@@ -54,8 +79,6 @@ jobs:
       run: |
         gcc --version
         python3 --version
-    - name: Duplicate USB VID/PID Check
-      run: python3 -u -m tools.ci_check_duplicate_usb_vid_pid
     - name: Build mpy-cross
       run: make -C mpy-cross -j2
     - name: Build unix port
@@ -160,25 +183,36 @@ jobs:
 
   mpy-cross-mac:
     runs-on: macos-11
+    needs: test
+    if: ${{ needs.test.outputs.boards-aarch != '[]' || needs.test.outputs.boards-arm != '[]' || needs.scheduler.test.boards-espressif != '[]' || needs.scheduler.test.boards-riscv != '[]' }}
+    env:
+      CP_VERSION: ${{ needs.test.outputs.cp-version }}
     steps:
-    - name: Dump GitHub context
+    - name: Set up repository
-      env:
+      uses: actions/checkout@v3
-        GITHUB_CONTEXT: ${{ toJson(github) }}
-      run: echo "$GITHUB_CONTEXT"
-    - uses: actions/checkout@v3
       with:
         submodules: false
         fetch-depth: 1
-    - name: Set up Python 3
+    - name: Set up python
       uses: actions/setup-python@v4
       with:
-        python-version: "3.10"
+        python-version: "3.x"
-    - name: Get CP deps
+    - name: Create submodule status
-      run: python tools/ci_fetch_deps.py mpy-cross-mac ${{ github.sha }}
+      run: git submodule status extmod/ulab frozen/ lib/ tools/ >> submodule_status
+    - name: Restore submodules
+      uses: actions/cache/restore@v3
+      with:
+        path: |
+          .git/modules/
+          extmod/ulab
+          frozen/
+          lib/
+          tools/
+        key: submodules-common-${{ hashFiles('submodule_status') }}
+    - name: CircuitPython dependencies
+      run: python tools/ci_fetch_deps.py ${{ github.job }}
     - name: CircuitPython version
-      run: |
+      run: tools/describe
-        tools/describe || git log --parents HEAD~4..
-        echo >>$GITHUB_ENV CP_VERSION=$(tools/describe)
     - name: Install dependencies
       run: |
         brew install gettext
@@ -222,21 +256,34 @@ jobs:
     runs-on: ubuntu-20.04
     needs: test
     if: ${{ needs.test.outputs.build-doc == 'True' }}
+    env:
+      CP_VERSION: ${{ needs.test.outputs.cp-version }}
     steps:
-    - uses: actions/checkout@v3
+    - name: Set up repository
+      uses: actions/checkout@v3
       with:
         submodules: false
         fetch-depth: 1
-    - name: Get CP deps
+    - name: Create submodule status
-      run: python tools/ci_fetch_deps.py docs ${{ github.sha }}
+      run: git submodule status extmod/ulab frozen/ lib/ tools/ >> submodule_status
+    - name: Restore submodules
+      uses: actions/cache/restore@v3
+      with:
+        path: |
+          .git/modules/
+          extmod/ulab
+          frozen/
+          lib/
+          tools/
+        key: submodules-common-${{ hashFiles('submodule_status') }}
+    - name: CircuitPython dependencies
+      run: python tools/ci_fetch_deps.py ${{ github.job }}
     - name: CircuitPython version
-      run: |
+      run: tools/describe
-        tools/describe || git log --parents HEAD~4..
+    - name: Set up python
-        echo >>$GITHUB_ENV CP_VERSION=$(tools/describe)
-    - name: Set up Python 3
       uses: actions/setup-python@v4
       with:
-        python-version: "3.10"
+        python-version: "3.x"
     - name: Install dependencies
       run: |
         sudo apt-get update
@@ -285,22 +332,37 @@ jobs:
   build-arm:
     runs-on: ubuntu-20.04
     needs: test
+    if: ${{ needs.test.outputs.boards-arm != '[]' }}
+    env:
+      CP_VERSION: ${{ needs.test.outputs.cp-version }}
     strategy:
       fail-fast: false
       matrix:
         board: ${{ fromJSON(needs.test.outputs.boards-arm) }}
-    if: ${{ needs.test.outputs.boards-arm != '[]' }}
     steps:
-    - name: Set up Python 3
+    - name: Set up repository
-      uses: actions/setup-python@v4
+      uses: actions/checkout@v3
-      with:
-        python-version: "3.10"
-    - uses: actions/checkout@v3
       with:
         submodules: false
         fetch-depth: 1
-    - name: Get CP deps
+    - name: Set up python
-      run: python tools/ci_fetch_deps.py ${{ matrix.board }} ${{ github.sha }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: "3.x"
+    - name: Create submodule status
+      run: git submodule status extmod/ulab frozen/ lib/ tools/ >> submodule_status
+    - name: Restore submodules
+      uses: actions/cache/restore@v3
+      with:
+        path: |
+          .git/modules/
+          extmod/ulab
+          frozen/
+          lib/
+          tools/
+        key: submodules-common-${{ hashFiles('submodule_status') }}
+    - name: CircuitPython dependencies
+      run: python tools/ci_fetch_deps.py ${{ matrix.board }}
     - uses: carlosperate/arm-none-eabi-gcc-action@v1
       with:
         release: '10-2020-q4'
@@ -338,22 +400,37 @@ jobs:
   build-riscv:
     runs-on: ubuntu-20.04
     needs: test
+    if: ${{ needs.test.outputs.boards-riscv != '[]' }}
+    env:
+      CP_VERSION: ${{ needs.test.outputs.cp-version }}
     strategy:
       fail-fast: false
       matrix:
         board: ${{ fromJSON(needs.test.outputs.boards-riscv) }}
-    if: ${{ needs.test.outputs.boards-riscv != '[]' }}
     steps:
-    - name: Set up Python 3
+    - name: Set up repository
-      uses: actions/setup-python@v4
+      uses: actions/checkout@v3
-      with:
-        python-version: "3.10"
-    - uses: actions/checkout@v3
       with:
         submodules: false
         fetch-depth: 1
-    - name: Get CP deps
+    - name: Set up python
-      run: python tools/ci_fetch_deps.py ${{ matrix.board }} ${{ github.sha }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: "3.x"
+    - name: Create submodule status
+      run: git submodule status extmod/ulab frozen/ lib/ tools/ >> submodule_status
+    - name: Restore submodules
+      uses: actions/cache/restore@v3
+      with:
+        path: |
+          .git/modules/
+          extmod/ulab
+          frozen/
+          lib/
+          tools/
+        key: submodules-common-${{ hashFiles('submodule_status') }}
+    - name: CircuitPython dependencies
+      run: python tools/ci_fetch_deps.py ${{ matrix.board }}
     - name: Install dependencies
       run: |
         sudo apt-get install -y gettext
@@ -390,27 +467,40 @@ jobs:
   build-espressif:
     runs-on: ubuntu-20.04
     needs: test
+    if: ${{ needs.test.outputs.boards-espressif != '[]' }}
+    env:
+      CP_VERSION: ${{ needs.test.outputs.cp-version }}
     strategy:
       fail-fast: false
       matrix:
         board: ${{ fromJSON(needs.test.outputs.boards-espressif) }}
-    if: ${{ needs.test.outputs.boards-espressif != '[]' }}
     steps:
-    - name: Set up Python 3
+    - name: Set up repository
+      uses: actions/checkout@v3
+      with:
+        submodules: false
+        fetch-depth: 1
+    - name: Set up python
       id: py3
       uses: actions/setup-python@v4
       with:
         python-version: "3.10"
-    - uses: actions/checkout@v3
+    - name: Create submodule status
+      run: git submodule status extmod/ulab frozen/ lib/ tools/ >> submodule_status
+    - name: Restore submodules
+      uses: actions/cache/restore@v3
       with:
-        submodules: false
+        path: |
-        fetch-depth: 1
+          .git/modules/
-    - name: Get CP deps
+          extmod/ulab
-      run: python tools/ci_fetch_deps.py ${{ matrix.board }} ${{ github.sha }}
+          frozen/
+          lib/
+          tools/
+        key: submodules-common-${{ hashFiles('submodule_status') }}
+    - name: CircuitPython dependencies
+      run: python tools/ci_fetch_deps.py ${{ matrix.board }}
     - name: CircuitPython version
-      run: |
+      run: tools/describe
-        tools/describe || git log --parents HEAD~4..
-        echo >>$GITHUB_ENV CP_VERSION=$(tools/describe)
     - uses: actions/cache@v3
       name: Fetch IDF tool cache
       id: idf-cache
@@ -481,22 +571,37 @@ jobs:
   build-aarch:
     runs-on: ubuntu-20.04
     needs: test
+    if: ${{ needs.test.outputs.boards-aarch != '[]' }}
+    env:
+      CP_VERSION: ${{ needs.test.outputs.cp-version }}
     strategy:
       fail-fast: false
       matrix:
         board: ${{ fromJSON(needs.test.outputs.boards-aarch) }}
-    if: ${{ needs.test.outputs.boards-aarch != '[]' }}
     steps:
-    - name: Set up Python 3
+    - name: Set up repository
-      uses: actions/setup-python@v4
+      uses: actions/checkout@v3
-      with:
-        python-version: "3.10"
-    - uses: actions/checkout@v3
       with:
         submodules: false
         fetch-depth: 1
-    - name: Get CP deps
+    - name: Set up python
-      run: python tools/ci_fetch_deps.py ${{ matrix.board }} ${{ github.sha }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: "3.x"
+    - name: Create submodule status
+      run: git submodule status extmod/ulab frozen/ lib/ tools/ >> submodule_status
+    - name: Restore submodules
+      uses: actions/cache/restore@v3
+      with:
+        path: |
+          .git/modules/
+          extmod/ulab
+          frozen/
+          lib/
+          tools/
+        key: submodules-common-${{ hashFiles('submodule_status') }}
+    - name: CircuitPython dependencies
+      run: python tools/ci_fetch_deps.py ${{ matrix.board }}
     - name: Install dependencies
       run: |
         sudo apt-get install -y gettext mtools

.github/workflows/ports_windows.yml

@@ -71,12 +71,32 @@ jobs:
         which python; python --version; python -c "import cascadetoml"
         which python3; python3 --version; python3 -c "import cascadetoml"
 
-    - uses: actions/checkout@v3
+    - name: Set up repository
+      uses: actions/checkout@v3
       with:
         submodules: false
         fetch-depth: 1
-    - name: Get CP deps
+    - name: Create submodule status
-      run: python tools/ci_fetch_deps.py windows ${{ github.sha }}
+      run: git submodule status extmod/ulab frozen/ lib/ tools/ >> submodule_status
+    - name: Restore submodules
+      uses: actions/cache/restore@v3
+      with:
+        path: |
+          .git/modules/
+          extmod/ulab
+          frozen/
+          lib/
+          tools/
+        key: submodules-common-${{ hashFiles('submodule_status') }}
+        enableCrossOsArchive: true
+    - name: CircuitPython dependencies
+      run: |
+        python tools/ci_fetch_deps.py windows
+        echo "::group::Fetch history and tags"
+        git fetch --no-recurse-submodules --shallow-since="2021-07-01" --tags https://github.com/adafruit/circuitpython HEAD
+        git fetch --no-recurse-submodules --shallow-since="2021-07-01" origin $GITHUB_SHA
+        git repack -d
+        echo "::endgroup::"
     - name: CircuitPython version
       run: |
         tools/describe || git log --parents HEAD~4..

tools/ci_fetch_deps.py

@@ -4,16 +4,12 @@ import subprocess
 import sys
 import time
 
-# target will be a board, "test", "docs", "mpy-cross-mac", or "windows"
+# Target will be a board, "test", "docs", "mpy-cross-mac", or "windows"
-
+TARGET = sys.argv[1]
-target = sys.argv[1]
-ref = sys.argv[2]
-
-print(target, ref)
 
 # Submodules needed by port builds outside of their ports directory.
 # Should we try and detect these?
-port_deps = {
+PORT_DEPS = {
     "atmel-samd": [
         "extmod/ulab/",
         "lib/adafruit_floppy/",
@@ -58,73 +54,58 @@ def run(title, command, check=True):
     try:
         subprocess.run(shlex.split(command), stderr=subprocess.STDOUT, check=check)
     finally:
-        print("Duration:", time.monotonic() - start, flush=True)
         print("::endgroup::", flush=True)
+        print("Duration:", time.monotonic() - start, flush=True)
 
 
-run(
+def main():
-    "Fetch back to the start of 2021 to get tag history",
+    submodules = []
-    'git fetch --tags --recurse-submodules=no --shallow-since="2021-07-01" https://github.com/adafruit/circuitpython HEAD',
+    print("Target:", TARGET)
-)
-run(
-    "Fetch back to the start of 2021 to get commit history",
-    f'git fetch --recurse-submodules=no --shallow-since="2021-07-01" origin {ref}',
-)
-# See https://stackoverflow.com/questions/63878612/git-fatal-error-in-object-unshallow-sha-1#comment118418373_63879454
-run('Fix for bug "fatal: error in object: unshallow"', "git repack -d")
-run("Init submodules", "git submodule init")
-run("Submodule status", "git submodule status")
 
-submodules = []
+    if TARGET == "scheduler":
+        submodules = ["extmod/ulab", "lib/", "tools/", "frozen/"]
+    elif TARGET == "mpy-cross-tests":
+        submodules = ["extmod/ulab", "lib/", "tools/"]
+    elif TARGET == "build-doc":
+        # used in .readthedocs.yml to generate RTD
+        submodules = ["extmod/ulab", "frozen/"]
+    elif TARGET == "mpy-cross-mac":
+        submodules = ["tools/"]  # for huffman
+    elif TARGET == "windows":
+        # This builds one board from a number of ports so fill out a bunch of submodules
+        submodules = ["extmod/ulab", "lib/", "tools/", "ports/", "data/nvm.toml"]
+    elif TARGET == "website":
+        submodules = ["tools/adabot/", "frozen/"]
+    else:
+        p = list(pathlib.Path(".").glob(f"ports/*/boards/{TARGET}/mpconfigboard.mk"))
+        if not p:
+            raise RuntimeError(f"Unsupported target: {TARGET}")
 
-if target == "test":
+        config = p[0]
-    submodules = ["extmod/", "lib/", "tools/", "extmod/ulab", "lib/berkeley-db-1.xx"]
+        # Add the ports folder to init submodules
-elif target == "docs":
+        port_folder = config.parents[2]
-    # used in .readthedocs.yml to generate RTD
+        port = port_folder.name
-    submodules = ["extmod/ulab/", "frozen/"]
+        submodules.append(str(port_folder))
-elif target == "mpy-cross-mac":
+        submodules.append("tools/")  # for huffman
-    submodules = ["tools/"]  # for huffman
+        submodules.extend(PORT_DEPS[port])
-elif target == "windows":
+        with config.open() as f:
-    # This builds one board from a number of ports so fill out a bunch of submodules
+            for line in f.readlines():
-    submodules = ["extmod/", "lib/", "tools/", "ports/", "data/nvm.toml/"]
+                prefix = "FROZEN_MPY_DIRS += $(TOP)/"
-elif target == "website":
+                if line.startswith(prefix):
-    submodules = ["tools/adabot/", "frozen/"]
+                    lib_folder = line.strip()[len(prefix) :]
-else:
+                    # Drop everything after the second folder because the frozen
-    p = list(pathlib.Path(".").glob(f"ports/*/boards/{target}/mpconfigboard.mk"))
+                    # folder may be inside the submodule.
-    if not p:
+                    if lib_folder.count("/") > 1:
-        raise RuntimeError(f"Unsupported target: {target}")
+                        lib_folder = lib_folder.split("/", maxsplit=2)
+                        lib_folder = "/".join(lib_folder[:2])
+                    submodules.append(lib_folder)
 
-    config = p[0]
+    print("Submodule paths:", submodules)
-    # Add the ports folder to init submodules
-    port_folder = config.parents[2]
-    port = port_folder.name
-    submodules.append(str(port_folder))
-    submodules.append("tools/")  # for huffman
-    submodules.extend(port_deps[port])
-    with config.open() as f:
-        for line in f.readlines():
-            prefix = "FROZEN_MPY_DIRS += $(TOP)/"
-            if line.startswith(prefix):
-                lib_folder = line.strip()[len(prefix) :]
-                # Drop everything after the second folder because the frozen
-                # folder may be inside the submodule.
-                if lib_folder.count("/") > 1:
-                    lib_folder = lib_folder.split("/", maxsplit=2)
-                    lib_folder = "/".join(lib_folder[:2])
-                submodules.append(lib_folder)
 
-print(submodules)
+    if submodules:
-if submodules:
+        submodules = " ".join(submodules)
-    submodules = " ".join(submodules)
+        run("Init the required submodules", f"git submodule update --init --depth=1 {submodules}")
-    # This line will fail because the submodule's need different commits than the tip of the branch. We
-    # fix it later.
-    run(
-        "Init the submodules we'll need",
-        f"git submodule update --init -N --depth 1 {submodules}",
-        check=False,
-    )
 
-    run(
+
-        "Fetch the submodule sha",
+if __name__ == "__main__":
-        "git submodule foreach 'git fetch --tags --depth 1 origin $sha1 && git checkout -q $sha1'",
+    main()
-    )

tools/describe

@@ -1,3 +1,6 @@
 #!/bin/sh
-# Add any supplied arguments.
+if [ -z "$CP_VERSION" ]; then
-git describe --first-parent --dirty --tags --match "[1-9].*" "$@"
+    git describe --first-parent --dirty --tags --match "[1-9].*" "$@"
+else
+    echo $CP_VERSION
+fi