From 434c2355a07c7ca2db7be16d545281f6d6cc598e Mon Sep 17 00:00:00 2001 From: MicroDev <70126934+MicroDev1@users.noreply.github.com> Date: Sat, 14 Jan 2023 17:10:40 +0530 Subject: [PATCH] multiple ci enhancements - selectively fetch submodule tags - conditionally build mpy-cross - remove frozen from submodule cache --- .github/workflows/build.yml | 45 +++++++++++++++-------------- .github/workflows/ports_windows.yml | 5 ++-- tools/ci_fetch_deps.py | 45 ++++++++++++++++++++++------- 3 files changed, 60 insertions(+), 35 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6a7a40c99c..7eb81bd8dd 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -43,14 +43,13 @@ jobs: - name: Duplicate USB VID/PID check run: python3 -u -m tools.ci_check_duplicate_usb_vid_pid - name: Create submodule status - run: git submodule status extmod/ulab frozen/ lib/ tools/ >> submodule_status + run: git submodule status extmod/ulab lib/ tools/ >> submodule_status - name: Cache submodules uses: actions/cache@v3 with: path: | .git/modules/ extmod/ulab - frozen/ lib/ tools/ key: submodules-common-${{ hashFiles('submodule_status') }} @@ -198,14 +197,13 @@ jobs: with: python-version: "3.x" - name: Create submodule status - run: git submodule status extmod/ulab frozen/ lib/ tools/ >> submodule_status + run: git submodule status extmod/ulab lib/ tools/ >> submodule_status - name: Restore submodules uses: actions/cache/restore@v3 with: path: | .git/modules/ extmod/ulab - frozen/ lib/ tools/ key: submodules-common-${{ hashFiles('submodule_status') }} @@ -265,14 +263,13 @@ jobs: submodules: false fetch-depth: 1 - name: Create submodule status - run: git submodule status extmod/ulab frozen/ lib/ tools/ >> submodule_status + run: git submodule status extmod/ulab lib/ tools/ >> submodule_status - name: Restore submodules uses: actions/cache/restore@v3 with: path: | .git/modules/ extmod/ulab - frozen/ lib/ tools/ key: submodules-common-${{ hashFiles('submodule_status') }} @@ -350,18 +347,18 @@ jobs: with: python-version: "3.x" - name: Create submodule status - run: git submodule status extmod/ulab frozen/ lib/ tools/ >> submodule_status + run: git submodule status extmod/ulab lib/ tools/ >> submodule_status - name: Restore submodules uses: actions/cache/restore@v3 with: path: | .git/modules/ extmod/ulab - frozen/ lib/ tools/ key: submodules-common-${{ hashFiles('submodule_status') }} - name: CircuitPython dependencies + id: cp-deps run: python tools/ci_fetch_deps.py ${{ matrix.board }} - uses: carlosperate/arm-none-eabi-gcc-action@v1 with: @@ -375,11 +372,12 @@ jobs: gcc --version arm-none-eabi-gcc --version python3 --version - - name: mpy-cross + - name: Build mpy-cross + if: ${{ steps.cp-deps.outputs.frozen_tags == 'True' }} run: make -C mpy-cross -j2 - name: Setup build failure matcher run: echo "::add-matcher::$GITHUB_WORKSPACE/.github/workflows/match-build-fail.json" - - name: build + - name: Build run: python3 -u build_release_files.py working-directory: tools env: @@ -418,18 +416,18 @@ jobs: with: python-version: "3.x" - name: Create submodule status - run: git submodule status extmod/ulab frozen/ lib/ tools/ >> submodule_status + run: git submodule status extmod/ulab lib/ tools/ >> submodule_status - name: Restore submodules uses: actions/cache/restore@v3 with: path: | .git/modules/ extmod/ulab - frozen/ lib/ tools/ key: submodules-common-${{ hashFiles('submodule_status') }} - name: CircuitPython dependencies + id: cp-deps run: python tools/ci_fetch_deps.py ${{ matrix.board }} - name: Install dependencies run: | @@ -442,11 +440,12 @@ jobs: gcc --version riscv64-unknown-elf-gcc --version python3 --version - - name: mpy-cross + - name: Build mpy-cross + if: ${{ steps.cp-deps.outputs.frozen_tags == 'True' }} run: make -C mpy-cross -j2 - name: Setup build failure matcher run: echo "::add-matcher::$GITHUB_WORKSPACE/.github/workflows/match-build-fail.json" - - name: build + - name: Build run: python3 -u build_release_files.py working-directory: tools env: @@ -486,18 +485,18 @@ jobs: with: python-version: "3.10" - name: Create submodule status - run: git submodule status extmod/ulab frozen/ lib/ tools/ >> submodule_status + run: git submodule status extmod/ulab lib/ tools/ >> submodule_status - name: Restore submodules uses: actions/cache/restore@v3 with: path: | .git/modules/ extmod/ulab - frozen/ lib/ tools/ key: submodules-common-${{ hashFiles('submodule_status') }} - name: CircuitPython dependencies + id: cp-deps run: python tools/ci_fetch_deps.py ${{ matrix.board }} - name: CircuitPython version run: tools/describe @@ -541,11 +540,12 @@ jobs: env: IDF_PATH: ${{ github.workspace }}/ports/espressif/esp-idf IDF_TOOLS_PATH: ${{ github.workspace }}/.idf_tools - - name: mpy-cross + - name: Build mpy-cross + if: ${{ steps.cp-deps.outputs.frozen_tags == 'True' }} run: make -C mpy-cross -j2 - name: Setup build failure matcher run: echo "::add-matcher::$GITHUB_WORKSPACE/.github/workflows/match-build-fail.json" - - name: build + - name: Build run: | source $IDF_PATH/export.sh python3 -u build_release_files.py @@ -589,18 +589,18 @@ jobs: with: python-version: "3.x" - name: Create submodule status - run: git submodule status extmod/ulab frozen/ lib/ tools/ >> submodule_status + run: git submodule status extmod/ulab lib/ tools/ >> submodule_status - name: Restore submodules uses: actions/cache/restore@v3 with: path: | .git/modules/ extmod/ulab - frozen/ lib/ tools/ key: submodules-common-${{ hashFiles('submodule_status') }} - name: CircuitPython dependencies + id: cp-deps run: python tools/ci_fetch_deps.py ${{ matrix.board }} - name: Install dependencies run: | @@ -627,11 +627,12 @@ jobs: arm-none-eabi-gcc --version python3 --version mkfs.fat --version || true - - name: mpy-cross + - name: Build mpy-cross + if: ${{ steps.cp-deps.outputs.frozen_tags == 'True' }} run: make -C mpy-cross -j2 - name: Setup build failure matcher run: echo "::add-matcher::$GITHUB_WORKSPACE/.github/workflows/match-build-fail.json" - - name: build + - name: Build run: python3 -u build_release_files.py working-directory: tools env: diff --git a/.github/workflows/ports_windows.yml b/.github/workflows/ports_windows.yml index 31187776be..61c6376ac7 100644 --- a/.github/workflows/ports_windows.yml +++ b/.github/workflows/ports_windows.yml @@ -4,7 +4,7 @@ on: push: pull_request: paths: - - '.github/workflows/*.yml' + - '.github/workflows/ports_windows.yml' - 'extmod/**' - 'lib/**' - 'mpy-cross/**' @@ -77,14 +77,13 @@ jobs: submodules: false fetch-depth: 1 - name: Create submodule status - run: git submodule status extmod/ulab frozen/ lib/ tools/ >> submodule_status + run: git submodule status extmod/ulab lib/ tools/ >> submodule_status - name: Restore submodules uses: actions/cache/restore@v3 with: path: | .git/modules/ extmod/ulab - frozen/ lib/ tools/ key: submodules-common-${{ hashFiles('submodule_status') }} diff --git a/tools/ci_fetch_deps.py b/tools/ci_fetch_deps.py index 43d708bbea..ddf7bf3bfd 100644 --- a/tools/ci_fetch_deps.py +++ b/tools/ci_fetch_deps.py @@ -1,8 +1,9 @@ -import pathlib -import shlex -import subprocess +import os import sys import time +import shlex +import pathlib +import subprocess # Target will be a board, "test", "docs", "mpy-cross-mac", or "windows" TARGET = sys.argv[1] @@ -58,13 +59,21 @@ def run(title, command, check=True): print("Duration:", time.monotonic() - start, flush=True) +def set_output(name, value): + if "GITHUB_OUTPUT" in os.environ: + with open(os.environ["GITHUB_OUTPUT"], "at") as f: + print(f"{name}={value}", file=f) + else: + print(f"Would set GitHub actions output {name} to '{value}'") + + def main(): submodules = [] + submodules_tags = [] + print("Target:", TARGET) - if TARGET == "scheduler": - submodules = ["extmod/ulab", "lib/", "tools/", "frozen/"] - elif TARGET == "mpy-cross-tests": + if TARGET == "test": submodules = ["extmod/ulab", "lib/", "tools/"] elif TARGET == "build-doc": # used in .readthedocs.yml to generate RTD @@ -98,13 +107,29 @@ def main(): if lib_folder.count("/") > 1: lib_folder = lib_folder.split("/", maxsplit=2) lib_folder = "/".join(lib_folder[:2]) - submodules.append(lib_folder) + submodules_tags.append(lib_folder) - print("Submodule paths:", submodules) + print("Submodule tags[Y]:", submodules_tags) + print("Submodule tags[N]:", submodules) + + if submodules_tags: + run( + "Init the submodules with tags", + f"git submodule update --init {' '.join(submodules_tags)}", + ) if submodules: - submodules = " ".join(submodules) - run("Init the required submodules", f"git submodule update --init --depth=1 {submodules}") + run( + "Init the submodules without tags", + f"git submodule update --init --depth=1 {' '.join(submodules)}", + ) + + for submodule in submodules_tags: + if submodule.startswith("frozen"): + set_output("frozen_tags", True) + break + else: + set_output("frozen_tags", False) if __name__ == "__main__":