diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b1f9698d4a..6a7a40c99c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -21,29 +21,54 @@ jobs: runs-on: ubuntu-20.04 outputs: build-doc: ${{ steps.set-matrix.outputs.build-doc }} - boards-arm: ${{ steps.set-matrix.outputs.boards-arm }} - boards-riscv: ${{ steps.set-matrix.outputs.boards-riscv }} - boards-espressif: ${{ steps.set-matrix.outputs.boards-espressif }} boards-aarch: ${{ steps.set-matrix.outputs.boards-aarch }} + boards-arm: ${{ steps.set-matrix.outputs.boards-arm }} + boards-espressif: ${{ steps.set-matrix.outputs.boards-espressif }} + boards-riscv: ${{ steps.set-matrix.outputs.boards-riscv }} + cp-version: ${{ steps.cp-version.outputs.cp-version }} steps: - name: Dump GitHub context run: echo "$GITHUB_CONTEXT" env: GITHUB_CONTEXT: ${{ toJson(github) }} - - uses: actions/checkout@v3 + - name: Set up repository + uses: actions/checkout@v3 with: submodules: false fetch-depth: 1 - - name: Set up Python 3 + - name: Set up python uses: actions/setup-python@v4 with: python-version: "3.x" - - name: Get CP deps - run: python tools/ci_fetch_deps.py test ${{ github.sha }} - - name: CircuitPython version + - name: Duplicate USB VID/PID check + run: python3 -u -m tools.ci_check_duplicate_usb_vid_pid + - name: Create submodule status + run: git submodule status extmod/ulab frozen/ lib/ tools/ >> submodule_status + - name: Cache submodules + uses: actions/cache@v3 + with: + path: | + .git/modules/ + extmod/ulab + frozen/ + lib/ + tools/ + key: submodules-common-${{ hashFiles('submodule_status') }} + - name: CircuitPython dependencies run: | - tools/describe || git log --parents HEAD~4.. - echo >>$GITHUB_ENV CP_VERSION=$(tools/describe) + python tools/ci_fetch_deps.py ${{ github.job }} + echo "::group::Fetch history and tags" + git fetch --no-recurse-submodules --shallow-since="2021-07-01" --tags https://github.com/adafruit/circuitpython HEAD + git fetch --no-recurse-submodules --shallow-since="2021-07-01" origin $GITHUB_SHA + git repack -d + echo "::endgroup::" + - name: CircuitPython version + id: cp-version + run: | + CP_VERSION=$(tools/describe) + echo "$CP_VERSION" + echo "CP_VERSION=$CP_VERSION" >> $GITHUB_ENV + echo "cp-version=$CP_VERSION" >> $GITHUB_OUTPUT - name: Install dependencies run: | sudo apt-get update @@ -54,8 +79,6 @@ jobs: run: | gcc --version python3 --version - - name: Duplicate USB VID/PID Check - run: python3 -u -m tools.ci_check_duplicate_usb_vid_pid - name: Build mpy-cross run: make -C mpy-cross -j2 - name: Build unix port @@ -160,25 +183,36 @@ jobs: mpy-cross-mac: runs-on: macos-11 + needs: test + if: ${{ needs.test.outputs.boards-aarch != '[]' || needs.test.outputs.boards-arm != '[]' || needs.scheduler.test.boards-espressif != '[]' || needs.scheduler.test.boards-riscv != '[]' }} + env: + CP_VERSION: ${{ needs.test.outputs.cp-version }} steps: - - name: Dump GitHub context - env: - GITHUB_CONTEXT: ${{ toJson(github) }} - run: echo "$GITHUB_CONTEXT" - - uses: actions/checkout@v3 + - name: Set up repository + uses: actions/checkout@v3 with: submodules: false fetch-depth: 1 - - name: Set up Python 3 + - name: Set up python uses: actions/setup-python@v4 with: - python-version: "3.10" - - name: Get CP deps - run: python tools/ci_fetch_deps.py mpy-cross-mac ${{ github.sha }} + python-version: "3.x" + - name: Create submodule status + run: git submodule status extmod/ulab frozen/ lib/ tools/ >> submodule_status + - name: Restore submodules + uses: actions/cache/restore@v3 + with: + path: | + .git/modules/ + extmod/ulab + frozen/ + lib/ + tools/ + key: submodules-common-${{ hashFiles('submodule_status') }} + - name: CircuitPython dependencies + run: python tools/ci_fetch_deps.py ${{ github.job }} - name: CircuitPython version - run: | - tools/describe || git log --parents HEAD~4.. - echo >>$GITHUB_ENV CP_VERSION=$(tools/describe) + run: tools/describe - name: Install dependencies run: | brew install gettext @@ -222,21 +256,34 @@ jobs: runs-on: ubuntu-20.04 needs: test if: ${{ needs.test.outputs.build-doc == 'True' }} + env: + CP_VERSION: ${{ needs.test.outputs.cp-version }} steps: - - uses: actions/checkout@v3 + - name: Set up repository + uses: actions/checkout@v3 with: submodules: false fetch-depth: 1 - - name: Get CP deps - run: python tools/ci_fetch_deps.py docs ${{ github.sha }} + - name: Create submodule status + run: git submodule status extmod/ulab frozen/ lib/ tools/ >> submodule_status + - name: Restore submodules + uses: actions/cache/restore@v3 + with: + path: | + .git/modules/ + extmod/ulab + frozen/ + lib/ + tools/ + key: submodules-common-${{ hashFiles('submodule_status') }} + - name: CircuitPython dependencies + run: python tools/ci_fetch_deps.py ${{ github.job }} - name: CircuitPython version - run: | - tools/describe || git log --parents HEAD~4.. - echo >>$GITHUB_ENV CP_VERSION=$(tools/describe) - - name: Set up Python 3 + run: tools/describe + - name: Set up python uses: actions/setup-python@v4 with: - python-version: "3.10" + python-version: "3.x" - name: Install dependencies run: | sudo apt-get update @@ -285,22 +332,37 @@ jobs: build-arm: runs-on: ubuntu-20.04 needs: test + if: ${{ needs.test.outputs.boards-arm != '[]' }} + env: + CP_VERSION: ${{ needs.test.outputs.cp-version }} strategy: fail-fast: false matrix: board: ${{ fromJSON(needs.test.outputs.boards-arm) }} - if: ${{ needs.test.outputs.boards-arm != '[]' }} steps: - - name: Set up Python 3 - uses: actions/setup-python@v4 - with: - python-version: "3.10" - - uses: actions/checkout@v3 + - name: Set up repository + uses: actions/checkout@v3 with: submodules: false fetch-depth: 1 - - name: Get CP deps - run: python tools/ci_fetch_deps.py ${{ matrix.board }} ${{ github.sha }} + - name: Set up python + uses: actions/setup-python@v4 + with: + python-version: "3.x" + - name: Create submodule status + run: git submodule status extmod/ulab frozen/ lib/ tools/ >> submodule_status + - name: Restore submodules + uses: actions/cache/restore@v3 + with: + path: | + .git/modules/ + extmod/ulab + frozen/ + lib/ + tools/ + key: submodules-common-${{ hashFiles('submodule_status') }} + - name: CircuitPython dependencies + run: python tools/ci_fetch_deps.py ${{ matrix.board }} - uses: carlosperate/arm-none-eabi-gcc-action@v1 with: release: '10-2020-q4' @@ -338,22 +400,37 @@ jobs: build-riscv: runs-on: ubuntu-20.04 needs: test + if: ${{ needs.test.outputs.boards-riscv != '[]' }} + env: + CP_VERSION: ${{ needs.test.outputs.cp-version }} strategy: fail-fast: false matrix: board: ${{ fromJSON(needs.test.outputs.boards-riscv) }} - if: ${{ needs.test.outputs.boards-riscv != '[]' }} steps: - - name: Set up Python 3 - uses: actions/setup-python@v4 - with: - python-version: "3.10" - - uses: actions/checkout@v3 + - name: Set up repository + uses: actions/checkout@v3 with: submodules: false fetch-depth: 1 - - name: Get CP deps - run: python tools/ci_fetch_deps.py ${{ matrix.board }} ${{ github.sha }} + - name: Set up python + uses: actions/setup-python@v4 + with: + python-version: "3.x" + - name: Create submodule status + run: git submodule status extmod/ulab frozen/ lib/ tools/ >> submodule_status + - name: Restore submodules + uses: actions/cache/restore@v3 + with: + path: | + .git/modules/ + extmod/ulab + frozen/ + lib/ + tools/ + key: submodules-common-${{ hashFiles('submodule_status') }} + - name: CircuitPython dependencies + run: python tools/ci_fetch_deps.py ${{ matrix.board }} - name: Install dependencies run: | sudo apt-get install -y gettext @@ -390,27 +467,40 @@ jobs: build-espressif: runs-on: ubuntu-20.04 needs: test + if: ${{ needs.test.outputs.boards-espressif != '[]' }} + env: + CP_VERSION: ${{ needs.test.outputs.cp-version }} strategy: fail-fast: false matrix: board: ${{ fromJSON(needs.test.outputs.boards-espressif) }} - if: ${{ needs.test.outputs.boards-espressif != '[]' }} steps: - - name: Set up Python 3 + - name: Set up repository + uses: actions/checkout@v3 + with: + submodules: false + fetch-depth: 1 + - name: Set up python id: py3 uses: actions/setup-python@v4 with: python-version: "3.10" - - uses: actions/checkout@v3 + - name: Create submodule status + run: git submodule status extmod/ulab frozen/ lib/ tools/ >> submodule_status + - name: Restore submodules + uses: actions/cache/restore@v3 with: - submodules: false - fetch-depth: 1 - - name: Get CP deps - run: python tools/ci_fetch_deps.py ${{ matrix.board }} ${{ github.sha }} + path: | + .git/modules/ + extmod/ulab + frozen/ + lib/ + tools/ + key: submodules-common-${{ hashFiles('submodule_status') }} + - name: CircuitPython dependencies + run: python tools/ci_fetch_deps.py ${{ matrix.board }} - name: CircuitPython version - run: | - tools/describe || git log --parents HEAD~4.. - echo >>$GITHUB_ENV CP_VERSION=$(tools/describe) + run: tools/describe - uses: actions/cache@v3 name: Fetch IDF tool cache id: idf-cache @@ -481,22 +571,37 @@ jobs: build-aarch: runs-on: ubuntu-20.04 needs: test + if: ${{ needs.test.outputs.boards-aarch != '[]' }} + env: + CP_VERSION: ${{ needs.test.outputs.cp-version }} strategy: fail-fast: false matrix: board: ${{ fromJSON(needs.test.outputs.boards-aarch) }} - if: ${{ needs.test.outputs.boards-aarch != '[]' }} steps: - - name: Set up Python 3 - uses: actions/setup-python@v4 - with: - python-version: "3.10" - - uses: actions/checkout@v3 + - name: Set up repository + uses: actions/checkout@v3 with: submodules: false fetch-depth: 1 - - name: Get CP deps - run: python tools/ci_fetch_deps.py ${{ matrix.board }} ${{ github.sha }} + - name: Set up python + uses: actions/setup-python@v4 + with: + python-version: "3.x" + - name: Create submodule status + run: git submodule status extmod/ulab frozen/ lib/ tools/ >> submodule_status + - name: Restore submodules + uses: actions/cache/restore@v3 + with: + path: | + .git/modules/ + extmod/ulab + frozen/ + lib/ + tools/ + key: submodules-common-${{ hashFiles('submodule_status') }} + - name: CircuitPython dependencies + run: python tools/ci_fetch_deps.py ${{ matrix.board }} - name: Install dependencies run: | sudo apt-get install -y gettext mtools diff --git a/.github/workflows/ports_windows.yml b/.github/workflows/ports_windows.yml index 6d38f3b853..31187776be 100644 --- a/.github/workflows/ports_windows.yml +++ b/.github/workflows/ports_windows.yml @@ -71,12 +71,32 @@ jobs: which python; python --version; python -c "import cascadetoml" which python3; python3 --version; python3 -c "import cascadetoml" - - uses: actions/checkout@v3 + - name: Set up repository + uses: actions/checkout@v3 with: submodules: false fetch-depth: 1 - - name: Get CP deps - run: python tools/ci_fetch_deps.py windows ${{ github.sha }} + - name: Create submodule status + run: git submodule status extmod/ulab frozen/ lib/ tools/ >> submodule_status + - name: Restore submodules + uses: actions/cache/restore@v3 + with: + path: | + .git/modules/ + extmod/ulab + frozen/ + lib/ + tools/ + key: submodules-common-${{ hashFiles('submodule_status') }} + enableCrossOsArchive: true + - name: CircuitPython dependencies + run: | + python tools/ci_fetch_deps.py windows + echo "::group::Fetch history and tags" + git fetch --no-recurse-submodules --shallow-since="2021-07-01" --tags https://github.com/adafruit/circuitpython HEAD + git fetch --no-recurse-submodules --shallow-since="2021-07-01" origin $GITHUB_SHA + git repack -d + echo "::endgroup::" - name: CircuitPython version run: | tools/describe || git log --parents HEAD~4.. diff --git a/tools/ci_fetch_deps.py b/tools/ci_fetch_deps.py index 04ae6fb0e1..43d708bbea 100644 --- a/tools/ci_fetch_deps.py +++ b/tools/ci_fetch_deps.py @@ -4,16 +4,12 @@ import subprocess import sys import time -# target will be a board, "test", "docs", "mpy-cross-mac", or "windows" - -target = sys.argv[1] -ref = sys.argv[2] - -print(target, ref) +# Target will be a board, "test", "docs", "mpy-cross-mac", or "windows" +TARGET = sys.argv[1] # Submodules needed by port builds outside of their ports directory. # Should we try and detect these? -port_deps = { +PORT_DEPS = { "atmel-samd": [ "extmod/ulab/", "lib/adafruit_floppy/", @@ -58,73 +54,58 @@ def run(title, command, check=True): try: subprocess.run(shlex.split(command), stderr=subprocess.STDOUT, check=check) finally: - print("Duration:", time.monotonic() - start, flush=True) print("::endgroup::", flush=True) + print("Duration:", time.monotonic() - start, flush=True) -run( - "Fetch back to the start of 2021 to get tag history", - 'git fetch --tags --recurse-submodules=no --shallow-since="2021-07-01" https://github.com/adafruit/circuitpython HEAD', -) -run( - "Fetch back to the start of 2021 to get commit history", - f'git fetch --recurse-submodules=no --shallow-since="2021-07-01" origin {ref}', -) -# See https://stackoverflow.com/questions/63878612/git-fatal-error-in-object-unshallow-sha-1#comment118418373_63879454 -run('Fix for bug "fatal: error in object: unshallow"', "git repack -d") -run("Init submodules", "git submodule init") -run("Submodule status", "git submodule status") +def main(): + submodules = [] + print("Target:", TARGET) -submodules = [] + if TARGET == "scheduler": + submodules = ["extmod/ulab", "lib/", "tools/", "frozen/"] + elif TARGET == "mpy-cross-tests": + submodules = ["extmod/ulab", "lib/", "tools/"] + elif TARGET == "build-doc": + # used in .readthedocs.yml to generate RTD + submodules = ["extmod/ulab", "frozen/"] + elif TARGET == "mpy-cross-mac": + submodules = ["tools/"] # for huffman + elif TARGET == "windows": + # This builds one board from a number of ports so fill out a bunch of submodules + submodules = ["extmod/ulab", "lib/", "tools/", "ports/", "data/nvm.toml"] + elif TARGET == "website": + submodules = ["tools/adabot/", "frozen/"] + else: + p = list(pathlib.Path(".").glob(f"ports/*/boards/{TARGET}/mpconfigboard.mk")) + if not p: + raise RuntimeError(f"Unsupported target: {TARGET}") -if target == "test": - submodules = ["extmod/", "lib/", "tools/", "extmod/ulab", "lib/berkeley-db-1.xx"] -elif target == "docs": - # used in .readthedocs.yml to generate RTD - submodules = ["extmod/ulab/", "frozen/"] -elif target == "mpy-cross-mac": - submodules = ["tools/"] # for huffman -elif target == "windows": - # This builds one board from a number of ports so fill out a bunch of submodules - submodules = ["extmod/", "lib/", "tools/", "ports/", "data/nvm.toml/"] -elif target == "website": - submodules = ["tools/adabot/", "frozen/"] -else: - p = list(pathlib.Path(".").glob(f"ports/*/boards/{target}/mpconfigboard.mk")) - if not p: - raise RuntimeError(f"Unsupported target: {target}") + config = p[0] + # Add the ports folder to init submodules + port_folder = config.parents[2] + port = port_folder.name + submodules.append(str(port_folder)) + submodules.append("tools/") # for huffman + submodules.extend(PORT_DEPS[port]) + with config.open() as f: + for line in f.readlines(): + prefix = "FROZEN_MPY_DIRS += $(TOP)/" + if line.startswith(prefix): + lib_folder = line.strip()[len(prefix) :] + # Drop everything after the second folder because the frozen + # folder may be inside the submodule. + if lib_folder.count("/") > 1: + lib_folder = lib_folder.split("/", maxsplit=2) + lib_folder = "/".join(lib_folder[:2]) + submodules.append(lib_folder) - config = p[0] - # Add the ports folder to init submodules - port_folder = config.parents[2] - port = port_folder.name - submodules.append(str(port_folder)) - submodules.append("tools/") # for huffman - submodules.extend(port_deps[port]) - with config.open() as f: - for line in f.readlines(): - prefix = "FROZEN_MPY_DIRS += $(TOP)/" - if line.startswith(prefix): - lib_folder = line.strip()[len(prefix) :] - # Drop everything after the second folder because the frozen - # folder may be inside the submodule. - if lib_folder.count("/") > 1: - lib_folder = lib_folder.split("/", maxsplit=2) - lib_folder = "/".join(lib_folder[:2]) - submodules.append(lib_folder) + print("Submodule paths:", submodules) -print(submodules) -if submodules: - submodules = " ".join(submodules) - # This line will fail because the submodule's need different commits than the tip of the branch. We - # fix it later. - run( - "Init the submodules we'll need", - f"git submodule update --init -N --depth 1 {submodules}", - check=False, - ) + if submodules: + submodules = " ".join(submodules) + run("Init the required submodules", f"git submodule update --init --depth=1 {submodules}") - run( - "Fetch the submodule sha", - "git submodule foreach 'git fetch --tags --depth 1 origin $sha1 && git checkout -q $sha1'", - ) + +if __name__ == "__main__": + main() diff --git a/tools/describe b/tools/describe index e20c2ffa44..0f3b541e3c 100755 --- a/tools/describe +++ b/tools/describe @@ -1,3 +1,6 @@ #!/bin/sh -# Add any supplied arguments. -git describe --first-parent --dirty --tags --match "[1-9].*" "$@" +if [ -z "$CP_VERSION" ]; then + git describe --first-parent --dirty --tags --match "[1-9].*" "$@" +else + echo $CP_VERSION +fi