diff --git a/Makefile b/Makefile index c834e413b1..382ddefa6f 100644 --- a/Makefile +++ b/Makefile @@ -327,7 +327,7 @@ clean-stm: .PHONY: fetch-all-submodules fetch-all-submodules: - tools/fetch-submodules.sh + $(PYTHON) tools/ci_fetch_deps.py all .PHONY: remove-all-submodules remove-all-submodules: diff --git a/py/circuitpy_defns.mk b/py/circuitpy_defns.mk index f3f81d9757..50dd139c7c 100644 --- a/py/circuitpy_defns.mk +++ b/py/circuitpy_defns.mk @@ -856,7 +856,12 @@ $(BUILD)/lib/libm/kf_rem_pio2.o: CFLAGS += -Wno-maybe-uninitialized # Fetch only submodules needed for this particular port. .PHONY: fetch-port-submodules fetch-port-submodules: - $(TOP)/tools/fetch-submodules.sh data extmod frozen lib tools ports/$(shell basename $(CURDIR)) + $(PYTHON) $(TOP)/tools/ci_fetch_deps.py $(shell basename $(CURDIR)) + +# Fetch only submodules needed for this particular board. +.PHONY: fetch-board-submodules +fetch-board-submodules: + $(PYTHON) $(TOP)/tools/ci_fetch_deps.py $(BOARD) .PHONY: invalid-board invalid-board: diff --git a/tools/ci_fetch_deps.py b/tools/ci_fetch_deps.py index 92b87ba1fa..b80f5559fd 100644 --- a/tools/ci_fetch_deps.py +++ b/tools/ci_fetch_deps.py @@ -3,10 +3,45 @@ import sys import time import shlex import pathlib +import re import subprocess -# Target will be a board, "test", "docs", "mpy-cross-mac", or "windows" -TARGET = sys.argv[1] +TOP = pathlib.Path(__file__).parent.parent + + +def _git_version(): + version_str = subprocess.check_output(["git", "--version"], encoding="ascii", errors="replace") + version_str = re.search("([0-9]\.*)*[0-9]", version_str).group(0) + return tuple(int(part) for part in version_str.split(".")) + + +clone_supports_filter = ( + False if "NO_USE_CLONE_FILTER" in os.environ else _git_version() >= (2, 27, 0) +) + +if clone_supports_filter: + filter_maybe = "--filter=blob:none" +else: + filter_maybe = "" + + +def _all_submodules(): + submodule_str = subprocess.check_output( + ["git", "submodule", "status"], encoding="ascii", errors="replace", cwd=TOP + ) + return [row.split()[1] for row in submodule_str.strip().split("\n")] + + +all_submodules = _all_submodules() + + +def matching_submodules(s): + if s.endswith("/"): + return [m for m in all_submodules if m.startswith(s)] + elif s not in all_submodules: + raise ValueError(f"{s!r} is not a submodule") + return [s] + # Submodules needed by port builds outside of their ports directory. # Should we try and detect these? @@ -49,68 +84,102 @@ PORT_DEPS = { } -def run(title, command, check=True): +def run(title, command, cwd): print("::group::" + title, flush=True) - print(command, flush=True) + print(f"{command} (in {cwd})", flush=True) start = time.monotonic() try: - subprocess.run(shlex.split(command), stderr=subprocess.STDOUT, check=check) + subprocess.run(shlex.split(command), stderr=subprocess.STDOUT, check=True, cwd=cwd) finally: print("::endgroup::", flush=True) print("Duration:", time.monotonic() - start, flush=True) +def matching_submodules(where): + for m in all_submodules: + if m in where: + yield m + for w in where: + if m.startswith(f"{w}/"): + yield m + break + + +def fetch(where): + if clone_supports_filter: + run( + "Init submodules (using filter)", + f"git submodule update --init {filter_maybe} {' '.join(where)}", + cwd=TOP, + ) + else: + run( + "Init submodules (using depth)", + f"git submodule update --init --depth 1 {' '.join(where)}", + cwd=TOP, + ) + for s in matching_submodules([w for w in where if w.startswith("frozen")]): + run(f"Ensure tags exist in {s}", "git fetch --tags --depth 1", cwd=TOP / s) + + def set_output(name, value): if "GITHUB_OUTPUT" in os.environ: with open(os.environ["GITHUB_OUTPUT"], "at") as f: print(f"{name}={value}", file=f) else: - print(f"Would set GitHub actions output {name} to '{value}'") + print(f"{name}: {value!r}") -def main(): +SUBMODULES_BY_TARGET = {} + + +def main(target): submodules = [] - submodules_tags = [] - print("Target:", TARGET) + print("Target:", target) - if TARGET == "scheduler": - # submodules = ["tools/"] + if target == "all": + submodules = [".", "frozen"] # explicitly list frozen to get tags + elif target == "scheduler": submodules = ["extmod/ulab", "lib/", "tools/"] - elif TARGET == "tests": - submodules = ["extmod/ulab", "lib/", "tools/"] - submodules_tags = [ + elif target == "tests": + submodules = [ + "extmod/ulab", + "lib/", + "tools/", "frozen/Adafruit_CircuitPython_asyncio", "frozen/Adafruit_CircuitPython_Ticks", ] - elif TARGET == "docs": + elif target == "docs": # used in .readthedocs.yml to generate RTD - submodules = ["extmod/ulab"] - submodules_tags = ["frozen/"] - elif TARGET == "mpy-cross" or TARGET == "mpy-cross-mac": + submodules = ["extmod/ulab", "frozen"] + elif target == "mpy-cross" or target == "mpy-cross-mac": submodules = ["tools/"] # for huffman - elif TARGET == "windows": + elif target == "windows": # This builds one board from a number of ports so fill out a bunch of submodules for port in ("atmel-samd", "nrf", "raspberrypi", "stm"): submodules.append(f"ports/{port}") submodules.extend(PORT_DEPS[port]) unique_submodules = set(submodules) submodules = list(unique_submodules) - elif TARGET == "website": - submodules = ["tools/adabot/"] - submodules_tags = ["frozen/"] - elif TARGET == "pre-commit": + elif target == "website": + submodules = ["tools/adabot", "frozen"] + elif target == "pre-commit": submodules = ["extmod/ulab"] + elif target in PORT_DEPS: + submodules = ["data", "extmod", "lib", "tools", "frozen", f"ports/{target}"] + PORT_DEPS[ + target + ] else: - p = list(pathlib.Path(".").glob(f"ports/*/boards/{TARGET}/mpconfigboard.mk")) + p = list(pathlib.Path(TOP).glob(f"ports/*/boards/{target}/mpconfigboard.mk")) if not p: - raise RuntimeError(f"Unsupported target: {TARGET}") + raise RuntimeError(f"Unsupported target: {target}") config = p[0] # Add the ports folder to init submodules port_folder = config.parents[2] port = port_folder.name - submodules.append(str(port_folder)) + submodules.append(f"ports/{port}") submodules.append("tools/") # for huffman submodules.extend(PORT_DEPS[port]) with config.open() as f: @@ -123,24 +192,14 @@ def main(): if lib_folder.count("/") > 1: lib_folder = lib_folder.split("/", maxsplit=2) lib_folder = "/".join(lib_folder[:2]) - submodules_tags.append(lib_folder) + submodules.append(lib_folder) - print("Submodule tags[Y]:", submodules_tags) - print("Submodule tags[N]:", submodules) - - if submodules_tags: - run( - "Init the submodules with tags", - f"git submodule update --init {' '.join(submodules_tags)}", - ) + print("Submodules:", " ".join(submodules)) if submodules: - run( - "Init the submodules without tags", - f"git submodule update --init --depth=1 {' '.join(submodules)}", - ) + fetch(submodules) - for submodule in submodules_tags: + for submodule in submodules: if submodule.startswith("frozen"): set_output("frozen_tags", True) break @@ -149,4 +208,11 @@ def main(): if __name__ == "__main__": - main() + if len(sys.argv) < 2: + raise SystemExit("Usage: ci_fetch_deps dep...") + + run("Sync submodule URLs", "git submodule sync --quiet", cwd=TOP) + + # Target will be a board, "test", "docs", "mpy-cross-mac", or "windows" + for target in sys.argv[1:]: + main(target) diff --git a/tools/fetch-submodules.sh b/tools/fetch-submodules.sh deleted file mode 100755 index 707edd992b..0000000000 --- a/tools/fetch-submodules.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env sh -# Pass the directories of submodules to fetch as command-line arguments. For example: -# ./fetch-submodules.sh lib tools -# If no arguments are passed, all submodules will be fetched. -# This script handles being called at other than the top level by making the paths -# for the submodules absolute. - -TOP=$(git rev-parse --show-toplevel) - -git submodule sync --quiet -# Prefix all the args with the absolute path to the top of the repo. -abs_submodules="" -for d in "$@"; do - abs_submodules="${abs_submodules} ${TOP}/${d}" -done -echo ${abs_submodules} - -# Fetch submodules as partial clones if possible. If that fails due to an older version of git, -# do a shallow init and fetch tags. -git submodule update --init --filter=blob:none ${abs_submodules} || \ - git submodule update --init --depth 1 ${abs_submodules} && \ - git submodule foreach 'git fetch --tags --depth 1' || \ - echo "ERROR: fetch-submodules.sh FAILED"