Merge pull request #8589 from jepler/issue8588
Rework ci_fetch_deps and use it from makefiles too
This commit is contained in:
commit
4bdd3e6150
2
Makefile
2
Makefile
|
@ -327,7 +327,7 @@ clean-stm:
|
|||
|
||||
.PHONY: fetch-all-submodules
|
||||
fetch-all-submodules:
|
||||
tools/fetch-submodules.sh
|
||||
$(PYTHON) tools/ci_fetch_deps.py all
|
||||
|
||||
.PHONY: remove-all-submodules
|
||||
remove-all-submodules:
|
||||
|
|
|
@ -856,7 +856,12 @@ $(BUILD)/lib/libm/kf_rem_pio2.o: CFLAGS += -Wno-maybe-uninitialized
|
|||
# Fetch only submodules needed for this particular port.
|
||||
.PHONY: fetch-port-submodules
|
||||
fetch-port-submodules:
|
||||
$(TOP)/tools/fetch-submodules.sh data extmod frozen lib tools ports/$(shell basename $(CURDIR))
|
||||
$(PYTHON) $(TOP)/tools/ci_fetch_deps.py $(shell basename $(CURDIR))
|
||||
|
||||
# Fetch only submodules needed for this particular board.
|
||||
.PHONY: fetch-board-submodules
|
||||
fetch-board-submodules:
|
||||
$(PYTHON) $(TOP)/tools/ci_fetch_deps.py $(BOARD)
|
||||
|
||||
.PHONY: invalid-board
|
||||
invalid-board:
|
||||
|
|
|
@ -3,10 +3,45 @@ import sys
|
|||
import time
|
||||
import shlex
|
||||
import pathlib
|
||||
import re
|
||||
import subprocess
|
||||
|
||||
# Target will be a board, "test", "docs", "mpy-cross-mac", or "windows"
|
||||
TARGET = sys.argv[1]
|
||||
TOP = pathlib.Path(__file__).parent.parent
|
||||
|
||||
|
||||
def _git_version():
|
||||
version_str = subprocess.check_output(["git", "--version"], encoding="ascii", errors="replace")
|
||||
version_str = re.search("([0-9]\.*)*[0-9]", version_str).group(0)
|
||||
return tuple(int(part) for part in version_str.split("."))
|
||||
|
||||
|
||||
clone_supports_filter = (
|
||||
False if "NO_USE_CLONE_FILTER" in os.environ else _git_version() >= (2, 27, 0)
|
||||
)
|
||||
|
||||
if clone_supports_filter:
|
||||
filter_maybe = "--filter=blob:none"
|
||||
else:
|
||||
filter_maybe = ""
|
||||
|
||||
|
||||
def _all_submodules():
|
||||
submodule_str = subprocess.check_output(
|
||||
["git", "submodule", "status"], encoding="ascii", errors="replace", cwd=TOP
|
||||
)
|
||||
return [row.split()[1] for row in submodule_str.strip().split("\n")]
|
||||
|
||||
|
||||
all_submodules = _all_submodules()
|
||||
|
||||
|
||||
def matching_submodules(s):
|
||||
if s.endswith("/"):
|
||||
return [m for m in all_submodules if m.startswith(s)]
|
||||
elif s not in all_submodules:
|
||||
raise ValueError(f"{s!r} is not a submodule")
|
||||
return [s]
|
||||
|
||||
|
||||
# Submodules needed by port builds outside of their ports directory.
|
||||
# Should we try and detect these?
|
||||
|
@ -66,68 +101,102 @@ PORT_DEPS = {
|
|||
}
|
||||
|
||||
|
||||
def run(title, command, check=True):
|
||||
def run(title, command, cwd):
|
||||
print("::group::" + title, flush=True)
|
||||
print(command, flush=True)
|
||||
print(f"{command} (in {cwd})", flush=True)
|
||||
start = time.monotonic()
|
||||
try:
|
||||
subprocess.run(shlex.split(command), stderr=subprocess.STDOUT, check=check)
|
||||
subprocess.run(shlex.split(command), stderr=subprocess.STDOUT, check=True, cwd=cwd)
|
||||
finally:
|
||||
print("::endgroup::", flush=True)
|
||||
print("Duration:", time.monotonic() - start, flush=True)
|
||||
|
||||
|
||||
def matching_submodules(where):
|
||||
for m in all_submodules:
|
||||
if m in where:
|
||||
yield m
|
||||
for w in where:
|
||||
if m.startswith(f"{w}/"):
|
||||
yield m
|
||||
break
|
||||
|
||||
|
||||
def fetch(where):
|
||||
if clone_supports_filter:
|
||||
run(
|
||||
"Init submodules (using filter)",
|
||||
f"git submodule update --init {filter_maybe} {' '.join(where)}",
|
||||
cwd=TOP,
|
||||
)
|
||||
else:
|
||||
run(
|
||||
"Init submodules (using depth)",
|
||||
f"git submodule update --init --depth 1 {' '.join(where)}",
|
||||
cwd=TOP,
|
||||
)
|
||||
for s in matching_submodules([w for w in where if w.startswith("frozen")]):
|
||||
run(f"Ensure tags exist in {s}", "git fetch --tags --depth 1", cwd=TOP / s)
|
||||
|
||||
|
||||
def set_output(name, value):
|
||||
if "GITHUB_OUTPUT" in os.environ:
|
||||
with open(os.environ["GITHUB_OUTPUT"], "at") as f:
|
||||
print(f"{name}={value}", file=f)
|
||||
else:
|
||||
print(f"Would set GitHub actions output {name} to '{value}'")
|
||||
print(f"{name}: {value!r}")
|
||||
|
||||
|
||||
def main():
|
||||
SUBMODULES_BY_TARGET = {}
|
||||
|
||||
|
||||
def main(target):
|
||||
submodules = []
|
||||
submodules_tags = []
|
||||
|
||||
print("Target:", TARGET)
|
||||
print("Target:", target)
|
||||
|
||||
if TARGET == "scheduler":
|
||||
# submodules = ["tools/"]
|
||||
if target == "all":
|
||||
submodules = [".", "frozen"] # explicitly list frozen to get tags
|
||||
elif target == "scheduler":
|
||||
submodules = ["extmod/ulab", "lib/", "tools/"]
|
||||
elif TARGET == "tests":
|
||||
submodules = ["extmod/ulab", "lib/", "tools/"]
|
||||
submodules_tags = [
|
||||
elif target == "tests":
|
||||
submodules = [
|
||||
"extmod/ulab",
|
||||
"lib/",
|
||||
"tools/",
|
||||
"frozen/Adafruit_CircuitPython_asyncio",
|
||||
"frozen/Adafruit_CircuitPython_Ticks",
|
||||
]
|
||||
elif TARGET == "docs":
|
||||
elif target == "docs":
|
||||
# used in .readthedocs.yml to generate RTD
|
||||
submodules = ["extmod/ulab"]
|
||||
submodules_tags = ["frozen/"]
|
||||
elif TARGET == "mpy-cross" or TARGET == "mpy-cross-mac":
|
||||
submodules = ["extmod/ulab", "frozen"]
|
||||
elif target == "mpy-cross" or target == "mpy-cross-mac":
|
||||
submodules = ["tools/"] # for huffman
|
||||
elif TARGET == "windows":
|
||||
elif target == "windows":
|
||||
# This builds one board from a number of ports so fill out a bunch of submodules
|
||||
for port in ("atmel-samd", "nrf", "raspberrypi", "stm"):
|
||||
submodules.append(f"ports/{port}")
|
||||
submodules.extend(PORT_DEPS[port])
|
||||
unique_submodules = set(submodules)
|
||||
submodules = list(unique_submodules)
|
||||
elif TARGET == "website":
|
||||
submodules = ["tools/adabot/"]
|
||||
submodules_tags = ["frozen/"]
|
||||
elif TARGET == "pre-commit":
|
||||
elif target == "website":
|
||||
submodules = ["tools/adabot", "frozen"]
|
||||
elif target == "pre-commit":
|
||||
submodules = ["extmod/ulab"]
|
||||
elif target in PORT_DEPS:
|
||||
submodules = ["data", "extmod", "lib", "tools", "frozen", f"ports/{target}"] + PORT_DEPS[
|
||||
target
|
||||
]
|
||||
else:
|
||||
p = list(pathlib.Path(".").glob(f"ports/*/boards/{TARGET}/mpconfigboard.mk"))
|
||||
p = list(pathlib.Path(TOP).glob(f"ports/*/boards/{target}/mpconfigboard.mk"))
|
||||
if not p:
|
||||
raise RuntimeError(f"Unsupported target: {TARGET}")
|
||||
raise RuntimeError(f"Unsupported target: {target}")
|
||||
|
||||
config = p[0]
|
||||
# Add the ports folder to init submodules
|
||||
port_folder = config.parents[2]
|
||||
port = port_folder.name
|
||||
submodules.append(str(port_folder))
|
||||
submodules.append(f"ports/{port}")
|
||||
submodules.append("tools/") # for huffman
|
||||
submodules.extend(PORT_DEPS[port])
|
||||
with config.open() as f:
|
||||
|
@ -140,24 +209,14 @@ def main():
|
|||
if lib_folder.count("/") > 1:
|
||||
lib_folder = lib_folder.split("/", maxsplit=2)
|
||||
lib_folder = "/".join(lib_folder[:2])
|
||||
submodules_tags.append(lib_folder)
|
||||
submodules.append(lib_folder)
|
||||
|
||||
print("Submodule tags[Y]:", submodules_tags)
|
||||
print("Submodule tags[N]:", submodules)
|
||||
|
||||
if submodules_tags:
|
||||
run(
|
||||
"Init the submodules with tags",
|
||||
f"git submodule update --init {' '.join(submodules_tags)}",
|
||||
)
|
||||
print("Submodules:", " ".join(submodules))
|
||||
|
||||
if submodules:
|
||||
run(
|
||||
"Init the submodules without tags",
|
||||
f"git submodule update --init --depth=1 {' '.join(submodules)}",
|
||||
)
|
||||
fetch(submodules)
|
||||
|
||||
for submodule in submodules_tags:
|
||||
for submodule in submodules:
|
||||
if submodule.startswith("frozen"):
|
||||
set_output("frozen_tags", True)
|
||||
break
|
||||
|
@ -166,4 +225,11 @@ def main():
|
|||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
if len(sys.argv) < 2:
|
||||
raise SystemExit("Usage: ci_fetch_deps dep...")
|
||||
|
||||
run("Sync submodule URLs", "git submodule sync --quiet", cwd=TOP)
|
||||
|
||||
# Target will be a board, "test", "docs", "mpy-cross-mac", or "windows"
|
||||
for target in sys.argv[1:]:
|
||||
main(target)
|
||||
|
|
|
@ -1,23 +0,0 @@
|
|||
#!/usr/bin/env sh
|
||||
# Pass the directories of submodules to fetch as command-line arguments. For example:
|
||||
# ./fetch-submodules.sh lib tools
|
||||
# If no arguments are passed, all submodules will be fetched.
|
||||
# This script handles being called at other than the top level by making the paths
|
||||
# for the submodules absolute.
|
||||
|
||||
TOP=$(git rev-parse --show-toplevel)
|
||||
|
||||
git submodule sync --quiet
|
||||
# Prefix all the args with the absolute path to the top of the repo.
|
||||
abs_submodules=""
|
||||
for d in "$@"; do
|
||||
abs_submodules="${abs_submodules} ${TOP}/${d}"
|
||||
done
|
||||
echo ${abs_submodules}
|
||||
|
||||
# Fetch submodules as partial clones if possible. If that fails due to an older version of git,
|
||||
# do a shallow init and fetch tags.
|
||||
git submodule update --init --filter=blob:none ${abs_submodules} || \
|
||||
git submodule update --init --depth 1 ${abs_submodules} && \
|
||||
git submodule foreach 'git fetch --tags --depth 1' || \
|
||||
echo "ERROR: fetch-submodules.sh FAILED"
|
Loading…
Reference in New Issue