djsundog/circuitpython
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #8589 from jepler/issue8588

Browse Source 
Rework ci_fetch_deps and use it from makefiles too
This commit is contained in:
Scott Shawcroft 2023-11-14 10:04:37 -08:00 committed by GitHub
commit 4bdd3e6150
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 114 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Makefile
View File

@ -327,7 +327,7 @@ clean-stm:
.PHONY: fetch-all-submodules
fetch-all-submodules:
tools/fetch-submodules.sh
$(PYTHON) tools/ci_fetch_deps.py all
.PHONY: remove-all-submodules
remove-all-submodules:

7
py/circuitpy_defns.mk
View File

@ -856,7 +856,12 @@ $(BUILD)/lib/libm/kf_rem_pio2.o: CFLAGS += -Wno-maybe-uninitialized
# Fetch only submodules needed for this particular port.
.PHONY: fetch-port-submodules
fetch-port-submodules:
$(TOP)/tools/fetch-submodules.sh data extmod frozen lib tools ports/$(shell basename $(CURDIR))
$(PYTHON) $(TOP)/tools/ci_fetch_deps.py $(shell basename $(CURDIR))
# Fetch only submodules needed for this particular board.
.PHONY: fetch-board-submodules
fetch-board-submodules:
$(PYTHON) $(TOP)/tools/ci_fetch_deps.py $(BOARD)
.PHONY: invalid-board
invalid-board:

148
tools/ci_fetch_deps.py
View File

@ -3,10 +3,45 @@ import sys
import time
import shlex
import pathlib
import re
import subprocess
# Target will be a board, "test", "docs", "mpy-cross-mac", or "windows"
TARGET = sys.argv[1]
TOP = pathlib.Path(__file__).parent.parent
def _git_version():
version_str = subprocess.check_output(["git", "--version"], encoding="ascii", errors="replace")
version_str = re.search("([0-9]\.*)*[0-9]", version_str).group(0)
return tuple(int(part) for part in version_str.split("."))
clone_supports_filter = (
False if "NO_USE_CLONE_FILTER" in os.environ else _git_version() >= (2, 27, 0)
)
if clone_supports_filter:
filter_maybe = "--filter=blob:none"
else:
filter_maybe = ""
def _all_submodules():
submodule_str = subprocess.check_output(
["git", "submodule", "status"], encoding="ascii", errors="replace", cwd=TOP
)
return [row.split()[1] for row in submodule_str.strip().split("\n")]
all_submodules = _all_submodules()
def matching_submodules(s):
if s.endswith("/"):
return [m for m in all_submodules if m.startswith(s)]
elif s not in all_submodules:
raise ValueError(f"{s!r} is not a submodule")
return [s]
# Submodules needed by port builds outside of their ports directory.
# Should we try and detect these?
@ -66,68 +101,102 @@ PORT_DEPS = {
}
def run(title, command, check=True):
def run(title, command, cwd):
print("::group::" + title, flush=True)
print(command, flush=True)
print(f"{command} (in {cwd})", flush=True)
start = time.monotonic()
try:
subprocess.run(shlex.split(command), stderr=subprocess.STDOUT, check=check)
subprocess.run(shlex.split(command), stderr=subprocess.STDOUT, check=True, cwd=cwd)
finally:
print("::endgroup::", flush=True)
print("Duration:", time.monotonic() - start, flush=True)
def matching_submodules(where):
for m in all_submodules:
if m in where:
yield m
for w in where:
if m.startswith(f"{w}/"):
yield m
break
def fetch(where):
if clone_supports_filter:
run(
"Init submodules (using filter)",
f"git submodule update --init {filter_maybe} {' '.join(where)}",
cwd=TOP,
)
else:
run(
"Init submodules (using depth)",
f"git submodule update --init --depth 1 {' '.join(where)}",
cwd=TOP,
)
for s in matching_submodules([w for w in where if w.startswith("frozen")]):
run(f"Ensure tags exist in {s}", "git fetch --tags --depth 1", cwd=TOP / s)
def set_output(name, value):
if "GITHUB_OUTPUT" in os.environ:
with open(os.environ["GITHUB_OUTPUT"], "at") as f:
print(f"{name}={value}", file=f)
else:
print(f"Would set GitHub actions output {name} to '{value}'")
print(f"{name}: {value!r}")
def main():
SUBMODULES_BY_TARGET = {}
def main(target):
submodules = []
submodules_tags = []
print("Target:", TARGET)
print("Target:", target)
if TARGET == "scheduler":
# submodules = ["tools/"]
if target == "all":
submodules = [".", "frozen"] # explicitly list frozen to get tags
elif target == "scheduler":
submodules = ["extmod/ulab", "lib/", "tools/"]
elif TARGET == "tests":
submodules = ["extmod/ulab", "lib/", "tools/"]
submodules_tags = [
elif target == "tests":
submodules = [
"extmod/ulab",
"lib/",
"tools/",
"frozen/Adafruit_CircuitPython_asyncio",
"frozen/Adafruit_CircuitPython_Ticks",
]
elif TARGET == "docs":
elif target == "docs":
# used in .readthedocs.yml to generate RTD
submodules = ["extmod/ulab"]
submodules_tags = ["frozen/"]
elif TARGET == "mpy-cross" or TARGET == "mpy-cross-mac":
submodules = ["extmod/ulab", "frozen"]
elif target == "mpy-cross" or target == "mpy-cross-mac":
submodules = ["tools/"] # for huffman
elif TARGET == "windows":
elif target == "windows":
# This builds one board from a number of ports so fill out a bunch of submodules
for port in ("atmel-samd", "nrf", "raspberrypi", "stm"):
submodules.append(f"ports/{port}")
submodules.extend(PORT_DEPS[port])
unique_submodules = set(submodules)
submodules = list(unique_submodules)
elif TARGET == "website":
submodules = ["tools/adabot/"]
submodules_tags = ["frozen/"]
elif TARGET == "pre-commit":
elif target == "website":
submodules = ["tools/adabot", "frozen"]
elif target == "pre-commit":
submodules = ["extmod/ulab"]
elif target in PORT_DEPS:
submodules = ["data", "extmod", "lib", "tools", "frozen", f"ports/{target}"] + PORT_DEPS[
target
]
else:
p = list(pathlib.Path(".").glob(f"ports/*/boards/{TARGET}/mpconfigboard.mk"))
p = list(pathlib.Path(TOP).glob(f"ports/*/boards/{target}/mpconfigboard.mk"))
if not p:
raise RuntimeError(f"Unsupported target: {TARGET}")
raise RuntimeError(f"Unsupported target: {target}")
config = p[0]
# Add the ports folder to init submodules
port_folder = config.parents[2]
port = port_folder.name
submodules.append(str(port_folder))
submodules.append(f"ports/{port}")
submodules.append("tools/") # for huffman
submodules.extend(PORT_DEPS[port])
with config.open() as f:
@ -140,24 +209,14 @@ def main():
if lib_folder.count("/") > 1:
lib_folder = lib_folder.split("/", maxsplit=2)
lib_folder = "/".join(lib_folder[:2])
submodules_tags.append(lib_folder)
submodules.append(lib_folder)
print("Submodule tags[Y]:", submodules_tags)
print("Submodule tags[N]:", submodules)
if submodules_tags:
run(
"Init the submodules with tags",
f"git submodule update --init {' '.join(submodules_tags)}",
)
print("Submodules:", " ".join(submodules))
if submodules:
run(
"Init the submodules without tags",
f"git submodule update --init --depth=1 {' '.join(submodules)}",
)
fetch(submodules)
for submodule in submodules_tags:
for submodule in submodules:
if submodule.startswith("frozen"):
set_output("frozen_tags", True)
break
@ -166,4 +225,11 @@ def main():
if __name__ == "__main__":
main()
if len(sys.argv) < 2:
raise SystemExit("Usage: ci_fetch_deps dep...")
run("Sync submodule URLs", "git submodule sync --quiet", cwd=TOP)
# Target will be a board, "test", "docs", "mpy-cross-mac", or "windows"
for target in sys.argv[1:]:
main(target)

23
tools/fetch-submodules.sh
View File

@ -1,23 +0,0 @@
#!/usr/bin/env sh
# Pass the directories of submodules to fetch as command-line arguments. For example:
# ./fetch-submodules.sh lib tools
# If no arguments are passed, all submodules will be fetched.
# This script handles being called at other than the top level by making the paths
# for the submodules absolute.
TOP=$(git rev-parse --show-toplevel)
git submodule sync --quiet
# Prefix all the args with the absolute path to the top of the repo.
abs_submodules=""
for d in "$@"; do
abs_submodules="${abs_submodules} ${TOP}/${d}"
done
echo ${abs_submodules}
# Fetch submodules as partial clones if possible. If that fails due to an older version of git,
# do a shallow init and fetch tags.
git submodule update --init --filter=blob:none ${abs_submodules} || \
git submodule update --init --depth 1 ${abs_submodules} && \
git submodule foreach 'git fetch --tags --depth 1' || \
echo "ERROR: fetch-submodules.sh FAILED"