circuitpython/tools/ci_fetch_deps.py

import os
import sys
import time
import shlex
import pathlib
import re
import subprocess

TOP = pathlib.Path(__file__).parent.parent


def _git_version():
    version_str = subprocess.check_output(["git", "--version"], encoding="ascii", errors="replace")
    version_str = re.search("([0-9]\.*)*[0-9]", version_str).group(0)
    return tuple(int(part) for part in version_str.split("."))


clone_supports_filter = (
    False if "NO_USE_CLONE_FILTER" in os.environ else _git_version() >= (2, 27, 0)
)

if clone_supports_filter:
    filter_maybe = "--filter=blob:none"
else:
    filter_maybe = ""


def _all_submodules():
    submodule_str = subprocess.check_output(
        ["git", "submodule", "status"], encoding="ascii", errors="replace", cwd=TOP
    )
    return [row.split()[1] for row in submodule_str.strip().split("\n")]


all_submodules = _all_submodules()


def matching_submodules(s):
    if s.endswith("/"):
        return [m for m in all_submodules if m.startswith(s)]
    elif s not in all_submodules:
        raise ValueError(f"{s!r} is not a submodule")
    return [s]


# Submodules needed by port builds outside of their ports directory.
# Should we try and detect these?
PORT_DEPS = {
    "atmel-samd": [
        "extmod/ulab/",
        "lib/adafruit_floppy/",
        "lib/mp3/",
        "lib/protomatter/",
        "lib/quirc/",
        "lib/tinyusb/",
        "lib/tlsf",
        "data/nvm.toml/",
    ],
    "broadcom": ["extmod/ulab/", "lib/tlsf", "lib/tinyusb/"],
    "cxd56": ["extmod/ulab/", "lib/tlsf", "lib/tinyusb/"],
    "espressif": [
        "extmod/ulab/",
        "lib/certificates/",
        "lib/protomatter/",
        "lib/quirc/",
        "lib/tlsf",
        "lib/tinyusb/",
    ],
    "litex": ["extmod/ulab/", "lib/tinyusb/", "lib/tlsf"],
    "mimxrt10xx": ["extmod/ulab/", "lib/tinyusb/", "lib/tlsf", "data/nvm.toml/"],
    "nrf": [
        "extmod/ulab/",
        "lib/mp3/",
        "lib/protomatter/",
        "lib/tinyusb/",
        "lib/tlsf",
        "data/nvm.toml/",
    ],
    "raspberrypi": [
        "extmod/ulab/",
        "lib/adafruit_floppy/",
        "lib/mbedtls/",
        "lib/mp3/",
        "lib/certificates/",
        "lib/protomatter/",
        "lib/quirc/",
        "lib/tinyusb/",
        "lib/tlsf",
        "data/nvm.toml/",
    ],
    "silabs": ["extmod/ulab/", "data/nvm.toml/", "lib/tlsf"],
    "stm": [
        "extmod/ulab/",
        "lib/mp3/",
        "lib/protomatter/",
        "lib/tinyusb/",
        "lib/tlsf",
        "data/nvm.toml/",
    ]
    # omit unix which is part of the "test" target below
}


def run(title, command, cwd):
    print("::group::" + title, flush=True)
    print(f"{command} (in {cwd})", flush=True)
    start = time.monotonic()
    try:
        subprocess.run(shlex.split(command), stderr=subprocess.STDOUT, check=True, cwd=cwd)
    finally:
        print("::endgroup::", flush=True)
        print("Duration:", time.monotonic() - start, flush=True)


def matching_submodules(where):
    for m in all_submodules:
        if m in where:
            yield m
        for w in where:
            if m.startswith(f"{w}/"):
                yield m
                break


def fetch(where):
    if clone_supports_filter:
        run(
            "Init submodules (using filter)",
            f"git submodule update --init {filter_maybe} {' '.join(where)}",
            cwd=TOP,
        )
    else:
        run(
            "Init submodules (using depth)",
            f"git submodule update --init --depth 1 {' '.join(where)}",
            cwd=TOP,
        )
        for s in matching_submodules([w for w in where if w.startswith("frozen")]):
            run(f"Ensure tags exist in {s}", "git fetch --tags --depth 1", cwd=TOP / s)


def set_output(name, value):
    if "GITHUB_OUTPUT" in os.environ:
        with open(os.environ["GITHUB_OUTPUT"], "at") as f:
            print(f"{name}={value}", file=f)
    else:
        print(f"{name}: {value!r}")


SUBMODULES_BY_TARGET = {}


def main(target):
    submodules = []

    print("Target:", target)

    if target == "all":
        submodules = [".", "frozen"]  # explicitly list frozen to get tags
    elif target == "scheduler":
        submodules = ["extmod/ulab", "lib/", "tools/"]
    elif target == "tests":
        submodules = [
            "extmod/ulab",
            "lib/",
            "tools/",
            "frozen/Adafruit_CircuitPython_asyncio",
            "frozen/Adafruit_CircuitPython_Ticks",
        ]
    elif target == "docs":
        # used in .readthedocs.yml to generate RTD
        submodules = ["extmod/ulab", "frozen"]
    elif target == "mpy-cross" or target == "mpy-cross-mac":
        submodules = ["tools/"]  # for huffman
    elif target == "windows":
        # This builds one board from a number of ports so fill out a bunch of submodules
        for port in ("atmel-samd", "nrf", "raspberrypi", "stm"):
            submodules.append(f"ports/{port}")
            submodules.extend(PORT_DEPS[port])
        unique_submodules = set(submodules)
        submodules = list(unique_submodules)
    elif target == "website":
        submodules = ["tools/adabot", "frozen"]
    elif target == "pre-commit":
        submodules = ["extmod/ulab"]
    elif target in PORT_DEPS:
        submodules = ["data", "extmod", "lib", "tools", "frozen", f"ports/{target}"] + PORT_DEPS[
            target
        ]
    else:
        p = list(pathlib.Path(TOP).glob(f"ports/*/boards/{target}/mpconfigboard.mk"))
        if not p:
            raise RuntimeError(f"Unsupported target: {target}")

        config = p[0]
        # Add the ports folder to init submodules
        port_folder = config.parents[2]
        port = port_folder.name
        submodules.append(f"ports/{port}")
        submodules.append("tools/")  # for huffman
        submodules.extend(PORT_DEPS[port])
        with config.open() as f:
            for line in f.readlines():
                prefix = "FROZEN_MPY_DIRS += $(TOP)/"
                if line.startswith(prefix):
                    lib_folder = line.strip()[len(prefix) :]
                    # Drop everything after the second folder because the frozen
                    # folder may be inside the submodule.
                    if lib_folder.count("/") > 1:
                        lib_folder = lib_folder.split("/", maxsplit=2)
                        lib_folder = "/".join(lib_folder[:2])
                    submodules.append(lib_folder)

    print("Submodules:", " ".join(submodules))

    if submodules:
        fetch(submodules)

    for submodule in submodules:
        if submodule.startswith("frozen"):
            set_output("frozen_tags", True)
            break
    else:
        set_output("frozen_tags", False)


if __name__ == "__main__":
    if len(sys.argv) < 2:
        raise SystemExit("Usage: ci_fetch_deps dep...")

    run("Sync submodule URLs", "git submodule sync --quiet", cwd=TOP)

    # Target will be a board, "test", "docs", "mpy-cross-mac", or "windows"
    for target in sys.argv[1:]:
        main(target)