384 lines
13 KiB
Python
384 lines
13 KiB
Python
# The MIT License (MIT)
|
|
#
|
|
# SPDX-FileCopyrightText: Copyright (c) 2019 Michael Schroeder
|
|
#
|
|
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
# of this software and associated documentation files (the "Software"), to deal
|
|
# in the Software without restriction, including without limitation the rights
|
|
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
# copies of the Software, and to permit persons to whom the Software is
|
|
# furnished to do so, subject to the following conditions:
|
|
#
|
|
# The above copyright notice and this permission notice shall be included in
|
|
# all copies or substantial portions of the Software.
|
|
#
|
|
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
# THE SOFTWARE.
|
|
#
|
|
|
|
import json
|
|
import os
|
|
import pathlib
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import functools
|
|
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
|
|
SUPPORTED_PORTS = [
|
|
"atmel-samd",
|
|
"broadcom",
|
|
"cxd56",
|
|
"espressif",
|
|
"litex",
|
|
"mimxrt10xx",
|
|
"nrf",
|
|
"raspberrypi",
|
|
"silabs",
|
|
"stm",
|
|
]
|
|
|
|
ALIASES_BY_BOARD = {
|
|
"circuitplayground_express": [
|
|
"circuitplayground_express_4h",
|
|
"circuitplayground_express_digikey_pycon2019",
|
|
],
|
|
"pybadge": ["edgebadge"],
|
|
"pyportal": ["pyportal_pynt"],
|
|
"gemma_m0": ["gemma_m0_pycon2018"],
|
|
}
|
|
|
|
ALIASES_BRAND_NAMES = {
|
|
"circuitplayground_express_4h": "Adafruit Circuit Playground Express 4-H",
|
|
"circuitplayground_express_digikey_pycon2019": "Circuit Playground Express Digi-Key PyCon 2019",
|
|
"edgebadge": "Adafruit EdgeBadge",
|
|
"pyportal_pynt": "Adafruit PyPortal Pynt",
|
|
"gemma_m0_pycon2018": "Adafruit Gemma M0 PyCon 2018",
|
|
}
|
|
|
|
ADDITIONAL_MODULES = {
|
|
"_asyncio": "MICROPY_PY_ASYNCIO",
|
|
"adafruit_bus_device": "CIRCUITPY_BUSDEVICE",
|
|
"adafruit_pixelbuf": "CIRCUITPY_PIXELBUF",
|
|
"array": "CIRCUITPY_ARRAY",
|
|
# always available, so depend on something that's always 1.
|
|
"builtins": "CIRCUITPY",
|
|
"builtins.pow3": "CIRCUITPY_BUILTINS_POW3",
|
|
"busio.SPI": "CIRCUITPY_BUSIO_SPI",
|
|
"busio.UART": "CIRCUITPY_BUSIO_UART",
|
|
"collections": "CIRCUITPY_COLLECTIONS",
|
|
"fontio": "CIRCUITPY_DISPLAYIO",
|
|
"io": "CIRCUITPY_IO",
|
|
"keypad.KeyMatrix": "CIRCUITPY_KEYPAD_KEYMATRIX",
|
|
"keypad.Keys": "CIRCUITPY_KEYPAD_KEYS",
|
|
"keypad.ShiftRegisterKeys": "CIRCUITPY_KEYPAD_SHIFTREGISTERKEYS",
|
|
"os.getenv": "CIRCUITPY_OS_GETENV",
|
|
"select": "MICROPY_PY_SELECT_SELECT",
|
|
"sys": "CIRCUITPY_SYS",
|
|
"terminalio": "CIRCUITPY_DISPLAYIO",
|
|
"usb": "CIRCUITPY_USB_HOST",
|
|
}
|
|
|
|
MODULES_NOT_IN_BINDINGS = [ "binascii", "errno", "json", "re", "ulab" ]
|
|
|
|
FROZEN_EXCLUDES = ["examples", "docs", "tests", "utils", "conf.py", "setup.py"]
|
|
"""Files and dirs at the root of a frozen directory that should be ignored.
|
|
This is the same list as in the preprocess_frozen_modules script."""
|
|
|
|
repository_urls = {}
|
|
"""Cache of repository URLs for frozen modules."""
|
|
|
|
root_dir = pathlib.Path(__file__).resolve().parent.parent
|
|
|
|
|
|
def get_circuitpython_root_dir():
|
|
"""The path to the root './circuitpython' directory."""
|
|
return root_dir
|
|
|
|
|
|
def get_bindings():
|
|
"""Get a list of modules in shared-bindings and ports/*/bindings based on folder names."""
|
|
shared_bindings_modules = [
|
|
module.name
|
|
for module in (get_circuitpython_root_dir() / "shared-bindings").iterdir()
|
|
if module.is_dir()
|
|
]
|
|
bindings_modules = []
|
|
for d in get_circuitpython_root_dir().glob("ports/*/bindings"):
|
|
bindings_modules.extend(module.name for module in d.iterdir() if d.is_dir())
|
|
return shared_bindings_modules + bindings_modules + MODULES_NOT_IN_BINDINGS + list(ADDITIONAL_MODULES.keys())
|
|
|
|
|
|
def get_board_mapping():
|
|
"""
|
|
Compiles the list of boards from the directories, with aliases and mapping
|
|
to the port.
|
|
"""
|
|
boards = {}
|
|
for port in SUPPORTED_PORTS:
|
|
board_path = root_dir / "ports" / port / "boards"
|
|
for board_path in os.scandir(board_path):
|
|
if board_path.is_dir():
|
|
board_files = os.listdir(board_path.path)
|
|
board_id = board_path.name
|
|
aliases = ALIASES_BY_BOARD.get(board_path.name, [])
|
|
boards[board_id] = {
|
|
"port": port,
|
|
"download_count": 0,
|
|
"aliases": aliases,
|
|
}
|
|
for alias in aliases:
|
|
boards[alias] = {
|
|
"port": port,
|
|
"download_count": 0,
|
|
"alias": True,
|
|
"aliases": [],
|
|
}
|
|
return boards
|
|
|
|
|
|
def build_module_map():
|
|
"""Establish the base of the JSON file, based on the contents from
|
|
`configs`. Base contains the module name and the controlling C macro name.
|
|
"""
|
|
base = dict()
|
|
modules = get_bindings()
|
|
for module in modules:
|
|
full_name = module
|
|
if module in ADDITIONAL_MODULES:
|
|
search_identifier = ADDITIONAL_MODULES[module]
|
|
else:
|
|
search_identifier = "CIRCUITPY_" + module.lstrip("_").upper()
|
|
|
|
base[module] = {
|
|
"name": full_name,
|
|
"key": search_identifier,
|
|
}
|
|
|
|
return base
|
|
|
|
|
|
def get_settings_from_makefile(port_dir, board_name):
|
|
"""Invoke make to print the value of critical build settings
|
|
|
|
This means that the effect of all Makefile directives is taken
|
|
into account, without having to re-encode the logic that sets them
|
|
in this script, something that has proved error-prone
|
|
|
|
This list must explicitly include any setting queried by tools/ci_set_matrix.py.
|
|
"""
|
|
contents = subprocess.run(
|
|
["make", "-C", port_dir, "-f", "Makefile", f"BOARD={board_name}", "print-CFLAGS", "print-CIRCUITPY_BUILD_EXTENSIONS", "print-FROZEN_MPY_DIRS", "print-SRC_PATTERNS", "print-SRC_SUPERVISOR"],
|
|
encoding="utf-8",
|
|
errors="replace",
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
)
|
|
# Make signals errors with exit status 2; 0 and 1 are "non-error" statuses
|
|
if contents.returncode not in (0, 1):
|
|
error_msg = (
|
|
f"Invoking '{' '.join(contents.args)}' exited with "
|
|
f"{contents.returncode}: {contents.stderr}"
|
|
)
|
|
raise RuntimeError(error_msg)
|
|
|
|
settings = {}
|
|
for line in contents.stdout.split("\n"):
|
|
if line.startswith('CFLAGS ='):
|
|
for m in re.findall(r'-D([A-Z][A-Z0-9_]*)=(\d+)', line):
|
|
settings[m[0]] = m[1]
|
|
elif m := re.match(r"^([A-Z][A-Z0-9_]*) = (.*)$", line):
|
|
settings[m.group(1)] = m.group(2)
|
|
|
|
return settings
|
|
|
|
|
|
def get_repository_url(directory):
|
|
if directory in repository_urls:
|
|
return repository_urls[directory]
|
|
readme = None
|
|
for readme_path in (
|
|
os.path.join(directory, "README.rst"),
|
|
os.path.join(os.path.dirname(directory), "README.rst"),
|
|
):
|
|
if os.path.exists(readme_path):
|
|
readme = readme_path
|
|
break
|
|
path = None
|
|
if readme:
|
|
with open(readme, "r") as fp:
|
|
for line in fp.readlines():
|
|
if m := re.match(
|
|
r"\s+:target:\s+(http\S+(docs.circuitpython|readthedocs)\S+)\s*",
|
|
line,
|
|
):
|
|
path = m.group(1)
|
|
break
|
|
if m := re.search(r"<(http[^>]+)>", line):
|
|
path = m.group(1)
|
|
break
|
|
if path is None:
|
|
contents = subprocess.run(
|
|
["git", "remote", "get-url", "origin"],
|
|
encoding="utf-8",
|
|
errors="replace",
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
cwd=directory,
|
|
)
|
|
path = contents.stdout.strip()
|
|
repository_urls[directory] = path
|
|
return path
|
|
|
|
def remove_prefix(s, prefix):
|
|
if not s.startswith(prefix):
|
|
raise ValueError(f"{s=} does not start with {prefix=}")
|
|
return s.removeprefix(prefix)
|
|
|
|
def frozen_modules_from_dirs(frozen_mpy_dirs, withurl):
|
|
"""
|
|
Go through the list of frozen directories and extract the python modules.
|
|
Paths are of the type:
|
|
$(TOP)/frozen/Adafruit_CircuitPython_CircuitPlayground
|
|
$(TOP)/frozen/circuitpython-stage/meowbit
|
|
Python modules are at the root of the path, and are python files or directories
|
|
containing python files. Except the ones in the FROZEN_EXCLUDES list.
|
|
"""
|
|
frozen_modules = []
|
|
for frozen_path in filter(lambda x: x, frozen_mpy_dirs.split(" ")):
|
|
frozen_path = remove_prefix(frozen_path, '../../')
|
|
source_dir = get_circuitpython_root_dir() / frozen_path
|
|
url_repository = get_repository_url(source_dir)
|
|
for sub in source_dir.glob("*"):
|
|
if sub.name in FROZEN_EXCLUDES:
|
|
continue
|
|
if sub.name.endswith(".py"):
|
|
if withurl:
|
|
frozen_modules.append((sub.name[:-3], url_repository))
|
|
else:
|
|
frozen_modules.append(sub.name[:-3])
|
|
continue
|
|
if next(sub.glob("**/*.py"), None): # tests if not empty
|
|
if withurl:
|
|
frozen_modules.append((sub.name, url_repository))
|
|
else:
|
|
frozen_modules.append(sub.name)
|
|
return frozen_modules
|
|
|
|
|
|
def lookup_setting(settings, key, default=""):
|
|
while True:
|
|
value = settings.get(key, default)
|
|
if not value.startswith("$"):
|
|
break
|
|
key = value[2:-1]
|
|
return value
|
|
|
|
|
|
@functools.cache
|
|
def all_ports_all_boards(ports=SUPPORTED_PORTS):
|
|
for port in ports:
|
|
port_dir = get_circuitpython_root_dir() / "ports" / port
|
|
for entry in (port_dir / "boards").iterdir():
|
|
if not entry.is_dir():
|
|
continue
|
|
yield (port, entry)
|
|
|
|
|
|
def support_matrix_by_board(use_branded_name=True, withurl=True):
|
|
"""Compiles a list of the available core modules available for each
|
|
board.
|
|
"""
|
|
base = build_module_map()
|
|
|
|
def support_matrix(arg):
|
|
port, entry = arg
|
|
port_dir = get_circuitpython_root_dir() / "ports" / port
|
|
settings = get_settings_from_makefile(str(port_dir), entry.name)
|
|
|
|
if use_branded_name:
|
|
with open(entry / "mpconfigboard.h") as get_name:
|
|
board_contents = get_name.read()
|
|
board_name_re = re.search(
|
|
r"(?<=MICROPY_HW_BOARD_NAME)\s+(.+)", board_contents
|
|
)
|
|
if board_name_re:
|
|
board_name = board_name_re.group(1).strip('"')
|
|
else:
|
|
board_name = entry.name
|
|
|
|
board_modules = []
|
|
for module in base:
|
|
key = base[module]["key"]
|
|
if int(lookup_setting(settings, key, "0")):
|
|
board_modules.append(base[module]["name"])
|
|
board_modules.sort()
|
|
|
|
if "CIRCUITPY_BUILD_EXTENSIONS" in settings:
|
|
board_extensions = [
|
|
extension.strip()
|
|
for extension in settings["CIRCUITPY_BUILD_EXTENSIONS"].split(",")
|
|
]
|
|
else:
|
|
raise OSError(f"Board extensions undefined: {board_name}.")
|
|
|
|
frozen_modules = []
|
|
if "FROZEN_MPY_DIRS" in settings:
|
|
frozen_modules = frozen_modules_from_dirs(
|
|
settings["FROZEN_MPY_DIRS"], withurl
|
|
)
|
|
if frozen_modules:
|
|
frozen_modules.sort()
|
|
|
|
# generate alias boards too
|
|
board_matrix = [
|
|
(
|
|
board_name,
|
|
{
|
|
"modules": board_modules,
|
|
"frozen_libraries": frozen_modules,
|
|
"extensions": board_extensions,
|
|
},
|
|
)
|
|
]
|
|
if entry.name in ALIASES_BY_BOARD:
|
|
for alias in ALIASES_BY_BOARD[entry.name]:
|
|
if use_branded_name:
|
|
if alias in ALIASES_BRAND_NAMES:
|
|
alias = ALIASES_BRAND_NAMES[alias]
|
|
else:
|
|
alias = alias.replace("_", " ").title()
|
|
board_matrix.append(
|
|
(
|
|
alias,
|
|
{
|
|
"modules": board_modules,
|
|
"frozen_libraries": frozen_modules,
|
|
"extensions": board_extensions,
|
|
},
|
|
)
|
|
)
|
|
|
|
return board_matrix # this is now a list of (board,modules)
|
|
|
|
executor = ThreadPoolExecutor(max_workers=os.cpu_count())
|
|
mapped_exec = executor.map(support_matrix, all_ports_all_boards())
|
|
# flatmap with comprehensions
|
|
boards = dict(
|
|
sorted(
|
|
[board for matrix in mapped_exec for board in matrix], key=lambda x: x[0]
|
|
)
|
|
)
|
|
|
|
return boards
|
|
|
|
|
|
if __name__ == "__main__":
|
|
print(json.dumps(support_matrix_by_board(), indent=2))
|