djsundog
/
circuitpython
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #7725 from dhalbert/port-specific-modules-in-support-matrix 

add port-specific modules to support matrix
This commit is contained in:
MicroDev 2023-03-15 09:32:10 +05:30 committed by GitHub
parent f708048d8e 069a058a83
commit 76d590bc01
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 137 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

201
docs/shared_bindings_matrix.py
View File

@ -31,7 +31,17 @@ import functools
from concurrent.futures import ThreadPoolExecutor from concurrent.futures import ThreadPoolExecutor
SUPPORTED_PORTS = ['atmel-samd', 'broadcom', 'cxd56', 'espressif', 'litex', 'mimxrt10xx', 'nrf', 'raspberrypi', 'stm'] SUPPORTED_PORTS = [
"atmel-samd",
"broadcom",
"cxd56",
"espressif",
"litex",
"mimxrt10xx",
"nrf",
"raspberrypi",
"stm",
]
ALIASES_BY_BOARD = { ALIASES_BY_BOARD = {
"circuitplayground_express": [ "circuitplayground_express": [
@ -44,16 +54,11 @@ ALIASES_BY_BOARD = {
} }
ALIASES_BRAND_NAMES = { ALIASES_BRAND_NAMES = {
"circuitplayground_express_4h": "circuitplayground_express_4h": "Adafruit Circuit Playground Express 4-H",
"Adafruit Circuit Playground Express 4-H", "circuitplayground_express_digikey_pycon2019": "Circuit Playground Express Digi-Key PyCon 2019",
"circuitplayground_express_digikey_pycon2019": "edgebadge": "Adafruit EdgeBadge",
"Circuit Playground Express Digi-Key PyCon 2019", "pyportal_pynt": "Adafruit PyPortal Pynt",
"edgebadge": "gemma_m0_pycon2018": "Adafruit Gemma M0 PyCon 2018",
"Adafruit EdgeBadge",
"pyportal_pynt":
"Adafruit PyPortal Pynt",
"gemma_m0_pycon2018":
"Adafruit Gemma M0 PyCon 2018",
} }
ADDITIONAL_MODULES = { ADDITIONAL_MODULES = {
@ -72,7 +77,19 @@ ADDITIONAL_MODULES = {
"usb": "CIRCUITPY_USB_HOST", "usb": "CIRCUITPY_USB_HOST",
} }
MODULES_NOT_IN_SHARED_BINDINGS = ["_asyncio", "array", "binascii", "builtins", "collections", "errno", "json", "re", "select", "sys", "ulab"] MODULES_NOT_IN_BINDINGS = [
"_asyncio",
"array",
"binascii",
"builtins",
"collections",
"errno",
"json",
"re",
"select",
"sys",
"ulab",
]
FROZEN_EXCLUDES = ["examples", "docs", "tests", "utils", "conf.py", "setup.py"] FROZEN_EXCLUDES = ["examples", "docs", "tests", "utils", "conf.py", "setup.py"]
"""Files and dirs at the root of a frozen directory that should be ignored. """Files and dirs at the root of a frozen directory that should be ignored.
@ -83,16 +100,23 @@ repository_urls = {}
root_dir = pathlib.Path(__file__).resolve().parent.parent root_dir = pathlib.Path(__file__).resolve().parent.parent
def get_circuitpython_root_dir(): def get_circuitpython_root_dir():
""" The path to the root './circuitpython' directory. """The path to the root './circuitpython' directory."""
"""
return root_dir return root_dir
def get_shared_bindings():
""" Get a list of modules in shared-bindings based on folder names. def get_bindings():
""" """Get a list of modules in shared-bindings and ports/*/bindings based on folder names."""
shared_bindings_dir = get_circuitpython_root_dir() / "shared-bindings" shared_bindings_modules = [
return [item.name for item in shared_bindings_dir.iterdir()] + MODULES_NOT_IN_SHARED_BINDINGS module.name
for module in (get_circuitpython_root_dir() / "shared-bindings").iterdir()
if module.is_dir()
]
bindings_modules = []
for d in get_circuitpython_root_dir().glob("ports/*/bindings"):
bindings_modules.extend(module.name for module in d.iterdir() if d.is_dir())
return shared_bindings_modules + bindings_modules + MODULES_NOT_IN_BINDINGS
def get_board_mapping(): def get_board_mapping():
@ -124,8 +148,7 @@ def get_board_mapping():
def read_mpconfig(): def read_mpconfig():
""" Open 'circuitpy_mpconfig.mk' and return the contents. """Open 'circuitpy_mpconfig.mk' and return the contents."""
"""
configs = [] configs = []
cpy_mpcfg = get_circuitpython_root_dir() / "py" / "circuitpy_mpconfig.mk" cpy_mpcfg = get_circuitpython_root_dir() / "py" / "circuitpy_mpconfig.mk"
with open(cpy_mpcfg) as mpconfig: with open(cpy_mpcfg) as mpconfig:
@ -135,14 +158,14 @@ def read_mpconfig():
def build_module_map(): def build_module_map():
""" Establish the base of the JSON file, based on the contents from """Establish the base of the JSON file, based on the contents from
`configs`. Base will contain module names, if they're part of `configs`. Base will contain module names, if they're part of
the `FULL_BUILD`, or their default value (0, 1, or a list of the `FULL_BUILD`, or their default value (0, 1, or a list of
modules that determine default [see audiocore, audiomixer, etc.]). modules that determine default [see audiocore, audiomixer, etc.]).
""" """
base = dict() base = dict()
modules = get_shared_bindings() modules = get_bindings()
configs = read_mpconfig() configs = read_mpconfig()
full_build = False full_build = False
for module in modules: for module in modules:
@ -150,7 +173,7 @@ def build_module_map():
if module in ADDITIONAL_MODULES: if module in ADDITIONAL_MODULES:
search_identifier = ADDITIONAL_MODULES[module] search_identifier = ADDITIONAL_MODULES[module]
else: else:
search_identifier = 'CIRCUITPY_'+module.lstrip("_").upper() search_identifier = "CIRCUITPY_" + module.lstrip("_").upper()
re_pattern = f"{re.escape(search_identifier)}\s*\??=\s*(.+)" re_pattern = f"{re.escape(search_identifier)}\s*\??=\s*(.+)"
find_config = re.findall(re_pattern, configs) find_config = re.findall(re_pattern, configs)
if not find_config: if not find_config:
@ -173,21 +196,22 @@ def build_module_map():
return base return base
def get_settings_from_makefile(port_dir, board_name):
""" Invoke make in a mode which prints the database, then parse it for
settings.
This means that the effect of all Makefile directives is taken def get_settings_from_makefile(port_dir, board_name):
into account, without having to re-encode the logic that sets them """Invoke make in a mode which prints the database, then parse it for
in this script, something that has proved error-prone settings.
This means that the effect of all Makefile directives is taken
into account, without having to re-encode the logic that sets them
in this script, something that has proved error-prone
""" """
contents = subprocess.run( contents = subprocess.run(
["make", "-C", port_dir, f"BOARD={board_name}", "-qp", "print-CC"], ["make", "-C", port_dir, f"BOARD={board_name}", "-qp", "print-CC"],
encoding="utf-8", encoding="utf-8",
errors="replace", errors="replace",
stdout=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.PIPE stderr=subprocess.PIPE,
) )
# Make signals errors with exit status 2; 0 and 1 are "non-error" statuses # Make signals errors with exit status 2; 0 and 1 are "non-error" statuses
if contents.returncode not in (0, 1): if contents.returncode not in (0, 1):
error_msg = ( error_msg = (
@ -197,22 +221,23 @@ def get_settings_from_makefile(port_dir, board_name):
raise RuntimeError(error_msg) raise RuntimeError(error_msg)
settings = {} settings = {}
for line in contents.stdout.split('\n'): for line in contents.stdout.split("\n"):
# Handle both = and := definitions. # Handle both = and := definitions.
m = re.match(r'^([A-Z][A-Z0-9_]*) :?= (.*)$', line) m = re.match(r"^([A-Z][A-Z0-9_]*) :?= (.*)$", line)
if m: if m:
settings[m.group(1)] = m.group(2) settings[m.group(1)] = m.group(2)
return settings return settings
def get_repository_url(directory): def get_repository_url(directory):
if directory in repository_urls: if directory in repository_urls:
return repository_urls[directory] return repository_urls[directory]
readme = None readme = None
for readme_path in ( for readme_path in (
os.path.join(directory, "README.rst"), os.path.join(directory, "README.rst"),
os.path.join(os.path.dirname(directory), "README.rst") os.path.join(os.path.dirname(directory), "README.rst"),
): ):
if os.path.exists(readme_path): if os.path.exists(readme_path):
readme = readme_path readme = readme_path
break break
@ -220,7 +245,10 @@ def get_repository_url(directory):
if readme: if readme:
with open(readme, "r") as fp: with open(readme, "r") as fp:
for line in fp.readlines(): for line in fp.readlines():
if m := re.match("\s+:target:\s+(http\S+(docs.circuitpython|readthedocs)\S+)\s*", line): if m := re.match(
"\s+:target:\s+(http\S+(docs.circuitpython|readthedocs)\S+)\s*",
line,
):
path = m.group(1) path = m.group(1)
break break
if m := re.search("<(http[^>]+)>", line): if m := re.search("<(http[^>]+)>", line):
@ -233,12 +261,13 @@ def get_repository_url(directory):
errors="replace", errors="replace",
stdout=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, stderr=subprocess.PIPE,
cwd=directory cwd=directory,
) )
path = contents.stdout.strip() path = contents.stdout.strip()
repository_urls[directory] = path repository_urls[directory] = path
return path return path
def frozen_modules_from_dirs(frozen_mpy_dirs, withurl): def frozen_modules_from_dirs(frozen_mpy_dirs, withurl):
""" """
Go through the list of frozen directories and extract the python modules. Go through the list of frozen directories and extract the python modules.
@ -261,34 +290,36 @@ def frozen_modules_from_dirs(frozen_mpy_dirs, withurl):
else: else:
frozen_modules.append(sub.name[:-3]) frozen_modules.append(sub.name[:-3])
continue continue
if next(sub.glob("**/*.py"), None): # tests if not empty if next(sub.glob("**/*.py"), None): # tests if not empty
if withurl: if withurl:
frozen_modules.append((sub.name, url_repository)) frozen_modules.append((sub.name, url_repository))
else: else:
frozen_modules.append(sub.name) frozen_modules.append(sub.name)
return frozen_modules return frozen_modules
def lookup_setting(settings, key, default=''):
def lookup_setting(settings, key, default=""):
while True: while True:
value = settings.get(key, default) value = settings.get(key, default)
if not value.startswith('$'): if not value.startswith("$"):
break break
key = value[2:-1] key = value[2:-1]
return value return value
@functools.cache @functools.cache
def all_ports_all_boards(ports=SUPPORTED_PORTS): def all_ports_all_boards(ports=SUPPORTED_PORTS):
for port in ports: for port in ports:
port_dir = get_circuitpython_root_dir() / "ports" / port port_dir = get_circuitpython_root_dir() / "ports" / port
for entry in (port_dir / "boards").iterdir(): for entry in (port_dir / "boards").iterdir():
if not entry.is_dir(): if not entry.is_dir():
continue continue
yield (port, entry) yield (port, entry)
def support_matrix_by_board(use_branded_name=True, withurl=True): def support_matrix_by_board(use_branded_name=True, withurl=True):
""" Compiles a list of the available core modules available for each """Compiles a list of the available core modules available for each
board. board.
""" """
base = build_module_map() base = build_module_map()
@ -300,8 +331,9 @@ def support_matrix_by_board(use_branded_name=True, withurl=True):
if use_branded_name: if use_branded_name:
with open(entry / "mpconfigboard.h") as get_name: with open(entry / "mpconfigboard.h") as get_name:
board_contents = get_name.read() board_contents = get_name.read()
board_name_re = re.search(r"(?<=MICROPY_HW_BOARD_NAME)\s+(.+)", board_name_re = re.search(
board_contents) r"(?<=MICROPY_HW_BOARD_NAME)\s+(.+)", board_contents
)
if board_name_re: if board_name_re:
board_name = board_name_re.group(1).strip('"') board_name = board_name_re.group(1).strip('"')
else: else:
@ -309,56 +341,69 @@ def support_matrix_by_board(use_branded_name=True, withurl=True):
board_modules = [] board_modules = []
for module in base: for module in base:
key = base[module]['key'] key = base[module]["key"]
if int(lookup_setting(settings, key, '0')): if int(lookup_setting(settings, key, "0")):
board_modules.append(base[module]['name']) board_modules.append(base[module]["name"])
board_modules.sort() board_modules.sort()
if "CIRCUITPY_BUILD_EXTENSIONS" in settings: if "CIRCUITPY_BUILD_EXTENSIONS" in settings:
board_extensions = [ board_extensions = [
extension.strip() for extension in extension.strip()
settings["CIRCUITPY_BUILD_EXTENSIONS"].split(",") for extension in settings["CIRCUITPY_BUILD_EXTENSIONS"].split(",")
] ]
else: else:
raise OSError(f"Board extensions undefined: {board_name}.") raise OSError(f"Board extensions undefined: {board_name}.")
frozen_modules = [] frozen_modules = []
if "FROZEN_MPY_DIRS" in settings: if "FROZEN_MPY_DIRS" in settings:
frozen_modules = frozen_modules_from_dirs(settings["FROZEN_MPY_DIRS"], withurl) frozen_modules = frozen_modules_from_dirs(
settings["FROZEN_MPY_DIRS"], withurl
)
if frozen_modules: if frozen_modules:
frozen_modules.sort() frozen_modules.sort()
# generate alias boards too # generate alias boards too
board_matrix = [( board_matrix = [
board_name, { (
"modules": board_modules, board_name,
"frozen_libraries": frozen_modules, {
"extensions": board_extensions, "modules": board_modules,
} "frozen_libraries": frozen_modules,
)] "extensions": board_extensions,
},
)
]
if entry.name in ALIASES_BY_BOARD: if entry.name in ALIASES_BY_BOARD:
for alias in ALIASES_BY_BOARD[entry.name]: for alias in ALIASES_BY_BOARD[entry.name]:
if use_branded_name: if use_branded_name:
if alias in ALIASES_BRAND_NAMES: if alias in ALIASES_BRAND_NAMES:
alias = ALIASES_BRAND_NAMES[alias] alias = ALIASES_BRAND_NAMES[alias]
else: else:
alias = alias.replace("_"," ").title() alias = alias.replace("_", " ").title()
board_matrix.append(( board_matrix.append(
alias, { (
"modules": board_modules, alias,
"frozen_libraries": frozen_modules, {
"extensions": board_extensions, "modules": board_modules,
}, "frozen_libraries": frozen_modules,
)) "extensions": board_extensions,
},
)
)
return board_matrix # this is now a list of (board,modules) return board_matrix # this is now a list of (board,modules)
executor = ThreadPoolExecutor(max_workers=os.cpu_count()) executor = ThreadPoolExecutor(max_workers=os.cpu_count())
mapped_exec = executor.map(support_matrix, all_ports_all_boards()) mapped_exec = executor.map(support_matrix, all_ports_all_boards())
# flatmap with comprehensions # flatmap with comprehensions
boards = dict(sorted([board for matrix in mapped_exec for board in matrix], key=lambda x: x[0])) boards = dict(
sorted(
[board for matrix in mapped_exec for board in matrix], key=lambda x: x[0]
)
)
return boards return boards
if __name__ == '__main__':
if __name__ == "__main__":
print(json.dumps(support_matrix_by_board(), indent=2)) print(json.dumps(support_matrix_by_board(), indent=2))

15
tools/build_board_info.py
View File

@ -6,6 +6,7 @@
import json import json
import os import os
import requests
import subprocess import subprocess
import sys import sys
import sh import sh
@ -96,7 +97,19 @@ def get_current_info():
response = response.json() response = response.json()
git_info = commit_sha, response["sha"] git_info = commit_sha, response["sha"]
current_list = json.loads(base64.b64decode(response["content"]).decode("utf-8"))
if response["content"] != "":
# if the file is there
current_list = json.loads(base64.b64decode(response["content"]).decode("utf-8"))
else:
# if too big, the file is not included
download_url = response["download_url"]
response = requests.get(download_url)
if not response.ok:
print(response.text)
raise RuntimeError("cannot get previous files.json")
current_list = response.json()
current_info = {} current_info = {}
for info in current_list: for info in current_list:
current_info[info["id"]] = info current_info[info["id"]] = info