djsundog/circuitpython
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #7725 from dhalbert/port-specific-modules-in-support-matrix

Browse Source 
add port-specific modules to support matrix
This commit is contained in:
MicroDev 2023-03-15 09:32:10 +05:30 committed by GitHub
commit 76d590bc01
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 137 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

151
docs/shared_bindings_matrix.py
View File

@ -31,7 +31,17 @@ import functools
from concurrent.futures import ThreadPoolExecutor from concurrent.futures import ThreadPoolExecutor
SUPPORTED_PORTS = ['atmel-samd', 'broadcom', 'cxd56', 'espressif', 'litex', 'mimxrt10xx', 'nrf', 'raspberrypi', 'stm'] SUPPORTED_PORTS = [
"atmel-samd",
"broadcom",
"cxd56",
"espressif",
"litex",
"mimxrt10xx",
"nrf",
"raspberrypi",
"stm",
]
ALIASES_BY_BOARD = { ALIASES_BY_BOARD = {
"circuitplayground_express": [ "circuitplayground_express": [
@ -44,16 +54,11 @@ ALIASES_BY_BOARD = {
} }
ALIASES_BRAND_NAMES = { ALIASES_BRAND_NAMES = {
"circuitplayground_express_4h": "circuitplayground_express_4h": "Adafruit Circuit Playground Express 4-H",
"Adafruit Circuit Playground Express 4-H", "circuitplayground_express_digikey_pycon2019": "Circuit Playground Express Digi-Key PyCon 2019",
"circuitplayground_express_digikey_pycon2019": "edgebadge": "Adafruit EdgeBadge",
"Circuit Playground Express Digi-Key PyCon 2019", "pyportal_pynt": "Adafruit PyPortal Pynt",
"edgebadge": "gemma_m0_pycon2018": "Adafruit Gemma M0 PyCon 2018",
"Adafruit EdgeBadge",
"pyportal_pynt":
"Adafruit PyPortal Pynt",
"gemma_m0_pycon2018":
"Adafruit Gemma M0 PyCon 2018",
} }
ADDITIONAL_MODULES = { ADDITIONAL_MODULES = {
@ -72,7 +77,19 @@ ADDITIONAL_MODULES = {
"usb": "CIRCUITPY_USB_HOST", "usb": "CIRCUITPY_USB_HOST",
} }
MODULES_NOT_IN_SHARED_BINDINGS = ["_asyncio", "array", "binascii", "builtins", "collections", "errno", "json", "re", "select", "sys", "ulab"] MODULES_NOT_IN_BINDINGS = [
"_asyncio",
"array",
"binascii",
"builtins",
"collections",
"errno",
"json",
"re",
"select",
"sys",
"ulab",
]
FROZEN_EXCLUDES = ["examples", "docs", "tests", "utils", "conf.py", "setup.py"] FROZEN_EXCLUDES = ["examples", "docs", "tests", "utils", "conf.py", "setup.py"]
"""Files and dirs at the root of a frozen directory that should be ignored. """Files and dirs at the root of a frozen directory that should be ignored.
@ -83,16 +100,23 @@ repository_urls = {}
root_dir = pathlib.Path(__file__).resolve().parent.parent root_dir = pathlib.Path(__file__).resolve().parent.parent
def get_circuitpython_root_dir(): def get_circuitpython_root_dir():
""" The path to the root './circuitpython' directory. """The path to the root './circuitpython' directory."""
"""
return root_dir return root_dir
def get_shared_bindings():
""" Get a list of modules in shared-bindings based on folder names. def get_bindings():
""" """Get a list of modules in shared-bindings and ports/*/bindings based on folder names."""
shared_bindings_dir = get_circuitpython_root_dir() / "shared-bindings" shared_bindings_modules = [
return [item.name for item in shared_bindings_dir.iterdir()] + MODULES_NOT_IN_SHARED_BINDINGS module.name
for module in (get_circuitpython_root_dir() / "shared-bindings").iterdir()
if module.is_dir()
]
bindings_modules = []
for d in get_circuitpython_root_dir().glob("ports/*/bindings"):
bindings_modules.extend(module.name for module in d.iterdir() if d.is_dir())
return shared_bindings_modules + bindings_modules + MODULES_NOT_IN_BINDINGS
def get_board_mapping(): def get_board_mapping():
@ -124,8 +148,7 @@ def get_board_mapping():
def read_mpconfig(): def read_mpconfig():
""" Open 'circuitpy_mpconfig.mk' and return the contents. """Open 'circuitpy_mpconfig.mk' and return the contents."""
"""
configs = [] configs = []
cpy_mpcfg = get_circuitpython_root_dir() / "py" / "circuitpy_mpconfig.mk" cpy_mpcfg = get_circuitpython_root_dir() / "py" / "circuitpy_mpconfig.mk"
with open(cpy_mpcfg) as mpconfig: with open(cpy_mpcfg) as mpconfig:
@ -135,14 +158,14 @@ def read_mpconfig():
def build_module_map(): def build_module_map():
""" Establish the base of the JSON file, based on the contents from """Establish the base of the JSON file, based on the contents from
`configs`. Base will contain module names, if they're part of `configs`. Base will contain module names, if they're part of
the `FULL_BUILD`, or their default value (0, 1, or a list of the `FULL_BUILD`, or their default value (0, 1, or a list of
modules that determine default [see audiocore, audiomixer, etc.]). modules that determine default [see audiocore, audiomixer, etc.]).
""" """
base = dict() base = dict()
modules = get_shared_bindings() modules = get_bindings()
configs = read_mpconfig() configs = read_mpconfig()
full_build = False full_build = False
for module in modules: for module in modules:
@ -150,7 +173,7 @@ def build_module_map():
if module in ADDITIONAL_MODULES: if module in ADDITIONAL_MODULES:
search_identifier = ADDITIONAL_MODULES[module] search_identifier = ADDITIONAL_MODULES[module]
else: else:
search_identifier = 'CIRCUITPY_'+module.lstrip("_").upper() search_identifier = "CIRCUITPY_" + module.lstrip("_").upper()
re_pattern = f"{re.escape(search_identifier)}\s*\??=\s*(.+)" re_pattern = f"{re.escape(search_identifier)}\s*\??=\s*(.+)"
find_config = re.findall(re_pattern, configs) find_config = re.findall(re_pattern, configs)
if not find_config: if not find_config:
@ -173,8 +196,9 @@ def build_module_map():
return base return base
def get_settings_from_makefile(port_dir, board_name): def get_settings_from_makefile(port_dir, board_name):
""" Invoke make in a mode which prints the database, then parse it for """Invoke make in a mode which prints the database, then parse it for
settings. settings.
This means that the effect of all Makefile directives is taken This means that the effect of all Makefile directives is taken
@ -186,7 +210,7 @@ def get_settings_from_makefile(port_dir, board_name):
encoding="utf-8", encoding="utf-8",
errors="replace", errors="replace",
stdout=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.PIPE stderr=subprocess.PIPE,
) )
# Make signals errors with exit status 2; 0 and 1 are "non-error" statuses # Make signals errors with exit status 2; 0 and 1 are "non-error" statuses
if contents.returncode not in (0, 1): if contents.returncode not in (0, 1):
@ -197,21 +221,22 @@ def get_settings_from_makefile(port_dir, board_name):
raise RuntimeError(error_msg) raise RuntimeError(error_msg)
settings = {} settings = {}
for line in contents.stdout.split('\n'): for line in contents.stdout.split("\n"):
# Handle both = and := definitions. # Handle both = and := definitions.
m = re.match(r'^([A-Z][A-Z0-9_]*) :?= (.*)$', line) m = re.match(r"^([A-Z][A-Z0-9_]*) :?= (.*)$", line)
if m: if m:
settings[m.group(1)] = m.group(2) settings[m.group(1)] = m.group(2)
return settings return settings
def get_repository_url(directory): def get_repository_url(directory):
if directory in repository_urls: if directory in repository_urls:
return repository_urls[directory] return repository_urls[directory]
readme = None readme = None
for readme_path in ( for readme_path in (
os.path.join(directory, "README.rst"), os.path.join(directory, "README.rst"),
os.path.join(os.path.dirname(directory), "README.rst") os.path.join(os.path.dirname(directory), "README.rst"),
): ):
if os.path.exists(readme_path): if os.path.exists(readme_path):
readme = readme_path readme = readme_path
@ -220,7 +245,10 @@ def get_repository_url(directory):
if readme: if readme:
with open(readme, "r") as fp: with open(readme, "r") as fp:
for line in fp.readlines(): for line in fp.readlines():
if m := re.match("\s+:target:\s+(http\S+(docs.circuitpython|readthedocs)\S+)\s*", line): if m := re.match(
"\s+:target:\s+(http\S+(docs.circuitpython|readthedocs)\S+)\s*",
line,
):
path = m.group(1) path = m.group(1)
break break
if m := re.search("<(http[^>]+)>", line): if m := re.search("<(http[^>]+)>", line):
@ -233,12 +261,13 @@ def get_repository_url(directory):
errors="replace", errors="replace",
stdout=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, stderr=subprocess.PIPE,
cwd=directory cwd=directory,
) )
path = contents.stdout.strip() path = contents.stdout.strip()
repository_urls[directory] = path repository_urls[directory] = path
return path return path
def frozen_modules_from_dirs(frozen_mpy_dirs, withurl): def frozen_modules_from_dirs(frozen_mpy_dirs, withurl):
""" """
Go through the list of frozen directories and extract the python modules. Go through the list of frozen directories and extract the python modules.
@ -268,26 +297,28 @@ def frozen_modules_from_dirs(frozen_mpy_dirs, withurl):
frozen_modules.append(sub.name) frozen_modules.append(sub.name)
return frozen_modules return frozen_modules
def lookup_setting(settings, key, default=''):
def lookup_setting(settings, key, default=""):
while True: while True:
value = settings.get(key, default) value = settings.get(key, default)
if not value.startswith('$'): if not value.startswith("$"):
break break
key = value[2:-1] key = value[2:-1]
return value return value
@functools.cache @functools.cache
def all_ports_all_boards(ports=SUPPORTED_PORTS): def all_ports_all_boards(ports=SUPPORTED_PORTS):
for port in ports: for port in ports:
port_dir = get_circuitpython_root_dir() / "ports" / port port_dir = get_circuitpython_root_dir() / "ports" / port
for entry in (port_dir / "boards").iterdir(): for entry in (port_dir / "boards").iterdir():
if not entry.is_dir(): if not entry.is_dir():
continue continue
yield (port, entry) yield (port, entry)
def support_matrix_by_board(use_branded_name=True, withurl=True): def support_matrix_by_board(use_branded_name=True, withurl=True):
""" Compiles a list of the available core modules available for each """Compiles a list of the available core modules available for each
board. board.
""" """
base = build_module_map() base = build_module_map()
@ -300,8 +331,9 @@ def support_matrix_by_board(use_branded_name=True, withurl=True):
if use_branded_name: if use_branded_name:
with open(entry / "mpconfigboard.h") as get_name: with open(entry / "mpconfigboard.h") as get_name:
board_contents = get_name.read() board_contents = get_name.read()
board_name_re = re.search(r"(?<=MICROPY_HW_BOARD_NAME)\s+(.+)", board_name_re = re.search(
board_contents) r"(?<=MICROPY_HW_BOARD_NAME)\s+(.+)", board_contents
)
if board_name_re: if board_name_re:
board_name = board_name_re.group(1).strip('"') board_name = board_name_re.group(1).strip('"')
else: else:
@ -309,56 +341,69 @@ def support_matrix_by_board(use_branded_name=True, withurl=True):
board_modules = [] board_modules = []
for module in base: for module in base:
key = base[module]['key'] key = base[module]["key"]
if int(lookup_setting(settings, key, '0')): if int(lookup_setting(settings, key, "0")):
board_modules.append(base[module]['name']) board_modules.append(base[module]["name"])
board_modules.sort() board_modules.sort()
if "CIRCUITPY_BUILD_EXTENSIONS" in settings: if "CIRCUITPY_BUILD_EXTENSIONS" in settings:
board_extensions = [ board_extensions = [
extension.strip() for extension in extension.strip()
settings["CIRCUITPY_BUILD_EXTENSIONS"].split(",") for extension in settings["CIRCUITPY_BUILD_EXTENSIONS"].split(",")
] ]
else: else:
raise OSError(f"Board extensions undefined: {board_name}.") raise OSError(f"Board extensions undefined: {board_name}.")
frozen_modules = [] frozen_modules = []
if "FROZEN_MPY_DIRS" in settings: if "FROZEN_MPY_DIRS" in settings:
frozen_modules = frozen_modules_from_dirs(settings["FROZEN_MPY_DIRS"], withurl) frozen_modules = frozen_modules_from_dirs(
settings["FROZEN_MPY_DIRS"], withurl
)
if frozen_modules: if frozen_modules:
frozen_modules.sort() frozen_modules.sort()
# generate alias boards too # generate alias boards too
board_matrix = [( board_matrix = [
board_name, { (
board_name,
{
"modules": board_modules, "modules": board_modules,
"frozen_libraries": frozen_modules, "frozen_libraries": frozen_modules,
"extensions": board_extensions, "extensions": board_extensions,
} },
)] )
]
if entry.name in ALIASES_BY_BOARD: if entry.name in ALIASES_BY_BOARD:
for alias in ALIASES_BY_BOARD[entry.name]: for alias in ALIASES_BY_BOARD[entry.name]:
if use_branded_name: if use_branded_name:
if alias in ALIASES_BRAND_NAMES: if alias in ALIASES_BRAND_NAMES:
alias = ALIASES_BRAND_NAMES[alias] alias = ALIASES_BRAND_NAMES[alias]
else: else:
alias = alias.replace("_"," ").title() alias = alias.replace("_", " ").title()
board_matrix.append(( board_matrix.append(
alias, { (
alias,
{
"modules": board_modules, "modules": board_modules,
"frozen_libraries": frozen_modules, "frozen_libraries": frozen_modules,
"extensions": board_extensions, "extensions": board_extensions,
}, },
)) )
)
return board_matrix # this is now a list of (board,modules) return board_matrix # this is now a list of (board,modules)
executor = ThreadPoolExecutor(max_workers=os.cpu_count()) executor = ThreadPoolExecutor(max_workers=os.cpu_count())
mapped_exec = executor.map(support_matrix, all_ports_all_boards()) mapped_exec = executor.map(support_matrix, all_ports_all_boards())
# flatmap with comprehensions # flatmap with comprehensions
boards = dict(sorted([board for matrix in mapped_exec for board in matrix], key=lambda x: x[0])) boards = dict(
sorted(
[board for matrix in mapped_exec for board in matrix], key=lambda x: x[0]
)
)
return boards return boards
if __name__ == '__main__':
if __name__ == "__main__":
print(json.dumps(support_matrix_by_board(), indent=2)) print(json.dumps(support_matrix_by_board(), indent=2))

13
tools/build_board_info.py
View File

@ -6,6 +6,7 @@
import json import json
import os import os
import requests
import subprocess import subprocess
import sys import sys
import sh import sh
@ -96,7 +97,19 @@ def get_current_info():
response = response.json() response = response.json()
git_info = commit_sha, response["sha"] git_info = commit_sha, response["sha"]
if response["content"] != "":
# if the file is there
current_list = json.loads(base64.b64decode(response["content"]).decode("utf-8")) current_list = json.loads(base64.b64decode(response["content"]).decode("utf-8"))
else:
# if too big, the file is not included
download_url = response["download_url"]
response = requests.get(download_url)
if not response.ok:
print(response.text)
raise RuntimeError("cannot get previous files.json")
current_list = response.json()
current_info = {} current_info = {}
for info in current_list: for info in current_list:
current_info[info["id"]] = info current_info[info["id"]] = info