From 81154b4ee6f611f4b255f9756b45d17a5438d07d Mon Sep 17 00:00:00 2001 From: Jeff Epler Date: Fri, 14 Oct 2022 11:32:58 -0500 Subject: [PATCH 1/7] Revert "don't build other ports due to common-hal changes" This reverts commit 91985cef7e9666d1f60ef6cef7f66c74896744d0. --- tools/ci_set_matrix.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/ci_set_matrix.py b/tools/ci_set_matrix.py index e165c23c27..0ae6010ac9 100644 --- a/tools/ci_set_matrix.py +++ b/tools/ci_set_matrix.py @@ -53,7 +53,7 @@ def set_output(name, value): with open(os.environ["GITHUB_OUTPUT"], "at") as f: print(f"{name}={value}", file=f) else: - print(f"Would set GitHub actions output {name} to '{value}'") + print("Would set GitHub actions output {name} to '{value}'") def set_boards_to_build(build_all): @@ -80,7 +80,9 @@ def set_boards_to_build(build_all): boards_to_build = set() board_pattern = re.compile(r"^ports/[^/]+/boards/([^/]+)/") port_pattern = re.compile(r"^ports/([^/]+)/") - module_pattern = re.compile(r"^(ports/[^/]+/shared-bindings|shared-module)/([^/]+)/") + module_pattern = re.compile( + r"^(ports/[^/]+/common-hal|shared-bindings|shared-module)/([^/]+)/" + ) for p in changed_files: # See if it is board specific board_matches = board_pattern.search(p) From cab40630573dc9444590782f88471cb8cb0625a4 Mon Sep 17 00:00:00 2001 From: Jeff Epler Date: Fri, 14 Oct 2022 09:23:35 -0500 Subject: [PATCH 2/7] Make it easier to locally test ci_set_matrix Now you can e.g., `tools/ci_set_matrix.py ports/raspberrypi/mpconfigport.h` and see what outputs would be set. --- docs/shared_bindings_matrix.py | 7 +++---- tools/ci_set_matrix.py | 28 ++++++++++++++++++++-------- 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/docs/shared_bindings_matrix.py b/docs/shared_bindings_matrix.py index 5b9b63d8fd..196415ca2f 100644 --- a/docs/shared_bindings_matrix.py +++ b/docs/shared_bindings_matrix.py @@ -80,12 +80,11 @@ This is the same list as in the preprocess_frozen_modules script.""" repository_urls = {} """Cache of repository URLs for frozen modules.""" +root_dir = pathlib.Path(__file__).resolve().parent.parent + def get_circuitpython_root_dir(): """ The path to the root './circuitpython' directory. """ - file_path = pathlib.Path(__file__).resolve() - root_dir = file_path.parent.parent - return root_dir def get_shared_bindings(): @@ -102,7 +101,7 @@ def get_board_mapping(): """ boards = {} for port in SUPPORTED_PORTS: - board_path = os.path.join("../ports", port, "boards") + board_path = root_dir / "ports" / port / "boards" for board_path in os.scandir(board_path): if board_path.is_dir(): board_files = os.listdir(board_path.path) diff --git a/tools/ci_set_matrix.py b/tools/ci_set_matrix.py index 0ae6010ac9..198aca899b 100644 --- a/tools/ci_set_matrix.py +++ b/tools/ci_set_matrix.py @@ -19,6 +19,13 @@ import os import sys import json import yaml +import pathlib + +tools_dir = pathlib.Path(__file__).resolve().parent +top_dir = tools_dir.parent + +sys.path.insert(0, str(tools_dir / "adabot")) +sys.path.insert(0, str(top_dir / "docs")) import build_board_info from shared_bindings_matrix import get_settings_from_makefile @@ -40,12 +47,17 @@ IGNORE = [ "tools/ci_check_duplicate_usb_vid_pid.py", ] -changed_files = {} -try: - changed_files = json.loads(os.environ["CHANGED_FILES"]) -except json.decoder.JSONDecodeError as exc: - if exc.msg != "Expecting value": - raise +if len(sys.argv) > 1: + print("Using files list on commandline") + changed_files = sys.argv[1:] +else: + c = os.environ["CHANGED_FILES"] + if c == "": + print("CHANGED_FILES is in environment, but value is empty") + changed_files = [] + else: + print("Using files list in CHANGED_FILES") + changed_files = json.loads(os.environ["CHANGED_FILES"]) def set_output(name, value): @@ -53,7 +65,7 @@ def set_output(name, value): with open(os.environ["GITHUB_OUTPUT"], "at") as f: print(f"{name}={value}", file=f) else: - print("Would set GitHub actions output {name} to '{value}'") + print(f"Would set GitHub actions output {name} to '{value}'") def set_boards_to_build(build_all): @@ -114,7 +126,7 @@ def set_boards_to_build(build_all): for board in all_board_ids: if board not in board_settings: board_settings[board] = get_settings_from_makefile( - "../ports/" + board_to_port[board], board + str(top_dir / "ports" / board_to_port[board]), board ) settings = board_settings[board] From 25164030e228b36f47a6caabdd084c417c061c80 Mon Sep 17 00:00:00 2001 From: Jeff Epler Date: Fri, 14 Oct 2022 11:06:35 -0500 Subject: [PATCH 3/7] Don't recompute 'all_ports_all_boards' This looks modestly expensive, and it's trivial to cache it. --- docs/shared_bindings_matrix.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/shared_bindings_matrix.py b/docs/shared_bindings_matrix.py index 196415ca2f..761e3e29f2 100644 --- a/docs/shared_bindings_matrix.py +++ b/docs/shared_bindings_matrix.py @@ -27,6 +27,7 @@ import pathlib import re import subprocess import sys +import functools from concurrent.futures import ThreadPoolExecutor @@ -275,6 +276,7 @@ def lookup_setting(settings, key, default=''): key = value[2:-1] return value +@functools.cache def all_ports_all_boards(ports=SUPPORTED_PORTS): for port in ports: From ad130e87f0bb45cf3ab2d4d1a55fcbf66f9de8c2 Mon Sep 17 00:00:00 2001 From: Jeff Epler Date: Fri, 14 Oct 2022 11:07:46 -0500 Subject: [PATCH 4/7] Parallelize finding board settings This reduces the _elapsed_ time running the script from ~90s to ~15s on my AMD Ryzen 7 5700U. The CPU time is still around 2 minutes. --- tools/ci_set_matrix.py | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/tools/ci_set_matrix.py b/tools/ci_set_matrix.py index 198aca899b..7cdc57cd52 100644 --- a/tools/ci_set_matrix.py +++ b/tools/ci_set_matrix.py @@ -20,6 +20,7 @@ import sys import json import yaml import pathlib +from concurrent.futures import ThreadPoolExecutor tools_dir = pathlib.Path(__file__).resolve().parent top_dir = tools_dir.parent @@ -28,7 +29,11 @@ sys.path.insert(0, str(tools_dir / "adabot")) sys.path.insert(0, str(top_dir / "docs")) import build_board_info -from shared_bindings_matrix import get_settings_from_makefile +from shared_bindings_matrix import ( + get_settings_from_makefile, + SUPPORTED_PORTS, + all_ports_all_boards, +) PORT_TO_ARCH = { "atmel-samd": "arm", @@ -86,6 +91,20 @@ def set_boards_to_build(build_all): port_to_boards[port].add(board_id) board_to_port[board_id] = port + def compute_board_settings(): + if board_settings: + return + + def get_settings(arg): + board = arg[1].name + return ( + board, + get_settings_from_makefile(str(top_dir / "ports" / board_to_port[board]), board), + ) + + with ThreadPoolExecutor(max_workers=os.cpu_count()) as ex: + board_settings.update(ex.map(get_settings, all_ports_all_boards())) + boards_to_build = all_board_ids if not build_all: @@ -123,11 +142,8 @@ def set_boards_to_build(build_all): # As a (nearly) last resort, for some certain files, we compute the settings from the # makefile for each board and determine whether to build them that way. if p.startswith("frozen") or p.startswith("supervisor") or module_matches: + compute_board_settings() for board in all_board_ids: - if board not in board_settings: - board_settings[board] = get_settings_from_makefile( - str(top_dir / "ports" / board_to_port[board]), board - ) settings = board_settings[board] # Check frozen files to see if they are in each board. From 3b600ac9b38fe0d6c67fadb4a5bd2f9cac3e53b9 Mon Sep 17 00:00:00 2001 From: Jeff Epler Date: Fri, 14 Oct 2022 11:19:54 -0500 Subject: [PATCH 5/7] Potentially compute settings of fewer boards .. when the file is within ports, just get the settings for boards within the port --- tools/ci_set_matrix.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) mode change 100644 => 100755 tools/ci_set_matrix.py diff --git a/tools/ci_set_matrix.py b/tools/ci_set_matrix.py old mode 100644 new mode 100755 index 7cdc57cd52..e65f3ed8b2 --- a/tools/ci_set_matrix.py +++ b/tools/ci_set_matrix.py @@ -91,19 +91,19 @@ def set_boards_to_build(build_all): port_to_boards[port].add(board_id) board_to_port[board_id] = port - def compute_board_settings(): - if board_settings: + def compute_board_settings(boards): + need = set(boards) - set(board_settings.keys()) + if not need: return - def get_settings(arg): - board = arg[1].name + def get_settings(board): return ( board, get_settings_from_makefile(str(top_dir / "ports" / board_to_port[board]), board), ) with ThreadPoolExecutor(max_workers=os.cpu_count()) as ex: - board_settings.update(ex.map(get_settings, all_ports_all_boards())) + board_settings.update(ex.map(get_settings, need)) boards_to_build = all_board_ids @@ -142,8 +142,13 @@ def set_boards_to_build(build_all): # As a (nearly) last resort, for some certain files, we compute the settings from the # makefile for each board and determine whether to build them that way. if p.startswith("frozen") or p.startswith("supervisor") or module_matches: - compute_board_settings() - for board in all_board_ids: + if port_matches: + port = port_matches.group(1) + board_ids = port_to_boards[port] + else: + board_ids = all_board_ids + compute_board_settings(board_ids) + for board in board_ids: settings = board_settings[board] # Check frozen files to see if they are in each board. From 641a398a542fcc9047ad3fe462919769d2f3b095 Mon Sep 17 00:00:00 2001 From: Jeff Epler Date: Fri, 14 Oct 2022 11:23:47 -0500 Subject: [PATCH 6/7] only build subset of boards for bindings changes --- tools/ci_set_matrix.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tools/ci_set_matrix.py b/tools/ci_set_matrix.py index e65f3ed8b2..30d31ec37d 100755 --- a/tools/ci_set_matrix.py +++ b/tools/ci_set_matrix.py @@ -12,6 +12,13 @@ on the event that triggered run. Pull request runs will compare to the base branch while pushes will compare to the current ref. We override this for the adafruit/circuitpython repo so we build all docs/boards for pushes. +When making changes to the script it is useful to manually test it. +You can for instance run +```shell +tools/ci_set_matrix ports/raspberrypi/common-hal/socket/SSLSocket.c +``` +and (at the time this comment was written) get a series of messages indicating +that only the single board raspberry_pi_pico_w would be built. """ import re @@ -112,7 +119,7 @@ def set_boards_to_build(build_all): board_pattern = re.compile(r"^ports/[^/]+/boards/([^/]+)/") port_pattern = re.compile(r"^ports/([^/]+)/") module_pattern = re.compile( - r"^(ports/[^/]+/common-hal|shared-bindings|shared-module)/([^/]+)/" + r"^(ports/[^/]+/(?:common-hal|bindings)|shared-bindings|shared-module)/([^/]+)/" ) for p in changed_files: # See if it is board specific From d08b43f704e2ef6c373a37f7b7ef30beb498956d Mon Sep 17 00:00:00 2001 From: Jeff Epler Date: Fri, 14 Oct 2022 11:28:29 -0500 Subject: [PATCH 7/7] We refer to port multiple times, make it convenient --- tools/ci_set_matrix.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tools/ci_set_matrix.py b/tools/ci_set_matrix.py index 30d31ec37d..f22840fad7 100755 --- a/tools/ci_set_matrix.py +++ b/tools/ci_set_matrix.py @@ -131,9 +131,9 @@ def set_boards_to_build(build_all): # See if it is port specific port_matches = port_pattern.search(p) + port = port_matches.group(1) if port_matches else None module_matches = module_pattern.search(p) - if port_matches and not module_matches: - port = port_matches.group(1) + if port and not module_matches: if port != "unix": boards_to_build.update(port_to_boards[port]) continue @@ -149,8 +149,7 @@ def set_boards_to_build(build_all): # As a (nearly) last resort, for some certain files, we compute the settings from the # makefile for each board and determine whether to build them that way. if p.startswith("frozen") or p.startswith("supervisor") or module_matches: - if port_matches: - port = port_matches.group(1) + if port: board_ids = port_to_boards[port] else: board_ids = all_board_ids