djsundog/circuitpython
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Selectively build boards based on changed files

Browse Source 
This uses a step output from the test job to set the build matrix
for the board build jobs. The built boards depends on which files
were changed.

* Changes contained within ports/*/boards/ will build each board.
* Changes contained within ports/* will build all boards from the
  port.
* All other file changes will build all boards.

All boards will be build for pushes in `adafruit/circuitpython` as
well.

A side-effect is that we no longer need to explicitly list the
boards to build. It will automatically update as new folders are
added.

Related to #4009
This commit is contained in:
Scott Shawcroft 2021-09-07 14:59:45 -07:00
parent daad0d0a86
commit a874043e1b
No known key found for this signature in database
GPG Key ID: 0DFD512649C052DA
3 changed files with 124 additions and 296 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

269
.github/workflows/build.yml vendored
View File

@ -15,6 +15,10 @@ on:
jobs:
test:
runs-on: ubuntu-20.04
outputs:
arm-boards: ${{ steps.set-matrix.outputs.arm-boards }}
riscv-boards: ${{ steps.set-matrix.outputs.riscv-boards }}
xtensa-boards: ${{ steps.set-matrix.outputs.xtensa-boards }}
steps:
- name: Dump GitHub context
env:
@ -43,9 +47,6 @@ jobs:
run: |
gcc --version
python3 --version
- name: New boards check
run: python3 -u ci_new_boards_check.py
working-directory: tools
- name: Duplicate USB VID/PID Check
run: python3 -u -m tools.ci_check_duplicate_usb_vid_pid
- name: Build and Validate Stubs
@ -139,6 +140,29 @@ jobs:
# setup.py sdist was run by 'make stubs'
[ -z "$TWINE_USERNAME" ] || echo "Uploading dev release to PyPi"
[ -z "$TWINE_USERNAME" ] || twine upload circuitpython-stubs/dist/*
- uses: dorny/paths-filter@v2
id: filter
with:
# Enable listing of files matching each filter.
# Paths to files will be available in `${FILTER_NAME}_files` output variable.
# Paths will be formatted as JSON array
list-files: json
# Compare against this branch. (Ignored for PRs.)
base: ${{ github.ref }}
# In this example all changed files are passed to the following action to do
# some custom processing.
filters: |
changed:
- '**'
- name: "Set boards to build"
id: set-matrix
working-directory: tools
env:
CHANGED_FILES: ${{ steps.filter.outputs.changed_files }}
run: |
python3 -u ci_changed_board_list.py
mpy-cross-mac:
runs-on: macos-10.15
@ -203,205 +227,7 @@ jobs:
strategy:
fail-fast: false
matrix:
board:
- "8086_commander"
- "ADM_B_NRF52840_1"
- "TG-Watch"
- "adafruit_feather_rp2040"
- "adafruit_itsybitsy_rp2040"
- "adafruit_led_glasses_nrf52840"
- "adafruit_macropad_rp2040"
- "adafruit_neokey_trinkey_m0"
- "adafruit_proxlight_trinkey_m0"
- "adafruit_qt2040_trinkey"
- "adafruit_qtpy_rp2040"
- "adafruit_rotary_trinkey_m0"
- "adafruit_slide_trinkey_m0"
- "aloriumtech_evo_m51"
- "aramcon2_badge"
- "aramcon_badge_2019"
- "arduino_mkr1300"
- "arduino_mkrzero"
- "arduino_nano_33_ble"
- "arduino_nano_33_iot"
- "arduino_nano_rp2040_connect"
- "arduino_zero"
- "bast_pro_mini_m0"
- "bastble"
- "bdmicro_vina_d21"
- "bdmicro_vina_d51"
- "bdmicro_vina_d51_pcb7"
- "bless_dev_board_multi_sensor"
- "blm_badge"
- "bluemicro840"
- "capablerobot_usbhub"
- "catwan_usbstick"
- "circuitbrains_basic_m0"
- "circuitbrains_deluxe_m4"
- "circuitplayground_bluefruit"
- "circuitplayground_express"
- "circuitplayground_express_crickit"
- "circuitplayground_express_displayio"
- "clue_nrf52840_express"
- "cp32-m4"
- "cp_sapling_m0"
- "cp_sapling_m0_revb"
- "cp_sapling_m0_spiflash"
- "cytron_maker_pi_rp2040"
- "datalore_ip_m4"
- "datum_distance"
- "datum_imu"
- "datum_light"
- "datum_weather"
- "dynalora_usb"
- "dynossat_edu_eps"
- "dynossat_edu_obc"
- "electronut_labs_blip"
- "electronut_labs_papyr"
- "escornabot_makech"
- "espruino_pico"
- "espruino_wifi"
- "feather_bluefruit_sense"
- "feather_m0_adalogger"
- "feather_m0_basic"
- "feather_m0_express"
- "feather_m0_express_crickit"
- "feather_m0_rfm69"
- "feather_m0_rfm9x"
- "feather_m0_supersized"
- "feather_m4_can"
- "feather_m4_express"
- "feather_m7_1011"
- "feather_mimxrt1011"
- "feather_mimxrt1062"
- "feather_nrf52840_express"
- "feather_stm32f405_express"
- "fluff_m0"
- "gemma_m0"
- "grandcentral_m4_express"
- "hallowing_m0_express"
- "hallowing_m4_express"
- "hiibot_bluefi"
- "huntercat_nfc"
- "ikigaisense_vita"
- "imxrt1010_evk"
- "imxrt1020_evk"
- "imxrt1060_evk"
- "itsybitsy_m0_express"
- "itsybitsy_m4_express"
- "itsybitsy_nrf52840_express"
- "jpconstantineau_encoderpad_rp2040"
- "kicksat-sprite"
- "loc_ber_m4_base_board"
- "makerdiary_m60_keyboard"
- "makerdiary_nrf52840_m2_devkit"
- "makerdiary_nrf52840_mdk"
- "makerdiary_nrf52840_mdk_usb_dongle"
- "matrixportal_m4"
- "meowbit_v121"
- "meowmeow"
- "metro_m0_express"
- "metro_m4_airlift_lite"
- "metro_m4_express"
- "metro_m7_1011"
- "metro_nrf52840_express"
- "microbit_v2"
- "mini_sam_m4"
- "monster_m4sk"
- "ndgarage_ndbit6"
- "ndgarage_ndbit6_v2"
- "neopixel_trinkey_m0"
- "nfc_copy_cat"
- "nice_nano"
- "nucleo_f746zg"
- "nucleo_f767zi"
- "nucleo_h743zi_2"
- "ohs2020_badge"
- "openbook_m4"
- "openmv_h7"
- "particle_argon"
- "particle_boron"
- "particle_xenon"
- "pca10056"
- "pca10059"
- "pca10100"
- "pewpew10"
- "pewpew_m4"
- "picoplanet"
- "pimoroni_interstate75"
- "pimoroni_keybow2040"
- "pimoroni_pga2040"
- "pimoroni_picolipo_16mb"
- "pimoroni_picolipo_4mb"
- "pimoroni_picosystem"
- "pimoroni_plasma2040"
- "pimoroni_tiny2040"
- "pitaya_go"
- "pyb_nano_v2"
- "pybadge"
- "pyboard_v11"
- "pycubed"
- "pycubed_mram"
- "pygamer"
- "pyportal"
- "pyportal_titano"
- "pyruler"
- "qtpy_m0"
- "qtpy_m0_haxpress"
- "raspberry_pi_pico"
- "raytac_mdbt50q-db-40"
- "raytac_mdbt50q-rx"
- "robohatmm1_m4"
- "sam32"
- "same54_xplained"
- "seeeduino_wio_terminal"
- "seeeduino_xiao"
- "sensebox_mcu"
- "serpente"
- "shirtty"
- "silicognition-m4-shim"
- "simmel"
- "snekboard"
- "sparkfun_lumidrive"
- "sparkfun_micromod_rp2040"
- "sparkfun_nrf52840_micromod"
- "sparkfun_nrf52840_mini"
- "sparkfun_pro_micro_rp2040"
- "sparkfun_qwiic_micro_no_flash"
- "sparkfun_qwiic_micro_with_flash"
- "sparkfun_redboard_turbo"
- "sparkfun_samd21_dev"
- "sparkfun_samd21_mini"
- "sparkfun_samd51_micromod"
- "sparkfun_samd51_thing_plus"
- "sparkfun_stm32f405_micromod"
- "sparkfun_thing_plus_rp2040"
- "spresense"
- "stackrduino_m0_pro"
- "stm32f411ce_blackpill"
- "stm32f411ce_blackpill_with_flash"
- "stm32f411ve_discovery"
- "stm32f412zg_discovery"
- "stm32f4_discovery"
- "stm32f746g_discovery"
- "stringcar_m0_express"
- "teensy40"
- "teensy41"
- "teknikio_bluebird"
- "thunderpack_v11"
- "thunderpack_v12"
- "tinkeringtech_scoutmakes_azul"
- "trellis_m4_express"
- "trinket_m0"
- "trinket_m0_haxpress"
- "uartlogger2"
- "uchip"
- "ugame10"
- "warmbit_bluepixel"
- "winterbloom_big_honking_button"
- "winterbloom_sol"
- "xinabox_cc03"
- "xinabox_cs11"
board: ${{ fromJSON(needs.test.outputs.arm-boards) }}
steps:
- name: Set up Python 3.8
@ -451,8 +277,7 @@ jobs:
strategy:
fail-fast: false
matrix:
board:
- "fomu"
board: ${{ fromJSON(needs.test.outputs.riscv-boards) }}
steps:
- name: Set up Python 3.8
@ -501,41 +326,7 @@ jobs:
strategy:
fail-fast: false
matrix:
board:
- "adafruit_feather_esp32s2_nopsram"
- "adafruit_feather_esp32s2_tftback_nopsram"
- "adafruit_funhouse"
- "adafruit_magtag_2.9_grayscale"
- "adafruit_metro_esp32s2"
- "ai_thinker_esp_12k_nodemcu"
- "artisense_rd00"
- "atmegazero_esp32s2"
- "crumpspace_crumps2"
- "electroniccats_bastwifi"
- "espressif_hmi_devkit_1"
- "espressif_kaluga_1"
- "espressif_kaluga_1.3"
- "espressif_saola_1_wroom"
- "espressif_saola_1_wrover"
- "franzininho_wifi_wroom"
- "franzininho_wifi_wrover"
- "gravitech_cucumber_m"
- "gravitech_cucumber_ms"
- "gravitech_cucumber_r"
- "gravitech_cucumber_rs"
- "lilygo_ttgo_t8_s2_st7789"
- "lolin_s2_mini"
- "microdev_micro_s2"
- "morpheans_morphesp-240"
- "muselab_nanoesp32_s2_wroom"
- "muselab_nanoesp32_s2_wrover"
- "odt_pixelwing_esp32_s2"
- "targett_module_clip_wroom"
- "targett_module_clip_wrover"
- "unexpectedmaker_feathers2"
- "unexpectedmaker_feathers2_neo"
- "unexpectedmaker_feathers2_prerelease"
- "unexpectedmaker_tinys2"
board: ${{ fromJSON(needs.test.outputs.xtensa-boards) }}
steps:
- name: Set up Python 3.8

94
tools/ci_changed_board_list.py Normal file
View File

@ -0,0 +1,94 @@
#! /usr/bin/env python3
"""This script is used in GitHub Actions to determine what boards are built
based on what files were changed. The base commit varies depending on the
event that triggered run. Pull request runs will compare to the base branch
while pushes will compare to the current ref. We override this for the
adafruit/circuitpython repo so we build all boards for pushes.
"""
# SPDX-FileCopyrightText: 2021 Scott Shawcroft
#
# SPDX-License-Identifier: MIT
import sys
import os
import json
import yaml
import re
import build_board_info
PORT_TO_ARCH = {
"atmel-samd": "arm",
"cxd56": "arm",
"esp32s2": "xtensa",
"litex": "riscv",
"mimxrt10xx": "arm",
"nrf": "arm",
"raspberrypi": "arm",
"stm": "arm",
}
changed_files = json.loads(os.environ["CHANGED_FILES"])
print("changed_files")
print(changed_files)
# Get boards in json format
boards_info_json = build_board_info.get_board_mapping()
all_board_ids = set()
port_to_boards = {}
board_to_port = {}
for board_id in boards_info_json:
info = boards_info_json[board_id]
if info.get("alias", False):
continue
all_board_ids.add(board_id)
port = info["port"]
if port not in port_to_boards:
port_to_boards[port] = set()
port_to_boards[port].add(board_id)
board_to_port[board_id] = port
# Build all boards if this is a push to an adafruit/circuitpython branch because we save the artifacts.
boards_to_build = all_board_ids
if not changed_files or (
os.environ.get("GITHUB_EVENT_NAME", "") == "push"
and os.environ.get("GITHUB_REPOSITORY", "") == "adafruit/circuitpython"
):
print("Building all boards because of adafruit/circuitpython branch")
else:
print("Adding boards to build based on changed files")
boards_to_build = set()
board_pattern = re.compile(r"ports/\w+/boards/(\w+)/")
port_pattern = re.compile(r"ports/(\w+)/")
for p in changed_files:
# See if it is board specific
board_matches = board_pattern.search(p)
if board_matches:
board = board_matches.group(1)
boards_to_build.add(board)
continue
# See if it is port specific
port_matches = port_pattern.search(p)
if port_matches:
port = port_matches.group(1)
boards_to_build.update(port_to_boards[port])
continue
# Otherwise build it all
boards_to_build = all_board_ids
break
# Split boards by architecture.
print("Building boards:")
arch_to_boards = {"arm": [], "riscv": [], "xtensa": []}
for board in boards_to_build:
print(" ", board)
arch = PORT_TO_ARCH[board_to_port[board]]
arch_to_boards[arch].append(board)
# Set the step outputs for each architecture
for arch in arch_to_boards:
print("::set-output name=" + arch + "-boards::" + json.dumps(sorted(arch_to_boards[arch])))

57
tools/ci_new_boards_check.py
View File

@ -1,57 +0,0 @@
#! /usr/bin/env python3
# SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors)
#
# SPDX-License-Identifier: MIT
import sys
import os
import json
import yaml
import build_board_info
workflow_file = ".github/workflows/build.yml"
# Get boards in json format
boards_info_json = build_board_info.get_board_mapping()
# Get all the boards out of the json format
info_boards = [
board for board in boards_info_json.keys() if not boards_info_json[board].get("alias", False)
]
# We need to know the path of the workflow file
base_path = os.path.dirname(__file__)
yml_path = os.path.abspath(os.path.join(base_path, "..", workflow_file))
# Loading board list based on build jobs in the workflow file.
ci_boards = []
with open(yml_path, "r") as f:
workflow = yaml.safe_load(f)
ok = True
for job in workflow["jobs"]:
if not job.startswith("build"):
continue
job_boards = workflow["jobs"][job]["strategy"]["matrix"]["board"]
if job_boards != sorted(job_boards):
print('Boards for job "{}" not sorted. Must be:'.format(job))
print(' - "' + '"\n - "'.join(sorted(job_boards)) + '"')
ok = False
ci_boards.extend(job_boards)
# All the travis_boards elements must be on info_boards
info_boards.sort()
ci_boards.sort()
missing_boards = set(info_boards) - set(ci_boards)
if missing_boards:
ok = False
print("Boards missing in {}:".format(workflow_file))
for board in missing_boards:
print(board)
if not ok:
sys.exit(1)