djsundog
/
circuitpython
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #7676 from microdev1/ci 

CI: Enhancements & Refactoring
This commit is contained in:
Scott Shawcroft 2023-03-08 10:36:05 -08:00 committed by GitHub
parent e346b731ca f0b509cd0d
commit 35e82a5f5e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
22 changed files with 158 additions and 190 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
.github/actions/deps/external/action.yml vendored
View File

@ -9,54 +9,31 @@ inputs:
- cache
- restore
platform:
port:
required: false
default: none
type: choice
options:
- arm
- aarch
- esp
- riscv
- none
type: string
runs:
using: composite
steps:
# aarch
- name: Get aarch toolchain
if: inputs.platform == 'aarch'
run: |
wget --no-verbose https://adafruit-circuit-python.s3.amazonaws.com/gcc-arm-10.3-2021.07-x86_64-aarch64-none-elf.tar.xz
sudo tar -C /usr --strip-components=1 -xaf gcc-arm-10.3-2021.07-x86_64-aarch64-none-elf.tar.xz
sudo apt-get install -y mtools
shell: bash
- name: Install mkfs.fat
if: inputs.platform == 'aarch'
run: |
wget https://github.com/dosfstools/dosfstools/releases/download/v4.2/dosfstools-4.2.tar.gz
tar -xaf dosfstools-4.2.tar.gz
cd dosfstools-4.2
./configure
make -j 2
cd src
echo >> $GITHUB_PATH $(pwd)
shell: bash
# arm
- name: Get arm toolchain
if: inputs.platform == 'aarch' || inputs.platform == 'arm'
if: >-
inputs.port != 'none' &&
inputs.port != 'litex' &&
inputs.port != 'espressif'
uses: carlosperate/arm-none-eabi-gcc-action@v1
with:
release: '10-2020-q4'
# esp
- name: Get esp toolchain
if: inputs.platform == 'esp'
# espressif
- name: Get espressif toolchain
if: inputs.port == 'espressif'
run: sudo apt-get install -y ninja-build
shell: bash
- name: Install IDF tools
if: inputs.platform == 'esp'
if: inputs.port == 'espressif'
run: |
echo "Installing ESP-IDF tools"
$IDF_PATH/tools/idf_tools.py --non-interactive install required
@ -66,24 +43,16 @@ runs:
rm -rf $IDF_TOOLS_PATH/dist
shell: bash
- name: Set environment
if: inputs.platform == 'esp'
if: inputs.port == 'espressif'
run: |
source $IDF_PATH/export.sh
echo >> $GITHUB_ENV "IDF_PYTHON_ENV_PATH=$IDF_PYTHON_ENV_PATH"
echo >> $GITHUB_PATH "$PATH"
shell: bash
# riscv
- name: Get riscv toolchain
if: inputs.platform == 'riscv'
run: |
wget https://static.dev.sifive.com/dev-tools/riscv64-unknown-elf-gcc-8.3.0-2019.08.0-x86_64-linux-centos6.tar.gz
sudo tar -C /usr --strip-components=1 -xaf riscv64-unknown-elf-gcc-8.3.0-2019.08.0-x86_64-linux-centos6.tar.gz
shell: bash
# common
- name: Cache python dependencies
if: inputs.platform != 'esp'
if: inputs.port != 'espressif'
uses: ./.github/actions/deps/python
with:
action: ${{ inputs.action }}

12
.github/actions/deps/ports/action.yml vendored
View File

@ -19,10 +19,18 @@ runs:
if [ -z $PORT ]; then (exit 1); else echo >> $GITHUB_OUTPUT "port=$PORT"; fi
shell: bash
- name: Set up espressif port
- name: Set up broadcom
if: steps.board-to-port.outputs.port == 'broadcom'
uses: ./.github/actions/deps/ports/broadcom
- name: Set up espressif
if: steps.board-to-port.outputs.port == 'espressif'
uses: ./.github/actions/deps/ports/espressif
- name: Set up nrf port
- name: Set up litex
if: steps.board-to-port.outputs.port == 'litex'
uses: ./.github/actions/deps/ports/litex
- name: Set up nrf
if: steps.board-to-port.outputs.port == 'nrf'
uses: ./.github/actions/deps/ports/nrf

21
.github/actions/deps/ports/broadcom/action.yml vendored Normal file
View File

@ -0,0 +1,21 @@
name: Fetch broadcom port deps
runs:
using: composite
steps:
- name: Get broadcom toolchain
run: |
wget --no-verbose https://adafruit-circuit-python.s3.amazonaws.com/gcc-arm-10.3-2021.07-x86_64-aarch64-none-elf.tar.xz
sudo tar -C /usr --strip-components=1 -xaf gcc-arm-10.3-2021.07-x86_64-aarch64-none-elf.tar.xz
sudo apt-get install -y mtools
shell: bash
- name: Install mkfs.fat
run: |
wget https://github.com/dosfstools/dosfstools/releases/download/v4.2/dosfstools-4.2.tar.gz
tar -xaf dosfstools-4.2.tar.gz
cd dosfstools-4.2
./configure
make -j 2
cd src
echo >> $GITHUB_PATH $(pwd)
shell: bash

10
.github/actions/deps/ports/litex/action.yml vendored Normal file
View File

@ -0,0 +1,10 @@
name: Fetch litex port deps
runs:
using: composite
steps:
- name: Get litex toolchain
run: |
wget https://static.dev.sifive.com/dev-tools/riscv64-unknown-elf-gcc-8.3.0-2019.08.0-x86_64-linux-centos6.tar.gz
sudo tar -C /usr --strip-components=1 -xaf riscv64-unknown-elf-gcc-8.3.0-2019.08.0-x86_64-linux-centos6.tar.gz
shell: bash

8
.github/workflows/build-boards.yml vendored
View File

@ -3,9 +3,6 @@ name: Build boards
on:
workflow_call:
inputs:
platform:
required: true
type: string
boards:
required: true
type: string
@ -19,7 +16,7 @@ on:
required: false
jobs:
build:
board:
runs-on: ubuntu-22.04
env:
CP_VERSION: ${{ inputs.cp-version }}
@ -38,6 +35,7 @@ jobs:
with:
python-version: 3.x
- name: Set up port
id: set-up-port
uses: ./.github/actions/deps/ports
with:
board: ${{ matrix.board }}
@ -47,7 +45,7 @@ jobs:
- name: Set up external
uses: ./.github/actions/deps/external
with:
platform: ${{ inputs.platform }}
port: ${{ steps.set-up-port.outputs.port }}
- name: Set up mpy-cross
if: steps.set-up-submodules.outputs.frozen == 'True'
uses: ./.github/actions/mpy_cross

76
.github/workflows/build.yml vendored
View File

@ -18,16 +18,11 @@ concurrency:
jobs:
scheduler:
runs-on: ubuntu-20.04
runs-on: ubuntu-22.04
outputs:
build-doc: ${{ steps.set-matrix.outputs.build-doc }}
build-boards: ${{ steps.set-matrix.outputs.build-boards }}
build-windows: ${{ steps.set-matrix.outputs.build-windows }}
boards-aarch: ${{ steps.set-matrix.outputs.boards-aarch }}
boards-arm: ${{ steps.set-matrix.outputs.boards-arm }}
boards-atmel: ${{ steps.set-matrix.outputs.boards-atmel }}
boards-esp: ${{ steps.set-matrix.outputs.boards-esp }}
boards-riscv: ${{ steps.set-matrix.outputs.boards-riscv }}
docs: ${{ steps.set-matrix.outputs.docs }}
ports: ${{ steps.set-matrix.outputs.ports }}
windows: ${{ steps.set-matrix.outputs.windows }}
cp-version: ${{ steps.set-up-submodules.outputs.version }}
steps:
- name: Dump GitHub context
@ -99,7 +94,7 @@ jobs:
mpy-cross:
needs: scheduler
if: needs.scheduler.outputs.build-boards == 'True'
if: needs.scheduler.outputs.ports != '{}'
uses: ./.github/workflows/build-mpy-cross.yml
secrets: inherit
with:
@ -108,7 +103,7 @@ jobs:
mpy-cross-mac:
runs-on: macos-11
needs: scheduler
if: needs.scheduler.outputs.build-boards == 'True'
if: needs.scheduler.outputs.ports != '{}'
env:
CP_VERSION: ${{ needs.scheduler.outputs.cp-version }}
steps:
@ -160,10 +155,10 @@ jobs:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
build-doc:
docs:
runs-on: ubuntu-22.04
needs: scheduler
if: needs.scheduler.outputs.build-doc == 'True'
if: needs.scheduler.outputs.docs == 'True'
env:
CP_VERSION: ${{ needs.scheduler.outputs.cp-version }}
steps:
@ -218,10 +213,10 @@ jobs:
[ -z "$TWINE_USERNAME" ] || echo "Uploading dev release to PyPi"
[ -z "$TWINE_USERNAME" ] || twine upload circuitpython-stubs/dist/*
build-windows:
windows:
runs-on: windows-2022
needs: scheduler
if: needs.scheduler.outputs.build-windows == 'True'
if: needs.scheduler.outputs.windows == 'True'
env:
CP_VERSION: ${{ needs.scheduler.outputs.cp-version }}
defaults:
@ -286,52 +281,15 @@ jobs:
# ERROR: Platform MINGW64_NT-10.0-17763-x86_64 appears to be unsupported
# https://github.com/espressif/esp-idf/issues/7062
aarch:
ports:
needs: [scheduler, mpy-cross, tests]
if: ${{ needs.scheduler.outputs.boards-aarch != '[]' }}
if: needs.scheduler.outputs.ports != '{}'
uses: ./.github/workflows/build-boards.yml
secrets: inherit
strategy:
fail-fast: false
matrix:
port: ${{ fromJSON(needs.scheduler.outputs.ports).ports }}
with:
platform: aarch
boards: ${{ needs.scheduler.outputs.boards-aarch }}
cp-version: ${{ needs.scheduler.outputs.cp-version }}
arm:
needs: [scheduler, mpy-cross, tests]
if: ${{ needs.scheduler.outputs.boards-arm != '[]' }}
uses: ./.github/workflows/build-boards.yml
secrets: inherit
with:
platform: arm
boards: ${{ needs.scheduler.outputs.boards-arm }}
cp-version: ${{ needs.scheduler.outputs.cp-version }}
atmel:
needs: [scheduler, mpy-cross, tests]
if: ${{ needs.scheduler.outputs.boards-atmel != '[]' }}
uses: ./.github/workflows/build-boards.yml
secrets: inherit
with:
platform: arm
boards: ${{ needs.scheduler.outputs.boards-atmel }}
cp-version: ${{ needs.scheduler.outputs.cp-version }}
esp:
needs: [scheduler, mpy-cross, tests]
if: ${{ needs.scheduler.outputs.boards-esp != '[]' }}
uses: ./.github/workflows/build-boards.yml
secrets: inherit
with:
platform: esp
boards: ${{ needs.scheduler.outputs.boards-esp }}
cp-version: ${{ needs.scheduler.outputs.cp-version }}
riscv:
needs: [scheduler, mpy-cross, tests]
if: ${{ needs.scheduler.outputs.boards-riscv != '[]' }}
uses: ./.github/workflows/build-boards.yml
secrets: inherit
with:
platform: riscv
boards: ${{ needs.scheduler.outputs.boards-riscv }}
boards: ${{ toJSON(fromJSON(needs.scheduler.outputs.ports)[matrix.port]) }}
cp-version: ${{ needs.scheduler.outputs.cp-version }}

0
.github/workflows/create_website_pr.yml → .github/workflows/create-website-pr.yml vendored
View File

12
.github/workflows/build-boards-custom.yml → .github/workflows/custom-board-build.yml vendored
View File

@ -32,16 +32,6 @@ run-name: ${{ inputs.board }}-${{ inputs.language }}-${{ inputs.version }}${{ in
jobs:
build:
runs-on: ubuntu-22.04
env:
PLATFORM_atmel-samd: arm
PLATFORM_broadcom: aarch
PLATFORM_cxd56: arm
PLATFORM_espressif: esp
PLATFORM_litex: riscv
PLATFORM_mimxrt10xx: arm
PLATFORM_nrf: arm
PLATFORM_raspberrypi: arm
PLATFORM_stm: arm
steps:
- name: Set up repository
run: |
@ -70,7 +60,7 @@ jobs:
uses: ./.github/actions/deps/external
with:
action: cache
platform: ${{ env[format('PLATFORM_{0}', steps.set-up-port.outputs.port)] }}
port: ${{ steps.set-up-port.outputs.port }}
- name: Set up mpy-cross
if: steps.set-up-submodules.outputs.frozen == 'True'
uses: ./.github/actions/mpy_cross

0
.github/workflows/notify.yml → .github/workflows/notify-on-issue-label.yml vendored
View File

4
.github/workflows/run-tests.yml vendored
View File

@ -9,7 +9,7 @@ on:
jobs:
run:
runs-on: ubuntu-20.04
runs-on: ubuntu-22.04
strategy:
fail-fast: false
matrix:
@ -31,7 +31,7 @@ jobs:
- name: Set up python
uses: actions/setup-python@v4
with:
python-version: 3.x
python-version: 3.8
- name: Set up submodules
uses: ./.github/actions/deps/submodules
with:

2
ports/atmel-samd/Makefile
View File

@ -367,7 +367,7 @@ $(BUILD)/firmware.elf: $(OBJ) $(GENERATED_LD_FILE)
$(STEPECHO) "LINK $@"
$(Q)echo $(OBJ) > $(BUILD)/firmware.objs
$(Q)$(CC) -o $@ $(LDFLAGS) @$(BUILD)/firmware.objs -Wl,--start-group $(LIBS) -Wl,--end-group
$(Q)$(SIZE) $@ | $(PYTHON) $(TOP)/tools/build_memory_info.py $(GENERATED_LD_FILE)
$(Q)$(SIZE) $@ | $(PYTHON) $(TOP)/tools/build_memory_info.py $(GENERATED_LD_FILE) $(BUILD)
$(BUILD)/firmware.bin: $(BUILD)/firmware.elf
$(STEPECHO) "Create $@"

2
ports/espressif/Makefile
View File

@ -434,7 +434,7 @@ $(BUILD)/firmware.elf: $(OBJ) | esp-idf-stamp
$(BUILD)/circuitpython-firmware.bin: $(BUILD)/firmware.elf | tools/build_memory_info.py
$(STEPECHO) "Create $@"
$(Q)esptool.py --chip $(IDF_TARGET) elf2image $(FLASH_FLAGS) --elf-sha256-offset 0xb0 -o $@ $^
$(Q)$(PYTHON) tools/build_memory_info.py $< $(BUILD)/esp-idf/sdkconfig $@
$(Q)$(PYTHON) tools/build_memory_info.py $< $(BUILD)/esp-idf/sdkconfig $@ $(BUILD)
$(BUILD)/firmware.bin: $(BUILD)/circuitpython-firmware.bin | esp-idf-stamp
$(Q)$(PYTHON) ../../tools/join_bins.py $@ $(BOOTLOADER_OFFSET) $(BUILD)/esp-idf/bootloader/bootloader.bin $(PARTITION_TABLE_OFFSET) $(BUILD)/esp-idf/partition_table/partition-table.bin $(FIRMWARE_OFFSET) $(BUILD)/circuitpython-firmware.bin

9
ports/espressif/tools/build_memory_info.py
View File

@ -6,13 +6,12 @@
# SPDX-License-Identifier: MIT
import csv
import json
import os
import re
import sys
from elftools.elf.elffile import ELFFile
print()
internal_memory = {
"esp32": [
@ -117,8 +116,12 @@ with open(sys.argv[1], "rb") as stream:
# This file is the bin
used_flash = os.stat(sys.argv[3]).st_size
free_flash = firmware_region - used_flash
with open(f"{sys.argv[4]}/firmware.size.json", "w") as f:
json.dump({"used_flash": used_flash, "firmware_region": firmware_region}, f)
print()
print(
"{:7} bytes used, {:7} bytes free in flash firmware space out of {} bytes ({}kB).".format(
used_flash, free_flash, firmware_region, firmware_region / 1024

2
ports/litex/Makefile
View File

@ -136,7 +136,7 @@ all: $(BUILD)/firmware.bin $(BUILD)/firmware.dfu
$(BUILD)/firmware.elf: $(OBJ)
$(STEPECHO) "LINK $@"
$(Q)$(CC) -o $@ $(LDFLAGS) $^ -Wl,--start-group $(LIBS) -Wl,--end-group
$(Q)$(SIZE) $@ | $(PYTHON) $(TOP)/tools/build_memory_info.py $(LD_FILE)
$(Q)$(SIZE) $@ | $(PYTHON) $(TOP)/tools/build_memory_info.py $(LD_FILE) $(BUILD)
$(BUILD)/firmware.bin: $(BUILD)/firmware.elf
$(STEPECHO) "Create $@"

2
ports/nrf/Makefile
View File

@ -212,7 +212,7 @@ $(BUILD)/firmware.elf: $(OBJ) $(GENERATED_LD_FILE)
$(STEPECHO) "LINK $@"
$(Q)echo $(OBJ) > $(BUILD)/firmware.objs
$(Q)$(CC) -o $@ $(LDFLAGS) @$(BUILD)/firmware.objs -Wl,--start-group $(LIBS) -Wl,--end-group
$(Q)$(SIZE) $@ | $(PYTHON) $(TOP)/tools/build_memory_info.py $(GENERATED_LD_FILE)
$(Q)$(SIZE) $@ | $(PYTHON) $(TOP)/tools/build_memory_info.py $(GENERATED_LD_FILE) $(BUILD)
$(BUILD)/firmware.bin: $(BUILD)/firmware.elf
$(STEPECHO) "Create $@"

2
ports/raspberrypi/Makefile
View File

@ -417,7 +417,7 @@ $(BUILD)/firmware.elf: $(OBJ) $(LINK_LD)
$(Q)echo $(OBJ) > $(BUILD)/firmware.objs
$(Q)echo $(PICO_LDFLAGS) > $(BUILD)/firmware.ldflags
$(Q)$(CC) -o $@ $(CFLAGS) @$(BUILD)/firmware.ldflags -Wl,-T,$(LINK_LD) -Wl,-Map=$@.map -Wl,-cref -Wl,--gc-sections @$(BUILD)/firmware.objs -Wl,-lc
$(Q)$(SIZE) $@ | $(PYTHON) $(TOP)/tools/build_memory_info.py $(LINK_LD)
$(Q)$(SIZE) $@ | $(PYTHON) $(TOP)/tools/build_memory_info.py $(LINK_LD) $(BUILD)
$(BUILD)/firmware.bin: $(BUILD)/firmware.elf
$(STEPECHO) "Create $@"

2
ports/stm/Makefile
View File

@ -268,7 +268,7 @@ $(BUILD)/firmware.elf: $(OBJ)
$(STEPECHO) "LINK $@"
$(Q)echo $^ > $(BUILD)/firmware.objs
$(Q)$(CC) -o $@ $(LDFLAGS) @$(BUILD)/firmware.objs -Wl,--start-group $(LIBS) -Wl,--end-group
$(Q)$(SIZE) $@ | $(PYTHON) $(TOP)/tools/build_memory_info.py $(LD_FILE)
$(Q)$(SIZE) $@ | $(PYTHON) $(TOP)/tools/build_memory_info.py $(LD_FILE) $(BUILD)
$(BUILD)/firmware.bin: $(BUILD)/firmware.elf
$(STEPECHO) "Create $@"

11
tools/build_memory_info.py
View File

@ -7,6 +7,8 @@
import re
import sys
import json
# Handle size constants with K or M suffixes (allowed in .ld but not in Python).
K_PATTERN = re.compile(r"([0-9]+)[kK]")
@ -15,11 +17,10 @@ K_REPLACE = r"(\1*1024)"
M_PATTERN = re.compile(r"([0-9]+)[mM]")
M_REPLACE = r"(\1*1024*1024)"
print()
text = 0
data = 0
bss = 0
# stdin is the linker output.
for line in sys.stdin:
# Uncomment to see linker output.
@ -29,6 +30,7 @@ for line in sys.stdin:
text, data, bss = map(int, line.split()[:3])
regions = {}
# This file is the linker script.
with open(sys.argv[1], "r") as f:
for line in f:
@ -51,6 +53,11 @@ used_flash = data + text
free_flash = firmware_region - used_flash
used_ram = data + bss
free_ram = ram_region - used_ram
with open(f"{sys.argv[2]}/firmware.size.json", "w") as f:
json.dump({"used_flash": used_flash, "firmware_region": firmware_region}, f)
print()
print(
"{} bytes used, {} bytes free in flash firmware space out of {} bytes ({}kB).".format(
used_flash, free_flash, firmware_region, firmware_region / 1024

23
tools/build_release_files.py
View File

@ -11,6 +11,7 @@ import subprocess
import shutil
import build_board_info as build_info
import time
import json
sys.path.append("../docs")
from shared_bindings_matrix import get_settings_from_makefile
@ -18,7 +19,7 @@ from shared_bindings_matrix import get_settings_from_makefile
for port in build_info.SUPPORTED_PORTS:
result = subprocess.run("rm -rf ../ports/{port}/build*".format(port=port), shell=True)
PARALLEL = "-j 5"
PARALLEL = "-j 4"
if "GITHUB_ACTION" in os.environ:
PARALLEL = "-j 2"
@ -29,6 +30,11 @@ if "BOARDS" in os.environ:
sha, version = build_info.get_version_info()
build_all = os.environ.get("GITHUB_EVENT_NAME") != "pull_request"
LANGUAGE_FIRST = "en_US"
LANGUAGE_THRESHOLD = 10 * 1024
languages = build_info.get_languages()
all_languages = build_info.get_languages(list_all=True)
@ -44,6 +50,9 @@ for board in build_boards:
board_info = all_boards[board]
board_settings = get_settings_from_makefile("../ports/" + board_info["port"], board)
languages.remove(LANGUAGE_FIRST)
languages.insert(0, LANGUAGE_FIRST)
for language in languages:
bin_directory = "../bin/{board}/{language}".format(board=board, language=language)
os.makedirs(bin_directory, exist_ok=True)
@ -130,4 +139,16 @@ for board in build_boards:
# Flush so we will see something before 10 minutes has passed.
print(flush=True)
if (not build_all) and (language is LANGUAGE_FIRST) and (exit_status is 0):
try:
with open(
f"../ports/{board_info['port']}/{build_dir}/firmware.size.json", "r"
) as f:
firmware = json.load(f)
if firmware["used_flash"] + LANGUAGE_THRESHOLD < firmware["firmware_region"]:
print("Skipping languages")
break
except FileNotFoundError:
pass
sys.exit(exit_status)

15
tools/ci_changes_per_commit.py
View File

@ -167,8 +167,8 @@ def get_commit_depth_and_check_suite(query_commits):
def get_bad_check_runs(query_check_runs):
bad_runs = {}
more_pages = True
bad_runs_by_matrix = {}
run_types = ["failed", "incomplete"]
@ -184,13 +184,12 @@ def get_bad_check_runs(query_check_runs):
for check_run in check_runs[run_type]["nodes"]:
name = check_run["name"]
res_matrix = regex_matrix.search(name)
if res_matrix:
matrix = name.split(" /", 1)[0]
matrix_job = name.split(" (", 1)[1][:-1]
bad_runs_by_matrix.setdefault(matrix, []).append(matrix_job)
if name.startswith("ports") or regex_matrix.search(name):
matrix = name.split(" ", 1)[0]
matrix_job = name.rsplit(" (", 1)[1][:-1]
bad_runs.setdefault(matrix, []).append(matrix_job)
elif name != "scheduler":
bad_runs_by_matrix[name] = True
bad_runs[name] = True
else:
return {}
@ -200,7 +199,7 @@ def get_bad_check_runs(query_check_runs):
query_check_runs.variables["include" + run_type_camel] = True
more_pages = True
return bad_runs_by_matrix
return bad_runs
def set_commit(commit):

4
tools/ci_fetch_deps.py
View File

@ -78,13 +78,13 @@ def main():
submodules = ["extmod/ulab", "lib/", "tools/"]
elif TARGET == "tests":
submodules = ["extmod/ulab", "lib/", "tools/"]
elif TARGET == "build-doc":
elif TARGET == "docs":
# used in .readthedocs.yml to generate RTD
submodules = ["extmod/ulab"]
submodules_tags = ["frozen/"]
elif TARGET == "mpy-cross" or TARGET == "mpy-cross-mac":
submodules = ["tools/"] # for huffman
elif TARGET == "build-windows":
elif TARGET == "windows":
# This builds one board from a number of ports so fill out a bunch of submodules
submodules = ["extmod/ulab", "lib/", "tools/", "ports/", "data/nvm.toml"]
elif TARGET == "website":

76
tools/ci_set_matrix.py
View File

@ -42,18 +42,6 @@ from shared_bindings_matrix import (
all_ports_all_boards,
)
PORT_TO_JOB = {
"atmel-samd": "atmel",
"broadcom": "aarch",
"cxd56": "arm",
"espressif": "esp",
"litex": "riscv",
"mimxrt10xx": "arm",
"nrf": "arm",
"raspberrypi": "arm",
"stm": "arm",
}
IGNORE = [
"tools/ci_set_matrix.py",
"tools/ci_check_duplicate_usb_vid_pid.py",
@ -109,7 +97,7 @@ def set_output(name: str, value):
print(f"Would set GitHub actions output {name} to '{value}'")
def set_boards_to_build(build_all: bool):
def set_boards(build_all: bool):
if last_failed_jobs.get("mpy-cross") or last_failed_jobs.get("tests"):
build_all = True
@ -224,18 +212,13 @@ def set_boards_to_build(build_all: bool):
boards_to_build = all_board_ids
break
# Split boards by job
job_to_boards = {"aarch": [], "arm": [], "atmel": [], "esp": [], "riscv": []}
# Append previously failed boards
for job in job_to_boards:
if job in last_failed_jobs:
for board in last_failed_jobs[job]:
boards_to_build.add(board)
boards_to_build.update(last_failed_jobs.get("ports") or [])
build_boards = bool(boards_to_build)
print("Building boards:", build_boards)
set_output("build-boards", build_boards)
print("Building boards:", bool(boards_to_build))
# Split boards by port
port_to_boards_to_build = {}
# Append boards according to job
for board in sorted(boards_to_build):
@ -244,17 +227,19 @@ def set_boards_to_build(build_all: bool):
# if this happens it's not in `board_to_port`.
if not port:
continue
job_to_boards[PORT_TO_JOB[port]].append(board)
port_to_boards_to_build.setdefault(port, []).append(board)
print(" ", board)
# Set the step outputs for each job
for job in job_to_boards:
set_output(f"boards-{job}", json.dumps(job_to_boards[job]))
if port_to_boards_to_build:
port_to_boards_to_build["ports"] = sorted(list(port_to_boards_to_build.keys()))
# Set the step outputs
set_output("ports", json.dumps(port_to_boards_to_build))
def set_docs_to_build(build_doc: bool):
def set_docs(build_doc: bool):
if not build_doc:
if last_failed_jobs.get("build-doc"):
if last_failed_jobs.get("docs"):
build_doc = True
else:
doc_pattern = re.compile(PATTERN_DOCS)
@ -277,12 +262,12 @@ def set_docs_to_build(build_doc: bool):
# Set the step outputs
print("Building docs:", build_doc)
set_output("build-doc", build_doc)
set_output("docs", build_doc)
def set_windows_to_build(build_windows):
def set_windows(build_windows: bool):
if not build_windows:
if last_failed_jobs.get("build-windows"):
if last_failed_jobs.get("windows"):
build_windows = True
else:
for file in changed_files:
@ -296,23 +281,22 @@ def set_windows_to_build(build_windows):
# Set the step outputs
print("Building windows:", build_windows)
set_output("build-windows", build_windows)
def check_changed_files():
if not changed_files:
print("Building all docs/boards")
return True
else:
print("Adding docs/boards to build based on changed files")
return False
set_output("windows", build_windows)
def main():
build_all = check_changed_files()
set_docs_to_build(build_all)
set_windows_to_build(build_all)
set_boards_to_build(build_all)
# Build all if no changed files
build_all = not changed_files
print(
"Building all docs/boards"
if build_all
else "Adding docs/boards to build based on changed files"
)
# Set jobs
set_docs(build_all)
set_windows(build_all)
set_boards(build_all)
if __name__ == "__main__":