djsundog
/
circuitpython
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

split boards dynamically

This commit is contained in:
MicroDev 2023-03-04 10:32:19 +05:30
parent 8a10069995
commit 055c9bfb40
No known key found for this signature in database
GPG Key ID: 2C0867BE60967730
10 changed files with 112 additions and 174 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
.github/actions/deps/external/action.yml vendored
View File

@ -9,54 +9,31 @@ inputs:
- cache
- restore
platform:
port:
required: false
default: none
type: choice
options:
- arm
- aarch
- esp
- riscv
- none
type: string
runs:
using: composite
steps:
# aarch
- name: Get aarch toolchain
if: inputs.platform == 'aarch'
run: |
wget --no-verbose https://adafruit-circuit-python.s3.amazonaws.com/gcc-arm-10.3-2021.07-x86_64-aarch64-none-elf.tar.xz
sudo tar -C /usr --strip-components=1 -xaf gcc-arm-10.3-2021.07-x86_64-aarch64-none-elf.tar.xz
sudo apt-get install -y mtools
shell: bash
- name: Install mkfs.fat
if: inputs.platform == 'aarch'
run: |
wget https://github.com/dosfstools/dosfstools/releases/download/v4.2/dosfstools-4.2.tar.gz
tar -xaf dosfstools-4.2.tar.gz
cd dosfstools-4.2
./configure
make -j 2
cd src
echo >> $GITHUB_PATH $(pwd)
shell: bash
# arm
- name: Get arm toolchain
if: inputs.platform == 'aarch' || inputs.platform == 'arm'
if: >-
inputs.port != 'none' &&
inputs.port != 'litex' &&
inputs.port != 'espressif'
uses: carlosperate/arm-none-eabi-gcc-action@v1
with:
release: '10-2020-q4'
# esp
- name: Get esp toolchain
if: inputs.platform == 'esp'
# espressif
- name: Get espressif toolchain
if: inputs.port == 'espressif'
run: sudo apt-get install -y ninja-build
shell: bash
- name: Install IDF tools
if: inputs.platform == 'esp'
if: inputs.port == 'espressif'
run: |
echo "Installing ESP-IDF tools"
$IDF_PATH/tools/idf_tools.py --non-interactive install required
@ -66,24 +43,16 @@ runs:
rm -rf $IDF_TOOLS_PATH/dist
shell: bash
- name: Set environment
if: inputs.platform == 'esp'
if: inputs.port == 'espressif'
run: |
source $IDF_PATH/export.sh
echo >> $GITHUB_ENV "IDF_PYTHON_ENV_PATH=$IDF_PYTHON_ENV_PATH"
echo >> $GITHUB_PATH "$PATH"
shell: bash
# riscv
- name: Get riscv toolchain
if: inputs.platform == 'riscv'
run: |
wget https://static.dev.sifive.com/dev-tools/riscv64-unknown-elf-gcc-8.3.0-2019.08.0-x86_64-linux-centos6.tar.gz
sudo tar -C /usr --strip-components=1 -xaf riscv64-unknown-elf-gcc-8.3.0-2019.08.0-x86_64-linux-centos6.tar.gz
shell: bash
# common
- name: Cache python dependencies
if: inputs.platform != 'esp'
if: inputs.port != 'espressif'
uses: ./.github/actions/deps/python
with:
action: ${{ inputs.action }}

12
.github/actions/deps/ports/action.yml vendored
View File

@ -19,10 +19,18 @@ runs:
if [ -z $PORT ]; then (exit 1); else echo >> $GITHUB_OUTPUT "port=$PORT"; fi
shell: bash
- name: Set up espressif port
- name: Set up broadcom
if: steps.board-to-port.outputs.port == 'broadcom'
uses: ./.github/actions/deps/ports/broadcom
- name: Set up espressif
if: steps.board-to-port.outputs.port == 'espressif'
uses: ./.github/actions/deps/ports/espressif
- name: Set up nrf port
- name: Set up litex
if: steps.board-to-port.outputs.port == 'litex'
uses: ./.github/actions/deps/ports/litex
- name: Set up nrf
if: steps.board-to-port.outputs.port == 'nrf'
uses: ./.github/actions/deps/ports/nrf

21
.github/actions/deps/ports/broadcom/action.yml vendored Normal file
View File

@ -0,0 +1,21 @@
name: Fetch broadcom port deps
runs:
using: composite
steps:
- name: Get broadcom toolchain
run: |
wget --no-verbose https://adafruit-circuit-python.s3.amazonaws.com/gcc-arm-10.3-2021.07-x86_64-broadcom64-none-elf.tar.xz
sudo tar -C /usr --strip-components=1 -xaf gcc-arm-10.3-2021.07-x86_64-broadcom64-none-elf.tar.xz
sudo apt-get install -y mtools
shell: bash
- name: Install mkfs.fat
run: |
wget https://github.com/dosfstools/dosfstools/releases/download/v4.2/dosfstools-4.2.tar.gz
tar -xaf dosfstools-4.2.tar.gz
cd dosfstools-4.2
./configure
make -j 2
cd src
echo >> $GITHUB_PATH $(pwd)
shell: bash

10
.github/actions/deps/ports/litex/action.yml vendored Normal file
View File

@ -0,0 +1,10 @@
name: Fetch litex port deps
runs:
using: composite
steps:
- name: Get litex toolchain
run: |
wget https://static.dev.sifive.com/dev-tools/riscv64-unknown-elf-gcc-8.3.0-2019.08.0-x86_64-linux-centos6.tar.gz
sudo tar -C /usr --strip-components=1 -xaf riscv64-unknown-elf-gcc-8.3.0-2019.08.0-x86_64-linux-centos6.tar.gz
shell: bash

12
.github/workflows/build-boards-custom.yml vendored
View File

@ -32,16 +32,6 @@ run-name: ${{ inputs.board }}-${{ inputs.language }}-${{ inputs.version }}${{ in
jobs:
build:
runs-on: ubuntu-22.04
env:
PLATFORM_atmel-samd: arm
PLATFORM_broadcom: aarch
PLATFORM_cxd56: arm
PLATFORM_espressif: esp
PLATFORM_litex: riscv
PLATFORM_mimxrt10xx: arm
PLATFORM_nrf: arm
PLATFORM_raspberrypi: arm
PLATFORM_stm: arm
steps:
- name: Set up repository
run: |
@ -70,7 +60,7 @@ jobs:
uses: ./.github/actions/deps/external
with:
action: cache
platform: ${{ env[format('PLATFORM_{0}', steps.set-up-port.outputs.port)] }}
port: ${{ steps.set-up-port.outputs.port }}
- name: Set up mpy-cross
if: steps.set-up-submodules.outputs.frozen == 'True'
uses: ./.github/actions/mpy_cross

6
.github/workflows/build-boards.yml vendored
View File

@ -3,9 +3,6 @@ name: Build boards
on:
workflow_call:
inputs:
platform:
required: true
type: string
boards:
required: true
type: string
@ -38,6 +35,7 @@ jobs:
with:
python-version: 3.x
- name: Set up port
id: set-up-port
uses: ./.github/actions/deps/ports
with:
board: ${{ matrix.board }}
@ -47,7 +45,7 @@ jobs:
- name: Set up external
uses: ./.github/actions/deps/external
with:
platform: ${{ inputs.platform }}
port: ${{ steps.set-up-port.outputs.port }}
- name: Set up mpy-cross
if: steps.set-up-submodules.outputs.frozen == 'True'
uses: ./.github/actions/mpy_cross

73
.github/workflows/build.yml vendored
View File

@ -20,14 +20,9 @@ jobs:
scheduler:
runs-on: ubuntu-20.04
outputs:
build-doc: ${{ steps.set-matrix.outputs.build-doc }}
build-boards: ${{ steps.set-matrix.outputs.build-boards }}
build-windows: ${{ steps.set-matrix.outputs.build-windows }}
boards-aarch: ${{ steps.set-matrix.outputs.boards-aarch }}
boards-arm: ${{ steps.set-matrix.outputs.boards-arm }}
boards-atmel: ${{ steps.set-matrix.outputs.boards-atmel }}
boards-esp: ${{ steps.set-matrix.outputs.boards-esp }}
boards-riscv: ${{ steps.set-matrix.outputs.boards-riscv }}
docs: ${{ steps.set-matrix.outputs.docs }}
boards: ${{ steps.set-matrix.outputs.boards }}
windows: ${{ steps.set-matrix.outputs.windows }}
cp-version: ${{ steps.set-up-submodules.outputs.version }}
steps:
- name: Dump GitHub context
@ -99,7 +94,7 @@ jobs:
mpy-cross:
needs: scheduler
if: needs.scheduler.outputs.build-boards == 'True'
if: needs.scheduler.outputs.boards != '{}'
uses: ./.github/workflows/build-mpy-cross.yml
secrets: inherit
with:
@ -108,7 +103,7 @@ jobs:
mpy-cross-mac:
runs-on: macos-11
needs: scheduler
if: needs.scheduler.outputs.build-boards == 'True'
if: needs.scheduler.outputs.boards != '{}'
env:
CP_VERSION: ${{ needs.scheduler.outputs.cp-version }}
steps:
@ -160,10 +155,10 @@ jobs:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
build-doc:
docs:
runs-on: ubuntu-22.04
needs: scheduler
if: needs.scheduler.outputs.build-doc == 'True'
if: needs.scheduler.outputs.docs == 'True'
env:
CP_VERSION: ${{ needs.scheduler.outputs.cp-version }}
steps:
@ -218,10 +213,10 @@ jobs:
[ -z "$TWINE_USERNAME" ] || echo "Uploading dev release to PyPi"
[ -z "$TWINE_USERNAME" ] || twine upload circuitpython-stubs/dist/*
build-windows:
windows:
runs-on: windows-2022
needs: scheduler
if: needs.scheduler.outputs.build-windows == 'True'
if: needs.scheduler.outputs.windows == 'True'
env:
CP_VERSION: ${{ needs.scheduler.outputs.cp-version }}
defaults:
@ -286,52 +281,14 @@ jobs:
# ERROR: Platform MINGW64_NT-10.0-17763-x86_64 appears to be unsupported
# https://github.com/espressif/esp-idf/issues/7062
aarch:
boards:
needs: [scheduler, mpy-cross, tests]
if: ${{ needs.scheduler.outputs.boards-aarch != '[]' }}
if: needs.scheduler.outputs.boards != '{}'
uses: ./.github/workflows/build-boards.yml
secrets: inherit
strategy:
matrix:
port: ${{ fromJSON(needs.scheduler.outputs.boards).ports }}
with:
platform: aarch
boards: ${{ needs.scheduler.outputs.boards-aarch }}
cp-version: ${{ needs.scheduler.outputs.cp-version }}
arm:
needs: [scheduler, mpy-cross, tests]
if: ${{ needs.scheduler.outputs.boards-arm != '[]' }}
uses: ./.github/workflows/build-boards.yml
secrets: inherit
with:
platform: arm
boards: ${{ needs.scheduler.outputs.boards-arm }}
cp-version: ${{ needs.scheduler.outputs.cp-version }}
atmel:
needs: [scheduler, mpy-cross, tests]
if: ${{ needs.scheduler.outputs.boards-atmel != '[]' }}
uses: ./.github/workflows/build-boards.yml
secrets: inherit
with:
platform: arm
boards: ${{ needs.scheduler.outputs.boards-atmel }}
cp-version: ${{ needs.scheduler.outputs.cp-version }}
esp:
needs: [scheduler, mpy-cross, tests]
if: ${{ needs.scheduler.outputs.boards-esp != '[]' }}
uses: ./.github/workflows/build-boards.yml
secrets: inherit
with:
platform: esp
boards: ${{ needs.scheduler.outputs.boards-esp }}
cp-version: ${{ needs.scheduler.outputs.cp-version }}
riscv:
needs: [scheduler, mpy-cross, tests]
if: ${{ needs.scheduler.outputs.boards-riscv != '[]' }}
uses: ./.github/workflows/build-boards.yml
secrets: inherit
with:
platform: riscv
boards: ${{ needs.scheduler.outputs.boards-riscv }}
boards: ${{ toJSON(fromJSON(needs.scheduler.outputs.boards)[matrix.port]) }}
cp-version: ${{ needs.scheduler.outputs.cp-version }}

15
tools/ci_changes_per_commit.py
View File

@ -167,8 +167,8 @@ def get_commit_depth_and_check_suite(query_commits):
def get_bad_check_runs(query_check_runs):
bad_runs = {}
more_pages = True
bad_runs_by_matrix = {}
run_types = ["failed", "incomplete"]
@ -184,13 +184,12 @@ def get_bad_check_runs(query_check_runs):
for check_run in check_runs[run_type]["nodes"]:
name = check_run["name"]
res_matrix = regex_matrix.search(name)
if res_matrix:
matrix = name.split(" /", 1)[0]
matrix_job = name.split(" (", 1)[1][:-1]
bad_runs_by_matrix.setdefault(matrix, []).append(matrix_job)
if name.startswith("boards") or regex_matrix.search(name):
matrix = name.split(" ", 1)[0]
matrix_job = name.rsplit(" (", 1)[1][:-1]
bad_runs.setdefault(matrix, []).append(matrix_job)
elif name != "scheduler":
bad_runs_by_matrix[name] = True
bad_runs[name] = True
else:
return {}
@ -200,7 +199,7 @@ def get_bad_check_runs(query_check_runs):
query_check_runs.variables["include" + run_type_camel] = True
more_pages = True
return bad_runs_by_matrix
return bad_runs
def set_commit(commit):

4
tools/ci_fetch_deps.py
View File

@ -78,13 +78,13 @@ def main():
submodules = ["extmod/ulab", "lib/", "tools/"]
elif TARGET == "tests":
submodules = ["extmod/ulab", "lib/", "tools/"]
elif TARGET == "build-doc":
elif TARGET == "docs":
# used in .readthedocs.yml to generate RTD
submodules = ["extmod/ulab"]
submodules_tags = ["frozen/"]
elif TARGET == "mpy-cross" or TARGET == "mpy-cross-mac":
submodules = ["tools/"] # for huffman
elif TARGET == "build-windows":
elif TARGET == "windows":
# This builds one board from a number of ports so fill out a bunch of submodules
submodules = ["extmod/ulab", "lib/", "tools/", "ports/", "data/nvm.toml"]
elif TARGET == "website":

78
tools/ci_set_matrix.py
View File

@ -42,18 +42,6 @@ from shared_bindings_matrix import (
all_ports_all_boards,
)
PORT_TO_JOB = {
"atmel-samd": "atmel",
"broadcom": "aarch",
"cxd56": "arm",
"espressif": "esp",
"litex": "riscv",
"mimxrt10xx": "arm",
"nrf": "arm",
"raspberrypi": "arm",
"stm": "arm",
}
IGNORE = [
"tools/ci_set_matrix.py",
"tools/ci_check_duplicate_usb_vid_pid.py",
@ -109,7 +97,7 @@ def set_output(name: str, value):
print(f"Would set GitHub actions output {name} to '{value}'")
def set_boards_to_build(build_all: bool):
def set_boards(build_all: bool):
if last_failed_jobs.get("mpy-cross") or last_failed_jobs.get("tests"):
build_all = True
@ -224,18 +212,15 @@ def set_boards_to_build(build_all: bool):
boards_to_build = all_board_ids
break
# Split boards by job
job_to_boards = {"aarch": [], "arm": [], "atmel": [], "esp": [], "riscv": []}
# Append previously failed boards
for job in job_to_boards:
if job in last_failed_jobs:
for board in last_failed_jobs[job]:
boards_to_build.add(board)
last_failed_boards = last_failed_jobs.get("boards")
if last_failed_boards:
boards_to_build.update(last_failed_boards)
build_boards = bool(boards_to_build)
print("Building boards:", build_boards)
set_output("build-boards", build_boards)
print("Building boards:", bool(boards_to_build))
# Split boards by port
port_to_boards_to_build = {}
# Append boards according to job
for board in sorted(boards_to_build):
@ -244,17 +229,19 @@ def set_boards_to_build(build_all: bool):
# if this happens it's not in `board_to_port`.
if not port:
continue
job_to_boards[PORT_TO_JOB[port]].append(board)
port_to_boards_to_build.setdefault(port, []).append(board)
print(" ", board)
# Set the step outputs for each job
for job in job_to_boards:
set_output(f"boards-{job}", json.dumps(job_to_boards[job]))
if port_to_boards_to_build:
port_to_boards_to_build["ports"] = sorted(list(port_to_boards_to_build.keys()))
# Set the step outputs
set_output("boards", json.dumps(port_to_boards_to_build))
def set_docs_to_build(build_doc: bool):
def set_docs(build_doc: bool):
if not build_doc:
if last_failed_jobs.get("build-doc"):
if last_failed_jobs.get("docs"):
build_doc = True
else:
doc_pattern = re.compile(PATTERN_DOCS)
@ -277,12 +264,12 @@ def set_docs_to_build(build_doc: bool):
# Set the step outputs
print("Building docs:", build_doc)
set_output("build-doc", build_doc)
set_output("docs", build_doc)
def set_windows_to_build(build_windows):
def set_windows(build_windows: bool):
if not build_windows:
if last_failed_jobs.get("build-windows"):
if last_failed_jobs.get("windows"):
build_windows = True
else:
for file in changed_files:
@ -296,23 +283,22 @@ def set_windows_to_build(build_windows):
# Set the step outputs
print("Building windows:", build_windows)
set_output("build-windows", build_windows)
def check_changed_files():
if not changed_files:
print("Building all docs/boards")
return True
else:
print("Adding docs/boards to build based on changed files")
return False
set_output("windows", build_windows)
def main():
build_all = check_changed_files()
set_docs_to_build(build_all)
set_windows_to_build(build_all)
set_boards_to_build(build_all)
# Build all if no changed files
build_all = not changed_files
print(
"Building all docs/boards"
if build_all
else "Adding docs/boards to build based on changed files"
)
# Set jobs
set_docs(build_all)
set_windows(build_all)
set_boards(build_all)
if __name__ == "__main__":