djsundog/circuitpython
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #7631 from MicroDev1/ci

Browse Source 
CI: More fixes and refactoring
This commit is contained in:
Scott Shawcroft 2023-02-23 11:49:28 -08:00 committed by GitHub
commit bf657e638a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 176 additions and 161 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
.github/actions/deps/ports/action.yml vendored Normal file
View File

@ -0,0 +1,24 @@
name: Fetch port deps
inputs:
board:
required: true
type: string
outputs:
port:
value: ${{ steps.board-to-port.outputs.port }}
runs:
using: composite
steps:
- name: Board to port
id: board-to-port
run: |
PORT=$(find ports/*/boards/ -type d -name ${{ inputs.board }} | sed 's/^ports\///g;s/\/boards.*//g')
if [ -z $PORT ]; then (exit 1); else echo >> $GITHUB_OUTPUT "port=$PORT"; fi
shell: bash
- name: Set up espressif port
if: steps.board-to-port.outputs.port == 'espressif'
uses: ./.github/actions/deps/ports/espressif

19
.github/workflows/build-boards-custom.yml vendored
View File

@ -51,20 +51,15 @@ jobs:
if: inputs.debug || inputs.flags != ''
run: |
> custom-build && git add custom-build
- name: Get port
id: get-port
run: |
PORT=$(find ports/*/boards/ -type d -name ${{ inputs.board }} | sed 's/^ports\///g;s/\/boards.*//g')
if [ -z $PORT ]; then (exit 1); else echo >> $GITHUB_OUTPUT "port=$PORT"; fi
- name: Port to platform
run: echo >> $GITHUB_ENV "PLATFORM=${{ env[format('PLATFORM_{0}', steps.get-port.outputs.port)] }}"
- name: Set up python
uses: actions/setup-python@v4
with:
python-version: 3.x
- name: Set up port
if: env.PLATFORM == 'esp'
uses: ./.github/actions/deps/ports/espressif
id: set-up-port
uses: ./.github/actions/deps/ports
with:
board: ${{ inputs.board }}
- name: Set up submodules
id: set-up-submodules
uses: ./.github/actions/deps/submodules
@ -75,7 +70,7 @@ jobs:
uses: ./.github/actions/deps/external
with:
action: cache
platform: ${{ env.PLATFORM }}
platform: ${{ env[format('PLATFORM_{0}', steps.set-up-port.outputs.port)] }}
- name: Set up mpy-cross
if: steps.set-up-submodules.outputs.frozen == 'True'
uses: ./.github/actions/mpy_cross
@ -96,9 +91,9 @@ jobs:
mkfs.fat --version || true
- name: Build board
run: make -j2 ${{ inputs.flags }} BOARD=${{ inputs.board }} DEBUG=${{ inputs.debug && '1' || '0' }} TRANSLATION=${{ inputs.language }}
working-directory: ports/${{ steps.get-port.outputs.port }}
working-directory: ports/${{ steps.set-up-port.outputs.port }}
- name: Upload artifact
uses: actions/upload-artifact@v3
with:
name: ${{ inputs.board }}-${{ inputs.language }}-${{ inputs.version }}${{ inputs.flags != '' && '-custom' || '' }}${{ inputs.debug && '-debug' || '' }}
path: ports/${{ steps.get-port.outputs.port }}/build-${{ inputs.board }}/firmware.*
path: ports/${{ steps.set-up-port.outputs.port }}/build-${{ inputs.board }}/firmware.*

5
.github/workflows/build-boards.yml vendored
View File

@ -38,8 +38,9 @@ jobs:
with:
python-version: 3.x
- name: Set up port
if: inputs.platform == 'esp'
uses: ./.github/actions/deps/ports/espressif
uses: ./.github/actions/deps/ports
with:
board: ${{ matrix.board }}
- name: Set up submodules
id: set-up-submodules
uses: ./.github/actions/deps/submodules

103
.github/workflows/build.yml vendored
View File

@ -22,11 +22,12 @@ jobs:
outputs:
build-doc: ${{ steps.set-matrix.outputs.build-doc }}
build-boards: ${{ steps.set-matrix.outputs.build-boards }}
build-windows: ${{ steps.set-matrix.outputs.build-windows }}
boards-aarch: ${{ steps.set-matrix.outputs.boards-aarch }}
boards-arm: ${{ steps.set-matrix.outputs.boards-arm }}
boards-atmel: ${{ steps.set-matrix.outputs.boards-atmel }}
boards-esp: ${{ steps.set-matrix.outputs.boards-esp }}
boards-riscv: ${{ steps.set-matrix.outputs.boards-riscv }}
boards-rpi: ${{ steps.set-matrix.outputs.boards-rpi }}
cp-version: ${{ steps.set-up-submodules.outputs.version }}
steps:
- name: Dump GitHub context
@ -54,6 +55,10 @@ jobs:
uses: ./.github/actions/deps/external
with:
action: cache
- name: Set up mpy-cross
uses: ./.github/actions/mpy_cross
with:
download: false
- name: Get last commit with checks
id: get-last-commit-with-checks
if: github.event_name == 'pull_request'
@ -63,14 +68,10 @@ jobs:
REPO: ${{ github.repository }}
PULL: ${{ github.event.number }}
GITHUB_TOKEN: ${{ github.token }}
EXCLUDE_COMMIT: ${{ github.event.after }}
- name: Set up mpy-cross
uses: ./.github/actions/mpy_cross
with:
download: false
EXCLUDE_COMMIT: ${{ github.event.pull_request.head.sha }}
- name: Set head sha
if: github.event_name == 'pull_request'
run: echo "HEAD_SHA=$(git show -s --format=%s $GITHUB_SHA | grep -o -P "(?<=Merge ).*(?= into)")" >> $GITHUB_ENV
run: echo "HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
- name: Set base sha
if: github.event_name == 'pull_request'
run: |
@ -218,6 +219,74 @@ jobs:
[ -z "$TWINE_USERNAME" ] || echo "Uploading dev release to PyPi"
[ -z "$TWINE_USERNAME" ] || twine upload circuitpython-stubs/dist/*
build-windows:
runs-on: windows-2022
needs: scheduler
if: ${{ needs.scheduler.outputs.build-windows == 'True' }}
env:
CP_VERSION: ${{ needs.scheduler.outputs.cp-version }}
defaults:
run:
# We define a custom shell script here, although `msys2.cmd` does neither exist nor is it available in the PATH yet
shell: msys2 {0}
steps:
# We want to change the configuration of the git command that actions/checkout will be using
# (since it is not possible to set autocrlf through the action yet, see actions/checkout#226).
- run: git config --global core.autocrlf input
shell: bash
- name: Check python coding (cmd)
run: python -c "import sys, locale; print(sys.getdefaultencoding(), locale.getpreferredencoding(False))"
shell: cmd
# We use a JS Action, which calls the system terminal or other custom terminals directly, if required
- uses: msys2/setup-msys2@v2
with:
install: base-devel git wget unzip gcc python-pip
# The goal of this was to test how things worked when the default file encoding (locale.getpreferedencoding())
# was not UTF-8. However, msys2 python does use utf-8 as the preferred file encoding, and using actions/setup-python
# python3.8 gave a broken build, so we're not really testing what we wanted to test.
# However, commandline length limits are being tested so that does some good.
- name: Check python coding (msys2)
run: |
locale -v
which python; python --version
python -c "import sys, locale; print(sys.getdefaultencoding(), locale.getpreferredencoding(False))"
which python3; python3 --version
python3 -c "import sys, locale; print(sys.getdefaultencoding(), locale.getpreferredencoding(False))"
- name: Install dependencies
run: |
wget --no-verbose -O gcc-arm.zip https://developer.arm.com/-/media/Files/downloads/gnu-rm/10-2020q4/gcc-arm-none-eabi-10-2020-q4-major-win32.zip
unzip -q -d /tmp gcc-arm.zip
tar -C /tmp/gcc-arm-none-* -cf - . | tar -C /usr/local -xf -
pip install wheel
# requirements_dev.txt doesn't install on windows. (with msys2 python)
# instead, pick a subset for what we want to do
pip install cascadetoml jinja2 typer click intelhex
# check that installed packages work....?
which python; python --version; python -c "import cascadetoml"
which python3; python3 --version; python3 -c "import cascadetoml"
- name: Set up repository
uses: actions/checkout@v3
with:
submodules: false
fetch-depth: 1
- name: Set up submodules
uses: ./.github/actions/deps/submodules
- name: build mpy-cross
run: make -j2 -C mpy-cross
- name: build rp2040
run: make -j2 -C ports/raspberrypi BOARD=adafruit_feather_rp2040 TRANSLATION=de_DE
- name: build samd21
run: make -j2 -C ports/atmel-samd BOARD=feather_m0_express TRANSLATION=zh_Latn_pinyin
- name: build samd51
run: make -j2 -C ports/atmel-samd BOARD=feather_m4_express TRANSLATION=es
- name: build nrf
run: make -j2 -C ports/nrf BOARD=feather_nrf52840_express TRANSLATION=fr
- name: build stm
run: make -j2 -C ports/stm BOARD=feather_stm32f405_express TRANSLATION=pt_BR
# I gave up trying to do esp builds on windows when I saw
# ERROR: Platform MINGW64_NT-10.0-17763-x86_64 appears to be unsupported
# https://github.com/espressif/esp-idf/issues/7062
aarch:
needs: [scheduler, mpy-cross, tests]
if: ${{ needs.scheduler.outputs.boards-aarch != '[]' }}
@ -238,6 +307,16 @@ jobs:
boards: ${{ needs.scheduler.outputs.boards-arm }}
cp-version: ${{ needs.scheduler.outputs.cp-version }}
atmel:
needs: [scheduler, mpy-cross, tests]
if: ${{ needs.scheduler.outputs.boards-atmel != '[]' }}
uses: ./.github/workflows/build-boards.yml
secrets: inherit
with:
platform: arm
boards: ${{ needs.scheduler.outputs.boards-atmel }}
cp-version: ${{ needs.scheduler.outputs.cp-version }}
esp:
needs: [scheduler, mpy-cross, tests]
if: ${{ needs.scheduler.outputs.boards-esp != '[]' }}
@ -257,13 +336,3 @@ jobs:
platform: riscv
boards: ${{ needs.scheduler.outputs.boards-riscv }}
cp-version: ${{ needs.scheduler.outputs.cp-version }}
rpi:
needs: [scheduler, mpy-cross, tests]
if: ${{ needs.scheduler.outputs.boards-rpi != '[]' }}
uses: ./.github/workflows/build-boards.yml
secrets: inherit
with:
platform: arm
boards: ${{ needs.scheduler.outputs.boards-rpi }}
cp-version: ${{ needs.scheduler.outputs.cp-version }}

111
.github/workflows/ports_windows.yml vendored
View File

@ -1,111 +0,0 @@
name: windows port
on:
push:
pull_request:
paths:
- '.github/workflows/ports_windows.yml'
- 'extmod/**'
- 'lib/**'
- 'mpy-cross/**'
- 'ports/unix/**'
- 'ports/windows/**'
- 'py/**'
- 'requirements*.txt'
- 'tools/**'
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
jobs:
windows:
runs-on: windows-2022
defaults:
run:
# We define a custom shell script here, although `msys2.cmd` does neither exist nor is it available in the PATH yet
shell: msys2 {0}
steps:
# We want to change the configuration of the git command that actions/checkout will be using (since it is not possible to set autocrlf through the action yet, see actions/checkout#226).
- run: git config --global core.autocrlf input
shell: bash
- name: Check python coding (cmd)
run: python -c "import sys, locale; print(sys.getdefaultencoding(), locale.getpreferredencoding(False))"
shell: cmd
# We use a JS Action, which calls the system terminal or other custom terminals directly, if required
- uses: msys2/setup-msys2@v2
with:
update: true
install: base-devel git wget unzip gcc python-pip
# The goal of this was to test how things worked when the default file
# encoding (locale.getpreferedencoding()) was not UTF-8. However, msys2
# python does use utf-8 as the preferred file encoding, and using
# actions/setup-python python3.8 gave a broken build, so we're not really
# testing what we wanted to test.
#
# however, commandline length limits are being tested so that does some
# good.
- name: Check python coding (msys2)
run: |
locale -v
which python; python --version
python -c "import sys, locale; print(sys.getdefaultencoding(), locale.getpreferredencoding(False))"
which python3; python3 --version
python3 -c "import sys, locale; print(sys.getdefaultencoding(), locale.getpreferredencoding(False))"
- name: Install dependencies
run: |
wget --no-verbose -O gcc-arm.zip https://developer.arm.com/-/media/Files/downloads/gnu-rm/10-2020q4/gcc-arm-none-eabi-10-2020-q4-major-win32.zip
unzip -q -d /tmp gcc-arm.zip
tar -C /tmp/gcc-arm-none-* -cf - . | tar -C /usr/local -xf -
pip install wheel
# requirements_dev.txt doesn't install on windows. (with msys2 python)
# instead, pick a subset for what we want to do
pip install cascadetoml jinja2 typer click intelhex
# check that installed packages work....?
which python; python --version; python -c "import cascadetoml"
which python3; python3 --version; python3 -c "import cascadetoml"
- name: Set up repository
uses: actions/checkout@v3
with:
submodules: false
fetch-depth: 1
- name: Set up submodules
uses: ./.github/actions/deps/submodules
with:
version: true
- name: build mpy-cross
run: make -j2 -C mpy-cross
- name: build rp2040
run: make -j2 -C ports/raspberrypi BOARD=adafruit_feather_rp2040 TRANSLATION=de_DE
- name: build samd21
run: make -j2 -C ports/atmel-samd BOARD=feather_m0_express TRANSLATION=zh_Latn_pinyin
- name: build samd51
run: make -j2 -C ports/atmel-samd BOARD=feather_m4_express TRANSLATION=es
- name: build nrf
run: make -j2 -C ports/nrf BOARD=feather_nrf52840_express TRANSLATION=fr
- name: build stm
run: make -j2 -C ports/stm BOARD=feather_stm32f405_express TRANSLATION=pt_BR
# I gave up trying to do esp32 builds on windows when I saw
# ERROR: Platform MINGW64_NT-10.0-17763-x86_64 appears to be unsupported
# https://github.com/espressif/esp-idf/issues/7062
#
# - name: prepare esp
# run: ports/espressif/esp-idf/install.bat
# shell: cmd
#
# - name: build esp
# run: . ports/espressif/esp-idf/export.sh && make -j2 -C ports/espressif BOARD=adafruit_metro_esp32s2

2
tools/ci_changes_per_commit.py
View File

@ -172,7 +172,7 @@ def get_bad_check_runs(query_check_runs):
run_types = ["failed", "incomplete"]
regex_matrix = re.compile("^[^\n ]+ \/ (build|run) \([^\n ]+\)$")
regex_matrix = re.compile(r"^\S+ \/ (build|run) \(\S+\)$")
while more_pages:
check_runs = query_check_runs.fetch()["data"]["node"]

8
tools/ci_fetch_deps.py
View File

@ -80,14 +80,16 @@ def main():
submodules = ["extmod/ulab", "lib/", "tools/"]
elif TARGET == "build-doc":
# used in .readthedocs.yml to generate RTD
submodules = ["extmod/ulab", "frozen/"]
submodules = ["extmod/ulab"]
submodules_tags = ["frozen/"]
elif TARGET == "mpy-cross" or TARGET == "mpy-cross-mac":
submodules = ["tools/"] # for huffman
elif TARGET == "windows":
elif TARGET == "build-windows":
# This builds one board from a number of ports so fill out a bunch of submodules
submodules = ["extmod/ulab", "lib/", "tools/", "ports/", "data/nvm.toml"]
elif TARGET == "website":
submodules = ["tools/adabot/", "frozen/"]
submodules = ["tools/adabot/"]
submodules_tags = ["frozen/"]
elif TARGET == "pre-commit":
submodules = ["extmod/ulab"]
else:

65
tools/ci_set_matrix.py
View File

@ -43,14 +43,14 @@ from shared_bindings_matrix import (
)
PORT_TO_ARCH = {
"atmel-samd": "arm",
"atmel-samd": "atmel",
"broadcom": "aarch",
"cxd56": "arm",
"espressif": "esp",
"litex": "riscv",
"mimxrt10xx": "arm",
"nrf": "arm",
"raspberrypi": "rpi",
"raspberrypi": "arm",
"stm": "arm",
}
@ -62,6 +62,24 @@ IGNORE = [
# Files in these directories never influence board builds
IGNORE_DIRS = ["tests", "docs", ".devcontainer"]
PATTERN_DOCS = (
r"^(?:\.github|docs|extmod\/ulab)|"
r"^(?:(?:ports\/\w+\/bindings|shared-bindings)\S+\.c|tools\/extract_pyi\.py|conf\.py|requirements-doc\.txt)$|"
r"(?:-stubs|\.(?:md|MD|rst|RST))$"
)
PATTERN_WINDOWS = [
".github/",
"extmod/",
"lib/",
"mpy-cross/",
"ports/unix/",
"ports/windows/",
"py/",
"requirements",
"tools/",
]
if len(sys.argv) > 1:
print("Using files list on commandline")
changed_files = sys.argv[1:]
@ -92,7 +110,7 @@ def set_output(name: str, value):
def set_boards_to_build(build_all: bool):
if "mpy_cross" in last_failed_jobs or "tests" in last_failed_jobs:
if last_failed_jobs.get("mpy_cross") or last_failed_jobs.get("tests"):
build_all = True
# Get boards in json format
@ -207,13 +225,12 @@ def set_boards_to_build(build_all: bool):
break
# Split boards by architecture.
arch_to_boards = {"aarch": [], "arm": [], "esp": [], "riscv": [], "rpi": []}
arch_to_boards = {"aarch": [], "arm": [], "atmel": [], "esp": [], "riscv": []}
# Append previously failed boards
for arch in arch_to_boards:
arch_to_job = f"build-{arch}"
if arch_to_job in last_failed_jobs:
for board in last_failed_jobs[arch_to_job]:
if arch in last_failed_jobs:
for board in last_failed_jobs[arch]:
if not board in boards_to_build:
boards_to_build.append(board)
@ -238,24 +255,22 @@ def set_boards_to_build(build_all: bool):
def set_docs_to_build(build_doc: bool):
if not build_doc:
if "build-doc" in last_failed_jobs:
if last_failed_jobs.get("build-doc"):
build_doc = True
else:
doc_pattern = re.compile(
r"^(?:\.github\/workflows\/|docs|extmod\/ulab|(?:(?:ports\/\w+\/bindings|shared-bindings)\S+\.c|conf\.py|tools\/extract_pyi\.py|requirements-doc\.txt)$)|(?:-stubs|\.(?:md|MD|rst|RST))$"
)
doc_pattern = re.compile(PATTERN_DOCS)
github_workspace = os.environ.get("GITHUB_WORKSPACE") or ""
github_workspace = github_workspace and github_workspace + "/"
for p in changed_files:
if doc_pattern.search(p) and (
for file in changed_files:
if doc_pattern.search(file) and (
(
subprocess.run(
f"git diff -U0 $BASE_SHA...$HEAD_SHA {github_workspace + p} | grep -o -m 1 '^[+-]\/\/|'",
f"git diff -U0 $BASE_SHA...$HEAD_SHA {github_workspace + file} | grep -o -m 1 '^[+-]\/\/|'",
capture_output=True,
shell=True,
).stdout
)
if p.endswith(".c")
if file.endswith(".c")
else True
):
build_doc = True
@ -266,6 +281,25 @@ def set_docs_to_build(build_doc: bool):
set_output("build-doc", build_doc)
def set_windows_to_build(build_windows):
if not build_windows:
if last_failed_jobs.get("build-windows"):
build_windows = True
else:
for file in changed_files:
for pattern in PATTERN_WINDOWS:
if file.startswith(pattern):
build_windows = True
break
else:
continue
break
# Set the step outputs
print("Building windows:", build_windows)
set_output("build-windows", build_windows)
def check_changed_files():
if not changed_files:
print("Building all docs/boards")
@ -278,6 +312,7 @@ def check_changed_files():
def main():
build_all = check_changed_files()
set_docs_to_build(build_all)
set_windows_to_build(build_all)
set_boards_to_build(build_all)