ci max parallelism

This commit is contained in:
MicroDev 2023-02-10 19:47:51 +05:30
parent 23bb17c240
commit 17e03c4768
No known key found for this signature in database
GPG Key ID: 2C0867BE60967730
7 changed files with 244 additions and 129 deletions

View File

@ -1,6 +1,11 @@
name: 'Fetch Submodules'
inputs:
target:
description: 'The target for ci_fetch_deps'
required: false
type: string
submodules:
description: 'The submodules to cache'
required: false
@ -63,7 +68,7 @@ runs:
- name: CircuitPython dependencies
id: cp-deps
run: python tools/ci_fetch_deps.py ${{ matrix.board || github.job }}
run: python tools/ci_fetch_deps.py ${{ inputs.target || matrix.board || github.job }}
shell: bash
- name: CircuitPython version

30
.github/actions/mpy_cross/action.yml vendored Normal file
View File

@ -0,0 +1,30 @@
name: Set up mpy-cross
runs:
using: composite
steps:
- name: Download mpy-cross
id: download-mpy-cross
continue-on-error: true
uses: actions/download-artifact@v3
with:
name: mpy-cross
path: mpy-cross
- name: Make mpy-cross executable
if: steps.download-mpy-cross.outcome == 'success'
run: sudo chmod +x mpy-cross/mpy-cross
shell: bash
- name: Build mpy-cross
if: steps.download-mpy-cross.outcome == 'failure'
run: make -C mpy-cross -j2
shell: bash
- name: Upload mpy-cross
continue-on-error: true
if: steps.download-mpy-cross.outcome == 'failure'
uses: actions/upload-artifact@v3
with:
name: mpy-cross
path: mpy-cross/mpy-cross

View File

@ -25,10 +25,6 @@ jobs:
matrix:
board: ${{ fromJSON(inputs.boards) }}
steps:
- name: Dump GitHub context
run: echo "$GITHUB_CONTEXT"
env:
GITHUB_CONTEXT: ${{ toJson(github) }}
- name: Set up repository
uses: actions/checkout@v3
with:
@ -49,6 +45,9 @@ jobs:
uses: ./.github/actions/external_deps
with:
platform: ${{ inputs.platform }}
- name: Set up mpy-cross
if: steps.set-up-submodules.outputs.frozen == 'True'
uses: ./.github/actions/mpy_cross
- name: Versions
run: |
@ -63,16 +62,6 @@ jobs:
riscv64-unknown-elf-gcc --version || true
mkfs.fat --version || true
- name: Download mpy-cross
if: ${{ steps.set-up-submodules.outputs.frozen == 'True' }}
uses: actions/download-artifact@v3
with:
name: mpy-cross
path: mpy-cross
- name: Make mpy-cross executable
if: ${{ steps.set-up-submodules.outputs.frozen == 'True' }}
run: sudo chmod +x mpy-cross/mpy-cross
- name: Set up build failure matcher
run: echo "::add-matcher::$GITHUB_WORKSPACE/.github/workflows/match-build-fail.json"
- name: Build

75
.github/workflows/build-mpy-cross.yml vendored Normal file
View File

@ -0,0 +1,75 @@
name: Build mpy-cross
on:
workflow_call:
inputs:
cp-version:
required: true
type: string
jobs:
build:
runs-on: ubuntu-22.04
strategy:
fail-fast: false
matrix:
mpy-cross: ["static", "static-aarch64", "static-mingw", "static-raspbian"]
env:
CP_VERSION: ${{ inputs.cp-version }}
EX_static-mingw: static.exe
OS_static: linux-amd64
OS_static-aarch64: linux-aarch64
OS_static-mingw: windows
OS_static-raspbian: linux-raspbian
steps:
- name: Set up repository
uses: actions/checkout@v3
with:
submodules: false
fetch-depth: 1
- name: Set up python
uses: actions/setup-python@v4
with:
python-version: 3.x
- name: Set up submodules
uses: ./.github/actions/fetch_submodules
with:
target: mpy-cross
- name: Install dependencies
run: |
sudo apt-get update
sudo apt-get install -y gettext
- name: Install toolchain (aarch64)
if: matrix.mpy-cross == 'static-aarch64'
run: sudo apt-get install -y gcc-aarch64-linux-gnu
- name: Install toolchain (mingw)
if: matrix.mpy-cross == 'static-mingw'
run: sudo apt-get install -y mingw-w64
- name: Build mpy-cross.${{ matrix.mpy-cross }}
run: make -C mpy-cross -j2 -f Makefile.${{ matrix.mpy-cross }}
- name: Upload artifact
uses: actions/upload-artifact@v3
with:
name: mpy-cross.${{ env[format('EX_{0}', matrix.mpy-cross)] || matrix.mpy-cross }}
path: mpy-cross/mpy-cross.${{ env[format('EX_{0}', matrix.mpy-cross)] || matrix.mpy-cross }}
- name: Upload to S3
if: >-
(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') ||
(github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))
env:
AWS_PAGER: ''
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
run: >-
[ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp
mpy-cross/mpy-cross.${{ env[format('EX_{0}', matrix.mpy-cross)] || matrix.mpy-cross }}
s3://adafruit-circuit-python/bin/mpy-cross/$OS_${{ matrix.mpy-cross }}/mpy-cross-$CP_VERSION.${{ env[format('EX_{0}', matrix.mpy-cross)] || matrix.mpy-cross }}
--no-progress --region us-east-1

View File

@ -17,7 +17,7 @@ concurrency:
cancel-in-progress: true
jobs:
test:
scheduler:
runs-on: ubuntu-20.04
outputs:
build-doc: ${{ steps.set-matrix.outputs.build-doc }}
@ -54,89 +54,8 @@ jobs:
version: true
- name: Install dependencies
run: |
sudo apt-get update
sudo apt-get install -y eatmydata
sudo eatmydata apt-get install -y gettext gcc-aarch64-linux-gnu mingw-w64
pip install -r requirements-dev.txt
- name: Versions
run: |
gcc --version
python3 --version
- name: Build mpy-cross
run: make -C mpy-cross -j2
- uses: actions/upload-artifact@v3
with:
name: mpy-cross
path: mpy-cross/mpy-cross
- name: Build unix port
run: make -C ports/unix VARIANT=coverage -j2
- name: Test all
run: ./run-tests.py -j2
working-directory: tests
- name: Print failure info
run: ./run-tests.py -j2 --print-failures
if: failure()
working-directory: tests
- name: Native Tests
run: ./run-tests.py -j2 --emit native
working-directory: tests
- name: mpy Tests
run: ./run-tests.py -j2 --via-mpy -d basics float micropython
working-directory: tests
- name: Native mpy Tests
run: ./run-tests.py -j2 --via-mpy --emit native -d basics float micropython
working-directory: tests
- name: Build native modules
run: |
make -C examples/natmod/features1
make -C examples/natmod/features2
make -C examples/natmod/btree
make -C examples/natmod/framebuf
make -C examples/natmod/uheapq
make -C examples/natmod/urandom
make -C examples/natmod/ure
make -C examples/natmod/uzlib
- name: Test native modules
run: ./run-natmodtests.py extmod/{btree*,framebuf*,uheapq*,ure*,uzlib*}.py
working-directory: tests
- name: Build mpy-cross.static-aarch64
run: make -C mpy-cross -j2 -f Makefile.static-aarch64
- uses: actions/upload-artifact@v3
with:
name: mpy-cross.static-aarch64
path: mpy-cross/mpy-cross.static-aarch64
- name: Build mpy-cross.static-raspbian
run: make -C mpy-cross -j2 -f Makefile.static-raspbian
- uses: actions/upload-artifact@v3
with:
name: mpy-cross.static-raspbian
path: mpy-cross/mpy-cross.static-raspbian
- name: Build mpy-cross.static
run: make -C mpy-cross -j2 -f Makefile.static
- uses: actions/upload-artifact@v3
with:
name: mpy-cross.static-amd64-linux
path: mpy-cross/mpy-cross.static
- name: Build mpy-cross.static-mingw
run: make -C mpy-cross -j2 -f Makefile.static-mingw
- uses: actions/upload-artifact@v3
with:
name: mpy-cross.static-x64-windows
path: mpy-cross/mpy-cross.static.exe
- name: Upload to S3
if: >-
(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') ||
(github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))
env:
AWS_PAGER: ''
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
run: |
pip install awscli
[ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross/mpy-cross.static-aarch64 s3://adafruit-circuit-python/bin/mpy-cross/mpy-cross.static-aarch64-${{ env.CP_VERSION }} --no-progress --region us-east-1
[ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross/mpy-cross.static-raspbian s3://adafruit-circuit-python/bin/mpy-cross/mpy-cross.static-raspbian-${{ env.CP_VERSION }} --no-progress --region us-east-1
[ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross/mpy-cross.static s3://adafruit-circuit-python/bin/mpy-cross/mpy-cross.static-amd64-linux-${{ env.CP_VERSION }} --no-progress --region us-east-1
[ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross/mpy-cross.static.exe s3://adafruit-circuit-python/bin/mpy-cross/mpy-cross.static-x64-windows-${{ env.CP_VERSION }}.exe --no-progress --region us-east-1
sudo apt-get install -y gettext
pip install requests requests-cache sh
# Disabled: Needs to be updated
# - name: Get last commit with checks
# id: get-last-commit-with-checks
@ -148,6 +67,8 @@ jobs:
# PULL: ${{ github.event.number }}
# GITHUB_TOKEN: ${{ github.token }}
# EXCLUDE_COMMIT: ${{ github.event.after }}
- name: Set up mpy-cross
uses: ./.github/actions/mpy_cross
- name: Set head sha
if: github.event_name == 'pull_request'
run: echo "HEAD_SHA=$(git show -s --format=%s $GITHUB_SHA | grep -o -P "(?<=Merge ).*(?= into)")" >> $GITHUB_ENV
@ -170,13 +91,28 @@ jobs:
CHANGED_FILES: ${{ steps.get-changes.outputs.changed_files }}
LAST_FAILED_JOBS: ${{ steps.get-last-commit-with-checks.outputs.check_runs }}
tests:
needs: scheduler
if: needs.scheduler.outputs.build-boards == 'True'
uses: ./.github/workflows/run-tests.yml
with:
cp-version: ${{ needs.scheduler.outputs.cp-version }}
mpy-cross:
needs: scheduler
if: needs.scheduler.outputs.build-boards == 'True'
uses: ./.github/workflows/build-mpy-cross.yml
with:
cp-version: ${{ needs.scheduler.outputs.cp-version }}
mpy-cross-mac:
runs-on: macos-11
needs: test
if: ${{ needs.test.outputs.build-boards == 'True' }}
needs: scheduler
if: ${{ needs.scheduler.outputs.build-boards == 'True' }}
env:
CP_VERSION: ${{ needs.test.outputs.cp-version }}
CP_VERSION: ${{ needs.scheduler.outputs.cp-version }}
steps:
- name: Set up repository
uses: actions/checkout@v3
@ -213,23 +149,25 @@ jobs:
name: mpy-cross-macos-11-universal
path: mpy-cross-macos-universal
- name: Upload mpy-cross build to S3
run: |
[ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross-macos-universal s3://adafruit-circuit-python/bin/mpy-cross/mpy-cross-macos-11-${{ env.CP_VERSION }}-universal --no-progress --region us-east-1
[ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross/mpy-cross-arm64 s3://adafruit-circuit-python/bin/mpy-cross/mpy-cross-macos-11-${{ env.CP_VERSION }}-arm64 --no-progress --region us-east-1
[ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross/mpy-cross s3://adafruit-circuit-python/bin/mpy-cross/mpy-cross-macos-11-${{ env.CP_VERSION }}-x64 --no-progress --region us-east-1
if: >-
(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') ||
(github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))
env:
AWS_PAGER: ''
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
if: (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') || (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))
run: |
[ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross-macos-universal s3://adafruit-circuit-python/bin/mpy-cross/mpy-cross-macos-11-${{ env.CP_VERSION }}-universal --no-progress --region us-east-1
[ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross/mpy-cross-arm64 s3://adafruit-circuit-python/bin/mpy-cross/mpy-cross-macos-11-${{ env.CP_VERSION }}-arm64 --no-progress --region us-east-1
[ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross/mpy-cross s3://adafruit-circuit-python/bin/mpy-cross/mpy-cross-macos-11-${{ env.CP_VERSION }}-x64 --no-progress --region us-east-1
build-doc:
runs-on: ubuntu-22.04
needs: test
if: ${{ needs.test.outputs.build-doc == 'True' }}
needs: scheduler
if: ${{ needs.scheduler.outputs.build-doc == 'True' }}
env:
CP_VERSION: ${{ needs.test.outputs.cp-version }}
CP_VERSION: ${{ needs.scheduler.outputs.cp-version }}
steps:
- name: Set up repository
uses: actions/checkout@v3
@ -276,7 +214,6 @@ jobs:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
run: |
pip install awscli
zip -9r circuitpython-stubs.zip circuitpython-stubs
[ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp circuitpython-stubs/dist/*.tar.gz s3://adafruit-circuit-python/bin/stubs/circuitpython-stubs-${{ env.CP_VERSION }}.zip --no-progress --region us-east-1
- name: Upload stubs to PyPi
@ -291,40 +228,40 @@ jobs:
aarch:
needs: test
if: ${{ needs.test.outputs.boards-aarch != '[]' }}
needs: [scheduler, mpy-cross, tests]
if: ${{ needs.scheduler.outputs.boards-aarch != '[]' }}
uses: ./.github/workflows/build-boards.yml
with:
platform: aarch
boards: ${{ needs.test.outputs.boards-aarch }}
cp-version: ${{ needs.test.outputs.cp-version }}
boards: ${{ needs.scheduler.outputs.boards-aarch }}
cp-version: ${{ needs.scheduler.outputs.cp-version }}
arm:
needs: test
if: ${{ needs.test.outputs.boards-arm != '[]' }}
needs: [scheduler, mpy-cross, tests]
if: ${{ needs.scheduler.outputs.boards-arm != '[]' }}
uses: ./.github/workflows/build-boards.yml
with:
platform: arm
boards: ${{ needs.test.outputs.boards-arm }}
cp-version: ${{ needs.test.outputs.cp-version }}
boards: ${{ needs.scheduler.outputs.boards-arm }}
cp-version: ${{ needs.scheduler.outputs.cp-version }}
espressif:
needs: test
if: ${{ needs.test.outputs.boards-espressif != '[]' }}
needs: [scheduler, mpy-cross, tests]
if: ${{ needs.scheduler.outputs.boards-espressif != '[]' }}
uses: ./.github/workflows/build-boards.yml
with:
platform: espressif
boards: ${{ needs.test.outputs.boards-espressif }}
cp-version: ${{ needs.test.outputs.cp-version }}
boards: ${{ needs.scheduler.outputs.boards-espressif }}
cp-version: ${{ needs.scheduler.outputs.cp-version }}
riscv:
needs: test
if: ${{ needs.test.outputs.boards-riscv != '[]' }}
needs: [scheduler, mpy-cross, tests]
if: ${{ needs.scheduler.outputs.boards-riscv != '[]' }}
uses: ./.github/workflows/build-boards.yml
with:
platform: riscv
boards: ${{ needs.test.outputs.boards-riscv }}
cp-version: ${{ needs.test.outputs.cp-version }}
boards: ${{ needs.scheduler.outputs.boards-riscv }}
cp-version: ${{ needs.scheduler.outputs.cp-version }}

76
.github/workflows/run-tests.yml vendored Normal file
View File

@ -0,0 +1,76 @@
name: Run tests
on:
workflow_call:
inputs:
cp-version:
required: true
type: string
jobs:
run:
runs-on: ubuntu-20.04
strategy:
fail-fast: false
matrix:
test: [all, mpy, native, native_mpy]
env:
CP_VERSION: ${{ inputs.cp-version }}
MICROPY_CPYTHON3: python3.8
MICROPY_MICROPYTHON: ../ports/unix/micropython-coverage
TEST_all:
TEST_mpy: --via-mpy -d basics float micropython
TEST_native: --emit native
TEST_native_mpy: --via-mpy --emit native -d basics float micropython
steps:
- name: Set up repository
uses: actions/checkout@v3
with:
submodules: false
fetch-depth: 1
- name: Set up python
uses: actions/setup-python@v4
with:
python-version: 3.x
- name: Set up submodules
uses: ./.github/actions/fetch_submodules
with:
target: tests
- name: Install dependencies
run: |
sudo apt-get update
sudo apt-get install -y gettext
- name: Set up mpy-cross
uses: ./.github/actions/mpy_cross
- name: Build unix port
run: make -C ports/unix VARIANT=coverage -j2
- name: Test all
run: ./run-tests.py -j2 $TEST_${{ matrix.test }}
working-directory: tests
- name: Print failure info
run: ./run-tests.py -j2 --print-failures
if: failure()
working-directory: tests
- name: Set up native modules
if: matrix.test == 'all'
run: pip install -r requirements-dev.txt
- name: Build native modules
if: matrix.test == 'all'
run: |
make -C examples/natmod/features1
make -C examples/natmod/features2
make -C examples/natmod/btree
make -C examples/natmod/framebuf
make -C examples/natmod/uheapq
make -C examples/natmod/urandom
make -C examples/natmod/ure
make -C examples/natmod/uzlib
- name: Test native modules
if: matrix.test == 'all'
run: ./run-natmodtests.py extmod/{btree*,framebuf*,uheapq*,ure*,uzlib*}.py
working-directory: tests

View File

@ -73,12 +73,15 @@ def main():
print("Target:", TARGET)
if TARGET == "test":
if TARGET == "scheduler":
# submodules = ["tools/"]
submodules = ["extmod/ulab", "lib/", "tools/"]
elif TARGET == "tests":
submodules = ["extmod/ulab", "lib/", "tools/"]
elif TARGET == "build-doc":
# used in .readthedocs.yml to generate RTD
submodules = ["extmod/ulab", "frozen/"]
elif TARGET == "mpy-cross-mac":
elif TARGET == "mpy-cross" or TARGET == "mpy-cross-mac":
submodules = ["tools/"] # for huffman
elif TARGET == "windows":
# This builds one board from a number of ports so fill out a bunch of submodules