Merge pull request #7453 from MicroDev1/ci

Multiple CI Enhancements
This commit is contained in:
Dan Halbert 2023-01-19 21:13:11 -05:00 committed by GitHub
commit 9c066825a7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 462 additions and 330 deletions

View File

@ -16,46 +16,67 @@ concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
env:
CACHE_SUBMODULES: "['extmod/ulab', 'lib/', 'tools/']"
jobs:
test:
runs-on: ubuntu-20.04
outputs:
build-doc: ${{ steps.set-matrix.outputs.build-doc }}
boards-arm: ${{ steps.set-matrix.outputs.boards-arm }}
boards-riscv: ${{ steps.set-matrix.outputs.boards-riscv }}
boards-espressif: ${{ steps.set-matrix.outputs.boards-espressif }}
boards-aarch: ${{ steps.set-matrix.outputs.boards-aarch }}
boards-arm: ${{ steps.set-matrix.outputs.boards-arm }}
boards-espressif: ${{ steps.set-matrix.outputs.boards-espressif }}
boards-riscv: ${{ steps.set-matrix.outputs.boards-riscv }}
cp-version: ${{ steps.cp-version.outputs.cp-version }}
steps:
- name: Dump GitHub context
run: echo "$GITHUB_CONTEXT"
env:
GITHUB_CONTEXT: ${{ toJson(github) }}
- uses: actions/checkout@v3
- name: Set up repository
uses: actions/checkout@v3
with:
submodules: false
fetch-depth: 1
- name: Set up Python 3
- name: Set up python
uses: actions/setup-python@v4
with:
python-version: "3.x"
- name: Get CP deps
run: python tools/ci_fetch_deps.py test ${{ github.sha }}
- name: CircuitPython version
- name: Duplicate USB VID/PID check
run: python3 -u -m tools.ci_check_duplicate_usb_vid_pid
- name: Create submodule status
run: git submodule status ${{ join(fromJSON(env.CACHE_SUBMODULES), ' ') }} >> submodule_status
- name: Cache submodules
uses: actions/cache@v3
with:
path: ".git/modules/\n${{ join(fromJSON(env.CACHE_SUBMODULES), '\n') }}"
key: submodules-common-${{ hashFiles('submodule_status') }}
- name: CircuitPython dependencies
run: |
tools/describe || git log --parents HEAD~4..
echo >>$GITHUB_ENV CP_VERSION=$(tools/describe)
python tools/ci_fetch_deps.py ${{ github.job }}
echo "::group::Fetch history and tags"
git fetch --no-recurse-submodules --shallow-since="2021-07-01" --tags https://github.com/adafruit/circuitpython HEAD
git fetch --no-recurse-submodules --shallow-since="2021-07-01" origin $GITHUB_SHA
git repack -d
echo "::endgroup::"
- name: CircuitPython version
id: cp-version
run: |
CP_VERSION=$(tools/describe)
echo "$CP_VERSION"
echo "CP_VERSION=$CP_VERSION" >> $GITHUB_ENV
echo "cp-version=$CP_VERSION" >> $GITHUB_OUTPUT
- name: Install dependencies
run: |
sudo apt-get update
sudo apt-get install -y eatmydata
sudo eatmydata apt-get install -y gettext gcc-aarch64-linux-gnu mingw-w64
pip install -r requirements-ci.txt -r requirements-dev.txt
pip install -r requirements-dev.txt
- name: Versions
run: |
gcc --version
python3 --version
- name: Duplicate USB VID/PID Check
run: python3 -u -m tools.ci_check_duplicate_usb_vid_pid
- name: Build mpy-cross
run: make -C mpy-cross -j2
- name: Build unix port
@ -114,13 +135,16 @@ jobs:
with:
name: mpy-cross.static-x64-windows
path: mpy-cross/mpy-cross.static.exe
- name: Upload mpy-cross builds to S3
if: (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') || (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))
- name: Upload to S3
if: >-
(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') ||
(github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))
env:
AWS_PAGER: ''
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
run: |
pip install awscli
[ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross/mpy-cross.static-aarch64 s3://adafruit-circuit-python/bin/mpy-cross/mpy-cross.static-aarch64-${{ env.CP_VERSION }} --no-progress --region us-east-1
[ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross/mpy-cross.static-raspbian s3://adafruit-circuit-python/bin/mpy-cross/mpy-cross.static-raspbian-${{ env.CP_VERSION }} --no-progress --region us-east-1
[ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross/mpy-cross.static s3://adafruit-circuit-python/bin/mpy-cross/mpy-cross.static-amd64-linux-${{ env.CP_VERSION }} --no-progress --region us-east-1
@ -160,29 +184,35 @@ jobs:
mpy-cross-mac:
runs-on: macos-11
needs: test
if: >-
needs.test.outputs.boards-aarch != '[]' ||
needs.test.outputs.boards-arm != '[]' ||
needs.test.outputs.boards-espressif != '[]' ||
needs.test.outputs.boards-riscv != '[]'
env:
CP_VERSION: ${{ needs.test.outputs.cp-version }}
steps:
- name: Dump GitHub context
env:
GITHUB_CONTEXT: ${{ toJson(github) }}
run: echo "$GITHUB_CONTEXT"
- uses: actions/checkout@v3
- name: Set up repository
uses: actions/checkout@v3
with:
submodules: false
fetch-depth: 1
- name: Set up Python 3
- name: Set up python
uses: actions/setup-python@v4
with:
python-version: "3.10"
- name: Get CP deps
run: python tools/ci_fetch_deps.py mpy-cross-mac ${{ github.sha }}
python-version: "3.x"
- name: Create submodule status
run: git submodule status ${{ join(fromJSON(env.CACHE_SUBMODULES), ' ') }} >> submodule_status
- name: Restore submodules
uses: actions/cache/restore@v3
with:
path: ".git/modules/\n${{ join(fromJSON(env.CACHE_SUBMODULES), '\n') }}"
key: submodules-common-${{ hashFiles('submodule_status') }}
- name: CircuitPython dependencies
run: python tools/ci_fetch_deps.py ${{ github.job }}
- name: CircuitPython version
run: |
tools/describe || git log --parents HEAD~4..
echo >>$GITHUB_ENV CP_VERSION=$(tools/describe)
- name: Install dependencies
run: |
brew install gettext
echo >>$GITHUB_PATH /usr/local/opt/gettext/bin
run: tools/describe
- name: Versions
run: |
gcc --version
@ -222,27 +252,35 @@ jobs:
runs-on: ubuntu-20.04
needs: test
if: ${{ needs.test.outputs.build-doc == 'True' }}
env:
CP_VERSION: ${{ needs.test.outputs.cp-version }}
steps:
- uses: actions/checkout@v3
- name: Set up repository
uses: actions/checkout@v3
with:
submodules: false
fetch-depth: 1
- name: Get CP deps
run: python tools/ci_fetch_deps.py docs ${{ github.sha }}
- name: Create submodule status
run: git submodule status ${{ join(fromJSON(env.CACHE_SUBMODULES), ' ') }} >> submodule_status
- name: Restore submodules
uses: actions/cache/restore@v3
with:
path: ".git/modules/\n${{ join(fromJSON(env.CACHE_SUBMODULES), '\n') }}"
key: submodules-common-${{ hashFiles('submodule_status') }}
- name: CircuitPython dependencies
run: python tools/ci_fetch_deps.py ${{ github.job }}
- name: CircuitPython version
run: |
tools/describe || git log --parents HEAD~4..
echo >>$GITHUB_ENV CP_VERSION=$(tools/describe)
- name: Set up Python 3
run: tools/describe
- name: Set up python
uses: actions/setup-python@v4
with:
python-version: "3.10"
python-version: "3.x"
- name: Install dependencies
run: |
sudo apt-get update
sudo apt-get install -y eatmydata
sudo eatmydata apt-get install -y latexmk librsvg2-bin texlive-fonts-recommended texlive-latex-recommended texlive-latex-extra
pip install -r requirements-ci.txt -r requirements-doc.txt
pip install -r requirements-doc.txt
- name: Build and Validate Stubs
run: make check-stubs -j2
- uses: actions/upload-artifact@v3
@ -262,13 +300,16 @@ jobs:
with:
name: docs
path: _build/latex
- name: Upload stubs to S3
if: (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') || (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))
- name: Upload to S3
if: >-
(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') ||
(github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))
env:
AWS_PAGER: ''
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
run: |
pip install awscli
zip -9r circuitpython-stubs.zip circuitpython-stubs
[ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp circuitpython-stubs/dist/*.tar.gz s3://adafruit-circuit-python/bin/stubs/circuitpython-stubs-${{ env.CP_VERSION }}.zip --no-progress --region us-east-1
- name: Upload stubs to PyPi
@ -282,225 +323,40 @@ jobs:
[ -z "$TWINE_USERNAME" ] || twine upload circuitpython-stubs/dist/*
build-arm:
runs-on: ubuntu-20.04
needs: test
strategy:
fail-fast: false
matrix:
board: ${{ fromJSON(needs.test.outputs.boards-arm) }}
if: ${{ needs.test.outputs.boards-arm != '[]' }}
steps:
- name: Set up Python 3
uses: actions/setup-python@v4
with:
python-version: "3.10"
- uses: actions/checkout@v3
with:
submodules: false
fetch-depth: 1
- name: Get CP deps
run: python tools/ci_fetch_deps.py ${{ matrix.board }} ${{ github.sha }}
- uses: carlosperate/arm-none-eabi-gcc-action@v1
with:
release: '10-2020-q4'
- name: Install dependencies
run: |
sudo apt-get install -y gettext
pip install -r requirements-ci.txt -r requirements-dev.txt
- name: Versions
run: |
gcc --version
arm-none-eabi-gcc --version
python3 --version
- name: mpy-cross
run: make -C mpy-cross -j2
- name: Setup build failure matcher
run: echo "::add-matcher::$GITHUB_WORKSPACE/.github/workflows/match-build-fail.json"
- name: build
run: python3 -u build_release_files.py
working-directory: tools
env:
BOARDS: ${{ matrix.board }}
- uses: actions/upload-artifact@v3
with:
name: ${{ matrix.board }}
path: bin/${{ matrix.board }}
- name: Upload to S3
run: "[ -z \"$AWS_ACCESS_KEY_ID\" ] || aws s3 cp bin/ s3://adafruit-circuit-python/bin/ --recursive --no-progress --region us-east-1"
env:
AWS_PAGER: ''
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
if: (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') || (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))
build-riscv:
runs-on: ubuntu-20.04
needs: test
strategy:
fail-fast: false
matrix:
board: ${{ fromJSON(needs.test.outputs.boards-riscv) }}
if: ${{ needs.test.outputs.boards-riscv != '[]' }}
steps:
- name: Set up Python 3
uses: actions/setup-python@v4
with:
python-version: "3.10"
- uses: actions/checkout@v3
with:
submodules: false
fetch-depth: 1
- name: Get CP deps
run: python tools/ci_fetch_deps.py ${{ matrix.board }} ${{ github.sha }}
- name: Install dependencies
run: |
sudo apt-get install -y gettext
pip install -r requirements-ci.txt -r requirements-dev.txt
wget https://static.dev.sifive.com/dev-tools/riscv64-unknown-elf-gcc-8.3.0-2019.08.0-x86_64-linux-centos6.tar.gz
sudo tar -C /usr --strip-components=1 -xaf riscv64-unknown-elf-gcc-8.3.0-2019.08.0-x86_64-linux-centos6.tar.gz
- name: Versions
run: |
gcc --version
riscv64-unknown-elf-gcc --version
python3 --version
- name: mpy-cross
run: make -C mpy-cross -j2
- name: Setup build failure matcher
run: echo "::add-matcher::$GITHUB_WORKSPACE/.github/workflows/match-build-fail.json"
- name: build
run: python3 -u build_release_files.py
working-directory: tools
env:
BOARDS: ${{ matrix.board }}
- uses: actions/upload-artifact@v3
with:
name: ${{ matrix.board }}
path: bin/${{ matrix.board }}
- name: Upload to S3
run: "[ -z \"$AWS_ACCESS_KEY_ID\" ] || aws s3 cp bin/ s3://adafruit-circuit-python/bin/ --recursive --no-progress --region us-east-1"
env:
AWS_PAGER: ''
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
if: (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') || (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))
build-espressif:
runs-on: ubuntu-20.04
needs: test
strategy:
fail-fast: false
matrix:
board: ${{ fromJSON(needs.test.outputs.boards-espressif) }}
if: ${{ needs.test.outputs.boards-espressif != '[]' }}
steps:
- name: Set up Python 3
id: py3
uses: actions/setup-python@v4
with:
python-version: "3.10"
- uses: actions/checkout@v3
with:
submodules: false
fetch-depth: 1
- name: Get CP deps
run: python tools/ci_fetch_deps.py ${{ matrix.board }} ${{ github.sha }}
- name: CircuitPython version
run: |
tools/describe || git log --parents HEAD~4..
echo >>$GITHUB_ENV CP_VERSION=$(tools/describe)
- uses: actions/cache@v3
name: Fetch IDF tool cache
id: idf-cache
with:
path: ${{ github.workspace }}/.idf_tools
key: ${{ runner.os }}-idf-tools-${{ hashFiles('.git/modules/ports/espressif/esp-idf/HEAD') }}-${{ steps.py3.outputs.python-path }}-20220404
- name: Clone IDF submodules
run: git submodule update --init $IDF_PATH
env:
IDF_PATH: ${{ github.workspace }}/ports/espressif/esp-idf
- name: Install IDF tools
run: |
echo "Installing ESP-IDF tools"
$IDF_PATH/tools/idf_tools.py --non-interactive install required
$IDF_PATH/tools/idf_tools.py --non-interactive install cmake
echo "Installing Python environment and packages"
$IDF_PATH/tools/idf_tools.py --non-interactive install-python-env
rm -rf $IDF_TOOLS_PATH/dist
env:
IDF_PATH: ${{ github.workspace }}/ports/espressif/esp-idf
IDF_TOOLS_PATH: ${{ github.workspace }}/.idf_tools
- name: Install dependencies
run: |
source $IDF_PATH/export.sh
sudo apt-get install -y gettext ninja-build
pip install -r requirements-ci.txt -r requirements-dev.txt
env:
IDF_PATH: ${{ github.workspace }}/ports/espressif/esp-idf
IDF_TOOLS_PATH: ${{ github.workspace }}/.idf_tools
- name: Versions
run: |
source $IDF_PATH/export.sh
gcc --version
python3 --version
ninja --version
cmake --version
shell: bash
env:
IDF_PATH: ${{ github.workspace }}/ports/espressif/esp-idf
IDF_TOOLS_PATH: ${{ github.workspace }}/.idf_tools
- name: mpy-cross
run: make -C mpy-cross -j2
- name: Setup build failure matcher
run: echo "::add-matcher::$GITHUB_WORKSPACE/.github/workflows/match-build-fail.json"
- name: build
run: |
source $IDF_PATH/export.sh
python3 -u build_release_files.py
working-directory: tools
shell: bash
env:
IDF_PATH: ${{ github.workspace }}/ports/espressif/esp-idf
IDF_TOOLS_PATH: ${{ github.workspace }}/.idf_tools
BOARDS: ${{ matrix.board }}
- uses: actions/upload-artifact@v3
with:
name: ${{ matrix.board }}
path: bin/${{ matrix.board }}
- name: Upload to S3
run: "[ -z \"$AWS_ACCESS_KEY_ID\" ] || aws s3 cp bin/ s3://adafruit-circuit-python/bin/ --recursive --no-progress --region us-east-1"
env:
AWS_PAGER: ''
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
if: (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') || (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))
build-aarch:
runs-on: ubuntu-20.04
needs: test
if: ${{ needs.test.outputs.boards-aarch != '[]' }}
env:
CP_VERSION: ${{ needs.test.outputs.cp-version }}
strategy:
fail-fast: false
matrix:
board: ${{ fromJSON(needs.test.outputs.boards-aarch) }}
if: ${{ needs.test.outputs.boards-aarch != '[]' }}
steps:
- name: Set up Python 3
uses: actions/setup-python@v4
with:
python-version: "3.10"
- uses: actions/checkout@v3
- name: Set up repository
uses: actions/checkout@v3
with:
submodules: false
fetch-depth: 1
- name: Get CP deps
run: python tools/ci_fetch_deps.py ${{ matrix.board }} ${{ github.sha }}
- name: Set up python
uses: actions/setup-python@v4
with:
python-version: "3.x"
- name: Create submodule status
run: git submodule status ${{ join(fromJSON(env.CACHE_SUBMODULES), ' ') }} >> submodule_status
- name: Restore submodules
uses: actions/cache/restore@v3
with:
path: ".git/modules/\n${{ join(fromJSON(env.CACHE_SUBMODULES), '\n') }}"
key: submodules-common-${{ hashFiles('submodule_status') }}
- name: CircuitPython dependencies
id: cp-deps
run: python tools/ci_fetch_deps.py ${{ matrix.board }}
- name: Install dependencies
run: |
sudo apt-get install -y gettext mtools
pip install -r requirements-ci.txt -r requirements-dev.txt
pip install -r requirements-dev.txt
wget --no-verbose https://adafruit-circuit-python.s3.amazonaws.com/gcc-arm-10.3-2021.07-x86_64-aarch64-none-elf.tar.xz
sudo tar -C /usr --strip-components=1 -xaf gcc-arm-10.3-2021.07-x86_64-aarch64-none-elf.tar.xz
- uses: carlosperate/arm-none-eabi-gcc-action@v1
@ -522,11 +378,12 @@ jobs:
arm-none-eabi-gcc --version
python3 --version
mkfs.fat --version || true
- name: mpy-cross
- name: Build mpy-cross
if: ${{ steps.cp-deps.outputs.frozen_tags == 'True' }}
run: make -C mpy-cross -j2
- name: Setup build failure matcher
run: echo "::add-matcher::$GITHUB_WORKSPACE/.github/workflows/match-build-fail.json"
- name: build
- name: Build
run: python3 -u build_release_files.py
working-directory: tools
env:
@ -536,9 +393,256 @@ jobs:
name: ${{ matrix.board }}
path: bin/${{ matrix.board }}
- name: Upload to S3
run: "[ -z \"$AWS_ACCESS_KEY_ID\" ] || aws s3 cp bin/ s3://adafruit-circuit-python/bin/ --recursive --no-progress --region us-east-1"
if: >-
(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') ||
(github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))
run: |
pip install awscli
[ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp bin/ s3://adafruit-circuit-python/bin/ --recursive --no-progress --region us-east-1
env:
AWS_PAGER: ''
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
build-arm:
runs-on: ubuntu-20.04
needs: test
if: ${{ needs.test.outputs.boards-arm != '[]' }}
env:
CP_VERSION: ${{ needs.test.outputs.cp-version }}
strategy:
fail-fast: false
matrix:
board: ${{ fromJSON(needs.test.outputs.boards-arm) }}
steps:
- name: Set up repository
uses: actions/checkout@v3
with:
submodules: false
fetch-depth: 1
- name: Set up python
uses: actions/setup-python@v4
with:
python-version: "3.x"
- name: Create submodule status
run: git submodule status ${{ join(fromJSON(env.CACHE_SUBMODULES), ' ') }} >> submodule_status
- name: Restore submodules
uses: actions/cache/restore@v3
with:
path: ".git/modules/\n${{ join(fromJSON(env.CACHE_SUBMODULES), '\n') }}"
key: submodules-common-${{ hashFiles('submodule_status') }}
- name: CircuitPython dependencies
id: cp-deps
run: python tools/ci_fetch_deps.py ${{ matrix.board }}
- uses: carlosperate/arm-none-eabi-gcc-action@v1
with:
release: '10-2020-q4'
- name: Install dependencies
run: |
sudo apt-get install -y gettext
pip install -r requirements-dev.txt
- name: Versions
run: |
gcc --version
arm-none-eabi-gcc --version
python3 --version
- name: Build mpy-cross
if: ${{ steps.cp-deps.outputs.frozen_tags == 'True' }}
run: make -C mpy-cross -j2
- name: Setup build failure matcher
run: echo "::add-matcher::$GITHUB_WORKSPACE/.github/workflows/match-build-fail.json"
- name: Build
run: python3 -u build_release_files.py
working-directory: tools
env:
BOARDS: ${{ matrix.board }}
- uses: actions/upload-artifact@v3
with:
name: ${{ matrix.board }}
path: bin/${{ matrix.board }}
- name: Upload to S3
if: >-
(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') ||
(github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))
run: |
pip install awscli
[ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp bin/ s3://adafruit-circuit-python/bin/ --recursive --no-progress --region us-east-1
env:
AWS_PAGER: ''
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
build-espressif:
runs-on: ubuntu-20.04
needs: test
if: ${{ needs.test.outputs.boards-espressif != '[]' }}
env:
CP_VERSION: ${{ needs.test.outputs.cp-version }}
IDF_PATH: ${{ github.workspace }}/ports/espressif/esp-idf
IDF_TOOLS_PATH: ${{ github.workspace }}/.idf_tools
strategy:
fail-fast: false
matrix:
board: ${{ fromJSON(needs.test.outputs.boards-espressif) }}
steps:
- name: Set up repository
uses: actions/checkout@v3
with:
submodules: false
fetch-depth: 1
- name: Set up python
id: setup-python
uses: actions/setup-python@v4
with:
python-version: "3.10"
- name: Create submodule status
run: git submodule status ${{ join(fromJSON(env.CACHE_SUBMODULES), ' ') }} >> submodule_status
- name: Restore submodules
uses: actions/cache/restore@v3
with:
path: ".git/modules/\n${{ join(fromJSON(env.CACHE_SUBMODULES), '\n') }}"
key: submodules-common-${{ hashFiles('submodule_status') }}
- name: Get IDF commit
id: idf-commit
run: |
COMMIT=$(git submodule status ports/espressif/esp-idf | grep -o -P '(?<=^-).*(?= )')
echo "$COMMIT"
echo "commit=$COMMIT" >> $GITHUB_OUTPUT
- name: Cache IDF submodules
uses: actions/cache@v3
with:
path: |
.git/modules/ports/espressif/esp-idf
ports/espressif/esp-idf
key: submodules-idf-${{ steps.idf-commit.outputs.commit }}
- name: CircuitPython dependencies
id: cp-deps
run: python tools/ci_fetch_deps.py ${{ matrix.board }}
- name: CircuitPython version
run: tools/describe
- name: Cache IDF tools
uses: actions/cache@v3
with:
path: ${{ env.IDF_TOOLS_PATH }}
key: ${{ runner.os }}-Python-${{ steps.setup-python.outputs.python-version }}-tools-idf-${{ steps.idf-commit.outputs.commit }}
- name: Clone IDF submodules
run: git submodule update --init --depth=1
working-directory: ${{ env.IDF_PATH }}
- name: Install IDF tools
run: |
echo "Installing ESP-IDF tools"
$IDF_PATH/tools/idf_tools.py --non-interactive install required
$IDF_PATH/tools/idf_tools.py --non-interactive install cmake
echo "Installing Python environment and packages"
$IDF_PATH/tools/idf_tools.py --non-interactive install-python-env
rm -rf $IDF_TOOLS_PATH/dist
- name: Install dependencies
run: |
source $IDF_PATH/export.sh
sudo apt-get install -y gettext ninja-build
pip install -r requirements-dev.txt
- name: Versions
run: |
source $IDF_PATH/export.sh
gcc --version
python3 --version
ninja --version
cmake --version
- name: Build mpy-cross
if: ${{ steps.cp-deps.outputs.frozen_tags == 'True' }}
run: make -C mpy-cross -j2
- name: Setup build failure matcher
run: echo "::add-matcher::$GITHUB_WORKSPACE/.github/workflows/match-build-fail.json"
- name: Build
run: |
source $IDF_PATH/export.sh
python3 -u build_release_files.py
working-directory: tools
env:
BOARDS: ${{ matrix.board }}
- uses: actions/upload-artifact@v3
with:
name: ${{ matrix.board }}
path: bin/${{ matrix.board }}
- name: Upload to S3
if: >-
(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') ||
(github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))
run: |
pip install awscli
[ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp bin/ s3://adafruit-circuit-python/bin/ --recursive --no-progress --region us-east-1
env:
AWS_PAGER: ''
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
build-riscv:
runs-on: ubuntu-20.04
needs: test
if: ${{ needs.test.outputs.boards-riscv != '[]' }}
env:
CP_VERSION: ${{ needs.test.outputs.cp-version }}
strategy:
fail-fast: false
matrix:
board: ${{ fromJSON(needs.test.outputs.boards-riscv) }}
steps:
- name: Set up repository
uses: actions/checkout@v3
with:
submodules: false
fetch-depth: 1
- name: Set up python
uses: actions/setup-python@v4
with:
python-version: "3.x"
- name: Create submodule status
run: git submodule status ${{ join(fromJSON(env.CACHE_SUBMODULES), ' ') }} >> submodule_status
- name: Restore submodules
uses: actions/cache/restore@v3
with:
path: ".git/modules/\n${{ join(fromJSON(env.CACHE_SUBMODULES), '\n') }}"
key: submodules-common-${{ hashFiles('submodule_status') }}
- name: CircuitPython dependencies
id: cp-deps
run: python tools/ci_fetch_deps.py ${{ matrix.board }}
- name: Install dependencies
run: |
sudo apt-get install -y gettext
pip install -r requirements-dev.txt
wget https://static.dev.sifive.com/dev-tools/riscv64-unknown-elf-gcc-8.3.0-2019.08.0-x86_64-linux-centos6.tar.gz
sudo tar -C /usr --strip-components=1 -xaf riscv64-unknown-elf-gcc-8.3.0-2019.08.0-x86_64-linux-centos6.tar.gz
- name: Versions
run: |
gcc --version
riscv64-unknown-elf-gcc --version
python3 --version
- name: Build mpy-cross
if: ${{ steps.cp-deps.outputs.frozen_tags == 'True' }}
run: make -C mpy-cross -j2
- name: Setup build failure matcher
run: echo "::add-matcher::$GITHUB_WORKSPACE/.github/workflows/match-build-fail.json"
- name: Build
run: python3 -u build_release_files.py
working-directory: tools
env:
BOARDS: ${{ matrix.board }}
- uses: actions/upload-artifact@v3
with:
name: ${{ matrix.board }}
path: bin/${{ matrix.board }}
- name: Upload to S3
if: >-
(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') ||
(github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))
run: |
pip install awscli
[ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp bin/ s3://adafruit-circuit-python/bin/ --recursive --no-progress --region us-east-1
env:
AWS_PAGER: ''
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
if: (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') || (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))

View File

@ -4,7 +4,7 @@ on:
push:
pull_request:
paths:
- '.github/workflows/*.yml'
- '.github/workflows/ports_windows.yml'
- 'extmod/**'
- 'lib/**'
- 'mpy-cross/**'
@ -71,12 +71,31 @@ jobs:
which python; python --version; python -c "import cascadetoml"
which python3; python3 --version; python3 -c "import cascadetoml"
- uses: actions/checkout@v3
- name: Set up repository
uses: actions/checkout@v3
with:
submodules: false
fetch-depth: 1
- name: Get CP deps
run: python tools/ci_fetch_deps.py windows ${{ github.sha }}
- name: Create submodule status
run: git submodule status extmod/ulab lib/ tools/ >> submodule_status
- name: Restore submodules
uses: actions/cache/restore@v3
with:
path: |
.git/modules/
extmod/ulab
lib/
tools/
key: submodules-common-${{ hashFiles('submodule_status') }}
enableCrossOsArchive: true
- name: CircuitPython dependencies
run: |
python tools/ci_fetch_deps.py windows
echo "::group::Fetch history and tags"
git fetch --no-recurse-submodules --shallow-since="2021-07-01" --tags https://github.com/adafruit/circuitpython HEAD
git fetch --no-recurse-submodules --shallow-since="2021-07-01" origin $GITHUB_SHA
git repack -d
echo "::endgroup::"
- name: CircuitPython version
run: |
tools/describe || git log --parents HEAD~4..

View File

@ -1,2 +1,2 @@
# For uploading artifacts
awscli
# awscli

View File

@ -1,19 +1,16 @@
import pathlib
import shlex
import subprocess
import os
import sys
import time
import shlex
import pathlib
import subprocess
# target will be a board, "test", "docs", "mpy-cross-mac", or "windows"
target = sys.argv[1]
ref = sys.argv[2]
print(target, ref)
# Target will be a board, "test", "docs", "mpy-cross-mac", or "windows"
TARGET = sys.argv[1]
# Submodules needed by port builds outside of their ports directory.
# Should we try and detect these?
port_deps = {
PORT_DEPS = {
"atmel-samd": [
"extmod/ulab/",
"lib/adafruit_floppy/",
@ -58,73 +55,82 @@ def run(title, command, check=True):
try:
subprocess.run(shlex.split(command), stderr=subprocess.STDOUT, check=check)
finally:
print("Duration:", time.monotonic() - start, flush=True)
print("::endgroup::", flush=True)
print("Duration:", time.monotonic() - start, flush=True)
run(
"Fetch back to the start of 2021 to get tag history",
'git fetch --tags --recurse-submodules=no --shallow-since="2021-07-01" https://github.com/adafruit/circuitpython HEAD',
)
run(
"Fetch back to the start of 2021 to get commit history",
f'git fetch --recurse-submodules=no --shallow-since="2021-07-01" origin {ref}',
)
# See https://stackoverflow.com/questions/63878612/git-fatal-error-in-object-unshallow-sha-1#comment118418373_63879454
run('Fix for bug "fatal: error in object: unshallow"', "git repack -d")
run("Init submodules", "git submodule init")
run("Submodule status", "git submodule status")
def set_output(name, value):
if "GITHUB_OUTPUT" in os.environ:
with open(os.environ["GITHUB_OUTPUT"], "at") as f:
print(f"{name}={value}", file=f)
else:
print(f"Would set GitHub actions output {name} to '{value}'")
submodules = []
if target == "test":
submodules = ["extmod/", "lib/", "tools/", "extmod/ulab", "lib/berkeley-db-1.xx"]
elif target == "docs":
# used in .readthedocs.yml to generate RTD
submodules = ["extmod/ulab/", "frozen/"]
elif target == "mpy-cross-mac":
submodules = ["tools/"] # for huffman
elif target == "windows":
# This builds one board from a number of ports so fill out a bunch of submodules
submodules = ["extmod/", "lib/", "tools/", "ports/", "data/nvm.toml/"]
elif target == "website":
submodules = ["tools/adabot/", "frozen/"]
else:
p = list(pathlib.Path(".").glob(f"ports/*/boards/{target}/mpconfigboard.mk"))
if not p:
raise RuntimeError(f"Unsupported target: {target}")
def main():
submodules = []
submodules_tags = []
config = p[0]
# Add the ports folder to init submodules
port_folder = config.parents[2]
port = port_folder.name
submodules.append(str(port_folder))
submodules.append("tools/") # for huffman
submodules.extend(port_deps[port])
with config.open() as f:
for line in f.readlines():
prefix = "FROZEN_MPY_DIRS += $(TOP)/"
if line.startswith(prefix):
lib_folder = line.strip()[len(prefix) :]
# Drop everything after the second folder because the frozen
# folder may be inside the submodule.
if lib_folder.count("/") > 1:
lib_folder = lib_folder.split("/", maxsplit=2)
lib_folder = "/".join(lib_folder[:2])
submodules.append(lib_folder)
print("Target:", TARGET)
print(submodules)
if submodules:
submodules = " ".join(submodules)
# This line will fail because the submodule's need different commits than the tip of the branch. We
# fix it later.
run(
"Init the submodules we'll need",
f"git submodule update --init -N --depth 1 {submodules}",
check=False,
)
if TARGET == "test":
submodules = ["extmod/ulab", "lib/", "tools/"]
elif TARGET == "build-doc":
# used in .readthedocs.yml to generate RTD
submodules = ["extmod/ulab", "frozen/"]
elif TARGET == "mpy-cross-mac":
submodules = ["tools/"] # for huffman
elif TARGET == "windows":
# This builds one board from a number of ports so fill out a bunch of submodules
submodules = ["extmod/ulab", "lib/", "tools/", "ports/", "data/nvm.toml"]
elif TARGET == "website":
submodules = ["tools/adabot/", "frozen/"]
else:
p = list(pathlib.Path(".").glob(f"ports/*/boards/{TARGET}/mpconfigboard.mk"))
if not p:
raise RuntimeError(f"Unsupported target: {TARGET}")
run(
"Fetch the submodule sha",
"git submodule foreach 'git fetch --tags --depth 1 origin $sha1 && git checkout -q $sha1'",
)
config = p[0]
# Add the ports folder to init submodules
port_folder = config.parents[2]
port = port_folder.name
submodules.append(str(port_folder))
submodules.append("tools/") # for huffman
submodules.extend(PORT_DEPS[port])
with config.open() as f:
for line in f.readlines():
prefix = "FROZEN_MPY_DIRS += $(TOP)/"
if line.startswith(prefix):
lib_folder = line.strip()[len(prefix) :]
# Drop everything after the second folder because the frozen
# folder may be inside the submodule.
if lib_folder.count("/") > 1:
lib_folder = lib_folder.split("/", maxsplit=2)
lib_folder = "/".join(lib_folder[:2])
submodules_tags.append(lib_folder)
print("Submodule tags[Y]:", submodules_tags)
print("Submodule tags[N]:", submodules)
if submodules_tags:
run(
"Init the submodules with tags",
f"git submodule update --init {' '.join(submodules_tags)}",
)
if submodules:
run(
"Init the submodules without tags",
f"git submodule update --init --depth=1 {' '.join(submodules)}",
)
for submodule in submodules_tags:
if submodule.startswith("frozen"):
set_output("frozen_tags", True)
break
else:
set_output("frozen_tags", False)
if __name__ == "__main__":
main()

View File

@ -1,3 +1,6 @@
#!/bin/sh
# Add any supplied arguments.
git describe --first-parent --dirty --tags --match "[1-9].*" "$@"
if [ -z "$CP_VERSION" ]; then
git describe --first-parent --dirty --tags --match "[1-9].*" "$@"
else
echo $CP_VERSION
fi