circuitpython/.github/workflows/build.yml

# SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors)
#
# SPDX-License-Identifier: MIT

name: Build CI

on:
  push:
  pull_request:
  release:
    types: [published]
  check_suite:
    types: [rerequested]

concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

jobs:
  test:
    runs-on: ubuntu-20.04
    outputs:
      build-doc: ${{ steps.set-matrix.outputs.build-doc }}
      build-boards: ${{ steps.set-matrix.outputs.build-boards }}
      boards-aarch: ${{ steps.set-matrix.outputs.boards-aarch }}
      boards-arm: ${{ steps.set-matrix.outputs.boards-arm }}
      boards-espressif: ${{ steps.set-matrix.outputs.boards-espressif }}
      boards-riscv: ${{ steps.set-matrix.outputs.boards-riscv }}
      cp-version: ${{ steps.set-up-submodules.outputs.version }}
    env:
      MICROPY_CPYTHON3: python3.8
      MICROPY_MICROPYTHON: ${{ github.workspace }}/ports/unix/micropython-coverage
    steps:
    - name: Dump GitHub context
      run: echo "$GITHUB_CONTEXT"
      env:
        GITHUB_CONTEXT: ${{ toJson(github) }}
    - name: Set up repository
      uses: actions/checkout@v3
      with:
        submodules: false
        fetch-depth: 1
    - name: Set up python
      uses: actions/setup-python@v4
      with:
        python-version: 3.x
    - name: Duplicate USB VID/PID check
      run: python3 -u -m tools.ci_check_duplicate_usb_vid_pid
    - name: Set up submodules
      id: set-up-submodules
      uses: ./.github/actions/fetch_submodules
      with:
        cache: cache
        version: true
    - name: Install dependencies
      run: |
        sudo apt-get update
        sudo apt-get install -y eatmydata
        sudo eatmydata apt-get install -y gettext gcc-aarch64-linux-gnu mingw-w64
        pip install -r requirements-dev.txt        
    - name: Versions
      run: |
        gcc --version
        python3 --version        
    - name: Build mpy-cross
      run: make -C mpy-cross -j2
    - uses: actions/upload-artifact@v3
      with:
        name: mpy-cross
        path: mpy-cross/mpy-cross
    - name: Build unix port
      run: make -C ports/unix VARIANT=coverage -j2
    - name: Test all
      run:  ./run-tests.py -j2
      working-directory: tests
    - name: Print failure info
      run: ./run-tests.py -j2 --print-failures
      if: failure()
      working-directory: tests
    - name: Native Tests
      run: ./run-tests.py -j2 --emit native
      working-directory: tests
    - name: mpy Tests
      run: ./run-tests.py -j2 --via-mpy -d basics float micropython
      working-directory: tests
    - name: Native mpy Tests
      run: ./run-tests.py -j2 --via-mpy --emit native -d basics float micropython
      working-directory: tests
    - name: Build native modules
      run: |
        make -C examples/natmod/features1
        make -C examples/natmod/features2
        make -C examples/natmod/btree
        make -C examples/natmod/framebuf
        make -C examples/natmod/uheapq
        make -C examples/natmod/urandom
        make -C examples/natmod/ure
        make -C examples/natmod/uzlib        
    - name: Test native modules
      run: ./run-natmodtests.py extmod/{btree*,framebuf*,uheapq*,ure*,uzlib*}.py
      working-directory: tests
    - name: Build mpy-cross.static-aarch64
      run: make -C mpy-cross -j2 -f Makefile.static-aarch64
    - uses: actions/upload-artifact@v3
      with:
        name: mpy-cross.static-aarch64
        path: mpy-cross/mpy-cross.static-aarch64
    - name: Build mpy-cross.static-raspbian
      run: make -C mpy-cross -j2 -f Makefile.static-raspbian
    - uses: actions/upload-artifact@v3
      with:
        name: mpy-cross.static-raspbian
        path: mpy-cross/mpy-cross.static-raspbian
    - name: Build mpy-cross.static
      run: make -C mpy-cross -j2 -f Makefile.static
    - uses: actions/upload-artifact@v3
      with:
        name: mpy-cross.static-amd64-linux
        path: mpy-cross/mpy-cross.static
    - name: Build mpy-cross.static-mingw
      run: make -C mpy-cross -j2 -f Makefile.static-mingw
    - uses: actions/upload-artifact@v3
      with:
        name: mpy-cross.static-x64-windows
        path: mpy-cross/mpy-cross.static.exe
    - name: Upload to S3
      if: >-
        (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') ||
        (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))        
      env:
        AWS_PAGER: ''
        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
      run: |
        pip install awscli
        [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross/mpy-cross.static-aarch64 s3://adafruit-circuit-python/bin/mpy-cross/mpy-cross.static-aarch64-${{ env.CP_VERSION }} --no-progress --region us-east-1
        [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross/mpy-cross.static-raspbian s3://adafruit-circuit-python/bin/mpy-cross/mpy-cross.static-raspbian-${{ env.CP_VERSION }} --no-progress --region us-east-1
        [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross/mpy-cross.static s3://adafruit-circuit-python/bin/mpy-cross/mpy-cross.static-amd64-linux-${{ env.CP_VERSION }} --no-progress --region us-east-1
        [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross/mpy-cross.static.exe s3://adafruit-circuit-python/bin/mpy-cross/mpy-cross.static-x64-windows-${{ env.CP_VERSION }}.exe --no-progress --region us-east-1        
    # Disabled: Needs to be updated
    # - name: Get last commit with checks
    #   id: get-last-commit-with-checks
    #   if: github.event_name == 'pull_request'
    #   working-directory: tools
    #   run: python3 -u ci_changes_per_commit.py
    #   env:
    #     REPO: ${{ github.repository }}
    #     PULL: ${{ github.event.number }}
    #     GITHUB_TOKEN: ${{ github.token }}
    #     EXCLUDE_COMMIT: ${{ github.event.after }}
    - name: Set head sha
      if: github.event_name == 'pull_request'
      run: echo "HEAD_SHA=$(git show -s --format=%s $GITHUB_SHA | grep -o -P "(?<=Merge ).*(?= into)")" >> $GITHUB_ENV
    - name: Set base sha
      if: github.event_name == 'pull_request'
      run: |
        git fetch --no-tags --no-recurse-submodules --depth=$((DEPTH + 1)) origin $HEAD_SHA
        echo "BASE_SHA=$(git rev-list $HEAD_SHA --skip=$DEPTH --max-count=1)" >> $GITHUB_ENV        
      env:
        DEPTH: ${{ steps.get-last-commit-with-checks.outputs.commit_depth || github.event.pull_request.commits }}
    - name: Get changes
      id: get-changes
      if: github.event_name == 'pull_request'
      run: echo $(git diff $BASE_SHA...$HEAD_SHA --name-only) | echo "changed_files=[\"$(sed "s/ /\", \"/g")\"]" >> $GITHUB_OUTPUT
    - name: Set matrix
      id: set-matrix
      working-directory: tools
      run: python3 -u ci_set_matrix.py
      env:
        CHANGED_FILES: ${{ steps.get-changes.outputs.changed_files }}
        LAST_FAILED_JOBS: ${{ steps.get-last-commit-with-checks.outputs.check_runs }}


  mpy-cross-mac:
    runs-on: macos-11
    needs: test
    if: ${{ needs.test.outputs.build-boards == 'True' }}
    env:
      CP_VERSION: ${{ needs.test.outputs.cp-version }}
    steps:
    - name: Set up repository
      uses: actions/checkout@v3
      with:
        submodules: false
        fetch-depth: 1
    - name: Set up python
      uses: actions/setup-python@v4
      with:
        python-version: 3.x
    - name: Set up submodules
      uses: ./.github/actions/fetch_submodules
    - name: Versions
      run: |
        gcc --version
        python3 --version
        msgfmt --version        
    - name: Build mpy-cross
      run: make -C mpy-cross -j2
    - uses: actions/upload-artifact@v3
      with:
        name: mpy-cross-macos-11-x64
        path: mpy-cross/mpy-cross
    - name: Build mpy-cross (arm64)
      run: make -C mpy-cross -j2 -f Makefile.m1 V=2
    - uses: actions/upload-artifact@v3
      with:
        name: mpy-cross-macos-11-arm64
        path: mpy-cross/mpy-cross-arm64
    - name: Make universal binary
      run: lipo -create -output mpy-cross-macos-universal mpy-cross/mpy-cross mpy-cross/mpy-cross-arm64
    - uses: actions/upload-artifact@v3
      with:
        name: mpy-cross-macos-11-universal
        path: mpy-cross-macos-universal
    - name: Upload mpy-cross build to S3
      run: |
        [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross-macos-universal s3://adafruit-circuit-python/bin/mpy-cross/mpy-cross-macos-11-${{ env.CP_VERSION }}-universal --no-progress --region us-east-1
        [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross/mpy-cross-arm64 s3://adafruit-circuit-python/bin/mpy-cross/mpy-cross-macos-11-${{ env.CP_VERSION }}-arm64 --no-progress --region us-east-1
        [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross/mpy-cross s3://adafruit-circuit-python/bin/mpy-cross/mpy-cross-macos-11-${{ env.CP_VERSION }}-x64 --no-progress --region us-east-1        
      env:
        AWS_PAGER: ''
        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
      if: (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') || (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))


  build-doc:
    runs-on: ubuntu-22.04
    needs: test
    if: ${{ needs.test.outputs.build-doc == 'True' }}
    env:
      CP_VERSION: ${{ needs.test.outputs.cp-version }}
    steps:
    - name: Set up repository
      uses: actions/checkout@v3
      with:
        submodules: false
        fetch-depth: 1
    - name: Set up python
      uses: actions/setup-python@v4
      with:
        python-version: 3.x
    - name: Set up submodules
      uses: ./.github/actions/fetch_submodules
    - name: Install dependencies
      run: |
        sudo apt-get update
        sudo apt-get install -y eatmydata
        sudo eatmydata apt-get install -y latexmk librsvg2-bin texlive-fonts-recommended texlive-latex-recommended texlive-latex-extra
        pip install -r requirements-doc.txt        
    - name: Build and Validate Stubs
      run: make check-stubs -j2
    - uses: actions/upload-artifact@v3
      with:
        name: stubs
        path: circuitpython-stubs/dist/*
    - name: Test Documentation Build (HTML)
      run: sphinx-build -E -W -b html -D version=${{ env.CP_VERSION }} -D release=${{ env.CP_VERSION }} . _build/html
    - uses: actions/upload-artifact@v3
      with:
        name: docs
        path: _build/html
    - name: Test Documentation Build (LaTeX/PDF)
      run: |
        make latexpdf        
    - uses: actions/upload-artifact@v3
      with:
        name: docs
        path: _build/latex
    - name: Upload to S3
      if: >-
        (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') ||
        (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))        
      env:
        AWS_PAGER: ''
        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
      run: |
        pip install awscli
        zip -9r circuitpython-stubs.zip circuitpython-stubs
        [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp circuitpython-stubs/dist/*.tar.gz s3://adafruit-circuit-python/bin/stubs/circuitpython-stubs-${{ env.CP_VERSION }}.zip --no-progress --region us-east-1        
    - name: Upload stubs to PyPi
      if: github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested')
      env:
        TWINE_USERNAME: ${{ secrets.pypi_username }}
        TWINE_PASSWORD: ${{ secrets.pypi_password }}
      run: |
        # setup.py sdist was run by 'make stubs'
        [ -z "$TWINE_USERNAME" ] || echo "Uploading dev release to PyPi"
        [ -z "$TWINE_USERNAME" ] || twine upload circuitpython-stubs/dist/*        


  aarch:
    needs: test
    if: ${{ needs.test.outputs.boards-aarch != '[]' }}
    uses: ./.github/workflows/build-boards.yml
    with:
      platform: aarch
      boards: ${{ needs.test.outputs.boards-aarch }}
      cp-version: ${{ needs.test.outputs.cp-version }}


  arm:
    needs: test
    if: ${{ needs.test.outputs.boards-arm != '[]' }}
    uses: ./.github/workflows/build-boards.yml
    with:
      platform: arm
      boards: ${{ needs.test.outputs.boards-arm }}
      cp-version: ${{ needs.test.outputs.cp-version }}


  espressif:
    needs: test
    if: ${{ needs.test.outputs.boards-espressif != '[]' }}
    uses: ./.github/workflows/build-boards.yml
    with:
      platform: espressif
      boards: ${{ needs.test.outputs.boards-espressif }}
      cp-version: ${{ needs.test.outputs.cp-version }}


  riscv:
    needs: test
    if: ${{ needs.test.outputs.boards-riscv != '[]' }}
    uses: ./.github/workflows/build-boards.yml
    with:
      platform: riscv
      boards: ${{ needs.test.outputs.boards-riscv }}
      cp-version: ${{ needs.test.outputs.cp-version }}