diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b1f9698d4a..c55f1ee06b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -16,46 +16,67 @@ concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true +env: + CACHE_SUBMODULES: "['extmod/ulab', 'lib/', 'tools/']" + jobs: test: runs-on: ubuntu-20.04 outputs: build-doc: ${{ steps.set-matrix.outputs.build-doc }} - boards-arm: ${{ steps.set-matrix.outputs.boards-arm }} - boards-riscv: ${{ steps.set-matrix.outputs.boards-riscv }} - boards-espressif: ${{ steps.set-matrix.outputs.boards-espressif }} boards-aarch: ${{ steps.set-matrix.outputs.boards-aarch }} + boards-arm: ${{ steps.set-matrix.outputs.boards-arm }} + boards-espressif: ${{ steps.set-matrix.outputs.boards-espressif }} + boards-riscv: ${{ steps.set-matrix.outputs.boards-riscv }} + cp-version: ${{ steps.cp-version.outputs.cp-version }} steps: - name: Dump GitHub context run: echo "$GITHUB_CONTEXT" env: GITHUB_CONTEXT: ${{ toJson(github) }} - - uses: actions/checkout@v3 + - name: Set up repository + uses: actions/checkout@v3 with: submodules: false fetch-depth: 1 - - name: Set up Python 3 + - name: Set up python uses: actions/setup-python@v4 with: python-version: "3.x" - - name: Get CP deps - run: python tools/ci_fetch_deps.py test ${{ github.sha }} - - name: CircuitPython version + - name: Duplicate USB VID/PID check + run: python3 -u -m tools.ci_check_duplicate_usb_vid_pid + - name: Create submodule status + run: git submodule status ${{ join(fromJSON(env.CACHE_SUBMODULES), ' ') }} >> submodule_status + - name: Cache submodules + uses: actions/cache@v3 + with: + path: ".git/modules/\n${{ join(fromJSON(env.CACHE_SUBMODULES), '\n') }}" + key: submodules-common-${{ hashFiles('submodule_status') }} + - name: CircuitPython dependencies run: | - tools/describe || git log --parents HEAD~4.. - echo >>$GITHUB_ENV CP_VERSION=$(tools/describe) + python tools/ci_fetch_deps.py ${{ github.job }} + echo "::group::Fetch history and tags" + git fetch --no-recurse-submodules --shallow-since="2021-07-01" --tags https://github.com/adafruit/circuitpython HEAD + git fetch --no-recurse-submodules --shallow-since="2021-07-01" origin $GITHUB_SHA + git repack -d + echo "::endgroup::" + - name: CircuitPython version + id: cp-version + run: | + CP_VERSION=$(tools/describe) + echo "$CP_VERSION" + echo "CP_VERSION=$CP_VERSION" >> $GITHUB_ENV + echo "cp-version=$CP_VERSION" >> $GITHUB_OUTPUT - name: Install dependencies run: | sudo apt-get update sudo apt-get install -y eatmydata sudo eatmydata apt-get install -y gettext gcc-aarch64-linux-gnu mingw-w64 - pip install -r requirements-ci.txt -r requirements-dev.txt + pip install -r requirements-dev.txt - name: Versions run: | gcc --version python3 --version - - name: Duplicate USB VID/PID Check - run: python3 -u -m tools.ci_check_duplicate_usb_vid_pid - name: Build mpy-cross run: make -C mpy-cross -j2 - name: Build unix port @@ -114,13 +135,16 @@ jobs: with: name: mpy-cross.static-x64-windows path: mpy-cross/mpy-cross.static.exe - - name: Upload mpy-cross builds to S3 - if: (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') || (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested')) + - name: Upload to S3 + if: >- + (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') || + (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested')) env: AWS_PAGER: '' AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} run: | + pip install awscli [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross/mpy-cross.static-aarch64 s3://adafruit-circuit-python/bin/mpy-cross/mpy-cross.static-aarch64-${{ env.CP_VERSION }} --no-progress --region us-east-1 [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross/mpy-cross.static-raspbian s3://adafruit-circuit-python/bin/mpy-cross/mpy-cross.static-raspbian-${{ env.CP_VERSION }} --no-progress --region us-east-1 [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross/mpy-cross.static s3://adafruit-circuit-python/bin/mpy-cross/mpy-cross.static-amd64-linux-${{ env.CP_VERSION }} --no-progress --region us-east-1 @@ -160,29 +184,35 @@ jobs: mpy-cross-mac: runs-on: macos-11 + needs: test + if: >- + needs.test.outputs.boards-aarch != '[]' || + needs.test.outputs.boards-arm != '[]' || + needs.test.outputs.boards-espressif != '[]' || + needs.test.outputs.boards-riscv != '[]' + env: + CP_VERSION: ${{ needs.test.outputs.cp-version }} steps: - - name: Dump GitHub context - env: - GITHUB_CONTEXT: ${{ toJson(github) }} - run: echo "$GITHUB_CONTEXT" - - uses: actions/checkout@v3 + - name: Set up repository + uses: actions/checkout@v3 with: submodules: false fetch-depth: 1 - - name: Set up Python 3 + - name: Set up python uses: actions/setup-python@v4 with: - python-version: "3.10" - - name: Get CP deps - run: python tools/ci_fetch_deps.py mpy-cross-mac ${{ github.sha }} + python-version: "3.x" + - name: Create submodule status + run: git submodule status ${{ join(fromJSON(env.CACHE_SUBMODULES), ' ') }} >> submodule_status + - name: Restore submodules + uses: actions/cache/restore@v3 + with: + path: ".git/modules/\n${{ join(fromJSON(env.CACHE_SUBMODULES), '\n') }}" + key: submodules-common-${{ hashFiles('submodule_status') }} + - name: CircuitPython dependencies + run: python tools/ci_fetch_deps.py ${{ github.job }} - name: CircuitPython version - run: | - tools/describe || git log --parents HEAD~4.. - echo >>$GITHUB_ENV CP_VERSION=$(tools/describe) - - name: Install dependencies - run: | - brew install gettext - echo >>$GITHUB_PATH /usr/local/opt/gettext/bin + run: tools/describe - name: Versions run: | gcc --version @@ -222,27 +252,35 @@ jobs: runs-on: ubuntu-20.04 needs: test if: ${{ needs.test.outputs.build-doc == 'True' }} + env: + CP_VERSION: ${{ needs.test.outputs.cp-version }} steps: - - uses: actions/checkout@v3 + - name: Set up repository + uses: actions/checkout@v3 with: submodules: false fetch-depth: 1 - - name: Get CP deps - run: python tools/ci_fetch_deps.py docs ${{ github.sha }} + - name: Create submodule status + run: git submodule status ${{ join(fromJSON(env.CACHE_SUBMODULES), ' ') }} >> submodule_status + - name: Restore submodules + uses: actions/cache/restore@v3 + with: + path: ".git/modules/\n${{ join(fromJSON(env.CACHE_SUBMODULES), '\n') }}" + key: submodules-common-${{ hashFiles('submodule_status') }} + - name: CircuitPython dependencies + run: python tools/ci_fetch_deps.py ${{ github.job }} - name: CircuitPython version - run: | - tools/describe || git log --parents HEAD~4.. - echo >>$GITHUB_ENV CP_VERSION=$(tools/describe) - - name: Set up Python 3 + run: tools/describe + - name: Set up python uses: actions/setup-python@v4 with: - python-version: "3.10" + python-version: "3.x" - name: Install dependencies run: | sudo apt-get update sudo apt-get install -y eatmydata sudo eatmydata apt-get install -y latexmk librsvg2-bin texlive-fonts-recommended texlive-latex-recommended texlive-latex-extra - pip install -r requirements-ci.txt -r requirements-doc.txt + pip install -r requirements-doc.txt - name: Build and Validate Stubs run: make check-stubs -j2 - uses: actions/upload-artifact@v3 @@ -262,13 +300,16 @@ jobs: with: name: docs path: _build/latex - - name: Upload stubs to S3 - if: (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') || (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested')) + - name: Upload to S3 + if: >- + (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') || + (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested')) env: AWS_PAGER: '' AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} run: | + pip install awscli zip -9r circuitpython-stubs.zip circuitpython-stubs [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp circuitpython-stubs/dist/*.tar.gz s3://adafruit-circuit-python/bin/stubs/circuitpython-stubs-${{ env.CP_VERSION }}.zip --no-progress --region us-east-1 - name: Upload stubs to PyPi @@ -282,225 +323,40 @@ jobs: [ -z "$TWINE_USERNAME" ] || twine upload circuitpython-stubs/dist/* - build-arm: - runs-on: ubuntu-20.04 - needs: test - strategy: - fail-fast: false - matrix: - board: ${{ fromJSON(needs.test.outputs.boards-arm) }} - if: ${{ needs.test.outputs.boards-arm != '[]' }} - steps: - - name: Set up Python 3 - uses: actions/setup-python@v4 - with: - python-version: "3.10" - - uses: actions/checkout@v3 - with: - submodules: false - fetch-depth: 1 - - name: Get CP deps - run: python tools/ci_fetch_deps.py ${{ matrix.board }} ${{ github.sha }} - - uses: carlosperate/arm-none-eabi-gcc-action@v1 - with: - release: '10-2020-q4' - - name: Install dependencies - run: | - sudo apt-get install -y gettext - pip install -r requirements-ci.txt -r requirements-dev.txt - - name: Versions - run: | - gcc --version - arm-none-eabi-gcc --version - python3 --version - - name: mpy-cross - run: make -C mpy-cross -j2 - - name: Setup build failure matcher - run: echo "::add-matcher::$GITHUB_WORKSPACE/.github/workflows/match-build-fail.json" - - name: build - run: python3 -u build_release_files.py - working-directory: tools - env: - BOARDS: ${{ matrix.board }} - - uses: actions/upload-artifact@v3 - with: - name: ${{ matrix.board }} - path: bin/${{ matrix.board }} - - name: Upload to S3 - run: "[ -z \"$AWS_ACCESS_KEY_ID\" ] || aws s3 cp bin/ s3://adafruit-circuit-python/bin/ --recursive --no-progress --region us-east-1" - env: - AWS_PAGER: '' - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - if: (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') || (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested')) - - - build-riscv: - runs-on: ubuntu-20.04 - needs: test - strategy: - fail-fast: false - matrix: - board: ${{ fromJSON(needs.test.outputs.boards-riscv) }} - if: ${{ needs.test.outputs.boards-riscv != '[]' }} - steps: - - name: Set up Python 3 - uses: actions/setup-python@v4 - with: - python-version: "3.10" - - uses: actions/checkout@v3 - with: - submodules: false - fetch-depth: 1 - - name: Get CP deps - run: python tools/ci_fetch_deps.py ${{ matrix.board }} ${{ github.sha }} - - name: Install dependencies - run: | - sudo apt-get install -y gettext - pip install -r requirements-ci.txt -r requirements-dev.txt - wget https://static.dev.sifive.com/dev-tools/riscv64-unknown-elf-gcc-8.3.0-2019.08.0-x86_64-linux-centos6.tar.gz - sudo tar -C /usr --strip-components=1 -xaf riscv64-unknown-elf-gcc-8.3.0-2019.08.0-x86_64-linux-centos6.tar.gz - - name: Versions - run: | - gcc --version - riscv64-unknown-elf-gcc --version - python3 --version - - name: mpy-cross - run: make -C mpy-cross -j2 - - name: Setup build failure matcher - run: echo "::add-matcher::$GITHUB_WORKSPACE/.github/workflows/match-build-fail.json" - - name: build - run: python3 -u build_release_files.py - working-directory: tools - env: - BOARDS: ${{ matrix.board }} - - uses: actions/upload-artifact@v3 - with: - name: ${{ matrix.board }} - path: bin/${{ matrix.board }} - - name: Upload to S3 - run: "[ -z \"$AWS_ACCESS_KEY_ID\" ] || aws s3 cp bin/ s3://adafruit-circuit-python/bin/ --recursive --no-progress --region us-east-1" - env: - AWS_PAGER: '' - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - if: (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') || (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested')) - - - build-espressif: - runs-on: ubuntu-20.04 - needs: test - strategy: - fail-fast: false - matrix: - board: ${{ fromJSON(needs.test.outputs.boards-espressif) }} - if: ${{ needs.test.outputs.boards-espressif != '[]' }} - steps: - - name: Set up Python 3 - id: py3 - uses: actions/setup-python@v4 - with: - python-version: "3.10" - - uses: actions/checkout@v3 - with: - submodules: false - fetch-depth: 1 - - name: Get CP deps - run: python tools/ci_fetch_deps.py ${{ matrix.board }} ${{ github.sha }} - - name: CircuitPython version - run: | - tools/describe || git log --parents HEAD~4.. - echo >>$GITHUB_ENV CP_VERSION=$(tools/describe) - - uses: actions/cache@v3 - name: Fetch IDF tool cache - id: idf-cache - with: - path: ${{ github.workspace }}/.idf_tools - key: ${{ runner.os }}-idf-tools-${{ hashFiles('.git/modules/ports/espressif/esp-idf/HEAD') }}-${{ steps.py3.outputs.python-path }}-20220404 - - name: Clone IDF submodules - run: git submodule update --init $IDF_PATH - env: - IDF_PATH: ${{ github.workspace }}/ports/espressif/esp-idf - - name: Install IDF tools - run: | - echo "Installing ESP-IDF tools" - $IDF_PATH/tools/idf_tools.py --non-interactive install required - $IDF_PATH/tools/idf_tools.py --non-interactive install cmake - echo "Installing Python environment and packages" - $IDF_PATH/tools/idf_tools.py --non-interactive install-python-env - rm -rf $IDF_TOOLS_PATH/dist - env: - IDF_PATH: ${{ github.workspace }}/ports/espressif/esp-idf - IDF_TOOLS_PATH: ${{ github.workspace }}/.idf_tools - - name: Install dependencies - run: | - source $IDF_PATH/export.sh - sudo apt-get install -y gettext ninja-build - pip install -r requirements-ci.txt -r requirements-dev.txt - env: - IDF_PATH: ${{ github.workspace }}/ports/espressif/esp-idf - IDF_TOOLS_PATH: ${{ github.workspace }}/.idf_tools - - name: Versions - run: | - source $IDF_PATH/export.sh - gcc --version - python3 --version - ninja --version - cmake --version - shell: bash - env: - IDF_PATH: ${{ github.workspace }}/ports/espressif/esp-idf - IDF_TOOLS_PATH: ${{ github.workspace }}/.idf_tools - - name: mpy-cross - run: make -C mpy-cross -j2 - - name: Setup build failure matcher - run: echo "::add-matcher::$GITHUB_WORKSPACE/.github/workflows/match-build-fail.json" - - name: build - run: | - source $IDF_PATH/export.sh - python3 -u build_release_files.py - working-directory: tools - shell: bash - env: - IDF_PATH: ${{ github.workspace }}/ports/espressif/esp-idf - IDF_TOOLS_PATH: ${{ github.workspace }}/.idf_tools - BOARDS: ${{ matrix.board }} - - uses: actions/upload-artifact@v3 - with: - name: ${{ matrix.board }} - path: bin/${{ matrix.board }} - - name: Upload to S3 - run: "[ -z \"$AWS_ACCESS_KEY_ID\" ] || aws s3 cp bin/ s3://adafruit-circuit-python/bin/ --recursive --no-progress --region us-east-1" - env: - AWS_PAGER: '' - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - if: (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') || (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested')) - - build-aarch: runs-on: ubuntu-20.04 needs: test + if: ${{ needs.test.outputs.boards-aarch != '[]' }} + env: + CP_VERSION: ${{ needs.test.outputs.cp-version }} strategy: fail-fast: false matrix: board: ${{ fromJSON(needs.test.outputs.boards-aarch) }} - if: ${{ needs.test.outputs.boards-aarch != '[]' }} steps: - - name: Set up Python 3 - uses: actions/setup-python@v4 - with: - python-version: "3.10" - - uses: actions/checkout@v3 + - name: Set up repository + uses: actions/checkout@v3 with: submodules: false fetch-depth: 1 - - name: Get CP deps - run: python tools/ci_fetch_deps.py ${{ matrix.board }} ${{ github.sha }} + - name: Set up python + uses: actions/setup-python@v4 + with: + python-version: "3.x" + - name: Create submodule status + run: git submodule status ${{ join(fromJSON(env.CACHE_SUBMODULES), ' ') }} >> submodule_status + - name: Restore submodules + uses: actions/cache/restore@v3 + with: + path: ".git/modules/\n${{ join(fromJSON(env.CACHE_SUBMODULES), '\n') }}" + key: submodules-common-${{ hashFiles('submodule_status') }} + - name: CircuitPython dependencies + id: cp-deps + run: python tools/ci_fetch_deps.py ${{ matrix.board }} - name: Install dependencies run: | sudo apt-get install -y gettext mtools - pip install -r requirements-ci.txt -r requirements-dev.txt + pip install -r requirements-dev.txt wget --no-verbose https://adafruit-circuit-python.s3.amazonaws.com/gcc-arm-10.3-2021.07-x86_64-aarch64-none-elf.tar.xz sudo tar -C /usr --strip-components=1 -xaf gcc-arm-10.3-2021.07-x86_64-aarch64-none-elf.tar.xz - uses: carlosperate/arm-none-eabi-gcc-action@v1 @@ -522,11 +378,12 @@ jobs: arm-none-eabi-gcc --version python3 --version mkfs.fat --version || true - - name: mpy-cross + - name: Build mpy-cross + if: ${{ steps.cp-deps.outputs.frozen_tags == 'True' }} run: make -C mpy-cross -j2 - name: Setup build failure matcher run: echo "::add-matcher::$GITHUB_WORKSPACE/.github/workflows/match-build-fail.json" - - name: build + - name: Build run: python3 -u build_release_files.py working-directory: tools env: @@ -536,9 +393,256 @@ jobs: name: ${{ matrix.board }} path: bin/${{ matrix.board }} - name: Upload to S3 - run: "[ -z \"$AWS_ACCESS_KEY_ID\" ] || aws s3 cp bin/ s3://adafruit-circuit-python/bin/ --recursive --no-progress --region us-east-1" + if: >- + (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') || + (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested')) + run: | + pip install awscli + [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp bin/ s3://adafruit-circuit-python/bin/ --recursive --no-progress --region us-east-1 + env: + AWS_PAGER: '' + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + + + build-arm: + runs-on: ubuntu-20.04 + needs: test + if: ${{ needs.test.outputs.boards-arm != '[]' }} + env: + CP_VERSION: ${{ needs.test.outputs.cp-version }} + strategy: + fail-fast: false + matrix: + board: ${{ fromJSON(needs.test.outputs.boards-arm) }} + steps: + - name: Set up repository + uses: actions/checkout@v3 + with: + submodules: false + fetch-depth: 1 + - name: Set up python + uses: actions/setup-python@v4 + with: + python-version: "3.x" + - name: Create submodule status + run: git submodule status ${{ join(fromJSON(env.CACHE_SUBMODULES), ' ') }} >> submodule_status + - name: Restore submodules + uses: actions/cache/restore@v3 + with: + path: ".git/modules/\n${{ join(fromJSON(env.CACHE_SUBMODULES), '\n') }}" + key: submodules-common-${{ hashFiles('submodule_status') }} + - name: CircuitPython dependencies + id: cp-deps + run: python tools/ci_fetch_deps.py ${{ matrix.board }} + - uses: carlosperate/arm-none-eabi-gcc-action@v1 + with: + release: '10-2020-q4' + - name: Install dependencies + run: | + sudo apt-get install -y gettext + pip install -r requirements-dev.txt + - name: Versions + run: | + gcc --version + arm-none-eabi-gcc --version + python3 --version + - name: Build mpy-cross + if: ${{ steps.cp-deps.outputs.frozen_tags == 'True' }} + run: make -C mpy-cross -j2 + - name: Setup build failure matcher + run: echo "::add-matcher::$GITHUB_WORKSPACE/.github/workflows/match-build-fail.json" + - name: Build + run: python3 -u build_release_files.py + working-directory: tools + env: + BOARDS: ${{ matrix.board }} + - uses: actions/upload-artifact@v3 + with: + name: ${{ matrix.board }} + path: bin/${{ matrix.board }} + - name: Upload to S3 + if: >- + (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') || + (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested')) + run: | + pip install awscli + [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp bin/ s3://adafruit-circuit-python/bin/ --recursive --no-progress --region us-east-1 + env: + AWS_PAGER: '' + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + + + + build-espressif: + runs-on: ubuntu-20.04 + needs: test + if: ${{ needs.test.outputs.boards-espressif != '[]' }} + env: + CP_VERSION: ${{ needs.test.outputs.cp-version }} + IDF_PATH: ${{ github.workspace }}/ports/espressif/esp-idf + IDF_TOOLS_PATH: ${{ github.workspace }}/.idf_tools + strategy: + fail-fast: false + matrix: + board: ${{ fromJSON(needs.test.outputs.boards-espressif) }} + steps: + - name: Set up repository + uses: actions/checkout@v3 + with: + submodules: false + fetch-depth: 1 + - name: Set up python + id: setup-python + uses: actions/setup-python@v4 + with: + python-version: "3.10" + - name: Create submodule status + run: git submodule status ${{ join(fromJSON(env.CACHE_SUBMODULES), ' ') }} >> submodule_status + - name: Restore submodules + uses: actions/cache/restore@v3 + with: + path: ".git/modules/\n${{ join(fromJSON(env.CACHE_SUBMODULES), '\n') }}" + key: submodules-common-${{ hashFiles('submodule_status') }} + - name: Get IDF commit + id: idf-commit + run: | + COMMIT=$(git submodule status ports/espressif/esp-idf | grep -o -P '(?<=^-).*(?= )') + echo "$COMMIT" + echo "commit=$COMMIT" >> $GITHUB_OUTPUT + - name: Cache IDF submodules + uses: actions/cache@v3 + with: + path: | + .git/modules/ports/espressif/esp-idf + ports/espressif/esp-idf + key: submodules-idf-${{ steps.idf-commit.outputs.commit }} + - name: CircuitPython dependencies + id: cp-deps + run: python tools/ci_fetch_deps.py ${{ matrix.board }} + - name: CircuitPython version + run: tools/describe + - name: Cache IDF tools + uses: actions/cache@v3 + with: + path: ${{ env.IDF_TOOLS_PATH }} + key: ${{ runner.os }}-Python-${{ steps.setup-python.outputs.python-version }}-tools-idf-${{ steps.idf-commit.outputs.commit }} + - name: Clone IDF submodules + run: git submodule update --init --depth=1 + working-directory: ${{ env.IDF_PATH }} + - name: Install IDF tools + run: | + echo "Installing ESP-IDF tools" + $IDF_PATH/tools/idf_tools.py --non-interactive install required + $IDF_PATH/tools/idf_tools.py --non-interactive install cmake + echo "Installing Python environment and packages" + $IDF_PATH/tools/idf_tools.py --non-interactive install-python-env + rm -rf $IDF_TOOLS_PATH/dist + - name: Install dependencies + run: | + source $IDF_PATH/export.sh + sudo apt-get install -y gettext ninja-build + pip install -r requirements-dev.txt + - name: Versions + run: | + source $IDF_PATH/export.sh + gcc --version + python3 --version + ninja --version + cmake --version + - name: Build mpy-cross + if: ${{ steps.cp-deps.outputs.frozen_tags == 'True' }} + run: make -C mpy-cross -j2 + - name: Setup build failure matcher + run: echo "::add-matcher::$GITHUB_WORKSPACE/.github/workflows/match-build-fail.json" + - name: Build + run: | + source $IDF_PATH/export.sh + python3 -u build_release_files.py + working-directory: tools + env: + BOARDS: ${{ matrix.board }} + - uses: actions/upload-artifact@v3 + with: + name: ${{ matrix.board }} + path: bin/${{ matrix.board }} + - name: Upload to S3 + if: >- + (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') || + (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested')) + run: | + pip install awscli + [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp bin/ s3://adafruit-circuit-python/bin/ --recursive --no-progress --region us-east-1 + env: + AWS_PAGER: '' + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + + + build-riscv: + runs-on: ubuntu-20.04 + needs: test + if: ${{ needs.test.outputs.boards-riscv != '[]' }} + env: + CP_VERSION: ${{ needs.test.outputs.cp-version }} + strategy: + fail-fast: false + matrix: + board: ${{ fromJSON(needs.test.outputs.boards-riscv) }} + steps: + - name: Set up repository + uses: actions/checkout@v3 + with: + submodules: false + fetch-depth: 1 + - name: Set up python + uses: actions/setup-python@v4 + with: + python-version: "3.x" + - name: Create submodule status + run: git submodule status ${{ join(fromJSON(env.CACHE_SUBMODULES), ' ') }} >> submodule_status + - name: Restore submodules + uses: actions/cache/restore@v3 + with: + path: ".git/modules/\n${{ join(fromJSON(env.CACHE_SUBMODULES), '\n') }}" + key: submodules-common-${{ hashFiles('submodule_status') }} + - name: CircuitPython dependencies + id: cp-deps + run: python tools/ci_fetch_deps.py ${{ matrix.board }} + - name: Install dependencies + run: | + sudo apt-get install -y gettext + pip install -r requirements-dev.txt + wget https://static.dev.sifive.com/dev-tools/riscv64-unknown-elf-gcc-8.3.0-2019.08.0-x86_64-linux-centos6.tar.gz + sudo tar -C /usr --strip-components=1 -xaf riscv64-unknown-elf-gcc-8.3.0-2019.08.0-x86_64-linux-centos6.tar.gz + - name: Versions + run: | + gcc --version + riscv64-unknown-elf-gcc --version + python3 --version + - name: Build mpy-cross + if: ${{ steps.cp-deps.outputs.frozen_tags == 'True' }} + run: make -C mpy-cross -j2 + - name: Setup build failure matcher + run: echo "::add-matcher::$GITHUB_WORKSPACE/.github/workflows/match-build-fail.json" + - name: Build + run: python3 -u build_release_files.py + working-directory: tools + env: + BOARDS: ${{ matrix.board }} + - uses: actions/upload-artifact@v3 + with: + name: ${{ matrix.board }} + path: bin/${{ matrix.board }} + - name: Upload to S3 + if: >- + (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') || + (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested')) + run: | + pip install awscli + [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp bin/ s3://adafruit-circuit-python/bin/ --recursive --no-progress --region us-east-1 env: AWS_PAGER: '' AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - if: (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') || (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested')) diff --git a/.github/workflows/ports_windows.yml b/.github/workflows/ports_windows.yml index 6d38f3b853..61c6376ac7 100644 --- a/.github/workflows/ports_windows.yml +++ b/.github/workflows/ports_windows.yml @@ -4,7 +4,7 @@ on: push: pull_request: paths: - - '.github/workflows/*.yml' + - '.github/workflows/ports_windows.yml' - 'extmod/**' - 'lib/**' - 'mpy-cross/**' @@ -71,12 +71,31 @@ jobs: which python; python --version; python -c "import cascadetoml" which python3; python3 --version; python3 -c "import cascadetoml" - - uses: actions/checkout@v3 + - name: Set up repository + uses: actions/checkout@v3 with: submodules: false fetch-depth: 1 - - name: Get CP deps - run: python tools/ci_fetch_deps.py windows ${{ github.sha }} + - name: Create submodule status + run: git submodule status extmod/ulab lib/ tools/ >> submodule_status + - name: Restore submodules + uses: actions/cache/restore@v3 + with: + path: | + .git/modules/ + extmod/ulab + lib/ + tools/ + key: submodules-common-${{ hashFiles('submodule_status') }} + enableCrossOsArchive: true + - name: CircuitPython dependencies + run: | + python tools/ci_fetch_deps.py windows + echo "::group::Fetch history and tags" + git fetch --no-recurse-submodules --shallow-since="2021-07-01" --tags https://github.com/adafruit/circuitpython HEAD + git fetch --no-recurse-submodules --shallow-since="2021-07-01" origin $GITHUB_SHA + git repack -d + echo "::endgroup::" - name: CircuitPython version run: | tools/describe || git log --parents HEAD~4.. diff --git a/requirements-ci.txt b/requirements-ci.txt index 81deb34beb..2f6e042fb8 100644 --- a/requirements-ci.txt +++ b/requirements-ci.txt @@ -1,2 +1,2 @@ # For uploading artifacts -awscli +# awscli diff --git a/tools/ci_fetch_deps.py b/tools/ci_fetch_deps.py index 04ae6fb0e1..ddf7bf3bfd 100644 --- a/tools/ci_fetch_deps.py +++ b/tools/ci_fetch_deps.py @@ -1,19 +1,16 @@ -import pathlib -import shlex -import subprocess +import os import sys import time +import shlex +import pathlib +import subprocess -# target will be a board, "test", "docs", "mpy-cross-mac", or "windows" - -target = sys.argv[1] -ref = sys.argv[2] - -print(target, ref) +# Target will be a board, "test", "docs", "mpy-cross-mac", or "windows" +TARGET = sys.argv[1] # Submodules needed by port builds outside of their ports directory. # Should we try and detect these? -port_deps = { +PORT_DEPS = { "atmel-samd": [ "extmod/ulab/", "lib/adafruit_floppy/", @@ -58,73 +55,82 @@ def run(title, command, check=True): try: subprocess.run(shlex.split(command), stderr=subprocess.STDOUT, check=check) finally: - print("Duration:", time.monotonic() - start, flush=True) print("::endgroup::", flush=True) + print("Duration:", time.monotonic() - start, flush=True) -run( - "Fetch back to the start of 2021 to get tag history", - 'git fetch --tags --recurse-submodules=no --shallow-since="2021-07-01" https://github.com/adafruit/circuitpython HEAD', -) -run( - "Fetch back to the start of 2021 to get commit history", - f'git fetch --recurse-submodules=no --shallow-since="2021-07-01" origin {ref}', -) -# See https://stackoverflow.com/questions/63878612/git-fatal-error-in-object-unshallow-sha-1#comment118418373_63879454 -run('Fix for bug "fatal: error in object: unshallow"', "git repack -d") -run("Init submodules", "git submodule init") -run("Submodule status", "git submodule status") +def set_output(name, value): + if "GITHUB_OUTPUT" in os.environ: + with open(os.environ["GITHUB_OUTPUT"], "at") as f: + print(f"{name}={value}", file=f) + else: + print(f"Would set GitHub actions output {name} to '{value}'") -submodules = [] -if target == "test": - submodules = ["extmod/", "lib/", "tools/", "extmod/ulab", "lib/berkeley-db-1.xx"] -elif target == "docs": - # used in .readthedocs.yml to generate RTD - submodules = ["extmod/ulab/", "frozen/"] -elif target == "mpy-cross-mac": - submodules = ["tools/"] # for huffman -elif target == "windows": - # This builds one board from a number of ports so fill out a bunch of submodules - submodules = ["extmod/", "lib/", "tools/", "ports/", "data/nvm.toml/"] -elif target == "website": - submodules = ["tools/adabot/", "frozen/"] -else: - p = list(pathlib.Path(".").glob(f"ports/*/boards/{target}/mpconfigboard.mk")) - if not p: - raise RuntimeError(f"Unsupported target: {target}") +def main(): + submodules = [] + submodules_tags = [] - config = p[0] - # Add the ports folder to init submodules - port_folder = config.parents[2] - port = port_folder.name - submodules.append(str(port_folder)) - submodules.append("tools/") # for huffman - submodules.extend(port_deps[port]) - with config.open() as f: - for line in f.readlines(): - prefix = "FROZEN_MPY_DIRS += $(TOP)/" - if line.startswith(prefix): - lib_folder = line.strip()[len(prefix) :] - # Drop everything after the second folder because the frozen - # folder may be inside the submodule. - if lib_folder.count("/") > 1: - lib_folder = lib_folder.split("/", maxsplit=2) - lib_folder = "/".join(lib_folder[:2]) - submodules.append(lib_folder) + print("Target:", TARGET) -print(submodules) -if submodules: - submodules = " ".join(submodules) - # This line will fail because the submodule's need different commits than the tip of the branch. We - # fix it later. - run( - "Init the submodules we'll need", - f"git submodule update --init -N --depth 1 {submodules}", - check=False, - ) + if TARGET == "test": + submodules = ["extmod/ulab", "lib/", "tools/"] + elif TARGET == "build-doc": + # used in .readthedocs.yml to generate RTD + submodules = ["extmod/ulab", "frozen/"] + elif TARGET == "mpy-cross-mac": + submodules = ["tools/"] # for huffman + elif TARGET == "windows": + # This builds one board from a number of ports so fill out a bunch of submodules + submodules = ["extmod/ulab", "lib/", "tools/", "ports/", "data/nvm.toml"] + elif TARGET == "website": + submodules = ["tools/adabot/", "frozen/"] + else: + p = list(pathlib.Path(".").glob(f"ports/*/boards/{TARGET}/mpconfigboard.mk")) + if not p: + raise RuntimeError(f"Unsupported target: {TARGET}") - run( - "Fetch the submodule sha", - "git submodule foreach 'git fetch --tags --depth 1 origin $sha1 && git checkout -q $sha1'", - ) + config = p[0] + # Add the ports folder to init submodules + port_folder = config.parents[2] + port = port_folder.name + submodules.append(str(port_folder)) + submodules.append("tools/") # for huffman + submodules.extend(PORT_DEPS[port]) + with config.open() as f: + for line in f.readlines(): + prefix = "FROZEN_MPY_DIRS += $(TOP)/" + if line.startswith(prefix): + lib_folder = line.strip()[len(prefix) :] + # Drop everything after the second folder because the frozen + # folder may be inside the submodule. + if lib_folder.count("/") > 1: + lib_folder = lib_folder.split("/", maxsplit=2) + lib_folder = "/".join(lib_folder[:2]) + submodules_tags.append(lib_folder) + + print("Submodule tags[Y]:", submodules_tags) + print("Submodule tags[N]:", submodules) + + if submodules_tags: + run( + "Init the submodules with tags", + f"git submodule update --init {' '.join(submodules_tags)}", + ) + + if submodules: + run( + "Init the submodules without tags", + f"git submodule update --init --depth=1 {' '.join(submodules)}", + ) + + for submodule in submodules_tags: + if submodule.startswith("frozen"): + set_output("frozen_tags", True) + break + else: + set_output("frozen_tags", False) + + +if __name__ == "__main__": + main() diff --git a/tools/describe b/tools/describe index e20c2ffa44..0f3b541e3c 100755 --- a/tools/describe +++ b/tools/describe @@ -1,3 +1,6 @@ #!/bin/sh -# Add any supplied arguments. -git describe --first-parent --dirty --tags --match "[1-9].*" "$@" +if [ -z "$CP_VERSION" ]; then + git describe --first-parent --dirty --tags --match "[1-9].*" "$@" +else + echo $CP_VERSION +fi