From e91fb247a372b023096871cf4fbdcd1ceaaddfdc Mon Sep 17 00:00:00 2001 From: microDev <70126934+microDev1@users.noreply.github.com> Date: Wed, 15 Sep 2021 15:05:05 +0530 Subject: [PATCH] build docs dynamically --- .github/workflows/build.yml | 125 +++++++++++--------- Makefile | 1 + ports/espressif/bindings/espidf/__init__.c | 8 +- requirements-ci.txt | 2 + requirements-dev.txt | 19 +-- requirements-doc.txt | 16 +++ tools/ci_changed_board_list.py | 94 --------------- tools/ci_set_matrix.py | 131 +++++++++++++++++++++ 8 files changed, 227 insertions(+), 169 deletions(-) create mode 100644 requirements-ci.txt create mode 100644 requirements-doc.txt delete mode 100644 tools/ci_changed_board_list.py create mode 100644 tools/ci_set_matrix.py diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 72a13987ed..2b652f7170 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -16,6 +16,7 @@ jobs: test: runs-on: ubuntu-20.04 outputs: + docs-build: ${{ steps.set-matrix.outputs.docs-build }} arm-boards: ${{ steps.set-matrix.outputs.arm-boards }} riscv-boards: ${{ steps.set-matrix.outputs.riscv-boards }} espressif-boards: ${{ steps.set-matrix.outputs.espressif-boards }} @@ -41,37 +42,14 @@ jobs: run: | sudo apt-get update sudo apt-get install -y eatmydata - sudo eatmydata apt-get install -y gettext librsvg2-bin mingw-w64 latexmk texlive-fonts-recommended texlive-latex-recommended texlive-latex-extra gcc-aarch64-linux-gnu - pip install -r requirements-dev.txt + sudo eatmydata apt-get install -y gettext gcc-aarch64-linux-gnu mingw-w64 + pip install -r requirements-ci.txt -r requirements-dev.txt - name: Versions run: | gcc --version python3 --version - name: Duplicate USB VID/PID Check run: python3 -u -m tools.ci_check_duplicate_usb_vid_pid - - name: Build and Validate Stubs - run: make check-stubs -j2 - - uses: actions/upload-artifact@v2 - with: - name: stubs - path: circuitpython-stubs/dist/* - - name: Install pypi dependencies - run: | - python -m pip install --upgrade pip - pip install setuptools wheel twine - - name: Test Documentation Build (HTML) - run: sphinx-build -E -W -b html -D version=${{ env.CP_VERSION }} -D release=${{ env.CP_VERSION }} . _build/html - - uses: actions/upload-artifact@v2 - with: - name: docs - path: _build/html - - name: Test Documentation Build (LaTeX/PDF) - run: | - make latexpdf - - uses: actions/upload-artifact@v2 - with: - name: docs - path: _build/latex - name: Build mpy-cross run: make -C mpy-cross -j2 - name: Build unix port @@ -130,39 +108,22 @@ jobs: [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross/mpy-cross.static.exe s3://adafruit-circuit-python/bin/mpy-cross/mpy-cross.static-x64-windows-${{ env.CP_VERSION }}.exe --no-progress --region us-east-1 zip -9r circuitpython-stubs.zip circuitpython-stubs [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp circuitpython-stubs/dist/*.tar.gz s3://adafruit-circuit-python/bin/stubs/circuitpython-stubs-${{ env.CP_VERSION }}.zip --no-progress --region us-east-1 - - - name: Upload stubs to PyPi - if: github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit' - env: - TWINE_USERNAME: ${{ secrets.pypi_username }} - TWINE_PASSWORD: ${{ secrets.pypi_password }} - run: | - # setup.py sdist was run by 'make stubs' - [ -z "$TWINE_USERNAME" ] || echo "Uploading dev release to PyPi" - [ -z "$TWINE_USERNAME" ] || twine upload circuitpython-stubs/dist/* - - uses: dorny/paths-filter@v2 + - name: "Get changes" + uses: dorny/paths-filter@v2 id: filter with: - # Enable listing of files matching each filter. - # Paths to files will be available in `${FILTER_NAME}_files` output variable. - # Paths will be formatted as JSON array list-files: json - - # Compare against this branch. (Ignored for PRs.) base: ${{ github.ref }} - - # In this example all changed files are passed to the following action to do - # some custom processing. filters: | changed: - '**' - - name: "Set boards to build" + - name: "Set matrix" id: set-matrix working-directory: tools env: CHANGED_FILES: ${{ steps.filter.outputs.changed_files }} - run: | - python3 -u ci_changed_board_list.py + run: python3 -u ci_set_matrix.py + mpy-cross-mac: runs-on: macos-10.15 @@ -221,6 +182,62 @@ jobs: if: (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') || (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested')) + build-doc: + runs-on: ubuntu-20.04 + needs: test + if: ${{ needs.test.outputs.docs-build == 'True' }} + steps: + - uses: actions/checkout@v2.2.0 + with: + submodules: false + fetch-depth: 0 + - name: Populate selected submodules + run: git submodule update --init extmod/ulab + - run: git fetch --recurse-submodules=no https://github.com/adafruit/circuitpython refs/tags/*:refs/tags/* + - name: CircuitPython version + run: | + git describe --dirty --tags + echo >>$GITHUB_ENV CP_VERSION=$(git describe --dirty --tags) + - name: Set up Python 3.8 + uses: actions/setup-python@v1 + with: + python-version: 3.8 + - name: Install deps + run: | + sudo apt-get update + sudo apt-get install -y eatmydata + sudo eatmydata apt-get install -y latexmk librsvg2-bin texlive-fonts-recommended texlive-latex-recommended texlive-latex-extra + pip install -r requirements-ci.txt -r requirements-doc.txt + - name: Build and Validate Stubs + run: make check-stubs -j2 + - uses: actions/upload-artifact@v2 + with: + name: stubs + path: circuitpython-stubs/dist/* + - name: Test Documentation Build (HTML) + run: sphinx-build -E -W -b html -D version=${{ env.CP_VERSION }} -D release=${{ env.CP_VERSION }} . _build/html + - uses: actions/upload-artifact@v2 + with: + name: docs + path: _build/html + - name: Test Documentation Build (LaTeX/PDF) + run: | + make latexpdf + - uses: actions/upload-artifact@v2 + with: + name: docs + path: _build/latex + - name: Upload stubs to PyPi + if: github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit' + env: + TWINE_USERNAME: ${{ secrets.pypi_username }} + TWINE_PASSWORD: ${{ secrets.pypi_password }} + run: | + # setup.py sdist was run by 'make stubs' + [ -z "$TWINE_USERNAME" ] || echo "Uploading dev release to PyPi" + [ -z "$TWINE_USERNAME" ] || twine upload circuitpython-stubs/dist/* + + build-arm: runs-on: ubuntu-20.04 needs: test @@ -229,7 +246,6 @@ jobs: matrix: board: ${{ fromJSON(needs.test.outputs.arm-boards) }} if: ${{ needs.test.outputs.arm-boards != '[]' }} - steps: - name: Set up Python 3.8 uses: actions/setup-python@v1 @@ -243,7 +259,7 @@ jobs: - name: Install deps run: | sudo apt-get install -y gettext - pip install -r requirements-dev.txt + pip install -r requirements-ci.txt -r requirements-dev.txt wget --no-verbose https://adafruit-circuit-python.s3.amazonaws.com/gcc-arm-none-eabi-10-2020-q4-major-x86_64-linux.tar.bz2 sudo tar -C /usr --strip-components=1 -xaf gcc-arm-none-eabi-10-2020-q4-major-x86_64-linux.tar.bz2 - name: Versions @@ -272,6 +288,7 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} if: (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') || (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested')) + build-riscv: runs-on: ubuntu-20.04 needs: test @@ -280,7 +297,6 @@ jobs: matrix: board: ${{ fromJSON(needs.test.outputs.riscv-boards) }} if: ${{ needs.test.outputs.riscv-boards != '[]' }} - steps: - name: Set up Python 3.8 uses: actions/setup-python@v1 @@ -294,7 +310,7 @@ jobs: - name: Install deps run: | sudo apt-get install -y gettext - pip install -r requirements-dev.txt + pip install -r requirements-ci.txt -r requirements-dev.txt wget https://static.dev.sifive.com/dev-tools/riscv64-unknown-elf-gcc-8.3.0-2019.08.0-x86_64-linux-centos6.tar.gz sudo tar -C /usr --strip-components=1 -xaf riscv64-unknown-elf-gcc-8.3.0-2019.08.0-x86_64-linux-centos6.tar.gz - name: Versions @@ -322,6 +338,8 @@ jobs: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} if: (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') || (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested')) + + build-espressif: runs-on: ubuntu-20.04 needs: test @@ -330,7 +348,6 @@ jobs: matrix: board: ${{ fromJSON(needs.test.outputs.espressif-boards) }} if: ${{ needs.test.outputs.espressif-boards != '[]' }} - steps: - name: Set up Python 3.8 uses: actions/setup-python@v1 @@ -363,11 +380,11 @@ jobs: env: IDF_PATH: ${{ github.workspace }}/ports/espressif/esp-idf IDF_TOOLS_PATH: ${{ github.workspace }}/.idf_tools - - name: Install CircuitPython deps + - name: Install deps run: | source $IDF_PATH/export.sh - pip install -r requirements-dev.txt sudo apt-get install -y gettext ninja-build + pip install -r requirements-ci.txt -r requirements-dev.txt env: IDF_PATH: ${{ github.workspace }}/ports/espressif/esp-idf IDF_TOOLS_PATH: ${{ github.workspace }}/.idf_tools diff --git a/Makefile b/Makefile index db0e871256..50b132b6b2 100644 --- a/Makefile +++ b/Makefile @@ -260,6 +260,7 @@ stubs: @$(PYTHON) tools/extract_pyi.py shared-bindings/ $(STUBDIR) @$(PYTHON) tools/extract_pyi.py extmod/ulab/code/ $(STUBDIR)/ulab @$(PYTHON) tools/extract_pyi.py ports/atmel-samd/bindings $(STUBDIR) + @$(PYTHON) tools/extract_pyi.py ports/espressif/bindings $(STUBDIR) @$(PYTHON) tools/extract_pyi.py ports/raspberrypi/bindings $(STUBDIR) @cp setup.py-stubs circuitpython-stubs/setup.py @cp README.rst-stubs circuitpython-stubs/README.rst diff --git a/ports/espressif/bindings/espidf/__init__.c b/ports/espressif/bindings/espidf/__init__.c index 9d79dd5195..4c4b941af6 100644 --- a/ports/espressif/bindings/espidf/__init__.c +++ b/ports/espressif/bindings/espidf/__init__.c @@ -110,10 +110,10 @@ const mp_obj_type_t mp_type_espidf_IDFError = { }; -//| class MemoryError(MemoryError): -//| """Raised when an ESP IDF memory allocation fails.""" -//| ... -//| +// class MemoryError(MemoryError): +// """Raised when an ESP IDF memory allocation fails.""" +// ... +// NORETURN void mp_raise_espidf_MemoryError(void) { nlr_raise(mp_obj_new_exception(&mp_type_espidf_MemoryError)); } diff --git a/requirements-ci.txt b/requirements-ci.txt new file mode 100644 index 0000000000..81deb34beb --- /dev/null +++ b/requirements-ci.txt @@ -0,0 +1,2 @@ +# For uploading artifacts +awscli diff --git a/requirements-dev.txt b/requirements-dev.txt index c3b33f8713..a57794ce1a 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,33 +6,18 @@ cascadetoml jinja2 typer -requests -requests-cache sh click -setuptools cpp-coveralls - -# For docs -Sphinx<4 -sphinx-rtd-theme -myst-parser -sphinx-autoapi -sphinxcontrib-svg2pdfconverter -readthedocs-sphinx-search +requests +requests-cache # For translate check polib # For pre-commit pyyaml -astroid -isort black -mypy - -# For uploading artifacts -awscli # for combining the Nordic SoftDevice with CircuitPython intelhex diff --git a/requirements-doc.txt b/requirements-doc.txt new file mode 100644 index 0000000000..cefd7dc51d --- /dev/null +++ b/requirements-doc.txt @@ -0,0 +1,16 @@ +# For docs +mypy +black +isort +twine +wheel +astroid +setuptools + +# For sphinx +Sphinx<4 +sphinx-autoapi +sphinx-rtd-theme +sphinxcontrib-svg2pdfconverter +readthedocs-sphinx-search +myst-parser diff --git a/tools/ci_changed_board_list.py b/tools/ci_changed_board_list.py deleted file mode 100644 index 086bd65418..0000000000 --- a/tools/ci_changed_board_list.py +++ /dev/null @@ -1,94 +0,0 @@ -#! /usr/bin/env python3 - -"""This script is used in GitHub Actions to determine what boards are built - based on what files were changed. The base commit varies depending on the - event that triggered run. Pull request runs will compare to the base branch - while pushes will compare to the current ref. We override this for the - adafruit/circuitpython repo so we build all boards for pushes. - """ - -# SPDX-FileCopyrightText: 2021 Scott Shawcroft -# -# SPDX-License-Identifier: MIT - -import sys -import os -import json -import yaml -import re - -import build_board_info - -PORT_TO_ARCH = { - "atmel-samd": "arm", - "cxd56": "arm", - "espressif": "espressif", - "litex": "riscv", - "mimxrt10xx": "arm", - "nrf": "arm", - "raspberrypi": "arm", - "stm": "arm", -} - -changed_files = json.loads(os.environ["CHANGED_FILES"]) -print("changed_files") -print(changed_files) - -# Get boards in json format -boards_info_json = build_board_info.get_board_mapping() -all_board_ids = set() -port_to_boards = {} -board_to_port = {} -for board_id in boards_info_json: - info = boards_info_json[board_id] - if info.get("alias", False): - continue - all_board_ids.add(board_id) - port = info["port"] - if port not in port_to_boards: - port_to_boards[port] = set() - port_to_boards[port].add(board_id) - board_to_port[board_id] = port - -# Build all boards if this is a push to an adafruit/circuitpython branch because we save the artifacts. -boards_to_build = all_board_ids -if not changed_files or ( - os.environ.get("GITHUB_EVENT_NAME", "") == "push" - and os.environ.get("GITHUB_REPOSITORY", "") == "adafruit/circuitpython" -): - print("Building all boards because of adafruit/circuitpython branch") -else: - print("Adding boards to build based on changed files") - boards_to_build = set() - board_pattern = re.compile(r"ports/\w+/boards/(\w+)/") - port_pattern = re.compile(r"ports/(\w+)/") - for p in changed_files: - # See if it is board specific - board_matches = board_pattern.search(p) - if board_matches: - board = board_matches.group(1) - boards_to_build.add(board) - continue - - # See if it is port specific - port_matches = port_pattern.search(p) - if port_matches: - port = port_matches.group(1) - boards_to_build.update(port_to_boards[port]) - continue - - # Otherwise build it all - boards_to_build = all_board_ids - break - -# Split boards by architecture. -print("Building boards:") -arch_to_boards = {"arm": [], "riscv": [], "espressif": []} -for board in boards_to_build: - print(" ", board) - arch = PORT_TO_ARCH[board_to_port[board]] - arch_to_boards[arch].append(board) - -# Set the step outputs for each architecture -for arch in arch_to_boards: - print("::set-output name=" + arch + "-boards::" + json.dumps(sorted(arch_to_boards[arch]))) diff --git a/tools/ci_set_matrix.py b/tools/ci_set_matrix.py new file mode 100644 index 0000000000..04ae23e22a --- /dev/null +++ b/tools/ci_set_matrix.py @@ -0,0 +1,131 @@ +#! /usr/bin/env python3 + +# SPDX-FileCopyrightText: 2021 Scott Shawcroft +# SPDX-FileCopyrightText: 2021 microDev +# +# SPDX-License-Identifier: MIT + +""" +This script is used in GitHub Actions to determine what docs/boards are +built based on what files were changed. The base commit varies depending +on the event that triggered run. Pull request runs will compare to the +base branch while pushes will compare to the current ref. We override this +for the adafruit/circuitpython repo so we build all docs/boards for pushes. + +""" + +import re +import os +import sys +import json +import yaml + +import build_board_info + +PORT_TO_ARCH = { + "atmel-samd": "arm", + "cxd56": "arm", + "espressif": "espressif", + "litex": "riscv", + "mimxrt10xx": "arm", + "nrf": "arm", + "raspberrypi": "arm", + "stm": "arm", +} + +changed_files = json.loads(os.environ["CHANGED_FILES"]) +print("changed_files") +print(changed_files) + + +def set_boards_to_build(build_all): + # Get boards in json format + boards_info_json = build_board_info.get_board_mapping() + all_board_ids = set() + port_to_boards = {} + board_to_port = {} + for board_id in boards_info_json: + info = boards_info_json[board_id] + if info.get("alias", False): + continue + all_board_ids.add(board_id) + port = info["port"] + if port not in port_to_boards: + port_to_boards[port] = set() + port_to_boards[port].add(board_id) + board_to_port[board_id] = port + + boards_to_build = all_board_ids + + if not build_all: + boards_to_build = set() + board_pattern = re.compile(r"ports\/\w+\/boards\/(\w+)\/") + port_pattern = re.compile(r"ports\/(\w+)\/") + for p in changed_files: + # See if it is board specific + board_matches = board_pattern.search(p) + if board_matches: + board = board_matches.group(1) + boards_to_build.add(board) + continue + + # See if it is port specific + port_matches = port_pattern.search(p) + if port_matches: + port = port_matches.group(1) + boards_to_build.update(port_to_boards[port]) + continue + + # Otherwise build it all + boards_to_build = all_board_ids + break + + # Split boards by architecture. + print("Building boards:") + arch_to_boards = {"arm": [], "riscv": [], "espressif": []} + for board in boards_to_build: + print(" ", board) + arch = PORT_TO_ARCH[board_to_port[board]] + arch_to_boards[arch].append(board) + + # Set the step outputs for each architecture + for arch in arch_to_boards: + print("::set-output name=" + arch + "-boards::" + json.dumps(sorted(arch_to_boards[arch]))) + + +def set_docs_to_build(build_all): + doc_match = build_all + if not build_all: + doc_pattern = re.compile( + r"extmod\/ulab\/code|ports\/\w+\/bindings|shared-bindings|\.(?:md|MD|rst|RST)$" + ) + for p in changed_files: + if doc_pattern.search(p): + doc_match = True + break + + # Set the step outputs + print("Building docs:", doc_match) + print("::set-output name=docs-build::" + str(doc_match)) + + +def check_changed_files(): + if not changed_files or ( + os.environ.get("GITHUB_EVENT_NAME", "") == "push" + and os.environ.get("GITHUB_REPOSITORY", "") == "adafruit/circuitpython" + ): + print("Building all docs/boards because of adafruit/circuitpython branch") + return True + else: + print("Adding docs/boards to build based on changed files") + return False + + +def main(): + build_all = check_changed_files() + set_docs_to_build(build_all) + set_boards_to_build(build_all) + + +if __name__ == "__main__": + main()