build docs dynamically

.github/workflows/build.yml

@@ -16,6 +16,7 @@ jobs:
   test:
     runs-on: ubuntu-20.04
     outputs:
+      docs-build: ${{ steps.set-matrix.outputs.docs-build }}
       arm-boards: ${{ steps.set-matrix.outputs.arm-boards }}
       riscv-boards: ${{ steps.set-matrix.outputs.riscv-boards }}
       espressif-boards: ${{ steps.set-matrix.outputs.espressif-boards }}
@@ -41,37 +42,14 @@ jobs:
       run: |
         sudo apt-get update
         sudo apt-get install -y eatmydata
-        sudo eatmydata apt-get install -y gettext librsvg2-bin mingw-w64 latexmk texlive-fonts-recommended texlive-latex-recommended texlive-latex-extra gcc-aarch64-linux-gnu
+        sudo eatmydata apt-get install -y gettext gcc-aarch64-linux-gnu mingw-w64
-        pip install -r requirements-dev.txt
+        pip install -r requirements-ci.txt -r requirements-dev.txt
     - name: Versions
       run: |
         gcc --version
         python3 --version
     - name: Duplicate USB VID/PID Check
       run: python3 -u -m tools.ci_check_duplicate_usb_vid_pid
-    - name: Build and Validate Stubs
-      run: make check-stubs -j2
-    - uses: actions/upload-artifact@v2
-      with:
-        name: stubs
-        path: circuitpython-stubs/dist/*
-    - name: Install pypi dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install setuptools wheel twine
-    - name: Test Documentation Build (HTML)
-      run: sphinx-build -E -W -b html -D version=${{ env.CP_VERSION }} -D release=${{ env.CP_VERSION }} . _build/html
-    - uses: actions/upload-artifact@v2
-      with:
-        name: docs
-        path: _build/html
-    - name: Test Documentation Build (LaTeX/PDF)
-      run: |
-        make latexpdf
-    - uses: actions/upload-artifact@v2
-      with:
-        name: docs
-        path: _build/latex
     - name: Build mpy-cross
       run: make -C mpy-cross -j2
     - name: Build unix port
@@ -130,39 +108,22 @@ jobs:
         [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross/mpy-cross.static.exe s3://adafruit-circuit-python/bin/mpy-cross/mpy-cross.static-x64-windows-${{ env.CP_VERSION }}.exe --no-progress --region us-east-1
         zip -9r circuitpython-stubs.zip circuitpython-stubs
         [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp circuitpython-stubs/dist/*.tar.gz s3://adafruit-circuit-python/bin/stubs/circuitpython-stubs-${{ env.CP_VERSION }}.zip --no-progress --region us-east-1
-
+    - name: "Get changes"
-    - name: Upload stubs to PyPi
+      uses: dorny/paths-filter@v2
-      if: github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit'
-      env:
-        TWINE_USERNAME: ${{ secrets.pypi_username }}
-        TWINE_PASSWORD: ${{ secrets.pypi_password }}
-      run: |
-        # setup.py sdist was run by 'make stubs'
-        [ -z "$TWINE_USERNAME" ] || echo "Uploading dev release to PyPi"
-        [ -z "$TWINE_USERNAME" ] || twine upload circuitpython-stubs/dist/*
-    - uses: dorny/paths-filter@v2
       id: filter
       with:
-        # Enable listing of files matching each filter.
-        # Paths to files will be available in `${FILTER_NAME}_files` output variable.
-        # Paths will be formatted as JSON array
         list-files: json
-
-        # Compare against this branch. (Ignored for PRs.)
         base: ${{ github.ref }}
-
-        # In this example all changed files are passed to the following action to do
-        # some custom processing.
         filters: |
           changed:
             - '**'
-    - name: "Set boards to build"
+    - name: "Set matrix"
       id: set-matrix
       working-directory: tools
       env:
         CHANGED_FILES: ${{ steps.filter.outputs.changed_files }}
-      run: |
+      run: python3 -u ci_set_matrix.py
-        python3 -u ci_changed_board_list.py
+
 
   mpy-cross-mac:
     runs-on: macos-10.15
@@ -221,6 +182,62 @@ jobs:
       if: (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') || (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))
 
 
+  build-doc:
+    runs-on: ubuntu-20.04
+    needs: test
+    if: ${{ needs.test.outputs.docs-build == 'True' }}
+    steps:
+    - uses: actions/checkout@v2.2.0
+      with:
+        submodules: false
+        fetch-depth: 0
+    - name: Populate selected submodules
+      run: git submodule update --init extmod/ulab
+    - run: git fetch --recurse-submodules=no https://github.com/adafruit/circuitpython refs/tags/*:refs/tags/*
+    - name: CircuitPython version
+      run: |
+        git describe --dirty --tags
+        echo >>$GITHUB_ENV CP_VERSION=$(git describe --dirty --tags)
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v1
+      with:
+        python-version: 3.8
+    - name: Install deps
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y eatmydata
+        sudo eatmydata apt-get install -y latexmk librsvg2-bin texlive-fonts-recommended texlive-latex-recommended texlive-latex-extra
+        pip install -r requirements-ci.txt -r requirements-doc.txt
+    - name: Build and Validate Stubs
+      run: make check-stubs -j2
+    - uses: actions/upload-artifact@v2
+      with:
+        name: stubs
+        path: circuitpython-stubs/dist/*
+    - name: Test Documentation Build (HTML)
+      run: sphinx-build -E -W -b html -D version=${{ env.CP_VERSION }} -D release=${{ env.CP_VERSION }} . _build/html
+    - uses: actions/upload-artifact@v2
+      with:
+        name: docs
+        path: _build/html
+    - name: Test Documentation Build (LaTeX/PDF)
+      run: |
+        make latexpdf
+    - uses: actions/upload-artifact@v2
+      with:
+        name: docs
+        path: _build/latex
+    - name: Upload stubs to PyPi
+      if: github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit'
+      env:
+        TWINE_USERNAME: ${{ secrets.pypi_username }}
+        TWINE_PASSWORD: ${{ secrets.pypi_password }}
+      run: |
+        # setup.py sdist was run by 'make stubs'
+        [ -z "$TWINE_USERNAME" ] || echo "Uploading dev release to PyPi"
+        [ -z "$TWINE_USERNAME" ] || twine upload circuitpython-stubs/dist/*
+
+
   build-arm:
     runs-on: ubuntu-20.04
     needs: test
@@ -229,7 +246,6 @@ jobs:
       matrix:
         board: ${{ fromJSON(needs.test.outputs.arm-boards) }}
     if: ${{ needs.test.outputs.arm-boards != '[]' }}
-
     steps:
     - name: Set up Python 3.8
       uses: actions/setup-python@v1
@@ -243,7 +259,7 @@ jobs:
     - name: Install deps
       run: |
         sudo apt-get install -y gettext
-        pip install -r requirements-dev.txt
+        pip install -r requirements-ci.txt -r requirements-dev.txt
         wget --no-verbose https://adafruit-circuit-python.s3.amazonaws.com/gcc-arm-none-eabi-10-2020-q4-major-x86_64-linux.tar.bz2
         sudo tar -C /usr --strip-components=1 -xaf gcc-arm-none-eabi-10-2020-q4-major-x86_64-linux.tar.bz2
     - name: Versions
@@ -272,6 +288,7 @@ jobs:
         AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
       if: (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') || (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))
 
+
   build-riscv:
     runs-on: ubuntu-20.04
     needs: test
@@ -280,7 +297,6 @@ jobs:
       matrix:
         board: ${{ fromJSON(needs.test.outputs.riscv-boards) }}
     if: ${{ needs.test.outputs.riscv-boards != '[]' }}
-
     steps:
     - name: Set up Python 3.8
       uses: actions/setup-python@v1
@@ -294,7 +310,7 @@ jobs:
     - name: Install deps
       run: |
         sudo apt-get install -y gettext
-        pip install -r requirements-dev.txt
+        pip install -r requirements-ci.txt -r requirements-dev.txt
         wget https://static.dev.sifive.com/dev-tools/riscv64-unknown-elf-gcc-8.3.0-2019.08.0-x86_64-linux-centos6.tar.gz
         sudo tar -C /usr --strip-components=1 -xaf riscv64-unknown-elf-gcc-8.3.0-2019.08.0-x86_64-linux-centos6.tar.gz
     - name: Versions
@@ -322,6 +338,8 @@ jobs:
         AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
         AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
       if: (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') || (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))
+
+
   build-espressif:
     runs-on: ubuntu-20.04
     needs: test
@@ -330,7 +348,6 @@ jobs:
       matrix:
         board: ${{ fromJSON(needs.test.outputs.espressif-boards) }}
     if: ${{ needs.test.outputs.espressif-boards != '[]' }}
-
     steps:
     - name: Set up Python 3.8
       uses: actions/setup-python@v1
@@ -363,11 +380,11 @@ jobs:
       env:
         IDF_PATH: ${{ github.workspace }}/ports/espressif/esp-idf
         IDF_TOOLS_PATH: ${{ github.workspace }}/.idf_tools
-    - name: Install CircuitPython deps
+    - name: Install deps
       run: |
         source $IDF_PATH/export.sh
-        pip install -r requirements-dev.txt
         sudo apt-get install -y gettext ninja-build
+        pip install -r requirements-ci.txt -r requirements-dev.txt
       env:
         IDF_PATH: ${{ github.workspace }}/ports/espressif/esp-idf
         IDF_TOOLS_PATH: ${{ github.workspace }}/.idf_tools

Makefile

@@ -260,6 +260,7 @@ stubs:
 	@$(PYTHON) tools/extract_pyi.py shared-bindings/ $(STUBDIR)
 	@$(PYTHON) tools/extract_pyi.py extmod/ulab/code/ $(STUBDIR)/ulab
 	@$(PYTHON) tools/extract_pyi.py ports/atmel-samd/bindings $(STUBDIR)
+	@$(PYTHON) tools/extract_pyi.py ports/espressif/bindings $(STUBDIR)
 	@$(PYTHON) tools/extract_pyi.py ports/raspberrypi/bindings $(STUBDIR)
 	@cp setup.py-stubs circuitpython-stubs/setup.py
 	@cp README.rst-stubs circuitpython-stubs/README.rst

ports/espressif/bindings/espidf/__init__.c

@@ -110,10 +110,10 @@ const mp_obj_type_t mp_type_espidf_IDFError = {
 };
 
 
-//| class MemoryError(MemoryError):
+// class MemoryError(MemoryError):
-//|     """Raised when an ESP IDF memory allocation fails."""
+//     """Raised when an ESP IDF memory allocation fails."""
-//|     ...
+//     ...
-//|
+//
 NORETURN void mp_raise_espidf_MemoryError(void) {
     nlr_raise(mp_obj_new_exception(&mp_type_espidf_MemoryError));
 }

requirements-ci.txt

@@ -0,0 +1,2 @@
+# For uploading artifacts
+awscli

requirements-dev.txt

@@ -6,33 +6,18 @@ cascadetoml
 jinja2
 typer
 
-requests
-requests-cache
 sh
 click
-setuptools
 cpp-coveralls
-
+requests
-# For docs
+requests-cache
-Sphinx<4
-sphinx-rtd-theme
-myst-parser
-sphinx-autoapi
-sphinxcontrib-svg2pdfconverter
-readthedocs-sphinx-search
 
 # For translate check
 polib
 
 # For pre-commit
 pyyaml
-astroid
-isort
 black
-mypy
-
-# For uploading artifacts
-awscli
 
 # for combining the Nordic SoftDevice with CircuitPython
 intelhex

requirements-doc.txt

@@ -0,0 +1,16 @@
+# For docs
+mypy
+black
+isort
+twine
+wheel
+astroid
+setuptools
+
+# For sphinx
+Sphinx<4
+sphinx-autoapi
+sphinx-rtd-theme
+sphinxcontrib-svg2pdfconverter
+readthedocs-sphinx-search
+myst-parser

tools/ci_changed_board_list.py

@@ -1,94 +0,0 @@
-#! /usr/bin/env python3
-
-"""This script is used in GitHub Actions to determine what boards are built
-  based on what files were changed. The base commit varies depending on the
-  event that triggered run. Pull request runs will compare to the base branch
-  while pushes will compare to the current ref. We override this for the
-  adafruit/circuitpython repo so we build all boards for pushes.
-  """
-
-# SPDX-FileCopyrightText: 2021 Scott Shawcroft
-#
-# SPDX-License-Identifier: MIT
-
-import sys
-import os
-import json
-import yaml
-import re
-
-import build_board_info
-
-PORT_TO_ARCH = {
-    "atmel-samd": "arm",
-    "cxd56": "arm",
-    "espressif": "espressif",
-    "litex": "riscv",
-    "mimxrt10xx": "arm",
-    "nrf": "arm",
-    "raspberrypi": "arm",
-    "stm": "arm",
-}
-
-changed_files = json.loads(os.environ["CHANGED_FILES"])
-print("changed_files")
-print(changed_files)
-
-# Get boards in json format
-boards_info_json = build_board_info.get_board_mapping()
-all_board_ids = set()
-port_to_boards = {}
-board_to_port = {}
-for board_id in boards_info_json:
-    info = boards_info_json[board_id]
-    if info.get("alias", False):
-        continue
-    all_board_ids.add(board_id)
-    port = info["port"]
-    if port not in port_to_boards:
-        port_to_boards[port] = set()
-    port_to_boards[port].add(board_id)
-    board_to_port[board_id] = port
-
-# Build all boards if this is a push to an adafruit/circuitpython branch because we save the artifacts.
-boards_to_build = all_board_ids
-if not changed_files or (
-    os.environ.get("GITHUB_EVENT_NAME", "") == "push"
-    and os.environ.get("GITHUB_REPOSITORY", "") == "adafruit/circuitpython"
-):
-    print("Building all boards because of adafruit/circuitpython branch")
-else:
-    print("Adding boards to build based on changed files")
-    boards_to_build = set()
-    board_pattern = re.compile(r"ports/\w+/boards/(\w+)/")
-    port_pattern = re.compile(r"ports/(\w+)/")
-    for p in changed_files:
-        # See if it is board specific
-        board_matches = board_pattern.search(p)
-        if board_matches:
-            board = board_matches.group(1)
-            boards_to_build.add(board)
-            continue
-
-        # See if it is port specific
-        port_matches = port_pattern.search(p)
-        if port_matches:
-            port = port_matches.group(1)
-            boards_to_build.update(port_to_boards[port])
-            continue
-
-        # Otherwise build it all
-        boards_to_build = all_board_ids
-        break
-
-# Split boards by architecture.
-print("Building boards:")
-arch_to_boards = {"arm": [], "riscv": [], "espressif": []}
-for board in boards_to_build:
-    print(" ", board)
-    arch = PORT_TO_ARCH[board_to_port[board]]
-    arch_to_boards[arch].append(board)
-
-# Set the step outputs for each architecture
-for arch in arch_to_boards:
-    print("::set-output name=" + arch + "-boards::" + json.dumps(sorted(arch_to_boards[arch])))

tools/ci_set_matrix.py

@@ -0,0 +1,131 @@
+#! /usr/bin/env python3
+
+# SPDX-FileCopyrightText: 2021 Scott Shawcroft
+# SPDX-FileCopyrightText: 2021 microDev
+#
+# SPDX-License-Identifier: MIT
+
+"""
+This script is used in GitHub Actions to determine what docs/boards are
+built based on what files were changed. The base commit varies depending
+on the event that triggered run. Pull request runs will compare to the
+base branch while pushes will compare to the current ref. We override this
+for the adafruit/circuitpython repo so we build all docs/boards for pushes.
+
+"""
+
+import re
+import os
+import sys
+import json
+import yaml
+
+import build_board_info
+
+PORT_TO_ARCH = {
+    "atmel-samd": "arm",
+    "cxd56": "arm",
+    "espressif": "espressif",
+    "litex": "riscv",
+    "mimxrt10xx": "arm",
+    "nrf": "arm",
+    "raspberrypi": "arm",
+    "stm": "arm",
+}
+
+changed_files = json.loads(os.environ["CHANGED_FILES"])
+print("changed_files")
+print(changed_files)
+
+
+def set_boards_to_build(build_all):
+    # Get boards in json format
+    boards_info_json = build_board_info.get_board_mapping()
+    all_board_ids = set()
+    port_to_boards = {}
+    board_to_port = {}
+    for board_id in boards_info_json:
+        info = boards_info_json[board_id]
+        if info.get("alias", False):
+            continue
+        all_board_ids.add(board_id)
+        port = info["port"]
+        if port not in port_to_boards:
+            port_to_boards[port] = set()
+        port_to_boards[port].add(board_id)
+        board_to_port[board_id] = port
+
+    boards_to_build = all_board_ids
+
+    if not build_all:
+        boards_to_build = set()
+        board_pattern = re.compile(r"ports\/\w+\/boards\/(\w+)\/")
+        port_pattern = re.compile(r"ports\/(\w+)\/")
+        for p in changed_files:
+            # See if it is board specific
+            board_matches = board_pattern.search(p)
+            if board_matches:
+                board = board_matches.group(1)
+                boards_to_build.add(board)
+                continue
+
+            # See if it is port specific
+            port_matches = port_pattern.search(p)
+            if port_matches:
+                port = port_matches.group(1)
+                boards_to_build.update(port_to_boards[port])
+                continue
+
+            # Otherwise build it all
+            boards_to_build = all_board_ids
+            break
+
+    # Split boards by architecture.
+    print("Building boards:")
+    arch_to_boards = {"arm": [], "riscv": [], "espressif": []}
+    for board in boards_to_build:
+        print(" ", board)
+        arch = PORT_TO_ARCH[board_to_port[board]]
+        arch_to_boards[arch].append(board)
+
+    # Set the step outputs for each architecture
+    for arch in arch_to_boards:
+        print("::set-output name=" + arch + "-boards::" + json.dumps(sorted(arch_to_boards[arch])))
+
+
+def set_docs_to_build(build_all):
+    doc_match = build_all
+    if not build_all:
+        doc_pattern = re.compile(
+            r"extmod\/ulab\/code|ports\/\w+\/bindings|shared-bindings|\.(?:md|MD|rst|RST)$"
+        )
+        for p in changed_files:
+            if doc_pattern.search(p):
+                doc_match = True
+                break
+
+    # Set the step outputs
+    print("Building docs:", doc_match)
+    print("::set-output name=docs-build::" + str(doc_match))
+
+
+def check_changed_files():
+    if not changed_files or (
+        os.environ.get("GITHUB_EVENT_NAME", "") == "push"
+        and os.environ.get("GITHUB_REPOSITORY", "") == "adafruit/circuitpython"
+    ):
+        print("Building all docs/boards because of adafruit/circuitpython branch")
+        return True
+    else:
+        print("Adding docs/boards to build based on changed files")
+        return False
+
+
+def main():
+    build_all = check_changed_files()
+    set_docs_to_build(build_all)
+    set_boards_to_build(build_all)
+
+
+if __name__ == "__main__":
+    main()