From 592fea682297177cd5710f4a507fcd33eec5ab96 Mon Sep 17 00:00:00 2001 From: MicroDev <70126934+MicroDev1@users.noreply.github.com> Date: Wed, 22 Feb 2023 20:08:13 +0530 Subject: [PATCH 1/4] split `atmel-samd` instead of `raspberrypi` --- .github/workflows/build.yml | 22 +++++++++++----------- tools/ci_set_matrix.py | 15 +++++++-------- 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7029cc4897..b275a6d86d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -24,9 +24,9 @@ jobs: build-boards: ${{ steps.set-matrix.outputs.build-boards }} boards-aarch: ${{ steps.set-matrix.outputs.boards-aarch }} boards-arm: ${{ steps.set-matrix.outputs.boards-arm }} + boards-atmel: ${{ steps.set-matrix.outputs.boards-atmel }} boards-esp: ${{ steps.set-matrix.outputs.boards-esp }} boards-riscv: ${{ steps.set-matrix.outputs.boards-riscv }} - boards-rpi: ${{ steps.set-matrix.outputs.boards-rpi }} cp-version: ${{ steps.set-up-submodules.outputs.version }} steps: - name: Dump GitHub context @@ -238,6 +238,16 @@ jobs: boards: ${{ needs.scheduler.outputs.boards-arm }} cp-version: ${{ needs.scheduler.outputs.cp-version }} + atmel: + needs: [scheduler, mpy-cross, tests] + if: ${{ needs.scheduler.outputs.boards-atmel != '[]' }} + uses: ./.github/workflows/build-boards.yml + secrets: inherit + with: + platform: arm + boards: ${{ needs.scheduler.outputs.boards-atmel }} + cp-version: ${{ needs.scheduler.outputs.cp-version }} + esp: needs: [scheduler, mpy-cross, tests] if: ${{ needs.scheduler.outputs.boards-esp != '[]' }} @@ -257,13 +267,3 @@ jobs: platform: riscv boards: ${{ needs.scheduler.outputs.boards-riscv }} cp-version: ${{ needs.scheduler.outputs.cp-version }} - - rpi: - needs: [scheduler, mpy-cross, tests] - if: ${{ needs.scheduler.outputs.boards-rpi != '[]' }} - uses: ./.github/workflows/build-boards.yml - secrets: inherit - with: - platform: arm - boards: ${{ needs.scheduler.outputs.boards-rpi }} - cp-version: ${{ needs.scheduler.outputs.cp-version }} diff --git a/tools/ci_set_matrix.py b/tools/ci_set_matrix.py index 8c248d8fd7..374e2f0c2b 100755 --- a/tools/ci_set_matrix.py +++ b/tools/ci_set_matrix.py @@ -43,14 +43,14 @@ from shared_bindings_matrix import ( ) PORT_TO_ARCH = { - "atmel-samd": "arm", + "atmel-samd": "atmel", "broadcom": "aarch", "cxd56": "arm", "espressif": "esp", "litex": "riscv", "mimxrt10xx": "arm", "nrf": "arm", - "raspberrypi": "rpi", + "raspberrypi": "arm", "stm": "arm", } @@ -92,7 +92,7 @@ def set_output(name: str, value): def set_boards_to_build(build_all: bool): - if "mpy_cross" in last_failed_jobs or "tests" in last_failed_jobs: + if last_failed_jobs.get("mpy_cross") or last_failed_jobs.get("tests"): build_all = True # Get boards in json format @@ -207,13 +207,12 @@ def set_boards_to_build(build_all: bool): break # Split boards by architecture. - arch_to_boards = {"aarch": [], "arm": [], "esp": [], "riscv": [], "rpi": []} + arch_to_boards = {"aarch": [], "arm": [], "atmel": [], "esp": [], "riscv": []} # Append previously failed boards for arch in arch_to_boards: - arch_to_job = f"build-{arch}" - if arch_to_job in last_failed_jobs: - for board in last_failed_jobs[arch_to_job]: + if arch in last_failed_jobs: + for board in last_failed_jobs[arch]: if not board in boards_to_build: boards_to_build.append(board) @@ -238,7 +237,7 @@ def set_boards_to_build(build_all: bool): def set_docs_to_build(build_doc: bool): if not build_doc: - if "build-doc" in last_failed_jobs: + if last_failed_jobs.get("build-doc"): build_doc = True else: doc_pattern = re.compile( From 1ea21cc8fc210f9a16f92c77c632a8bd40812671 Mon Sep 17 00:00:00 2001 From: MicroDev <70126934+MicroDev1@users.noreply.github.com> Date: Wed, 22 Feb 2023 20:11:38 +0530 Subject: [PATCH 2/4] refactor and simplify fetching port deps --- .github/actions/deps/ports/action.yml | 24 +++++++++++++++++++++++ .github/workflows/build-boards-custom.yml | 19 +++++++----------- .github/workflows/build-boards.yml | 5 +++-- 3 files changed, 34 insertions(+), 14 deletions(-) create mode 100644 .github/actions/deps/ports/action.yml diff --git a/.github/actions/deps/ports/action.yml b/.github/actions/deps/ports/action.yml new file mode 100644 index 0000000000..2aa50a1cc2 --- /dev/null +++ b/.github/actions/deps/ports/action.yml @@ -0,0 +1,24 @@ +name: Fetch port deps + +inputs: + board: + required: true + type: string + +outputs: + port: + value: ${{ steps.board-to-port.outputs.port }} + +runs: + using: composite + steps: + - name: Board to port + id: board-to-port + run: | + PORT=$(find ports/*/boards/ -type d -name ${{ inputs.board }} | sed 's/^ports\///g;s/\/boards.*//g') + if [ -z $PORT ]; then (exit 1); else echo >> $GITHUB_OUTPUT "port=$PORT"; fi + shell: bash + + - name: Set up espressif port + if: steps.board-to-port.outputs.port == 'espressif' + uses: ./.github/actions/deps/ports/espressif diff --git a/.github/workflows/build-boards-custom.yml b/.github/workflows/build-boards-custom.yml index 22ba0f62f4..df8197d9ae 100644 --- a/.github/workflows/build-boards-custom.yml +++ b/.github/workflows/build-boards-custom.yml @@ -51,20 +51,15 @@ jobs: if: inputs.debug || inputs.flags != '' run: | > custom-build && git add custom-build - - name: Get port - id: get-port - run: | - PORT=$(find ports/*/boards/ -type d -name ${{ inputs.board }} | sed 's/^ports\///g;s/\/boards.*//g') - if [ -z $PORT ]; then (exit 1); else echo >> $GITHUB_OUTPUT "port=$PORT"; fi - - name: Port to platform - run: echo >> $GITHUB_ENV "PLATFORM=${{ env[format('PLATFORM_{0}', steps.get-port.outputs.port)] }}" - name: Set up python uses: actions/setup-python@v4 with: python-version: 3.x - name: Set up port - if: env.PLATFORM == 'esp' - uses: ./.github/actions/deps/ports/espressif + id: set-up-port + uses: ./.github/actions/deps/ports + with: + board: ${{ inputs.board }} - name: Set up submodules id: set-up-submodules uses: ./.github/actions/deps/submodules @@ -75,7 +70,7 @@ jobs: uses: ./.github/actions/deps/external with: action: cache - platform: ${{ env.PLATFORM }} + platform: ${{ env[format('PLATFORM_{0}', steps.set-up-port.outputs.port)] }} - name: Set up mpy-cross if: steps.set-up-submodules.outputs.frozen == 'True' uses: ./.github/actions/mpy_cross @@ -96,9 +91,9 @@ jobs: mkfs.fat --version || true - name: Build board run: make -j2 ${{ inputs.flags }} BOARD=${{ inputs.board }} DEBUG=${{ inputs.debug && '1' || '0' }} TRANSLATION=${{ inputs.language }} - working-directory: ports/${{ steps.get-port.outputs.port }} + working-directory: ports/${{ steps.set-up-port.outputs.port }} - name: Upload artifact uses: actions/upload-artifact@v3 with: name: ${{ inputs.board }}-${{ inputs.language }}-${{ inputs.version }}${{ inputs.flags != '' && '-custom' || '' }}${{ inputs.debug && '-debug' || '' }} - path: ports/${{ steps.get-port.outputs.port }}/build-${{ inputs.board }}/firmware.* + path: ports/${{ steps.set-up-port.outputs.port }}/build-${{ inputs.board }}/firmware.* diff --git a/.github/workflows/build-boards.yml b/.github/workflows/build-boards.yml index 457fce5b98..b975c131ee 100644 --- a/.github/workflows/build-boards.yml +++ b/.github/workflows/build-boards.yml @@ -38,8 +38,9 @@ jobs: with: python-version: 3.x - name: Set up port - if: inputs.platform == 'esp' - uses: ./.github/actions/deps/ports/espressif + uses: ./.github/actions/deps/ports + with: + board: ${{ matrix.board }} - name: Set up submodules id: set-up-submodules uses: ./.github/actions/deps/submodules From f4f95ada79808ab95ba51f3b0fd0535ea0906921 Mon Sep 17 00:00:00 2001 From: MicroDev <70126934+MicroDev1@users.noreply.github.com> Date: Wed, 22 Feb 2023 20:30:21 +0530 Subject: [PATCH 3/4] fix empty exclude commit --- .github/workflows/build.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b275a6d86d..453804a26a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -54,6 +54,10 @@ jobs: uses: ./.github/actions/deps/external with: action: cache + - name: Set up mpy-cross + uses: ./.github/actions/mpy_cross + with: + download: false - name: Get last commit with checks id: get-last-commit-with-checks if: github.event_name == 'pull_request' @@ -63,14 +67,10 @@ jobs: REPO: ${{ github.repository }} PULL: ${{ github.event.number }} GITHUB_TOKEN: ${{ github.token }} - EXCLUDE_COMMIT: ${{ github.event.after }} - - name: Set up mpy-cross - uses: ./.github/actions/mpy_cross - with: - download: false + EXCLUDE_COMMIT: ${{ github.event.pull_request.head.sha }} - name: Set head sha if: github.event_name == 'pull_request' - run: echo "HEAD_SHA=$(git show -s --format=%s $GITHUB_SHA | grep -o -P "(?<=Merge ).*(?= into)")" >> $GITHUB_ENV + run: echo "HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV - name: Set base sha if: github.event_name == 'pull_request' run: | From 60a9c7e5b2570a84c07c6c0c78f73c63322937f0 Mon Sep 17 00:00:00 2001 From: MicroDev <70126934+MicroDev1@users.noreply.github.com> Date: Thu, 23 Feb 2023 12:49:34 +0530 Subject: [PATCH 4/4] move windows workflow to build ci and more --- .github/workflows/build.yml | 69 +++++++++++++++++ .github/workflows/ports_windows.yml | 111 ---------------------------- tools/ci_changes_per_commit.py | 2 +- tools/ci_fetch_deps.py | 8 +- tools/ci_set_matrix.py | 50 +++++++++++-- 5 files changed, 118 insertions(+), 122 deletions(-) delete mode 100644 .github/workflows/ports_windows.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 453804a26a..0c8b927de8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -22,6 +22,7 @@ jobs: outputs: build-doc: ${{ steps.set-matrix.outputs.build-doc }} build-boards: ${{ steps.set-matrix.outputs.build-boards }} + build-windows: ${{ steps.set-matrix.outputs.build-windows }} boards-aarch: ${{ steps.set-matrix.outputs.boards-aarch }} boards-arm: ${{ steps.set-matrix.outputs.boards-arm }} boards-atmel: ${{ steps.set-matrix.outputs.boards-atmel }} @@ -218,6 +219,74 @@ jobs: [ -z "$TWINE_USERNAME" ] || echo "Uploading dev release to PyPi" [ -z "$TWINE_USERNAME" ] || twine upload circuitpython-stubs/dist/* + build-windows: + runs-on: windows-2022 + needs: scheduler + if: ${{ needs.scheduler.outputs.build-windows == 'True' }} + env: + CP_VERSION: ${{ needs.scheduler.outputs.cp-version }} + defaults: + run: + # We define a custom shell script here, although `msys2.cmd` does neither exist nor is it available in the PATH yet + shell: msys2 {0} + steps: + # We want to change the configuration of the git command that actions/checkout will be using + # (since it is not possible to set autocrlf through the action yet, see actions/checkout#226). + - run: git config --global core.autocrlf input + shell: bash + - name: Check python coding (cmd) + run: python -c "import sys, locale; print(sys.getdefaultencoding(), locale.getpreferredencoding(False))" + shell: cmd + # We use a JS Action, which calls the system terminal or other custom terminals directly, if required + - uses: msys2/setup-msys2@v2 + with: + install: base-devel git wget unzip gcc python-pip + # The goal of this was to test how things worked when the default file encoding (locale.getpreferedencoding()) + # was not UTF-8. However, msys2 python does use utf-8 as the preferred file encoding, and using actions/setup-python + # python3.8 gave a broken build, so we're not really testing what we wanted to test. + # However, commandline length limits are being tested so that does some good. + - name: Check python coding (msys2) + run: | + locale -v + which python; python --version + python -c "import sys, locale; print(sys.getdefaultencoding(), locale.getpreferredencoding(False))" + which python3; python3 --version + python3 -c "import sys, locale; print(sys.getdefaultencoding(), locale.getpreferredencoding(False))" + - name: Install dependencies + run: | + wget --no-verbose -O gcc-arm.zip https://developer.arm.com/-/media/Files/downloads/gnu-rm/10-2020q4/gcc-arm-none-eabi-10-2020-q4-major-win32.zip + unzip -q -d /tmp gcc-arm.zip + tar -C /tmp/gcc-arm-none-* -cf - . | tar -C /usr/local -xf - + pip install wheel + # requirements_dev.txt doesn't install on windows. (with msys2 python) + # instead, pick a subset for what we want to do + pip install cascadetoml jinja2 typer click intelhex + # check that installed packages work....? + which python; python --version; python -c "import cascadetoml" + which python3; python3 --version; python3 -c "import cascadetoml" + - name: Set up repository + uses: actions/checkout@v3 + with: + submodules: false + fetch-depth: 1 + - name: Set up submodules + uses: ./.github/actions/deps/submodules + - name: build mpy-cross + run: make -j2 -C mpy-cross + - name: build rp2040 + run: make -j2 -C ports/raspberrypi BOARD=adafruit_feather_rp2040 TRANSLATION=de_DE + - name: build samd21 + run: make -j2 -C ports/atmel-samd BOARD=feather_m0_express TRANSLATION=zh_Latn_pinyin + - name: build samd51 + run: make -j2 -C ports/atmel-samd BOARD=feather_m4_express TRANSLATION=es + - name: build nrf + run: make -j2 -C ports/nrf BOARD=feather_nrf52840_express TRANSLATION=fr + - name: build stm + run: make -j2 -C ports/stm BOARD=feather_stm32f405_express TRANSLATION=pt_BR + # I gave up trying to do esp builds on windows when I saw + # ERROR: Platform MINGW64_NT-10.0-17763-x86_64 appears to be unsupported + # https://github.com/espressif/esp-idf/issues/7062 + aarch: needs: [scheduler, mpy-cross, tests] if: ${{ needs.scheduler.outputs.boards-aarch != '[]' }} diff --git a/.github/workflows/ports_windows.yml b/.github/workflows/ports_windows.yml deleted file mode 100644 index ba2042660a..0000000000 --- a/.github/workflows/ports_windows.yml +++ /dev/null @@ -1,111 +0,0 @@ -name: windows port - -on: - push: - pull_request: - paths: - - '.github/workflows/ports_windows.yml' - - 'extmod/**' - - 'lib/**' - - 'mpy-cross/**' - - 'ports/unix/**' - - 'ports/windows/**' - - 'py/**' - - 'requirements*.txt' - - 'tools/**' - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -jobs: - windows: - runs-on: windows-2022 - defaults: - run: - # We define a custom shell script here, although `msys2.cmd` does neither exist nor is it available in the PATH yet - shell: msys2 {0} - steps: - - # We want to change the configuration of the git command that actions/checkout will be using (since it is not possible to set autocrlf through the action yet, see actions/checkout#226). - - run: git config --global core.autocrlf input - shell: bash - - - name: Check python coding (cmd) - run: python -c "import sys, locale; print(sys.getdefaultencoding(), locale.getpreferredencoding(False))" - shell: cmd - - # We use a JS Action, which calls the system terminal or other custom terminals directly, if required - - uses: msys2/setup-msys2@v2 - with: - update: true - install: base-devel git wget unzip gcc python-pip - - # The goal of this was to test how things worked when the default file - # encoding (locale.getpreferedencoding()) was not UTF-8. However, msys2 - # python does use utf-8 as the preferred file encoding, and using - # actions/setup-python python3.8 gave a broken build, so we're not really - # testing what we wanted to test. - # - # however, commandline length limits are being tested so that does some - # good. - - name: Check python coding (msys2) - run: | - locale -v - which python; python --version - python -c "import sys, locale; print(sys.getdefaultencoding(), locale.getpreferredencoding(False))" - which python3; python3 --version - python3 -c "import sys, locale; print(sys.getdefaultencoding(), locale.getpreferredencoding(False))" - - - name: Install dependencies - run: | - wget --no-verbose -O gcc-arm.zip https://developer.arm.com/-/media/Files/downloads/gnu-rm/10-2020q4/gcc-arm-none-eabi-10-2020-q4-major-win32.zip - unzip -q -d /tmp gcc-arm.zip - tar -C /tmp/gcc-arm-none-* -cf - . | tar -C /usr/local -xf - - pip install wheel - # requirements_dev.txt doesn't install on windows. (with msys2 python) - # instead, pick a subset for what we want to do - pip install cascadetoml jinja2 typer click intelhex - # check that installed packages work....? - which python; python --version; python -c "import cascadetoml" - which python3; python3 --version; python3 -c "import cascadetoml" - - - name: Set up repository - uses: actions/checkout@v3 - with: - submodules: false - fetch-depth: 1 - - - name: Set up submodules - uses: ./.github/actions/deps/submodules - with: - version: true - - - name: build mpy-cross - run: make -j2 -C mpy-cross - - - name: build rp2040 - run: make -j2 -C ports/raspberrypi BOARD=adafruit_feather_rp2040 TRANSLATION=de_DE - - - name: build samd21 - run: make -j2 -C ports/atmel-samd BOARD=feather_m0_express TRANSLATION=zh_Latn_pinyin - - - name: build samd51 - run: make -j2 -C ports/atmel-samd BOARD=feather_m4_express TRANSLATION=es - - - name: build nrf - run: make -j2 -C ports/nrf BOARD=feather_nrf52840_express TRANSLATION=fr - - - name: build stm - run: make -j2 -C ports/stm BOARD=feather_stm32f405_express TRANSLATION=pt_BR - -# I gave up trying to do esp32 builds on windows when I saw -# ERROR: Platform MINGW64_NT-10.0-17763-x86_64 appears to be unsupported -# https://github.com/espressif/esp-idf/issues/7062 -# -# - name: prepare esp -# run: ports/espressif/esp-idf/install.bat -# shell: cmd -# -# - name: build esp -# run: . ports/espressif/esp-idf/export.sh && make -j2 -C ports/espressif BOARD=adafruit_metro_esp32s2 diff --git a/tools/ci_changes_per_commit.py b/tools/ci_changes_per_commit.py index 6452f241c0..d5f621d263 100644 --- a/tools/ci_changes_per_commit.py +++ b/tools/ci_changes_per_commit.py @@ -172,7 +172,7 @@ def get_bad_check_runs(query_check_runs): run_types = ["failed", "incomplete"] - regex_matrix = re.compile("^[^\n ]+ \/ (build|run) \([^\n ]+\)$") + regex_matrix = re.compile(r"^\S+ \/ (build|run) \(\S+\)$") while more_pages: check_runs = query_check_runs.fetch()["data"]["node"] diff --git a/tools/ci_fetch_deps.py b/tools/ci_fetch_deps.py index bdf2967af8..3b495f8758 100644 --- a/tools/ci_fetch_deps.py +++ b/tools/ci_fetch_deps.py @@ -80,14 +80,16 @@ def main(): submodules = ["extmod/ulab", "lib/", "tools/"] elif TARGET == "build-doc": # used in .readthedocs.yml to generate RTD - submodules = ["extmod/ulab", "frozen/"] + submodules = ["extmod/ulab"] + submodules_tags = ["frozen/"] elif TARGET == "mpy-cross" or TARGET == "mpy-cross-mac": submodules = ["tools/"] # for huffman - elif TARGET == "windows": + elif TARGET == "build-windows": # This builds one board from a number of ports so fill out a bunch of submodules submodules = ["extmod/ulab", "lib/", "tools/", "ports/", "data/nvm.toml"] elif TARGET == "website": - submodules = ["tools/adabot/", "frozen/"] + submodules = ["tools/adabot/"] + submodules_tags = ["frozen/"] elif TARGET == "pre-commit": submodules = ["extmod/ulab"] else: diff --git a/tools/ci_set_matrix.py b/tools/ci_set_matrix.py index 374e2f0c2b..43a9790160 100755 --- a/tools/ci_set_matrix.py +++ b/tools/ci_set_matrix.py @@ -62,6 +62,24 @@ IGNORE = [ # Files in these directories never influence board builds IGNORE_DIRS = ["tests", "docs", ".devcontainer"] +PATTERN_DOCS = ( + r"^(?:\.github|docs|extmod\/ulab)|" + r"^(?:(?:ports\/\w+\/bindings|shared-bindings)\S+\.c|tools\/extract_pyi\.py|conf\.py|requirements-doc\.txt)$|" + r"(?:-stubs|\.(?:md|MD|rst|RST))$" +) + +PATTERN_WINDOWS = [ + ".github/", + "extmod/", + "lib/", + "mpy-cross/", + "ports/unix/", + "ports/windows/", + "py/", + "requirements", + "tools/", +] + if len(sys.argv) > 1: print("Using files list on commandline") changed_files = sys.argv[1:] @@ -240,21 +258,19 @@ def set_docs_to_build(build_doc: bool): if last_failed_jobs.get("build-doc"): build_doc = True else: - doc_pattern = re.compile( - r"^(?:\.github\/workflows\/|docs|extmod\/ulab|(?:(?:ports\/\w+\/bindings|shared-bindings)\S+\.c|conf\.py|tools\/extract_pyi\.py|requirements-doc\.txt)$)|(?:-stubs|\.(?:md|MD|rst|RST))$" - ) + doc_pattern = re.compile(PATTERN_DOCS) github_workspace = os.environ.get("GITHUB_WORKSPACE") or "" github_workspace = github_workspace and github_workspace + "/" - for p in changed_files: - if doc_pattern.search(p) and ( + for file in changed_files: + if doc_pattern.search(file) and ( ( subprocess.run( - f"git diff -U0 $BASE_SHA...$HEAD_SHA {github_workspace + p} | grep -o -m 1 '^[+-]\/\/|'", + f"git diff -U0 $BASE_SHA...$HEAD_SHA {github_workspace + file} | grep -o -m 1 '^[+-]\/\/|'", capture_output=True, shell=True, ).stdout ) - if p.endswith(".c") + if file.endswith(".c") else True ): build_doc = True @@ -265,6 +281,25 @@ def set_docs_to_build(build_doc: bool): set_output("build-doc", build_doc) +def set_windows_to_build(build_windows): + if not build_windows: + if last_failed_jobs.get("build-windows"): + build_windows = True + else: + for file in changed_files: + for pattern in PATTERN_WINDOWS: + if file.startswith(pattern): + build_windows = True + break + else: + continue + break + + # Set the step outputs + print("Building windows:", build_windows) + set_output("build-windows", build_windows) + + def check_changed_files(): if not changed_files: print("Building all docs/boards") @@ -277,6 +312,7 @@ def check_changed_files(): def main(): build_all = check_changed_files() set_docs_to_build(build_all) + set_windows_to_build(build_all) set_boards_to_build(build_all)