diff --git a/.github/actions/deps/external/action.yml b/.github/actions/deps/external/action.yml new file mode 100644 index 0000000000..3503e5b194 --- /dev/null +++ b/.github/actions/deps/external/action.yml @@ -0,0 +1,84 @@ +name: Fetch external deps + +inputs: + platform: + required: false + default: none + type: choice + options: + - arm + - aarch + - esp + - riscv + - none + +runs: + using: composite + steps: + # aarch + - name: Get aarch toolchain + if: inputs.platform == 'aarch' + run: | + wget --no-verbose https://adafruit-circuit-python.s3.amazonaws.com/gcc-arm-10.3-2021.07-x86_64-aarch64-none-elf.tar.xz + sudo tar -C /usr --strip-components=1 -xaf gcc-arm-10.3-2021.07-x86_64-aarch64-none-elf.tar.xz + sudo apt-get install -y mtools + shell: bash + - name: Install mkfs.fat + if: inputs.platform == 'aarch' + run: | + wget https://github.com/dosfstools/dosfstools/releases/download/v4.2/dosfstools-4.2.tar.gz + tar -xaf dosfstools-4.2.tar.gz + cd dosfstools-4.2 + ./configure + make -j 2 + cd src + echo >> $GITHUB_PATH $(pwd) + shell: bash + + # arm + - name: Get arm toolchain + if: inputs.platform == 'aarch' || inputs.platform == 'arm' + uses: carlosperate/arm-none-eabi-gcc-action@v1 + with: + release: '10-2020-q4' + + # esp + - name: Get esp toolchain + if: inputs.platform == 'esp' + run: sudo apt-get install -y ninja-build + shell: bash + - name: Install IDF tools + if: inputs.platform == 'esp' + run: | + echo "Installing ESP-IDF tools" + $IDF_PATH/tools/idf_tools.py --non-interactive install required + $IDF_PATH/tools/idf_tools.py --non-interactive install cmake + echo "Installing Python environment and packages" + $IDF_PATH/tools/idf_tools.py --non-interactive install-python-env + rm -rf $IDF_TOOLS_PATH/dist + shell: bash + - name: Set environment + if: inputs.platform == 'esp' + run: | + source $IDF_PATH/export.sh + echo >> $GITHUB_ENV "IDF_PYTHON_ENV_PATH=$IDF_PYTHON_ENV_PATH" + echo >> $GITHUB_PATH "$PATH" + shell: bash + + # riscv + - name: Get riscv toolchain + if: inputs.platform == 'riscv' + run: | + wget https://static.dev.sifive.com/dev-tools/riscv64-unknown-elf-gcc-8.3.0-2019.08.0-x86_64-linux-centos6.tar.gz + sudo tar -C /usr --strip-components=1 -xaf riscv64-unknown-elf-gcc-8.3.0-2019.08.0-x86_64-linux-centos6.tar.gz + shell: bash + + # common + - name: Cache python dependencies + if: inputs.platform != 'esp' + uses: ./.github/actions/deps/python + with: + action: ${{ fromJSON('["restore", "cache"]')[github.job == 'scheduler'] }} + - name: Install python dependencies + run: pip install -r requirements-dev.txt + shell: bash diff --git a/.github/actions/deps/ports/espressif/action.yml b/.github/actions/deps/ports/espressif/action.yml new file mode 100644 index 0000000000..aa1e98f8a0 --- /dev/null +++ b/.github/actions/deps/ports/espressif/action.yml @@ -0,0 +1,36 @@ +name: Fetch espressif port deps + +runs: + using: composite + steps: + - name: Set IDF env + run: | + echo >> $GITHUB_ENV "IDF_PATH=$GITHUB_WORKSPACE/ports/espressif/esp-idf" + echo >> $GITHUB_ENV "IDF_TOOLS_PATH=$GITHUB_WORKSPACE/.idf_tools" + shell: bash + + - name: Get IDF commit + id: idf-commit + run: | + COMMIT=$(git submodule status ports/espressif/esp-idf | grep -o -P '(?<=^-).*(?= )') + echo "$COMMIT" + echo "commit=$COMMIT" >> $GITHUB_OUTPUT + shell: bash + + - name: Cache IDF submodules + uses: actions/cache@v3 + with: + path: | + .git/modules/ports/espressif/esp-idf + ports/espressif/esp-idf + key: submodules-idf-${{ steps.idf-commit.outputs.commit }} + + - name: Cache IDF tools + uses: actions/cache@v3 + with: + path: ${{ env.IDF_TOOLS_PATH }} + key: ${{ runner.os }}-${{ env.pythonLocation }}-tools-idf-${{ steps.idf-commit.outputs.commit }} + + - name: Initialize IDF submodules + run: git submodule update --init --depth=1 --recursive $IDF_PATH + shell: bash diff --git a/.github/actions/deps/python/action.yml b/.github/actions/deps/python/action.yml new file mode 100644 index 0000000000..9b3732c9e0 --- /dev/null +++ b/.github/actions/deps/python/action.yml @@ -0,0 +1,42 @@ +name: Fetch python deps + +inputs: + action: + description: The cache action to use + required: false + default: restore + type: choice + options: + - cache + - restore + +runs: + using: composite + steps: + - name: Cache python dependencies + id: cache-python-deps + if: inputs.action == 'cache' + uses: actions/cache@v3 + with: + path: .cp_tools + key: ${{ runner.os }}-${{ env.pythonLocation }}-tools-cp-${{ hashFiles('requirements-dev.txt') }} + + - name: Restore python dependencies + id: restore-python-deps + if: inputs.action == 'restore' + uses: actions/cache/restore@v3 + with: + path: .cp_tools + key: ${{ runner.os }}-${{ env.pythonLocation }}-tools-cp-${{ hashFiles('requirements-dev.txt') }} + + - name: Set up venv + if: inputs.action == 'cache' && !steps.cache-python-deps.outputs.cache-hit + run: python -m venv .cp_tools + shell: bash + + - name: Activate venv + if: inputs.action == 'cache' || (inputs.action == 'restore' && steps.restore-python-deps.outputs.cache-hit) + run: | + source .cp_tools/bin/activate + echo >> $GITHUB_PATH "$PATH" + shell: bash diff --git a/.github/actions/fetch_submodules/action.yml b/.github/actions/deps/submodules/action.yml similarity index 89% rename from .github/actions/fetch_submodules/action.yml rename to .github/actions/deps/submodules/action.yml index 407259967c..bca76b204c 100644 --- a/.github/actions/fetch_submodules/action.yml +++ b/.github/actions/deps/submodules/action.yml @@ -1,13 +1,18 @@ name: 'Fetch Submodules' inputs: + target: + description: 'The target for ci_fetch_deps' + required: false + type: string + submodules: description: 'The submodules to cache' required: false default: '["extmod/ulab", "lib/", "tools/"]' type: string - cache: + action: description: 'The cache action to use' required: false default: 'restore' @@ -42,7 +47,7 @@ runs: shell: bash - name: Cache submodules - if: ${{ inputs.cache == 'cache' }} + if: ${{ inputs.action == 'cache' }} uses: actions/cache@v3 with: path: ".git/modules/\n${{ join(fromJSON(steps.create-submodule-status.outputs.submodules), '\n') }}" @@ -50,7 +55,7 @@ runs: enableCrossOsArchive: true - name: Restore submodules - if: ${{ inputs.cache == 'restore' }} + if: ${{ inputs.action == 'restore' }} uses: actions/cache/restore@v3 with: path: ".git/modules/\n${{ join(fromJSON(steps.create-submodule-status.outputs.submodules), '\n') }}" @@ -63,7 +68,7 @@ runs: - name: CircuitPython dependencies id: cp-deps - run: python tools/ci_fetch_deps.py ${{ matrix.board || github.job }} + run: python tools/ci_fetch_deps.py ${{ inputs.target || matrix.board || github.job }} shell: bash - name: CircuitPython version diff --git a/.github/actions/mpy_cross/action.yml b/.github/actions/mpy_cross/action.yml new file mode 100644 index 0000000000..d9fe54fdc7 --- /dev/null +++ b/.github/actions/mpy_cross/action.yml @@ -0,0 +1,37 @@ +name: Set up mpy-cross + +inputs: + download: + required: false + default: true + type: boolean + +runs: + using: composite + steps: + - name: Download mpy-cross + id: download-mpy-cross + if: inputs.download == 'true' + continue-on-error: true + uses: actions/download-artifact@v3 + with: + name: mpy-cross + path: mpy-cross + + - name: Make mpy-cross executable + if: inputs.download == 'true' && steps.download-mpy-cross.outcome == 'success' + run: sudo chmod +x mpy-cross/mpy-cross + shell: bash + + - name: Build mpy-cross + if: inputs.download == 'false' || steps.download-mpy-cross.outcome == 'failure' + run: make -C mpy-cross -j2 + shell: bash + + - name: Upload mpy-cross + if: inputs.download == 'false' || steps.download-mpy-cross.outcome == 'failure' + continue-on-error: true + uses: actions/upload-artifact@v3 + with: + name: mpy-cross + path: mpy-cross/mpy-cross diff --git a/.github/actions/upload_aws/action.yml b/.github/actions/upload_aws/action.yml new file mode 100644 index 0000000000..7aee8c7e78 --- /dev/null +++ b/.github/actions/upload_aws/action.yml @@ -0,0 +1,32 @@ +name: Upload to AWS S3 + +inputs: + source: + required: true + type: string + + destination: + required: false + type: string + + AWS_ACCESS_KEY_ID: + required: true + + AWS_SECRET_ACCESS_KEY: + required: true + +runs: + using: composite + steps: + - name: Upload to S3 + if: >- + (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') || + (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested')) + run: >- + [ -z "$AWS_ACCESS_KEY_ID" ] || + aws s3 cp ${{ inputs.source }} s3://adafruit-circuit-python/bin/${{ inputs.destination }} --recursive --no-progress --region us-east-1 + env: + AWS_PAGER: '' + AWS_ACCESS_KEY_ID: ${{ inputs.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ inputs.AWS_SECRET_ACCESS_KEY }} + shell: bash diff --git a/.github/workflows/build-boards.yml b/.github/workflows/build-boards.yml new file mode 100644 index 0000000000..b9e6577a19 --- /dev/null +++ b/.github/workflows/build-boards.yml @@ -0,0 +1,82 @@ +name: Build boards + +on: + workflow_call: + inputs: + platform: + required: true + type: string + + boards: + required: true + type: string + + cp-version: + required: true + type: string + +jobs: + build: + runs-on: ubuntu-22.04 + env: + CP_VERSION: ${{ inputs.cp-version }} + strategy: + fail-fast: false + matrix: + board: ${{ fromJSON(inputs.boards) }} + steps: + - name: Set up repository + uses: actions/checkout@v3 + with: + submodules: false + fetch-depth: 1 + - name: Set up python + uses: actions/setup-python@v4 + with: + python-version: 3.x + - name: Set up port + if: inputs.platform == 'esp' + uses: ./.github/actions/deps/ports/espressif + - name: Set up submodules + id: set-up-submodules + uses: ./.github/actions/deps/submodules + - name: Set up external + uses: ./.github/actions/deps/external + with: + platform: ${{ inputs.platform }} + - name: Set up mpy-cross + if: steps.set-up-submodules.outputs.frozen == 'True' + uses: ./.github/actions/mpy_cross + + - name: Versions + run: | + gcc --version + python3 --version + cmake --version || true + ninja --version || true + aarch64-none-elf-gcc --version || true + arm-none-eabi-gcc --version || true + xtensa-esp32-elf-gcc --version || true + riscv32-esp-elf-gcc --version || true + riscv64-unknown-elf-gcc --version || true + mkfs.fat --version || true + + - name: Set up build failure matcher + run: echo "::add-matcher::$GITHUB_WORKSPACE/.github/workflows/match-build-fail.json" + - name: Build board + run: python3 -u build_release_files.py + working-directory: tools + env: + BOARDS: ${{ matrix.board }} + + - name: Upload artifact + uses: actions/upload-artifact@v3 + with: + name: ${{ matrix.board }} + path: bin/${{ matrix.board }} + - name: Upload to S3 + uses: ./.github/actions/upload_aws + with: + source: bin/ + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/build-mpy-cross.yml b/.github/workflows/build-mpy-cross.yml new file mode 100644 index 0000000000..c17c9da1cb --- /dev/null +++ b/.github/workflows/build-mpy-cross.yml @@ -0,0 +1,65 @@ +name: Build mpy-cross + +on: + workflow_call: + inputs: + cp-version: + required: true + type: string + +jobs: + build: + runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + mpy-cross: ["static", "static-aarch64", "static-mingw", "static-raspbian"] + env: + CP_VERSION: ${{ inputs.cp-version }} + EX_static-mingw: static.exe + OS_static: linux-amd64 + OS_static-aarch64: linux-aarch64 + OS_static-mingw: windows + OS_static-raspbian: linux-raspbian + steps: + - name: Set up repository + uses: actions/checkout@v3 + with: + submodules: false + fetch-depth: 1 + - name: Set up python + uses: actions/setup-python@v4 + with: + python-version: 3.x + - name: Set up submodules + uses: ./.github/actions/deps/submodules + with: + target: mpy-cross + + - name: Install toolchain (aarch64) + if: matrix.mpy-cross == 'static-aarch64' + run: sudo apt-get install -y gcc-aarch64-linux-gnu + - name: Install toolchain (mingw) + if: matrix.mpy-cross == 'static-mingw' + run: sudo apt-get install -y mingw-w64 + + - name: Build mpy-cross.${{ matrix.mpy-cross }} + run: make -C mpy-cross -j2 -f Makefile.${{ matrix.mpy-cross }} + + - name: Set output + run: | + echo >> $GITHUB_ENV "EX=${{ env[format('EX_{0}', matrix.mpy-cross)] || matrix.mpy-cross }}" + echo >> $GITHUB_ENV "OS=${{ env[format('OS_{0}', matrix.mpy-cross)] }}" + + - name: Upload artifact + uses: actions/upload-artifact@v3 + with: + name: mpy-cross.${{ env.EX }} + path: mpy-cross/mpy-cross.${{ env.EX }} + - name: Upload to S3 + uses: ./.github/actions/upload_aws + with: + source: mpy-cross/mpy-cross.${{ env.EX }} + destination: mpy-cross/${{ env.OS }}/mpy-cross-${{ env.OS }}-${{ env.CP_VERSION }}.${{ env.EX }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f7d57041e6..dae1ffc271 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -17,14 +17,16 @@ concurrency: cancel-in-progress: true jobs: - test: + scheduler: runs-on: ubuntu-20.04 outputs: build-doc: ${{ steps.set-matrix.outputs.build-doc }} + build-boards: ${{ steps.set-matrix.outputs.build-boards }} boards-aarch: ${{ steps.set-matrix.outputs.boards-aarch }} boards-arm: ${{ steps.set-matrix.outputs.boards-arm }} - boards-espressif: ${{ steps.set-matrix.outputs.boards-espressif }} + boards-esp: ${{ steps.set-matrix.outputs.boards-esp }} boards-riscv: ${{ steps.set-matrix.outputs.boards-riscv }} + boards-rpi: ${{ steps.set-matrix.outputs.boards-rpi }} cp-version: ${{ steps.set-up-submodules.outputs.version }} steps: - name: Dump GitHub context @@ -39,107 +41,32 @@ jobs: - name: Set up python uses: actions/setup-python@v4 with: - python-version: "3.x" + python-version: 3.x - name: Duplicate USB VID/PID check run: python3 -u -m tools.ci_check_duplicate_usb_vid_pid - name: Set up submodules id: set-up-submodules - uses: ./.github/actions/fetch_submodules + uses: ./.github/actions/deps/submodules with: - cache: "cache" + action: cache version: true - - name: Install dependencies - run: | - sudo apt-get update - sudo apt-get install -y eatmydata - sudo eatmydata apt-get install -y gettext gcc-aarch64-linux-gnu mingw-w64 - pip install -r requirements-dev.txt - - name: Versions - run: | - gcc --version - python3 --version - - name: Build mpy-cross - run: make -C mpy-cross -j2 - - name: Build unix port - run: | - make -C ports/unix VARIANT=coverage -j2 - - name: Test all - run: MICROPY_CPYTHON3=python3.8 MICROPY_MICROPYTHON=../ports/unix/micropython-coverage ./run-tests.py -j1 - working-directory: tests - - name: Print failure info - run: MICROPY_CPYTHON3=python3.8 MICROPY_MICROPYTHON=../ports/unix/micropython-coverage ./run-tests.py -j1 --print-failures - if: failure() - working-directory: tests - - name: Native Tests - run: MICROPY_CPYTHON3=python3.8 MICROPY_MICROPYTHON=../ports/unix/micropython-coverage ./run-tests.py -j1 --emit native - working-directory: tests - - name: mpy Tests - run: MICROPY_CPYTHON3=python3.8 MICROPY_MICROPYTHON=../ports/unix/micropython-coverage ./run-tests.py -j1 --via-mpy -d basics float micropython - working-directory: tests - - name: Native mpy Tests - run: MICROPY_CPYTHON3=python3.8 MICROPY_MICROPYTHON=../ports/unix/micropython-coverage ./run-tests.py -j1 --via-mpy --emit native -d basics float micropython - working-directory: tests - - name: Build native modules - run: | - make -C examples/natmod/features1 - make -C examples/natmod/features2 - make -C examples/natmod/btree - make -C examples/natmod/framebuf - make -C examples/natmod/uheapq - make -C examples/natmod/urandom - make -C examples/natmod/ure - make -C examples/natmod/uzlib - - name: Test native modules - run: MICROPY_CPYTHON3=python3.8 MICROPY_MICROPYTHON=../ports/unix/micropython-coverage ./run-natmodtests.py extmod/{btree*,framebuf*,uheapq*,ure*,uzlib*}.py - working-directory: tests - - name: Build mpy-cross.static-aarch64 - run: make -C mpy-cross -j2 -f Makefile.static-aarch64 - - uses: actions/upload-artifact@v3 + - name: Set up external + uses: ./.github/actions/deps/external + # Disabled: Needs to be updated + # - name: Get last commit with checks + # id: get-last-commit-with-checks + # if: github.event_name == 'pull_request' + # working-directory: tools + # run: python3 -u ci_changes_per_commit.py + # env: + # REPO: ${{ github.repository }} + # PULL: ${{ github.event.number }} + # GITHUB_TOKEN: ${{ github.token }} + # EXCLUDE_COMMIT: ${{ github.event.after }} + - name: Set up mpy-cross + uses: ./.github/actions/mpy_cross with: - name: mpy-cross.static-aarch64 - path: mpy-cross/mpy-cross.static-aarch64 - - name: Build mpy-cross.static-raspbian - run: make -C mpy-cross -j2 -f Makefile.static-raspbian - - uses: actions/upload-artifact@v3 - with: - name: mpy-cross.static-raspbian - path: mpy-cross/mpy-cross.static-raspbian - - name: Build mpy-cross.static - run: make -C mpy-cross -j2 -f Makefile.static - - uses: actions/upload-artifact@v3 - with: - name: mpy-cross.static-amd64-linux - path: mpy-cross/mpy-cross.static - - name: Build mpy-cross.static-mingw - run: make -C mpy-cross -j2 -f Makefile.static-mingw - - uses: actions/upload-artifact@v3 - with: - name: mpy-cross.static-x64-windows - path: mpy-cross/mpy-cross.static.exe - - name: Upload to S3 - if: >- - (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') || - (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested')) - env: - AWS_PAGER: '' - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - run: | - pip install awscli - [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross/mpy-cross.static-aarch64 s3://adafruit-circuit-python/bin/mpy-cross/mpy-cross.static-aarch64-${{ env.CP_VERSION }} --no-progress --region us-east-1 - [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross/mpy-cross.static-raspbian s3://adafruit-circuit-python/bin/mpy-cross/mpy-cross.static-raspbian-${{ env.CP_VERSION }} --no-progress --region us-east-1 - [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross/mpy-cross.static s3://adafruit-circuit-python/bin/mpy-cross/mpy-cross.static-amd64-linux-${{ env.CP_VERSION }} --no-progress --region us-east-1 - [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross/mpy-cross.static.exe s3://adafruit-circuit-python/bin/mpy-cross/mpy-cross.static-x64-windows-${{ env.CP_VERSION }}.exe --no-progress --region us-east-1 - - name: Get last commit with checks - id: get-last-commit-with-checks - if: github.event_name == 'pull_request' - working-directory: tools - env: - REPO: ${{ github.repository }} - PULL: ${{ github.event.number }} - GITHUB_TOKEN: ${{ github.token }} - EXCLUDE_COMMIT: ${{ github.event.after }} - run: python3 -u ci_changes_per_commit.py + download: false - name: Set head sha if: github.event_name == 'pull_request' run: echo "HEAD_SHA=$(git show -s --format=%s $GITHUB_SHA | grep -o -P "(?<=Merge ).*(?= into)")" >> $GITHUB_ENV @@ -162,17 +89,28 @@ jobs: CHANGED_FILES: ${{ steps.get-changes.outputs.changed_files }} LAST_FAILED_JOBS: ${{ steps.get-last-commit-with-checks.outputs.check_runs }} + tests: + needs: scheduler + if: needs.scheduler.outputs.build-boards == 'True' + uses: ./.github/workflows/run-tests.yml + with: + cp-version: ${{ needs.scheduler.outputs.cp-version }} + + + mpy-cross: + needs: scheduler + if: needs.scheduler.outputs.build-boards == 'True' + uses: ./.github/workflows/build-mpy-cross.yml + with: + cp-version: ${{ needs.scheduler.outputs.cp-version }} + mpy-cross-mac: runs-on: macos-11 - needs: test - if: >- - needs.test.outputs.boards-aarch != '[]' || - needs.test.outputs.boards-arm != '[]' || - needs.test.outputs.boards-espressif != '[]' || - needs.test.outputs.boards-riscv != '[]' + needs: scheduler + if: ${{ needs.scheduler.outputs.build-boards == 'True' }} env: - CP_VERSION: ${{ needs.test.outputs.cp-version }} + CP_VERSION: ${{ needs.scheduler.outputs.cp-version }} steps: - name: Set up repository uses: actions/checkout@v3 @@ -182,9 +120,9 @@ jobs: - name: Set up python uses: actions/setup-python@v4 with: - python-version: "3.x" + python-version: 3.x - name: Set up submodules - uses: ./.github/actions/fetch_submodules + uses: ./.github/actions/deps/submodules - name: Versions run: | gcc --version @@ -204,11 +142,15 @@ jobs: path: mpy-cross/mpy-cross-arm64 - name: Make universal binary run: lipo -create -output mpy-cross-macos-universal mpy-cross/mpy-cross mpy-cross/mpy-cross-arm64 - - uses: actions/upload-artifact@v3 + - name: Upload artifact + uses: actions/upload-artifact@v3 with: name: mpy-cross-macos-11-universal path: mpy-cross-macos-universal - - name: Upload mpy-cross build to S3 + - name: Upload to S3 + if: >- + (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') || + (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested')) run: | [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross-macos-universal s3://adafruit-circuit-python/bin/mpy-cross/mpy-cross-macos-11-${{ env.CP_VERSION }}-universal --no-progress --region us-east-1 [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross/mpy-cross-arm64 s3://adafruit-circuit-python/bin/mpy-cross/mpy-cross-macos-11-${{ env.CP_VERSION }}-arm64 --no-progress --region us-east-1 @@ -217,15 +159,13 @@ jobs: AWS_PAGER: '' AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - if: (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') || (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested')) - build-doc: runs-on: ubuntu-22.04 - needs: test - if: ${{ needs.test.outputs.build-doc == 'True' }} + needs: scheduler + if: ${{ needs.scheduler.outputs.build-doc == 'True' }} env: - CP_VERSION: ${{ needs.test.outputs.cp-version }} + CP_VERSION: ${{ needs.scheduler.outputs.cp-version }} steps: - name: Set up repository uses: actions/checkout@v3 @@ -235,14 +175,12 @@ jobs: - name: Set up python uses: actions/setup-python@v4 with: - python-version: "3.x" + python-version: 3.x - name: Set up submodules - uses: ./.github/actions/fetch_submodules + uses: ./.github/actions/deps/submodules - name: Install dependencies run: | - sudo apt-get update - sudo apt-get install -y eatmydata - sudo eatmydata apt-get install -y latexmk librsvg2-bin texlive-fonts-recommended texlive-latex-recommended texlive-latex-extra + sudo apt-get install -y latexmk librsvg2-bin texlive-fonts-recommended texlive-latex-recommended texlive-latex-extra pip install -r requirements-doc.txt - name: Build and Validate Stubs run: make check-stubs -j2 @@ -264,17 +202,12 @@ jobs: name: docs path: _build/latex - name: Upload to S3 - if: >- - (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') || - (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested')) - env: - AWS_PAGER: '' + uses: ./.github/actions/upload_aws + with: + source: circuitpython-stubs/dist/*.tar.gz + destination: stubs/circuitpython-stubs-${{ env.CP_VERSION }}.zip AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - run: | - pip install awscli - zip -9r circuitpython-stubs.zip circuitpython-stubs - [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp circuitpython-stubs/dist/*.tar.gz s3://adafruit-circuit-python/bin/stubs/circuitpython-stubs-${{ env.CP_VERSION }}.zip --no-progress --region us-east-1 - name: Upload stubs to PyPi if: github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested') env: @@ -286,294 +219,50 @@ jobs: [ -z "$TWINE_USERNAME" ] || twine upload circuitpython-stubs/dist/* - build-aarch: - runs-on: ubuntu-22.04 - needs: test - if: ${{ needs.test.outputs.boards-aarch != '[]' }} - env: - CP_VERSION: ${{ needs.test.outputs.cp-version }} - strategy: - fail-fast: false - matrix: - board: ${{ fromJSON(needs.test.outputs.boards-aarch) }} - steps: - - name: Set up repository - uses: actions/checkout@v3 - with: - submodules: false - fetch-depth: 1 - - name: Set up python - uses: actions/setup-python@v4 - with: - python-version: "3.x" - - name: Set up submodules - id: set-up-submodules - uses: ./.github/actions/fetch_submodules - - name: Install dependencies - run: | - sudo apt-get install -y gettext mtools - pip install -r requirements-dev.txt - wget --no-verbose https://adafruit-circuit-python.s3.amazonaws.com/gcc-arm-10.3-2021.07-x86_64-aarch64-none-elf.tar.xz - sudo tar -C /usr --strip-components=1 -xaf gcc-arm-10.3-2021.07-x86_64-aarch64-none-elf.tar.xz - - uses: carlosperate/arm-none-eabi-gcc-action@v1 - with: - release: '10-2020-q4' - - name: Install mkfs.fat - run: | - wget https://github.com/dosfstools/dosfstools/releases/download/v4.2/dosfstools-4.2.tar.gz - tar -xaf dosfstools-4.2.tar.gz - cd dosfstools-4.2 - ./configure - make -j 2 - cd src - echo >>$GITHUB_PATH $(pwd) - - name: Versions - run: | - gcc --version - aarch64-none-elf-gcc --version - arm-none-eabi-gcc --version - python3 --version - mkfs.fat --version || true - - name: Build mpy-cross - if: ${{ steps.set-up-submodules.outputs.frozen == 'True' }} - run: make -C mpy-cross -j2 - - name: Setup build failure matcher - run: echo "::add-matcher::$GITHUB_WORKSPACE/.github/workflows/match-build-fail.json" - - name: Build - run: python3 -u build_release_files.py - working-directory: tools - env: - BOARDS: ${{ matrix.board }} - - uses: actions/upload-artifact@v3 - with: - name: ${{ matrix.board }} - path: bin/${{ matrix.board }} - - name: Upload to S3 - if: >- - (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') || - (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested')) - run: | - pip install awscli - [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp bin/ s3://adafruit-circuit-python/bin/ --recursive --no-progress --region us-east-1 - env: - AWS_PAGER: '' - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aarch: + needs: [scheduler, mpy-cross, tests] + if: ${{ needs.scheduler.outputs.boards-aarch != '[]' }} + uses: ./.github/workflows/build-boards.yml + with: + platform: aarch + boards: ${{ needs.scheduler.outputs.boards-aarch }} + cp-version: ${{ needs.scheduler.outputs.cp-version }} - build-arm: - runs-on: ubuntu-22.04 - needs: test - if: ${{ needs.test.outputs.boards-arm != '[]' }} - env: - CP_VERSION: ${{ needs.test.outputs.cp-version }} - strategy: - fail-fast: false - matrix: - board: ${{ fromJSON(needs.test.outputs.boards-arm) }} - steps: - - name: Set up repository - uses: actions/checkout@v3 - with: - submodules: false - fetch-depth: 1 - - name: Set up python - uses: actions/setup-python@v4 - with: - python-version: "3.x" - - name: Set up submodules - id: set-up-submodules - uses: ./.github/actions/fetch_submodules - - uses: carlosperate/arm-none-eabi-gcc-action@v1 - with: - release: '10-2020-q4' - - name: Install dependencies - run: | - sudo apt-get install -y gettext - pip install -r requirements-dev.txt - - name: Versions - run: | - gcc --version - arm-none-eabi-gcc --version - python3 --version - - name: Build mpy-cross - if: ${{ steps.set-up-submodules.outputs.frozen == 'True' }} - run: make -C mpy-cross -j2 - - name: Setup build failure matcher - run: echo "::add-matcher::$GITHUB_WORKSPACE/.github/workflows/match-build-fail.json" - - name: Build - run: python3 -u build_release_files.py - working-directory: tools - env: - BOARDS: ${{ matrix.board }} - - uses: actions/upload-artifact@v3 - with: - name: ${{ matrix.board }} - path: bin/${{ matrix.board }} - - name: Upload to S3 - if: >- - (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') || - (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested')) - run: | - pip install awscli - [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp bin/ s3://adafruit-circuit-python/bin/ --recursive --no-progress --region us-east-1 - env: - AWS_PAGER: '' - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + arm: + needs: [scheduler, mpy-cross, tests] + if: ${{ needs.scheduler.outputs.boards-arm != '[]' }} + uses: ./.github/workflows/build-boards.yml + with: + platform: arm + boards: ${{ needs.scheduler.outputs.boards-arm }} + cp-version: ${{ needs.scheduler.outputs.cp-version }} - build-espressif: - runs-on: ubuntu-22.04 - needs: test - if: ${{ needs.test.outputs.boards-espressif != '[]' }} - env: - CP_VERSION: ${{ needs.test.outputs.cp-version }} - IDF_PATH: ${{ github.workspace }}/ports/espressif/esp-idf - IDF_TOOLS_PATH: ${{ github.workspace }}/.idf_tools - strategy: - fail-fast: false - matrix: - board: ${{ fromJSON(needs.test.outputs.boards-espressif) }} - steps: - - name: Set up repository - uses: actions/checkout@v3 - with: - submodules: false - fetch-depth: 1 - - name: Set up python - id: setup-python - uses: actions/setup-python@v4 - with: - python-version: "3.10" - - name: Get IDF commit - id: idf-commit - run: | - COMMIT=$(git submodule status ports/espressif/esp-idf | grep -o -P '(?<=^-).*(?= )') - echo "$COMMIT" - echo "commit=$COMMIT" >> $GITHUB_OUTPUT - - name: Cache IDF submodules - uses: actions/cache@v3 - with: - path: | - .git/modules/ports/espressif/esp-idf - ports/espressif/esp-idf - key: submodules-idf-${{ steps.idf-commit.outputs.commit }} - - name: Cache IDF tools - uses: actions/cache@v3 - with: - path: ${{ env.IDF_TOOLS_PATH }} - key: ${{ runner.os }}-Python-${{ steps.setup-python.outputs.python-version }}-tools-idf-${{ steps.idf-commit.outputs.commit }} - - name: Initialize IDF submodules - run: git submodule update --init --depth=1 --recursive $IDF_PATH - - name: Install IDF tools - run: | - echo "Installing ESP-IDF tools" - $IDF_PATH/tools/idf_tools.py --non-interactive install required - $IDF_PATH/tools/idf_tools.py --non-interactive install cmake - echo "Installing Python environment and packages" - $IDF_PATH/tools/idf_tools.py --non-interactive install-python-env - rm -rf $IDF_TOOLS_PATH/dist - - name: Set up submodules - id: set-up-submodules - uses: ./.github/actions/fetch_submodules - - name: Install dependencies - run: | - source $IDF_PATH/export.sh - sudo apt-get install -y gettext ninja-build - pip install -r requirements-dev.txt - - name: Versions - run: | - source $IDF_PATH/export.sh - gcc --version - python3 --version - ninja --version - cmake --version - - name: Build mpy-cross - if: ${{ steps.set-up-submodules.outputs.frozen == 'True' }} - run: make -C mpy-cross -j2 - - name: Setup build failure matcher - run: echo "::add-matcher::$GITHUB_WORKSPACE/.github/workflows/match-build-fail.json" - - name: Build - run: | - source $IDF_PATH/export.sh - python3 -u build_release_files.py - working-directory: tools - env: - BOARDS: ${{ matrix.board }} - - uses: actions/upload-artifact@v3 - with: - name: ${{ matrix.board }} - path: bin/${{ matrix.board }} - - name: Upload to S3 - if: >- - (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') || - (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested')) - run: | - pip install awscli - [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp bin/ s3://adafruit-circuit-python/bin/ --recursive --no-progress --region us-east-1 - env: - AWS_PAGER: '' - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + esp: + needs: [scheduler, mpy-cross, tests] + if: ${{ needs.scheduler.outputs.boards-esp != '[]' }} + uses: ./.github/workflows/build-boards.yml + with: + platform: esp + boards: ${{ needs.scheduler.outputs.boards-esp }} + cp-version: ${{ needs.scheduler.outputs.cp-version }} - build-riscv: - runs-on: ubuntu-22.04 - needs: test - if: ${{ needs.test.outputs.boards-riscv != '[]' }} - env: - CP_VERSION: ${{ needs.test.outputs.cp-version }} - strategy: - fail-fast: false - matrix: - board: ${{ fromJSON(needs.test.outputs.boards-riscv) }} - steps: - - name: Set up repository - uses: actions/checkout@v3 - with: - submodules: false - fetch-depth: 1 - - name: Set up python - uses: actions/setup-python@v4 - with: - python-version: "3.x" - - name: Set up submodules - id: set-up-submodules - uses: ./.github/actions/fetch_submodules - - name: Install dependencies - run: | - sudo apt-get install -y gettext - pip install -r requirements-dev.txt - wget https://static.dev.sifive.com/dev-tools/riscv64-unknown-elf-gcc-8.3.0-2019.08.0-x86_64-linux-centos6.tar.gz - sudo tar -C /usr --strip-components=1 -xaf riscv64-unknown-elf-gcc-8.3.0-2019.08.0-x86_64-linux-centos6.tar.gz - - name: Versions - run: | - gcc --version - riscv64-unknown-elf-gcc --version - python3 --version - - name: Build mpy-cross - if: ${{ steps.set-up-submodules.outputs.frozen == 'True' }} - run: make -C mpy-cross -j2 - - name: Setup build failure matcher - run: echo "::add-matcher::$GITHUB_WORKSPACE/.github/workflows/match-build-fail.json" - - name: Build - run: python3 -u build_release_files.py - working-directory: tools - env: - BOARDS: ${{ matrix.board }} - - uses: actions/upload-artifact@v3 - with: - name: ${{ matrix.board }} - path: bin/${{ matrix.board }} - - name: Upload to S3 - if: >- - (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') || - (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested')) - run: | - pip install awscli - [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp bin/ s3://adafruit-circuit-python/bin/ --recursive --no-progress --region us-east-1 - env: - AWS_PAGER: '' - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + riscv: + needs: [scheduler, mpy-cross, tests] + if: ${{ needs.scheduler.outputs.boards-riscv != '[]' }} + uses: ./.github/workflows/build-boards.yml + with: + platform: riscv + boards: ${{ needs.scheduler.outputs.boards-riscv }} + cp-version: ${{ needs.scheduler.outputs.cp-version }} + + rpi: + needs: [scheduler, mpy-cross, tests] + if: ${{ needs.scheduler.outputs.boards-rpi != '[]' }} + uses: ./.github/workflows/build-boards.yml + with: + platform: arm + boards: ${{ needs.scheduler.outputs.boards-rpi }} + cp-version: ${{ needs.scheduler.outputs.cp-version }} diff --git a/.github/workflows/create_website_pr.yml b/.github/workflows/create_website_pr.yml index 19004d9bea..eeae8c8299 100644 --- a/.github/workflows/create_website_pr.yml +++ b/.github/workflows/create_website_pr.yml @@ -24,13 +24,13 @@ jobs: - name: Set up python uses: actions/setup-python@v4 with: - python-version: "3.x" + python-version: 3.x - name: Set up submodules - uses: ./.github/actions/fetch_submodules + uses: ./.github/actions/deps/submodules with: version: true - - name: Install dependencies - run: pip install -r requirements-dev.txt + - name: Set up external + uses: ./.github/actions/deps/external - name: Versions run: | gcc --version diff --git a/.github/workflows/ports_windows.yml b/.github/workflows/ports_windows.yml index 66d09c2418..ba2042660a 100644 --- a/.github/workflows/ports_windows.yml +++ b/.github/workflows/ports_windows.yml @@ -77,7 +77,7 @@ jobs: fetch-depth: 1 - name: Set up submodules - uses: ./.github/actions/fetch_submodules + uses: ./.github/actions/deps/submodules with: version: true diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 02aed6e9b4..c0dff33564 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -24,13 +24,13 @@ jobs: - name: Set up python uses: actions/setup-python@v4 with: - python-version: "3.x" - - name: CircuitPython dependencies - run: python tools/ci_fetch_deps.py ${{ github.job }} + python-version: 3.x + - name: Set up submodules + uses: ./.github/actions/deps/submodules + - name: Set up external + uses: ./.github/actions/deps/external - name: Install dependencies - run: | - sudo apt-get install -y gettext uncrustify - pip3 install black polib pyyaml + run: sudo apt-get install -y gettext uncrustify - name: Run pre-commit uses: pre-commit/action@v3.0.0 - name: Make patch diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml new file mode 100644 index 0000000000..12e8d2ef74 --- /dev/null +++ b/.github/workflows/run-tests.yml @@ -0,0 +1,67 @@ +name: Run tests + +on: + workflow_call: + inputs: + cp-version: + required: true + type: string + +jobs: + run: + runs-on: ubuntu-20.04 + strategy: + fail-fast: false + matrix: + test: [all, mpy, native, native_mpy] + env: + CP_VERSION: ${{ inputs.cp-version }} + MICROPY_CPYTHON3: python3.8 + MICROPY_MICROPYTHON: ../ports/unix/micropython-coverage + TEST_all: + TEST_mpy: --via-mpy -d basics float micropython + TEST_native: --emit native + TEST_native_mpy: --via-mpy --emit native -d basics float micropython + steps: + - name: Set up repository + uses: actions/checkout@v3 + with: + submodules: false + fetch-depth: 1 + - name: Set up python + uses: actions/setup-python@v4 + with: + python-version: 3.x + - name: Set up submodules + uses: ./.github/actions/deps/submodules + with: + target: tests + - name: Set up external + if: matrix.test == 'all' + uses: ./.github/actions/deps/external + - name: Set up mpy-cross + uses: ./.github/actions/mpy_cross + - name: Build unix port + run: make -C ports/unix VARIANT=coverage -j2 + - name: Run tests + run: ./run-tests.py -j2 ${{ env[format('TEST_{0}', matrix.test)] }} + working-directory: tests + - name: Print failure info + run: ./run-tests.py -j2 --print-failures + if: failure() + working-directory: tests + - name: Build native modules + if: matrix.test == 'all' + run: | + make -C examples/natmod/features1 + make -C examples/natmod/features2 + make -C examples/natmod/btree + make -C examples/natmod/framebuf + make -C examples/natmod/uheapq + make -C examples/natmod/urandom + make -C examples/natmod/ure + make -C examples/natmod/uzlib + - name: Test native modules + if: matrix.test == 'all' + run: ./run-natmodtests.py extmod/{btree*,framebuf*,uheapq*,ure*,uzlib*}.py + working-directory: tests diff --git a/py/py.mk b/py/py.mk index 0cbfd252f2..c598f44cbd 100644 --- a/py/py.mk +++ b/py/py.mk @@ -243,7 +243,7 @@ $(HEADER_BUILD)/mpversion.h: FORCE | $(HEADER_BUILD) MPCONFIGPORT_MK = $(wildcard mpconfigport.mk) $(HEADER_BUILD)/$(TRANSLATION).mo: $(TOP)/locale/$(TRANSLATION).po | $(HEADER_BUILD) - $(Q)msgfmt -o $@ $^ + $(Q)$(PYTHON) $(TOP)/tools/msgfmt.py -o $@ $^ $(HEADER_BUILD)/qstrdefs.preprocessed.h: $(PY_QSTR_DEFS) $(QSTR_DEFS) $(QSTR_DEFS_COLLECTED) mpconfigport.h $(MPCONFIGPORT_MK) $(PY_SRC)/mpconfig.h | $(HEADER_BUILD) $(STEPECHO) "GEN $@" diff --git a/requirements-dev.txt b/requirements-dev.txt index 0dcfad3956..5e8fe3d3f6 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,6 +6,7 @@ typer sh click cpp-coveralls + requests requests-cache @@ -13,8 +14,8 @@ requests-cache polib # For pre-commit -pyyaml black +pyyaml pre-commit # for combining the Nordic SoftDevice with CircuitPython @@ -23,9 +24,6 @@ intelhex # for building & testing natmods pyelftools -# for stubs and annotations -adafruit-circuitpython-typing - # for mbedtls certificate store cryptography diff --git a/tools/ci_fetch_deps.py b/tools/ci_fetch_deps.py index 5cf2279314..bdf2967af8 100644 --- a/tools/ci_fetch_deps.py +++ b/tools/ci_fetch_deps.py @@ -73,12 +73,15 @@ def main(): print("Target:", TARGET) - if TARGET == "test": + if TARGET == "scheduler": + # submodules = ["tools/"] + submodules = ["extmod/ulab", "lib/", "tools/"] + elif TARGET == "tests": submodules = ["extmod/ulab", "lib/", "tools/"] elif TARGET == "build-doc": # used in .readthedocs.yml to generate RTD submodules = ["extmod/ulab", "frozen/"] - elif TARGET == "mpy-cross-mac": + elif TARGET == "mpy-cross" or TARGET == "mpy-cross-mac": submodules = ["tools/"] # for huffman elif TARGET == "windows": # This builds one board from a number of ports so fill out a bunch of submodules diff --git a/tools/ci_set_matrix.py b/tools/ci_set_matrix.py index bf899127ae..6e7a4f1229 100755 --- a/tools/ci_set_matrix.py +++ b/tools/ci_set_matrix.py @@ -46,11 +46,11 @@ PORT_TO_ARCH = { "atmel-samd": "arm", "broadcom": "aarch", "cxd56": "arm", - "espressif": "espressif", + "espressif": "esp", "litex": "riscv", "mimxrt10xx": "arm", "nrf": "arm", - "raspberrypi": "arm", + "raspberrypi": "rpi", "stm": "arm", } @@ -204,29 +204,33 @@ def set_boards_to_build(build_all: bool): break # Split boards by architecture. - print("Building boards:") - arch_to_boards = {"aarch": [], "arm": [], "riscv": [], "espressif": []} + arch_to_boards = {"aarch": [], "arm": [], "esp": [], "riscv": [], "rpi": []} + + # Append previously failed boards + for arch in arch_to_boards: + arch_to_job = f"build-{arch}" + if arch_to_job in last_failed_jobs: + for board in last_failed_jobs[arch_to_job]: + if not board in boards_to_build: + boards_to_build.append(board) + + build_boards = bool(boards_to_build) + print("Building boards:", build_boards) + set_output("build-boards", build_boards) + + # Append boards according to arch for board in sorted(boards_to_build): - print(" ", board) port = board_to_port.get(board) # A board can appear due to its _deletion_ (rare) # if this happens it's not in `board_to_port`. if not port: continue - arch = PORT_TO_ARCH[port] - arch_to_boards[arch].append(board) + arch_to_boards[PORT_TO_ARCH[port]].append(board) + print(" ", board) # Set the step outputs for each architecture for arch in arch_to_boards: - # Append previous failed jobs - if f"build-{arch}" in last_failed_jobs: - failed_boards = last_failed_jobs[f"build-{arch}"] - for board in failed_boards: - if not board in arch_to_boards[arch]: - print(" ", board) - arch_to_boards[arch].append(board) - # Set Output - set_output(f"boards-{arch}", json.dumps(sorted(arch_to_boards[arch]))) + set_output(f"boards-{arch}", json.dumps(arch_to_boards[arch])) def set_docs_to_build(build_doc: bool): diff --git a/tools/msgfmt.py b/tools/msgfmt.py new file mode 100644 index 0000000000..1ed594b84f --- /dev/null +++ b/tools/msgfmt.py @@ -0,0 +1,244 @@ +#! /usr/bin/env python3 +# Written by Martin v. Löwis + +"""Generate binary message catalog from textual translation description. +This program converts a textual Uniforum-style message catalog (.po file) into +a binary GNU catalog (.mo file). This is essentially the same function as the +GNU msgfmt program, however, it is a simpler implementation. Currently it +does not handle plural forms but it does handle message contexts. +Usage: msgfmt.py [OPTIONS] filename.po +Options: + -o file + --output-file=file + Specify the output file to write to. If omitted, output will go to a + file named filename.mo (based off the input file name). + -h + --help + Print this message and exit. + -V + --version + Display version information and exit. +""" + +import os +import sys +import ast +import getopt +import struct +import array +from email.parser import HeaderParser + +__version__ = "1.2" + +MESSAGES = {} + + +def usage(code, msg=""): + print(__doc__, file=sys.stderr) + if msg: + print(msg, file=sys.stderr) + sys.exit(code) + + +def add(ctxt, id, str, fuzzy): + "Add a non-fuzzy translation to the dictionary." + global MESSAGES + if not fuzzy and str: + if ctxt is None: + MESSAGES[id] = str + else: + MESSAGES[b"%b\x04%b" % (ctxt, id)] = str + + +def generate(): + "Return the generated output." + global MESSAGES + # the keys are sorted in the .mo file + keys = sorted(MESSAGES.keys()) + offsets = [] + ids = strs = b"" + for id in keys: + # For each string, we need size and file offset. Each string is NUL + # terminated; the NUL does not count into the size. + offsets.append((len(ids), len(id), len(strs), len(MESSAGES[id]))) + ids += id + b"\0" + strs += MESSAGES[id] + b"\0" + output = "" + # The header is 7 32-bit unsigned integers. We don't use hash tables, so + # the keys start right after the index tables. + # translated string. + keystart = 7 * 4 + 16 * len(keys) + # and the values start after the keys + valuestart = keystart + len(ids) + koffsets = [] + voffsets = [] + # The string table first has the list of keys, then the list of values. + # Each entry has first the size of the string, then the file offset. + for o1, l1, o2, l2 in offsets: + koffsets += [l1, o1 + keystart] + voffsets += [l2, o2 + valuestart] + offsets = koffsets + voffsets + output = struct.pack( + "Iiiiiii", + 0x950412DE, # Magic + 0, # Version + len(keys), # # of entries + 7 * 4, # start of key index + 7 * 4 + len(keys) * 8, # start of value index + 0, + 0, + ) # size and offset of hash table + output += array.array("i", offsets).tobytes() + output += ids + output += strs + return output + + +def make(filename, outfile): + ID = 1 + STR = 2 + CTXT = 3 + + # Compute .mo name from .po name and arguments + if filename.endswith(".po"): + infile = filename + else: + infile = filename + ".po" + if outfile is None: + outfile = os.path.splitext(infile)[0] + ".mo" + + try: + with open(infile, "rb") as f: + lines = f.readlines() + except IOError as msg: + print(msg, file=sys.stderr) + sys.exit(1) + + section = msgctxt = None + fuzzy = 0 + + # Start off assuming Latin-1, so everything decodes without failure, + # until we know the exact encoding + encoding = "latin-1" + + # Parse the catalog + lno = 0 + for l in lines: + l = l.decode(encoding) + lno += 1 + # If we get a comment line after a msgstr, this is a new entry + if l[0] == "#" and section == STR: + add(msgctxt, msgid, msgstr, fuzzy) + section = msgctxt = None + fuzzy = 0 + # Record a fuzzy mark + if l[:2] == "#," and "fuzzy" in l: + fuzzy = 1 + # Skip comments + if l[0] == "#": + continue + # Now we are in a msgid or msgctxt section, output previous section + if l.startswith("msgctxt"): + if section == STR: + add(msgctxt, msgid, msgstr, fuzzy) + section = CTXT + l = l[7:] + msgctxt = b"" + elif l.startswith("msgid") and not l.startswith("msgid_plural"): + if section == STR: + add(msgctxt, msgid, msgstr, fuzzy) + if not msgid: + # See whether there is an encoding declaration + p = HeaderParser() + charset = p.parsestr(msgstr.decode(encoding)).get_content_charset() + if charset: + encoding = charset + section = ID + l = l[5:] + msgid = msgstr = b"" + is_plural = False + # This is a message with plural forms + elif l.startswith("msgid_plural"): + if section != ID: + print( + "msgid_plural not preceded by msgid on %s:%d" % (infile, lno), file=sys.stderr + ) + sys.exit(1) + l = l[12:] + msgid += b"\0" # separator of singular and plural + is_plural = True + # Now we are in a msgstr section + elif l.startswith("msgstr"): + section = STR + if l.startswith("msgstr["): + if not is_plural: + print("plural without msgid_plural on %s:%d" % (infile, lno), file=sys.stderr) + sys.exit(1) + l = l.split("]", 1)[1] + if msgstr: + msgstr += b"\0" # Separator of the various plural forms + else: + if is_plural: + print( + "indexed msgstr required for plural on %s:%d" % (infile, lno), + file=sys.stderr, + ) + sys.exit(1) + l = l[6:] + # Skip empty lines + l = l.strip() + if not l: + continue + l = ast.literal_eval(l) + if section == CTXT: + msgctxt += l.encode(encoding) + elif section == ID: + msgid += l.encode(encoding) + elif section == STR: + msgstr += l.encode(encoding) + else: + print("Syntax error on %s:%d" % (infile, lno), "before:", file=sys.stderr) + print(l, file=sys.stderr) + sys.exit(1) + # Add last entry + if section == STR: + add(msgctxt, msgid, msgstr, fuzzy) + + # Compute output + output = generate() + + try: + with open(outfile, "wb") as f: + f.write(output) + except IOError as msg: + print(msg, file=sys.stderr) + + +def main(): + try: + opts, args = getopt.getopt(sys.argv[1:], "hVo:", ["help", "version", "output-file="]) + except getopt.error as msg: + usage(1, msg) + + outfile = None + # parse options + for opt, arg in opts: + if opt in ("-h", "--help"): + usage(0) + elif opt in ("-V", "--version"): + print("msgfmt.py", __version__) + sys.exit(0) + elif opt in ("-o", "--output-file"): + outfile = arg + # do it + if not args: + print("No input file given", file=sys.stderr) + print("Try `msgfmt --help' for more information.", file=sys.stderr) + return + + for filename in args: + make(filename, outfile) + + +if __name__ == "__main__": + main()