cache common submodules and share cp version

This commit is contained in:
MicroDev 2023-01-14 10:58:04 +05:30
parent 79bd88349e
commit 6bc0274aec
No known key found for this signature in database
GPG Key ID: 2C0867BE60967730
4 changed files with 249 additions and 140 deletions

View File

@ -21,29 +21,54 @@ jobs:
runs-on: ubuntu-20.04
outputs:
build-doc: ${{ steps.set-matrix.outputs.build-doc }}
boards-arm: ${{ steps.set-matrix.outputs.boards-arm }}
boards-riscv: ${{ steps.set-matrix.outputs.boards-riscv }}
boards-espressif: ${{ steps.set-matrix.outputs.boards-espressif }}
boards-aarch: ${{ steps.set-matrix.outputs.boards-aarch }}
boards-arm: ${{ steps.set-matrix.outputs.boards-arm }}
boards-espressif: ${{ steps.set-matrix.outputs.boards-espressif }}
boards-riscv: ${{ steps.set-matrix.outputs.boards-riscv }}
cp-version: ${{ steps.cp-version.outputs.cp-version }}
steps:
- name: Dump GitHub context
run: echo "$GITHUB_CONTEXT"
env:
GITHUB_CONTEXT: ${{ toJson(github) }}
- uses: actions/checkout@v3
- name: Set up repository
uses: actions/checkout@v3
with:
submodules: false
fetch-depth: 1
- name: Set up Python 3
- name: Set up python
uses: actions/setup-python@v4
with:
python-version: "3.x"
- name: Get CP deps
run: python tools/ci_fetch_deps.py test ${{ github.sha }}
- name: CircuitPython version
- name: Duplicate USB VID/PID check
run: python3 -u -m tools.ci_check_duplicate_usb_vid_pid
- name: Create submodule status
run: git submodule status extmod/ulab frozen/ lib/ tools/ >> submodule_status
- name: Cache submodules
uses: actions/cache@v3
with:
path: |
.git/modules/
extmod/ulab
frozen/
lib/
tools/
key: submodules-common-${{ hashFiles('submodule_status') }}
- name: CircuitPython dependencies
run: |
tools/describe || git log --parents HEAD~4..
echo >>$GITHUB_ENV CP_VERSION=$(tools/describe)
python tools/ci_fetch_deps.py ${{ github.job }}
echo "::group::Fetch history and tags"
git fetch --no-recurse-submodules --shallow-since="2021-07-01" --tags https://github.com/adafruit/circuitpython HEAD
git fetch --no-recurse-submodules --shallow-since="2021-07-01" origin $GITHUB_SHA
git repack -d
echo "::endgroup::"
- name: CircuitPython version
id: cp-version
run: |
CP_VERSION=$(tools/describe)
echo "$CP_VERSION"
echo "CP_VERSION=$CP_VERSION" >> $GITHUB_ENV
echo "cp-version=$CP_VERSION" >> $GITHUB_OUTPUT
- name: Install dependencies
run: |
sudo apt-get update
@ -54,8 +79,6 @@ jobs:
run: |
gcc --version
python3 --version
- name: Duplicate USB VID/PID Check
run: python3 -u -m tools.ci_check_duplicate_usb_vid_pid
- name: Build mpy-cross
run: make -C mpy-cross -j2
- name: Build unix port
@ -160,25 +183,36 @@ jobs:
mpy-cross-mac:
runs-on: macos-11
needs: test
if: ${{ needs.test.outputs.boards-aarch != '[]' || needs.test.outputs.boards-arm != '[]' || needs.scheduler.test.boards-espressif != '[]' || needs.scheduler.test.boards-riscv != '[]' }}
env:
CP_VERSION: ${{ needs.test.outputs.cp-version }}
steps:
- name: Dump GitHub context
env:
GITHUB_CONTEXT: ${{ toJson(github) }}
run: echo "$GITHUB_CONTEXT"
- uses: actions/checkout@v3
- name: Set up repository
uses: actions/checkout@v3
with:
submodules: false
fetch-depth: 1
- name: Set up Python 3
- name: Set up python
uses: actions/setup-python@v4
with:
python-version: "3.10"
- name: Get CP deps
run: python tools/ci_fetch_deps.py mpy-cross-mac ${{ github.sha }}
python-version: "3.x"
- name: Create submodule status
run: git submodule status extmod/ulab frozen/ lib/ tools/ >> submodule_status
- name: Restore submodules
uses: actions/cache/restore@v3
with:
path: |
.git/modules/
extmod/ulab
frozen/
lib/
tools/
key: submodules-common-${{ hashFiles('submodule_status') }}
- name: CircuitPython dependencies
run: python tools/ci_fetch_deps.py ${{ github.job }}
- name: CircuitPython version
run: |
tools/describe || git log --parents HEAD~4..
echo >>$GITHUB_ENV CP_VERSION=$(tools/describe)
run: tools/describe
- name: Install dependencies
run: |
brew install gettext
@ -222,21 +256,34 @@ jobs:
runs-on: ubuntu-20.04
needs: test
if: ${{ needs.test.outputs.build-doc == 'True' }}
env:
CP_VERSION: ${{ needs.test.outputs.cp-version }}
steps:
- uses: actions/checkout@v3
- name: Set up repository
uses: actions/checkout@v3
with:
submodules: false
fetch-depth: 1
- name: Get CP deps
run: python tools/ci_fetch_deps.py docs ${{ github.sha }}
- name: Create submodule status
run: git submodule status extmod/ulab frozen/ lib/ tools/ >> submodule_status
- name: Restore submodules
uses: actions/cache/restore@v3
with:
path: |
.git/modules/
extmod/ulab
frozen/
lib/
tools/
key: submodules-common-${{ hashFiles('submodule_status') }}
- name: CircuitPython dependencies
run: python tools/ci_fetch_deps.py ${{ github.job }}
- name: CircuitPython version
run: |
tools/describe || git log --parents HEAD~4..
echo >>$GITHUB_ENV CP_VERSION=$(tools/describe)
- name: Set up Python 3
run: tools/describe
- name: Set up python
uses: actions/setup-python@v4
with:
python-version: "3.10"
python-version: "3.x"
- name: Install dependencies
run: |
sudo apt-get update
@ -285,22 +332,37 @@ jobs:
build-arm:
runs-on: ubuntu-20.04
needs: test
if: ${{ needs.test.outputs.boards-arm != '[]' }}
env:
CP_VERSION: ${{ needs.test.outputs.cp-version }}
strategy:
fail-fast: false
matrix:
board: ${{ fromJSON(needs.test.outputs.boards-arm) }}
if: ${{ needs.test.outputs.boards-arm != '[]' }}
steps:
- name: Set up Python 3
uses: actions/setup-python@v4
with:
python-version: "3.10"
- uses: actions/checkout@v3
- name: Set up repository
uses: actions/checkout@v3
with:
submodules: false
fetch-depth: 1
- name: Get CP deps
run: python tools/ci_fetch_deps.py ${{ matrix.board }} ${{ github.sha }}
- name: Set up python
uses: actions/setup-python@v4
with:
python-version: "3.x"
- name: Create submodule status
run: git submodule status extmod/ulab frozen/ lib/ tools/ >> submodule_status
- name: Restore submodules
uses: actions/cache/restore@v3
with:
path: |
.git/modules/
extmod/ulab
frozen/
lib/
tools/
key: submodules-common-${{ hashFiles('submodule_status') }}
- name: CircuitPython dependencies
run: python tools/ci_fetch_deps.py ${{ matrix.board }}
- uses: carlosperate/arm-none-eabi-gcc-action@v1
with:
release: '10-2020-q4'
@ -338,22 +400,37 @@ jobs:
build-riscv:
runs-on: ubuntu-20.04
needs: test
if: ${{ needs.test.outputs.boards-riscv != '[]' }}
env:
CP_VERSION: ${{ needs.test.outputs.cp-version }}
strategy:
fail-fast: false
matrix:
board: ${{ fromJSON(needs.test.outputs.boards-riscv) }}
if: ${{ needs.test.outputs.boards-riscv != '[]' }}
steps:
- name: Set up Python 3
uses: actions/setup-python@v4
with:
python-version: "3.10"
- uses: actions/checkout@v3
- name: Set up repository
uses: actions/checkout@v3
with:
submodules: false
fetch-depth: 1
- name: Get CP deps
run: python tools/ci_fetch_deps.py ${{ matrix.board }} ${{ github.sha }}
- name: Set up python
uses: actions/setup-python@v4
with:
python-version: "3.x"
- name: Create submodule status
run: git submodule status extmod/ulab frozen/ lib/ tools/ >> submodule_status
- name: Restore submodules
uses: actions/cache/restore@v3
with:
path: |
.git/modules/
extmod/ulab
frozen/
lib/
tools/
key: submodules-common-${{ hashFiles('submodule_status') }}
- name: CircuitPython dependencies
run: python tools/ci_fetch_deps.py ${{ matrix.board }}
- name: Install dependencies
run: |
sudo apt-get install -y gettext
@ -390,27 +467,40 @@ jobs:
build-espressif:
runs-on: ubuntu-20.04
needs: test
if: ${{ needs.test.outputs.boards-espressif != '[]' }}
env:
CP_VERSION: ${{ needs.test.outputs.cp-version }}
strategy:
fail-fast: false
matrix:
board: ${{ fromJSON(needs.test.outputs.boards-espressif) }}
if: ${{ needs.test.outputs.boards-espressif != '[]' }}
steps:
- name: Set up Python 3
- name: Set up repository
uses: actions/checkout@v3
with:
submodules: false
fetch-depth: 1
- name: Set up python
id: py3
uses: actions/setup-python@v4
with:
python-version: "3.10"
- uses: actions/checkout@v3
- name: Create submodule status
run: git submodule status extmod/ulab frozen/ lib/ tools/ >> submodule_status
- name: Restore submodules
uses: actions/cache/restore@v3
with:
submodules: false
fetch-depth: 1
- name: Get CP deps
run: python tools/ci_fetch_deps.py ${{ matrix.board }} ${{ github.sha }}
path: |
.git/modules/
extmod/ulab
frozen/
lib/
tools/
key: submodules-common-${{ hashFiles('submodule_status') }}
- name: CircuitPython dependencies
run: python tools/ci_fetch_deps.py ${{ matrix.board }}
- name: CircuitPython version
run: |
tools/describe || git log --parents HEAD~4..
echo >>$GITHUB_ENV CP_VERSION=$(tools/describe)
run: tools/describe
- uses: actions/cache@v3
name: Fetch IDF tool cache
id: idf-cache
@ -481,22 +571,37 @@ jobs:
build-aarch:
runs-on: ubuntu-20.04
needs: test
if: ${{ needs.test.outputs.boards-aarch != '[]' }}
env:
CP_VERSION: ${{ needs.test.outputs.cp-version }}
strategy:
fail-fast: false
matrix:
board: ${{ fromJSON(needs.test.outputs.boards-aarch) }}
if: ${{ needs.test.outputs.boards-aarch != '[]' }}
steps:
- name: Set up Python 3
uses: actions/setup-python@v4
with:
python-version: "3.10"
- uses: actions/checkout@v3
- name: Set up repository
uses: actions/checkout@v3
with:
submodules: false
fetch-depth: 1
- name: Get CP deps
run: python tools/ci_fetch_deps.py ${{ matrix.board }} ${{ github.sha }}
- name: Set up python
uses: actions/setup-python@v4
with:
python-version: "3.x"
- name: Create submodule status
run: git submodule status extmod/ulab frozen/ lib/ tools/ >> submodule_status
- name: Restore submodules
uses: actions/cache/restore@v3
with:
path: |
.git/modules/
extmod/ulab
frozen/
lib/
tools/
key: submodules-common-${{ hashFiles('submodule_status') }}
- name: CircuitPython dependencies
run: python tools/ci_fetch_deps.py ${{ matrix.board }}
- name: Install dependencies
run: |
sudo apt-get install -y gettext mtools

View File

@ -71,12 +71,32 @@ jobs:
which python; python --version; python -c "import cascadetoml"
which python3; python3 --version; python3 -c "import cascadetoml"
- uses: actions/checkout@v3
- name: Set up repository
uses: actions/checkout@v3
with:
submodules: false
fetch-depth: 1
- name: Get CP deps
run: python tools/ci_fetch_deps.py windows ${{ github.sha }}
- name: Create submodule status
run: git submodule status extmod/ulab frozen/ lib/ tools/ >> submodule_status
- name: Restore submodules
uses: actions/cache/restore@v3
with:
path: |
.git/modules/
extmod/ulab
frozen/
lib/
tools/
key: submodules-common-${{ hashFiles('submodule_status') }}
enableCrossOsArchive: true
- name: CircuitPython dependencies
run: |
python tools/ci_fetch_deps.py windows
echo "::group::Fetch history and tags"
git fetch --no-recurse-submodules --shallow-since="2021-07-01" --tags https://github.com/adafruit/circuitpython HEAD
git fetch --no-recurse-submodules --shallow-since="2021-07-01" origin $GITHUB_SHA
git repack -d
echo "::endgroup::"
- name: CircuitPython version
run: |
tools/describe || git log --parents HEAD~4..

View File

@ -4,16 +4,12 @@ import subprocess
import sys
import time
# target will be a board, "test", "docs", "mpy-cross-mac", or "windows"
target = sys.argv[1]
ref = sys.argv[2]
print(target, ref)
# Target will be a board, "test", "docs", "mpy-cross-mac", or "windows"
TARGET = sys.argv[1]
# Submodules needed by port builds outside of their ports directory.
# Should we try and detect these?
port_deps = {
PORT_DEPS = {
"atmel-samd": [
"extmod/ulab/",
"lib/adafruit_floppy/",
@ -58,73 +54,58 @@ def run(title, command, check=True):
try:
subprocess.run(shlex.split(command), stderr=subprocess.STDOUT, check=check)
finally:
print("Duration:", time.monotonic() - start, flush=True)
print("::endgroup::", flush=True)
print("Duration:", time.monotonic() - start, flush=True)
run(
"Fetch back to the start of 2021 to get tag history",
'git fetch --tags --recurse-submodules=no --shallow-since="2021-07-01" https://github.com/adafruit/circuitpython HEAD',
)
run(
"Fetch back to the start of 2021 to get commit history",
f'git fetch --recurse-submodules=no --shallow-since="2021-07-01" origin {ref}',
)
# See https://stackoverflow.com/questions/63878612/git-fatal-error-in-object-unshallow-sha-1#comment118418373_63879454
run('Fix for bug "fatal: error in object: unshallow"', "git repack -d")
run("Init submodules", "git submodule init")
run("Submodule status", "git submodule status")
def main():
submodules = []
print("Target:", TARGET)
submodules = []
if TARGET == "scheduler":
submodules = ["extmod/ulab", "lib/", "tools/", "frozen/"]
elif TARGET == "mpy-cross-tests":
submodules = ["extmod/ulab", "lib/", "tools/"]
elif TARGET == "build-doc":
# used in .readthedocs.yml to generate RTD
submodules = ["extmod/ulab", "frozen/"]
elif TARGET == "mpy-cross-mac":
submodules = ["tools/"] # for huffman
elif TARGET == "windows":
# This builds one board from a number of ports so fill out a bunch of submodules
submodules = ["extmod/ulab", "lib/", "tools/", "ports/", "data/nvm.toml"]
elif TARGET == "website":
submodules = ["tools/adabot/", "frozen/"]
else:
p = list(pathlib.Path(".").glob(f"ports/*/boards/{TARGET}/mpconfigboard.mk"))
if not p:
raise RuntimeError(f"Unsupported target: {TARGET}")
if target == "test":
submodules = ["extmod/", "lib/", "tools/", "extmod/ulab", "lib/berkeley-db-1.xx"]
elif target == "docs":
# used in .readthedocs.yml to generate RTD
submodules = ["extmod/ulab/", "frozen/"]
elif target == "mpy-cross-mac":
submodules = ["tools/"] # for huffman
elif target == "windows":
# This builds one board from a number of ports so fill out a bunch of submodules
submodules = ["extmod/", "lib/", "tools/", "ports/", "data/nvm.toml/"]
elif target == "website":
submodules = ["tools/adabot/", "frozen/"]
else:
p = list(pathlib.Path(".").glob(f"ports/*/boards/{target}/mpconfigboard.mk"))
if not p:
raise RuntimeError(f"Unsupported target: {target}")
config = p[0]
# Add the ports folder to init submodules
port_folder = config.parents[2]
port = port_folder.name
submodules.append(str(port_folder))
submodules.append("tools/") # for huffman
submodules.extend(PORT_DEPS[port])
with config.open() as f:
for line in f.readlines():
prefix = "FROZEN_MPY_DIRS += $(TOP)/"
if line.startswith(prefix):
lib_folder = line.strip()[len(prefix) :]
# Drop everything after the second folder because the frozen
# folder may be inside the submodule.
if lib_folder.count("/") > 1:
lib_folder = lib_folder.split("/", maxsplit=2)
lib_folder = "/".join(lib_folder[:2])
submodules.append(lib_folder)
config = p[0]
# Add the ports folder to init submodules
port_folder = config.parents[2]
port = port_folder.name
submodules.append(str(port_folder))
submodules.append("tools/") # for huffman
submodules.extend(port_deps[port])
with config.open() as f:
for line in f.readlines():
prefix = "FROZEN_MPY_DIRS += $(TOP)/"
if line.startswith(prefix):
lib_folder = line.strip()[len(prefix) :]
# Drop everything after the second folder because the frozen
# folder may be inside the submodule.
if lib_folder.count("/") > 1:
lib_folder = lib_folder.split("/", maxsplit=2)
lib_folder = "/".join(lib_folder[:2])
submodules.append(lib_folder)
print("Submodule paths:", submodules)
print(submodules)
if submodules:
submodules = " ".join(submodules)
# This line will fail because the submodule's need different commits than the tip of the branch. We
# fix it later.
run(
"Init the submodules we'll need",
f"git submodule update --init -N --depth 1 {submodules}",
check=False,
)
if submodules:
submodules = " ".join(submodules)
run("Init the required submodules", f"git submodule update --init --depth=1 {submodules}")
run(
"Fetch the submodule sha",
"git submodule foreach 'git fetch --tags --depth 1 origin $sha1 && git checkout -q $sha1'",
)
if __name__ == "__main__":
main()

View File

@ -1,3 +1,6 @@
#!/bin/sh
# Add any supplied arguments.
git describe --first-parent --dirty --tags --match "[1-9].*" "$@"
if [ -z "$CP_VERSION" ]; then
git describe --first-parent --dirty --tags --match "[1-9].*" "$@"
else
echo $CP_VERSION
fi