diff --git a/tools/autobuild/autobuild.sh b/tools/autobuild/autobuild.sh new file mode 100755 index 0000000000..45822f2b7d --- /dev/null +++ b/tools/autobuild/autobuild.sh @@ -0,0 +1,90 @@ +#!/bin/bash +# +# Build firmware for ports. +# +# Requirements: +# - All toolchains must be in path (arm-none-eabi-gcc, xtensa-lx106-elf) +# - IDF_PATH_V4 must be set +# - MICROPY_AUTOBUILD_MICROPYTHON_REPO must be set to location of micropython repository +# - MICROPY_AUTOBUILD_MAKE must be set to the make command to use, eg "make -j2" +# +# Optional settings: +# - MICROPY_AUTOBUILD_REMOTE_MACHINE can be set to a remote ssh machine to copy files to +# - MICROPY_AUTOBUILD_REMOTE_DIR can be set to destination directory on remote machine + +if [ ! -d "$IDF_PATH_V4" ]; then + echo "must set IDF_PATH_V4" + exit 1 +fi + +if [ ! -d "$MICROPY_AUTOBUILD_MICROPYTHON_REPO" ]; then + echo "must set MICROPY_AUTOBUILD_MICROPYTHON_REPO" + exit 1 +fi + +if [ -z "$MICROPY_AUTOBUILD_MAKE" ]; then + echo "must set MICROPY_AUTOBUILD_MAKE" + exit 1 +fi + +######################################## +# Initialisation + +# get directory of this script for access to other build scripts +AUTODIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +# make local directory to put firmware +LOCAL_FIRMWARE=/tmp/autobuild-firmware-$$ +mkdir -p ${LOCAL_FIRMWARE} + +# get latest MicroPython +git -C ${MICROPY_AUTOBUILD_MICROPYTHON_REPO} pull +git -C ${MICROPY_AUTOBUILD_MICROPYTHON_REPO} submodule update --init +git -C ${MICROPY_AUTOBUILD_MICROPYTHON_REPO}/lib/pico-sdk submodule update --init + +######################################## +# Build all firmware + +pushd ${MICROPY_AUTOBUILD_MICROPYTHON_REPO} + +# build cross compiler +make -C mpy-cross + +# make the firmware tag +FW_DATE=$(date '+%Y%m%d') +FW_GIT="$(git describe --dirty || echo unknown)" +FW_TAG="-$FW_DATE-unstable-$FW_GIT" + +# build new firmware +cd ports/stm32 +${AUTODIR}/build-stm32-latest.sh ${FW_TAG} ${LOCAL_FIRMWARE} +cd ../cc3200 +${AUTODIR}/build-cc3200-latest.sh ${FW_TAG} ${LOCAL_FIRMWARE} +cd ../esp8266 +${AUTODIR}/build-esp8266-latest.sh ${FW_TAG} ${LOCAL_FIRMWARE} +cd ../esp32 +${AUTODIR}/build-esp32-latest.sh ${IDF_PATH_V4} ${FW_TAG} ${LOCAL_FIRMWARE} +cd ../rp2 +${AUTODIR}/build-rp2-latest.sh ${FW_TAG} ${LOCAL_FIRMWARE} + +popd + +######################################## +# Copy firmware to remote machine + +if [ -z "$MICROPY_AUTOBUILD_REMOTE_MACHINE" -o -z "$MICROPY_AUTOBUILD_REMOTE_DIR" ]; then + echo "No remote given, leaving firmware in ${LOCAL_FIRMWARE}" + exit 0 +fi + +# copy new firmware to remote machine +scp ${LOCAL_FIRMWARE}/* ${MICROPY_AUTOBUILD_REMOTE_MACHINE}:${MICROPY_AUTOBUILD_REMOTE_DIR}/ + +# remove old firmware +${AUTODIR}/remove_old_firmware.py ${MICROPY_AUTOBUILD_REMOTE_MACHINE} ${MICROPY_AUTOBUILD_REMOTE_DIR} + +######################################## +# Clean up + +/bin/rm -v ${LOCAL_FIRMWARE}/* +/bin/rmdir ${LOCAL_FIRMWARE} diff --git a/tools/autobuild/build-cc3200-latest.sh b/tools/autobuild/build-cc3200-latest.sh new file mode 100755 index 0000000000..3c6d400365 --- /dev/null +++ b/tools/autobuild/build-cc3200-latest.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# function for building firmware +function do_build() { + descr=$1 + board=$2 + shift + shift + echo "building $descr $board" + build_dir=/tmp/cc3200-build-$board + $MICROPY_AUTOBUILD_MAKE -B $@ BTARGET=application BOARD=$board BUILD=$build_dir || exit 1 + zip $dest_dir/$descr$fw_tag.zip $build_dir/mcuimg.bin + rm -rf $build_dir +} + +# check/get parameters +if [ $# != 2 ]; then + echo "usage: $0 " + exit 1 +fi + +fw_tag=$1 +dest_dir=$2 + +# check we are in the correct directory +if [ ! -r application.mk ]; then + echo "must be in cc3200 directory" + exit 1 +fi + +# build the versions +do_build wipy WIPY diff --git a/tools/autobuild/build-esp32-latest.sh b/tools/autobuild/build-esp32-latest.sh new file mode 100755 index 0000000000..e433332c9e --- /dev/null +++ b/tools/autobuild/build-esp32-latest.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# +# Build esp32 port + +# for debugging +#exec &> /tmp/esp-log-$$.txt + +# function for building firmware +function do_build() { + descr=$1 + board=$2 + shift + shift + echo "building $descr $board" + build_dir=/tmp/esp32-build-$board + rm -rf $build_dir # be sure we don't have anything leftover from a previous build + make $@ BOARD=$board BUILD=$build_dir || exit 1 + mv $build_dir/firmware.bin $dest_dir/$descr$fw_tag.bin + mv $build_dir/micropython.elf $dest_dir/$descr$fw_tag.elf + mv $build_dir/micropython.map $dest_dir/$descr$fw_tag.map + rm -rf $build_dir +} + +# check/get parameters +if [ $# != 3 ]; then + echo "usage: $0 " + exit 1 +fi + +idf_path=$1 +fw_tag=$2 +dest_dir=$3 + +# check we are in the correct directory +if [ ! -r modesp32.c ]; then + echo "must be in esp32 directory" + exit 1 +fi + +source $idf_path/export.sh + +# build the versions +do_build esp32 GENERIC FROZEN_MANIFEST=$(pwd)/boards/manifest_release.py +do_build esp32spiram GENERIC_SPIRAM FROZEN_MANIFEST=$(pwd)/boards/manifest_release.py +do_build tinypico UM_TINYPICO diff --git a/tools/autobuild/build-esp8266-latest.sh b/tools/autobuild/build-esp8266-latest.sh new file mode 100755 index 0000000000..fe79587fd1 --- /dev/null +++ b/tools/autobuild/build-esp8266-latest.sh @@ -0,0 +1,66 @@ +#!/bin/bash + +PYTHON3=python3 +yaota8266=$HOME/yaota8266 + +# for debugging +#exec &> /tmp/esp-log-$$.txt + +# function for building firmware +function do_build() { + descr=$1 + board=$2 + shift + shift + echo "building $descr $board" + #build_dir=/tmp/esp8266-build-$board + build_dir=build-$board # until esp8266.ld is fixed + rm -rf $build_dir # be sure we don't have anything leftover from a previous build + $MICROPY_AUTOBUILD_MAKE $@ BOARD=$board BUILD=$build_dir || exit 1 + mv $build_dir/firmware-combined.bin $dest_dir/$descr$fw_tag.bin + mv $build_dir/firmware.elf $dest_dir/$descr$fw_tag.elf + mv $build_dir/firmware.map $dest_dir/$descr$fw_tag.map + rm -rf $build_dir +} + +function do_build_ota() { + descr=$1 + board=$2 + shift + shift + echo "building $descr $board" + #build_dir=/tmp/esp8266-build-$board + build_dir=build-$board # until esp8266.ld is fixed + rm -rf $build_dir # be sure we don't have anything leftover from a previous build + $MICROPY_AUTOBUILD_MAKE $@ BOARD=$board BUILD=$build_dir || exit 1 + cat $yaota8266/yaota8266.bin $build_dir/firmware-ota.bin > $dest_dir/$descr$fw_tag.bin + cwd=$(pwd) + pushd $yaota8266/ota-client + $PYTHON3 ota_client.py sign $cwd/$build_dir/firmware-ota.bin + popd + mv $build_dir/firmware-ota.bin.ota $dest_dir/$descr$fw_tag.ota + mv $build_dir/firmware.elf $dest_dir/$descr$fw_tag.elf + mv $build_dir/firmware.map $dest_dir/$descr$fw_tag.map + #rm -rf $build_dir +} + +# check/get parameters +if [ $# != 2 ]; then + echo "usage: $0 " + exit 1 +fi + +fw_tag=$1 +dest_dir=$2 + +# check we are in the correct directory +if [ ! -r boards/esp8266_common.ld ]; then + echo "must be in esp8266 directory" + exit 1 +fi + +# build the versions +do_build esp8266 GENERIC +do_build esp8266-512k GENERIC_512K +do_build esp8266-1m GENERIC_1M +do_build_ota esp8266-ota GENERIC ota diff --git a/tools/autobuild/build-rp2-latest.sh b/tools/autobuild/build-rp2-latest.sh new file mode 100755 index 0000000000..5124686e21 --- /dev/null +++ b/tools/autobuild/build-rp2-latest.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# function for building firmware +function do_build() { + descr=$1 + board=$2 + shift + shift + echo "building $descr $board" + build_dir=/tmp/rp2-build-$board + $MICROPY_AUTOBUILD_MAKE $@ BOARD=$board BUILD=$build_dir || exit 1 + mv $build_dir/firmware.uf2 $dest_dir/$descr$fw_tag.uf2 + rm -rf $build_dir +} + +# check/get parameters +if [ $# != 2 ]; then + echo "usage: $0 " + exit 1 +fi + +fw_tag=$1 +dest_dir=$2 + +# check we are in the correct directory +if [ ! -r modrp2.c ]; then + echo "must be in rp2 directory" + exit 1 +fi + +# build the boards +do_build rp2-pico PICO diff --git a/tools/autobuild/build-stm32-latest.sh b/tools/autobuild/build-stm32-latest.sh new file mode 100755 index 0000000000..226c14a41e --- /dev/null +++ b/tools/autobuild/build-stm32-latest.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +# function for building firmware +function do_build() { + descr=$1 + board=$2 + shift + shift + echo "building $descr $board" + build_dir=/tmp/stm-build-$board + $MICROPY_AUTOBUILD_MAKE -B $@ BOARD=$board BUILD=$build_dir || exit 1 + mv $build_dir/firmware.dfu $dest_dir/$descr$fw_tag.dfu + rm -rf $build_dir +} + +# check/get parameters +if [ $# != 2 ]; then + echo "usage: $0 " + exit 1 +fi + +fw_tag=$1 +dest_dir=$2 + +# check we are in the correct directory +if [ ! -r modpyb.c ]; then + echo "must be in stm directory" + exit 1 +fi + +# build the versions +do_build pybv3 PYBV3 +do_build pybv3-network PYBV3 MICROPY_PY_WIZNET5K=5200 MICROPY_PY_CC3K=1 +do_build pybv10 PYBV10 +do_build pybv10-dp PYBV10 MICROPY_FLOAT_IMPL=double +do_build pybv10-thread PYBV10 CFLAGS_EXTRA='-DMICROPY_PY_THREAD=1' +do_build pybv10-dp-thread PYBV10 MICROPY_FLOAT_IMPL=double CFLAGS_EXTRA='-DMICROPY_PY_THREAD=1' +do_build pybv10-network PYBV10 MICROPY_PY_WIZNET5K=5200 MICROPY_PY_CC3K=1 +do_build pybv11 PYBV11 +do_build pybv11-dp PYBV11 MICROPY_FLOAT_IMPL=double +do_build pybv11-thread PYBV11 CFLAGS_EXTRA='-DMICROPY_PY_THREAD=1' +do_build pybv11-dp-thread PYBV11 MICROPY_FLOAT_IMPL=double CFLAGS_EXTRA='-DMICROPY_PY_THREAD=1' +do_build pybv11-network PYBV11 MICROPY_PY_WIZNET5K=5200 MICROPY_PY_CC3K=1 +do_build pyblitev10 PYBLITEV10 +do_build pyblitev10-dp PYBLITEV10 MICROPY_FLOAT_IMPL=double +do_build pyblitev10-thread PYBLITEV10 CFLAGS_EXTRA='-DMICROPY_PY_THREAD=1' +do_build pyblitev10-dp-thread PYBLITEV10 MICROPY_FLOAT_IMPL=double CFLAGS_EXTRA='-DMICROPY_PY_THREAD=1' +do_build pyblitev10-network PYBLITEV10 MICROPY_PY_WIZNET5K=5200 MICROPY_PY_CC3K=1 +do_build PYBD-SF2 PYBD_SF2 +do_build PYBD-SF3 PYBD_SF3 +do_build PYBD-SF6 PYBD_SF6 + +for board in boards/{NUCLEO_*,STM32F*DISC,B_L*,USBDONGLE_WB55,ESPRUINO_PICO} ; do + bd=$(basename $board) + do_build $bd $bd USE_MBOOT=0 MBOOT_ENABLE_PACKING=0 +done diff --git a/tools/autobuild/remove_old_firmware.py b/tools/autobuild/remove_old_firmware.py new file mode 100755 index 0000000000..e9d2e8aae8 --- /dev/null +++ b/tools/autobuild/remove_old_firmware.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 + +import re, subprocess, sys + + +DEBUG = False +DRY_RUN = False +NUM_KEEP_PER_BOARD = 4 + + +def main(): + ssh_machine = sys.argv[1] + ssh_firmware_dir = sys.argv[2] + + # SSH to get list of existing files. + p = subprocess.run( + ["ssh", ssh_machine, "find", ssh_firmware_dir, "-name", "\\*-unstable-v\\*"], + capture_output=True, + ) + if p.returncode != 0: + print(p.stderr) + return + all_files = p.stdout.split(b"\n") + + # Parse all files to organise into boards/date/version. + boards = {} + for file in all_files: + m = re.match( + rb"([a-z/.]+)/([A-Za-z0-9_-]+)-(20[0-9]{6})-unstable-(v[0-9.-]+-g[0-9a-f]+).", + file, + ) + if not m: + continue + dir, board, date, version = m.groups() + if board not in boards: + boards[board] = {} + if (date, version) not in boards[board]: + boards[board][(date, version)] = [] + boards[board][(date, version)].append(file) + + # Collect files to remove based on date and version. + remove = [] + for board in boards.values(): + filelist = [(date, version, files) for (date, version), files in board.items()] + filelist.sort(reverse=True) + keep = [] + for date, version, files in filelist: + if keep and version == keep[-1]: + remove.extend(files) + elif len(keep) >= NUM_KEEP_PER_BOARD: + remove.extend(files) + else: + keep.append(version) + + if DEBUG: + all_files.sort(reverse=True) + for file in all_files: + print(file, file in remove) + print(len(remove), "/", len(all_files)) + + # Do removal of files. + for file in remove: + file = str(file, "ascii") + print("remove:", file) + if not DRY_RUN: + p = subprocess.run(["ssh", ssh_machine, "/bin/rm", file], capture_output=True) + if p.returncode != 0: + print(p.stderr) + + +if __name__ == "__main__": + main()