From 3588d14ae6a611764a0f83867754da691fba7127 Mon Sep 17 00:00:00 2001
From: Damien George <damien@micropython.org>
Date: Mon, 21 Jun 2021 14:56:37 +1000
Subject: [PATCH] tools/autobuild: Add scripts to build release firmware.

Signed-off-by: Damien George <damien@micropython.org>
---
 tools/autobuild/autobuild.sh            | 90 +++++++++++++++++++++++++
 tools/autobuild/build-cc3200-latest.sh  | 32 +++++++++
 tools/autobuild/build-esp32-latest.sh   | 45 +++++++++++++
 tools/autobuild/build-esp8266-latest.sh | 66 ++++++++++++++++++
 tools/autobuild/build-rp2-latest.sh     | 32 +++++++++
 tools/autobuild/build-stm32-latest.sh   | 56 +++++++++++++++
 tools/autobuild/remove_old_firmware.py  | 72 ++++++++++++++++++++
 7 files changed, 393 insertions(+)
 create mode 100755 tools/autobuild/autobuild.sh
 create mode 100755 tools/autobuild/build-cc3200-latest.sh
 create mode 100755 tools/autobuild/build-esp32-latest.sh
 create mode 100755 tools/autobuild/build-esp8266-latest.sh
 create mode 100755 tools/autobuild/build-rp2-latest.sh
 create mode 100755 tools/autobuild/build-stm32-latest.sh
 create mode 100755 tools/autobuild/remove_old_firmware.py

diff --git a/tools/autobuild/autobuild.sh b/tools/autobuild/autobuild.sh
new file mode 100755
index 0000000000..45822f2b7d
--- /dev/null
+++ b/tools/autobuild/autobuild.sh
@@ -0,0 +1,90 @@
+#!/bin/bash
+#
+# Build firmware for ports.
+#
+# Requirements:
+# - All toolchains must be in path (arm-none-eabi-gcc, xtensa-lx106-elf)
+# - IDF_PATH_V4 must be set
+# - MICROPY_AUTOBUILD_MICROPYTHON_REPO must be set to location of micropython repository
+# - MICROPY_AUTOBUILD_MAKE must be set to the make command to use, eg "make -j2"
+#
+# Optional settings:
+# - MICROPY_AUTOBUILD_REMOTE_MACHINE can be set to a remote ssh machine to copy files to
+# - MICROPY_AUTOBUILD_REMOTE_DIR can be set to destination directory on remote machine
+
+if [ ! -d "$IDF_PATH_V4" ]; then
+    echo "must set IDF_PATH_V4"
+    exit 1
+fi
+
+if [ ! -d "$MICROPY_AUTOBUILD_MICROPYTHON_REPO" ]; then
+    echo "must set MICROPY_AUTOBUILD_MICROPYTHON_REPO"
+    exit 1
+fi
+
+if [ -z "$MICROPY_AUTOBUILD_MAKE" ]; then
+    echo "must set MICROPY_AUTOBUILD_MAKE"
+    exit 1
+fi
+
+########################################
+# Initialisation
+
+# get directory of this script for access to other build scripts
+AUTODIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+# make local directory to put firmware
+LOCAL_FIRMWARE=/tmp/autobuild-firmware-$$
+mkdir -p ${LOCAL_FIRMWARE}
+
+# get latest MicroPython
+git -C ${MICROPY_AUTOBUILD_MICROPYTHON_REPO} pull
+git -C ${MICROPY_AUTOBUILD_MICROPYTHON_REPO} submodule update --init
+git -C ${MICROPY_AUTOBUILD_MICROPYTHON_REPO}/lib/pico-sdk submodule update --init
+
+########################################
+# Build all firmware
+
+pushd ${MICROPY_AUTOBUILD_MICROPYTHON_REPO}
+
+# build cross compiler
+make -C mpy-cross
+
+# make the firmware tag
+FW_DATE=$(date '+%Y%m%d')
+FW_GIT="$(git describe --dirty || echo unknown)"
+FW_TAG="-$FW_DATE-unstable-$FW_GIT"
+
+# build new firmware
+cd ports/stm32
+${AUTODIR}/build-stm32-latest.sh ${FW_TAG} ${LOCAL_FIRMWARE}
+cd ../cc3200
+${AUTODIR}/build-cc3200-latest.sh ${FW_TAG} ${LOCAL_FIRMWARE}
+cd ../esp8266
+${AUTODIR}/build-esp8266-latest.sh ${FW_TAG} ${LOCAL_FIRMWARE}
+cd ../esp32
+${AUTODIR}/build-esp32-latest.sh ${IDF_PATH_V4} ${FW_TAG} ${LOCAL_FIRMWARE}
+cd ../rp2
+${AUTODIR}/build-rp2-latest.sh ${FW_TAG} ${LOCAL_FIRMWARE}
+
+popd
+
+########################################
+# Copy firmware to remote machine
+
+if [ -z "$MICROPY_AUTOBUILD_REMOTE_MACHINE" -o -z "$MICROPY_AUTOBUILD_REMOTE_DIR" ]; then
+    echo "No remote given, leaving firmware in ${LOCAL_FIRMWARE}"
+    exit 0
+fi
+
+# copy new firmware to remote machine
+scp ${LOCAL_FIRMWARE}/* ${MICROPY_AUTOBUILD_REMOTE_MACHINE}:${MICROPY_AUTOBUILD_REMOTE_DIR}/
+
+# remove old firmware
+${AUTODIR}/remove_old_firmware.py ${MICROPY_AUTOBUILD_REMOTE_MACHINE} ${MICROPY_AUTOBUILD_REMOTE_DIR}
+
+########################################
+# Clean up
+
+/bin/rm -v ${LOCAL_FIRMWARE}/*
+/bin/rmdir ${LOCAL_FIRMWARE}
diff --git a/tools/autobuild/build-cc3200-latest.sh b/tools/autobuild/build-cc3200-latest.sh
new file mode 100755
index 0000000000..3c6d400365
--- /dev/null
+++ b/tools/autobuild/build-cc3200-latest.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+
+# function for building firmware
+function do_build() {
+    descr=$1
+    board=$2
+    shift
+    shift
+    echo "building $descr $board"
+    build_dir=/tmp/cc3200-build-$board
+    $MICROPY_AUTOBUILD_MAKE -B $@ BTARGET=application BOARD=$board BUILD=$build_dir || exit 1
+    zip $dest_dir/$descr$fw_tag.zip $build_dir/mcuimg.bin
+    rm -rf $build_dir
+}
+
+# check/get parameters
+if [ $# != 2 ]; then
+    echo "usage: $0 <fw-tag> <dest-dir>"
+    exit 1
+fi
+
+fw_tag=$1
+dest_dir=$2
+
+# check we are in the correct directory
+if [ ! -r application.mk ]; then
+    echo "must be in cc3200 directory"
+    exit 1
+fi
+
+# build the versions
+do_build wipy WIPY
diff --git a/tools/autobuild/build-esp32-latest.sh b/tools/autobuild/build-esp32-latest.sh
new file mode 100755
index 0000000000..e433332c9e
--- /dev/null
+++ b/tools/autobuild/build-esp32-latest.sh
@@ -0,0 +1,45 @@
+#!/bin/bash
+#
+# Build esp32 port
+
+# for debugging
+#exec &> /tmp/esp-log-$$.txt
+
+# function for building firmware
+function do_build() {
+    descr=$1
+    board=$2
+    shift
+    shift
+    echo "building $descr $board"
+    build_dir=/tmp/esp32-build-$board
+    rm -rf $build_dir # be sure we don't have anything leftover from a previous build
+    make $@ BOARD=$board BUILD=$build_dir || exit 1
+    mv $build_dir/firmware.bin $dest_dir/$descr$fw_tag.bin
+    mv $build_dir/micropython.elf $dest_dir/$descr$fw_tag.elf
+    mv $build_dir/micropython.map $dest_dir/$descr$fw_tag.map
+    rm -rf $build_dir
+}
+
+# check/get parameters
+if [ $# != 3 ]; then
+    echo "usage: $0 <idf-path> <fw-tag> <dest-dir>"
+    exit 1
+fi
+
+idf_path=$1
+fw_tag=$2
+dest_dir=$3
+
+# check we are in the correct directory
+if [ ! -r modesp32.c ]; then
+    echo "must be in esp32 directory"
+    exit 1
+fi
+
+source $idf_path/export.sh
+
+# build the versions
+do_build esp32 GENERIC FROZEN_MANIFEST=$(pwd)/boards/manifest_release.py
+do_build esp32spiram GENERIC_SPIRAM FROZEN_MANIFEST=$(pwd)/boards/manifest_release.py
+do_build tinypico UM_TINYPICO
diff --git a/tools/autobuild/build-esp8266-latest.sh b/tools/autobuild/build-esp8266-latest.sh
new file mode 100755
index 0000000000..fe79587fd1
--- /dev/null
+++ b/tools/autobuild/build-esp8266-latest.sh
@@ -0,0 +1,66 @@
+#!/bin/bash
+
+PYTHON3=python3
+yaota8266=$HOME/yaota8266
+
+# for debugging
+#exec &> /tmp/esp-log-$$.txt
+
+# function for building firmware
+function do_build() {
+    descr=$1
+    board=$2
+    shift
+    shift
+    echo "building $descr $board"
+    #build_dir=/tmp/esp8266-build-$board
+    build_dir=build-$board # until esp8266.ld is fixed
+    rm -rf $build_dir # be sure we don't have anything leftover from a previous build
+    $MICROPY_AUTOBUILD_MAKE $@ BOARD=$board BUILD=$build_dir || exit 1
+    mv $build_dir/firmware-combined.bin $dest_dir/$descr$fw_tag.bin
+    mv $build_dir/firmware.elf $dest_dir/$descr$fw_tag.elf
+    mv $build_dir/firmware.map $dest_dir/$descr$fw_tag.map
+    rm -rf $build_dir
+}
+
+function do_build_ota() {
+    descr=$1
+    board=$2
+    shift
+    shift
+    echo "building $descr $board"
+    #build_dir=/tmp/esp8266-build-$board
+    build_dir=build-$board # until esp8266.ld is fixed
+    rm -rf $build_dir # be sure we don't have anything leftover from a previous build
+    $MICROPY_AUTOBUILD_MAKE $@ BOARD=$board BUILD=$build_dir || exit 1
+    cat $yaota8266/yaota8266.bin $build_dir/firmware-ota.bin > $dest_dir/$descr$fw_tag.bin
+    cwd=$(pwd)
+    pushd $yaota8266/ota-client
+    $PYTHON3 ota_client.py sign $cwd/$build_dir/firmware-ota.bin 
+    popd
+    mv $build_dir/firmware-ota.bin.ota $dest_dir/$descr$fw_tag.ota
+    mv $build_dir/firmware.elf $dest_dir/$descr$fw_tag.elf
+    mv $build_dir/firmware.map $dest_dir/$descr$fw_tag.map
+    #rm -rf $build_dir
+}
+
+# check/get parameters
+if [ $# != 2 ]; then
+    echo "usage: $0 <fw-tag> <dest-dir>"
+    exit 1
+fi
+
+fw_tag=$1
+dest_dir=$2
+
+# check we are in the correct directory
+if [ ! -r boards/esp8266_common.ld ]; then
+    echo "must be in esp8266 directory"
+    exit 1
+fi
+
+# build the versions
+do_build esp8266 GENERIC
+do_build esp8266-512k GENERIC_512K
+do_build esp8266-1m GENERIC_1M
+do_build_ota esp8266-ota GENERIC ota
diff --git a/tools/autobuild/build-rp2-latest.sh b/tools/autobuild/build-rp2-latest.sh
new file mode 100755
index 0000000000..5124686e21
--- /dev/null
+++ b/tools/autobuild/build-rp2-latest.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+
+# function for building firmware
+function do_build() {
+    descr=$1
+    board=$2
+    shift
+    shift
+    echo "building $descr $board"
+    build_dir=/tmp/rp2-build-$board
+    $MICROPY_AUTOBUILD_MAKE $@ BOARD=$board BUILD=$build_dir || exit 1
+    mv $build_dir/firmware.uf2 $dest_dir/$descr$fw_tag.uf2
+    rm -rf $build_dir
+}
+
+# check/get parameters
+if [ $# != 2 ]; then
+    echo "usage: $0 <fw-tag> <dest-dir>"
+    exit 1
+fi
+
+fw_tag=$1
+dest_dir=$2
+
+# check we are in the correct directory
+if [ ! -r modrp2.c ]; then
+    echo "must be in rp2 directory"
+    exit 1
+fi
+
+# build the boards
+do_build rp2-pico PICO
diff --git a/tools/autobuild/build-stm32-latest.sh b/tools/autobuild/build-stm32-latest.sh
new file mode 100755
index 0000000000..226c14a41e
--- /dev/null
+++ b/tools/autobuild/build-stm32-latest.sh
@@ -0,0 +1,56 @@
+#!/bin/bash
+
+# function for building firmware
+function do_build() {
+    descr=$1
+    board=$2
+    shift
+    shift
+    echo "building $descr $board"
+    build_dir=/tmp/stm-build-$board
+    $MICROPY_AUTOBUILD_MAKE -B $@ BOARD=$board BUILD=$build_dir || exit 1
+    mv $build_dir/firmware.dfu $dest_dir/$descr$fw_tag.dfu
+    rm -rf $build_dir
+}
+
+# check/get parameters
+if [ $# != 2 ]; then
+    echo "usage: $0 <fw-tag> <dest-dir>"
+    exit 1
+fi
+
+fw_tag=$1
+dest_dir=$2
+
+# check we are in the correct directory
+if [ ! -r modpyb.c ]; then
+    echo "must be in stm directory"
+    exit 1
+fi
+
+# build the versions
+do_build pybv3 PYBV3
+do_build pybv3-network PYBV3 MICROPY_PY_WIZNET5K=5200 MICROPY_PY_CC3K=1
+do_build pybv10 PYBV10
+do_build pybv10-dp PYBV10 MICROPY_FLOAT_IMPL=double
+do_build pybv10-thread PYBV10 CFLAGS_EXTRA='-DMICROPY_PY_THREAD=1'
+do_build pybv10-dp-thread PYBV10 MICROPY_FLOAT_IMPL=double CFLAGS_EXTRA='-DMICROPY_PY_THREAD=1'
+do_build pybv10-network PYBV10 MICROPY_PY_WIZNET5K=5200 MICROPY_PY_CC3K=1
+do_build pybv11 PYBV11
+do_build pybv11-dp PYBV11 MICROPY_FLOAT_IMPL=double
+do_build pybv11-thread PYBV11 CFLAGS_EXTRA='-DMICROPY_PY_THREAD=1'
+do_build pybv11-dp-thread PYBV11 MICROPY_FLOAT_IMPL=double CFLAGS_EXTRA='-DMICROPY_PY_THREAD=1'
+do_build pybv11-network PYBV11 MICROPY_PY_WIZNET5K=5200 MICROPY_PY_CC3K=1
+do_build pyblitev10 PYBLITEV10
+do_build pyblitev10-dp PYBLITEV10 MICROPY_FLOAT_IMPL=double
+do_build pyblitev10-thread PYBLITEV10 CFLAGS_EXTRA='-DMICROPY_PY_THREAD=1'
+do_build pyblitev10-dp-thread PYBLITEV10 MICROPY_FLOAT_IMPL=double CFLAGS_EXTRA='-DMICROPY_PY_THREAD=1'
+do_build pyblitev10-network PYBLITEV10 MICROPY_PY_WIZNET5K=5200 MICROPY_PY_CC3K=1
+do_build PYBD-SF2 PYBD_SF2
+do_build PYBD-SF3 PYBD_SF3
+do_build PYBD-SF6 PYBD_SF6
+
+for board in boards/{NUCLEO_*,STM32F*DISC,B_L*,USBDONGLE_WB55,ESPRUINO_PICO} ; do
+    bd=$(basename $board)
+    do_build $bd $bd USE_MBOOT=0 MBOOT_ENABLE_PACKING=0
+done
diff --git a/tools/autobuild/remove_old_firmware.py b/tools/autobuild/remove_old_firmware.py
new file mode 100755
index 0000000000..e9d2e8aae8
--- /dev/null
+++ b/tools/autobuild/remove_old_firmware.py
@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+
+import re, subprocess, sys
+
+
+DEBUG = False
+DRY_RUN = False
+NUM_KEEP_PER_BOARD = 4
+
+
+def main():
+    ssh_machine = sys.argv[1]
+    ssh_firmware_dir = sys.argv[2]
+
+    # SSH to get list of existing files.
+    p = subprocess.run(
+        ["ssh", ssh_machine, "find", ssh_firmware_dir, "-name", "\\*-unstable-v\\*"],
+        capture_output=True,
+    )
+    if p.returncode != 0:
+        print(p.stderr)
+        return
+    all_files = p.stdout.split(b"\n")
+
+    # Parse all files to organise into boards/date/version.
+    boards = {}
+    for file in all_files:
+        m = re.match(
+            rb"([a-z/.]+)/([A-Za-z0-9_-]+)-(20[0-9]{6})-unstable-(v[0-9.-]+-g[0-9a-f]+).",
+            file,
+        )
+        if not m:
+            continue
+        dir, board, date, version = m.groups()
+        if board not in boards:
+            boards[board] = {}
+        if (date, version) not in boards[board]:
+            boards[board][(date, version)] = []
+        boards[board][(date, version)].append(file)
+
+    # Collect files to remove based on date and version.
+    remove = []
+    for board in boards.values():
+        filelist = [(date, version, files) for (date, version), files in board.items()]
+        filelist.sort(reverse=True)
+        keep = []
+        for date, version, files in filelist:
+            if keep and version == keep[-1]:
+                remove.extend(files)
+            elif len(keep) >= NUM_KEEP_PER_BOARD:
+                remove.extend(files)
+            else:
+                keep.append(version)
+
+    if DEBUG:
+        all_files.sort(reverse=True)
+        for file in all_files:
+            print(file, file in remove)
+        print(len(remove), "/", len(all_files))
+
+    # Do removal of files.
+    for file in remove:
+        file = str(file, "ascii")
+        print("remove:", file)
+        if not DRY_RUN:
+            p = subprocess.run(["ssh", ssh_machine, "/bin/rm", file], capture_output=True)
+            if p.returncode != 0:
+                print(p.stderr)
+
+
+if __name__ == "__main__":
+    main()