fetch-port-submodules; fetch-all-submodules

2023-06-05 17:16:25 -04:00 · 2023-06-05 17:16:25 -04:00 · 38f6fa27e3
parent 0aaf5a4a98
commit 38f6fa27e3
16 changed files with 104 additions and 40 deletions
--- a/BUILDING.md
+++ b/BUILDING.md
@ -27,7 +27,11 @@ This project has a bunch of git submodules.  You will need to update them regula

 In the root folder of the CircuitPython repository, execute the following:

-    make fetch-submodules
+    make fetch-all-submodules
+
+Or, in the ports directory for the particular port you are building, do:
+
+    make fetch-port-submodules

 ### Required Python Packages

--- a/26
+++ b/26
@ -61,7 +61,6 @@ TRANSLATE_SOURCES_EXC = -path "ports/*/build-*" \

 help:
 	@echo "Please use \`make <target>' where <target> is one of"
-	@echo "  fetch-submodules	to fetch dependencies from submodules, run this right after you clone the repo"
 	@echo "  html       to make standalone HTML files"
 	@echo "  dirhtml    to make HTML files named index.html in directories"
 	@echo "  singlehtml to make a single large HTML file"
@ -84,6 +83,8 @@ help:
 	@echo "  pseudoxml  to make pseudoxml-XML files for display purposes"
 	@echo "  linkcheck  to check all external links for integrity"
 	@echo "  doctest    to run all doctests embedded in the documentation (if enabled)"
+	@echo "  fetch-all-submodules	to fetch submodules for all ports"
+	@echo "  remove-all-submodules	remove all submodules, including files and .git/ data"

 clean:
 	rm -rf $(BUILDDIR)/*
@ -324,25 +325,12 @@ clean-stm:
 	$(MAKE) -C ports/stm BOARD=feather_stm32f405_express clean


-# If available, do blobless partial clones of submodules to save time and space.
-# A blobless partial clone lazily fetches data as needed, but has all the metadata available (tags, etc.)
-# so it does not have the idiosyncrasies of a shallow clone.
-#
-# If not available, do a fetch that will fail, and then fix it up with a second fetch.
-# (Only works for git servers that allow sha fetches.)
-.PHONY: fetch-submodules
-fetch-submodules:
-	git submodule sync
-	#####################################################################################
-	# NOTE: Ideally, use git version 2.36.0 or later, to do partial clones of submodules.
-	# If an older git is used, submodules will be cloned with a shallow clone of depth 1.
-	# You will see a git usage message first if the git version is too old to do
-	# clones of submodules.
-	#####################################################################################
-	git submodule update --init --filter=blob:none || git submodule update --init -N --depth 1 || git submodule foreach 'git fetch --tags --depth 1 origin $$sha1 && git checkout -q $$sha1' || echo 'make fetch-submodules FAILED'
+.PHONY: fetch-all-submodules
+fetch-all-submodules:
+	tools/fetch-submodules.sh

-.PHONY: remove-submodules
-remove-submodules:
+.PHONY: remove-all-submodules
+remove-all-submodules:
 	git submodule deinit -f --all
 	rm -rf .git/modules/*

--- a/ports/atmel-samd/Makefile
+++ b/ports/atmel-samd/Makefile
@ -363,11 +363,15 @@ SRC_QSTR_PREPROCESSOR += peripherals/samd/$(PERIPHERALS_CHIP_FAMILY)/clocks.c

 all: $(BUILD)/firmware.bin $(BUILD)/firmware.uf2

+ifeq ($(VALID_BOARD),)
+$(BUILD)/firmware.elf: invalid-board
+else
 $(BUILD)/firmware.elf: $(OBJ) $(GENERATED_LD_FILE)
 	$(STEPECHO) "LINK $@"
 	$(Q)echo $(OBJ) > $(BUILD)/firmware.objs
 	$(Q)$(CC) -o $@ $(LDFLAGS) @$(BUILD)/firmware.objs -Wl,--print-memory-usage -Wl,--start-group $(LIBS) -Wl,--end-group
 	$(Q)$(SIZE) $@ | $(PYTHON) $(TOP)/tools/build_memory_info.py $(GENERATED_LD_FILE) $(BUILD)
+endif

 $(BUILD)/firmware.bin: $(BUILD)/firmware.elf
 	$(STEPECHO) "Create $@"
--- a/ports/broadcom/Makefile
+++ b/ports/broadcom/Makefile
@ -142,10 +142,14 @@ all: $(BUILD)/firmware.kernel$(SUFFIX).img $(BUILD)/firmware.disk.img.zip
 %.o: %.c
 	$(CC) $(CFLAGS) -c $< -o $@

+ifeq ($(VALID_BOARD),)
+$(BUILD)/kernel$(SUFFIX).elf: invalid-board
+else
 $(BUILD)/kernel$(SUFFIX).elf: $(OBJ)
 	$(STEPECHO) "LINK $@"
 	$(Q)echo $(OBJ) > $(BUILD)/firmware.objs
 	$(Q)$(CC) -o $@ $(LDFLAGS) @$(BUILD)/firmware.objs -Wl,--start-group $(LIBS) -Wl,--end-group
+endif

 $(BUILD)/kernel$(SUFFIX).img: $(BUILD)/kernel$(SUFFIX).elf
 	$(STEPECHO) "Create $@"
--- a/ports/cxd56/Makefile
+++ b/ports/cxd56/Makefile
@ -183,9 +183,13 @@ $(BUILD)/firmware.elf: $(BUILD)/libmpy.a
 $(MKSPK):
 	$(MAKE) -C mkspk

+ifeq ($(VALID_BOARD),)
+$(BUILD)/firmware.spk: invalid-board
+else
 $(BUILD)/firmware.spk: $(BUILD)/firmware.elf $(MKSPK)
 	$(ECHO) "Creating $@"
 	$(MKSPK) -c 2 $(BUILD)/firmware.elf nuttx $(BUILD)/firmware.spk
+endif

 flash: $(BUILD)/firmware.spk
 	$(ECHO) "Writing $< to the board"
--- a/ports/espressif/Makefile
+++ b/ports/espressif/Makefile
@ -411,6 +411,7 @@ ESP_IDF_COMPONENTS_EXPANDED += $(BUILD)/esp-idf/esp-idf/esp32-camera/libesp32-ca
 #$(error $(ESP_IDF_COMPONENTS_EXPANDED))
 endif

+ifneq ($(VALID_BOARD),)
 # BOOTLOADER_OFFSET is determined by chip type, based on the ROM bootloader, and is not changeable.
 ifeq ($(IDF_TARGET),esp32)
 BOOTLOADER_OFFSET = 0x1000
@ -423,6 +424,7 @@ BOOTLOADER_OFFSET = 0x1000
 else
 $(error unknown IDF_TARGET $(IDF_TARGET))
 endif
+endif

 IDF_CMAKE_TARGETS = \
 		bootloader/bootloader.bin \
@ -457,8 +459,12 @@ $(BUILD)/circuitpython-firmware.bin: $(BUILD)/firmware.elf | tools/build_memory_
 	$(Q)esptool.py --chip $(IDF_TARGET) elf2image $(FLASH_FLAGS) --elf-sha256-offset 0xb0 -o $@ $^
 	$(Q)$(PYTHON) tools/build_memory_info.py $< $(BUILD)/esp-idf/sdkconfig $@ $(BUILD)

+ifeq ($(VALID_BOARD),)
+$(BUILD)/firmware.bin: invalid-board
+else
 $(BUILD)/firmware.bin: $(BUILD)/circuitpython-firmware.bin | esp-idf-stamp
 	$(Q)$(PYTHON) ../../tools/join_bins.py $@ $(BOOTLOADER_OFFSET) $(BUILD)/esp-idf/bootloader/bootloader.bin $(PARTITION_TABLE_OFFSET) $(BUILD)/esp-idf/partition_table/partition-table.bin $(FIRMWARE_OFFSET) $(BUILD)/circuitpython-firmware.bin
+endif

 UF2_FAMILY_ID_esp32s2 = 0xbfdd4eee
 UF2_FAMILY_ID_esp32s3 = 0xc47e5767
--- a/ports/litex/Makefile
+++ b/ports/litex/Makefile
@ -133,10 +133,14 @@ SRC_QSTR_PREPROCESSOR +=

 all: $(BUILD)/firmware.bin $(BUILD)/firmware.dfu

+ifeq ($(VALID_BOARD),)
+$(BUILD)/firmware.elf: invalid-board
+else
 $(BUILD)/firmware.elf: $(OBJ)
 	$(STEPECHO) "LINK $@"
 	$(Q)$(CC) -o $@ $(LDFLAGS) $^ -Wl,--start-group $(LIBS) -Wl,--end-group
 	$(Q)$(SIZE) $@ | $(PYTHON) $(TOP)/tools/build_memory_info.py $(LD_FILE) $(BUILD)
+endif

 $(BUILD)/firmware.bin: $(BUILD)/firmware.elf
 	$(STEPECHO) "Create $@"
--- a/ports/mimxrt10xx/Makefile
+++ b/ports/mimxrt10xx/Makefile
@ -200,9 +200,13 @@ SRC_QSTR += $(SRC_C) $(SRC_SUPERVISOR) $(SRC_COMMON_HAL_EXPANDED) $(SRC_SHARED_M

 all: $(BUILD)/firmware.bin $(BUILD)/firmware.uf2 $(BUILD)/firmware.hex

+ifeq ($(VALID_BOARD),)
+$(BUILD)/firmware.elf: invalid-board
+else
 $(BUILD)/firmware.elf: $(OBJ) $(LD_FILES)
 	$(STEPECHO) "LINK $@"
 	$(Q)$(CC) -o $@ $(LDFLAGS) $(filter-out %.ld, $^) -Wl,--print-memory-usage -Wl,--start-group $(LIBS) -Wl,--end-group
+endif

 # -R excludes sections from the output files.
 $(BUILD)/firmware.bin: $(BUILD)/firmware.elf
--- a/ports/nrf/Makefile
+++ b/ports/nrf/Makefile
@ -208,11 +208,15 @@ UF2_FAMILY_ID_nrf52833 = 0x621E937A

 all: $(BUILD)/firmware.bin $(BUILD)/firmware.uf2 $(BUILD)/firmware.combined.hex

+ifeq ($(VALID_BOARD),)
+$(BUILD)/firmware.elf: invalid-board
+else
 $(BUILD)/firmware.elf: $(OBJ) $(GENERATED_LD_FILE)
 	$(STEPECHO) "LINK $@"
 	$(Q)echo $(OBJ) > $(BUILD)/firmware.objs
 	$(Q)$(CC) -o $@ $(LDFLAGS) @$(BUILD)/firmware.objs -Wl,--start-group $(LIBS) -Wl,--end-group
 	$(Q)$(SIZE) $@ | $(PYTHON) $(TOP)/tools/build_memory_info.py $(GENERATED_LD_FILE) $(BUILD)
+endif

 $(BUILD)/firmware.bin: $(BUILD)/firmware.elf
 	$(STEPECHO) "Create $@"
--- a/ports/raspberrypi/Makefile
+++ b/ports/raspberrypi/Makefile
@ -434,11 +434,15 @@ endif

 LINKER_SCRIPTS += -Wl,-T,link.ld

+ifeq ($(VALID_BOARD),)
+$(BUILD)/firmware.elf: invalid-board
+else
 $(BUILD)/firmware.elf: $(OBJ) $(BOARD_LD) link.ld
 	$(STEPECHO) "LINK $@"
 	$(Q)echo $(OBJ) > $(BUILD)/firmware.objs
 	$(Q)echo $(PICO_LDFLAGS) > $(BUILD)/firmware.ldflags
 	$(Q)$(CC) -o $@ $(CFLAGS) @$(BUILD)/firmware.ldflags $(LINKER_SCRIPTS) -Wl,--print-memory-usage -Wl,-Map=$@.map -Wl,-cref -Wl,--gc-sections @$(BUILD)/firmware.objs -Wl,-lc
+endif

 $(BUILD)/firmware.bin: $(BUILD)/firmware.elf
 	$(STEPECHO) "Create $@"
--- a/ports/silabs/Makefile
+++ b/ports/silabs/Makefile
@ -137,9 +137,13 @@ MCU_SECTIONS = $^ $@
 # Default goal
 all: $(OUTPUT_DIR)/firmware.bin

+ifeq ($(VALID_BOARD),)
+$(BUILD)/firmware.bin: invalid-board
+else
 $(OUTPUT_DIR)/firmware.bin: $(SILABS_BUILD)/$(PROJECTNAME).Makefile $(OUTPUT_DIR)/firmware.hex
 	+@$(MAKE) --no-print-directory $(OUTPUT_DIR)/firmware.out
 	@echo 'Done.'
+endif

 $(SILABS_BUILD)/$(PROJECTNAME).Makefile:
 	+@$(MAKE) --no-print-directory slc-generate
--- a/ports/silabs/README.md
+++ b/ports/silabs/README.md
@ -47,8 +47,8 @@ Ensure your clone of Circuitpython is ready to build by following the [guide on
 Clone the source code of CircuitPython from Github:

    $  git clone https://github.com/SiliconLabs/circuitpython.git
-    $  cd circuitpython
-    $  make fetch-submodules
+    $  cd circuitpython/ports/silabs
+    $  make fetch-port-submodules

 Checkout the branch or tag you want to build. For example:

--- a/ports/stm/Makefile
+++ b/ports/stm/Makefile
@ -52,7 +52,7 @@ INC += -I../../supervisor/shared/usb
 #Debugging/Optimization
 ifeq ($(DEBUG), 1)
 	CFLAGS += -ggdb3
-	# You may want to enable these flags to make setting breakpoints easier.
+# You may want to enable these flags to make setting breakpoints easier.
 	CFLAGS += -fno-inline -fno-ipa-sra
 else
 	CFLAGS += -DNDEBUG
@ -264,11 +264,15 @@ endif

 all: $(BUILD)/firmware.bin $(BUILD)/firmware.uf2

+ifeq ($(VALID_BOARD),)
+$(BUILD)/firmware.elf: invalid-board
+else
 $(BUILD)/firmware.elf: $(OBJ)
 	$(STEPECHO) "LINK $@"
 	$(Q)echo $^ > $(BUILD)/firmware.objs
 	$(Q)$(CC) -o $@ $(LDFLAGS) @$(BUILD)/firmware.objs -Wl,--start-group $(LIBS) -Wl,--end-group
 	$(Q)$(SIZE) $@ | $(PYTHON) $(TOP)/tools/build_memory_info.py $(LD_FILE) $(BUILD)
+endif

 $(BUILD)/firmware.bin: $(BUILD)/firmware.elf
 	$(STEPECHO) "Create $@"
--- a/py/circuitpy_defns.mk
+++ b/py/circuitpy_defns.mk
@ -829,3 +829,15 @@ check-release-needs-clean-build:

 # Ignore these errors
 $(BUILD)/lib/libm/kf_rem_pio2.o: CFLAGS += -Wno-maybe-uninitialized
+
+# Fetch only submodules needed for this particular port.
+.PHONY: fetch-port-submodules
+fetch-port-submodules:
+	$(TOP)/tools/fetch-submodules.sh data extmod frozen lib $(CURDIR)
+
+.PHONY: invalid-board
+invalid-board:
+	$(Q)if [ -z "$(BOARD)" ] ; then echo "ERROR: No BOARD specified" ; else echo "ERROR: Invalid BOARD $(BOARD) specified"; fi && \
+	echo "Valid boards:" && \
+	printf '%s\n' $(ALL_BOARDS_IN_PORT) | column -xc $$(tput cols || echo 80) 1>&2 && \
+	false
--- a/py/circuitpy_mkenv.mk
+++ b/py/circuitpy_mkenv.mk
@ -24,22 +24,10 @@

 # Common Makefile items that can be shared across CircuitPython ports.

-# Select the board to build for.
-define show_board_error
-$(info Valid boards:)
-$(shell printf '%s\n' $(patsubst boards/%/mpconfigboard.mk,%,$(wildcard boards/*/mpconfigboard.mk)) | column -xc $$(tput cols || echo 80) 1>&2)
-$(error Rerun with $(MAKE) BOARD=<board>)
-endef
-
-ifeq ($(BOARD),)
-    $(info No BOARD specified)
-    $(call show_board_error)
-else
-  ifeq ($(wildcard boards/$(BOARD)/.),)
-    $(info Invalid BOARD specified)
-    $(call show_board_error)
-  endif
-endif
+ALL_BOARDS_IN_PORT := $(patsubst boards/%/mpconfigboard.mk,%,$(wildcard boards/*/mpconfigboard.mk))
+# An incorrect BOARD might have been specified, so check against the list.
+# There is deliberately no space after the :=
+VALID_BOARD :=$(filter $(BOARD),$(ALL_BOARDS_IN_PORT))

 # If the flash PORT is not given, use the default /dev/tty.SLAB_USBtoUART.
 PORT ?= /dev/tty.SLAB_USBtoUART
@ -47,16 +35,22 @@ PORT ?= /dev/tty.SLAB_USBtoUART
 # If the build directory is not given, make it reflect the board name.
 BUILD ?= build-$(BOARD)

+# First makefile with targets. Defines the default target.
 include ../../py/mkenv.mk

-# Board-specific
+# Board-specific. Skip if the rule requested is not board-specific.
+ifneq ($(VALID_BOARD),)
 include boards/$(BOARD)/mpconfigboard.mk
+endif

 # Port-specific
 include mpconfigport.mk

+# Also board-specific. Skip if the rule requested is not board-specific.
+ifneq ($(VALID_BOARD),)
 # CircuitPython-specific
 include $(TOP)/py/circuitpy_mpconfig.mk
+endif

 # qstr definitions (must come before including py.mk)
 QSTR_DEFS = qstrdefsport.h
--- a/tools/fetch-submodules.sh
+++ b/tools/fetch-submodules.sh
@ -0,0 +1,24 @@
+#!/usr/bin/env sh
+# Pass the directories of submodules to fetch as command-line arguments. For example:
+# ./fetch-submodules.sh lib tools
+# If no arguments are passed, all submodules will be fetched.
+# This script handles being called at other than the top level by making the paths
+# for the submodules absolute.
+
+TOP=$(git rev-parse --show-toplevel)
+
+git submodule sync --quiet
+# Prefix all the args with the absolute path to the top of the repo.
+abs_submodules=""
+for d in "$@"; do
+    abs_submodules="${abs_submodules} ${TOP}/${d}"
+done
+echo ${abs_submodules}
+
+# Fetch submodules as partial clones if possible. If that fails due to an older version of git,
+# do a shallow init with no fetch and then check out the proper commit.
+git submodule update --init --filter=blob:none ${abs_submodules} || \
+    git submodule update --init --no-fetch --depth 1 ${abs_submodules} || \
+    git submodule foreach $* \
+        'git fetch --tags --depth 1 origin $$sha1 && git checkout -q $$sha1' || \
+    echo "ERROR: fetch-submodules.sh FAILED"