From 2dcb2f06cbfaacb5cf362cac1ba547dd45d13e9a Mon Sep 17 00:00:00 2001 From: Dan Halbert Date: Sat, 23 Dec 2017 23:46:19 -0500 Subject: [PATCH 1/3] Add git version info to frozen modules. --- .gitmodules | 3 ++ py/mkenv.mk | 1 + py/mkrules.mk | 7 ++- tools/preprocess_frozen_modules.py | 69 ++++++++++++++++++++++++++++++ tools/python-semver | 1 + 5 files changed, 77 insertions(+), 4 deletions(-) create mode 100755 tools/preprocess_frozen_modules.py create mode 160000 tools/python-semver diff --git a/.gitmodules b/.gitmodules index faad88a857..ef87dd1ba1 100644 --- a/.gitmodules +++ b/.gitmodules @@ -29,3 +29,6 @@ [submodule "frozen/Adafruit_CircuitPython_BusDevice"] path = frozen/Adafruit_CircuitPython_BusDevice url = https://github.com/adafruit/Adafruit_CircuitPython_BusDevice.git +[submodule "tools/python-semver"] + path = tools/python-semver + url = https://github.com/k-bx/python-semver.git diff --git a/py/mkenv.mk b/py/mkenv.mk index 5a118854aa..5f8cd902bf 100644 --- a/py/mkenv.mk +++ b/py/mkenv.mk @@ -70,6 +70,7 @@ endif MAKE_FROZEN = $(TOP)/tools/make-frozen.py MPY_CROSS = $(TOP)/mpy-cross/mpy-cross MPY_TOOL = $(TOP)/tools/mpy-tool.py +PREPROCESS_FROZEN_MODULES = PYTHONPATH=$(TOP)/tools/python-semver $(TOP)/tools/preprocess_frozen_modules.py all: .PHONY: all diff --git a/py/mkrules.mk b/py/mkrules.mk index 1b33555128..713e69c622 100644 --- a/py/mkrules.mk +++ b/py/mkrules.mk @@ -114,14 +114,13 @@ $(MPY_CROSS): $(TOP)/py/*.[ch] $(TOP)/mpy-cross/*.[ch] $(TOP)/windows/fmode.c $(Q)$(MAKE) -C $(TOP)/mpy-cross # Copy all the modules and single python files to freeze to a common area, omitting top-level dirs (the repo names). -# Remove any conf.py (sphinx config) and setup.py (module install info) files, which are not meant to be frozen. -# Also remove the library examples directory, so it won't be included. +# Do any preprocessing necessary: currently, this adds version information, removes examples, and +# non-library .py files in the modules (setup.py and conf.py) # Then compile .mpy files from all the .py files, placing them in the same directories as the .py files. $(BUILD)/frozen_mpy: $(FROZEN_MPY_DIRS) $(ECHO) FREEZE $(FROZEN_MPY_DIRS) $(Q)$(MKDIR) -p $@ - $(Q)$(RSYNC) -rL --include="*/" --include='*.py' --exclude="*" $(addsuffix /*,$(FROZEN_MPY_DIRS)) $@ - $(Q)$(RM) -rf $@/conf.py $@/setup.py $@/examples + $(Q)$(PREPROCESS_FROZEN_MODULES) -o $@ $(FROZEN_MPY_DIRS) $(Q)$(CD) $@ && \ $(FIND) -L . -type f -name '*.py' | sed 's=^\./==' | \ xargs -n1 $(abspath $(MPY_CROSS)) $(MPY_CROSS_FLAGS) diff --git a/tools/preprocess_frozen_modules.py b/tools/preprocess_frozen_modules.py new file mode 100755 index 0000000000..e29b54b16f --- /dev/null +++ b/tools/preprocess_frozen_modules.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +import argparse +import os +import os.path +from pathlib import Path +import semver +import subprocess + +def version_string(path=None, *, valid_semver=False): + version = None + tag = subprocess.run('git describe --tags --exact-match', shell=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=path) + if tag.returncode == 0: + version = tag.stdout.strip().decode("utf-8", "strict") + else: + describe = subprocess.run("git describe --tags", shell=True, stdout=subprocess.PIPE, cwd=path) + tag, additional_commits, commitish = describe.stdout.strip().decode("utf-8", "strict").rsplit("-", maxsplit=2) + commitish = commitish[1:] + if valid_semver: + version_info = semver.parse_version_info(tag) + if not version_info.prerelease: + version = semver.bump_patch(tag) + "-alpha.0.plus." + additional_commits + "+" + commitish + else: + version = tag + ".plus." + additional_commits + "+" + commitish + else: + version = commitish + return version + +# Visit all the .py files in topdir. Replace any __version__ = "0.0.0-auto.0" type of info +# with actual version info derived from git. +def copy_and_process(in_dir, out_dir): + for root, subdirs, files in os.walk(in_dir): + + # Skip library examples directories. + if Path(root).name == 'examples': + continue + + for file in files: + # Skip top-level setup.py (module install info) and conf.py (sphinx config), + # which are not part of the library + if (root == in_dir) and file in ('conf.py', 'setup.py'): + continue + + input_file_path = Path(root, file) + output_file_path = Path(out_dir, input_file_path.relative_to(in_dir)) + + if file.endswith(".py"): + output_file_path.parent.mkdir(parents=True, exist_ok=True) + with input_file_path.open("r") as input, output_file_path.open("w") as output: + for line in input: + if line.startswith("__version__"): + module_version = version_string(root, valid_semver=True) + line = line.replace("0.0.0-auto.0", module_version) + output.write(line) + +if __name__ == '__main__': + argparser = argparse.ArgumentParser(description="""\ + Copy and pre-process .py files into output directory, before freezing. + 1. Remove top-level repo directory. + 2. Update __version__ info. + 3. Remove examples. + 4. Remove non-library setup.py and conf.py""") + argparser.add_argument("in_dirs", metavar="input-dir", nargs="+", + help="top-level code dirs (may be git repo dirs)") + argparser.add_argument("-o", "--out_dir", help="output directory") + args = argparser.parse_args() + + for in_dir in args.in_dirs: + copy_and_process(in_dir, args.out_dir) diff --git a/tools/python-semver b/tools/python-semver new file mode 160000 index 0000000000..2001c62d1a --- /dev/null +++ b/tools/python-semver @@ -0,0 +1 @@ +Subproject commit 2001c62d1a0361c44acc7076d8ce91e1d1c66141 From 644ad74ea1a33ef7112da1537e83c802b2b15202 Mon Sep 17 00:00:00 2001 From: Dan Halbert Date: Sun, 24 Dec 2017 13:49:58 -0500 Subject: [PATCH 2/3] preprocess_frozen_modules.py: make compatible with Python 3.4. --- tools/preprocess_frozen_modules.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/preprocess_frozen_modules.py b/tools/preprocess_frozen_modules.py index e29b54b16f..b425ffa2d2 100755 --- a/tools/preprocess_frozen_modules.py +++ b/tools/preprocess_frozen_modules.py @@ -45,7 +45,9 @@ def copy_and_process(in_dir, out_dir): output_file_path = Path(out_dir, input_file_path.relative_to(in_dir)) if file.endswith(".py"): - output_file_path.parent.mkdir(parents=True, exist_ok=True) + # mkdir() takes an exists_ok=True, but not until Python 3.5. + if not output_file_path.parent.exists(): + output_file_path.parent.mkdir(parents=True) with input_file_path.open("r") as input, output_file_path.open("w") as output: for line in input: if line.startswith("__version__"): From 31be20744deaeb24a18205fc8099d2628531830d Mon Sep 17 00:00:00 2001 From: Dan Halbert Date: Sun, 24 Dec 2017 14:29:03 -0500 Subject: [PATCH 3/3] preprocess_frozen_modules.py: yet more Python 3.4 compatibility changes --- tools/preprocess_frozen_modules.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tools/preprocess_frozen_modules.py b/tools/preprocess_frozen_modules.py index b425ffa2d2..0233fba614 100755 --- a/tools/preprocess_frozen_modules.py +++ b/tools/preprocess_frozen_modules.py @@ -6,15 +6,16 @@ from pathlib import Path import semver import subprocess +# Compatible with Python 3.4 due to travis using trusty as default. + def version_string(path=None, *, valid_semver=False): version = None - tag = subprocess.run('git describe --tags --exact-match', shell=True, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=path) - if tag.returncode == 0: - version = tag.stdout.strip().decode("utf-8", "strict") - else: - describe = subprocess.run("git describe --tags", shell=True, stdout=subprocess.PIPE, cwd=path) - tag, additional_commits, commitish = describe.stdout.strip().decode("utf-8", "strict").rsplit("-", maxsplit=2) + try: + tag = subprocess.check_output('git describe --tags --exact-match', shell=True, cwd=path) + version = tag.strip().decode("utf-8", "strict") + except subprocess.CalledProcessError: + describe = subprocess.check_output("git describe --tags", shell=True, cwd=path) + tag, additional_commits, commitish = describe.strip().decode("utf-8", "strict").rsplit("-", maxsplit=2) commitish = commitish[1:] if valid_semver: version_info = semver.parse_version_info(tag) @@ -45,7 +46,6 @@ def copy_and_process(in_dir, out_dir): output_file_path = Path(out_dir, input_file_path.relative_to(in_dir)) if file.endswith(".py"): - # mkdir() takes an exists_ok=True, but not until Python 3.5. if not output_file_path.parent.exists(): output_file_path.parent.mkdir(parents=True) with input_file_path.open("r") as input, output_file_path.open("w") as output: