From 1b60a6dc4e755e14453e024364c8ef5f07780d80 Mon Sep 17 00:00:00 2001
From: Paul Sokolovsky <pfalcon@users.sourceforge.net>
Date: Tue, 19 Apr 2016 14:39:08 +0300
Subject: [PATCH] py: Divide "split" and "cat" phases of qstr extraction for
 better efficiency.

E.g. for stmhal, accumulated preprocessed output may grow large due to
bloated vendor headers, and then reprocessing tens of megabytes on each
build make take couple of seconds on fast hardware (=> potentially dozens
of seconds on slow hardware). So instead, split once after each change,
and only cat repetitively (guaranteed to be fast, as there're thousands
of lines involved at most).
---
 py/makeqstrdefs.py | 14 +++++++-------
 py/mkrules.mk      |  9 +++++++--
 2 files changed, 14 insertions(+), 9 deletions(-)

diff --git a/py/makeqstrdefs.py b/py/makeqstrdefs.py
index 0240505579..d02fb7e3a4 100644
--- a/py/makeqstrdefs.py
+++ b/py/makeqstrdefs.py
@@ -87,6 +87,8 @@ def cat_together():
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description='Generates qstr definitions from a specified source')
 
+    parser.add_argument('command',
+        help='Command (split/cat)')
     parser.add_argument('input_filename',
         help='Name of the input file (when not specified, the script reads standard input)')
     parser.add_argument('output_dir',
@@ -100,11 +102,9 @@ if __name__ == "__main__":
     except OSError:
         pass
 
-    if args.input_filename:
-        infile = open(args.input_filename, 'r')
-    else:
-        infile = sys.stdin
+    if args.command == "split":
+        with open(args.input_filename) as infile:
+            process_file(infile)
 
-    file_data = process_file(infile)
-    infile.close()
-    cat_together()
+    if args.command == "cat":
+        cat_together()
diff --git a/py/mkrules.mk b/py/mkrules.mk
index 7f1bbe8687..fd68a75b18 100644
--- a/py/mkrules.mk
+++ b/py/mkrules.mk
@@ -90,9 +90,14 @@ $(HEADER_BUILD)/qstr.i.last: $(SRC_QSTR) | $(HEADER_BUILD)/mpversion.h
 	    $(CPP) $(QSTR_GEN_EXTRA_CFLAGS) $(CFLAGS) $? >$(HEADER_BUILD)/qstr.i.last; \
 	fi
 
-$(QSTR_DEFS_COLLECTED): $(HEADER_BUILD)/qstr.i.last
+$(HEADER_BUILD)/qstr.split: $(HEADER_BUILD)/qstr.i.last
 	$(ECHO) "GEN $@"
-	$(Q)$(PYTHON) $(PY_SRC)/makeqstrdefs.py $(HEADER_BUILD)/qstr.i.last $(HEADER_BUILD)/qstr $(QSTR_DEFS_COLLECTED)
+	$(Q)$(PYTHON) $(PY_SRC)/makeqstrdefs.py split $(HEADER_BUILD)/qstr.i.last $(HEADER_BUILD)/qstr $(QSTR_DEFS_COLLECTED)
+	$(Q)touch $@
+
+$(QSTR_DEFS_COLLECTED): $(HEADER_BUILD)/qstr.split
+	$(ECHO) "GEN $@"
+	$(Q)$(PYTHON) $(PY_SRC)/makeqstrdefs.py cat $(HEADER_BUILD)/qstr.i.last $(HEADER_BUILD)/qstr $(QSTR_DEFS_COLLECTED)
 
 # $(sort $(var)) removes duplicates
 #