From c618f91e22613b2e530f7c21be0be789a7e8eed5 Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Tue, 19 Apr 2016 11:30:06 +0300 Subject: [PATCH] py: Rework QSTR extraction to work in simple and obvious way. When there're C files to be (re)compiled, they're all passed first to preprocessor. QSTR references are extracted from preprocessed output and split per original C file. Then all available qstr files (including those generated previously) are catenated together. Only if the resulting content has changed, the output file is written (causing almost global rebuild to pick up potentially renumbered qstr's). Otherwise, it's not updated to not cause spurious rebuilds. Related make rules are split to minimize amount of commands executed in the interim case (when some C files were updated, but no qstrs were changed). --- py/makeqstrdefs.py | 102 ++++++++++++++++++++++++++++----------------- py/mkrules.mk | 12 +++--- py/qstr.c | 2 + py/qstr.h | 2 + 4 files changed, 74 insertions(+), 44 deletions(-) diff --git a/py/makeqstrdefs.py b/py/makeqstrdefs.py index de0fefe5b5..35d7658e90 100644 --- a/py/makeqstrdefs.py +++ b/py/makeqstrdefs.py @@ -20,39 +20,85 @@ def debug(message): pass +def write_out(fname, output): + if output: + fname = fname.replace("/", "__").replace("..", "@@") + with open(args.output_dir + "/" + fname + ".qstr", "w") as f: + f.write("\n".join(output) + "\n") + def process_file(f): output = [] + last_fname = None + outf = None for line in f: + if line and line[0] == "#": + comp = line.split() + fname = comp[2] + assert fname[0] == '"' and fname[-1] == '"' + fname = fname[1:-1] + if fname[0] == "/" or not fname.endswith(".c"): + continue + if fname != last_fname: + write_out(last_fname, output) + output = [] + last_fname = fname + continue for match in re.findall(r'MP_QSTR_[_a-zA-Z0-9]+', line): name = match.replace('MP_QSTR_', '') if name not in QSTRING_BLACK_LIST: output.append('Q(' + name + ')') - # make sure there is a newline at the end of the output - output.append('') + write_out(last_fname, output) + return "" - return '\n'.join(output) + +def cat_together(): + import glob + import hashlib + hasher = hashlib.md5() + all_lines = [] + outf = open(args.output_dir + "/out", "wb") + for fname in glob.glob(args.output_dir + "/*.qstr"): + with open(fname, "rb") as f: + lines = f.readlines() + all_lines += lines + all_lines.sort() + all_lines = b"\n".join(all_lines) + outf.write(all_lines) + outf.close() + hasher.update(all_lines) + new_hash = hasher.hexdigest() + #print(new_hash) + old_hash = None + try: + with open(args.output_file + ".hash") as f: + old_hash = f.read() + except IOError: + pass + if old_hash != new_hash: + print("QSTR updated") + os.rename(args.output_dir + "/out", args.output_file) + with open(args.output_file + ".hash", "w") as f: + f.write(new_hash) + else: + print("QSTR not updated") if __name__ == "__main__": parser = argparse.ArgumentParser(description='Generates qstr definitions from a specified source') - parser.add_argument('-o', '--output-file', dest='output_filename', - help='Output filename (defaults to stdout)') - parser.add_argument('input_filename', nargs='?', - help='Name of the input file (when not specified, the script reads standard input') - parser.add_argument('-s', '--skip-write-when-same', dest='skip_write_when_same', - action='store_true', default=False, - help="Don't write the output file if it already exists and the contents have not changed (disabled by default)") + parser.add_argument('input_filename', + help='Name of the input file (when not specified, the script reads standard input)') + parser.add_argument('output_dir', + help='Output directory to store individual qstr files') + parser.add_argument('output_file', + help='Name of the output file with collected qstrs') args = parser.parse_args() - - # Check if the file contents changed from last time - write_file = True - - # By default write into STDOUT - outfile = sys.stdout - real_output_filename = 'STDOUT' + try: + os.makedirs(args.output_dir) + except OSError: + pass if args.input_filename: infile = open(args.input_filename, 'r') @@ -61,24 +107,4 @@ if __name__ == "__main__": file_data = process_file(infile) infile.close() - - # Detect custom output file name - if args.output_filename: - real_output_filename = args.output_filename - if os.path.isfile(args.output_filename) and args.skip_write_when_same: - with open(args.output_filename, 'r') as f: - existing_data = f.read() - if existing_data == file_data: - debug("Skip regeneration of: %s\n" % real_output_filename) - write_file = False - else: - debug("File HAS changed, overwriting\n") - outfile = open(args.output_filename, 'w') - else: - outfile = open(args.output_filename, 'w') - - # Only write the file if we the data has changed - if write_file: - sys.stderr.write("QSTR %s\n" % real_output_filename) - outfile.write(file_data) - outfile.close() + cat_together() diff --git a/py/mkrules.mk b/py/mkrules.mk index 8ab06cfdc4..9a739b8b7b 100644 --- a/py/mkrules.mk +++ b/py/mkrules.mk @@ -52,7 +52,7 @@ EMPTY_QSTRDEFS_GENERATED_H = $(BUILD)/tmp/genhdr/qstrdefs.generated.h # List all native flags since the current build system doesn't have # the micropython configuration available. However, these flags are # needed to extract all qstrings -QSTR_GEN_EXTRA_CFLAGS += -P -DN_X64 -DN_X86 -DN_THUMB -DN_ARM +QSTR_GEN_EXTRA_CFLAGS += -D__QSTR_EXTRACT -DN_X64 -DN_X86 -DN_THUMB -DN_ARM QSTR_GEN_EXTRA_CFLAGS += -I$(BUILD)/tmp vpath %.c . $(TOP) @@ -81,13 +81,13 @@ $(EMPTY_QSTRDEFS_GENERATED_H): # to get built before we try to compile any of them. $(OBJ): | $(HEADER_BUILD)/qstrdefs.generated.h $(HEADER_BUILD)/mpversion.h -# This rule joins all generated qstr files -$(QSTR_DEFS_COLLECTED): $(addprefix $(HEADER_BUILD)/,$(addsuffix .qstr,$(SRC_QSTR))) +$(HEADER_BUILD)/qstr.i.last: $(SRC_QSTR) | $(HEADER_BUILD)/mpversion.h $(ECHO) "GEN $@" - $(Q)cat $^ > $@ + $(Q)$(CPP) $(QSTR_GEN_EXTRA_CFLAGS) $(CFLAGS) $? >$(HEADER_BUILD)/qstr.i.last - -# +$(QSTR_DEFS_COLLECTED): $(HEADER_BUILD)/qstr.i.last + $(ECHO) "GEN $@" + $(Q)$(PYTHON) $(PY_SRC)/makeqstrdefs.py $(HEADER_BUILD)/qstr.i.last $(HEADER_BUILD)/qstr $(QSTR_DEFS_COLLECTED) # $(sort $(var)) removes duplicates # diff --git a/py/qstr.c b/py/qstr.c index ef31a682ee..24793ff8a3 100644 --- a/py/qstr.c +++ b/py/qstr.c @@ -93,9 +93,11 @@ const qstr_pool_t mp_qstr_const_pool = { 10, // set so that the first dynamically allocated pool is twice this size; must be <= the len (just below) MP_QSTRnumber_of, // corresponds to number of strings in array just below { +#ifndef __QSTR_EXTRACT #define QDEF(id, str) str, #include "genhdr/qstrdefs.generated.h" #undef QDEF +#endif }, }; diff --git a/py/qstr.h b/py/qstr.h index b5c261f0d5..1989ebe962 100644 --- a/py/qstr.h +++ b/py/qstr.h @@ -37,9 +37,11 @@ // first entry in enum will be MP_QSTR_NULL=0, which indicates invalid/no qstr enum { +#ifndef __QSTR_EXTRACT #define QDEF(id, str) id, #include "genhdr/qstrdefs.generated.h" #undef QDEF +#endif MP_QSTRnumber_of, // no underscore so it can't clash with any of the above };