py: Rework QSTR extraction to work in simple and obvious way.

When there're C files to be (re)compiled, they're all passed first to
preprocessor. QSTR references are extracted from preprocessed output and
split per original C file. Then all available qstr files (including those
generated previously) are catenated together. Only if the resulting content
has changed, the output file is written (causing almost global rebuild
to pick up potentially renumbered qstr's). Otherwise, it's not updated
to not cause spurious rebuilds. Related make rules are split to minimize
amount of commands executed in the interim case (when some C files were
updated, but no qstrs were changed).
This commit is contained in:
Paul Sokolovsky 2016-04-19 11:30:06 +03:00
parent 8aa3cbf153
commit c618f91e22
4 changed files with 74 additions and 44 deletions

View File

@ -20,39 +20,85 @@ def debug(message):
pass pass
def write_out(fname, output):
if output:
fname = fname.replace("/", "__").replace("..", "@@")
with open(args.output_dir + "/" + fname + ".qstr", "w") as f:
f.write("\n".join(output) + "\n")
def process_file(f): def process_file(f):
output = [] output = []
last_fname = None
outf = None
for line in f: for line in f:
if line and line[0] == "#":
comp = line.split()
fname = comp[2]
assert fname[0] == '"' and fname[-1] == '"'
fname = fname[1:-1]
if fname[0] == "/" or not fname.endswith(".c"):
continue
if fname != last_fname:
write_out(last_fname, output)
output = []
last_fname = fname
continue
for match in re.findall(r'MP_QSTR_[_a-zA-Z0-9]+', line): for match in re.findall(r'MP_QSTR_[_a-zA-Z0-9]+', line):
name = match.replace('MP_QSTR_', '') name = match.replace('MP_QSTR_', '')
if name not in QSTRING_BLACK_LIST: if name not in QSTRING_BLACK_LIST:
output.append('Q(' + name + ')') output.append('Q(' + name + ')')
# make sure there is a newline at the end of the output write_out(last_fname, output)
output.append('') return ""
return '\n'.join(output)
def cat_together():
import glob
import hashlib
hasher = hashlib.md5()
all_lines = []
outf = open(args.output_dir + "/out", "wb")
for fname in glob.glob(args.output_dir + "/*.qstr"):
with open(fname, "rb") as f:
lines = f.readlines()
all_lines += lines
all_lines.sort()
all_lines = b"\n".join(all_lines)
outf.write(all_lines)
outf.close()
hasher.update(all_lines)
new_hash = hasher.hexdigest()
#print(new_hash)
old_hash = None
try:
with open(args.output_file + ".hash") as f:
old_hash = f.read()
except IOError:
pass
if old_hash != new_hash:
print("QSTR updated")
os.rename(args.output_dir + "/out", args.output_file)
with open(args.output_file + ".hash", "w") as f:
f.write(new_hash)
else:
print("QSTR not updated")
if __name__ == "__main__": if __name__ == "__main__":
parser = argparse.ArgumentParser(description='Generates qstr definitions from a specified source') parser = argparse.ArgumentParser(description='Generates qstr definitions from a specified source')
parser.add_argument('-o', '--output-file', dest='output_filename', parser.add_argument('input_filename',
help='Output filename (defaults to stdout)') help='Name of the input file (when not specified, the script reads standard input)')
parser.add_argument('input_filename', nargs='?', parser.add_argument('output_dir',
help='Name of the input file (when not specified, the script reads standard input') help='Output directory to store individual qstr files')
parser.add_argument('-s', '--skip-write-when-same', dest='skip_write_when_same', parser.add_argument('output_file',
action='store_true', default=False, help='Name of the output file with collected qstrs')
help="Don't write the output file if it already exists and the contents have not changed (disabled by default)")
args = parser.parse_args() args = parser.parse_args()
try:
# Check if the file contents changed from last time os.makedirs(args.output_dir)
write_file = True except OSError:
pass
# By default write into STDOUT
outfile = sys.stdout
real_output_filename = 'STDOUT'
if args.input_filename: if args.input_filename:
infile = open(args.input_filename, 'r') infile = open(args.input_filename, 'r')
@ -61,24 +107,4 @@ if __name__ == "__main__":
file_data = process_file(infile) file_data = process_file(infile)
infile.close() infile.close()
cat_together()
# Detect custom output file name
if args.output_filename:
real_output_filename = args.output_filename
if os.path.isfile(args.output_filename) and args.skip_write_when_same:
with open(args.output_filename, 'r') as f:
existing_data = f.read()
if existing_data == file_data:
debug("Skip regeneration of: %s\n" % real_output_filename)
write_file = False
else:
debug("File HAS changed, overwriting\n")
outfile = open(args.output_filename, 'w')
else:
outfile = open(args.output_filename, 'w')
# Only write the file if we the data has changed
if write_file:
sys.stderr.write("QSTR %s\n" % real_output_filename)
outfile.write(file_data)
outfile.close()

View File

@ -52,7 +52,7 @@ EMPTY_QSTRDEFS_GENERATED_H = $(BUILD)/tmp/genhdr/qstrdefs.generated.h
# List all native flags since the current build system doesn't have # List all native flags since the current build system doesn't have
# the micropython configuration available. However, these flags are # the micropython configuration available. However, these flags are
# needed to extract all qstrings # needed to extract all qstrings
QSTR_GEN_EXTRA_CFLAGS += -P -DN_X64 -DN_X86 -DN_THUMB -DN_ARM QSTR_GEN_EXTRA_CFLAGS += -D__QSTR_EXTRACT -DN_X64 -DN_X86 -DN_THUMB -DN_ARM
QSTR_GEN_EXTRA_CFLAGS += -I$(BUILD)/tmp QSTR_GEN_EXTRA_CFLAGS += -I$(BUILD)/tmp
vpath %.c . $(TOP) vpath %.c . $(TOP)
@ -81,13 +81,13 @@ $(EMPTY_QSTRDEFS_GENERATED_H):
# to get built before we try to compile any of them. # to get built before we try to compile any of them.
$(OBJ): | $(HEADER_BUILD)/qstrdefs.generated.h $(HEADER_BUILD)/mpversion.h $(OBJ): | $(HEADER_BUILD)/qstrdefs.generated.h $(HEADER_BUILD)/mpversion.h
# This rule joins all generated qstr files $(HEADER_BUILD)/qstr.i.last: $(SRC_QSTR) | $(HEADER_BUILD)/mpversion.h
$(QSTR_DEFS_COLLECTED): $(addprefix $(HEADER_BUILD)/,$(addsuffix .qstr,$(SRC_QSTR)))
$(ECHO) "GEN $@" $(ECHO) "GEN $@"
$(Q)cat $^ > $@ $(Q)$(CPP) $(QSTR_GEN_EXTRA_CFLAGS) $(CFLAGS) $? >$(HEADER_BUILD)/qstr.i.last
$(QSTR_DEFS_COLLECTED): $(HEADER_BUILD)/qstr.i.last
# $(ECHO) "GEN $@"
$(Q)$(PYTHON) $(PY_SRC)/makeqstrdefs.py $(HEADER_BUILD)/qstr.i.last $(HEADER_BUILD)/qstr $(QSTR_DEFS_COLLECTED)
# $(sort $(var)) removes duplicates # $(sort $(var)) removes duplicates
# #

View File

@ -93,9 +93,11 @@ const qstr_pool_t mp_qstr_const_pool = {
10, // set so that the first dynamically allocated pool is twice this size; must be <= the len (just below) 10, // set so that the first dynamically allocated pool is twice this size; must be <= the len (just below)
MP_QSTRnumber_of, // corresponds to number of strings in array just below MP_QSTRnumber_of, // corresponds to number of strings in array just below
{ {
#ifndef __QSTR_EXTRACT
#define QDEF(id, str) str, #define QDEF(id, str) str,
#include "genhdr/qstrdefs.generated.h" #include "genhdr/qstrdefs.generated.h"
#undef QDEF #undef QDEF
#endif
}, },
}; };

View File

@ -37,9 +37,11 @@
// first entry in enum will be MP_QSTR_NULL=0, which indicates invalid/no qstr // first entry in enum will be MP_QSTR_NULL=0, which indicates invalid/no qstr
enum { enum {
#ifndef __QSTR_EXTRACT
#define QDEF(id, str) id, #define QDEF(id, str) id,
#include "genhdr/qstrdefs.generated.h" #include "genhdr/qstrdefs.generated.h"
#undef QDEF #undef QDEF
#endif
MP_QSTRnumber_of, // no underscore so it can't clash with any of the above MP_QSTRnumber_of, // no underscore so it can't clash with any of the above
}; };