From f853e3e106b151ec2819df729fd68815dce693fb Mon Sep 17 00:00:00 2001 From: Jim Mussared Date: Wed, 8 Dec 2021 16:47:33 +1100 Subject: [PATCH] tools/makemanifest.py: Merge make-frozen.py. Takes the functionality from tools/make-frozen.py, adds support for multiple frozen directories, and moves it to tools/makemanifest.py. Signed-off-by: Jim Mussared --- tools/make-frozen.py | 85 ------------------------------------------- tools/makemanifest.py | 74 +++++++++++++++++++++++++++++++++---- 2 files changed, 66 insertions(+), 93 deletions(-) delete mode 100755 tools/make-frozen.py diff --git a/tools/make-frozen.py b/tools/make-frozen.py deleted file mode 100755 index bc35d38342..0000000000 --- a/tools/make-frozen.py +++ /dev/null @@ -1,85 +0,0 @@ -#!/usr/bin/env python -# -# Create frozen modules structure for MicroPython. -# -# Usage: -# -# Have a directory with modules to be frozen (only modules, not packages -# supported so far): -# -# frozen/foo.py -# frozen/bar.py -# -# Run script, passing path to the directory above: -# -# ./make-frozen.py frozen > frozen.c -# -# Include frozen.c in your build, having defined MICROPY_MODULE_FROZEN_STR in -# config. -# -from __future__ import print_function -import sys -import os - - -def module_name(f): - return f - - -modules = [] - -if len(sys.argv) > 1: - root = sys.argv[1].rstrip("/") - root_len = len(root) - - for dirpath, dirnames, filenames in os.walk(root): - for f in filenames: - fullpath = dirpath + "/" + f - st = os.stat(fullpath) - modules.append((fullpath[root_len + 1 :], st)) - -print("#include ") -print("const char mp_frozen_str_names[] = {") -for f, st in modules: - m = module_name(f) - print('"%s\\0"' % m) -print('"\\0"};') - -print("const uint32_t mp_frozen_str_sizes[] = {") - -for f, st in modules: - print("%d," % st.st_size) - -print("0};") - -print("const char mp_frozen_str_content[] = {") -for f, st in modules: - data = open(sys.argv[1] + "/" + f, "rb").read() - - # We need to properly escape the script data to create a C string. - # When C parses hex characters of the form \x00 it keeps parsing the hex - # data until it encounters a non-hex character. Thus one must create - # strings of the form "data\x01" "abc" to properly encode this kind of - # data. We could just encode all characters as hex digits but it's nice - # to be able to read the resulting C code as ASCII when possible. - - data = bytearray(data) # so Python2 extracts each byte as an integer - esc_dict = {ord("\n"): "\\n", ord("\r"): "\\r", ord('"'): '\\"', ord("\\"): "\\\\"} - chrs = ['"'] - break_str = False - for c in data: - try: - chrs.append(esc_dict[c]) - except KeyError: - if 32 <= c <= 126: - if break_str: - chrs.append('" "') - break_str = False - chrs.append(chr(c)) - else: - chrs.append("\\x%02x" % c) - break_str = True - chrs.append('\\0"') - print("".join(chrs)) - -print('"\\0"};') diff --git a/tools/makemanifest.py b/tools/makemanifest.py index fc608ed449..e37ae74cfd 100644 --- a/tools/makemanifest.py +++ b/tools/makemanifest.py @@ -201,8 +201,6 @@ def freeze_internal(kind, path, script, opt): if not os.path.isdir(path): raise FreezeError("freeze path must be a directory: {}".format(path)) if script is None and kind == KIND_AS_STR: - if any(f[0] == KIND_AS_STR for f in manifest_list): - raise FreezeError("can only freeze one str directory") manifest_list.append((KIND_AS_STR, path, script, opt)) elif script is None or isinstance(script, str) and script.find(".") == -1: # Recursively search `path` for files to freeze, optionally restricted @@ -235,6 +233,70 @@ def freeze_internal(kind, path, script, opt): manifest_list.append((kind, path, script, opt)) +def generate_frozen_str_content(paths): + def module_name(f): + return f + + modules = [] + output = [] + + for path in paths: + root = path.rstrip("/") + root_len = len(root) + + for dirpath, dirnames, filenames in os.walk(root): + for f in filenames: + fullpath = dirpath + "/" + f + st = os.stat(fullpath) + modules.append((path, fullpath[root_len + 1 :], st)) + + output.append("#include \n") + output.append("const char mp_frozen_str_names[] = {\n") + for _path, f, st in modules: + m = module_name(f) + output.append('"%s\\0"\n' % m) + output.append('"\\0"};\n') + + output.append("const uint32_t mp_frozen_str_sizes[] = {\n") + + for _path, f, st in modules: + output.append("%d," % st.st_size) + + output.append("0};\n") + + output.append("const char mp_frozen_str_content[] = {\n") + for path, f, st in modules: + data = open(path + "/" + f, "rb").read() + + # We need to properly escape the script data to create a C string. + # When C parses hex characters of the form \x00 it keeps parsing the hex + # data until it encounters a non-hex character. Thus one must create + # strings of the form "data\x01" "abc" to properly encode this kind of + # data. We could just encode all characters as hex digits but it's nice + # to be able to read the resulting C code as ASCII when possible. + + data = bytearray(data) # so Python2 extracts each byte as an integer + esc_dict = {ord("\n"): "\\n", ord("\r"): "\\r", ord('"'): '\\"', ord("\\"): "\\\\"} + output.append('"') + break_str = False + for c in data: + try: + output.append(esc_dict[c]) + except KeyError: + if 32 <= c <= 126: + if break_str: + output.append('" "') + break_str = False + output.append(chr(c)) + else: + output.append("\\x%02x" % c) + break_str = True + output.append('\\0"\n') + + output.append('"\\0"};\n') + return "".join(output) + + def main(): # Parse arguments import argparse @@ -264,7 +326,6 @@ def main(): sys.exit(1) # Get paths to tools - MAKE_FROZEN = VARS["MPY_DIR"] + "/tools/make-frozen.py" MPY_CROSS = VARS["MPY_DIR"] + "/mpy-cross/mpy-cross" if sys.platform == "win32": MPY_CROSS += ".exe" @@ -327,10 +388,7 @@ def main(): return # Freeze paths as strings - res, output_str = system([sys.executable, MAKE_FROZEN] + str_paths) - if res != 0: - print("error freezing strings {}: {}".format(str_paths, output_str)) - sys.exit(1) + output_str = generate_frozen_str_content(str_paths) # Freeze .mpy files if mpy_files: @@ -365,7 +423,7 @@ def main(): mkdir(args.output) with open(args.output, "wb") as f: f.write(b"//\n// Content for MICROPY_MODULE_FROZEN_STR\n//\n") - f.write(output_str) + f.write(output_str.encode()) f.write(b"//\n// Content for MICROPY_MODULE_FROZEN_MPY\n//\n") f.write(output_mpy)