Add CIRCUITPY_MESSAGE_COMPRESSION_LEVEL
to trade compile speed & flash size Initially enable the faster mode on rp2040 and espressif, where there's usually plenty of flash available (these advanced techniques save hundreds to thousands of bytes, which is important on a lot of old samd21 boards but is a drop in the lake of a 4MB flash chip)
This commit is contained in:
Jeff Epler
parent
770f22e5a7
commit
7ab5252cdd
@ -63,6 +63,8 @@ endif
|
||||
OBJ = $(PY_CORE_O)
|
||||
OBJ += $(addprefix $(BUILD)/, $(SRC_C:.c=.o))
|
||||
|
||||
# CIRCUITPY
|
||||
$(BUILD)/supervisor/shared/translate/translate.o: $(HEADER_BUILD)/qstrdefs.generated.h $(HEADER_BUILD)/compressed_translations.generated.h
|
||||
CIRCUITPY_MESSAGE_COMPRESSION_LEVEL = 1
|
||||
|
||||
include $(TOP)/py/mkrules.mk
|
||||
|
||||
@ -148,3 +148,6 @@ endif
|
||||
# only if something else is turned off, such as HID.
|
||||
USB_NUM_ENDPOINT_PAIRS = 7
|
||||
USB_NUM_IN_ENDPOINTS = 5
|
||||
|
||||
# Usually lots of flash space available
|
||||
CIRCUITPY_MESSAGE_COMPRESSION_LEVEL ?= 1
|
||||
|
||||
@ -52,3 +52,6 @@ USB_NUM_ENDPOINT_PAIRS = 8
|
||||
|
||||
INTERNAL_FLASH_FILESYSTEM = 1
|
||||
CIRCUITPY_SETTABLE_PROCESSOR_FREQUENCY = 1
|
||||
|
||||
# Usually lots of flash space available
|
||||
CIRCUITPY_MESSAGE_COMPRESSION_LEVEL ?= 1
|
||||
|
||||
@ -50,5 +50,6 @@ MICROPY_VFS_LFS2 = 0
|
||||
|
||||
# CIRCUITPY
|
||||
CIRCUITPY_ULAB = 1
|
||||
CIRCUITPY_MESSAGE_COMPRESSION_LEVEL = 1
|
||||
MICROPY_EMIT_NATIVE = 0
|
||||
CFLAGS += -DCIRCUITPY=1
|
||||
|
||||
@ -92,3 +92,4 @@ CFLAGS += \
|
||||
|
||||
SRC_C += coverage.c
|
||||
SRC_CXX += coveragecpp.cpp
|
||||
CIRCUITPY_MESSAGE_COMPRESSION_LEVEL = 1
|
||||
|
||||
@ -52,6 +52,10 @@ CFLAGS += -DCIRCUITPY=$(CIRCUITPY)
|
||||
CIRCUITPY_FULL_BUILD ?= 1
|
||||
CFLAGS += -DCIRCUITPY_FULL_BUILD=$(CIRCUITPY_FULL_BUILD)
|
||||
|
||||
# By default, aggressively reduce the size of in-flash messages, at the cost of
|
||||
# increased build time
|
||||
CIRCUITPY_MESSAGE_COMPRESSION_LEVEL ?= 9
|
||||
|
||||
# Reduce the size of in-flash properties. Requires support in the .ld linker
|
||||
# file, so not enabled by default.
|
||||
CIRCUITPY_OPTIMIZE_PROPERTY_FLASH_SIZE ?= 0
|
||||
|
||||
@ -174,7 +174,7 @@ class EncodingTable:
|
||||
qstrs_inv: object
|
||||
|
||||
|
||||
def compute_huffman_coding(qstrs, translation_name, translations, f):
|
||||
def compute_huffman_coding(qstrs, translation_name, translations, f, compression_level):
|
||||
# possible future improvement: some languages are better when consider len(k) > 2. try both?
|
||||
qstrs = dict((k, v) for k, v in qstrs.items() if len(k) > 3)
|
||||
qstr_strs = list(qstrs.keys())
|
||||
@ -209,6 +209,8 @@ def compute_huffman_coding(qstrs, translation_name, translations, f):
|
||||
if 0x80 <= ord_c < 0xFF:
|
||||
end_unused = min(ord_c, end_unused)
|
||||
max_words = end_unused - 0x80
|
||||
if compression_level < 5:
|
||||
max_words = 0
|
||||
|
||||
bits_per_codepoint = 16 if max_ord > 255 else 8
|
||||
values_type = "uint16_t" if max_ord > 255 else "uint8_t"
|
||||
@ -298,8 +300,12 @@ def compute_huffman_coding(qstrs, translation_name, translations, f):
|
||||
word = scores[0][0]
|
||||
words.append(word)
|
||||
|
||||
splitters = words[:]
|
||||
if compression_level > 3:
|
||||
splitters.extend(qstr_strs)
|
||||
|
||||
words.sort(key=len)
|
||||
extractor = TextSplitter(words + qstr_strs)
|
||||
extractor = TextSplitter(splitters)
|
||||
counter = collections.Counter()
|
||||
used_qstr = 0
|
||||
for t in texts:
|
||||
@ -356,8 +362,8 @@ def compute_huffman_coding(qstrs, translation_name, translations, f):
|
||||
len(translation.encode("utf-8")) for (original, translation) in translations
|
||||
)
|
||||
|
||||
maxlen = len(words[-1])
|
||||
minlen = len(words[0])
|
||||
maxlen = len(words[-1]) if words else 0
|
||||
minlen = len(words[0]) if words else 0
|
||||
wlencount = [len([None for w in words if len(w) == l]) for l in range(minlen, maxlen + 1)]
|
||||
|
||||
translation_qstr_bits = used_qstr.bit_length()
|
||||
@ -596,6 +602,12 @@ if __name__ == "__main__":
|
||||
parser.add_argument(
|
||||
"--translation", default=None, type=str, help="translations for i18n() items"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--compression_level",
|
||||
type=int,
|
||||
default=9,
|
||||
help="degree of compression (>5: construct dictionary; >3: use qstrs)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--compression_filename",
|
||||
type=argparse.FileType("w", encoding="UTF-8"),
|
||||
@ -619,6 +631,6 @@ if __name__ == "__main__":
|
||||
i18ns = sorted(i18ns)
|
||||
translations = translate(args.translation, i18ns)
|
||||
encoding_table = compute_huffman_coding(
|
||||
qstrs, args.translation, translations, args.compression_filename
|
||||
qstrs, args.translation, translations, args.compression_filename, args.compression_level
|
||||
)
|
||||
output_translation_data(encoding_table, translations, args.translation_filename)
|
||||
|
||||
2
py/py.mk
2
py/py.mk
@ -269,7 +269,7 @@ $(PY_BUILD)/translations-$(TRANSLATION).c: $(HEADER_BUILD)/compressed_translatio
|
||||
$(HEADER_BUILD)/compressed_translations.generated.h: $(PY_SRC)/maketranslationdata.py $(HEADER_BUILD)/$(TRANSLATION).mo $(HEADER_BUILD)/qstrdefs.generated.h
|
||||
$(STEPECHO) "GEN $@"
|
||||
$(Q)mkdir -p $(PY_BUILD)
|
||||
$(Q)$(PYTHON) $(PY_SRC)/maketranslationdata.py --compression_filename $(HEADER_BUILD)/compressed_translations.generated.h --translation $(HEADER_BUILD)/$(TRANSLATION).mo --translation_filename $(PY_BUILD)/translations-$(TRANSLATION).c --qstrdefs_filename $(HEADER_BUILD)/qstrdefs.generated.h $(HEADER_BUILD)/qstrdefs.preprocessed.h
|
||||
$(Q)$(PYTHON) $(PY_SRC)/maketranslationdata.py --compression_filename $(HEADER_BUILD)/compressed_translations.generated.h --translation $(HEADER_BUILD)/$(TRANSLATION).mo --translation_filename $(PY_BUILD)/translations-$(TRANSLATION).c --qstrdefs_filename $(HEADER_BUILD)/qstrdefs.generated.h --compression_level $(CIRCUITPY_MESSAGE_COMPRESSION_LEVEL) $(HEADER_BUILD)/qstrdefs.preprocessed.h
|
||||
|
||||
PY_CORE_O += $(PY_BUILD)/translations-$(TRANSLATION).o
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user