From 08ed09acc6c73bae5475735e6e39b10ccd951d36 Mon Sep 17 00:00:00 2001 From: Jeff Epler Date: Sun, 16 Aug 2020 20:38:05 -0500 Subject: [PATCH 1/4] makeqstrdata: don't print "compression incrased length" messages This check as implemented is misleading, because it compares the compressed size in bytes (including the length indication) with the source string length in Unicode code points. For English this is approximately fair, but for Japanese this is quite unfair and produces an excess of "increased length" messages. This message might have existed for one of two reasons: * to alert to an improperly function huffman compression * to call attention to a need for a "string is stored uncompressed" case We know by now that the huffman compression is functioning as designed and effective in general. Just to be on the safe side, I did some back-of-the-envelope estimates. I considered these three replacements for "the true source string size, in bytes": + decompressed_len_utf8 = len(decompressed.encode('utf-8')) + decompressed_len_utf16 = len(decompressed.encode('utf-16be')) + decompressed_len_bitsize = ((1+len(decompressed)) * math.ceil(math.log(1+len(values), 2)) + 7) // 8 The third counts how many bits each character requires (fewer than 128 characters in the source character set = 7, fewer than 256 = 8, fewer than 512 = 9, etc, adding a string-terminating value) and is in some way representative of the best way we would be able to store "uncompressed strings". The Japanese translation (largest as of writing) has just a few strings which increase by this metric. However, the amount of loss due to expansion in those cases is outweighed by the cost of adding 1 bit per string to indicate whether it's compressed or not. For instance, in the BOARD=trinket_m0 TRANSLATION=ja build the loss is 47 bytes over 300 strings. Adding 1 bit to each of 300 strings will cost about 37 bytes, leaving just 5 Thumb instructions to implement the code to check and decode "uncompressed" strings in order to break even. --- py/makeqstrdata.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/py/makeqstrdata.py b/py/makeqstrdata.py index df2c687e5c..6132298a51 100644 --- a/py/makeqstrdata.py +++ b/py/makeqstrdata.py @@ -259,8 +259,6 @@ def compress(encoding_table, decompressed, encoded_length_bits, len_translation_ current_bit -= 1 if current_bit != 7: current_byte += 1 - if current_byte > len(decompressed): - print("Note: compression increased length", repr(decompressed), len(decompressed), current_byte, file=sys.stderr) return enc[:current_byte] def qstr_escape(qst): From 880fe1a6942a6e37a0e11b86b5773682cfa280fb Mon Sep 17 00:00:00 2001 From: Jeff Epler Date: Sun, 16 Aug 2020 20:58:24 -0500 Subject: [PATCH 2/4] gen_display_resources: silence 'missing character' message Since we made the decision to allow translations which do not have coverage in the terminal font, these routinely occur and are expected. The message is unhelpful and due to their voume make it harder to find relevant information particularly in github actions results. --- tools/gen_display_resources.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/gen_display_resources.py b/tools/gen_display_resources.py index 478a2f22f9..e4197aabf4 100644 --- a/tools/gen_display_resources.py +++ b/tools/gen_display_resources.py @@ -61,7 +61,6 @@ f.load_glyphs(set(ord(c) for c in all_characters)) # Get each glyph. for c in set(all_characters): if ord(c) not in f._glyphs: - print("Font missing character:", c, ord(c)) filtered_characters = filtered_characters.replace(c, "") continue g = f.get_glyph(ord(c)) From ead2554c78d698bf5e70a3afd33d1154885e60d5 Mon Sep 17 00:00:00 2001 From: Jeff Epler Date: Sun, 16 Aug 2020 21:17:41 -0500 Subject: [PATCH 3/4] mpy-cross: Silently clone when building for static-raspbian Also move the construct earlier. This prevents the message ``` make: pitools/arm-bcm2708/gcc-linaro-arm-linux-gnueabihf-raspbian-x64/bin/arm-linux-gnueabihf-gcc: Command not found ``` from occurring during the first build, since make was evaluating the line ``` mpy-cross.mk:COMPILER_TARGET := $(shell $(CC) -dumpmachine) ``` before $(CC) was created by the clone. Nothing bad seems to have come of this, but it's nice to fix it anyhow. (And interesting that it was lost among the spewed messages we're removing now) --- mpy-cross/Makefile.static-raspbian | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mpy-cross/Makefile.static-raspbian b/mpy-cross/Makefile.static-raspbian index f895f998b5..6329ebdb56 100644 --- a/mpy-cross/Makefile.static-raspbian +++ b/mpy-cross/Makefile.static-raspbian @@ -6,7 +6,7 @@ PROG=mpy-cross.static-raspbian BUILD=build-static-raspbian STATIC_BUILD=1 +$(shell if ! [ -d pitools ]; then echo 1>&2 "Fetching pi build tools. This may take awhile."; git clone -q https://github.com/raspberrypi/tools.git --depth=1 pitools; fi) CROSS_COMPILE = pitools/arm-bcm2708/gcc-linaro-arm-linux-gnueabihf-raspbian-x64/bin/arm-linux-gnueabihf- include mpy-cross.mk -$(shell [ -d pitools ] || git clone --progress --verbose https://github.com/raspberrypi/tools.git --depth=1 pitools) From b1d210cca728066a7ab0188ab76428e96b4e0ced Mon Sep 17 00:00:00 2001 From: Jeff Epler Date: Sun, 16 Aug 2020 21:33:36 -0500 Subject: [PATCH 4/4] remove trailing whitespace --- mpy-cross/Makefile.static-raspbian | 1 - 1 file changed, 1 deletion(-) diff --git a/mpy-cross/Makefile.static-raspbian b/mpy-cross/Makefile.static-raspbian index 6329ebdb56..8fe78b0aff 100644 --- a/mpy-cross/Makefile.static-raspbian +++ b/mpy-cross/Makefile.static-raspbian @@ -9,4 +9,3 @@ STATIC_BUILD=1 $(shell if ! [ -d pitools ]; then echo 1>&2 "Fetching pi build tools. This may take awhile."; git clone -q https://github.com/raspberrypi/tools.git --depth=1 pitools; fi) CROSS_COMPILE = pitools/arm-bcm2708/gcc-linaro-arm-linux-gnueabihf-raspbian-x64/bin/arm-linux-gnueabihf- include mpy-cross.mk -