2016-10-25 17:27:59 -04:00
|
|
|
/*
|
2017-08-27 15:02:50 -04:00
|
|
|
* This file is part of the MicroPython project, http://micropython.org/
|
2016-10-25 17:27:59 -04:00
|
|
|
*
|
|
|
|
* The MIT License (MIT)
|
|
|
|
*
|
2018-07-31 19:53:54 -04:00
|
|
|
* Copyright (c) 2018 Scott Shawcroft for Adafruit Industries
|
2016-10-25 17:27:59 -04:00
|
|
|
*
|
|
|
|
* Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
|
|
* of this software and associated documentation files (the "Software"), to deal
|
|
|
|
* in the Software without restriction, including without limitation the rights
|
|
|
|
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
|
|
* copies of the Software, and to permit persons to whom the Software is
|
|
|
|
* furnished to do so, subject to the following conditions:
|
|
|
|
*
|
|
|
|
* The above copyright notice and this permission notice shall be included in
|
|
|
|
* all copies or substantial portions of the Software.
|
|
|
|
*
|
|
|
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
|
|
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
|
|
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
|
|
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
|
|
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
|
|
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
|
|
* THE SOFTWARE.
|
|
|
|
*/
|
|
|
|
|
2018-07-31 19:53:54 -04:00
|
|
|
#include "supervisor/shared/translate.h"
|
2016-10-25 17:27:59 -04:00
|
|
|
|
2018-08-15 21:32:37 -04:00
|
|
|
#include <stdbool.h>
|
|
|
|
#include <stdint.h>
|
2018-07-31 19:53:54 -04:00
|
|
|
#include <string.h>
|
2016-10-28 23:16:39 -04:00
|
|
|
|
2018-08-15 21:32:37 -04:00
|
|
|
#ifndef NO_QSTR
|
|
|
|
#include "genhdr/compression.generated.h"
|
|
|
|
#endif
|
|
|
|
|
2018-12-06 17:24:20 -05:00
|
|
|
#include "supervisor/serial.h"
|
|
|
|
|
|
|
|
void serial_write_compressed(const compressed_string_t* compressed) {
|
|
|
|
char decompressed[compressed->length];
|
|
|
|
decompress(compressed, decompressed);
|
|
|
|
serial_write(decompressed);
|
|
|
|
}
|
|
|
|
|
translation: Compress as unicode, not bytes
By treating each unicode code-point as a single entity for huffman
compression, the overall compression rate can be somewhat improved
without changing the algorithm. On the decompression side, when
compressed values above 127 are encountered, they need to be
converted from a 16-bit Unicode code point into a UTF-8 byte
sequence.
Doing this returns approximately 1.5kB of flash storage with the
zh_Latn_pinyin translation. (292 -> 1768 bytes remaining in my build
of trinket_m0)
Other "more ASCII" translations benefit less, and in fact
zh_Latn_pinyin is no longer the most constrained translation!
(de_DE 1156 -> 1384 bytes free in flash, I didn't check others
before pushing for CI)
English is slightly pessimized, 2840 -> 2788 bytes, probably mostly
because the "values" array was changed from uint8_t to uint16_t,
which is strictly not required for an all-ASCII translation. This
could probably be avoided in this case, but as English is not the
most constrained translation it doesn't really matter.
Testing performed: built for feather nRF52840 express and trinket m0
in English and zh_Latn_pinyin; ran and verified the localized
messages such as
Àn xià rènhé jiàn jìnrù REPL. Shǐyòng CTRL-D chóngxīn jiāzài.
and
Press any key to enter the REPL. Use CTRL-D to reload.
were properly displayed.
2019-12-02 09:41:03 -05:00
|
|
|
STATIC int put_utf8(char *buf, int u) {
|
|
|
|
if(u <= 0x7f) {
|
|
|
|
*buf = u;
|
|
|
|
return 1;
|
|
|
|
} else if(u <= 0x07ff) {
|
|
|
|
*buf++ = 0b11000000 | (u >> 6);
|
|
|
|
*buf = 0b10000000 | (u & 0b00111111);
|
|
|
|
return 2;
|
|
|
|
} else { // u <= 0xffff)
|
|
|
|
*buf++ = 0b11000000 | (u >> 12);
|
|
|
|
*buf = 0b10000000 | ((u >> 6) & 0b00111111);
|
|
|
|
*buf = 0b10000000 | (u & 0b00111111);
|
|
|
|
return 3;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-08-15 21:32:37 -04:00
|
|
|
char* decompress(const compressed_string_t* compressed, char* decompressed) {
|
|
|
|
uint8_t this_byte = 0;
|
|
|
|
uint8_t this_bit = 7;
|
|
|
|
uint8_t b = compressed->data[this_byte];
|
|
|
|
// Stop one early because the last byte is always NULL.
|
translation: Compress as unicode, not bytes
By treating each unicode code-point as a single entity for huffman
compression, the overall compression rate can be somewhat improved
without changing the algorithm. On the decompression side, when
compressed values above 127 are encountered, they need to be
converted from a 16-bit Unicode code point into a UTF-8 byte
sequence.
Doing this returns approximately 1.5kB of flash storage with the
zh_Latn_pinyin translation. (292 -> 1768 bytes remaining in my build
of trinket_m0)
Other "more ASCII" translations benefit less, and in fact
zh_Latn_pinyin is no longer the most constrained translation!
(de_DE 1156 -> 1384 bytes free in flash, I didn't check others
before pushing for CI)
English is slightly pessimized, 2840 -> 2788 bytes, probably mostly
because the "values" array was changed from uint8_t to uint16_t,
which is strictly not required for an all-ASCII translation. This
could probably be avoided in this case, but as English is not the
most constrained translation it doesn't really matter.
Testing performed: built for feather nRF52840 express and trinket m0
in English and zh_Latn_pinyin; ran and verified the localized
messages such as
Àn xià rènhé jiàn jìnrù REPL. Shǐyòng CTRL-D chóngxīn jiāzài.
and
Press any key to enter the REPL. Use CTRL-D to reload.
were properly displayed.
2019-12-02 09:41:03 -05:00
|
|
|
for (uint16_t i = 0; i < compressed->length - 1;) {
|
2018-08-15 21:32:37 -04:00
|
|
|
uint32_t bits = 0;
|
|
|
|
uint8_t bit_length = 0;
|
|
|
|
uint32_t max_code = lengths[0];
|
|
|
|
uint32_t searched_length = lengths[0];
|
|
|
|
while (true) {
|
|
|
|
bits <<= 1;
|
|
|
|
if ((0x80 & b) != 0) {
|
|
|
|
bits |= 1;
|
|
|
|
}
|
|
|
|
b <<= 1;
|
|
|
|
bit_length += 1;
|
|
|
|
if (this_bit == 0) {
|
|
|
|
this_bit = 7;
|
|
|
|
this_byte += 1;
|
|
|
|
b = compressed->data[this_byte]; // This may read past the end but its never used.
|
|
|
|
} else {
|
|
|
|
this_bit -= 1;
|
|
|
|
}
|
|
|
|
if (max_code > 0 && bits < max_code) {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
max_code = (max_code << 1) + lengths[bit_length];
|
|
|
|
searched_length += lengths[bit_length];
|
|
|
|
}
|
translation: Compress as unicode, not bytes
By treating each unicode code-point as a single entity for huffman
compression, the overall compression rate can be somewhat improved
without changing the algorithm. On the decompression side, when
compressed values above 127 are encountered, they need to be
converted from a 16-bit Unicode code point into a UTF-8 byte
sequence.
Doing this returns approximately 1.5kB of flash storage with the
zh_Latn_pinyin translation. (292 -> 1768 bytes remaining in my build
of trinket_m0)
Other "more ASCII" translations benefit less, and in fact
zh_Latn_pinyin is no longer the most constrained translation!
(de_DE 1156 -> 1384 bytes free in flash, I didn't check others
before pushing for CI)
English is slightly pessimized, 2840 -> 2788 bytes, probably mostly
because the "values" array was changed from uint8_t to uint16_t,
which is strictly not required for an all-ASCII translation. This
could probably be avoided in this case, but as English is not the
most constrained translation it doesn't really matter.
Testing performed: built for feather nRF52840 express and trinket m0
in English and zh_Latn_pinyin; ran and verified the localized
messages such as
Àn xià rènhé jiàn jìnrù REPL. Shǐyòng CTRL-D chóngxīn jiāzài.
and
Press any key to enter the REPL. Use CTRL-D to reload.
were properly displayed.
2019-12-02 09:41:03 -05:00
|
|
|
i += put_utf8(decompressed + i, values[searched_length + bits - max_code]);
|
2018-08-15 21:32:37 -04:00
|
|
|
}
|
|
|
|
|
|
|
|
decompressed[compressed->length-1] = '\0';
|
|
|
|
return decompressed;
|
|
|
|
}
|
|
|
|
|
|
|
|
inline __attribute__((always_inline)) const compressed_string_t* translate(const char* original) {
|
2018-07-31 19:53:54 -04:00
|
|
|
#ifndef NO_QSTR
|
|
|
|
#define QDEF(id, str)
|
2018-08-16 16:34:12 -04:00
|
|
|
#define TRANSLATION(id, len, compressed...) if (strcmp(original, id) == 0) { static const compressed_string_t v = {.length = len, .data = compressed}; return &v; } else
|
2018-07-31 19:53:54 -04:00
|
|
|
#include "genhdr/qstrdefs.generated.h"
|
|
|
|
#undef TRANSLATION
|
|
|
|
#undef QDEF
|
|
|
|
#endif
|
2018-08-15 21:32:37 -04:00
|
|
|
return NULL;
|
2018-07-31 19:53:54 -04:00
|
|
|
}
|