diff --git a/py/makeqstrdata.py b/py/makeqstrdata.py index 04c8938766..28aed3df97 100644 --- a/py/makeqstrdata.py +++ b/py/makeqstrdata.py @@ -100,9 +100,30 @@ def translate(translation_file, i18ns): translations.append((original, translation)) return translations +def frequent_ngrams(corpus, sz, n): + return collections.Counter(corpus[i:i+sz] for i in range(len(corpus)-sz)).most_common(n) + +def ngrams_to_pua(translation, ngrams): + if len(ngrams) > 32: + start = 0xe000 + else: + start = 0x80 + for i, g in enumerate(ngrams): + translation = translation.replace(g, chr(start + i)) + return translation + +def pua_to_ngrams(compressed, ngrams): + if len(ngrams) > 32: + start, end = 0xe000, 0xf8ff + else: + start, end = 0x80, 0xbf + return "".join(ngrams[ord(c) - start] if (start <= ord(c) <= end) else c for c in compressed) + def compute_huffman_coding(translations, qstrs, compression_filename): all_strings = [x[1] for x in translations] all_strings_concat = "".join(all_strings) + ngrams = [i[0] for i in frequent_ngrams(all_strings_concat, 2, 32)] + all_strings_concat = ngrams_to_pua(all_strings_concat, ngrams) counts = collections.Counter(all_strings_concat) cb = huffman.codebook(counts.items()) values = [] @@ -128,7 +149,8 @@ def compute_huffman_coding(translations, qstrs, compression_filename): for i in range(1, max(length_count) + 2): lengths.append(length_count.get(i, 0)) print("// values", values, "lengths", len(lengths), lengths) - print("// estimated total memory size", len(lengths) + 2*len(values) + sum(len(cb[u]) for u in all_strings_concat)) + ngramdata = [ord(ni) for i in ngrams for ni in i] + print("// estimated total memory size", len(lengths) + 2*len(values) + 2 * len(ngramdata) + sum((len(cb[u]) + 7)//8 for u in all_strings_concat)) print("//", values, lengths) values_type = "uint16_t" if max(ord(u) for u in values) > 255 else "uint8_t" max_translation_encoded_length = max(len(translation.encode("utf-8")) for original,translation in translations) @@ -136,10 +158,11 @@ def compute_huffman_coding(translations, qstrs, compression_filename): f.write("const uint8_t lengths[] = {{ {} }};\n".format(", ".join(map(str, lengths)))) f.write("const {} values[] = {{ {} }};\n".format(values_type, ", ".join(str(ord(u)) for u in values))) f.write("#define compress_max_length_bits ({})\n".format(max_translation_encoded_length.bit_length())) - return values, lengths + f.write("const {} ngrams[] = {{ {} }};\n".format(values_type, ", ".join(str(u) for u in ngramdata))) + return values, lengths, ngrams def decompress(encoding_table, encoded, encoded_length_bits): - values, lengths = encoding_table + values, lengths, ngrams = encoding_table dec = [] this_byte = 0 this_bit = 7 @@ -187,6 +210,7 @@ def decompress(encoding_table, encoded, encoded_length_bits): searched_length += lengths[bit_length] v = values[searched_length + bits - max_code] + v = pua_to_ngrams(v, ngrams) i += len(v.encode('utf-8')) dec.append(v) return ''.join(dec) @@ -194,7 +218,8 @@ def decompress(encoding_table, encoded, encoded_length_bits): def compress(encoding_table, decompressed, encoded_length_bits, len_translation_encoded): if not isinstance(decompressed, str): raise TypeError() - values, lengths = encoding_table + values, lengths, ngrams = encoding_table + decompressed = ngrams_to_pua(decompressed, ngrams) enc = bytearray(len(decompressed) * 3) #print(decompressed) #print(lengths) diff --git a/supervisor/shared/translate.c b/supervisor/shared/translate.c index 606f8fa91a..49ee8f143f 100644 --- a/supervisor/shared/translate.c +++ b/supervisor/shared/translate.c @@ -34,6 +34,7 @@ #include "genhdr/compression.generated.h" #endif +#include "py/misc.h" #include "supervisor/serial.h" void serial_write_compressed(const compressed_string_t* compressed) { @@ -46,10 +47,18 @@ STATIC int put_utf8(char *buf, int u) { if(u <= 0x7f) { *buf = u; return 1; + } else if(MP_ARRAY_SIZE(ngrams) <= 64 && u <= 0xbf) { + int n = (u - 0x80) * 2; + int ret = put_utf8(buf, ngrams[n]); + return ret + put_utf8(buf + ret, ngrams[n+1]); } else if(u <= 0x07ff) { *buf++ = 0b11000000 | (u >> 6); *buf = 0b10000000 | (u & 0b00111111); return 2; + } else if(MP_ARRAY_SIZE(ngrams) > 64 && u >= 0xe000 && u <= 0xf8ff) { + int n = (u - 0xe000) * 2; + int ret = put_utf8(buf, ngrams[n]); + return ret + put_utf8(buf + ret, ngrams[n+1]); } else { // u <= 0xffff) *buf++ = 0b11000000 | (u >> 12); *buf = 0b10000000 | ((u >> 6) & 0b00111111);