use python based msgfmt

.github/actions/deps/external/action.yml

@@ -12,16 +12,6 @@ inputs:
     - riscv
     - none
 
-  apt:
-    required: false
-    default: true
-    type: boolean
-
-  python:
-    required: false
-    default: true
-    type: boolean
-
 runs:
   using: composite
   steps:
@@ -85,15 +75,10 @@ runs:
 
     # common
     - name: Cache python dependencies
-      if: inputs.python == 'true' && inputs.platform != 'espressif'
+      if: inputs.platform != 'espressif'
       uses: ./.github/actions/deps/python
       with:
         action: ${{ fromJSON('["restore", "cache"]')[github.job == 'scheduler'] }}
     - name: Install python dependencies
-      if: inputs.python == 'true'
       run: pip install -r requirements-dev.txt
       shell: bash
-    - name: Install dependencies
-      if: inputs.apt == 'true'
-      run: sudo apt-get install -y gettext
-      shell: bash

.github/actions/deps/python/action.yml

@@ -14,22 +14,29 @@ runs:
   using: composite
   steps:
     - name: Cache python dependencies
-      if: ${{ inputs.action == 'cache' }}
+      id: cache-python-deps
+      if: inputs.action == 'cache'
       uses: actions/cache@v3
       with:
         path: .cp_tools
         key: ${{ runner.os }}-${{ env.pythonLocation }}-tools-cp-${{ hashFiles('requirements-dev.txt') }}
 
     - name: Restore python dependencies
-      if: ${{ inputs.action == 'restore' }}
+      id: restore-python-deps
+      if: inputs.action == 'restore'
       uses: actions/cache/restore@v3
       with:
         path: .cp_tools
         key: ${{ runner.os }}-${{ env.pythonLocation }}-tools-cp-${{ hashFiles('requirements-dev.txt') }}
 
     - name: Set up venv
+      if: inputs.action == 'cache' && !steps.cache-python-deps.outputs.cache-hit
+      run: python -m venv .cp_tools
+      shell: bash
+
+    - name: Activate venv
+      if: inputs.action == 'cache' || (inputs.action == 'restore' && steps.restore-python-deps.outputs.cache-hit)
       run: |
-        python -m venv .cp_tools
         source .cp_tools/bin/activate
         echo >> $GITHUB_PATH "$PATH"
       shell: bash

.github/workflows/build-mpy-cross.yml

@@ -35,10 +35,6 @@ jobs:
       uses: ./.github/actions/deps/submodules
       with:
         target: mpy-cross
-    - name: Set up external
-      uses: ./.github/actions/deps/external
-      with:
-        python: false
 
     - name: Install toolchain (aarch64)
       if: matrix.mpy-cross == 'static-aarch64'

.github/workflows/build.yml

@@ -179,9 +179,7 @@ jobs:
       uses: ./.github/actions/deps/submodules
     - name: Install dependencies
       run: |
-        sudo apt-get update
-        sudo apt-get install -y eatmydata
-        sudo eatmydata apt-get install -y latexmk librsvg2-bin texlive-fonts-recommended texlive-latex-recommended texlive-latex-extra
+        sudo apt-get install -y latexmk librsvg2-bin texlive-fonts-recommended texlive-latex-recommended texlive-latex-extra
         pip install -r requirements-doc.txt
     - name: Build and Validate Stubs
       run: make check-stubs -j2

.github/workflows/create_website_pr.yml

@@ -31,8 +31,6 @@ jobs:
         version: true
     - name: Set up external
       uses: ./.github/actions/deps/external
-      with:
-        apt: false
     - name: Versions
       run: |
         gcc --version

.github/workflows/pre-commit.yml

@@ -30,7 +30,7 @@ jobs:
     - name: Set up external
       uses: ./.github/actions/deps/external
     - name: Install dependencies
-      run: sudo apt-get install -y uncrustify
+      run: sudo apt-get install -y gettext uncrustify
     - name: Run pre-commit
       uses: pre-commit/action@v3.0.0
     - name: Make patch

.github/workflows/run-tests.yml

@@ -37,6 +37,7 @@ jobs:
       with:
         target: tests
     - name: Set up external
+      if: matrix.test == 'all'
       uses: ./.github/actions/deps/external
     - name: Set up mpy-cross
       uses: ./.github/actions/mpy_cross

py/py.mk

@@ -243,7 +243,7 @@ $(HEADER_BUILD)/mpversion.h: FORCE | $(HEADER_BUILD)
 MPCONFIGPORT_MK = $(wildcard mpconfigport.mk)
 
 $(HEADER_BUILD)/$(TRANSLATION).mo: $(TOP)/locale/$(TRANSLATION).po | $(HEADER_BUILD)
-	$(Q)msgfmt -o $@ $^
+	$(Q)$(PYTHON) $(TOP)/tools/msgfmt.py -o $@ $^
 
 $(HEADER_BUILD)/qstrdefs.preprocessed.h: $(PY_QSTR_DEFS) $(QSTR_DEFS) $(QSTR_DEFS_COLLECTED) mpconfigport.h $(MPCONFIGPORT_MK) $(PY_SRC)/mpconfig.h | $(HEADER_BUILD)
 	$(STEPECHO) "GEN $@"

tools/msgfmt.py

@@ -0,0 +1,244 @@
+#! /usr/bin/env python3
+# Written by Martin v. Löwis <loewis@informatik.hu-berlin.de>
+
+"""Generate binary message catalog from textual translation description.
+This program converts a textual Uniforum-style message catalog (.po file) into
+a binary GNU catalog (.mo file).  This is essentially the same function as the
+GNU msgfmt program, however, it is a simpler implementation.  Currently it
+does not handle plural forms but it does handle message contexts.
+Usage: msgfmt.py [OPTIONS] filename.po
+Options:
+    -o file
+    --output-file=file
+        Specify the output file to write to.  If omitted, output will go to a
+        file named filename.mo (based off the input file name).
+    -h
+    --help
+        Print this message and exit.
+    -V
+    --version
+        Display version information and exit.
+"""
+
+import os
+import sys
+import ast
+import getopt
+import struct
+import array
+from email.parser import HeaderParser
+
+__version__ = "1.2"
+
+MESSAGES = {}
+
+
+def usage(code, msg=""):
+    print(__doc__, file=sys.stderr)
+    if msg:
+        print(msg, file=sys.stderr)
+    sys.exit(code)
+
+
+def add(ctxt, id, str, fuzzy):
+    "Add a non-fuzzy translation to the dictionary."
+    global MESSAGES
+    if not fuzzy and str:
+        if ctxt is None:
+            MESSAGES[id] = str
+        else:
+            MESSAGES[b"%b\x04%b" % (ctxt, id)] = str
+
+
+def generate():
+    "Return the generated output."
+    global MESSAGES
+    # the keys are sorted in the .mo file
+    keys = sorted(MESSAGES.keys())
+    offsets = []
+    ids = strs = b""
+    for id in keys:
+        # For each string, we need size and file offset.  Each string is NUL
+        # terminated; the NUL does not count into the size.
+        offsets.append((len(ids), len(id), len(strs), len(MESSAGES[id])))
+        ids += id + b"\0"
+        strs += MESSAGES[id] + b"\0"
+    output = ""
+    # The header is 7 32-bit unsigned integers.  We don't use hash tables, so
+    # the keys start right after the index tables.
+    # translated string.
+    keystart = 7 * 4 + 16 * len(keys)
+    # and the values start after the keys
+    valuestart = keystart + len(ids)
+    koffsets = []
+    voffsets = []
+    # The string table first has the list of keys, then the list of values.
+    # Each entry has first the size of the string, then the file offset.
+    for o1, l1, o2, l2 in offsets:
+        koffsets += [l1, o1 + keystart]
+        voffsets += [l2, o2 + valuestart]
+    offsets = koffsets + voffsets
+    output = struct.pack(
+        "Iiiiiii",
+        0x950412DE,  # Magic
+        0,  # Version
+        len(keys),  # # of entries
+        7 * 4,  # start of key index
+        7 * 4 + len(keys) * 8,  # start of value index
+        0,
+        0,
+    )  # size and offset of hash table
+    output += array.array("i", offsets).tobytes()
+    output += ids
+    output += strs
+    return output
+
+
+def make(filename, outfile):
+    ID = 1
+    STR = 2
+    CTXT = 3
+
+    # Compute .mo name from .po name and arguments
+    if filename.endswith(".po"):
+        infile = filename
+    else:
+        infile = filename + ".po"
+    if outfile is None:
+        outfile = os.path.splitext(infile)[0] + ".mo"
+
+    try:
+        with open(infile, "rb") as f:
+            lines = f.readlines()
+    except IOError as msg:
+        print(msg, file=sys.stderr)
+        sys.exit(1)
+
+    section = msgctxt = None
+    fuzzy = 0
+
+    # Start off assuming Latin-1, so everything decodes without failure,
+    # until we know the exact encoding
+    encoding = "latin-1"
+
+    # Parse the catalog
+    lno = 0
+    for l in lines:
+        l = l.decode(encoding)
+        lno += 1
+        # If we get a comment line after a msgstr, this is a new entry
+        if l[0] == "#" and section == STR:
+            add(msgctxt, msgid, msgstr, fuzzy)
+            section = msgctxt = None
+            fuzzy = 0
+        # Record a fuzzy mark
+        if l[:2] == "#," and "fuzzy" in l:
+            fuzzy = 1
+        # Skip comments
+        if l[0] == "#":
+            continue
+        # Now we are in a msgid or msgctxt section, output previous section
+        if l.startswith("msgctxt"):
+            if section == STR:
+                add(msgctxt, msgid, msgstr, fuzzy)
+            section = CTXT
+            l = l[7:]
+            msgctxt = b""
+        elif l.startswith("msgid") and not l.startswith("msgid_plural"):
+            if section == STR:
+                add(msgctxt, msgid, msgstr, fuzzy)
+                if not msgid:
+                    # See whether there is an encoding declaration
+                    p = HeaderParser()
+                    charset = p.parsestr(msgstr.decode(encoding)).get_content_charset()
+                    if charset:
+                        encoding = charset
+            section = ID
+            l = l[5:]
+            msgid = msgstr = b""
+            is_plural = False
+        # This is a message with plural forms
+        elif l.startswith("msgid_plural"):
+            if section != ID:
+                print(
+                    "msgid_plural not preceded by msgid on %s:%d" % (infile, lno), file=sys.stderr
+                )
+                sys.exit(1)
+            l = l[12:]
+            msgid += b"\0"  # separator of singular and plural
+            is_plural = True
+        # Now we are in a msgstr section
+        elif l.startswith("msgstr"):
+            section = STR
+            if l.startswith("msgstr["):
+                if not is_plural:
+                    print("plural without msgid_plural on %s:%d" % (infile, lno), file=sys.stderr)
+                    sys.exit(1)
+                l = l.split("]", 1)[1]
+                if msgstr:
+                    msgstr += b"\0"  # Separator of the various plural forms
+            else:
+                if is_plural:
+                    print(
+                        "indexed msgstr required for plural on  %s:%d" % (infile, lno),
+                        file=sys.stderr,
+                    )
+                    sys.exit(1)
+                l = l[6:]
+        # Skip empty lines
+        l = l.strip()
+        if not l:
+            continue
+        l = ast.literal_eval(l)
+        if section == CTXT:
+            msgctxt += l.encode(encoding)
+        elif section == ID:
+            msgid += l.encode(encoding)
+        elif section == STR:
+            msgstr += l.encode(encoding)
+        else:
+            print("Syntax error on %s:%d" % (infile, lno), "before:", file=sys.stderr)
+            print(l, file=sys.stderr)
+            sys.exit(1)
+    # Add last entry
+    if section == STR:
+        add(msgctxt, msgid, msgstr, fuzzy)
+
+    # Compute output
+    output = generate()
+
+    try:
+        with open(outfile, "wb") as f:
+            f.write(output)
+    except IOError as msg:
+        print(msg, file=sys.stderr)
+
+
+def main():
+    try:
+        opts, args = getopt.getopt(sys.argv[1:], "hVo:", ["help", "version", "output-file="])
+    except getopt.error as msg:
+        usage(1, msg)
+
+    outfile = None
+    # parse options
+    for opt, arg in opts:
+        if opt in ("-h", "--help"):
+            usage(0)
+        elif opt in ("-V", "--version"):
+            print("msgfmt.py", __version__)
+            sys.exit(0)
+        elif opt in ("-o", "--output-file"):
+            outfile = arg
+    # do it
+    if not args:
+        print("No input file given", file=sys.stderr)
+        print("Try `msgfmt --help' for more information.", file=sys.stderr)
+        return
+
+    for filename in args:
+        make(filename, outfile)
+
+
+if __name__ == "__main__":
+    main()