Fixes for PR3638 (#3680)

.github/workflows/translations.yaml

@@ -43,7 +43,9 @@ jobs:
           ./scripts/utils/generate_ts.sh
 
       - name: Uploading
-        uses: actions/upload-artifact@v1
+        uses: actions/upload-artifact@v3
         with:
-            name: Translation file
-            path: translations.ts
+            name: Translation files
+            path: |
+              translations.ts
+              addon_ts/*.ts

.gitignore

@@ -33,6 +33,9 @@ linux/netfilter/vendor/
 __pycache__
 lottie.mjs
 
+# Translation pipeline
+addon_ts/
+
 # CMake build artifacts
 cmake_install.cmake
 CMakeCache.txt
@@ -47,7 +50,7 @@ src/mozillavpn_autogen/
 node_modules
 
 #Github Codespaces
-.venv 
+.venv
 
 asset_catalog_compiler.Info.plist
 
@@ -60,7 +63,7 @@ android/local.properties
 android/**/*.cxx
 android/.cxx/*
 !android/res/debug/**/*
-!android/*/**/Makefile 
+!android/*/**/Makefile
 xcode.xconfig
 .gradle/
 .gradle_cache/
@@ -130,7 +133,7 @@ windows/split-tunnel/*.sys
 windows/split-tunnel/*.dll
 windows/split-tunnel/.status
 
-#Rust
+# Rust
 .cargo_home/
 
 

scripts/addon/build.py

@@ -6,7 +6,7 @@
 import argparse
 import json
 import os
-import xml.etree.ElementTree as ET
+from lxml import etree as ET
 import tempfile
 import shutil
 import sys
@@ -58,9 +58,9 @@ def retrieve_strings_tutorial(manifest, filename):
     }
 
     for step in tutorial_json["steps"]:
-        if not "id" in step:
+        if "id" not in step:
             exit(f"Tutorial {filename} does not have an id for one of the steps")
-        if not "tooltip" in step:
+        if "tooltip" not in step:
             exit(
                 f"Tutorial {filename} does not have a tooltip for step id {step['id']}"
             )
@@ -81,13 +81,13 @@ def retrieve_strings_guide(manifest, filename):
     guide_strings = {}
 
     guide_json = manifest["guide"]
-    if not "id" in guide_json:
+    if "id" not in guide_json:
         exit(f"Guide {filename} does not have an id")
-    if not "title" in guide_json:
+    if "title" not in guide_json:
         exit(f"Guide {filename} does not have a title")
-    if not "subtitle" in guide_json:
+    if "subtitle" not in guide_json:
         exit(f"Guide {filename} does not have a subtitle")
-    if not "blocks" in guide_json:
+    if "blocks" not in guide_json:
         exit(f"Guide {filename} does not have a blocks")
 
     guide_id = guide_json["id"]
@@ -103,11 +103,11 @@ def retrieve_strings_guide(manifest, filename):
     }
 
     for block in guide_json["blocks"]:
-        if not "id" in block:
+        if "id" not in block:
             exit(f"Guide {filename} does not have an id for one of the blocks")
-        if not "type" in block:
+        if "type" not in block:
             exit(f"Guide {filename} does not have a type for block id {block['id']}")
-        if not "content" in block:
+        if "content" not in block:
             exit(f"Guide {filename} does not have a content for block id {block['id']}")
 
         block_id = block["id"]
@@ -124,11 +124,11 @@ def retrieve_strings_guide(manifest, filename):
             continue
 
         for subblock in block["content"]:
-            if not "id" in subblock:
+            if "id" not in subblock:
                 exit(
                     f"Guide {filename} does not have an id for one of the subblocks of block {block_id}"
                 )
-            if not "content" in subblock:
+            if "content" not in subblock:
                 exit(
                     f"Guide file {filename} does not have a content for subblock id {subblock['id']}"
                 )
@@ -160,6 +160,9 @@ def write_en_language(filename, strings):
         message = ET.SubElement(context, "message")
         message.set("id", key)
 
+        location = ET.SubElement(message, "location")
+        location.set("filename", "addon.qml")
+
         source = ET.SubElement(message, "source")
         source.text = value["value"]
 
@@ -171,7 +174,7 @@ def write_en_language(filename, strings):
             extracomment.text = value["comments"]
 
     with open(filename, "w", encoding="utf-8") as f:
-        f.write(ET.tostring(ts, encoding="unicode"))
+        f.write(ET.tostring(ts, encoding="unicode", pretty_print=True))
 
 
 def copy_files(path, dest_path):
@@ -182,7 +185,7 @@ def copy_files(path, dest_path):
         file_path = os.path.join(path, file)
         if os.path.isfile(file_path):
             if file_path.endswith((".ts", ".qrc", ".rcc")):
-                exit(f"Unexpected {ext} file found: {os.path.join(path, file)}")
+                exit(f"Unexpected extension file found: {os.path.join(path, file)}")
 
             shutil.copyfile(file_path, os.path.join(dest_path, file))
             continue
@@ -298,11 +301,11 @@ def qtquery(qmake, propname):
 with open(args.source, "r", encoding="utf-8") as file:
     manifest = json.load(file)
 
-    print(f"Copying files in a temporary folder...")
+    print("Copying files in a temporary folder...")
     tmp_path = tempfile.mkdtemp()
     copy_files(os.path.dirname(args.source), tmp_path)
 
-    print(f"Retrieving strings...")
+    print("Retrieving strings...")
     strings = {}
     if manifest["type"] == "tutorial":
         strings = retrieve_strings_tutorial(manifest, args.source)
@@ -311,7 +314,7 @@ def qtquery(qmake, propname):
     else:
         exit(f"Unupported manifest type `{manifest['type']}`")
 
-    print(f"Create localization file...")
+    print("Create localization file...")
     os.mkdir(os.path.join(tmp_path, "i18n"))
     template_ts_file = os.path.join(args.dest, f"{manifest['id']}.ts")
     write_en_language(template_ts_file, strings)
@@ -340,7 +343,7 @@ def qtquery(qmake, propname):
             os.system(f"{lconvert} -if xlf -i {xliff_path} -o {locale_file}")
             os.system(f"{lrelease} -idbased {locale_file}")
 
-    print(f"Generate the RC file...")
+    print("Generate the RC file...")
     files = get_file_list(tmp_path, "")
 
     qrc_file = os.path.join(tmp_path, f"{manifest['id']}.qrc")
@@ -353,7 +356,7 @@ def qtquery(qmake, propname):
             elm.text = file
         f.write(ET.tostring(rcc_elm, encoding="unicode"))
 
-    print(f"Creating the final addon...")
+    print("Creating the final addon...")
     rcc_file = os.path.join(args.dest, f"{manifest['id']}.rcc")
     os.system(f"{rcc} {qrc_file} -o {rcc_file} -binary")
     print(f"Done: {rcc_file}")

scripts/utils/generate_strings.py

@@ -7,6 +7,7 @@
 import yaml
 import argparse
 
+
 def stop(string_id):
     exit(
         f"Each key must be a string or a list with 1 or more items. Fix string ID `{string_id}`"
@@ -34,6 +35,7 @@ def construct_mapping(self, node, deep=False):
             mapping.append(key)
         return super().construct_mapping(node, deep)
 
+
 def parseTranslationStrings(yamlfile):
     if not os.path.isfile(yamlfile):
         exit(f"Unable to find {yamlfile}")
@@ -108,9 +110,10 @@ def parseTranslationStrings(yamlfile):
                     "value": value,
                     "comments": comments,
                 }
-        
+
         return yaml_strings
 
+
 # Render a dictionary of strings into the l18nstrings module.
 def generateStrings(strings, outdir):
     os.makedirs(outdir, exist_ok=True)
@@ -162,7 +165,9 @@ class L18nStrings final : public QQmlPropertyMap {
 """
         )
 
-    with open(os.path.join(outdir, "l18nstrings_p.cpp"), "w", encoding="utf-8") as output:
+    with open(
+        os.path.join(outdir, "l18nstrings_p.cpp"), "w", encoding="utf-8"
+    ) as output:
         output.write(
             """/* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
@@ -204,31 +209,46 @@ def serialize(string):
         # Generate the retranslate() method.
         output.write("void L18nStrings::retranslate() {\n")
         for key in strings:
-            output.write(f"    insert(\"{key}\", qtTrId(_ids[{key}]));\n")
+            output.write(f'    insert("{key}", qtTrId(_ids[{key}]));\n')
         output.write("}")
 
 
 if __name__ == "__main__":
     # Parse arguments to locate the input and output files.
     parser = argparse.ArgumentParser(
-        description='Generate internationaliation strings database from a YAML source')
-    parser.add_argument('source', metavar='SOURCE', type=str, action='store', nargs='?',
-        help='YAML strings file to process')
-    parser.add_argument('-o', '--output', metavar='DIR', type=str, action='store',
-        help='Output directory for generated files')
+        description="Generate internationaliation strings database from a YAML source"
+    )
+    parser.add_argument(
+        "source",
+        metavar="SOURCE",
+        type=str,
+        action="store",
+        nargs="?",
+        help="YAML strings file to process",
+    )
+    parser.add_argument(
+        "-o",
+        "--output",
+        metavar="DIR",
+        type=str,
+        action="store",
+        help="Output directory for generated files",
+    )
     args = parser.parse_args()
 
     # If no source was provided, find it relative to this script file.
     if args.source is None:
-        rootpath = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir))
-        args.source = os.path.join(rootpath, 'translations', 'strings.yaml')
-
+        rootpath = os.path.abspath(
+            os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
+        )
+        args.source = os.path.join(rootpath, "translations", "strings.yaml")
+
     # If no output directory was provided, use the current directory.
     if args.output is None:
         args.output = os.getcwd()
-    
+
     # Parse the inputs for their sweet juicy strings.
     strings = parseTranslationStrings(args.source)
-    
+
     # Render the strings into generated content.
     generateStrings(strings, args.output)

scripts/utils/generate_ts.sh

@@ -43,15 +43,40 @@ print G "done"
 print Y "Generating the main translation file... "
 lupdate translations/generated/dummy.pro -ts translations.ts || die
 
-for branch in $(git branch -r | grep origin/releases); do
-  printn Y "Importing strings from $branch..."
+printn Y "Generating strings for addons... "
+python scripts/addon/generate_all.py
+mkdir -p addon_ts || die
+cp addons/generated/addons/*.ts addon_ts
+print G "done."
 
+for branch in $(git branch -r | grep origin/releases); do
   git checkout $branch &>/dev/null || die
+
+  printn Y "Importing main strings from $branch..."
   python cache/generate_strings.py -o translations/generated || die
   lupdate translations/generated/dummy.pro -ts branch.ts || die
   lconvert -i translations.ts branch.ts -o tmp.ts || die
   mv tmp.ts translations.ts || die
   rm branch.ts || die
+
+  if [ -f "scripts/addon/generate_all.py" ]; then
+    printn Y "Importing addon strings from $branch..."
+    python scripts/addon/generate_all.py
+    ts_files="addons/generated/addons/*.ts"
+    for f in $ts_files
+    do
+      ts_name=$(basename "$f")
+      if [ -f "addon_ts/${ts_name}"]; then
+        printn Y "File ${ts_name} exists, updating with branch strings..."
+        lconvert -i "cache_rs/${ts_name}" "addons/generated/addons/${ts_name}" -o tmp.ts || die
+        mv tmp.ts "addon_ts/${ts_name}"
+      else
+        printn Y "File ${ts_name} does not exist, copying over..."
+        cp "addons/generated/addons/${ts_name}" addon_ts/
+      fi
+    done
+    rm addons/generated/addons/*.ts || die
+  fi
 done
 
 printn Y "Remove cache... "