VPN-5785 - Create tool for checking translation status on main and re…

…lease branches (#8633)

.github/workflows/gh_pages.yaml

@@ -21,7 +21,7 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  wasm_chrome: 
+  wasm_chrome:
     name: Add Wasm_Chrome
     runs-on: ubuntu-20.04
     steps:
@@ -41,7 +41,7 @@ jobs:
             name: Wasm-Chrome
             path: tools/wasm_chrome
 
-  logviewer: 
+  logviewer:
       name: Add Logviewer
       runs-on: ubuntu-20.04
       steps:
@@ -53,7 +53,7 @@ jobs:
               name: Logviewer
               path: tools/logviewer
 
-  addonStats: 
+  addonStats:
       name: Add Addon-Stats
       runs-on: ubuntu-20.04
       steps:
@@ -74,8 +74,8 @@ jobs:
       - name: Building
         shell: bash
         run: |
-            sudo apt-get update 
-            sudo apt-get install doxygen -y 
+            sudo apt-get update
+            sudo apt-get install doxygen -y
             doxygen
       - name: Uploading
         uses: actions/upload-artifact@v3
@@ -107,12 +107,36 @@ jobs:
             name: Inspector Build
             path: tools/inspector/dist
 
+  translationsreport:
+    name: Add Translations Report
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Clone repository
+        uses: actions/checkout@v3
+
+      - uses: actions/setup-node@v3
+        with:
+          node-version: '16'
+
+      - name: Building
+        shell: bash
+        run: |
+          cd tools/translationsreport/app
+          npm install
+          npm run build
+
+      - name: Uploading
+        uses: actions/upload-artifact@v3
+        with:
+            name: Translations Report
+            path: tools/translationsreport/app/build
+
   addons:
     runs-on: ubuntu-22.04
     steps:
       - name: Clone repository
         uses: actions/checkout@v3
-        with: 
+        with:
           submodules: 'true'
       - name: Install Linux packages
         run: |
@@ -155,7 +179,7 @@ jobs:
 
   ghPages:
     runs-on: ubuntu-20.04
-    needs: [wasm_chrome, inspector,addons,logviewer,addonstats,docs]
+    needs: [wasm_chrome, inspector, translationsreport, addons, logviewer, addonstats, docs]
     name: Compile Github Page from Components
     env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -198,6 +222,12 @@ jobs:
           name: Inspector Build
           path: _site/inspector
 
+      - name: Download the Translations Report artifact
+        uses: actions/download-artifact@v3
+        with:
+          name: Inspector Build
+          path: _site/translationsreport
+
       - name: Upload artifact
         uses: actions/upload-pages-artifact@v1
 
@@ -210,6 +240,6 @@ jobs:
       needs: ghPages
       if: github.ref == 'refs/heads/main'
       steps:
-        - name: Deploy to GitHub Pages  
+        - name: Deploy to GitHub Pages
           id: deployment
           uses: actions/deploy-pages@v1

.gitignore

@@ -221,3 +221,6 @@ target/
 
 # MSVC Windows builds of rustc generate these, which store debugging information
 *.pdb
+
+# Translations Report
+tools/translationsreport/output.json

package.json

@@ -5,7 +5,9 @@
     "wasm:dev": "node tests/functional/wasm.js",
     "qml-reload": "node ./tools/qml_hot_reload/hot.js",
     "languages:check": "cd ./tools/languagelocalizer/ && node languageLocalizer.js -c",
-    "languages:update": "cd ./tools/languagelocalizer/ && node languageLocalizer.js"
+    "languages:update": "cd ./tools/languagelocalizer/ && node languageLocalizer.js",
+    "translationsreport:start": "cd ./tools/translationsreport/app && npm run start",
+    "translationsreport:build": "cd ./tools/translationsreport/app && npm run build"
   },
   "devDependencies": {
     "appium-adb": "^9.14.4",
@@ -21,6 +23,7 @@
     "websocket": "^1.0.34"
   },
   "workspaces":[
-    "tools/languagelocalizer"
+    "tools/languagelocalizer",
+    "tools/translationsreport"
   ]
 }

taskcluster/ci/build-tools/kind.yml

@@ -0,0 +1,37 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+---
+
+loader: taskgraph.loader.transform:loader
+
+transforms:
+    - mozillavpn_taskgraph.transforms.release_index:transforms
+    - taskgraph.transforms.job:transforms
+    - taskgraph.transforms.task:transforms
+
+tasks:
+    translation-report:
+        worker-type: b-linux
+        worker:
+            docker-image: {in-tree: base}
+            max-run-time: 3600
+            artifacts:
+              - type: directory
+                name: public/build
+                path: /builds/worker/artifacts
+        description: "Generate translations report"
+        add-index-routes: tools
+        treeherder:
+            symbol: TL(I18NREPORT)
+            kind: build
+            platform: tools/dev
+            tier: 1
+        run:
+            using: run-task
+            use-caches: true
+            cwd: '{checkout}'
+            command: >-
+                pip3 install -r requirements.txt &&
+                python3 tools/translationsreport/analyzer.py &&
+                mv tools/translationsreport/output.json /builds/worker/artifacts

tools/translationsreport/analyzer.py

@@ -0,0 +1,141 @@
+import os
+import xml.etree.ElementTree as ET
+import argparse
+from jinja2 import Environment, FileSystemLoader
+import subprocess
+
+
+def get_current_branch():
+    try:
+        result = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD'], universal_newlines=True)
+        return result.strip()
+    except subprocess.CalledProcessError:
+        return "Unknown"
+
+
+def get_current_commit():
+    try:
+        result = subprocess.check_output(['git', 'rev-parse', 'HEAD'], universal_newlines=True)
+        return result.strip()
+    except subprocess.CalledProcessError:
+        return "Unknown"
+
+
+def extract_translated_strings(xliff_path):
+    tree = ET.parse(xliff_path)
+    root = tree.getroot()
+
+    translated_strings = set()
+    not_translated_strings = set()
+
+    # Define the namespace used in XLIFF files
+    namespace = {"xliff": "urn:oasis:names:tc:xliff:document:1.2"}
+
+    for trans_unit in root.findall(".//xliff:trans-unit", namespace):
+        source = trans_unit.find(".//xliff:source", namespace).text
+        target_element = trans_unit.find(".//xliff:target", namespace)
+
+        # Big hack, but this string is breaking the template.
+        # Jinja feel obligated to add a newline before % and
+        # I (Bea) don't know why
+        if "Please confirm you would like to remove" in source:
+            source = "Please confirm you would like to remove"
+
+        if target_element is not None and target_element.text is not None:
+            translated_strings.add(source)
+        else:
+            not_translated_strings.add(source)
+
+
+    return (translated_strings, not_translated_strings)
+
+
+def get_language_code_from_path(path):
+    components = path.split(os.path.sep)
+
+    # Find the index of "i18n" in the path
+    i18n_index = components.index("i18n") if "i18n" in components else -1
+
+    # If "i18n" is found and there is a folder after it, return that folder
+    if i18n_index != -1 and i18n_index < len(components) - 1:
+        return components[i18n_index + 1]
+    else:
+        raise Exception(f"Unexpected folder path! {path}")
+
+
+def get_addon_name_from_xliff_path(path):
+    # Split the path into components
+    components = path.rstrip(os.path.sep).split(os.path.sep)
+
+    # Return the last component, which is the last folder name
+    return components[-2]
+
+
+def analyze_folder(folder_path):
+    main_app_translations = {}
+    addon_translations = {}
+
+    script_directory = os.path.dirname(os.path.abspath(__file__))
+    addons_directory = os.path.join(script_directory, "../../addons")
+    for folder in os.listdir(addons_directory):
+        if os.path.isdir(os.path.join(addons_directory, folder)):
+            addon_translations[folder] = {}
+
+    for root, dirs, files in os.walk(folder_path):
+        for file in files:
+            if file.endswith(".xliff"):
+                xliff_file = os.path.join(root, file)
+                (translations, non_translations) = extract_translated_strings(xliff_file)
+                language_code = get_language_code_from_path(xliff_file)
+
+                if language_code == "en":
+                    translations = non_translations
+
+                destination = main_app_translations
+
+                # Determine if the strings are from the root or an addon
+                is_addon = "addons" in os.path.relpath(xliff_file, folder_path)
+                if is_addon:
+                    addon_name = get_addon_name_from_xliff_path(xliff_file)
+                    if addon_name not in addon_translations:
+                        addon_translations[addon_name] = {}
+
+                    destination = addon_translations[addon_name]
+
+                for translation in translations:
+                    if translation not in destination:
+                        destination[translation] = set()
+
+                    destination[translation].add(language_code)
+
+
+    return (main_app_translations, addon_translations)
+
+
+def generate_html(result):
+    script_directory = os.path.dirname(os.path.abspath(__file__))
+    template_loader = FileSystemLoader(script_directory)
+    env = Environment(loader=template_loader,autoescape=True)
+
+    template = env.get_template('template.jinja')
+
+    (main_app_translations, addon_translations) = result
+    output = template.render(
+        branch=get_current_branch(),
+        commit=get_current_commit(),
+        main_app_translations=main_app_translations,
+        addon_translations=addon_translations
+    )
+
+    with open(os.path.join(script_directory, "output.json"), 'w') as f:
+        f.write(output)
+
+
+def main():
+    script_directory = os.path.dirname(os.path.abspath(__file__))
+    result = analyze_folder(os.path.join(script_directory, "../../src/translations/i18n"))
+    generate_html(result)
+
+
+if __name__ == "__main__":
+    main()

tools/translationsreport/app/.gitignore

@@ -0,0 +1,23 @@
+# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
+
+# dependencies
+/node_modules
+/.pnp
+.pnp.js
+
+# testing
+/coverage
+
+# production
+/build
+
+# misc
+.DS_Store
+.env.local
+.env.development.local
+.env.test.local
+.env.production.local
+
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*