Work on loading UI translations (#2969)

* Load translations for Javascript in page template

* Normalise language codes to gettext format with underscores

* .mo files need to be under LC_MESSAGES as well

* remove unused JS code

* Normalise result in test

* Fix for opening files on Py 2

* Fix location of I18N directory

* Add translation files to package_data

.gitignore

@@ -6,7 +6,8 @@ docs/man/*.gz
 docs/source/api/generated
 docs/source/config.rst
 docs/gh-pages
-notebook/i18n/*/*.mo
+notebook/i18n/*/LC_MESSAGES/*.mo
+notebook/i18n/*/LC_MESSAGES/nbjs.json
 notebook/static/components
 notebook/static/style/*.min.css*
 notebook/static/*/js/built/

notebook/base/handlers.py

@@ -34,6 +34,7 @@
 
 import notebook
 from notebook._tz import utcnow
+from notebook.i18n import combine_translations
 from notebook.utils import is_hidden, url_path_join, url_is_absolute, url_escape
 from notebook.services.security import csp_report_uri
 
@@ -409,6 +410,8 @@ def template_namespace(self):
             xsrf_form_html=self.xsrf_form_html,
             token=self.token,
             xsrf_token=self.xsrf_token.decode('utf8'),
+            nbjs_translations=json.dumps(combine_translations(
+                self.request.headers.get('Accept-Language', ''))),
             **self.jinja_template_vars
         )
 

notebook/i18n/README.md

@@ -62,8 +62,8 @@ code for your desired language ( i.e. German = "de", Japanese = "ja", etc. ).
 use at runtime.
 
 ```shell
-pybabel compile -D notebook -f -l ${LANG} -i ${LANG}/LC_MESSAGES/notebook.po -o ${LANG}/notebook.mo
-pybabel compile -D nbui -f -l ${LANG} -i ${LANG}/LC_MESSAGES/nbui.po -o ${LANG}/nbui.mo
+pybabel compile -D notebook -f -l ${LANG} -i ${LANG}/LC_MESSAGES/notebook.po -o ${LANG}/LC_MESSAGES/notebook.mo
+pybabel compile -D nbui -f -l ${LANG} -i ${LANG}/LC_MESSAGES/nbui.po -o ${LANG}/LC_MESSAGES/nbui.mo
 ```
 
 *nbjs.po* needs to be converted to JSON for use within the JavaScript code, with  *po2json*, as follows:

notebook/i18n/__init__.py

@@ -0,0 +1,99 @@
+"""Server functions for loading translations
+"""
+from collections import defaultdict
+import errno
+import io
+import json
+from os.path import dirname, join as pjoin
+import re
+
+I18N_DIR = dirname(__file__)
+# Cache structure:
+# {'nbjs': {   # Domain
+#   'zh-CN': {  # Language code
+#     <english string>: <translated string>
+#     ...
+#   }
+# }}
+TRANSLATIONS_CACHE = {'nbjs': {}}
+
+
+_accept_lang_re = re.compile(r'''
+(?P<lang>[a-zA-Z]{1,8}(-[a-zA-Z]{1,8})?)
+(\s*;\s*q\s*=\s*
+  (?P<qvalue>[01](.\d+)?)
+)?''', re.VERBOSE)
+
+def parse_accept_lang_header(accept_lang):
+    """Parses the 'Accept-Language' HTTP header.
+
+    Returns a list of language codes in *ascending* order of preference
+    (with the most preferred language last).
+    """
+    by_q = defaultdict(list)
+    for part in accept_lang.split(','):
+        m = _accept_lang_re.match(part.strip())
+        if not m:
+            continue
+        lang, qvalue = m.group('lang', 'qvalue')
+        # Browser header format is zh-CN, gettext uses zh_CN
+        lang = lang.replace('-', '_')
+        if qvalue is None:
+            qvalue = 1.
+        else:
+            qvalue = float(qvalue)
+        if qvalue == 0:
+            continue  # 0 means not accepted
+        by_q[qvalue].append(lang)
+
+    res = []
+    for qvalue, langs in sorted(by_q.items()):
+        res.extend(sorted(langs))
+    return res
+
+def load(language, domain='nbjs'):
+    """Load translations from an nbjs.json file"""
+    try:
+        f = io.open(pjoin(I18N_DIR, language, 'LC_MESSAGES', 'nbjs.json'),
+                    encoding='utf-8')
+    except IOError as e:
+        if e.errno != errno.ENOENT:
+            raise
+        return {}
+
+    with f:
+        data = json.load(f)
+    return data["locale_data"][domain]
+
+def cached_load(language, domain='nbjs'):
+    """Load translations for one language, using in-memory cache if available"""
+    domain_cache = TRANSLATIONS_CACHE[domain]
+    try:
+        return domain_cache[language]
+    except KeyError:
+        data = load(language, domain)
+        domain_cache[language] = data
+        return data
+
+def combine_translations(accept_language, domain='nbjs'):
+    """Combine translations for multiple accepted languages.
+
+    Returns data re-packaged in jed1.x format.
+    """
+    lang_codes = parse_accept_lang_header(accept_language)
+    combined = {}
+    for language in lang_codes:
+        if language == 'en':
+            # en is default, all translations are in frontend.
+            combined.clear()
+        else:
+            combined.update(cached_load(language, domain))
+
+    combined[''] = {"domain":"nbjs"}
+
+    return {
+        "domain": domain,
+        "locale_data": {
+            domain: combined
+        }
+    }

notebook/static/base/js/i18n.js

@@ -4,51 +4,13 @@
 // Module to handle i18n ( Internationalization ) and translated UI
 
 define([
-    'jed',
-    'moment',
-    'json!../../../i18n/nbjs.json',
-    'base/js/i18nload',
-	], function(Jed, moment, nbjs, i18nload) {
+    'jed'
+	], function(Jed) {
     "use strict";
-
-    // Setup language related stuff
-    var ui_lang = navigator.languages && navigator.languages[0] || // Chrome / Firefox
-    navigator.language ||   // All browsers
-    navigator.userLanguage; // IE <= 10
 
-    var init = function() {
-    	var msg_promise;
-    	if (nbjs.supported_languages.indexOf(ui_lang) >= 0) {
-    		moment.locale(ui_lang);
-    		msg_promise = new Promise( function (resolve, reject) {
-    			require([i18nload.id+"!"+ui_lang], function (data) {
-    				var newi18n = new Jed(data);
-    				newi18n._ = newi18n.gettext;
-    				resolve(newi18n);
-    			}, function (error) {
-    	            console.log("Error loading translations for language: "+ui_lang);
-    	            var newi18n = new Jed(nbjs);
-    	            newi18n._ = newi18n.gettext;
-    	            resolve(newi18n);
-    			});
-    	   });
-    	} else {
-    		msg_promise = new Promise( function (resolve, reject) {
-			    var newi18n = new Jed(nbjs);
-				newi18n._ = newi18n.gettext;
-				resolve(newi18n);
-    		});
-    	}
-    	return msg_promise;
-    }
-    var i18n = new Jed(nbjs);   
+    var i18n = new Jed(document.nbjs_translations);
     i18n._ = i18n.gettext;
     i18n.msg = i18n; // Just a place holder until the init promise resolves.
-
-    init().then(function (msg) {
-    	i18n.msg = msg;
-    	i18n.msg._ = i18n.msg.gettext;
-    });
-
+
     return i18n;
 });

notebook/templates/page.html

@@ -97,6 +97,8 @@
               return {};
           }
       })
+
+    document.nbjs_translations = {{ nbjs_translations|safe }};
     </script>
 
     {% block meta %}

notebook/tests/test_i18n.py

@@ -0,0 +1,10 @@
+import nose.tools as nt
+
+from notebook import i18n
+
+def test_parse_accept_lang_header():
+    palh = i18n.parse_accept_lang_header
+    nt.assert_equal(palh(''), [])
+    nt.assert_equal(palh('zh-CN,en-GB;q=0.7,en;q=0.3'),
+                    ['en', 'en_GB', 'zh_CN'])
+    nt.assert_equal(palh('nl,fr;q=0'), ['nl'])

setupbase.py

@@ -210,6 +210,7 @@ def find_package_data():
         'notebook.tests' : js_tests,
         'notebook.bundler.tests': ['resources/*', 'resources/*/*', 'resources/*/*/.*'],
         'notebook.services.api': ['api.yaml'],
+        'notebook.i18n': ['*/LC_MESSAGES/*.*'],
     }
 
     return package_data