Fix #349, do parallel transcription with DeepSpeech

This doesn't effect anything a user sees, but will test the accuracy of DeepSpeech in Telemetry.

extension/background/intentRunner.js

@@ -27,7 +27,7 @@ this.intentRunner = (function() {
 
     failed(message) {
       telemetry.add({ intentSuccess: false });
-      telemetry.send({});
+      telemetry.sendSoon();
       try {
         this.onError(message);
       } catch (e) {
@@ -76,6 +76,7 @@ this.intentRunner = (function() {
         intentParseSuccess: !this.fallback,
         intentSuccess: true,
         utterance: this.utterance,
+        utteranceChars: this.utterance.length,
         utteranceParsed: { slots: this.slots },
       });
     }
@@ -167,7 +168,7 @@ this.intentRunner = (function() {
       }
       // FIXME: this isn't necessarily the right time to send the ping, if the intent
       // isn't actually complete:
-      telemetry.send();
+      telemetry.sendSoon();
     } catch (e) {
       const display = e.displayMessage || `Internal error: ${e}`;
       context.failed(display);

extension/background/main.js

@@ -40,8 +40,6 @@ this.main = (function() {
       return settings.saveSettings(message.settings);
     } else if (message.type === "addTelemetry") {
       return telemetry.add(message.properties);
-    } else if (message.type === "sendTelemetry") {
-      return telemetry.send();
     } else if (message.type === "addFeedback") {
       return telemetry.addFeedback(message.properties);
     } else if (message.type === "openRecordingTab") {

extension/background/telemetry.js

@@ -83,6 +83,11 @@ this.telemetry = (function() {
     ping = null;
   };
 
+  exports.sendSoon = async function() {
+    await util.sleep(1000);
+    return exports.send();
+  };
+
   exports.addFeedback = function({ feedback, rating }) {
     const ping = Object.assign(
       { intentId: lastIntentId, timestamp: Date.now() },

extension/background/voiceSchema.js

@@ -1,6 +1,6 @@
 /* eslint-disable */
 // This is a copy of:
-// https://github.com/mozilla-services/mozilla-pipeline-schemas/blob/master/schemas/telemetry/voice/voice.1.schema.json
+// https://github.com/mozilla-services/mozilla-pipeline-schemas/blob/master/schemas/telemetry/voice/voice.4.schema.json
 
 this.voiceSchema = {
   "$schema": "http://json-schema.org/draft-04/schema#",
@@ -98,6 +98,12 @@ this.voiceSchema = {
                   "isSystem": {
                     "type": "boolean"
                   },
+                  "isWebExtension": {
+                    "type": "boolean"
+                  },
+                  "multiprocessCompatible": {
+                    "type": "boolean"
+                  },
                   "name": {
                     "type": "string"
                   },
@@ -306,7 +312,10 @@ this.voiceSchema = {
               "type": "boolean"
             },
             "vendor": {
-              "type": "string"
+              "type": [
+                "string",
+                "null"
+              ]
             },
             "version": {
               "pattern": "^[0-9]{2,3}\\.",
@@ -333,6 +342,12 @@ this.voiceSchema = {
             "properties": {
               "branch": {
                 "type": "string"
+              },
+              "enrollmentId": {
+                "type": "string"
+              },
+              "type": {
+                "type": "string"
               }
             },
             "type": "object"
@@ -397,6 +412,17 @@ this.voiceSchema = {
           },
           "type": "object"
         },
+        "services": {
+          "properties": {
+            "accountEnabled": {
+              "type": "boolean"
+            },
+            "syncEnabled": {
+              "type": "boolean"
+            }
+          },
+          "type": "object"
+        },
         "settings": {
           "properties": {
             "addonCompatibilityCheckEnabled": {
@@ -430,6 +456,29 @@ this.voiceSchema = {
             "blocklistEnabled": {
               "type": "boolean"
             },
+            "defaultPrivateSearchEngine": {
+              "type": "string"
+            },
+            "defaultPrivateSearchEngineData": {
+              "properties": {
+                "loadPath": {
+                  "type": [
+                    "string",
+                    "null"
+                  ]
+                },
+                "name": {
+                  "type": "string"
+                },
+                "origin": {
+                  "type": "string"
+                },
+                "submissionURL": {
+                  "type": "string"
+                }
+              },
+              "type": "object"
+            },
             "defaultSearchEngine": {
               "type": "string"
             },
@@ -516,6 +565,12 @@ this.voiceSchema = {
                 "null"
               ]
             },
+            "isInOptoutSample": {
+              "type": [
+                "boolean",
+                "null"
+              ]
+            },
             "launcherProcessState": {
               "type": "integer"
             },
@@ -557,6 +612,14 @@ this.voiceSchema = {
               "type": "object"
             },
             "userPrefs": {
+              "additionalProperties": {
+                "type": [
+                  "boolean",
+                  "string",
+                  "number",
+                  "null"
+                ]
+              },
               "type": "object"
             }
           },
@@ -739,6 +802,12 @@ this.voiceSchema = {
                   "properties": {
                     "advancedLayers": {
                       "properties": {
+                        "noConstantBufferOffsetting": {
+                          "type": [
+                            "boolean",
+                            "null"
+                          ]
+                        },
                         "status": {
                           "type": [
                             "string",
@@ -816,6 +885,28 @@ this.voiceSchema = {
                         }
                       },
                       "type": "object"
+                    },
+                    "webrender": {
+                      "properties": {
+                        "status": {
+                          "type": [
+                            "string",
+                            "null"
+                          ]
+                        }
+                      },
+                      "type": "object"
+                    },
+                    "wrQualified": {
+                      "properties": {
+                        "status": {
+                          "type": [
+                            "string",
+                            "null"
+                          ]
+                        }
+                      },
+                      "type": "object"
                     }
                   },
                   "type": "object"
@@ -938,13 +1029,22 @@ this.voiceSchema = {
             "isWow64": {
               "type": "boolean"
             },
+            "isWowARM64": {
+              "type": [
+                "boolean",
+                "null"
+              ]
+            },
             "memoryMB": {
               "type": "number"
             },
             "os": {
               "properties": {
                 "installYear": {
-                  "type": "number"
+                  "type": [
+                    "number",
+                    "null"
+                  ]
                 },
                 "kernelVersion": {
                   "type": "string"
@@ -963,8 +1063,7 @@ this.voiceSchema = {
                 },
                 "version": {
                   "type": [
-                    "string",
-                    "integer"
+                    "string"
                   ]
                 },
                 "windowsBuildNumber": {
@@ -1035,6 +1134,10 @@ this.voiceSchema = {
     },
     "payload": {
       "properties": {
+        "deepSpeechMatches": {
+          "title": "Did the DeepSpeech transcription match with Google's?",
+          "type": "boolean"
+        },
         "extensionInstallationChannel": {
           "description": "Where the user first installed the extension (e.g., internal_beta, external_beta, etc)",
           "title": "First install channel",
@@ -1170,6 +1273,19 @@ this.voiceSchema = {
           "title": "Text utterance by user",
           "type": "string"
         },
+        "utteranceChars": {
+          "title": "Number of characters in utterance",
+          "type": "number"
+        },
+        "utteranceDeepSpeech": {
+          "description": "An alternate transcription for verification",
+          "title": "Utterance as transcribed by DeepSpeech",
+          "type": "string"
+        },
+        "utteranceDeepSpeechChars": {
+          "title": "Number of characters in DeepSpeech transcription",
+          "type": "number"
+        },
         "utteranceParsed": {
           "description": "Typically the slots found in the utterance",
           "title": "Parsed form of the utterance",
@@ -1190,8 +1306,8 @@ this.voiceSchema = {
       "type": "string"
     },
     "version": {
-      "maximum": 1,
-      "minimum": 1,
+      "maximum": 4,
+      "minimum": 4,
       "type": "number"
     }
   },

extension/buildSettings.js.ejs

@@ -1,7 +1,9 @@
 this.buildSettings = {
   logLevel: <%= env.LOG_LEVEL || "debug" %>,
   inDevelopment: null,
-  sstServer: "https://speaktome-2.services.mozilla.com",
+  sttServer: "https://speaktome-2.services.mozilla.com",
+  deepSpeechServer: "https://dev.speaktome.nonprod.cloudops.mozgcp.net",
+  sendToDeepSpeech: true,
   gitCommit: <%= gitCommit %>,
   buildTime: <%= buildTime %>,
   backgroundTabRecorder: <%= !env.FOREGROUND %>,

extension/popup/voice.js

@@ -1,10 +1,11 @@
-/* globals vad, buildSettings */
+/* globals vad, buildSettings, log, util */
 
 this.voice = (function() {
   const exports = {};
 
   const STT_SERVER_URL =
-    buildSettings.sstServer || "https://speaktome-2.services.mozilla.com";
+    buildSettings.sttServer || "https://speaktome-2.services.mozilla.com";
+  const DEEP_SPEECH_URL = buildSettings.deepSpeechServer;
   const LANGUAGE = "en-US";
 
   exports.Recorder = class Recorder {
@@ -178,6 +179,46 @@ this.voice = (function() {
         return;
       }
       const json = await response.json();
+      if (buildSettings.sendToDeepSpeech) {
+        // We will send it right away, but we don't want block on this:
+        setTimeout(async () => {
+          let response;
+          try {
+            response = await fetch(DEEP_SPEECH_URL, {
+              method: "POST",
+              body: blob,
+              headers: {
+                "Accept-Language-STT": LANGUAGE,
+                "Product-Tag": "vf",
+              },
+            });
+          } catch (e) {
+            log.warn("Error sending audio to DeepSpeech:", String(e), e);
+            return;
+          }
+          if (!response.ok) {
+            log.warn(
+              "Server error for DeepSpeech:",
+              response.status,
+              response.statusText
+            );
+            return;
+          }
+          const deepJson = await response.json();
+          const utterance = deepJson.data[0].text;
+          browser.runtime.sendMessage({
+            type: "addTelemetry",
+            properties: {
+              utteranceDeepSpeech: utterance,
+              utteranceDeepSpeechChars: utterance.length,
+              deepSpeechMatches: util.normalizedStringsMatch(
+                utterance,
+                json.data[0].text
+              ),
+            },
+          });
+        });
+      }
       browser.runtime.sendMessage({
         type: "addTelemetry",
         properties: {

extension/util.js

@@ -80,6 +80,16 @@ this.util = (function() {
     return 0;
   };
 
+  exports.normalizedStringsMatch = function(a, b) {
+    function norm(s) {
+      s = s.toLowerCase().trim();
+      s = s.replace(/\s\s+/, " ");
+      s = s.replace(/[^a-z0-9]/, "");
+      return s;
+    }
+    return norm(a) === norm(b);
+  };
+
   exports.randomString = function randomString(length, chars) {
     const randomStringChars =
       "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";