fix: None feature was affecting too much when no so many intents.

axa-group · Aug 20, 2019 · 344885c · 344885c
1 parent 1cdf6b6
commit 344885c
Show file tree

Hide file tree

Showing 4 changed files with 30 additions and 6 deletions.
diff --git a/lib/classifiers/neural-network.js b/lib/classifiers/neural-network.js
@@ -113,6 +113,24 @@ class NeuralNetwork {
   }
 
   train(srcData) {
+    const useNoneFeature =
+      srcData[srcData.length - 1].input.nonefeature !== undefined;
+    if (useNoneFeature) {
+      const intents = {};
+      for (let i = 0; i < srcData.length - 1; i += 1) {
+        const tokens = Object.keys(srcData[i].output);
+        for (let j = 0; j < tokens.length; j += 1) {
+          if (!intents[tokens[j]]) {
+            intents[tokens[j]] = 1;
+          }
+        }
+      }
+      const current = srcData[srcData.length - 1];
+      const keys = Object.keys(intents);
+      for (let i = 0; i < keys.length; i += 1) {
+        current.output[keys[i]] = 0.0000001;
+      }
+    }
     const data = this.formatData(srcData);
     if (!this.status) {
       this.status = { error: Infinity, deltaError: Infinity, iterations: 0 };

diff --git a/lib/nlp/nlp-manager.js b/lib/nlp/nlp-manager.js
@@ -167,15 +167,21 @@ class NlpManager {
    */
   addDocument(locale, srcUtterance, intent) {
     const utterance = this.translateUtterance(srcUtterance);
-    this.nluManager.addDocument(locale, utterance, intent);
     const entities = this.nerManager.getEntitiesFromUtterance(utterance);
     this.slotManager.addBatch(intent, entities);
     const optionalUtterance = this.nerManager.generateNamedEntityUtterance(
       utterance,
       locale
     );
     if (optionalUtterance) {
-      this.nluManager.addDocument(locale, optionalUtterance, intent);
+      this.nluManager.addDocument(locale, `${optionalUtterance}`, intent);
+      this.nluManager.addDocument(
+        locale,
+        `${optionalUtterance} ${utterance}`,
+        intent
+      );
+    } else {
+      this.nluManager.addDocument(locale, utterance, intent);
     }
   }
 

diff --git a/lib/nlu/base-nlu.js b/lib/nlu/base-nlu.js
@@ -179,13 +179,13 @@ class BaseNLU {
   }
 
   getWhitelist(tokens) {
-    const result = [];
+    const result = {};
     for (let i = 0; i < this.docs.length; i += 1) {
       if (this.someSimilar(tokens, this.docs[i].tokens)) {
-        result.push(this.docs[i].intent);
+        result[this.docs[i].intent] = 1;
       }
     }
-    return result;
+    return Object.keys(result);
   }
 
   /**

diff --git a/lib/nlu/brain-nlu.js b/lib/nlu/brain-nlu.js
@@ -79,7 +79,7 @@ class BrainNLU extends BaseNLU {
   getClassifications(utterance) {
     const tokens = this.textToFeatures(utterance);
     const whitelist = this.getWhitelist(tokens);
-    if (!whitelist.includes.None) {
+    if (!whitelist.includes('None')) {
       whitelist.push('None');
     }
     const classifications = this.classifier.getClassifications(tokens);