feat(server): introduce main NLP model and resolvers NLP model

.gitignore

@@ -25,6 +25,6 @@ packages/**/config/config.json
 skills/**/src/config.json
 packages/**/data/db/*.json
 skills/**/memory/*.json
-core/data/leon-model.nlp
+core/data/models/*.nlp
 package.json.backup
 .python-version

package.json

@@ -48,7 +48,7 @@
     "build:server": "npm run delete-dist:server && npm run train && npm run generate:skills-endpoints && babel ./server/src -d ./server/dist --copy-files && shx mkdir -p server/dist/tmp",
     "start:tcp-server": "cross-env PIPENV_PIPFILE=bridges/python/Pipfile pipenv run python bridges/python/tcp_server/main.py",
     "start": "cross-env LEON_NODE_ENV=production node ./server/dist/index.js",
-    "train": "babel-node scripts/run-train.js",
+    "train": "babel-node scripts/train/run-train.js",
     "prepare-release": "babel-node scripts/release/prepare-release.js",
     "check": "babel-node scripts/run-check.js",
     "docker:build": "docker build -t leonai/leon .",

scripts/check.js

@@ -22,7 +22,8 @@ export default () => new Promise(async (resolve, reject) => {
     const googleCloudPath = 'core/config/voice/google-cloud.json'
     const watsonSttPath = 'core/config/voice/watson-stt.json'
     const watsonTtsPath = 'core/config/voice/watson-tts.json'
-    const nlpModelPath = 'core/data/leon-model.nlp'
+    const resolversNlpModelPath = 'core/data/models/leon-resolvers-model.nlp'
+    const mainNlpModelPath = 'core/data/models/leon-main-model.nlp'
     const report = {
       can_run: { title: 'Run', type: 'error', v: true },
       can_run_module: { title: 'Run modules', type: 'error', v: true },
@@ -87,13 +88,26 @@ export default () => new Promise(async (resolve, reject) => {
       log.error(`${e}\n`)
     }
 
-    // NLP model checking
+    // Resolvers NLP model checking
 
-    log.info('NLP model state')
-    if (!fs.existsSync(nlpModelPath) || !Object.keys(fs.readFileSync(nlpModelPath)).length) {
+    log.info('Resolvers NLP model state')
+    if (!fs.existsSync(resolversNlpModelPath)
+      || !Object.keys(fs.readFileSync(resolversNlpModelPath)).length) {
       report.can_text.v = false
       Object.keys(report).forEach((item) => { if (item.indexOf('stt') !== -1 || item.indexOf('tts') !== -1) report[item].v = false })
-      log.error('NLP model not found or broken. Try to generate a new one: "npm run train"\n')
+      log.error('Resolvers NLP model not found or broken. Try to generate a new one: "npm run train"\n')
+    } else {
+      log.success('Found and valid\n')
+    }
+
+    // Main NLP model checking
+
+    log.info('Main NLP model state')
+    if (!fs.existsSync(mainNlpModelPath)
+      || !Object.keys(fs.readFileSync(mainNlpModelPath)).length) {
+      report.can_text.v = false
+      Object.keys(report).forEach((item) => { if (item.indexOf('stt') !== -1 || item.indexOf('tts') !== -1) report[item].v = false })
+      log.error('Main NLP model not found or broken. Try to generate a new one: "npm run train"\n')
     } else {
       log.success('Found and valid\n')
     }

scripts/setup/setup.js

@@ -1,7 +1,7 @@
 import loader from '@/helpers/loader'
 import log from '@/helpers/log'
 
-import train from '../train'
+import train from '../train/train'
 import generateHttpApiKey from '../generate/generate-http-api-key'
 import setupDotenv from './setup-dotenv'
 import setupCore from './setup-core'

scripts/run-train.js → scripts/train/run-train.js

@@ -3,7 +3,7 @@ import log from '@/helpers/log'
 import train from './train'
 
 /**
- * Execute the training script
+ * Execute the training scripts
  */
 (async () => {
   try {

scripts/train/train-main-model/train-global-entities.js

@@ -0,0 +1,40 @@
+import path from 'path'
+import fs from 'fs'
+
+import log from '@/helpers/log'
+
+/**
+ * Train global entities
+ */
+export default (lang, nlp) => new Promise((resolve) => {
+  log.title('Global entities training')
+
+  const globalEntitiesPath = path.join(process.cwd(), 'core/data', lang, 'global-entities')
+  const globalEntityFiles = fs.readdirSync(globalEntitiesPath)
+  const newEntitiesObj = { }
+
+  // Add global entities annotations (@...)
+  for (let i = 0; i < globalEntityFiles.length; i += 1) {
+    const globalEntityFileName = globalEntityFiles[i]
+    const [entityName] = globalEntityFileName.split('.')
+    const globalEntityPath = path.join(globalEntitiesPath, globalEntityFileName)
+    const { options } = JSON.parse(fs.readFileSync(globalEntityPath, 'utf8'))
+    const optionKeys = Object.keys(options)
+    const optionsObj = { }
+
+    log.info(`[${lang}] Adding "${entityName}" global entity...`)
+
+    optionKeys.forEach((optionKey) => {
+      const { synonyms } = options[optionKey]
+
+      optionsObj[optionKey] = synonyms
+    })
+
+    newEntitiesObj[entityName] = { options: optionsObj }
+    log.success(`[${lang}] "${entityName}" global entity added`)
+  }
+
+  nlp.addEntities(newEntitiesObj, lang)
+
+  resolve()
+})