Update the .zap file when Studio project updates to a new GSDK (#1431)

- Update the uc_upgrade target for apack.json
- Add the upgrade zap file function which updates the top level json packages for the .zap file based on the gsdk being used for the project.
- The code goes through a temporary directory where .zap files are stored for Studio and updates all the .zap files in that directory
- The top level upgrade packages are matched based on type and category
- Other minor cleanup
- Cleaning up the directory flag to -d and adding the no load failure flag when packages are not present
- Adding unit tests for the upgrade process
- JIRA: ZAPP-1362

apack.json

@@ -64,7 +64,7 @@
       "cmd": "$(zap-cli) generate --noUi --noServer -o ${generationOutput} --packageMatch fuzzy --zcl ${sdkRoot}/app/zcl/zcl-zap.json --zcl ${sdkRoot}/extension/matter_extension/src/app/zap-templates/zcl/zcl.json --generationTemplate ${sdkRoot}/protocol/zigbee/app/framework/gen-template/gen-templates.json --generationTemplate ${sdkRoot}/extension/matter_extension/src/app/zap-templates/app-templates.json --in ${contentFolder} --noLoadingFailure --appendGenerationSubdirectory"
     },
     "uc_upgrade": {
-      "cmd": "$(zap-cli) convert  --results ${results} --noop"
+      "cmd": "$(zap-cli) upgrade  --results ${results} -d ${tempContentFolder} --zcl ${sdkRoot}/app/zcl/zcl-zap.json --zcl ${sdkRoot}/extension/matter_extension/src/app/zap-templates/zcl/zcl.json --generationTemplate ${sdkRoot}/protocol/zigbee/app/framework/gen-template/gen-templates.json --generationTemplate ${sdkRoot}/extension/matter_extension/src/app/zap-templates/app-templates.json --noLoadingFailure"
     },
     "zapHelp": {
       "cmd": "$(zap) --help"

src-electron/db/query-package.js

@@ -605,7 +605,8 @@ SELECT
   SP.PACKAGE_REF,
   SESSION_PARTITION.SESSION_REF,
   SP.REQUIRED,
-  P.TYPE
+  P.TYPE,
+  SESSION_PARTITION.SESSION_PARTITION_ID
 FROM
   SESSION_PARTITION
 INNER JOIN

src-electron/importexport/import-json.js

@@ -307,7 +307,7 @@ async function importSinglePackage(
     let packageNameMatch = packages.find(
       (p) => p.path.includes(filePathToSearch) && fs.existsSync(p.path)
     )
-    if (packageMatch) {
+    if (packageMatch && packageNameMatch) {
       p = packageNameMatch
       env.logError(
         `None of packages exist for ${pkg.path}, so using one which matches the file name: ${p.path} from ${pkgPaths}.`
@@ -1761,7 +1761,9 @@ async function jsonDataLoader(
   sessionId,
   packageMatch,
   defaultZclMetafile,
-  defaultTemplateFile
+  defaultTemplateFile,
+  upgradeZclPackages,
+  upgradeTemplatePackages
 ) {
   // Initially clean up all the packages from the session.
   let sessionPartitions =
@@ -1806,6 +1808,21 @@ async function jsonDataLoader(
 
   mainPackageData.sessionId = sessionId
 
+  // Updating main packages
+  if (
+    upgradeZclPackages &&
+    mainPackageData.zclPackageIds.length == upgradeZclPackages.length
+  ) {
+    mainPackageData.zclPackageIds = upgradeZclPackages.map((pkg) => pkg.id)
+  }
+
+  if (
+    upgradeTemplatePackages &&
+    mainPackageData.templateIds.length == upgradeTemplatePackages.length
+  ) {
+    mainPackageData.templateIds = upgradeTemplatePackages.map((pkg) => pkg.id)
+  }
+
   let mainPackagePromise = []
   for (let i = 0; i < mainPackageData.zclPackageIds.length; i++) {
     mainPackagePromise.push(
@@ -1972,7 +1989,8 @@ async function jsonDataLoader(
   await Promise.all(promisesStage2)
   await querySession.setSessionClean(db, sessionId)
 
-  if ('package' in state) {
+  // No need to go through this when upgrading packages.
+  if ('package' in state && !upgradeZclPackages && !upgradeTemplatePackages) {
     await Promise.all(
       state.package.map(async (pkg) => {
         let pkgFilePath = getPkgPath(pkg, state.filePath)
@@ -2055,16 +2073,9 @@ function cleanJsonState(state) {
  *
  * @param {*} filePath
  * @param {*} data
- * @param {*} defaultZclMetafile
- * @param {*} defaultTemplateFile
  * @returns Promise of parsed JSON object
  */
-async function readJsonData(
-  filePath,
-  data,
-  defaultZclMetafile,
-  defaultTemplateFile
-) {
+async function readJsonData(filePath, data) {
   let state = JSON.parse(data)
 
   cleanJsonState(state)

src-electron/importexport/import.js

@@ -38,24 +38,14 @@ const util = require('../util/util.js')
  *
  * @param {*} filePath
  * @param {*} defaultZclMetafile
- * @param {*} defaultTemplateFile
  * @returns Promise of file reading.
  */
-async function readDataFromFile(
-  filePath,
-  defaultZclMetafile,
-  defaultTemplateFile
-) {
+async function readDataFromFile(filePath, defaultZclMetafile) {
   let data = await fsp.readFile(filePath)
 
   let stringData = data.toString().trim()
   if (stringData.startsWith('{')) {
-    return importJson.readJsonData(
-      filePath,
-      data,
-      defaultZclMetafile,
-      defaultTemplateFile
-    )
+    return importJson.readJsonData(filePath, data)
   } else if (stringData.startsWith('#ISD')) {
     return importIsc.readIscData(
       filePath,
@@ -110,11 +100,8 @@ async function importDataFromFile(
     packageMatch: dbEnum.packageMatch.fuzzy
   }
 ) {
-  let state = await readDataFromFile(
-    filePath,
-    options.defaultZclMetafile,
-    options.defaultTemplateFile
-  )
+  let state = await readDataFromFile(filePath, options.defaultZclMetafile)
+
   state = ff.convertFromFile(state)
   try {
     await dbApi.dbBeginTransaction(db)
@@ -126,7 +113,9 @@ async function importDataFromFile(
         sid,
         {
           zcl: env.builtinSilabsZclMetafile(),
-          template: env.builtinTemplateMetafile()
+          template: env.builtinTemplateMetafile(),
+          upgradeZclPackages: options.upgradeZclPackages,
+          upgradeTemplatePackages: options.upgradeTemplatePackages
         },
         null,
         null
@@ -162,7 +151,9 @@ async function importDataFromFile(
       sid,
       options.packageMatch,
       options.defaultZclMetafile,
-      options.defaultTemplateFile
+      options.defaultTemplateFile,
+      options.upgradeZclPackages,
+      options.upgradeTemplatePackages
     )
     if (options.postImportScript != null) {
       await executePostImportScript(

src-electron/main-process/startup.js

@@ -37,12 +37,12 @@ const ipcServer = require('../server/ipc-server')
 const ipcClient = require('../client/ipc-client')
 const generatorEngine = require('../generator/generation-engine.js')
 const querySession = require('../db/query-session.js')
+const queryPackage = require('../db/query-package.js')
 const util = require('../util/util.js')
 const importJs = require('../importexport/import.js')
 const exportJs = require('../importexport/export.js')
 const watchdog = require('./watchdog')
 const sdkUtil = require('../util/sdk-util')
-const { argv } = require('process')
 
 let mainDatabase = null
 
@@ -192,6 +192,173 @@ async function noopConvert(resultsFile, logger) {
   }
 }
 
+/**
+ * Find all zap files in a given directory
+ * @param {*} dir
+ * @returns all .zap files in the given directory
+ */
+function findZapFiles(dir) {
+  let zapFiles = []
+
+  // Read all items in the directory
+  const items = fs.readdirSync(dir)
+
+  // Loop through each item
+  items.forEach((item) => {
+    const itemPath = path.join(dir, item)
+    const stats = fs.statSync(itemPath)
+
+    // If it's a directory, search recursively
+    if (stats.isDirectory()) {
+      zapFiles = zapFiles.concat(findZapFiles(itemPath))
+    }
+
+    // If it's a file and has .zap extension, add to the list
+    if (stats.isFile() && path.extname(item) === '.zap') {
+      zapFiles.push(itemPath)
+    }
+  })
+
+  return zapFiles
+}
+
+/**
+ * Go over the zap file's top level packages and see if they can be upgraded
+ * based on the upgrade packages given.
+ *
+ * @param {*} db
+ * @param {*} upgradePackages
+ * @param {*} zapFilePackages
+ * @param {*} packageType
+ * @returns list of packages
+ */
+async function getUpgradePackageMatch(
+  db,
+  upgradePackages,
+  zapFilePackages,
+  packageType
+) {
+  let matchedUpgradePackages = []
+  if (Array.isArray(upgradePackages) && Array.isArray(zapFilePackages)) {
+    for (let i = 0; i < upgradePackages.length; i++) {
+      let upgradePackage = await queryPackage.getPackageByPathAndType(
+        db,
+        upgradePackages[i],
+        packageType
+      )
+      if (upgradePackage) {
+        for (let j = 0; j < zapFilePackages.length; j++) {
+          if (
+            zapFilePackages[j].category == upgradePackage.category &&
+            zapFilePackages[j].type == upgradePackage.type
+          ) {
+            matchedUpgradePackages.push(upgradePackage)
+          }
+        }
+      }
+    }
+  }
+  return matchedUpgradePackages
+}
+
+/**
+ * Upgrade the top level packages(.json files) of a .zap file when a gsdk is
+ * updated.
+ *
+ * @param {*} argv
+ * @param {*} options
+ */
+async function upgradeZapFile(argv, options) {
+  let zapFiles = findZapFiles(argv.d)
+  let upgrade_results = argv.results
+  for (let i = 0; i < zapFiles.length; i++) {
+    let zapFile = zapFiles[i]
+    options.logger(`🤖 Update started for file: ${zapFile}`)
+    let dbFile = env.sqliteFile('upgrade')
+    let db = await dbApi.initDatabaseAndLoadSchema(
+      dbFile,
+      env.schemaFile(),
+      env.zapVersion()
+    )
+    options.logger('    🐝 database and schema initialized')
+    await zclLoader.loadZclMetafiles(db, argv.zclProperties, {
+      failOnLoadingError: !argv.noLoadingFailure
+    })
+    options.logger(`    🐝 New zcl package loaded: ${argv.zclProperties}`)
+    if (argv.generationTemplate != null) {
+      let ctx = await generatorEngine.loadTemplates(
+        db,
+        argv.generationTemplate,
+        {
+          failOnLoadingError: !argv.noLoadingFailure
+        }
+      )
+      if (ctx.error) {
+        throw ctx.error
+      }
+      options.logger(`    🐝 New templates loaded: ${argv.generationTemplate}`)
+    }
+    let state = await importJs.readDataFromFile(zapFile)
+    let upgradeZclPackages = await getUpgradePackageMatch(
+      db,
+      argv.zclProperties,
+      state.package,
+      dbEnum.packageType.zclProperties
+    )
+    let upgradeTemplatePackages = await getUpgradePackageMatch(
+      db,
+      argv.generationTemplate,
+      state.package,
+      dbEnum.packageType.genTemplatesJson
+    )
+
+    let importResult = await importJs.importDataFromFile(db, zapFile, {
+      defaultZclMetafile: argv.zclProperties,
+      postImportScript: argv.postImportScript,
+      packageMatch: argv.packageMatch,
+      upgradeZclPackages: upgradeZclPackages,
+      upgradeTemplatePackages: upgradeTemplatePackages
+    })
+    let sessionId = importResult.sessionId
+    await util.ensurePackagesAndPopulateSessionOptions(db, sessionId, {
+      zcl: argv.zclProperties,
+      template: argv.generationTemplate,
+      upgradeZclPackages: upgradeZclPackages,
+      upgradeTemplatePackages: upgradeTemplatePackages
+    })
+    options.logger(`    👈 read in: ${zapFile}`)
+    let of = outputFile(zapFile, zapFile)
+    let parent = path.dirname(of)
+    if (!fs.existsSync(parent)) {
+      fs.mkdirSync(parent, { recursive: true })
+    }
+    // Now we need to write the sessionKey for the file path
+    await querySession.updateSessionKeyValue(
+      db,
+      sessionId,
+      dbEnum.sessionKey.filePath,
+      of
+    )
+    let outputPath = await exportJs.exportDataIntoFile(db, sessionId, of, {
+      removeLog: argv.noZapFileLog,
+      createBackup: true,
+      fileFormat: argv.saveFileFormat
+    })
+    options.logger(`    👉 write out: ${outputPath}`)
+  }
+  try {
+    if (upgrade_results != null)
+      await writeConversionResultsFile(upgrade_results)
+    options.logger(`    👉 write out: ${upgrade_results}`)
+  } catch (error) {
+    options.logger(`    ⚠️  failed to write out: ${upgrade_results}`)
+  }
+  options.logger('😎 Upgrade done!')
+  if (options.quitFunction != null) {
+    options.quitFunction()
+  }
+}
+
 /**
  * Perform file conversion.
  *
@@ -868,6 +1035,14 @@ async function startUpMainInstance(argv, callbacks) {
       console.log(code)
       cleanExit(argv.cleanupDelay, 0)
     })
+  } else if (argv._.includes('upgrade')) {
+    return upgradeZapFile(argv, {
+      logger: console.log,
+      quitFunction: quitFunction
+    }).catch((code) => {
+      console.log(code)
+      cleanExit(argv.cleanupDelay, 0)
+    })
   } else if (argv._.includes('stop')) {
     console.log('No server running, nothing to stop.')
     cleanExit(argv.cleanupDelay, 0)
@@ -930,3 +1105,4 @@ exports.startUpSecondaryInstance = startUpSecondaryInstance
 exports.shutdown = shutdown
 exports.quit = quit
 exports.generateSingleFile = generateSingleFile
+exports.upgradeZapFile = upgradeZapFile