Make zap files smaller

apack.json

@@ -4,7 +4,7 @@
   "description": "Graphical configuration tool for application and libraries based on Zigbee Cluster Library.",
   "path": [".", "node_modules/.bin/", "ZAP.app/Contents/MacOS"],
   "requiredFeatureLevel": "apack.core:9",
-  "featureLevel": 95,
+  "featureLevel": 96,
   "uc.triggerExtension": "zap",
   "executable": {
     "zap:win32.x86_64": {

package.json

@@ -88,8 +88,8 @@
     "genmeta2": "node src-script/zap-generate.js --genResultFile --stateDirectory ~/.zap/genmeta --appendGenerationSubdirectory -o ./tmp -i  ./test/resource/test-meta.zap --packageMatch fuzzy",
     "gentest": "node src-script/zap-generate.js --genResultFile --stateDirectory ~/.zap/gentest -z ./zcl-builtin/silabs/zcl.json -g ./test/gen-template/test/gen-test.json -o ./tmp",
     "gendotdot": "node src-script/zap-generate.js --genResultFile -z ./zcl-builtin/dotdot/library.xml -g ./test/gen-template/dotdot/dotdot-templates.json -o ./tmp",
-    "convert": "node src-script/zap-convert.js -o {basename}.zap -z ./zcl-builtin/silabs/zcl.json -g test/gen-template/zigbee/gen-templates.json ./test/resource/isc/*.isc ./test/resource/*.zap",
-    "convert-dl": "node src-script/zap-convert.js -o {basename}.zap -z ./zcl-builtin/silabs/zcl.json -g test/gen-template/zigbee/gen-templates.json ./test/resource/isc/ha-door-lock.isc",
+    "convert": "node src-script/zap-convert.js -o {basename}.zap -z ./zcl-builtin/silabs/zcl.json -g test/gen-template/zigbee/gen-templates.json -i ./test/resource/isc/*.isc ./test/resource/*.zap",
+    "convert-dl": "node src-script/zap-convert.js -o {basename}.zap -z ./zcl-builtin/silabs/zcl.json -g test/gen-template/zigbee/gen-templates.json -i ./test/resource/isc/ha-door-lock.isc",
     "refresh-zap": "node src-script/zap-convert.js -z zcl-builtin/silabs/zcl.json test/resource/three-endpoint-device.zap -g test/gen-template/zigbee/gen-templates.json -o test/resource/three-endpoint-device.zap",
     "publish-linux": "exec electron-builder -l -p always",
     "linuxpack-ui": "./dist/linux-unpacked/zap --zclProperties zcl-builtin/silabs/zcl-test.properties --genTemplateJson test/gen-template/zigbee/gen-templates.json",

src-electron/db/query-command.js

@@ -1176,7 +1176,7 @@ ON
   WHERE PACKAGE_REF IN (${dbApi.toInClause(packageIds)})
 GROUP BY
   COMMAND.COMMAND_ID
-ORDER BY CODE`
+ORDER BY COMMAND.CODE, COMMAND.NAME`
     )
     .then((rows) => rows.map(dbMapping.map.command))
 }

src-electron/importexport/export.js

@@ -27,6 +27,7 @@ const env = require('../util/env')
 const querySession = require('../db/query-session.js')
 const queryImpExp = require('../db/query-impexp.js')
 const dbEnum = require('../../src-shared/db-enum.js')
+const ff = require('./file-format')
 
 async function exportEndpointType(db, endpointType) {
   let data = await queryImpExp.exportClustersFromEndpointType(
@@ -157,12 +158,24 @@ async function exportDataIntoFile(
     createBackup: false,
   }
 ) {
+  let fileFormat = env.defaultFileFormat()
+
   env.logDebug(`Writing state from session ${sessionId} into file ${filePath}`)
   let state = await createStateFromDatabase(db, sessionId)
-  if (options.removeLog) delete state.log
-
+  if (fileFormat > 0) {
+    state.fileFormat = fileFormat
+  } else {
+    delete state.fileFormat
+  }
   // avoid unncessary Studio integration id from being saved in file.
-  state['keyValuePairs'] = state['keyValuePairs'].filter(x => x.key != dbEnum.sessionKey.ideProjectPath)
+  if (state.keyValuePairs) {
+    state.keyValuePairs = state.keyValuePairs.filter(
+      (x) => x.key != dbEnum.sessionKey.ideProjectPath
+    )
+  }
+  state = ff.convertToFile(state)
+
+  if (options.removeLog) delete state.log
 
   if (fs.existsSync(filePath)) {
     fs.copyFileSync(filePath, filePath + '~')
@@ -210,6 +223,7 @@ async function getSessionKeyValues(db, sessionId, excludedKeys) {
  */
 async function createStateFromDatabase(db, sessionId) {
   let state = {
+    fileFormat: 0,
     featureLevel: env.zapVersion().featureLevel,
     creator: 'zap',
   }

src-electron/importexport/file-format.js

@@ -0,0 +1,310 @@
+/**
+ *
+ *    Copyright (c) 2023 Silicon Labs
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License");
+ *    you may not use this file except in compliance with the License.
+ *    You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS,
+ *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *    See the License for the specific language governing permissions and
+ *    limitations under the License.
+ */
+
+const _ = require('lodash')
+const types = require('../util/types')
+
+// Converts attribute storage string to internal representation
+function unpackAttribute(a) {
+  let data
+  if (a.includes('=>')) {
+    data = a.split('=>')[0]
+  } else {
+    data = a
+  }
+  let toks = data.split(' | ').map((x) => x.trim())
+  if (toks.length != 12) throw new Error(`Invalid format: ${a}`)
+
+  let attr = {}
+  attr.included = toks[0] === '+' ? 1 : 0
+  attr.code = types.hexStringToInt(toks[1])
+  if (toks[2].length == 0) {
+    attr.mfgCode = null
+  } else {
+    attr.mfgCode = types.hexStringToInt(toks[2])
+  }
+  attr.side = toks[3]
+  attr.storageOption = toks[4] === 'Ext' ? 'External' : toks[4]
+  attr.singleton = toks[5] === 'singleton' ? 1 : 0
+  attr.bounded = toks[6] === 'bound' ? 1 : 0
+  attr.defaultValue = toks[7]
+  attr.reportable = parseInt(toks[8])
+  attr.minInterval = parseInt(toks[9])
+  attr.maxInterval = parseInt(toks[10])
+  attr.reportableChange = parseInt(toks[11])
+  return attr
+}
+
+// Converts attribute object for internal representation.
+function packAttribute(a) {
+  let data = [
+    a.included === 1 ? '+' : '-',
+    types.intToHexString(a.code, 2),
+    a.mfgCode != null ? types.intToHexString(a.mfgCode, 2) : '      ',
+    a.side,
+    a.storageOption === 'External' ? 'Ext' : a.storageOption,
+    a.singleton ? 'singleton' : '         ',
+    a.bounded ? 'bound' : '     ',
+    a.defaultValue.padStart(20, ' '),
+    a.reportable,
+    a.minInterval,
+    a.maxInterval,
+    a.reportableChange,
+  ].join(' | ')
+  return `${data} => ${a.name} [${a.type}]`
+}
+
+// Converts command storage string to internal representation
+function unpackCommand(c) {
+  let data
+  if (c.includes('=>')) {
+    data = c.split('=>')[0]
+  } else {
+    data = c
+  }
+  let toks = data.split(' | ').map((x) => x.trim())
+  if (toks.length != 5) throw new Error(`Invalid format: ${a}`)
+
+  let cmd = {}
+  cmd.code = types.hexStringToInt(toks[0])
+  if (toks[1].length == 0) {
+    cmd.mfgCode = null
+  } else {
+    cmd.mfgCode = types.hexStringToInt(toks[1])
+  }
+  cmd.source = toks[2]
+  cmd.incoming = toks[3] === '1' ? 1 : 0
+  cmd.outgoing = toks[4] === '1' ? 1 : 0
+  return cmd
+}
+
+// Converts command object for file representation.
+function packCommand(cmd) {
+  let data = [
+    types.intToHexString(cmd.code, 2),
+    cmd.mfgCode != null ? types.intToHexString(cmd.mfgCode, 2) : '      ',
+    cmd.source,
+    cmd.incoming,
+    cmd.outgoing,
+  ].join(' | ')
+  return `${data} => ${cmd.name}`
+}
+
+// Convert string representation to internal object representation
+function unpackEvent(ev) {
+  let data
+  if (ev.includes('=>')) {
+    data = ev.split('=>')[0]
+  } else {
+    data = ev
+  }
+  let toks = data.split(' | ').map((x) => x.trim())
+  if (toks.length != 4) throw new Error(`Invalid format: ${a}`)
+
+  let evnt = {}
+  evnt.included = toks[0] === '+' ? 1 : 0
+  evnt.code = types.hexStringToInt(toks[1])
+  if (toks[2].length == 0) {
+    evnt.mfgCode = null
+  } else {
+    evnt.mfgCode = types.hexStringToInt(toks[2])
+  }
+  evnt.side = toks[3]
+
+  return evnt
+}
+
+// Converts event object for file representation
+function packEvent(ev) {
+  let data = [
+    ev.included === 1 ? '+' : '-',
+    types.intToHexString(ev.code, 2),
+    ev.mfgCode != null ? types.intToHexString(ev.mfgCode, 2) : '      ',
+    ev.side,
+  ].join(' | ')
+  return `${data} => ${ev.name}`
+}
+
+// Converts the key value pairs in the file into internal representation
+function unpackKeyValuePairs(keyValuePairs) {
+  let kvps = []
+  for (let kvp of keyValuePairs) {
+    if (_.isString(kvp)) {
+      let pair = kvp.split('=').map((x) => x.trim())
+      kvps.push({
+        key: pair[0],
+        value: pair[1],
+      })
+    } else {
+      kvps.push(kvp)
+    }
+  }
+  return kvps
+}
+
+// Packs key value pairs for extenrnal representation
+function packKeyValuePairs(keyValuePairs) {
+  let props = []
+  for (let kvp of keyValuePairs) {
+    props.push(`${kvp.key} = ${kvp.value}`)
+  }
+  return props
+}
+
+// Cleanses toplevel cluster data.
+function cleanseCluster(c) {
+  c.code = '0x' + c.code.toString(16).padStart(4, '0')
+  if (c.mfgCode != null) {
+    c.mfgCode = '0x' + c.mfgCode.toString(16).padStart(4, '0')
+  } else {
+    delete c.mfgCode
+  }
+}
+
+// Uncleanses the toplevel cluster data.
+function uncleanseCluster(c) {
+  if (_.isString(c.code)) {
+    let code = c.code
+    if (code.startsWith('0x')) code = code.substring(2)
+    c.code = parseInt(code, 16)
+  }
+
+  if (_.isString(c.mfgCode)) {
+    let code = c.mfgCode
+    if (code.startsWith('0x')) code = code.substring(2)
+    c.mfgCode = parseInt(code, 16)
+  }
+}
+
+/**
+ * This function gets the state from database and converts it for a given file format.
+ *
+ * @param {*} state
+ * @param {*} fileFormat
+ */
+function convertToFile(state) {
+  if (state.fileFormat && state.fileFormat > 0) {
+    // Convert key value pairs
+    if (state.keyValuePairs) {
+      state.keyValuePairs = packKeyValuePairs(state.keyValuePairs)
+    }
+
+    for (let ept of state.endpointTypes) {
+      // Now cleanse the clusters
+      for (let c of ept.clusters) {
+        cleanseCluster(c)
+      }
+
+      for (let c of ept.clusters) {
+        // Now we convert all the attributes...
+        if (c.attributes) {
+          let atts = []
+          for (let a of c.attributes) {
+            atts.push(packAttribute(a))
+          }
+          c.attributes = atts
+        }
+
+        // ... and commands...
+        if (c.commands) {
+          let cmds = []
+          for (let cmd of c.commands) {
+            cmds.push(packCommand(cmd))
+          }
+          c.commands = cmds
+        }
+
+        // ... and events.
+        if (c.events) {
+          let evs = []
+          for (let ev of c.events) {
+            evs.push(packEvent(ev))
+          }
+          c.events = evs
+        }
+      }
+    }
+
+    return state
+  } else {
+    return state
+  }
+}
+
+/**
+ * This function gets the JSON from the file, and converts it to the correct database state
+ */
+function convertFromFile(state) {
+  if (state.fileFormat && state.fileFormat > 0) {
+    // Convert key value pairs
+    if (state.keyValuePairs) {
+      state.keyValuePairs = unpackKeyValuePairs(state.keyValuePairs)
+    }
+
+    for (let ept of state.endpointTypes) {
+      // Now uncleanse the clusters
+      for (let c of ept.clusters) {
+        uncleanseCluster(c)
+
+        // Now we convert all the attributes...
+        if (c.attributes) {
+          let atts = []
+          for (let a of c.attributes) {
+            if (_.isString(a)) {
+              atts.push(unpackAttribute(a))
+            } else {
+              atts.push(a)
+            }
+          }
+          c.attributes = atts
+        }
+
+        // ... and commands.
+        if (c.commands) {
+          let cmds = []
+          for (let cmd of c.commands) {
+            if (_.isString(cmd)) {
+              cmds.push(unpackCommand(cmd))
+            } else {
+              cmds.push(cmd)
+            }
+          }
+          c.commands = cmds
+        }
+
+        // ... and events.
+        if (c.events) {
+          let evs = []
+          for (let ev of c.events) {
+            if (_.isString(ev)) {
+              evs.push(unpackEvent(ev))
+            } else {
+              evs.push(ev)
+            }
+          }
+          c.events = evs
+        }
+      }
+    }
+    return state
+  } else {
+    return state
+  }
+}
+
+exports.convertFromFile = convertFromFile
+exports.convertToFile = convertToFile

src-electron/importexport/import.js

@@ -27,6 +27,7 @@ const querySession = require('../db/query-session.js')
 const env = require('../util/env')
 const script = require('../util/script')
 const dbEnum = require('../../src-shared/db-enum')
+const ff = require('./file-format.js')
 
 /**
  * Reads the data from the file and resolves with the state object if all is good.
@@ -88,6 +89,7 @@ async function importDataFromFile(
   }
 ) {
   let state = await readDataFromFile(filePath, options.defaultZclMetafile)
+  state = ff.convertFromFile(state)
   try {
     await dbApi.dbBeginTransaction(db)
     let sid