feat: Live loading of external JS (converters/extensions) (#24764)

* feat: Live loading of external JS (converters/extensions) * Fix imports * Improve error message on MQTT save * Handle non-existing base path * Throw on bad converter * Add tests * Fix use of ext conv in network map tests. * More coverage. * Dont mock zhc for basics, tests actual live loading * Update * feat: Live loading of external JS (converters/extensions) * Fix imports * Improve error message on MQTT save * Handle non-existing base path * Throw on bad converter * Add tests * Fix use of ext conv in network map tests. * More coverage. * Dont mock zhc for basics, tests actual live loading * Update * Fix rebase * Fix * Bump zhc * pretty * fix typing * Cleanup `external_converters` setting remnants. --------- Co-authored-by: Koen Kanters <[email protected]>
Koenkk · Dec 1, 2024 · 9907005 · 9907005
1 parent cc61f67
commit 9907005
Show file tree

Hide file tree

Showing 22 changed files with 750 additions and 477 deletions.
diff --git a/lib/controller.ts b/lib/controller.ts
@@ -15,7 +15,7 @@ import ExtensionBind from './extension/bind';
 import ExtensionBridge from './extension/bridge';
 import ExtensionConfigure from './extension/configure';
 import ExtensionExternalConverters from './extension/externalConverters';
-import ExtensionExternalExtension from './extension/externalExtension';
+import ExtensionExternalExtensions from './extension/externalExtensions';
 // Extensions
 import ExtensionFrontend from './extension/frontend';
 import ExtensionGroups from './extension/groups';
@@ -47,7 +47,7 @@ const AllExtensions = [
     ExtensionOTAUpdate,
     ExtensionExternalConverters,
     ExtensionFrontend,
-    ExtensionExternalExtension,
+    ExtensionExternalExtensions,
     ExtensionAvailability,
 ];
 
@@ -106,18 +106,15 @@ export class Controller {
             new ExtensionGroups(...this.extensionArgs),
             new ExtensionBind(...this.extensionArgs),
             new ExtensionOTAUpdate(...this.extensionArgs),
-            new ExtensionExternalExtension(...this.extensionArgs),
+            new ExtensionExternalExtensions(...this.extensionArgs),
+            new ExtensionExternalConverters(...this.extensionArgs),
             new ExtensionAvailability(...this.extensionArgs),
         ];
 
         if (settings.get().frontend) {
             this.extensions.push(new ExtensionFrontend(...this.extensionArgs));
         }
 
-        if (settings.get().external_converters.length) {
-            this.extensions.push(new ExtensionExternalConverters(...this.extensionArgs));
-        }
-
         if (settings.get().homeassistant) {
             this.extensions.push(new ExtensionHomeAssistant(...this.extensionArgs));
         }

diff --git a/lib/extension/externalConverters.ts b/lib/extension/externalConverters.ts
@@ -1,11 +1,13 @@
-import * as zhc from 'zigbee-herdsman-converters';
+import type * as zhc from 'zigbee-herdsman-converters';
+
+import {addDefinition, removeExternalDefinitions} from 'zigbee-herdsman-converters';
 
 import logger from '../util/logger';
-import * as settings from '../util/settings';
-import {loadExternalConverter} from '../util/utils';
-import Extension from './extension';
+import ExternalJSExtension from './externalJS';
+
+type ModuleExports = zhc.Definition | zhc.Definition[];
 
-export default class ExternalConverters extends Extension {
+export default class ExternalConverters extends ExternalJSExtension<ModuleExports> {
     constructor(
         zigbee: Zigbee,
         mqtt: MQTT,
@@ -16,25 +18,51 @@ export default class ExternalConverters extends Extension {
         restartCallback: () => Promise<void>,
         addExtension: (extension: Extension) => Promise<void>,
     ) {
-        super(zigbee, mqtt, state, publishEntityState, eventBus, enableDisableExtension, restartCallback, addExtension);
-
-        for (const file of settings.get().external_converters) {
-            try {
-                for (const definition of loadExternalConverter(file)) {
-                    zhc.addDefinition(definition);
-                }
-                logger.info(`Loaded external converter '${file}'`);
-            } catch (error) {
-                logger.error(`Failed to load external converter file '${file}' (${(error as Error).message})`);
-                logger.error(
-                    `Probably there is a syntax error in the file or the external converter is not ` +
-                        `compatible with the current Zigbee2MQTT version`,
-                );
-                logger.error(
-                    `Note that external converters are not meant for long term usage, it's meant for local ` +
-                        `testing after which a pull request should be created to add out-of-the-box support for the device`,
-                );
+        super(
+            zigbee,
+            mqtt,
+            state,
+            publishEntityState,
+            eventBus,
+            enableDisableExtension,
+            restartCallback,
+            addExtension,
+            'converter',
+            'external_converters',
+        );
+    }
+
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    protected async removeJS(name: string, module: ModuleExports): Promise<void> {
+        removeExternalDefinitions(name);
+
+        await this.zigbee.resolveDevicesDefinitions(true);
+    }
+
+    protected async loadJS(name: string, module: ModuleExports): Promise<void> {
+        try {
+            removeExternalDefinitions(name);
+
+            for (const definition of this.getDefinitions(module)) {
+                definition.externalConverterName = name;
+
+                addDefinition(definition);
+                logger.info(`Loaded external converter '${name}'.`);
             }
+
+            await this.zigbee.resolveDevicesDefinitions(true);
+        } catch (error) {
+            logger.error(`Failed to load external converter '${name}'`);
+            logger.error(`Check the code for syntax error and make sure it is up to date with the current Zigbee2MQTT version.`);
+            logger.error(
+                `External converters are not meant for long term usage, but for local testing after which a pull request should be created to add out-of-the-box support for the device`,
+            );
+
+            throw error;
         }
     }
+
+    private getDefinitions(module: ModuleExports): zhc.Definition[] {
+        return Array.isArray(module) ? module : [module];
+    }
 }
diff --git a/lib/extension/externalExtension.ts b/lib/extension/externalExtension.ts
diff --git a/lib/extension/externalExtensions.ts b/lib/extension/externalExtensions.ts
@@ -0,0 +1,59 @@
+import type Extension from './extension';
+
+import logger from '../util/logger';
+import * as settings from '../util/settings';
+import ExternalJSExtension from './externalJS';
+
+type ModuleExports = typeof Extension;
+
+export default class ExternalExtensions extends ExternalJSExtension<ModuleExports> {
+    constructor(
+        zigbee: Zigbee,
+        mqtt: MQTT,
+        state: State,
+        publishEntityState: PublishEntityState,
+        eventBus: EventBus,
+        enableDisableExtension: (enable: boolean, name: string) => Promise<void>,
+        restartCallback: () => Promise<void>,
+        addExtension: (extension: Extension) => Promise<void>,
+    ) {
+        super(
+            zigbee,
+            mqtt,
+            state,
+            publishEntityState,
+            eventBus,
+            enableDisableExtension,
+            restartCallback,
+            addExtension,
+            'extension',
+            'external_extensions',
+        );
+    }
+
+    protected async removeJS(name: string, module: ModuleExports): Promise<void> {
+        await this.enableDisableExtension(false, module.name);
+    }
+
+    protected async loadJS(name: string, module: ModuleExports): Promise<void> {
+        // stop if already started
+        await this.enableDisableExtension(false, module.name);
+        await this.addExtension(
+            // @ts-expect-error `module` is the interface, not the actual passed class
+            new module(
+                this.zigbee,
+                this.mqtt,
+                this.state,
+                this.publishEntityState,
+                this.eventBus,
+                this.enableDisableExtension,
+                this.restartCallback,
+                this.addExtension,
+                settings,
+                logger,
+            ),
+        );
+
+        logger.info(`Loaded external extension '${name}'.`);
+    }
+}