feat: add websocket server nodejs extension which depends on third-pa…

…rty packages (#504) * feat: add websocket server nodejs extension which depends on third-party packages * fix: avoid properties overwriten in http server nodejs extension * feat: add test case for mixing python and nodejs * fix: python dependency * fix: cleanup in the final stage of nodejs testing cases * fix: fix asan detect error * feat: add Python websocket extension and test case * fix: refine codes --------- Co-authored-by: Hu Yueh-Wei <[email protected]>
TEN-framework · Jan 8, 2025 · 52d3258 · 52d3258
1 parent 1f81cde
commit 52d3258
Show file tree

Hide file tree

Showing 108 changed files with 2,554 additions and 58 deletions.
diff --git a/.github/tools/setup_pytest_dependencies.py b/.github/tools/setup_pytest_dependencies.py
@@ -99,6 +99,15 @@ def setup():
             "python-dotenv",
         ]
     )
+    utils.run_cmd_with_retry(
+        [
+            sys.executable,
+            "-m",
+            "pip",
+            "install",
+            "websocket-client",
+        ]
+    )
 
 
 if __name__ == "__main__":

diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -673,9 +673,9 @@
       "request": "launch",
       "program": "node",
       "args": [
-        "${workspaceFolder}/out/linux/x64/tests/ten_runtime/integration/nodejs/http_server_nodejs/http_server_nodejs_app/build/start.js"
+        "${workspaceFolder}/out/linux/x64/tests/ten_runtime/integration/nodejs/mix_python_ext_nodejs/mix_python_ext_nodejs_app/build/start.js"
       ],
-      "cwd": "${workspaceFolder}/out/linux/x64/tests/ten_runtime/integration/nodejs/http_server_nodejs/http_server_nodejs_app/",
+      "cwd": "${workspaceFolder}/out/linux/x64/tests/ten_runtime/integration/nodejs/mix_python_ext_nodejs/mix_python_ext_nodejs_app/",
       "env": {
         "LD_PRELOAD": "ten_packages/system/ten_runtime/lib/libasan.so",
         "NODE_PATH": "ten_packages/system/ten_runtime_nodejs/lib:$NODE_PATH",
@@ -689,9 +689,9 @@
       "program": "/usr/bin/node",
       "args": [
         "--expose-gc",
-        "${workspaceFolder}/out/linux/x64/tests/ten_runtime/integration/nodejs/http_server_nodejs/http_server_nodejs_app/build/start.js"
+        "${workspaceFolder}/out/linux/x64/tests/ten_runtime/integration/nodejs/mix_python_ext_nodejs/mix_python_ext_nodejs_app/build/start.js"
       ],
-      "cwd": "${workspaceFolder}/out/linux/x64/tests/ten_runtime/integration/nodejs/http_server_nodejs/http_server_nodejs_app/",
+      "cwd": "${workspaceFolder}/out/linux/x64/tests/ten_runtime/integration/nodejs/mix_python_ext_nodejs/mix_python_ext_nodejs_app/",
       "environment": [
         {
           "name": "LD_PRELOAD",
@@ -708,6 +708,10 @@
         {
           "name": "LSAN_OPTIONS",
           "value": "verbosity=1:log_threads=1"
+        },
+        {
+          "name": "TEN_ENABLE_MEMORY_TRACKING",
+          "value": "true"
         }
       ],
       "sourceFileMap": {

diff --git a/build/ten_runtime/feature/build_pkg.py b/build/ten_runtime/feature/build_pkg.py
@@ -208,6 +208,11 @@ def build_nodejs_extensions(app_root_path: str) -> int:
                 # Change to extension directory.
                 os.chdir(extension_path)
 
+                status = npm_install()
+                if status != 0:
+                    print(f"Failed to npm install in {extension_path}")
+                    return 1
+
                 now = datetime.now()
                 cmd = ["npm", "run", "build"]
                 returncode, output = cmd_exec.run_cmd(cmd)

diff --git a/packages/example_extensions/BUILD.gn b/packages/example_extensions/BUILD.gn
@@ -25,5 +25,12 @@ group("example_extensions") {
         "pil_demo_python",
       ]
     }
+
+    if (ten_enable_nodejs_binding) {
+      deps += [
+        "http_server_extension_nodejs",
+        "websocket_server_nodejs",
+      ]
+    }
   }
 }
diff --git a/packages/example_extensions/http_server_extension_nodejs/src/index.ts b/packages/example_extensions/http_server_extension_nodejs/src/index.ts
@@ -81,9 +81,9 @@ class HttpServerExtension extends Extension {
         } else if ("name" in _ten) {
           const name = _ten["name"];
           const cmd = Cmd.Create(name);
+          cmd.setPropertyFromJson("", body);
           cmd.setPropertyString("method", req.method!);
           cmd.setPropertyString("url", req.url!);
-          cmd.setPropertyFromJson("", body);
 
           this.tenEnv!.sendCmd(cmd).then(
             ([cmdResult, error]: [CmdResult | null, Error | null]) => {

diff --git a/packages/example_extensions/pil_demo_python/__init__.py b/packages/example_extensions/pil_demo_python/__init__.py
@@ -1,3 +1,8 @@
+#
+# This file is part of TEN Framework, an open source project.
+# Licensed under the Apache License, Version 2.0.
+# See the LICENSE file for more information.
+#
 from . import main
 
 print("pil demo extension loaded")
diff --git a/packages/example_extensions/simple_echo_python/BUILD.gn b/packages/example_extensions/simple_echo_python/BUILD.gn
@@ -0,0 +1,32 @@
+#
+# This file is part of TEN Framework, an open source project.
+# Licensed under the Apache License, Version 2.0.
+# See the LICENSE file for more information.
+#
+import("//build/feature/ten_package.gni")
+import("//build/ten_runtime/feature/publish.gni")
+import("//build/ten_runtime/glob.gni")
+import("//build/ten_runtime/options.gni")
+
+ten_package("simple_echo_python") {
+  package_kind = "extension"
+
+  resources = [
+    "__init__.py",
+    "addon.py",
+    "extension.py",
+    "manifest.json",
+    "property.json",
+    "requirements.txt",
+  ]
+
+  deps = [ "//core/src/ten_runtime" ]
+}
+
+if (ten_enable_ten_manager) {
+  ten_package_publish("upload_simple_echo_python_to_server") {
+    base_dir =
+        rebase_path("${root_out_dir}/ten_packages/extension/simple_echo_python")
+    deps = [ ":simple_echo_python" ]
+  }
+}
diff --git a/packages/example_extensions/simple_echo_python/LICENSE b/packages/example_extensions/simple_echo_python/LICENSE
@@ -0,0 +1,13 @@
+Copyright © 2025 Agora
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
diff --git a/packages/example_extensions/simple_echo_python/__init__.py b/packages/example_extensions/simple_echo_python/__init__.py
@@ -0,0 +1,6 @@
+#
+# This file is part of TEN Framework, an open source project.
+# Licensed under the Apache License, Version 2.0.
+# See the LICENSE file for more information.
+#
+from . import addon
diff --git a/packages/example_extensions/simple_echo_python/addon.py b/packages/example_extensions/simple_echo_python/addon.py
@@ -0,0 +1,17 @@
+#
+# This file is part of TEN Framework, an open source project.
+# Licensed under the Apache License, Version 2.0.
+# See the LICENSE file for more information.
+#
+from ten import (
+    Addon,
+    register_addon_as_extension,
+    TenEnv,
+)
+from .extension import SimpleEchoExtension
+
+
+@register_addon_as_extension("simple_echo_python")
+class SimpleEchoExtensionAddon(Addon):
+    def on_create_instance(self, ten_env: TenEnv, name: str, context) -> None:
+        ten_env.on_create_instance_done(SimpleEchoExtension(name), context)
diff --git a/packages/example_extensions/simple_echo_python/extension.py b/packages/example_extensions/simple_echo_python/extension.py
@@ -0,0 +1,102 @@
+#
+# This file is part of TEN Framework, an open source project.
+# Licensed under the Apache License, Version 2.0.
+# See the LICENSE file for more information.
+#
+from ten import (
+    AudioFrame,
+    VideoFrame,
+    AsyncExtension,
+    AsyncTenEnv,
+    Cmd,
+    StatusCode,
+    CmdResult,
+    Data,
+)
+
+
+class SimpleEchoExtension(AsyncExtension):
+    async def on_init(self, ten_env: AsyncTenEnv) -> None:
+        ten_env.log_debug("on_init")
+
+    async def on_start(self, ten_env: AsyncTenEnv) -> None:
+        ten_env.log_debug("on_start")
+
+        # TODO: read properties, initialize resources
+
+    async def on_stop(self, ten_env: AsyncTenEnv) -> None:
+        ten_env.log_debug("on_stop")
+
+        # TODO: clean up resources
+
+    async def on_deinit(self, ten_env: AsyncTenEnv) -> None:
+        ten_env.log_debug("on_deinit")
+
+    async def on_cmd(self, ten_env: AsyncTenEnv, cmd: Cmd) -> None:
+        cmd_name = cmd.get_name()
+        ten_env.log_debug("on_cmd name {}".format(cmd_name))
+
+        cmd_result = CmdResult.create(StatusCode.OK)
+        cmd_result.set_property_string("detail", cmd_name + ", too")
+
+        await ten_env.return_result(cmd_result, cmd)
+
+    async def on_data(self, ten_env: AsyncTenEnv, data: Data) -> None:
+        data_name = data.get_name()
+        ten_env.log_debug("on_data name {}".format(data_name))
+
+        buf = data.get_buf()
+
+        new_data = Data.create(data_name)
+        new_data.alloc_buf(len(buf))
+        new_buf = new_data.lock_buf()
+        new_buf[:] = buf
+        new_data.unlock_buf(new_buf)
+
+        await ten_env.send_data(new_data)
+
+    async def on_audio_frame(
+        self, ten_env: AsyncTenEnv, audio_frame: AudioFrame
+    ) -> None:
+        audio_frame_name = audio_frame.get_name()
+        ten_env.log_debug("on_audio_frame name {}".format(audio_frame_name))
+
+        buf = audio_frame.get_buf()
+        new_audio_frame = AudioFrame.create(audio_frame_name)
+        new_audio_frame.alloc_buf(len(buf))
+        new_buf = new_audio_frame.lock_buf()
+        new_buf[:] = buf
+        new_audio_frame.unlock_buf(new_buf)
+
+        new_audio_frame.set_bytes_per_sample(audio_frame.get_bytes_per_sample())
+        new_audio_frame.set_sample_rate(audio_frame.get_sample_rate())
+        new_audio_frame.set_data_fmt(audio_frame.get_data_fmt())
+        new_audio_frame.set_eof(audio_frame.is_eof())
+        new_audio_frame.set_line_size(audio_frame.get_line_size())
+        new_audio_frame.set_number_of_channels(
+            audio_frame.get_number_of_channels()
+        )
+        new_audio_frame.set_timestamp(audio_frame.get_timestamp())
+
+        await ten_env.send_audio_frame(new_audio_frame)
+
+    async def on_video_frame(
+        self, ten_env: AsyncTenEnv, video_frame: VideoFrame
+    ) -> None:
+        video_frame_name = video_frame.get_name()
+        ten_env.log_debug("on_video_frame name {}".format(video_frame_name))
+
+        buf = video_frame.get_buf()
+        new_video_frame = VideoFrame.create(video_frame_name)
+        new_video_frame.alloc_buf(len(buf))
+        new_buf = new_video_frame.lock_buf()
+        new_buf[:] = buf
+        new_video_frame.unlock_buf(new_buf)
+
+        new_video_frame.set_eof(video_frame.is_eof())
+        new_video_frame.set_height(video_frame.get_height())
+        new_video_frame.set_width(video_frame.get_width())
+        new_video_frame.set_timestamp(video_frame.get_timestamp())
+        new_video_frame.set_pixel_fmt(video_frame.get_pixel_fmt())
+
+        await ten_env.send_video_frame(new_video_frame)
diff --git a/packages/example_extensions/simple_echo_python/manifest.json b/packages/example_extensions/simple_echo_python/manifest.json
@@ -0,0 +1,13 @@
+{
+  "type": "extension",
+  "name": "simple_echo_python",
+  "version": "0.1.0",
+  "dependencies": [
+    {
+      "type": "system",
+      "name": "ten_runtime_python",
+      "version": "0.6.0"
+    }
+  ],
+  "api": {}
+}
diff --git a/packages/example_extensions/simple_echo_python/property.json b/packages/example_extensions/simple_echo_python/property.json
@@ -0,0 +1 @@
+{}
diff --git a/packages/example_extensions/simple_echo_python/requirements.txt b/packages/example_extensions/simple_echo_python/requirements.txt
diff --git a/packages/example_extensions/websocket_server_nodejs/BUILD.gn b/packages/example_extensions/websocket_server_nodejs/BUILD.gn
@@ -0,0 +1,32 @@
+#
+# This file is part of TEN Framework, an open source project.
+# Licensed under the Apache License, Version 2.0.
+# See the LICENSE file for more information.
+#
+import("//build/feature/ten_package.gni")
+import("//build/ten_runtime/feature/publish.gni")
+import("//build/ten_runtime/glob.gni")
+import("//build/ten_runtime/options.gni")
+
+ten_package("websocket_server_nodejs") {
+  package_kind = "extension"
+
+  resources = [
+    "BUILD.gn",
+    "manifest.json",
+    "package.json",
+    "property.json",
+    "src/index.ts",
+    "tsconfig.json",
+  ]
+
+  deps = [ "//core/src/ten_runtime" ]
+}
+
+if (ten_enable_ten_manager) {
+  ten_package_publish("upload_websocket_server_nodejs_to_server") {
+    base_dir = rebase_path(
+            "${root_out_dir}/ten_packages/extension/websocket_server_nodejs")
+    deps = [ ":websocket_server_nodejs" ]
+  }
+}
diff --git a/packages/example_extensions/websocket_server_nodejs/LICENSE b/packages/example_extensions/websocket_server_nodejs/LICENSE
@@ -0,0 +1,13 @@
+Copyright © 2025 Agora
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
diff --git a/packages/example_extensions/websocket_server_nodejs/manifest.json b/packages/example_extensions/websocket_server_nodejs/manifest.json
@@ -0,0 +1,23 @@
+{
+  "type": "extension",
+  "name": "websocket_server_nodejs",
+  "version": "0.6.0",
+  "dependencies": [
+    {
+      "type": "system",
+      "name": "ten_runtime_nodejs",
+      "version": "0.6.0"
+    }
+  ],
+  "package": {
+    "include": [
+      "manifest.json",
+      "property.json",
+      "BUILD.gn",
+      "src/**",
+      "tsconfig.json",
+      "package.json"
+    ]
+  },
+  "api": {}
+}