From ba04161d4a48a2bbda1a96936941ed6eb6899d31 Mon Sep 17 00:00:00 2001 From: Edison Fernandez Date: Tue, 9 Jul 2024 17:30:57 -0600 Subject: [PATCH] First version --- README.md | 6 +- config/agent/api_mapping.json | 21 ++++ config/agent/prompt.txt | 58 ++++++++++ config/analytics/configuration.json | 14 +++ config/demo-nginx.conf | 52 +++++++++ config/vst/vst_config.json | 172 ++++++++++++++++++++++++++++ config/vst/vst_storage.json | 5 + docker-compose.yaml | 98 ++++++++++++++++ 8 files changed, 425 insertions(+), 1 deletion(-) create mode 100644 config/agent/api_mapping.json create mode 100644 config/agent/prompt.txt create mode 100644 config/analytics/configuration.json create mode 100644 config/demo-nginx.conf create mode 100644 config/vst/vst_config.json create mode 100644 config/vst/vst_storage.json create mode 100644 docker-compose.yaml diff --git a/README.md b/README.md index 5a718c6..f938000 100644 --- a/README.md +++ b/README.md @@ -1 +1,5 @@ -# smart-seek-360 \ No newline at end of file +# Smart Seek 360 + +The SmartSeek360 searches for objects specified by the user in a 360-degree video stream, once the object is found the system follows it through the video leveraging the PTZ functions, and also triggers a video recording. + +For a more complete description and usage instructions visit [the project page](https://developer.ridgerun.com/wiki/index.php/Metropolis_Microservices/RidgeRun_Microservices_Demo). diff --git a/config/agent/api_mapping.json b/config/agent/api_mapping.json new file mode 100644 index 0000000..899bb3e --- /dev/null +++ b/config/agent/api_mapping.json @@ -0,0 +1,21 @@ +{ + "search_object": { + "ip": "127.0.0.1", + "port": 5030, + "path": "search", + "method": "GET", + "properties": { + "objects": "input", + "thresholds": 0.4 + } + }, + "move_camera": { + "port": 5020, + "path": "position", + "method": "PUT", + "body": { + "pan": "pan_angle", + "tilt": 0 + } + } +} diff --git a/config/agent/prompt.txt b/config/agent/prompt.txt new file mode 100644 index 0000000..a8b3f4c --- /dev/null +++ b/config/agent/prompt.txt @@ -0,0 +1,58 @@ +[INST] You have access to the following functions. Use them if required:\n\n +[ + { + "type": "function", + "function": { + "name": "search_object", + "description": "Search for objects in live video", + "parameters": { + "type": "object", + "properties": { + "input": { + "type": "string", + "description": "List of objects that will be searched, objects must be separated by comma, objects must include articles 'a' or 'an' e.g '[a dog, a cat]' " + } + }, + "required": [ + "input" + ] + } + } + }, + { + "type": "function", + "function": { + "name": "move_camera", + "description": "Move the camera to the left or right", + "parameters": { + "type": "object", + "properties": { + "pan_angle": { + "type": "float", + "description": "Angle to move the camera right or left, right is a positive angle while left is a negative angle, default is 0" + } + }, + "required": [ + "pan_angle" + ] + } + } + } +] +\n\n +[INST] search for a car and a flower [/INST] +{ + "name": "search_object", + "arguments": { + "input": "a car, a flower" + } +} +\n\n +[INST] move camera 30 degrees to the left [/INST] +{ + "name": "move_camera", + "arguments": { + "pan_angle": -30 + } +} + diff --git a/config/analytics/configuration.json b/config/analytics/configuration.json new file mode 100644 index 0000000..f43c5eb --- /dev/null +++ b/config/analytics/configuration.json @@ -0,0 +1,14 @@ +{ + "move_camera": { + "enable": 1, + "port": 5020, + "ip": "127.0.0.1", + "time_threshold": 10 + }, + "record": { + "enable": 0, + "port": 81, + "ip": "127.0.0.1", + "time_threshold": 20 + } +} diff --git a/config/demo-nginx.conf b/config/demo-nginx.conf new file mode 100644 index 0000000..8795e35 --- /dev/null +++ b/config/demo-nginx.conf @@ -0,0 +1,52 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# specify your service discovery config here + +location /agent/ { + rewrite ^/agent/?(.*)$ /$1 break; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + access_log /var/log/nginx/access.log timed_combined; + proxy_pass http://localhost:5010; +} + +location /analytics/ { + rewrite ^/analytics/?(.*)$ /$1 break; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + access_log /var/log/nginx/access.log timed_combined; + proxy_pass http://localhost:5040; +} + +location /detect/ { + rewrite ^/detect/?(.*)$ /$1 break; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + access_log /var/log/nginx/access.log timed_combined; + proxy_pass http://localhost:5030; +} + +location /ptz/ { + rewrite ^/ptz/?(.*)$ /$1 break; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + access_log /var/log/nginx/access.log timed_combined; + proxy_pass http://localhost:5020; +} diff --git a/config/vst/vst_config.json b/config/vst/vst_config.json new file mode 100644 index 0000000..ca2b2a7 --- /dev/null +++ b/config/vst/vst_config.json @@ -0,0 +1,172 @@ +{ + "network": + { + "http_port": 81, + "server_domain_name":"", + "stunurl_list": ["stun.l.google.com:19302","stun1.l.google.com:19302"], + "static_turnurl_list": [], + "use_coturn_auth_secret": false, + "coturn_turnurl_list_with_secret": [], + "use_twilio_stun_turn": false, + "twilio_account_sid": "", + "twilio_auth_token": "", + "use_reverse_proxy": false, + "reverse_proxy_server_address": "REVERSE_PROXY_SERVER_ADDRESS:100", + "ntp_servers": [], + "max_webrtc_out_connections": 1, + "max_webrtc_in_connections": 1, + "webservice_access_control_list":"", + "rtsp_server_port": -1, + "rtsp_preferred_network_iface": "eth0", + "rtsp_in_base_udp_port_num": -1, + "rtsp_out_base_udp_port_num": -1, + "rtsp_streaming_over_tcp": false, + "rtsp_server_reclamation_client_timeout_sec": 10, + "rx_socket_buffer_size":1000000, + "tx_socket_buffer_size":1000000, + "stream_monitor_interval_secs": 2, + "rtp_udp_port_range" : "31000-31200", + "udp_latency_ms": 200, + "udp_drop_on_latency": false, + "webrtc_latency_ms": 1000, + "enable_frame_drop": true, + "webrtc_video_quality_tunning": + { + "resolution_2160": + { + "bitrate_start" : 20000, "bitrate_range" : [15000,25000], + "qp_range_I" : [0,30], "qp_range_P" : [0,51] + }, + "resolution_1440": + { + "bitrate_start" : 10000, "bitrate_range" : [8000,15000], + "qp_range_I" : [10,30], "qp_range_P" : [10,30] + }, + "resolution_1080": + { + "bitrate_start" : 5000, "bitrate_range" : [3000,8000], + "qp_range_I" : [10,30], "qp_range_P" : [10,30] + }, + "resolution_720": + { + "bitrate_start" : 3000, "bitrate_range" : [2000,5000], + "qp_range_I" : [10,30], "qp_range_P" : [10,30] + }, + "resolution_480": + { + "bitrate_start" : 1000, "bitrate_range" : [800,3000], + "qp_range_I" : [10,30], "qp_range_P" : [10,30] + } + }, + "webrtc_peer_conn_timeout_sec": 10, + "enable_grpc": false, + "grpc_server_port": "50051", + "webrtc_in_audio_sender_max_bitrate": 128000, + "webrtc_in_video_degradation_preference": "resolution", + "webrtc_in_video_sender_max_framerate": 30, + "remote_vst_address": "", + "webrtc_port_range": {"min":0, "max":0}, + "enable_websocket_pingpong": false, + "websocket_keep_alive_ms": 5000 + }, + "onvif": + { + "device_discovery_timeout_secs":10, + "onvif_request_timeout_secs":20, + "device_discovery_freq_secs":15, + "device_discovery_interfaces": ["eth1"], + "max_devices_supported": 8, + "default_bitrate_kbps": 8000, + "default_framerate": 30, + "default_resolution": "1920x1080", + "default_gov_length": 60 + }, + "data": + { + "storage_config_file": "/root/vst_release/configs/vst_storage.json", + "storage_threshold_percentage": 95, + "storage_monitoring_frequency_secs": 2, + "nv_streamer_directory_path": "/home/vst/vst_release/streamer_videos/", + "nv_streamer_loop_playback":true, + "nv_streamer_seekable":false, + "nv_streamer_max_upload_file_size_MB": 10000, + "nv_streamer_media_container_supported": ["mp4","mkv"], + "nv_streamer_metadata_container_supported": ["json"], + "nv_streamer_rtsp_server_output_buffer_size_kb": 1000, + "supported_video_codecs": ["h264","h265"], + "supported_audio_codecs": ["pcmu","pcma","mpeg4-generic"], + "enable_aging_policy": false, + "max_video_download_size_MB":1000, + "always_recording": false, + "event_recording": true, + "event_record_length_secs": 10, + "record_buffer_length_secs": 2, + "use_software_path": false, + "use_webrtc_inbuilt_encoder": "h264", + "webrtc_in_fixed_resolution": "1280x720", + "webrtc_in_max_framerate": 30, + "webrtc_in_video_bitrate_thresold_percentage": 50, + "webrtc_in_passthrough": false, + "webrtc_sender_quality": "pass_through", + "enable_rtsp_server_sei_metadata": false, + "enable_proxy_server_sei_metadata": false, + "gpu_indices": [], + "webrtc_out_enable_insert_sps_pps" : true, + "webrtc_out_set_iframe_interval" : 30, + "webrtc_out_set_idr_interval" : 256, + "webrtc_out_min_drc_interval" : 5, + "webrtc_out_encode_fallback_option" : "software", + "device_name" : "VST", + "device_location" : "", + "enable_dec_low_latency_mode": true, + "webrtc_out_default_resolution": "" + }, + "notifications": + { + "enable_notification": true, + "use_message_broker": "redis", + "message_broker_topic": "vst.event", + "message_broker_payload_key": "sensor.id", + "message_broker_metadata_topic": "test", + "redis_server_env_var":"localhost:6379", + "kafka_server_address": "localhost:9092" + }, + "debug": + { + "enable_perf_logging":true, + "enable_qos_monitoring":true, + "qos_logfile_path":"/root/vst_release/webroot/log/", + "qos_data_capture_interval_sec":1, + "qos_data_publish_interval_sec":5, + "enable_gst_debug_probes":true, + "enable_prometheus":true, + "prometheus_port": "8080", + "enable_highlighting_logs":false, + "enable_debug_apis": true, + "dump_webrtc_input_stats": false, + "enable_frameid_in_webrtc_stream": false, + "enable_network_bandwidth_notification" : false, + "enable_latency_logging": true + }, + "overlay": + { + "video_metadata_server": "", + "video_metadata_query_batch_size_num_frames": 300, + "use_video_metadata_protobuf": false, + "enable_gem_drawing": true, + "analytic_server_address": "", + "overlay_text_font_type": "DejaVuSansMono.ttf" + }, + "security": + { + "use_https": false, + "use_rtsp_authentication": false, + "use_http_digest_authentication": false, + "use_multi_user": false, + "enable_user_cleanup": false, + "session_max_age_sec": 2592000, + "multi_user_extra_options": ["Secure", "SameSite=none"], + "nv_org_id": "", + "nv_ngc_key": "" + } +} diff --git a/config/vst/vst_storage.json b/config/vst/vst_storage.json new file mode 100644 index 0000000..7ad5a44 --- /dev/null +++ b/config/vst/vst_storage.json @@ -0,0 +1,5 @@ +{ + "data_path": "/root/store/data/", + "video_path": "/root/store/video/", + "total_video_storage_size_MB": 10000 +} diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000..d4351a2 --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,98 @@ +# Copyright (C) 2024 RidgeRun, LLC (http://www.ridgerun.com) +# All Rights Reserved. +# +# The contents of this software are proprietary and confidential to RidgeRun, +# LLC. No part of this program may be photocopied, reproduced or translated +# into another programming language without prior written consent of +# RidgeRun, LLC. The user is free to modify the source code after obtaining +# a software license from RidgeRun. All source code changes must be provided +# back to RidgeRun without any encumbrance. + +services: + detection: + image: ridgerun/detection-service + user: "0:0" + network_mode: "host" + runtime: nvidia + logging: + driver: "json-file" + options: + max-size: "8192m" + max-file: "3" + restart: always + container_name: detection-service + entrypoint: detection --horizontal-slices=1 --vertical-slices=1 + deploy: + resources: + limits: + memory: 5600M + restart_policy: + condition: always + + analytics: + image: ridgerun/analytics-service + user: "0:0" + network_mode: "host" + runtime: nvidia + logging: + driver: "json-file" + options: + max-size: "8192m" + max-file: "3" + restart: always + volumes: + - ./config/analytics/:/analytics-config/ + container_name: analytics-service + entrypoint: analytics --config-file /analytics-config/configuration.json + depends_on: + detection: + condition: service_started + deploy: + resources: + limits: + memory: 5600M + restart_policy: + condition: always + + agent: + image: ridgerun/ai-agent-service + user: "0:0" + network_mode: "host" + runtime: nvidia + logging: + driver: "json-file" + options: + max-size: "8192m" + max-file: "3" + restart: always + volumes: + - ./config/agent/:/ai-agent-config/ + container_name: agent-service + entrypoint: ai-agent --system_prompt ai-agent-config/prompt.txt --api_map ai-agent-config/api_mapping.json + depends_on: + detection: + condition: service_started + deploy: + restart_policy: + condition: always + + ptz: + image: ridgerun/ptz-service + user: "0:0" + network_mode: "host" + runtime: nvidia + logging: + driver: "json-file" + options: + max-size: "8192m" + max-file: "3" + restart: always + container_name: ptz-service + entrypoint: ptz --host 127.0.0.1 --port 5020 + depends_on: + detection: + condition: service_started + deploy: + restart_policy: + condition: always +