From 8ed0ec51f1f5f1dc6bfb6871042e63feb38e4abf Mon Sep 17 00:00:00 2001 From: Nir-Az Date: Wed, 8 Jun 2022 09:35:31 +0300 Subject: [PATCH 01/25] comment out and validate libCI without redundent frame copies --- src/sensor.cpp | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/src/sensor.cpp b/src/sensor.cpp index 316e1b1bd8..17a42db249 100644 --- a/src/sensor.cpp +++ b/src/sensor.cpp @@ -267,11 +267,12 @@ void log_callback_end( uint32_t fps, { auto system_time = environment::get_instance().get_time_service()->get_time(); auto fr = std::make_shared(); - byte* pix = (byte*)fo.pixels; - std::vector pixels(pix, pix + fo.frame_size); - fr->data = pixels; + + //REMOVED! - no need to add 2 copies to a frame + //byte* pix = (byte*)fo.pixels; + //std::vector pixels(pix, pix + fo.frame_size); + //fr->data = pixels; fr->set_stream(profile); - frame_additional_data additional_data(0, 0, system_time, @@ -398,9 +399,12 @@ void log_callback_end( uint32_t fps, if (fh.frame) { - assert( expected_size == sizeof(byte) * fr->data.size() ); + assert( expected_size == sizeof(byte) * /*fr->data.size()*/f.frame_size ); + + memcpy( (void *)fh->get_frame_data(), + /*fr->data.data()*/ f.pixels, + expected_size ); - memcpy((void*)fh->get_frame_data(), fr->data.data(), expected_size); auto&& video = (video_frame*)fh.frame; video->assign(width, height, width * bpp / 8, bpp); video->set_timestamp_domain(timestamp_domain); @@ -898,7 +902,9 @@ void log_callback_end( uint32_t fps, last_frame_number = frame_counter; last_timestamp = timestamp; frame_holder frame = _source.alloc_frame(RS2_EXTENSION_MOTION_FRAME, data_size, fr->additional_data, true); - memcpy((void*)frame->get_frame_data(), fr->data.data(), sizeof(byte)*fr->data.size()); + memcpy( (void *)frame->get_frame_data(), + /*fr->data.data()*/ sensor_data.fo.pixels, + /*fr->data.size()*/ sizeof( byte ) * sensor_data.fo.frame_size ); if (!frame) { LOG_INFO("Dropped frame. alloc_frame(...) returned nullptr"); From 8b635a1de0ec8f9bff8ae7a88b95da210d49ac78 Mon Sep 17 00:00:00 2001 From: ohadmeir Date: Mon, 13 Jun 2022 11:25:30 +0300 Subject: [PATCH 02/25] Add live fps testing --- unit-tests/live/frames/test-fps.py | 99 ++++++++++++++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 unit-tests/live/frames/test-fps.py diff --git a/unit-tests/live/frames/test-fps.py b/unit-tests/live/frames/test-fps.py new file mode 100644 index 0000000000..eef60d27da --- /dev/null +++ b/unit-tests/live/frames/test-fps.py @@ -0,0 +1,99 @@ +# License: Apache 2.0. See LICENSE file in root directory. +# Copyright(c) 2022 Intel Corporation. All Rights Reserved. + +# test:device L500* +# test:device D400* +# test:donotrun:!nightly + +import pyrealsense2 as rs +from rspy.stopwatch import Stopwatch +from rspy import test, log +import time +import platform + +# Start depth + color streams and measure frame frequency using sensor API. +# Verify that actual fps is as requested + +def measure_fps(sensor, profile): + """ + Wait for 2 seconds to be sure that frames are at steady state after start + Count number of received frames for 5 seconds and compare actual fps to requested fps + """ + seconds_till_steady_state = 2 + seconds_to_count_frames = 5 + + steady_state = False + frames_received = 0 + fps_stopwatch = Stopwatch() + + def frame_cb(frame): + nonlocal steady_state, frames_received + if steady_state: + frames_received += 1 + + sensor.open(profile) + sensor.start(frame_cb) + + time.sleep(seconds_till_steady_state) + + fps_stopwatch.reset() + steady_state = True + + time.sleep(seconds_to_count_frames) # Time to count frames + + steady_state = False # Stop counting + + sensor.stop() + sensor.close() + + fps = frames_received / seconds_to_count_frames + return fps + + +acceptable_exception_rate_Hz = 0.5 +tested_fps = [6, 15, 30, 60, 90] + +dev = test.find_first_device_or_exit() +product_line = dev.get_info(rs.camera_info.product_line) + +##################################################################################################### +test.start("Testing depth fps " + product_line + " device - "+ platform.system() + " OS") + +for requested_fps in tested_fps: + ds = dev.first_depth_sensor() + try: + dp = next(p for p in ds.profiles + if p.fps() == requested_fps + and p.stream_type() == rs.stream.depth + and p.format() == rs.format.z16) + except StopIteration: + print("Requested fps: {:.1f} [Hz], not supported".format(requested_fps)) + else: + fps = measure_fps(ds, dp) + print("Requested fps: {:.1f} [Hz], actual fps: {:.1f} [Hz] ".format(requested_fps, fps)) + test.check(fps <= (requested_fps + acceptable_exception_rate_Hz) and fps >= (requested_fps - acceptable_exception_rate_Hz)) +test.finish() + + +##################################################################################################### +test.start("Testing color fps " + product_line + " device - "+ platform.system() + " OS") + +for requested_fps in tested_fps: + dev = test.find_first_device_or_exit() + cs = dev.first_color_sensor() + try: + cp = next(p for p in cs.profiles + if p.fps() == requested_fps + and p.stream_type() == rs.stream.color + and p.format() == rs.format.rgb8) + except StopIteration: + print("Requested fps: {:.1f} [Hz], not supported".format(requested_fps)) + else: + fps = measure_fps(cs, cp) + print("Requested fps: {:.1f} [Hz], actual fps: {:.1f} [Hz] ".format(requested_fps, fps)) + test.check(fps <= (requested_fps + acceptable_exception_rate_Hz) and fps >= (requested_fps - acceptable_exception_rate_Hz)) + +test.finish() + +##################################################################################################### +test.print_results_and_exit() From 8a14565682ffb5269bf55255dcd19b8fa0f3c3a8 Mon Sep 17 00:00:00 2001 From: ohadmeir Date: Mon, 13 Jun 2022 12:30:37 +0300 Subject: [PATCH 03/25] Test fps mean for 20 seconds --- unit-tests/live/frames/test-fps.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/unit-tests/live/frames/test-fps.py b/unit-tests/live/frames/test-fps.py index eef60d27da..01074e2135 100644 --- a/unit-tests/live/frames/test-fps.py +++ b/unit-tests/live/frames/test-fps.py @@ -20,7 +20,7 @@ def measure_fps(sensor, profile): Count number of received frames for 5 seconds and compare actual fps to requested fps """ seconds_till_steady_state = 2 - seconds_to_count_frames = 5 + seconds_to_count_frames = 20 steady_state = False frames_received = 0 @@ -50,7 +50,7 @@ def frame_cb(frame): return fps -acceptable_exception_rate_Hz = 0.5 +acceptable_exception_rate_Hz = 1 tested_fps = [6, 15, 30, 60, 90] dev = test.find_first_device_or_exit() From cc794605ff8a7499d253c43d46f6cb90cdef6963 Mon Sep 17 00:00:00 2001 From: ohadmeir Date: Mon, 13 Jun 2022 16:08:36 +0300 Subject: [PATCH 04/25] Handle auto-exposure (affects fps) --- unit-tests/live/frames/test-fps.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/unit-tests/live/frames/test-fps.py b/unit-tests/live/frames/test-fps.py index 01074e2135..801109482e 100644 --- a/unit-tests/live/frames/test-fps.py +++ b/unit-tests/live/frames/test-fps.py @@ -3,7 +3,8 @@ # test:device L500* # test:device D400* -# test:donotrun:!nightly + +# test:timeout 230 import pyrealsense2 as rs from rspy.stopwatch import Stopwatch @@ -36,8 +37,8 @@ def frame_cb(frame): time.sleep(seconds_till_steady_state) - fps_stopwatch.reset() steady_state = True + fps_stopwatch.reset() time.sleep(seconds_to_count_frames) # Time to count frames @@ -61,6 +62,9 @@ def frame_cb(frame): for requested_fps in tested_fps: ds = dev.first_depth_sensor() + if product_line == "D400": + ds.set_option(rs.option.enable_auto_exposure, 1) + try: dp = next(p for p in ds.profiles if p.fps() == requested_fps @@ -81,6 +85,11 @@ def frame_cb(frame): for requested_fps in tested_fps: dev = test.find_first_device_or_exit() cs = dev.first_color_sensor() + if product_line == "D400": + ds.set_option(rs.option.enable_auto_exposure, 1) + elif product_line == "L500": + cs.set_option(rs.option.enable_auto_exposure, 0) + try: cp = next(p for p in cs.profiles if p.fps() == requested_fps From 194af8eaa5b5e0a5739bfc3acb2d4e8646382bca Mon Sep 17 00:00:00 2001 From: ohadmeir Date: Mon, 13 Jun 2022 16:08:36 +0300 Subject: [PATCH 05/25] Handle auto-exposure (affects fps) --- unit-tests/live/frames/test-fps.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/unit-tests/live/frames/test-fps.py b/unit-tests/live/frames/test-fps.py index 01074e2135..ff83ef490a 100644 --- a/unit-tests/live/frames/test-fps.py +++ b/unit-tests/live/frames/test-fps.py @@ -4,6 +4,7 @@ # test:device L500* # test:device D400* # test:donotrun:!nightly +# test:timeout 230 import pyrealsense2 as rs from rspy.stopwatch import Stopwatch @@ -36,8 +37,8 @@ def frame_cb(frame): time.sleep(seconds_till_steady_state) - fps_stopwatch.reset() steady_state = True + fps_stopwatch.reset() time.sleep(seconds_to_count_frames) # Time to count frames @@ -61,6 +62,9 @@ def frame_cb(frame): for requested_fps in tested_fps: ds = dev.first_depth_sensor() + if product_line == "D400": + ds.set_option(rs.option.enable_auto_exposure, 1) + try: dp = next(p for p in ds.profiles if p.fps() == requested_fps @@ -81,6 +85,11 @@ def frame_cb(frame): for requested_fps in tested_fps: dev = test.find_first_device_or_exit() cs = dev.first_color_sensor() + if product_line == "D400": + ds.set_option(rs.option.enable_auto_exposure, 1) + elif product_line == "L500": + cs.set_option(rs.option.enable_auto_exposure, 0) + try: cp = next(p for p in cs.profiles if p.fps() == requested_fps From 0cd42d99f52add52aa9524b8c8f0367625908aa7 Mon Sep 17 00:00:00 2001 From: ohadmeir Date: Sun, 19 Jun 2022 13:17:14 +0300 Subject: [PATCH 06/25] Handle PR comments --- unit-tests/live/frames/test-fps.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/unit-tests/live/frames/test-fps.py b/unit-tests/live/frames/test-fps.py index ff83ef490a..39e66be390 100644 --- a/unit-tests/live/frames/test-fps.py +++ b/unit-tests/live/frames/test-fps.py @@ -51,7 +51,7 @@ def frame_cb(frame): return fps -acceptable_exception_rate_Hz = 1 +delta_Hz = 1 tested_fps = [6, 15, 30, 60, 90] dev = test.find_first_device_or_exit() @@ -62,6 +62,7 @@ def frame_cb(frame): for requested_fps in tested_fps: ds = dev.first_depth_sensor() + #Set auto-exposure option as it might take precedence over requested FPS if product_line == "D400": ds.set_option(rs.option.enable_auto_exposure, 1) @@ -75,7 +76,7 @@ def frame_cb(frame): else: fps = measure_fps(ds, dp) print("Requested fps: {:.1f} [Hz], actual fps: {:.1f} [Hz] ".format(requested_fps, fps)) - test.check(fps <= (requested_fps + acceptable_exception_rate_Hz) and fps >= (requested_fps - acceptable_exception_rate_Hz)) + test.check(fps <= (requested_fps + delta_Hz) and fps >= (requested_fps - delta_Hz)) test.finish() @@ -83,8 +84,8 @@ def frame_cb(frame): test.start("Testing color fps " + product_line + " device - "+ platform.system() + " OS") for requested_fps in tested_fps: - dev = test.find_first_device_or_exit() cs = dev.first_color_sensor() + #Set auto-exposure option as it might take precedence over requested FPS if product_line == "D400": ds.set_option(rs.option.enable_auto_exposure, 1) elif product_line == "L500": @@ -100,7 +101,7 @@ def frame_cb(frame): else: fps = measure_fps(cs, cp) print("Requested fps: {:.1f} [Hz], actual fps: {:.1f} [Hz] ".format(requested_fps, fps)) - test.check(fps <= (requested_fps + acceptable_exception_rate_Hz) and fps >= (requested_fps - acceptable_exception_rate_Hz)) + test.check(fps <= (requested_fps + delta_Hz) and fps >= (requested_fps - delta_Hz)) test.finish() From dd713e809d72e480077a1de361ba7dbd4042f8d4 Mon Sep 17 00:00:00 2001 From: Remi Bettan Date: Wed, 22 Jun 2022 13:02:47 +0300 Subject: [PATCH 07/25] copy frame only when needed --- src/sensor.cpp | 46 ++++++++++++++++++++++------------------------ 1 file changed, 22 insertions(+), 24 deletions(-) diff --git a/src/sensor.cpp b/src/sensor.cpp index 9de92ed5a0..be5bce8026 100644 --- a/src/sensor.cpp +++ b/src/sensor.cpp @@ -284,28 +284,15 @@ void log_callback_end( uint32_t fps, auto system_time = environment::get_instance().get_time_service()->get_time(); auto fr = std::make_shared(); - //REMOVED! - no need to add 2 copies to a frame - //byte* pix = (byte*)fo.pixels; - std::vector pixels; + fr->set_stream(profile); + + // D457 dev - computing relevant frame size const auto&& vsp = As(profile); int width = vsp ? vsp->get_width() : 0; int height = vsp ? vsp->get_height() : 0; int bpp = get_image_bpp(profile->get_format()); + auto frame_size = compute_frame_expected_size(width, height, bpp); - // method should be limited to use of MIPI - not for USB - // the aim is to grab the data from a bigger buffer, which is aligned to 64 bytes, - // when the resolution's width is not aligned to 64 - if (width % 64 != 0 && fo.frame_size > compute_frame_expected_size(width, height, bpp)) - { - pixels = align_width_to_64(width, height, bpp, pix); - } - else - { - pixels = std::vector(pix, pix + fo.frame_size); - } - - //fr->data = pixels; - fr->set_stream(profile); frame_additional_data additional_data(0, 0, system_time, @@ -316,7 +303,7 @@ void log_callback_end( uint32_t fps, last_frame_number, false, 0, - (uint32_t)pixels.size()); + (uint32_t)frame_size); if (_metadata_modifier) _metadata_modifier(additional_data); @@ -442,12 +429,23 @@ void log_callback_end( uint32_t fps, if (fh.frame) { - assert( expected_size == sizeof(byte) * /*fr->data.size()*/f.frame_size ); - //expected_size == sizeof(byte) * fr->data.size() + 68); // added for D457 - need to understand why this happens (68 is size of md) - - memcpy( (void *)fh->get_frame_data(), - /*fr->data.data()*/ f.pixels, - expected_size ); + // method should be limited to use of MIPI - not for USB + // the aim is to grab the data from a bigger buffer, which is aligned to 64 bytes, + // when the resolution's width is not aligned to 64 + if (width % 64 != 0 && f.frame_size > expected_size) + { + byte* pix = (byte*)f.pixels; + std::vector pixels; + pixels = align_width_to_64(width, height, bpp, pix); + fr->data = pixels; + assert( expected_size == sizeof(byte) * fr->data.size()); + memcpy( (void *)fh->get_frame_data(), fr->data.data(), expected_size ); + } + else + { + assert( expected_size == sizeof(byte) * f.frame_size ); + memcpy( (void *)fh->get_frame_data(), f.pixels, expected_size ); + } auto&& video = dynamic_cast(fh.frame); if (video) From cb022fcc44780a5848e9d06f0ded1385e1faea6b Mon Sep 17 00:00:00 2001 From: Remi Bettan Date: Wed, 22 Jun 2022 14:58:40 +0300 Subject: [PATCH 08/25] code review corrections --- src/sensor.cpp | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/sensor.cpp b/src/sensor.cpp index be5bce8026..9928ad85a8 100644 --- a/src/sensor.cpp +++ b/src/sensor.cpp @@ -432,14 +432,11 @@ void log_callback_end( uint32_t fps, // method should be limited to use of MIPI - not for USB // the aim is to grab the data from a bigger buffer, which is aligned to 64 bytes, // when the resolution's width is not aligned to 64 - if (width % 64 != 0 && f.frame_size > expected_size) + if ((width * bpp >> 3) % 64 != 0 && f.frame_size > expected_size) { - byte* pix = (byte*)f.pixels; - std::vector pixels; - pixels = align_width_to_64(width, height, bpp, pix); - fr->data = pixels; - assert( expected_size == sizeof(byte) * fr->data.size()); - memcpy( (void *)fh->get_frame_data(), fr->data.data(), expected_size ); + std::vector pixels = align_width_to_64(width, height, bpp, (byte*)f.pixels); + assert( expected_size == sizeof(byte) * pixels.size()); + memcpy( (void *)fh->get_frame_data(), pixels.data(), expected_size ); } else { From 0fc9b42f8cdf23bf904b047d1572e37918c90bd4 Mon Sep 17 00:00:00 2001 From: Remi Bettan Date: Sun, 3 Jul 2022 15:12:35 +0300 Subject: [PATCH 09/25] hwmc - restoring usb api --- src/option.cpp | 38 +++++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/src/option.cpp b/src/option.cpp index faebb41a22..5abb35165e 100644 --- a/src/option.cpp +++ b/src/option.cpp @@ -120,6 +120,42 @@ std::vector librealsense::command_transfer_over_xu::send_receive(const std::dec << data.size() << " exceeds permitted limit " << HW_MONITOR_BUFFER_SIZE); } + std::vector transmit_buf(HW_MONITOR_BUFFER_SIZE, 0); + std::copy(data.begin(), data.end(), transmit_buf.begin()); + + if (!dev.set_xu(_xu, _ctrl, transmit_buf.data(), static_cast(transmit_buf.size()))) + throw invalid_value_exception(to_string() << "set_xu(ctrl=" << unsigned(_ctrl) << ") failed!" << " Last Error: " << strerror(errno)); + + if (require_response) + { + result.resize(HW_MONITOR_BUFFER_SIZE); + if (!dev.get_xu(_xu, _ctrl, result.data(), static_cast(result.size()))) + throw invalid_value_exception(to_string() << "get_xu(ctrl=" << unsigned(_ctrl) << ") failed!" << " Last Error: " << strerror(errno)); + + // Returned data size located in the last 4 bytes + auto data_size = *(reinterpret_cast(result.data() + HW_MONITOR_DATA_SIZE_OFFSET)) + SIZE_OF_HW_MONITOR_HEADER; + result.resize(data_size); + } + return result; + }); +} +/* +std::vector librealsense::command_transfer_over_xu::send_receive(const std::vector& data, int, bool require_response) +{ + return _uvc.invoke_powered([this, &data, require_response] + (platform::uvc_device& dev) + { + std::vector result; + std::lock_guard lock(dev); + + if (data.size() > HW_MONITOR_BUFFER_SIZE) + { + LOG_ERROR("XU command size is invalid"); + throw invalid_value_exception(to_string() << "Requested XU command size " << + std::dec << data.size() << " exceeds permitted limit " << HW_MONITOR_BUFFER_SIZE); + } + + std::vector transmit_buf(HW_MONITOR_BUFFER_SIZE, 0); // D457 - size of 1028 needed instead of 1024 (HW_MONITOR_BUFFER_SIZE) std::vector transmit_buf(HW_MONITOR_BUFFER_SIZE + SIZE_OF_HW_MONITOR_HEADER, 0); std::copy(data.begin(), data.end(), transmit_buf.begin()); @@ -145,7 +181,7 @@ std::vector librealsense::command_transfer_over_xu::send_receive(const } return result; }); -} +}*/ librealsense::polling_errors_disable::~polling_errors_disable() { From b136188c1aeac9d1e6bda6795fac3cc37561f3b4 Mon Sep 17 00:00:00 2001 From: Remi Bettan Date: Sun, 3 Jul 2022 15:13:08 +0300 Subject: [PATCH 10/25] comment added for usb enumeration successfull within this branch --- src/linux/backend-v4l2.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/linux/backend-v4l2.cpp b/src/linux/backend-v4l2.cpp index 33860af97c..bd762ff85b 100644 --- a/src/linux/backend-v4l2.cpp +++ b/src/linux/backend-v4l2.cpp @@ -655,7 +655,7 @@ namespace librealsense // /sys/devices/pci0000:00/0000:00:xx.0/ABC/M-N/version usb_specification = get_usb_connection_type(real_path + "/../../../"); } - else // Video4Linux Devices that are not listed as UVC + else // COMMENT THIS TO MAKE USB ENUMERATION POSSIBLE - Video4Linux Devices that are not listed as UVC { //LOG_INFO("Enumerating v4l " << name << " realpath=" << real_path); v4l_node = true; From e09a2123a632c1a2d4012b076889969c1e9ed4ba Mon Sep 17 00:00:00 2001 From: Remi Bettan Date: Mon, 4 Jul 2022 14:19:36 +0300 Subject: [PATCH 11/25] adding ifdef flags for mipi/usb work --- src/linux/backend-v4l2.cpp | 3 +++ src/option.cpp | 12 +++++++++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/linux/backend-v4l2.cpp b/src/linux/backend-v4l2.cpp index bd762ff85b..9412ddc4ae 100644 --- a/src/linux/backend-v4l2.cpp +++ b/src/linux/backend-v4l2.cpp @@ -655,6 +655,8 @@ namespace librealsense // /sys/devices/pci0000:00/0000:00:xx.0/ABC/M-N/version usb_specification = get_usb_connection_type(real_path + "/../../../"); } +#undef enumerate_MIPI +#ifdef enumerate_MIPI else // COMMENT THIS TO MAKE USB ENUMERATION POSSIBLE - Video4Linux Devices that are not listed as UVC { //LOG_INFO("Enumerating v4l " << name << " realpath=" << real_path); @@ -706,6 +708,7 @@ namespace librealsense break; } } +#endif // enumerate_MIPI // D457 Dev - skip unsupported devices. Do no upstream! // manually rectify descriptor inconsistencies diff --git a/src/option.cpp b/src/option.cpp index 5abb35165e..1fb93a2482 100644 --- a/src/option.cpp +++ b/src/option.cpp @@ -105,6 +105,9 @@ const char* librealsense::uvc_pu_option::get_description() const } } +#undef HWM_MIPI +#ifndef HWM_MIPI + std::vector librealsense::command_transfer_over_xu::send_receive(const std::vector& data, int, bool require_response) { return _uvc.invoke_powered([this, &data, require_response] @@ -139,7 +142,9 @@ std::vector librealsense::command_transfer_over_xu::send_receive(const return result; }); } -/* + +#else + std::vector librealsense::command_transfer_over_xu::send_receive(const std::vector& data, int, bool require_response) { return _uvc.invoke_powered([this, &data, require_response] @@ -155,7 +160,6 @@ std::vector librealsense::command_transfer_over_xu::send_receive(const std::dec << data.size() << " exceeds permitted limit " << HW_MONITOR_BUFFER_SIZE); } - std::vector transmit_buf(HW_MONITOR_BUFFER_SIZE, 0); // D457 - size of 1028 needed instead of 1024 (HW_MONITOR_BUFFER_SIZE) std::vector transmit_buf(HW_MONITOR_BUFFER_SIZE + SIZE_OF_HW_MONITOR_HEADER, 0); std::copy(data.begin(), data.end(), transmit_buf.begin()); @@ -181,7 +185,9 @@ std::vector librealsense::command_transfer_over_xu::send_receive(const } return result; }); -}*/ +} + +#endif // HWM_MIPI librealsense::polling_errors_disable::~polling_errors_disable() { From d3e3d8b7f4443f7400d413f0c9c4672a6cbe1437 Mon Sep 17 00:00:00 2001 From: Remi Bettan Date: Mon, 4 Jul 2022 14:20:18 +0300 Subject: [PATCH 12/25] fix minor bug in ds5-color --- src/ds5/ds5-color.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ds5/ds5-color.cpp b/src/ds5/ds5-color.cpp index 9cbd35e30a..c8431ce1ab 100644 --- a/src/ds5/ds5-color.cpp +++ b/src/ds5/ds5-color.cpp @@ -240,8 +240,8 @@ namespace librealsense } if (_pid != ds::RS457_PID) { - color_ep.register_processing_block(processing_block_factory::create_pbf_vector(RS2_FORMAT_UYVY, map_supported_color_formats(RS2_FORMAT_UYVY), RS2_STREAM_COLOR)); color_ep.register_processing_block(processing_block_factory::create_pbf_vector(RS2_FORMAT_YUYV, map_supported_color_formats(RS2_FORMAT_YUYV), RS2_STREAM_COLOR)); + color_ep.register_processing_block(processing_block_factory::create_id_pbf(RS2_FORMAT_RAW16, RS2_STREAM_COLOR)); } else { From 9e4390d76154c9df0313d888fb354b1bfd5632ae Mon Sep 17 00:00:00 2001 From: Remi Bettan Date: Mon, 4 Jul 2022 14:21:23 +0300 Subject: [PATCH 13/25] fix profiles unordered_set - permits usb depth stream --- src/sensor.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/sensor.cpp b/src/sensor.cpp index 9928ad85a8..3637818454 100644 --- a/src/sensor.cpp +++ b/src/sensor.cpp @@ -653,8 +653,9 @@ void log_callback_end( uint32_t fps, stream_profiles uvc_sensor::init_stream_profiles() { - // D457 development - std::unordered_set> profiles; - std::unordered_set> profiles; + std::unordered_set> video_profiles; + // D457 development + std::unordered_set> motion_profiles; power on(std::dynamic_pointer_cast(shared_from_this())); _uvc_profiles = _device->get_profiles(); @@ -673,7 +674,7 @@ void log_callback_end( uint32_t fps, profile->set_stream_index(0); profile->set_format(rs2_fmt); profile->set_framerate(p.fps); - profiles.insert(profile); + motion_profiles.insert(profile); } else { @@ -683,11 +684,12 @@ void log_callback_end( uint32_t fps, profile->set_stream_index(0); profile->set_format(rs2_fmt); profile->set_framerate(p.fps); - profiles.insert(profile); + video_profiles.insert(profile); } } - stream_profiles result{ profiles.begin(), profiles.end() }; + stream_profiles result{ video_profiles.begin(), video_profiles.end() }; + result.insert(result.end(), motion_profiles.begin(), motion_profiles.end()); return result; } @@ -1550,8 +1552,6 @@ void log_callback_end( uint32_t fps, for (auto source : requests) add_source_profile_missing_data(source); - - const auto&& resolved_req = resolve_requests(requests); _raw_sensor->set_source_owner(this); From 7d791fa71170b700b4c4735b35e83fbf66134325 Mon Sep 17 00:00:00 2001 From: Remi Bettan Date: Mon, 4 Jul 2022 16:51:13 +0300 Subject: [PATCH 14/25] hack hwm to make mipi and usb work togethter - temp --- src/linux/backend-v4l2.cpp | 2 +- src/option.cpp | 60 ++++++++++++++++++++++++++++---------- 2 files changed, 46 insertions(+), 16 deletions(-) diff --git a/src/linux/backend-v4l2.cpp b/src/linux/backend-v4l2.cpp index 9412ddc4ae..14b4dc17ce 100644 --- a/src/linux/backend-v4l2.cpp +++ b/src/linux/backend-v4l2.cpp @@ -655,7 +655,7 @@ namespace librealsense // /sys/devices/pci0000:00/0000:00:xx.0/ABC/M-N/version usb_specification = get_usb_connection_type(real_path + "/../../../"); } -#undef enumerate_MIPI +#define enumerate_MIPI #ifdef enumerate_MIPI else // COMMENT THIS TO MAKE USB ENUMERATION POSSIBLE - Video4Linux Devices that are not listed as UVC { diff --git a/src/option.cpp b/src/option.cpp index 1fb93a2482..d886128fb5 100644 --- a/src/option.cpp +++ b/src/option.cpp @@ -123,23 +123,53 @@ std::vector librealsense::command_transfer_over_xu::send_receive(const std::dec << data.size() << " exceeds permitted limit " << HW_MONITOR_BUFFER_SIZE); } - std::vector transmit_buf(HW_MONITOR_BUFFER_SIZE, 0); - std::copy(data.begin(), data.end(), transmit_buf.begin()); - - if (!dev.set_xu(_xu, _ctrl, transmit_buf.data(), static_cast(transmit_buf.size()))) - throw invalid_value_exception(to_string() << "set_xu(ctrl=" << unsigned(_ctrl) << ") failed!" << " Last Error: " << strerror(errno)); - - if (require_response) + try{ + std::vector transmit_buf(HW_MONITOR_BUFFER_SIZE, 0); + std::copy(data.begin(), data.end(), transmit_buf.begin()); + + if (!dev.set_xu(_xu, _ctrl, transmit_buf.data(), static_cast(transmit_buf.size()))) + throw invalid_value_exception(to_string() << "set_xu(ctrl=" << unsigned(_ctrl) << ") failed!" << " Last Error: " << strerror(errno)); + + if (require_response) + { + result.resize(HW_MONITOR_BUFFER_SIZE); + if (!dev.get_xu(_xu, _ctrl, result.data(), static_cast(result.size()))) + throw invalid_value_exception(to_string() << "get_xu(ctrl=" << unsigned(_ctrl) << ") failed!" << " Last Error: " << strerror(errno)); + + // Returned data size located in the last 4 bytes + auto data_size = *(reinterpret_cast(result.data() + HW_MONITOR_DATA_SIZE_OFFSET)) + SIZE_OF_HW_MONITOR_HEADER; + result.resize(data_size); + } + return result; + } + catch(...) { - result.resize(HW_MONITOR_BUFFER_SIZE); - if (!dev.get_xu(_xu, _ctrl, result.data(), static_cast(result.size()))) - throw invalid_value_exception(to_string() << "get_xu(ctrl=" << unsigned(_ctrl) << ") failed!" << " Last Error: " << strerror(errno)); - - // Returned data size located in the last 4 bytes - auto data_size = *(reinterpret_cast(result.data() + HW_MONITOR_DATA_SIZE_OFFSET)) + SIZE_OF_HW_MONITOR_HEADER; - result.resize(data_size); + // D457 - size of 1028 needed instead of 1024 (HW_MONITOR_BUFFER_SIZE) + std::vector transmit_buf(HW_MONITOR_BUFFER_SIZE + SIZE_OF_HW_MONITOR_HEADER, 0); + std::copy(data.begin(), data.end(), transmit_buf.begin()); + + if (!dev.set_xu(_xu, _ctrl, transmit_buf.data(), static_cast(transmit_buf.size()))) + throw invalid_value_exception(to_string() << "set_xu(ctrl=" << unsigned(_ctrl) << ") failed!" << " Last Error: " << strerror(errno)); + + if (require_response) + { + result.resize(HW_MONITOR_BUFFER_SIZE); + if (!dev.get_xu(_xu, _ctrl, result.data(), static_cast(result.size()))) + throw invalid_value_exception(to_string() << "get_xu(ctrl=" << unsigned(_ctrl) << ") failed!" << " Last Error: " << strerror(errno)); + + // Returned data size located in the last 4 bytes + auto data_size = *(reinterpret_cast(result.data() + HW_MONITOR_DATA_SIZE_OFFSET + SIZE_OF_HW_MONITOR_HEADER)) ; + result.resize(data_size); + + // D457 code - stepping over 24 bytes: + // 4 bytes: header and magic number + // 20 bytes: input command + // this code may be removed after some bug correction in the kernel code + result.insert(result.begin(),transmit_buf.begin() + 24,transmit_buf.begin() + 24 + data_size); + } + return result; } - return result; + }); } From fa7f2a7d3bde0ddc2b557bc7decba8640e25d9b5 Mon Sep 17 00:00:00 2001 From: Remi Bettan Date: Thu, 7 Jul 2022 11:14:02 +0300 Subject: [PATCH 15/25] permitting usb metadata - v4l_uvc_device checked --- src/linux/backend-v4l2.cpp | 110 +++++++++++++++++++++++++------------ 1 file changed, 74 insertions(+), 36 deletions(-) diff --git a/src/linux/backend-v4l2.cpp b/src/linux/backend-v4l2.cpp index 14b4dc17ce..740e6cab2d 100644 --- a/src/linux/backend-v4l2.cpp +++ b/src/linux/backend-v4l2.cpp @@ -1184,54 +1184,92 @@ namespace librealsense } else { - if (!is_metadata_streamed()) // when metadata is not enabled at all, streaming only video + if (!_info.has_metadata_node) { - auto timestamp = (double)buf.timestamp.tv_sec * 1000.f + (double)buf.timestamp.tv_usec / 1000.f; - timestamp = monotonic_to_realtime(timestamp); - - LOG_DEBUG_V4L("no metadata streamed"); - if (buf_mgr.verify_vd_md_sync()) + if(has_metadata()) { - buffer->attach_buffer(buf); - buf_mgr.handle_buffer(e_video_buf, -1); // transfer new buffer request to the frame callback - - - auto frame_sz = buf_mgr.md_node_present() ? buf.bytesused : - std::min(buf.bytesused - buf_mgr.metadata_size(), - buffer->get_length_frame_only()); + auto timestamp = (double)buf.timestamp.tv_sec*1000.f + (double)buf.timestamp.tv_usec/1000.f; + timestamp = monotonic_to_realtime(timestamp); - uint8_t md_size = buf_mgr.metadata_size(); - void* md_start = buf_mgr.metadata_start(); + // Read metadata. Metadata node performs a blocking call to ensure video and metadata sync + acquire_metadata(buf_mgr,fds,compressed_format); + md_extracted = true; - // D457 development - hid over uvc - md size for IMU is 64 - metadata_hid_raw meta_data{}; - if (md_size == 0 && buffer->get_length_frame_only() <= 64) + if (wa_applied) { - // Populate HID IMU data - Header - meta_data.header.report_type = md_hid_report_type::hid_report_imu; - meta_data.header.length = hid_header_size + metadata_imu_report_size; - meta_data.header.timestamp = *(reinterpret_cast(buffer->get_frame_start() + offsetof(hid_mipi_data, hwTs))); - // Payload: - meta_data.report_type.imu_report.header.md_type_id = md_type::META_DATA_HID_IMU_REPORT_ID; - meta_data.report_type.imu_report.header.md_size = metadata_imu_report_size; - - md_size = sizeof(metadata_hid_raw); - md_start = &meta_data; + auto fn = *(uint32_t*)((char*)(buf_mgr.metadata_start())+28); + LOG_DEBUG_V4L("Extracting md buff, fn = " << fn); } - frame_object fo{ frame_sz, md_size, - buffer->get_frame_start(), md_start, timestamp }; + auto frame_sz = buf_mgr.md_node_present() ? buf.bytesused : + std::min(buf.bytesused - buf_mgr.metadata_size(), buffer->get_length_frame_only()); + frame_object fo{ frame_sz, buf_mgr.metadata_size(), + buffer->get_frame_start(), buf_mgr.metadata_start(), timestamp }; - //Invoke user callback and enqueue next frame - _callback(_profile, fo, [buf_mgr]() mutable { - buf_mgr.request_next_frame(); - }); + buffer->attach_buffer(buf); + buf_mgr.handle_buffer(e_video_buf,-1); // transfer new buffer request to the frame callback + if (buf_mgr.verify_vd_md_sync()) + { + //Invoke user callback and enqueue next frame + _callback(_profile, fo, [buf_mgr]() mutable { + buf_mgr.request_next_frame(); + }); + } + else + { + LOG_WARNING("Video frame dropped, video and metadata buffers inconsistency"); + } } - else + else // when metadata is not enabled at all, streaming only video { - LOG_WARNING("Video frame dropped, video and metadata buffers inconsistency"); + auto timestamp = (double)buf.timestamp.tv_sec * 1000.f + (double)buf.timestamp.tv_usec / 1000.f; + timestamp = monotonic_to_realtime(timestamp); + + LOG_DEBUG_V4L("no metadata streamed"); + if (buf_mgr.verify_vd_md_sync()) + { + buffer->attach_buffer(buf); + buf_mgr.handle_buffer(e_video_buf, -1); // transfer new buffer request to the frame callback + + + auto frame_sz = buf_mgr.md_node_present() ? buf.bytesused : + std::min(buf.bytesused - buf_mgr.metadata_size(), + buffer->get_length_frame_only()); + + uint8_t md_size = buf_mgr.metadata_size(); + void* md_start = buf_mgr.metadata_start(); + + // D457 development - hid over uvc - md size for IMU is 64 + metadata_hid_raw meta_data{}; + if (md_size == 0 && buffer->get_length_frame_only() <= 64) + { + // Populate HID IMU data - Header + meta_data.header.report_type = md_hid_report_type::hid_report_imu; + meta_data.header.length = hid_header_size + metadata_imu_report_size; + meta_data.header.timestamp = *(reinterpret_cast(buffer->get_frame_start() + offsetof(hid_mipi_data, hwTs))); + // Payload: + meta_data.report_type.imu_report.header.md_type_id = md_type::META_DATA_HID_IMU_REPORT_ID; + meta_data.report_type.imu_report.header.md_size = metadata_imu_report_size; + + md_size = sizeof(metadata_hid_raw); + md_start = &meta_data; + } + + frame_object fo{ frame_sz, md_size, + buffer->get_frame_start(), md_start, timestamp }; + + //Invoke user callback and enqueue next frame + _callback(_profile, fo, [buf_mgr]() mutable { + buf_mgr.request_next_frame(); + }); + } + else + { + LOG_WARNING("Video frame dropped, video and metadata buffers inconsistency"); + } } + } else { From f40255bd4315cdc56732d0cdb7621827053c8d57 Mon Sep 17 00:00:00 2001 From: Remi Bettan Date: Thu, 7 Jul 2022 12:38:05 +0300 Subject: [PATCH 16/25] removing ifdef flags --- src/linux/backend-v4l2.cpp | 3 --- src/option.cpp | 49 -------------------------------------- 2 files changed, 52 deletions(-) diff --git a/src/linux/backend-v4l2.cpp b/src/linux/backend-v4l2.cpp index 740e6cab2d..02706ff304 100644 --- a/src/linux/backend-v4l2.cpp +++ b/src/linux/backend-v4l2.cpp @@ -655,8 +655,6 @@ namespace librealsense // /sys/devices/pci0000:00/0000:00:xx.0/ABC/M-N/version usb_specification = get_usb_connection_type(real_path + "/../../../"); } -#define enumerate_MIPI -#ifdef enumerate_MIPI else // COMMENT THIS TO MAKE USB ENUMERATION POSSIBLE - Video4Linux Devices that are not listed as UVC { //LOG_INFO("Enumerating v4l " << name << " realpath=" << real_path); @@ -708,7 +706,6 @@ namespace librealsense break; } } -#endif // enumerate_MIPI // D457 Dev - skip unsupported devices. Do no upstream! // manually rectify descriptor inconsistencies diff --git a/src/option.cpp b/src/option.cpp index d886128fb5..14dea30df0 100644 --- a/src/option.cpp +++ b/src/option.cpp @@ -105,9 +105,6 @@ const char* librealsense::uvc_pu_option::get_description() const } } -#undef HWM_MIPI -#ifndef HWM_MIPI - std::vector librealsense::command_transfer_over_xu::send_receive(const std::vector& data, int, bool require_response) { return _uvc.invoke_powered([this, &data, require_response] @@ -173,52 +170,6 @@ std::vector librealsense::command_transfer_over_xu::send_receive(const }); } -#else - -std::vector librealsense::command_transfer_over_xu::send_receive(const std::vector& data, int, bool require_response) -{ - return _uvc.invoke_powered([this, &data, require_response] - (platform::uvc_device& dev) - { - std::vector result; - std::lock_guard lock(dev); - - if (data.size() > HW_MONITOR_BUFFER_SIZE) - { - LOG_ERROR("XU command size is invalid"); - throw invalid_value_exception(to_string() << "Requested XU command size " << - std::dec << data.size() << " exceeds permitted limit " << HW_MONITOR_BUFFER_SIZE); - } - - // D457 - size of 1028 needed instead of 1024 (HW_MONITOR_BUFFER_SIZE) - std::vector transmit_buf(HW_MONITOR_BUFFER_SIZE + SIZE_OF_HW_MONITOR_HEADER, 0); - std::copy(data.begin(), data.end(), transmit_buf.begin()); - - if (!dev.set_xu(_xu, _ctrl, transmit_buf.data(), static_cast(transmit_buf.size()))) - throw invalid_value_exception(to_string() << "set_xu(ctrl=" << unsigned(_ctrl) << ") failed!" << " Last Error: " << strerror(errno)); - - if (require_response) - { - result.resize(HW_MONITOR_BUFFER_SIZE); - if (!dev.get_xu(_xu, _ctrl, result.data(), static_cast(result.size()))) - throw invalid_value_exception(to_string() << "get_xu(ctrl=" << unsigned(_ctrl) << ") failed!" << " Last Error: " << strerror(errno)); - - // Returned data size located in the last 4 bytes - auto data_size = *(reinterpret_cast(result.data() + HW_MONITOR_DATA_SIZE_OFFSET + SIZE_OF_HW_MONITOR_HEADER)) ; - result.resize(data_size); - - // D457 code - stepping over 24 bytes: - // 4 bytes: header and magic number - // 20 bytes: input command - // this code may be removed after some bug correction in the kernel code - result.insert(result.begin(),transmit_buf.begin() + 24,transmit_buf.begin() + 24 + data_size); - } - return result; - }); -} - -#endif // HWM_MIPI - librealsense::polling_errors_disable::~polling_errors_disable() { if (auto handler = _polling_error_handler.lock()) From 7d1da08c0359e407e76039e006592c1042aec2cf Mon Sep 17 00:00:00 2001 From: Remi Bettan Date: Thu, 7 Jul 2022 15:48:51 +0300 Subject: [PATCH 17/25] adding private methods --- src/linux/backend-v4l2.cpp | 137 ++++++++++++++++++++----------------- src/linux/backend-v4l2.h | 2 + 2 files changed, 76 insertions(+), 63 deletions(-) diff --git a/src/linux/backend-v4l2.cpp b/src/linux/backend-v4l2.cpp index 02706ff304..d51815f12e 100644 --- a/src/linux/backend-v4l2.cpp +++ b/src/linux/backend-v4l2.cpp @@ -1242,15 +1242,7 @@ namespace librealsense if (md_size == 0 && buffer->get_length_frame_only() <= 64) { // Populate HID IMU data - Header - meta_data.header.report_type = md_hid_report_type::hid_report_imu; - meta_data.header.length = hid_header_size + metadata_imu_report_size; - meta_data.header.timestamp = *(reinterpret_cast(buffer->get_frame_start() + offsetof(hid_mipi_data, hwTs))); - // Payload: - meta_data.report_type.imu_report.header.md_type_id = md_type::META_DATA_HID_IMU_REPORT_ID; - meta_data.report_type.imu_report.header.md_size = metadata_imu_report_size; - - md_size = sizeof(metadata_hid_raw); - md_start = &meta_data; + populate_imu_data(meta_data, buffer->get_frame_start(), md_size, &md_start); } frame_object fo{ frame_sz, md_size, @@ -1266,7 +1258,6 @@ namespace librealsense LOG_WARNING("Video frame dropped, video and metadata buffers inconsistency"); } } - } else { @@ -1288,70 +1279,90 @@ namespace librealsense LOG_DEBUG("FD_ISSET: no data on video node sink"); } - // uploading to user's callback - std::shared_ptr video_v4l2_buffer; - std::shared_ptr md_v4l2_buffer; + // pulling synchronized video and metadata and uploading them to user's callback + upload_video_and_metadata_from_syncer(buf_mgr); + } + } + else // (val==0) + { + LOG_WARNING("Frames didn't arrived within 5 seconds"); + librealsense::notification n = {RS2_NOTIFICATION_CATEGORY_FRAMES_TIMEOUT, 0, RS2_LOG_SEVERITY_WARN, "Frames didn't arrived within 5 seconds"}; - if (_is_started && is_metadata_streamed()) - { - int video_fd = -1, md_fd = -1; - if (_video_md_syncer.pull_video_with_metadata(video_v4l2_buffer, md_v4l2_buffer, video_fd, md_fd)) - { - // Preparing video buffer - auto video_buffer = get_video_buffer(video_v4l2_buffer->index); - video_buffer->attach_buffer(*video_v4l2_buffer); + _error_handler(n); + } + } + } - // happens when the video did not arrive on - // the current polling iteration (was taken from the syncer's video queue) - if (buf_mgr.get_buffers()[e_video_buf]._file_desc == -1) - { - buf_mgr.handle_buffer(e_video_buf, video_fd, *video_v4l2_buffer, video_buffer); - } - buf_mgr.handle_buffer(e_video_buf, -1); // transfer new buffer request to the frame callback + void v4l_uvc_device::populate_imu_data(metadata_hid_raw& meta_data, uint8_t* frame_start, uint8_t& md_size, void** md_start) const + { + meta_data.header.report_type = md_hid_report_type::hid_report_imu; + meta_data.header.length = hid_header_size + metadata_imu_report_size; + meta_data.header.timestamp = *(reinterpret_cast(frame_start + offsetof(hid_mipi_data, hwTs))); + // Payload: + meta_data.report_type.imu_report.header.md_type_id = md_type::META_DATA_HID_IMU_REPORT_ID; + meta_data.report_type.imu_report.header.md_size = metadata_imu_report_size; - // Preparing metadata buffer - static const size_t uvc_md_start_offset = sizeof(uvc_meta_buffer::ns) + sizeof(uvc_meta_buffer::sof); - auto metadata_buffer = get_md_buffer(md_v4l2_buffer->index); - buf_mgr.set_md_attributes(md_v4l2_buffer->bytesused, - metadata_buffer->get_frame_start()); - metadata_buffer->attach_buffer(*md_v4l2_buffer); + md_size = sizeof(metadata_hid_raw); + *md_start = &meta_data; + } - if (buf_mgr.get_buffers()[e_metadata_buf]._file_desc == -1) - { - buf_mgr.handle_buffer(e_metadata_buf, md_fd, *md_v4l2_buffer, metadata_buffer); - } - buf_mgr.handle_buffer(e_metadata_buf, -1); // transfer new buffer request to the frame callback - auto frame_sz = buf_mgr.md_node_present() ? video_v4l2_buffer->bytesused : - std::min(video_v4l2_buffer->bytesused - buf_mgr.metadata_size(), - video_buffer->get_length_frame_only()); + void v4l_uvc_device::upload_video_and_metadata_from_syncer(buffers_mgr& buf_mgr) + { + // uploading to user's callback + std::shared_ptr video_v4l2_buffer; + std::shared_ptr md_v4l2_buffer; - auto timestamp = (double)video_v4l2_buffer->timestamp.tv_sec * 1000.f + (double)video_v4l2_buffer->timestamp.tv_usec / 1000.f; - timestamp = monotonic_to_realtime(timestamp); + if (_is_started && is_metadata_streamed()) + { + int video_fd = -1, md_fd = -1; + if (_video_md_syncer.pull_video_with_metadata(video_v4l2_buffer, md_v4l2_buffer, video_fd, md_fd)) + { + // Preparing video buffer + auto video_buffer = get_video_buffer(video_v4l2_buffer->index); + video_buffer->attach_buffer(*video_v4l2_buffer); - // D457 work - to work with "normal camera", use frame_sz as the first input to the following frame_object: - //frame_object fo{ buf.bytesused - MAX_META_DATA_SIZE, buf_mgr.metadata_size(), - frame_object fo{ frame_sz, buf_mgr.metadata_size(), - video_buffer->get_frame_start(), buf_mgr.metadata_start(), timestamp }; + // happens when the video did not arrive on + // the current polling iteration (was taken from the syncer's video queue) + if (buf_mgr.get_buffers()[e_video_buf]._file_desc == -1) + { + buf_mgr.handle_buffer(e_video_buf, video_fd, *video_v4l2_buffer, video_buffer); + } + buf_mgr.handle_buffer(e_video_buf, -1); // transfer new buffer request to the frame callback - //Invoke user callback and enqueue next frame - _callback(_profile, fo, [buf_mgr]() mutable { - buf_mgr.request_next_frame(); - }); - } - else - { - LOG_DEBUG("video_md_syncer - synchronized video and md could not be pulled"); - } - } + // Preparing metadata buffer + static const size_t uvc_md_start_offset = sizeof(uvc_meta_buffer::ns) + sizeof(uvc_meta_buffer::sof); + auto metadata_buffer = get_md_buffer(md_v4l2_buffer->index); + buf_mgr.set_md_attributes(md_v4l2_buffer->bytesused, + metadata_buffer->get_frame_start()); + metadata_buffer->attach_buffer(*md_v4l2_buffer); + + if (buf_mgr.get_buffers()[e_metadata_buf]._file_desc == -1) + { + buf_mgr.handle_buffer(e_metadata_buf, md_fd, *md_v4l2_buffer, metadata_buffer); } + buf_mgr.handle_buffer(e_metadata_buf, -1); // transfer new buffer request to the frame callback + + auto frame_sz = buf_mgr.md_node_present() ? video_v4l2_buffer->bytesused : + std::min(video_v4l2_buffer->bytesused - buf_mgr.metadata_size(), + video_buffer->get_length_frame_only()); + + auto timestamp = (double)video_v4l2_buffer->timestamp.tv_sec * 1000.f + (double)video_v4l2_buffer->timestamp.tv_usec / 1000.f; + timestamp = monotonic_to_realtime(timestamp); + + // D457 work - to work with "normal camera", use frame_sz as the first input to the following frame_object: + //frame_object fo{ buf.bytesused - MAX_META_DATA_SIZE, buf_mgr.metadata_size(), + frame_object fo{ frame_sz, buf_mgr.metadata_size(), + video_buffer->get_frame_start(), buf_mgr.metadata_start(), timestamp }; + + //Invoke user callback and enqueue next frame + _callback(_profile, fo, [buf_mgr]() mutable { + buf_mgr.request_next_frame(); + }); } - else // (val==0) + else { - LOG_WARNING("Frames didn't arrived within 5 seconds"); - librealsense::notification n = {RS2_NOTIFICATION_CATEGORY_FRAMES_TIMEOUT, 0, RS2_LOG_SEVERITY_WARN, "Frames didn't arrived within 5 seconds"}; - - _error_handler(n); + LOG_DEBUG("video_md_syncer - synchronized video and md could not be pulled"); } } } diff --git a/src/linux/backend-v4l2.h b/src/linux/backend-v4l2.h index 1d7e3711a2..4be33ff342 100644 --- a/src/linux/backend-v4l2.h +++ b/src/linux/backend-v4l2.h @@ -354,6 +354,8 @@ namespace librealsense void subscribe_to_ctrl_event(uint32_t control_id); void unsubscribe_from_ctrl_event(uint32_t control_id); bool pend_for_ctrl_status_event(); + void upload_video_and_metadata_from_syncer(buffers_mgr& buf_mgr); + void populate_imu_data(metadata_hid_raw& meta_data, uint8_t* frame_start, uint8_t& md_size, void** md_start) const; // checking if metadata is streamed virtual inline bool is_metadata_streamed() const { return false;} virtual inline std::shared_ptr get_video_buffer(__u32 index) const {return _buffers[index];} From 101d6b595adfd8279cd1f8ebfe42f19369e3e205 Mon Sep 17 00:00:00 2001 From: Remi Bettan Date: Sun, 10 Jul 2022 13:17:51 +0300 Subject: [PATCH 18/25] v4l2_ext_controls initialized to fit different kernel versions --- src/linux/backend-v4l2.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/linux/backend-v4l2.cpp b/src/linux/backend-v4l2.cpp index d51815f12e..5c54d23e4d 100644 --- a/src/linux/backend-v4l2.cpp +++ b/src/linux/backend-v4l2.cpp @@ -2189,7 +2189,7 @@ namespace librealsense xctrl.value = xctrl.value ? V4L2_EXPOSURE_APERTURE_PRIORITY : V4L2_EXPOSURE_MANUAL; // Extract the control group from the underlying control query - v4l2_ext_controls ctrls_block { xctrl.id&0xffff0000, 1, 0, {0, 0}, &xctrl }; + v4l2_ext_controls ctrls_block { xctrl.id&0xffff0000, 1, 0, 0, 0, &xctrl }; int retVal = xioctl(_fd, VIDIOC_S_EXT_CTRLS, &ctrls_block); if (retVal < 0) @@ -2207,7 +2207,7 @@ namespace librealsense v4l2_ext_control xctrl{xu_to_cid(xu,control), uint32_t(size), 0, 0}; xctrl.p_u8 = data; - struct v4l2_ext_controls ext {xctrl.id & 0xffff0000, 1, 0, {0, 0}, &xctrl}; + v4l2_ext_controls ext {xctrl.id & 0xffff0000, 1, 0, 0, 0, &xctrl}; // the ioctl fails once when performing send and receive right after it // it succeeds on the second time From 0a37ce438209020542cf15e02be9a96ed74d3822 Mon Sep 17 00:00:00 2001 From: Remi Bettan Date: Tue, 12 Jul 2022 12:12:01 +0300 Subject: [PATCH 19/25] hwm w/a - improved by setting mipi or usb api once instead of try catch at each call --- src/ds5/ds5-device.cpp | 2 +- src/option.cpp | 5 +++-- src/option.h | 5 +++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/ds5/ds5-device.cpp b/src/ds5/ds5-device.cpp index 82f3960a85..7c9c335c62 100644 --- a/src/ds5/ds5-device.cpp +++ b/src/ds5/ds5-device.cpp @@ -774,7 +774,7 @@ namespace librealsense _hw_monitor = std::make_shared( std::make_shared( std::make_shared( - raw_sensor, depth_xu, DS5_HWMONITOR), + raw_sensor, depth_xu, DS5_HWMONITOR, mipi_sensor), raw_sensor)); } else diff --git a/src/option.cpp b/src/option.cpp index 14dea30df0..c75b220e05 100644 --- a/src/option.cpp +++ b/src/option.cpp @@ -120,7 +120,8 @@ std::vector librealsense::command_transfer_over_xu::send_receive(const std::dec << data.size() << " exceeds permitted limit " << HW_MONITOR_BUFFER_SIZE); } - try{ + if (!_is_mipi) + { std::vector transmit_buf(HW_MONITOR_BUFFER_SIZE, 0); std::copy(data.begin(), data.end(), transmit_buf.begin()); @@ -139,7 +140,7 @@ std::vector librealsense::command_transfer_over_xu::send_receive(const } return result; } - catch(...) + else { // D457 - size of 1028 needed instead of 1024 (HW_MONITOR_BUFFER_SIZE) std::vector transmit_buf(HW_MONITOR_BUFFER_SIZE + SIZE_OF_HW_MONITOR_HEADER, 0); diff --git a/src/option.h b/src/option.h index 9f37d56d46..4c3a702d8c 100644 --- a/src/option.h +++ b/src/option.h @@ -492,14 +492,15 @@ namespace librealsense std::vector send_receive(const std::vector& data, int, bool require_response) override; command_transfer_over_xu(uvc_sensor& uvc, - platform::extension_unit xu, uint8_t ctrl) - : _uvc(uvc), _xu(std::move(xu)), _ctrl(ctrl) + platform::extension_unit xu, uint8_t ctrl, bool is_mipi = false) + : _uvc(uvc), _xu(std::move(xu)), _ctrl(ctrl), _is_mipi(is_mipi) {} private: uvc_sensor& _uvc; platform::extension_unit _xu; uint8_t _ctrl; + bool _is_mipi; // D457 dev- should be removed after hwm discrepancy USB/MIPI fixed in driver }; class polling_error_handler; From 0354229b2e9c709696fc50330f9e1a0908718555 Mon Sep 17 00:00:00 2001 From: Remi Bettan Date: Tue, 12 Jul 2022 13:24:09 +0300 Subject: [PATCH 20/25] fix metadata offset acc to v4l device class --- src/linux/backend-v4l2.cpp | 19 +++++++++++++++---- src/linux/backend-v4l2.h | 2 ++ 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/src/linux/backend-v4l2.cpp b/src/linux/backend-v4l2.cpp index 5c54d23e4d..57fc794f76 100644 --- a/src/linux/backend-v4l2.cpp +++ b/src/linux/backend-v4l2.cpp @@ -1331,10 +1331,8 @@ namespace librealsense buf_mgr.handle_buffer(e_video_buf, -1); // transfer new buffer request to the frame callback // Preparing metadata buffer - static const size_t uvc_md_start_offset = sizeof(uvc_meta_buffer::ns) + sizeof(uvc_meta_buffer::sof); auto metadata_buffer = get_md_buffer(md_v4l2_buffer->index); - buf_mgr.set_md_attributes(md_v4l2_buffer->bytesused, - metadata_buffer->get_frame_start()); + set_metadata_attributes(buf_mgr, md_v4l2_buffer->bytesused, metadata_buffer->get_frame_start()); metadata_buffer->attach_buffer(*md_v4l2_buffer); if (buf_mgr.get_buffers()[e_metadata_buf]._file_desc == -1) @@ -1367,7 +1365,20 @@ namespace librealsense } } - void v4l_uvc_device::acquire_metadata(buffers_mgr & buf_mgr,fd_set &, bool compressed_format) + void v4l_uvc_device::set_metadata_attributes(buffers_mgr& buf_mgr, __u32 bytesused, uint8_t* md_start) + { + size_t uvc_md_start_offset = sizeof(uvc_meta_buffer::ns) + sizeof(uvc_meta_buffer::sof); + buf_mgr.set_md_attributes(bytesused - uvc_md_start_offset, + md_start + uvc_md_start_offset); + } + + void v4l_mipi_device::set_metadata_attributes(buffers_mgr& buf_mgr, __u32 bytesused, uint8_t* md_start) + { + buf_mgr.set_md_attributes(bytesused, md_start); + } + + + void v4l_uvc_device::acquire_metadata(buffers_mgr& buf_mgr,fd_set &, bool compressed_format) { if (has_metadata()) buf_mgr.set_md_from_video_node(compressed_format); diff --git a/src/linux/backend-v4l2.h b/src/linux/backend-v4l2.h index 4be33ff342..510c8e1dfb 100644 --- a/src/linux/backend-v4l2.h +++ b/src/linux/backend-v4l2.h @@ -351,6 +351,7 @@ namespace librealsense virtual void prepare_capture_buffers() override; virtual void stop_data_capture() override; virtual void acquire_metadata(buffers_mgr & buf_mgr,fd_set &fds, bool compressed_format = false) override; + virtual void set_metadata_attributes(buffers_mgr& buf_mgr, __u32 bytesused, uint8_t* md_start); void subscribe_to_ctrl_event(uint32_t control_id); void unsubscribe_from_ctrl_event(uint32_t control_id); bool pend_for_ctrl_status_event(); @@ -430,6 +431,7 @@ namespace librealsense bool get_xu(const extension_unit& xu, uint8_t control, uint8_t* data, int size) const override; control_range get_xu_range(const extension_unit& xu, uint8_t control, int len) const override; control_range get_pu_range(rs2_option option) const override; + void set_metadata_attributes(buffers_mgr& buf_mgr, __u32 bytesused, uint8_t* md_start) override; protected: virtual uint32_t get_cid(rs2_option option) const; uint32_t xu_to_cid(const extension_unit& xu, uint8_t control) const; // Find the mapping of XU to the underlying control From 9521c1a142787f5caeff85ed8d222a4bbe6480cd Mon Sep 17 00:00:00 2001 From: Remi Bettan Date: Wed, 13 Jul 2022 14:45:20 +0300 Subject: [PATCH 21/25] video-md syncer - override md buffer when same sequence number is pushed --- src/linux/backend-v4l2.cpp | 23 +++++++++++++++++++---- src/linux/backend-v4l2.h | 1 + 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/linux/backend-v4l2.cpp b/src/linux/backend-v4l2.cpp index 57fc794f76..c8195ca085 100644 --- a/src/linux/backend-v4l2.cpp +++ b/src/linux/backend-v4l2.cpp @@ -2414,14 +2414,18 @@ namespace librealsense if (_video_queue.size() > 2) { // Enqueue of video buffer before throwing its content away - enqueue_buffer_before_throwing_it(_video_queue.front()); - _video_queue.pop(); + enqueue_front_buffer_before_throwing_it(_video_queue); } } void v4l2_video_md_syncer::push_metadata(const sync_buffer& md_buffer) { std::lock_guard lock(_syncer_mutex); + + // override front buffer if it has the same sequence that the new buffer - happens with metadata sequence 0 + if (_md_queue.size() > 0 && _md_queue.front()._v4l2_buf->sequence == md_buffer._v4l2_buf->sequence) + enqueue_front_buffer_before_throwing_it(_md_queue); + _md_queue.push(md_buffer); LOG_DEBUG_V4L("video_md_syncer - md pushed with sequence " << md_buffer._v4l2_buf->sequence); @@ -2429,8 +2433,7 @@ namespace librealsense if (_md_queue.size() > 2) { // Enqueue of md buffer before throwing its content away - enqueue_buffer_before_throwing_it(_md_queue.front()); - _md_queue.pop(); + enqueue_front_buffer_before_throwing_it(_md_queue); } } @@ -2524,6 +2527,18 @@ namespace librealsense } } + void v4l2_video_md_syncer::enqueue_front_buffer_before_throwing_it(std::queue& sync_queue) + { + // Enqueue of buffer before throwing its content away + LOG_DEBUG_V4L("video_md_syncer - Enqueue buf " << std::dec << sync_queue.front()._buffer_index << " for fd " << sync_queue.front()._fd << " before dropping it"); + if (xioctl(sync_queue.front()._fd, VIDIOC_QBUF, sync_queue.front()._v4l2_buf.get()) < 0) + { + LOG_ERROR("xioctl(VIDIOC_QBUF) failed when requesting new frame! fd: " << sync_queue.front()._fd << " error: " << strerror(errno)); + } + sync_queue.pop(); + } + + void v4l2_video_md_syncer::flush() { // Empty queues diff --git a/src/linux/backend-v4l2.h b/src/linux/backend-v4l2.h index 510c8e1dfb..c177866f42 100644 --- a/src/linux/backend-v4l2.h +++ b/src/linux/backend-v4l2.h @@ -278,6 +278,7 @@ namespace librealsense private: void enqueue_buffer_before_throwing_it(const sync_buffer& sb) const; + void enqueue_front_buffer_before_throwing_it(std::queue& sync_queue); std::mutex _syncer_mutex; std::queue _video_queue; From 1e69df1e51b40edffaf10e5e1ac6880ff1361409 Mon Sep 17 00:00:00 2001 From: Remi Bettan Date: Sun, 17 Jul 2022 13:45:49 +0300 Subject: [PATCH 22/25] DFU via MIPI - started --- common/fw-update-helper.cpp | 44 +++++++++++++++++++++++++++++++++++++ common/fw-update-helper.h | 3 ++- 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/common/fw-update-helper.cpp b/common/fw-update-helper.cpp index 43707dfc60..29a7efe2e9 100644 --- a/common/fw-update-helper.cpp +++ b/common/fw-update-helper.cpp @@ -194,10 +194,54 @@ namespace rs2 return false; } + void firmware_update_manager::process_flow_mipi() + { + if (!_is_signed) + { + LOG_INFO("Only Signed Firmware can be burnt on MIPI device"); + return; + } + + // Enter DFU mode + auto device_debug = _dev.as(); + uint32_t dfu_opcode = 0x1e; + device_debug.build_command(dfu_opcode, 1); + + _progress = 30; + // Grant permissions for writing + // skipped for now - must be done in sudo + //chmod("/dev/d4xx-dfu504", __S_IREAD|__S_IWRITE); + + // Write signed firmware to appropriate file descritptor + std::ofstream fw_path_in_device("/dev/d4xx-dfu504", std::ios::binary); + if (fw_path_in_device) + { + fw_path_in_device.write(reinterpret_cast(_fw.data()), _fw.size()); + } + else + { + fail("Firmware Update failed - wrong path or permissions missing"); + return; + } + LOG_INFO("Firmware Update for MIPI device done."); + fw_path_in_device.close(); + + _progress = 100; + _done = true; + // need to find a way to update the fw version field in the viewer + } + void firmware_update_manager::process_flow( std::function cleanup, invoker invoke) { + auto str = _dev.get_info(RS2_CAMERA_INFO_PRODUCT_ID); + if (!strcmp(_dev.get_info(RS2_CAMERA_INFO_PRODUCT_ID), "ABCD")) // if device is D457 + { + process_flow_mipi(); + return; + } + std::string serial = ""; if (_dev.supports(RS2_CAMERA_INFO_FIRMWARE_UPDATE_ID)) serial = _dev.get_info(RS2_CAMERA_INFO_FIRMWARE_UPDATE_ID); diff --git a/common/fw-update-helper.h b/common/fw-update-helper.h index a6ee4a74b8..1f2b05ea73 100644 --- a/common/fw-update-helper.h +++ b/common/fw-update-helper.h @@ -31,6 +31,7 @@ namespace rs2 private: void process_flow(std::function cleanup, invoker invoke) override; + void process_flow_mipi(); bool check_for( std::function action, std::function cleanup, std::chrono::system_clock::duration delta); @@ -53,4 +54,4 @@ namespace rs2 void draw_expanded(ux_window& win, std::string& error_message) override; int calc_height() override; }; -} \ No newline at end of file +} From bef6e811c5d95eabbf3d6a22d94d2048d21e17a3 Mon Sep 17 00:00:00 2001 From: Remi Bettan Date: Tue, 19 Jul 2022 18:45:09 +0300 Subject: [PATCH 23/25] usb hwm api used also for mipi --- src/linux/backend-v4l2.cpp | 12 ++++---- src/option.cpp | 61 ++++++++++---------------------------- 2 files changed, 21 insertions(+), 52 deletions(-) diff --git a/src/linux/backend-v4l2.cpp b/src/linux/backend-v4l2.cpp index c8195ca085..3efb6d948a 100644 --- a/src/linux/backend-v4l2.cpp +++ b/src/linux/backend-v4l2.cpp @@ -74,7 +74,7 @@ constexpr uint32_t RS_CAMERA_CID_AE_SETPOINT_GET = (RS_CAMERA_CID_BAS constexpr uint32_t RS_CAMERA_CID_AE_SETPOINT_SET = (RS_CAMERA_CID_BASE+12); constexpr uint32_t RS_CAMERA_CID_ERB = (RS_CAMERA_CID_BASE+13); constexpr uint32_t RS_CAMERA_CID_EWB = (RS_CAMERA_CID_BASE+14); -constexpr uint32_t RS_CAMERA_CID_HWMC = (RS_CAMERA_CID_BASE+15); +constexpr uint32_t RS_CAMERA_CID_HWMC_LEGACY = (RS_CAMERA_CID_BASE+15); //const uint32_t RS_CAMERA_GENERIC_XU = (RS_CAMERA_CID_BASE+15); // RS_CAMERA_CID_HWMC duplicate?? constexpr uint32_t RS_CAMERA_CID_LASER_POWER_MODE = (RS_CAMERA_CID_BASE+16); // RS_CAMERA_CID_LASER_POWER duplicate ??? @@ -84,6 +84,7 @@ constexpr uint32_t RS_CAMERA_CID_EXPOSURE_MODE = (RS_CAMERA_CID_BAS constexpr uint32_t RS_CAMERA_CID_WHITE_BALANCE_MODE = (RS_CAMERA_CID_BASE+20); // Similar to RS_CAMERA_CID_EWB ?? constexpr uint32_t RS_CAMERA_CID_PRESET = (RS_CAMERA_CID_BASE+21); +constexpr uint32_t RS_CAMERA_CID_HWMC = (RS_CAMERA_CID_BASE+32); /* refe4rence for kernel 4.9 to be removed #define UVC_CID_GENERIC_XU (V4L2_CID_PRIVATE_BASE+15) @@ -2232,14 +2233,13 @@ namespace librealsense continue; } - // TODO check if parsing for non-integer values D457 - //memcpy(data,(void*)(&ctrl.value),size); - if (control == RS_ENABLE_AUTO_EXPOSURE) xctrl.value = (V4L2_EXPOSURE_MANUAL == xctrl.value) ? 0 : 1; - // used to parse values that have size > 1 - memcpy(data,(void*)(&xctrl.value), size); + // used to parse the data when only a value is returned (e.g. laser power), + // and not a pointer to a buffer of data (e.g. gvd) + if (size < sizeof(__s64)) + memcpy(data,(void*)(&xctrl.value), size); return true; } diff --git a/src/option.cpp b/src/option.cpp index c75b220e05..92d16eb1ff 100644 --- a/src/option.cpp +++ b/src/option.cpp @@ -120,54 +120,23 @@ std::vector librealsense::command_transfer_over_xu::send_receive(const std::dec << data.size() << " exceeds permitted limit " << HW_MONITOR_BUFFER_SIZE); } - if (!_is_mipi) - { - std::vector transmit_buf(HW_MONITOR_BUFFER_SIZE, 0); - std::copy(data.begin(), data.end(), transmit_buf.begin()); - - if (!dev.set_xu(_xu, _ctrl, transmit_buf.data(), static_cast(transmit_buf.size()))) - throw invalid_value_exception(to_string() << "set_xu(ctrl=" << unsigned(_ctrl) << ") failed!" << " Last Error: " << strerror(errno)); - - if (require_response) - { - result.resize(HW_MONITOR_BUFFER_SIZE); - if (!dev.get_xu(_xu, _ctrl, result.data(), static_cast(result.size()))) - throw invalid_value_exception(to_string() << "get_xu(ctrl=" << unsigned(_ctrl) << ") failed!" << " Last Error: " << strerror(errno)); - - // Returned data size located in the last 4 bytes - auto data_size = *(reinterpret_cast(result.data() + HW_MONITOR_DATA_SIZE_OFFSET)) + SIZE_OF_HW_MONITOR_HEADER; - result.resize(data_size); - } - return result; - } - else + std::vector transmit_buf(HW_MONITOR_BUFFER_SIZE, 0); + std::copy(data.begin(), data.end(), transmit_buf.begin()); + + if (!dev.set_xu(_xu, _ctrl, transmit_buf.data(), static_cast(transmit_buf.size()))) + throw invalid_value_exception(to_string() << "set_xu(ctrl=" << unsigned(_ctrl) << ") failed!" << " Last Error: " << strerror(errno)); + + if (require_response) { - // D457 - size of 1028 needed instead of 1024 (HW_MONITOR_BUFFER_SIZE) - std::vector transmit_buf(HW_MONITOR_BUFFER_SIZE + SIZE_OF_HW_MONITOR_HEADER, 0); - std::copy(data.begin(), data.end(), transmit_buf.begin()); - - if (!dev.set_xu(_xu, _ctrl, transmit_buf.data(), static_cast(transmit_buf.size()))) - throw invalid_value_exception(to_string() << "set_xu(ctrl=" << unsigned(_ctrl) << ") failed!" << " Last Error: " << strerror(errno)); - - if (require_response) - { - result.resize(HW_MONITOR_BUFFER_SIZE); - if (!dev.get_xu(_xu, _ctrl, result.data(), static_cast(result.size()))) - throw invalid_value_exception(to_string() << "get_xu(ctrl=" << unsigned(_ctrl) << ") failed!" << " Last Error: " << strerror(errno)); - - // Returned data size located in the last 4 bytes - auto data_size = *(reinterpret_cast(result.data() + HW_MONITOR_DATA_SIZE_OFFSET + SIZE_OF_HW_MONITOR_HEADER)) ; - result.resize(data_size); - - // D457 code - stepping over 24 bytes: - // 4 bytes: header and magic number - // 20 bytes: input command - // this code may be removed after some bug correction in the kernel code - result.insert(result.begin(),transmit_buf.begin() + 24,transmit_buf.begin() + 24 + data_size); - } - return result; - } + result.resize(HW_MONITOR_BUFFER_SIZE); + if (!dev.get_xu(_xu, _ctrl, result.data(), static_cast(result.size()))) + throw invalid_value_exception(to_string() << "get_xu(ctrl=" << unsigned(_ctrl) << ") failed!" << " Last Error: " << strerror(errno)); + // Returned data size located in the last 4 bytes + auto data_size = *(reinterpret_cast(result.data() + HW_MONITOR_DATA_SIZE_OFFSET)) + SIZE_OF_HW_MONITOR_HEADER; + result.resize(data_size); + } + return result; }); } From 04b3f2554717f4f48b019d74a6b28fee5cfec675 Mon Sep 17 00:00:00 2001 From: Remi Bettan Date: Sun, 24 Jul 2022 14:18:31 +0300 Subject: [PATCH 24/25] check fw compatibility for dfu via mipi --- common/fw-update-helper.cpp | 11 +++++++++-- common/fw-update-helper.h | 21 +++++++++++++++++++-- src/ds5/ds5-private.h | 3 ++- 3 files changed, 30 insertions(+), 5 deletions(-) diff --git a/common/fw-update-helper.cpp b/common/fw-update-helper.cpp index 29a7efe2e9..8ebe80f6b3 100644 --- a/common/fw-update-helper.cpp +++ b/common/fw-update-helper.cpp @@ -194,13 +194,19 @@ namespace rs2 return false; } - void firmware_update_manager::process_flow_mipi() + void firmware_update_manager_mipi::process_flow(std::function cleanup, invoker invoke) { if (!_is_signed) { LOG_INFO("Only Signed Firmware can be burnt on MIPI device"); return; } + auto dev_updatable = _dev.as(); + if(!(dev_updatable && dev_updatable.check_firmware_compatibility(_fw))) + { + fail("Firmware Update failed - fw version must be newer than version 5.13.1.1"); + return; + } // Enter DFU mode auto device_debug = _dev.as(); @@ -238,7 +244,8 @@ namespace rs2 auto str = _dev.get_info(RS2_CAMERA_INFO_PRODUCT_ID); if (!strcmp(_dev.get_info(RS2_CAMERA_INFO_PRODUCT_ID), "ABCD")) // if device is D457 { - process_flow_mipi(); + firmware_update_manager_mipi fw_update_mgr_mipi(_not_model, _model, _dev, _ctx, _fw, _is_signed); + fw_update_mgr_mipi.process_flow(cleanup, invoke); return; } diff --git a/common/fw-update-helper.h b/common/fw-update-helper.h index 1f2b05ea73..73f23846f6 100644 --- a/common/fw-update-helper.h +++ b/common/fw-update-helper.h @@ -4,6 +4,7 @@ #pragma once #include "notifications.h" +#include "../src/fw-update/fw-update-device-interface.h" namespace rs2 @@ -28,10 +29,9 @@ namespace rs2 const device_model& get_device_model() const { return _model; } std::shared_ptr get_protected_notification_model() { return _not_model.lock(); }; - private: + protected: void process_flow(std::function cleanup, invoker invoke) override; - void process_flow_mipi(); bool check_for( std::function action, std::function cleanup, std::chrono::system_clock::duration delta); @@ -44,6 +44,23 @@ namespace rs2 device_model& _model; }; + class firmware_update_manager_mipi : public process_manager + { + public: + firmware_update_manager_mipi(std::weak_ptr not_model, device_model& model, device dev, context ctx, std::vector fw, bool is_signed) + : process_manager("Firmware Update Mipi"), _not_model(not_model), _model(model), + _fw(fw), _is_signed(is_signed), _dev(dev), _ctx(ctx) {} + void process_flow(std::function cleanup, invoker invoke); + + private: + std::weak_ptr _not_model; + device _dev; + context _ctx; + std::vector _fw; + bool _is_signed; + device_model& _model; + }; + struct fw_update_notification_model : public process_notification_model { fw_update_notification_model(std::string name, diff --git a/src/ds5/ds5-private.h b/src/ds5/ds5-private.h index 29262e2812..84a26fd0df 100644 --- a/src/ds5/ds5-private.h +++ b/src/ds5/ds5-private.h @@ -735,7 +735,8 @@ namespace librealsense {RS465_PID, "5.12.7.100" }, {RS416_RGB_PID, "5.8.15.0" }, {RS405_PID, "5.12.11.8" }, - {RS455_PID, "5.12.7.100" } + {RS455_PID, "5.12.7.100" }, + {RS457_PID, "5.13.1.1" } }; From 38323464a4d8fcab9c082b817c4fbc9955b24486 Mon Sep 17 00:00:00 2001 From: Remi Bettan Date: Mon, 1 Aug 2022 09:49:22 +0300 Subject: [PATCH 25/25] code review minor changes --- src/ds5/ds5-device.cpp | 2 +- src/linux/backend-v4l2.cpp | 2 +- src/option.h | 5 ++--- src/sensor.cpp | 6 +++--- 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/ds5/ds5-device.cpp b/src/ds5/ds5-device.cpp index 7c9c335c62..82f3960a85 100644 --- a/src/ds5/ds5-device.cpp +++ b/src/ds5/ds5-device.cpp @@ -774,7 +774,7 @@ namespace librealsense _hw_monitor = std::make_shared( std::make_shared( std::make_shared( - raw_sensor, depth_xu, DS5_HWMONITOR, mipi_sensor), + raw_sensor, depth_xu, DS5_HWMONITOR), raw_sensor)); } else diff --git a/src/linux/backend-v4l2.cpp b/src/linux/backend-v4l2.cpp index 3efb6d948a..9f47483327 100644 --- a/src/linux/backend-v4l2.cpp +++ b/src/linux/backend-v4l2.cpp @@ -656,7 +656,7 @@ namespace librealsense // /sys/devices/pci0000:00/0000:00:xx.0/ABC/M-N/version usb_specification = get_usb_connection_type(real_path + "/../../../"); } - else // COMMENT THIS TO MAKE USB ENUMERATION POSSIBLE - Video4Linux Devices that are not listed as UVC + else // Video4Linux Devices that are not listed as UVC { //LOG_INFO("Enumerating v4l " << name << " realpath=" << real_path); v4l_node = true; diff --git a/src/option.h b/src/option.h index 4c3a702d8c..9f37d56d46 100644 --- a/src/option.h +++ b/src/option.h @@ -492,15 +492,14 @@ namespace librealsense std::vector send_receive(const std::vector& data, int, bool require_response) override; command_transfer_over_xu(uvc_sensor& uvc, - platform::extension_unit xu, uint8_t ctrl, bool is_mipi = false) - : _uvc(uvc), _xu(std::move(xu)), _ctrl(ctrl), _is_mipi(is_mipi) + platform::extension_unit xu, uint8_t ctrl) + : _uvc(uvc), _xu(std::move(xu)), _ctrl(ctrl) {} private: uvc_sensor& _uvc; platform::extension_unit _xu; uint8_t _ctrl; - bool _is_mipi; // D457 dev- should be removed after hwm discrepancy USB/MIPI fixed in driver }; class polling_error_handler; diff --git a/src/sensor.cpp b/src/sensor.cpp index 3637818454..47a2a281fe 100644 --- a/src/sensor.cpp +++ b/src/sensor.cpp @@ -654,7 +654,7 @@ void log_callback_end( uint32_t fps, stream_profiles uvc_sensor::init_stream_profiles() { std::unordered_set> video_profiles; - // D457 development + // D457 development - only via mipi imu frames com from uvc instead of hid std::unordered_set> motion_profiles; power on(std::dynamic_pointer_cast(shared_from_this())); @@ -965,8 +965,8 @@ void log_callback_end( uint32_t fps, last_timestamp = timestamp; frame_holder frame = _source.alloc_frame(RS2_EXTENSION_MOTION_FRAME, data_size, fr->additional_data, true); memcpy( (void *)frame->get_frame_data(), - /*fr->data.data()*/ sensor_data.fo.pixels, - /*fr->data.size()*/ sizeof( byte ) * sensor_data.fo.frame_size ); + sensor_data.fo.pixels, + sizeof( byte ) * sensor_data.fo.frame_size ); if (!frame) { LOG_INFO("Dropped frame. alloc_frame(...) returned nullptr");