diff --git a/CMake/lrs_options.cmake b/CMake/lrs_options.cmake index 9ff68ad50a..04b6ef2dce 100644 --- a/CMake/lrs_options.cmake +++ b/CMake/lrs_options.cmake @@ -23,6 +23,7 @@ option(BUILD_CV_EXAMPLES "Build OpenCV examples" OFF) option(BUILD_DLIB_EXAMPLES "Build DLIB examples - requires DLIB_DIR" OFF) option(BUILD_OPENVINO_EXAMPLES "Build Intel OpenVINO Toolkit examples - requires INTEL_OPENVINO_DIR" OFF) option(BUILD_PCL_EXAMPLES "Build PCL examples" OFF) +option(BUILD_OPEN3D_EXAMPLES "Build Open3D examples" OFF) option(BUILD_NODEJS_BINDINGS "Build Node.js bindings" OFF) option(BUILD_OPENNI2_BINDINGS "Build OpenNI bindings" OFF) option(IMPORT_DEPTH_CAM_FW "Download the latest firmware for the depth cameras" ON) diff --git a/wrappers/CMakeLists.txt b/wrappers/CMakeLists.txt index fcde2b9d70..bfdca772d9 100644 --- a/wrappers/CMakeLists.txt +++ b/wrappers/CMakeLists.txt @@ -25,6 +25,10 @@ if (BUILD_OPENVINO_EXAMPLES) add_subdirectory(openvino) endif() +if (BUILD_OPEN3D_EXAMPLES) + add_subdirectory(open3d) +endif() + if(BUILD_MATLAB_BINDINGS) add_subdirectory(matlab) endif() diff --git a/wrappers/open3d/CMakeLists.txt b/wrappers/open3d/CMakeLists.txt new file mode 100644 index 0000000000..060634ef13 --- /dev/null +++ b/wrappers/open3d/CMakeLists.txt @@ -0,0 +1,47 @@ +cmake_minimum_required(VERSION 3.12.0) +set (CMAKE_CXX_STANDARD 14) + +if(POLICY CMP0091) + # https://stackoverflow.com/a/56490614 + cmake_policy(SET CMP0091 NEW) +endif() + +# The options need to be the same as Open3D's default +# If Open3D is configured and built with custom options, you'll also need to +# specify the same custom options. +option(STATIC_WINDOWS_RUNTIME "Use static (MT/MTd) Windows runtime" ON) +# This needs cmake_policy(SET CMP0091 NEW) +if (STATIC_WINDOWS_RUNTIME) + set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>") +else() + set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>DLL") +endif() + +project(Open3D_RS_examples LANGUAGES CXX) + +# Find installed Open3D, which exports Open3D::Open3D +if(WIN32) + find_package(Open3D HINTS ${CMAKE_INSTALL_PREFIX}/CMake) +else() + find_package(Open3D HINTS ${CMAKE_INSTALL_PREFIX}/lib/cmake) +endif() +if(NOT Open3D_FOUND OR Open3D_VERSION VERSION_LESS 0.12) + message(FATAL_ERROR "Open3D v0.12+ not found, please use -DCMAKE_INSTALL_PREFIX=open3d_install_dir") +endif() + +foreach(EXAMPLE RealSenseBagReader RealSenseRecorder) + add_executable(${EXAMPLE} cpp/${EXAMPLE}.cpp) + target_link_libraries(${EXAMPLE} Open3D::Open3D) + + # On Windows, when BUILD_SHARED_LIBS, copy .dll to the executable directory + if(WIN32) + get_target_property(open3d_type Open3D::Open3D TYPE) + if(open3d_type STREQUAL "SHARED_LIBRARY") + message(STATUS "Will copy Open3D.dll to ${CMAKE_CURRENT_BINARY_DIR}/$") + add_custom_command(TARGET ${EXAMPLE} POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy + ${CMAKE_INSTALL_PREFIX}/bin/Open3D.dll + ${CMAKE_CURRENT_BINARY_DIR}/$) + endif() + endif() +endforeach() diff --git a/wrappers/open3d/cpp/RealSenseBagReader.cpp b/wrappers/open3d/cpp/RealSenseBagReader.cpp new file mode 100755 index 0000000000..22d325c2d9 --- /dev/null +++ b/wrappers/open3d/cpp/RealSenseBagReader.cpp @@ -0,0 +1,198 @@ +// ---------------------------------------------------------------------------- +// - Open3D: www.open3d.org - +// ---------------------------------------------------------------------------- +// The MIT License (MIT) +// +// Copyright (c) 2019 www.open3d.org +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +// IN THE SOFTWARE. +// ---------------------------------------------------------------------------- + +#include +#include +#include +#include +#include +#include + +#include "open3d/Open3D.h" + +using namespace open3d; +namespace sc = std::chrono; + +void PrintUsage() { + PrintOpen3DVersion(); + utility::LogInfo("Usage:"); + // clang-format off + utility::LogInfo("RealSenseBagReader [-V] --input input.bag [--output path]"); + // clang-format on +} + +int main(int argc, char **argv) { + if (!utility::ProgramOptionExists(argc, argv, "--input")) { + PrintUsage(); + return 1; + } + if (utility::ProgramOptionExists(argc, argv, "-V")) { + utility::SetVerbosityLevel(utility::VerbosityLevel::Debug); + } else { + utility::SetVerbosityLevel(utility::VerbosityLevel::Info); + } + std::string bag_filename = + utility::GetProgramOptionAsString(argc, argv, "--input"); + + bool write_image = false; + std::string output_path; + if (!utility::ProgramOptionExists(argc, argv, "--output")) { + utility::LogInfo("No output image path, only play bag."); + } else { + output_path = utility::GetProgramOptionAsString(argc, argv, "--output"); + if (output_path.empty()) { + utility::LogError("Output path {} is empty, only play bag.", output_path); + return 1; + } + if (utility::filesystem::DirectoryExists(output_path)) { + utility::LogWarning("Output path {} already existing, only play bag.", + output_path); + return 1; + } else if (!utility::filesystem::MakeDirectory(output_path)) { + utility::LogWarning("Unable to create path {}, only play bag.", + output_path); + return 1; + } else { + utility::LogInfo("Decompress images to {}", output_path); + utility::filesystem::MakeDirectoryHierarchy(output_path + "/color"); + utility::filesystem::MakeDirectoryHierarchy(output_path + "/depth"); + write_image = true; + } + } + + t::io::RSBagReader bag_reader; + bag_reader.Open(bag_filename); + if (!bag_reader.IsOpened()) { + utility::LogError("Unable to open {}", bag_filename); + return 1; + } + + bool flag_exit = false; + bool flag_play = true; + visualization::VisualizerWithKeyCallback vis; + visualization::SetGlobalColorMap( + visualization::ColorMap::ColorMapOption::Gray); + vis.RegisterKeyCallback(GLFW_KEY_ESCAPE, [&](visualization::Visualizer *vis) { + flag_exit = true; + return true; + }); + vis.RegisterKeyCallback(GLFW_KEY_SPACE, [&](visualization::Visualizer *vis) { + if (flag_play) { + utility::LogInfo("Playback paused, press [SPACE] to continue"); + } else { + utility::LogInfo("Playback resumed, press [SPACE] to pause"); + } + flag_play = !flag_play; + return true; + }); + vis.RegisterKeyCallback(GLFW_KEY_LEFT, [&](visualization::Visualizer *vis) { + uint64_t now = bag_reader.GetTimestamp(); + if (bag_reader.SeekTimestamp(now < 1'000'000 ? 0 : now - 1'000'000)) + utility::LogInfo("Seek back 1s"); + else + utility::LogWarning("Seek back 1s failed"); + return true; + }); + vis.RegisterKeyCallback(GLFW_KEY_RIGHT, [&](visualization::Visualizer *vis) { + uint64_t now = bag_reader.GetTimestamp(); + if (bag_reader.SeekTimestamp(now + 1'000'000)) + utility::LogInfo("Seek forward 1s"); + else + utility::LogWarning("Seek forward 1s failed"); + return true; + }); + + vis.CreateVisualizerWindow("Open3D Intel RealSense bag player", 1920, 540); + utility::LogInfo("Starting to play. Press [SPACE] to pause. Press [ESC] to " + "exit."); + + bool is_geometry_added = false; + int idx = 0; + const auto bag_metadata = bag_reader.GetMetadata(); + utility::LogInfo("Recorded with device {}", bag_metadata.device_name_); + utility::LogInfo(" Serial number: {}", bag_metadata.serial_number_); + utility::LogInfo("Video resolution: {}x{}", bag_metadata.width_, + bag_metadata.height_); + utility::LogInfo(" frame rate: {}", bag_metadata.fps_); + utility::LogInfo(" duration: {:.6f}s", + static_cast(bag_metadata.stream_length_usec_) * + 1e-6); + utility::LogInfo(" color pixel format: {}", bag_metadata.color_format_); + utility::LogInfo(" depth pixel format: {}", bag_metadata.depth_format_); + + if (write_image) { + io::WriteIJsonConvertibleToJSON( + fmt::format("{}/intrinsic.json", output_path), bag_metadata); + } + const auto frame_interval = sc::duration(1. / bag_metadata.fps_); + + auto last_frame_time = std::chrono::steady_clock::now() - frame_interval; + using legacyRGBDImage = open3d::geometry::RGBDImage; + legacyRGBDImage im_rgbd; + while (!bag_reader.IsEOF() && !flag_exit) { + if (flag_play) { + std::this_thread::sleep_until(last_frame_time + frame_interval); + last_frame_time = std::chrono::steady_clock::now(); + im_rgbd = bag_reader.NextFrame().ToLegacyRGBDImage(); + // create shared_ptr with no-op deleter for stack RGBDImage + auto ptr_im_rgbd = + std::shared_ptr(&im_rgbd, [](legacyRGBDImage *) {}); + // Improve depth visualization by scaling + /* im_rgbd.depth_.LinearTransform(0.25); */ + if (ptr_im_rgbd->IsEmpty()) + continue; + + if (!is_geometry_added) { + vis.AddGeometry(ptr_im_rgbd); + is_geometry_added = true; + } + + ++idx; + if (write_image) +#pragma omp parallel sections + { +#pragma omp section + { + auto color_file = + fmt::format("{0}/color/{1:05d}.jpg", output_path, idx); + utility::LogInfo("Writing to {}", color_file); + io::WriteImage(color_file, im_rgbd.color_); + } +#pragma omp section + { + auto depth_file = + fmt::format("{0}/depth/{1:05d}.png", output_path, idx); + utility::LogInfo("Writing to {}", depth_file); + io::WriteImage(depth_file, im_rgbd.depth_); + } + } + vis.UpdateGeometry(); + vis.UpdateRender(); + } + vis.PollEvents(); + } + bag_reader.Close(); +} diff --git a/wrappers/open3d/cpp/RealSenseRecorder.cpp b/wrappers/open3d/cpp/RealSenseRecorder.cpp new file mode 100644 index 0000000000..b9a8f82e23 --- /dev/null +++ b/wrappers/open3d/cpp/RealSenseRecorder.cpp @@ -0,0 +1,199 @@ +// ---------------------------------------------------------------------------- +// - Open3D: www.open3d.org - +// ---------------------------------------------------------------------------- +// The MIT License (MIT) +// +// Copyright (c) 2018 www.open3d.org +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +// IN THE SOFTWARE. +// ---------------------------------------------------------------------------- + +#include +#include +#include + +#include "open3d/Open3D.h" + +using namespace open3d; +namespace tio = open3d::t::io; +namespace sc = std::chrono; + +void PrintUsage() { + PrintOpen3DVersion(); + // clang-format off + utility::LogInfo( + "Open a RealSense camera and display live color and depth streams. You can set\n" + "frame sizes and frame rates for each stream and the depth stream can be\n" + "optionally aligned to the color stream. NOTE: An error of 'UNKNOWN: Couldn't\n" + "resolve requests' implies unsupported stream format settings."); + // clang-format on + utility::LogInfo("Usage:"); + utility::LogInfo( + "RealSenseRecorder [-h|--help] [-V] [-l|--list-devices] [--align]\n" + "[--record rgbd_video_file.bag] [-c|--config rs-config.json]"); +} + +int main(int argc, char **argv) { + // Parse command line arguments + if (utility::ProgramOptionExists(argc, argv, "--help") || + utility::ProgramOptionExists(argc, argv, "-h")) { + PrintUsage(); + return 0; + } + if (utility::ProgramOptionExists(argc, argv, "--list-devices") || + utility::ProgramOptionExists(argc, argv, "-l")) { + tio::RealSenseSensor::ListDevices(); + return 0; + } + if (utility::ProgramOptionExists(argc, argv, "-V")) { + utility::SetVerbosityLevel(utility::VerbosityLevel::Debug); + } else { + utility::SetVerbosityLevel(utility::VerbosityLevel::Info); + } + bool align_streams = false; + std::string config_file, bag_file; + + if (utility::ProgramOptionExists(argc, argv, "-c")) { + config_file = utility::GetProgramOptionAsString(argc, argv, "-c"); + } else if (utility::ProgramOptionExists(argc, argv, "--config")) { + config_file = utility::GetProgramOptionAsString(argc, argv, "--config"); + } else { + utility::LogError("config json file required."); + PrintUsage(); + return 1; + } + if (utility::ProgramOptionExists(argc, argv, "--align")) { + align_streams = true; + } + if (utility::ProgramOptionExists(argc, argv, "--record")) { + bag_file = utility::GetProgramOptionAsString(argc, argv, "--record"); + } + + // Read in camera configuration. + tio::RealSenseSensorConfig rs_cfg; + open3d::io::ReadIJsonConvertible(config_file, rs_cfg); + + // Initialize camera. + tio::RealSenseSensor rs; + rs.ListDevices(); + rs.InitSensor(rs_cfg, 0, bag_file); + utility::LogInfo("{}", rs.GetMetadata().ToString()); + + // Create windows to show depth and color streams. + bool flag_start = false, flag_record = flag_start, flag_exit = false; + visualization::VisualizerWithKeyCallback depth_vis, color_vis; + auto callback_exit = [&](visualization::Visualizer *vis) { + flag_exit = true; + if (flag_start) { + utility::LogInfo("Recording finished."); + } else { + utility::LogInfo("Nothing has been recorded."); + } + return false; + }; + depth_vis.RegisterKeyCallback(GLFW_KEY_ESCAPE, callback_exit); + color_vis.RegisterKeyCallback(GLFW_KEY_ESCAPE, callback_exit); + auto callback_toggle_record = [&](visualization::Visualizer *vis) { + if (flag_record) { + rs.PauseRecord(); + utility::LogInfo( + "Recording paused. " + "Press [SPACE] to continue. " + "Press [ESC] to save and exit."); + flag_record = false; + } else { + rs.ResumeRecord(); + flag_record = true; + if (!flag_start) { + utility::LogInfo( + "Recording started. " + "Press [SPACE] to pause. " + "Press [ESC] to save and exit."); + flag_start = true; + } else { + utility::LogInfo( + "Recording resumed, video may be discontinuous. " + "Press [SPACE] to pause. " + "Press [ESC] to save and exit."); + } + } + return false; + }; + if (!bag_file.empty()) { + depth_vis.RegisterKeyCallback(GLFW_KEY_SPACE, callback_toggle_record); + color_vis.RegisterKeyCallback(GLFW_KEY_SPACE, callback_toggle_record); + utility::LogInfo( + "In the visulizer window, " + "press [SPACE] to start recording, " + "press [ESC] to exit."); + } else { + utility::LogInfo("In the visulizer window, press [ESC] to exit."); + } + + using legacyRGBDImage = open3d::geometry::RGBDImage; + using legacyImage = open3d::geometry::Image; + std::shared_ptr depth_image_ptr, color_image_ptr; + + // Loop over frames from device + legacyRGBDImage im_rgbd; + bool is_geometry_added = false; + size_t frame_id = 0; + rs.StartCapture(flag_start); + do { + im_rgbd = rs.CaptureFrame(true, align_streams).ToLegacyRGBDImage(); + + // Improve depth visualization by scaling + /* im_rgbd.depth_.LinearTransform(0.25); */ + depth_image_ptr = std::shared_ptr( + &im_rgbd.depth_, [](open3d::geometry::Image *) {}); + color_image_ptr = std::shared_ptr( + &im_rgbd.color_, [](open3d::geometry::Image *) {}); + + if (!is_geometry_added) { + if (!depth_vis.CreateVisualizerWindow( + "Open3D || RealSense || Depth", depth_image_ptr->width_, + depth_image_ptr->height_, 15, 50) || + !depth_vis.AddGeometry(depth_image_ptr) || + !color_vis.CreateVisualizerWindow( + "Open3D || RealSense || Color", color_image_ptr->width_, + color_image_ptr->height_, 675, 50) || + !color_vis.AddGeometry(color_image_ptr)) { + utility::LogError("Window creation failed!"); + return 0; + } + is_geometry_added = true; + } + + depth_vis.UpdateGeometry(); + color_vis.UpdateGeometry(); + depth_vis.PollEvents(); + color_vis.PollEvents(); + depth_vis.UpdateRender(); + color_vis.UpdateRender(); + + if (frame_id++ % 30 == 0) { + utility::LogInfo("Time: {}s, Frame {}", + static_cast(rs.GetTimestamp()) * 1e-6, + frame_id - 1); + } + } while (!flag_exit); + + rs.StopCapture(); + return 0; +} diff --git a/wrappers/open3d/readme.md b/wrappers/open3d/readme.md new file mode 100644 index 0000000000..c7f3e5e301 --- /dev/null +++ b/wrappers/open3d/readme.md @@ -0,0 +1,243 @@ +RealSense with Open3D +===================== + +[Open3D](http://www.open3d.org/) is an open-source library that supports rapid +development of software for 3D data processing, including scene reconstruction, +visualization and 3D machine learning. + +RealSense (`librealsense` SDK v2) is integrated into Open3D (v0.12+) and you can +use it through both C++ and Python APIs without a separate `librealsense` SDK +installation on Linux, macOS and Windows. Older versions of Open3D support +RealSense through a separate install of `librealsense` SDK v1 and `pyrealsense`. + +Obtaining Open3D with RealSense support +--------------------------------------- + +Please refer to [Open3D +documentation](http://www.open3d.org/docs/latest/introduction.html) for more +details such as: +- Obtaining Open3D source code and building from source. +- Building C++ projects using the Open3D library. +- Tutorials and examples. +- C++ and Python API documentation. + +### Python + +Install Open3D from PyPI (a virtual environment is recommended): + + pip install open3d + +### Compile from source (C++) + +To build Open3D from source with RealSense support, set +`BUILD_LIBREALSENSE=ON` at CMake config step. You can add other +configuration options as well (e.g.: `BUILD_GUI=ON` and +`BUILD_PYTHON_MODULE=ON` may be useful). + + cmake -D BUILD_LIBREALSENSE=ON -D /path/to/Open3D/source/ + +Reading from RealSense bag files +-------------------------------- + +### Sample RealSense bag files + +You con download sample RealSense bag datasets with this script (in the Open3D repository): + + python examples/python/reconstruction_system/scripts/download_dataset.py L515_test + +Check the script for more RS bag datasets. + +Here is a C++ code snippet that shows how to read a RealSense bag file +recorded with Open3D or the `realsense-viewer`. Note that general ROSbag +files are not supported. See more details and available functionality +(such as getting timestamps, aligning the depth stream to the color +stream and getting intrinsic calibration) in the C++ API in the +[RSBagReader +documentation](http://www.open3d.org/docs/latest/cpp_api/classopen3d_1_1t_1_1io_1_1_r_s_bag_reader.html) + + #include + using namespace open3d; + t::io::RSBagReader bag_reader; + bag_reader.Open(bag_filename); + auto im_rgbd = bag_reader.NextFrame(); + while (!bag_reader.IsEOF()) { + // process im_rgbd.depth_ and im_rgbd.color_ + im_rgbd = bag_reader.NextFrame(); + } + bag_reader.Close(); + +Here is the corresponding Python code. The Python API has equivalent functionality: + + import open3d as o3d + bag_reader = o3d.t.io.RSBagReader() + bag_reader.open(bag_filename) + im_rgbd = bag_reader.next_frame() + while not bag_reader.is_eof(): + # process im_rgbd.depth and im_rgbd.color + im_rgbd = bag_reader.next_frame() + + bag_reader.close() + +### Examples + +#### C++ RS bag file viewer + +This C++ example that plays back the color and depth streams from a +RealSense bag file. It also prints out metadata about the video streams +in the file. Press [SPACE] to pause/resume and [ESC] to exit.: + + make RealSenseBagReader + wrappers/open3d/RealSenseBagReader --input L515_test.bag + +![RealSenseBagReader example](https://storage.googleapis.com/open3d-bin/docs/images/RSbagviewer.jpg) + +### Running the Open3D scene reconstruction pipeline + +You can provide an RS bag file directly to the reconstruction pipeline (part of the Open3D Python +examples) and colormap optimization pipelines. It will be automatically converted to a directory of +depth and color frames and the camera intrinsics. Edit the +`examples/python/reconstruction_system/config/realsense.json` file with the path to your RS bag file +and leave path_intrinsic empty. Update other configuration parameters if needed (see the +reconstruction pipeline documentation for more details, including other required packages): + + cd examples/python/reconstruction_system/ + python run_system.py --make --register --refine --integrate config/realsense.json + python ../pipelines/color_map_optimization_for_reconstruction_system.py --config config/realsense.json + +The reconstruction result below was obtained with the `L515_JackJack` +dataset with the configuration changes: + + "path_dataset": "/path/to/downloaded/L515_JackJack.bag" + "max_depth": 0.85, + "tsdf_cubic_size": 0.75, + "voxel_size": 0.025, + "max_depth_diff": 0.03 + + +RealSense camera configuration, live capture, processing and recording +---------------------------------------------------------------------- + +### RealSense camera discovery + +You can list all RealSense devices connected to the system and get their +capabilities (supported resolutions, frame rates, etc.) with the code +snippet below. + + #include + open3d::t::io::RealSenseSensor::ListDevices(); + + import open3d as o3d + o3d.t.io.RealSenseSensor.list_devices() + +Here is sample output when only one L515 camera is connected: + + [Open3D INFO] [0] Intel RealSense L515: f0141095 + [Open3D INFO] color_format: [RS2_FORMAT_BGR8 | RS2_FORMAT_BGRA8 | RS2_FORMAT_RGB8 | RS2_FORMAT_RGBA8 | RS2_FORMAT_Y16 | RS2_FORMAT_YUYV] + [Open3D INFO] color_resolution: [1280,720 | 1920,1080 | 960,540] + [Open3D INFO] color_fps: [15 | 30 | 6 | 60] + [Open3D INFO] depth_format: [RS2_FORMAT_Z16] + [Open3D INFO] depth_resolution: [1024,768 | 320,240 | 640,480] + [Open3D INFO] depth_fps: [30] + [Open3D INFO] visual_preset: [RS2_L500_VISUAL_PRESET_CUSTOM | RS2_L500_VISUAL_PRESET_DEFAULT | RS2_L500_VISUAL_PRESET_LOW_AMBIENT | RS2_L500_VISUAL_PRESET_MAX_RANGE | RS2_L500_VISUAL_PRESET_NO_AMBIENT | RS2_L500_VISUAL_PRESET_SHORT_RANGE] + [Open3D INFO] Open3D only supports synchronized color and depth capture (color_fps = depth_fps). + +This data can also be obtained programmatically to configure a camera +based on custom specifications (e.g.: resolution less than 720p) and to +independently configure multiple cameras. + +### RealSense camera configuration + +RealSense cameras can be configured with a simple `json` configuration +file. See [RealSense +documentation](https://intelrealsense.github.io/librealsense/doxygen/rs__option_8h.html) +for the set of configuration values. Supported configuration options +will be depend on the device and other chosen options. Here are the +options supported by Open3D: + +- **serial**: Pick a specific device, leave empty to pick the first + available device. +- **color_format**: Pixel format for color frames. +- **color_resolution**: (width, height): Leave 0 to let RealSense + pick a supported width or height. +- **depth_format**: Pixel format for depth frames. +- **depth_resolution**: (width, height): Leave 0 to let RealSense + pick a supported width or height. +- **fps**: Common frame rate for both depth and color streams. Leave 0 + to let RealSense pick a supported frame rate. +- **visual_preset**: Controls depth computation on the device. + Supported values are specific to product line (SR300, RS400, L500). + Leave empty to pick the default. + +Here is an example `json` configuration file to capture 30fps, 540p +color and 480p depth video from any connected RealSense camera. The +video width is picked by RealSense. We also set the `visual_preset` to +`RS2_L500_VISUAL_PRESET_MAX_RANGE` to better capture far away objects. + + { + "serial": "", + "color_format": "RS2_FORMAT_RGB8", + "color_resolution": "0,540", + "depth_format": "RS2_FORMAT_Z16", + "depth_resolution": "0,480", + "fps": "30", + "visual_preset": "RS2_L500_VISUAL_PRESET_MAX_RANGE" + } + +### RealSense camera capture, processing and recording + +The following code snippets show how to capture live RGBD video from a +RealSense camera. They capture the first 150 frames and also record them +to an RS bag file. The bag file can be played back with Open3D tools, +realsense-viewer. You can also use ROS tools such as +[rosbag](http://wiki.ros.org/rosbag), +[rqt\_bag](http://wiki.ros.org/rqt_bag) and +[rviz](https://wiki.ros.org/rviz) to examine, play and modify the bag +file. You can adapt the snippets to your needs by processing or +displaying the frames after capture. + + #include + open3d::t::io::RealSenseSensorConfig rs_cfg; + open3d::io::ReadIJsonConvertible(config_filename, rs_cfg); + RealSenseSensor rs; + rs.InitSensor(rs_cfg, 0, bag_filename); + rs.StartCapture(true); // true: start recording with capture + for(size_t fid = 0; fid<150; ++fid) { + im_rgbd = rs.CaptureFrame(true, true); // wait for frames and align them + // process im_rgbd.depth_ and im_rgbd.color_ + } + rs.StopCapture(); + + import json + import open3d as o3d + with open(config_filename) as cf: + rs_cfg = o3d.t.io.RealSenseSensorConfig(json.load(cf)) + + rs = o3d.t.io.RealSenseSensor() + rs.init_sensor(rs_cfg, 0, bag_filename) + rs.start_capture(True) # true: start recording with capture + for fid in range(150): + im_rgbd = rs.capture_frame(True, True) # wait for frames and align them + # process im_rgbd.depth and im_rgbd.color + + rs.stop_capture() + +Note that for any real time application such as live capture and +processing, it is important to complete frame processing in the frame +interval (\~33ms for 30fps recording). You may experience frame drops +otherwise. For high resolution capture, you can defer frame alignment by +setting `align_depth_to_color=false` during capture and performing it +while reading the bag file instead. + +This is a complete C++ example that shows visualizing live capture and +recording to a bag file. The recording can be paused / resumed with +[SPACE]. Use [ESC] to stop capture and quit. You can use this example to +capture your own dataset: + + make RealSenseRecorder + wrappers/open3d/RealSenseRecorder --config ../wrappers/open3d/rs_default_config.json --record test_data.bag + +![RealSenseRecorder example](https://storage.googleapis.com/open3d-bin/docs/images/RealSenseRecorder.jpg)