diff --git a/common/model-views.cpp b/common/model-views.cpp index d062c72f73..78d3306ba5 100644 --- a/common/model-views.cpp +++ b/common/model-views.cpp @@ -22,6 +22,8 @@ #include "fw-update-helper.h" #include "on-chip-calib.h" #include "viewer.h" +#include "post-processing-filters-list.h" +#include "post-processing-block-model.h" #include #include @@ -856,7 +858,7 @@ namespace rs2 std::shared_ptr block, std::function invoker, std::string& error_message, bool enable) - : _owner(owner), _name(name), _block(block), _invoker(invoker), enabled(enable) + : _owner(owner), _name(name), _block(block), _invoker(invoker), _enabled(enable) { std::stringstream ss; ss << "##" << ((owner) ? owner->dev.get_info(RS2_CAMERA_INFO_NAME) : _name) @@ -868,15 +870,15 @@ namespace rs2 else _full_name = _name; - enabled = restore_processing_block(_full_name.c_str(), - block, enabled); + _enabled = restore_processing_block(_full_name.c_str(), + block, _enabled); populate_options(ss.str().c_str(), owner, owner ? &owner->_options_invalidated : nullptr, error_message); } void processing_block_model::save_to_config_file() { - save_processing_block_to_config_file(_full_name.c_str(), _block, enabled); + save_processing_block_to_config_file(_full_name.c_str(), _block, _enabled); } option_model& processing_block_model::get_option(rs2_option opt) @@ -902,24 +904,34 @@ namespace rs2 subdevice_model::subdevice_model( device& dev, - std::shared_ptr s, std::string& error_message, viewer_model& viewer) - : s(s), dev(dev), tm2(), ui(), last_valid_ui(), + std::shared_ptr s, + std::shared_ptr< atomic_objects_in_frame > device_detected_objects, + std::string& error_message, + viewer_model& viewer + ) + : s( s ), dev( dev ), tm2(), ui(), last_valid_ui(), streaming(false), _pause(false), depth_colorizer(std::make_shared()), yuy2rgb(std::make_shared()), depth_decoder(std::make_shared()), - viewer(viewer) + viewer(viewer), + detected_objects( device_detected_objects ) { supported_options = s->get_supported_options(); restore_processing_block("colorizer", depth_colorizer); restore_processing_block("yuy2rgb", yuy2rgb); + std::string device_name( dev.get_info( RS2_CAMERA_INFO_NAME )); + std::string sensor_name( s->get_info( RS2_CAMERA_INFO_NAME )); + std::stringstream ss; ss << configurations::viewer::post_processing - << "." << dev.get_info(RS2_CAMERA_INFO_NAME) - << "." << s->get_info(RS2_CAMERA_INFO_NAME); + << "." << device_name + << "." << sensor_name; auto key = ss.str(); + bool const is_rgb_camera = s->is< color_sensor >(); + if (config_file::instance().contains(key.c_str())) { post_processing_enabled = config_file::instance().get(key.c_str()); @@ -978,22 +990,39 @@ namespace rs2 viewer.zo_sensors++; } else - model->enabled = false; + model->enable( false ); } if (shared_filter->is()) - model->enabled = false; + model->enable( false ); if (shared_filter->is()) { - std::string sn_name(s->get_info(RS2_CAMERA_INFO_NAME)); - if (sn_name == "RGB Camera") - model->enabled = false; + if( is_rgb_camera ) + model->enable( false ); } post_processing.push_back(model); } + if( is_rgb_camera ) + { + for( auto & create_filter : post_processing_filters_list::get() ) + { + auto filter = create_filter(); + if( !filter ) + continue; + filter->start( *this ); + std::shared_ptr< processing_block_model > model( + new post_processing_block_model { + this, filter, + [=]( rs2::frame f ) { return filter->process( f ); }, + error_message + } ); + post_processing.push_back( model ); + } + } + auto colorizer = std::make_shared( this, "Depth Visualization", depth_colorizer, [=](rs2::frame f) { return depth_colorizer->colorize(f); }, error_message); @@ -1753,6 +1782,13 @@ namespace rs2 streaming = false; _pause = false; + + if( profiles[0].stream_type() == RS2_STREAM_COLOR ) + { + std::lock_guard< std::mutex > lock( detected_objects->mutex ); + detected_objects->clear(); + detected_objects->sensor_is_on = false; + } s->stop(); @@ -1799,7 +1835,7 @@ namespace rs2 //The function decides if specific frame should be sent to the syncer bool subdevice_model::is_synchronized_frame(viewer_model& viewer, const frame& f) { - if(zero_order_artifact_fix && zero_order_artifact_fix->enabled && + if(zero_order_artifact_fix && zero_order_artifact_fix->is_enabled() && (f.get_profile().stream_type() == RS2_STREAM_DEPTH || f.get_profile().stream_type() == RS2_STREAM_INFRARED || f.get_profile().stream_type() == RS2_STREAM_CONFIDENCE)) return true; if (!viewer.is_3d_view || viewer.is_3d_depth_source(f) || viewer.is_3d_texture_source(f)) @@ -1810,7 +1846,7 @@ namespace rs2 void subdevice_model::play(const std::vector& profiles, viewer_model& viewer, std::shared_ptr syncer) { - if (post_processing_enabled && zero_order_artifact_fix && zero_order_artifact_fix->enabled) + if (post_processing_enabled && zero_order_artifact_fix && zero_order_artifact_fix->is_enabled()) { verify_zero_order_conditions(); } @@ -1861,6 +1897,11 @@ namespace rs2 _options_invalidated = true; streaming = true; + if( s->is< color_sensor >() ) + { + std::lock_guard< std::mutex > lock( detected_objects->mutex ); + detected_objects->sensor_is_on = true; + } } void subdevice_model::update(std::string& error_message, notifications_model& notifications) { @@ -3334,13 +3375,14 @@ namespace rs2 : dev(dev), syncer(viewer.syncer), _update_readonly_options_timer(std::chrono::seconds(6)) + , _detected_objects( std::make_shared< atomic_objects_in_frame >() ) { auto name = get_device_name(dev); id = to_string() << name.first << ", " << name.second; for (auto&& sub : dev.query_sensors()) { - auto model = std::make_shared(dev, std::make_shared(sub), error_message, viewer); + auto model = std::make_shared(dev, std::make_shared(sub), _detected_objects, error_message, viewer); subdevices.push_back(model); } @@ -3445,7 +3487,7 @@ namespace rs2 continue; for(auto&& pp : sub->post_processing) - if (pp->enabled) + if (pp->is_enabled()) res = pp->invoke(res); } @@ -3768,7 +3810,7 @@ namespace rs2 fps = s.second.profile.fps(); } auto curr_frame = p.get_position(); - uint64_t step = 1000.0 / (float)fps * 1e6; + uint64_t step = uint64_t( 1000.0 / (float)fps * 1e6 ); p.seek(std::chrono::nanoseconds(curr_frame - step)); } if (ImGui::IsItemHovered()) @@ -3868,7 +3910,7 @@ namespace rs2 fps = s.second.profile.fps(); } auto curr_frame = p.get_position(); - uint64_t step = 1000.0 / (float)fps * 1e6; + uint64_t step = uint64_t( 1000.0 / (float)fps * 1e6 ); p.seek(std::chrono::nanoseconds(curr_frame + step)); } if (ImGui::IsItemHovered()) @@ -5780,11 +5822,18 @@ namespace rs2 if (ImGui::Button(label.c_str(), { 30,24 })) { - if (sub->zero_order_artifact_fix && sub->zero_order_artifact_fix->enabled) + if (sub->zero_order_artifact_fix && sub->zero_order_artifact_fix->is_enabled()) sub->verify_zero_order_conditions(); sub->post_processing_enabled = true; config_file::instance().set(get_device_sensor_name(sub.get()).c_str(), sub->post_processing_enabled); + for( auto&& pb : sub->post_processing ) + { + if( !pb->visible ) + continue; + if( pb->is_enabled() ) + pb->processing_block_enable_disable( true ); + } } if (ImGui::IsItemHovered()) { @@ -5803,6 +5852,13 @@ namespace rs2 sub->post_processing_enabled = false; config_file::instance().set(get_device_sensor_name(sub.get()).c_str(), sub->post_processing_enabled); + for( auto&& pb : sub->post_processing ) + { + if( !pb->visible ) + continue; + if( pb->is_enabled() ) + pb->processing_block_enable_disable( false ); + } } if (ImGui::IsItemHovered()) { @@ -5848,7 +5904,7 @@ namespace rs2 if (!sub->post_processing_enabled) { - if (!pb->enabled) + if (!pb->is_enabled()) { std::string label = to_string() << " " << textual_icons::toggle_off << "##" << id << "," << sub->s->get_info(RS2_CAMERA_INFO_NAME) << "," << pb->get_name(); @@ -5866,7 +5922,7 @@ namespace rs2 } else { - if (!pb->enabled) + if (!pb->is_enabled()) { std::string label = to_string() << " " << textual_icons::toggle_off << "##" << id << "," << sub->s->get_info(RS2_CAMERA_INFO_NAME) << "," << pb->get_name(); @@ -5877,7 +5933,7 @@ namespace rs2 { if (pb->get_block()->is()) sub->verify_zero_order_conditions(); - pb->enabled = true; + pb->enable( true ); pb->save_to_config_file(); } if (ImGui::IsItemHovered()) @@ -5895,7 +5951,7 @@ namespace rs2 if (ImGui::Button(label.c_str(), { 25,24 })) { - pb->enabled = false; + pb->enable( false ); pb->save_to_config_file(); } if (ImGui::IsItemHovered()) diff --git a/common/model-views.h b/common/model-views.h index e60a31b661..95e1e96eb8 100644 --- a/common/model-views.h +++ b/common/model-views.h @@ -22,6 +22,8 @@ #include "imgui-fonts-karla.hpp" #include "imgui-fonts-fontawesome.hpp" #include "../third-party/json.hpp" +#include "objects-in-frame.h" +#include "processing-block-model.h" #include "realsense-ui-advanced-mode.h" #include "fw-update-helper.h" @@ -379,47 +381,6 @@ namespace rs2 void save_processing_block_to_config_file(const char* name, std::shared_ptr pb, bool enable = true); - class processing_block_model - { - public: - processing_block_model(subdevice_model* owner, - const std::string& name, - std::shared_ptr block, - std::function invoker, - std::string& error_message, - bool enabled = true); - - const std::string& get_name() const { return _name; } - - option_model& get_option(rs2_option opt); - - rs2::frame invoke(rs2::frame f) const { return _invoker(f); } - - void save_to_config_file(); - - std::vector get_option_list() - { - return _block->get_supported_options(); - } - - void populate_options(const std::string& opt_base_label, - subdevice_model* model, - bool* options_invalidated, - std::string& error_message); - - std::shared_ptr get_block() { return _block; } - - bool enabled = true; - bool visible = true; - private: - std::shared_ptr _block; - std::map options_metadata; - std::string _name; - std::string _full_name; - std::function _invoker; - subdevice_model* _owner; - }; - class syncer_model { public: @@ -597,7 +558,7 @@ namespace rs2 bool* options_invalidated, std::string& error_message); - subdevice_model(device& dev, std::shared_ptr s, std::string& error_message, viewer_model& viewer); + subdevice_model(device& dev, std::shared_ptr s, std::shared_ptr< atomic_objects_in_frame > objects, std::string& error_message, viewer_model& viewer); ~subdevice_model(); bool is_there_common_fps() ; @@ -665,6 +626,7 @@ namespace rs2 std::shared_ptr s; device dev; tm2_model tm2; + std::shared_ptr< atomic_objects_in_frame > detected_objects; std::map options_metadata; std::vector resolutions; @@ -725,6 +687,7 @@ namespace rs2 void outline_rect(const rect& r); void draw_rect(const rect& r, int line_width = 1); + class stream_model { public: @@ -815,6 +778,8 @@ namespace rs2 viewer_model& viewer, std::string& error_message); void begin_update_unsigned(viewer_model& viewer, std::string& error_message); + std::shared_ptr< atomic_objects_in_frame > get_detected_objects() const { return _detected_objects; } + std::vector> subdevices; std::shared_ptr syncer; std::shared_ptr dev_syncer; @@ -868,6 +833,7 @@ namespace rs2 std::vector> live_subdevices; periodic_timer _update_readonly_options_timer; bool pause_required = false; + std::shared_ptr< atomic_objects_in_frame > _detected_objects; }; class viewer_model; diff --git a/common/objects-in-frame.h b/common/objects-in-frame.h new file mode 100644 index 0000000000..8c58a84b8b --- /dev/null +++ b/common/objects-in-frame.h @@ -0,0 +1,35 @@ +#pragma once + +#include "rendering.h" // for rs2::rect +#include +#include +#include +#include + + +struct object_in_frame +{ + rs2::rect normalized_color_bbox, normalized_depth_bbox; + std::string name; + float mean_depth; + size_t id; + + object_in_frame( size_t id, std::string const & name, rs2::rect bbox_color, rs2::rect bbox_depth, float depth ) + : id( id ) + , name( name ) + , normalized_color_bbox( bbox_color ) + , normalized_depth_bbox( bbox_depth ) + , mean_depth( depth ) + { + } +}; + + +typedef std::vector< object_in_frame > objects_in_frame; + + +struct atomic_objects_in_frame : public objects_in_frame +{ + std::mutex mutex; + bool sensor_is_on = true; +}; diff --git a/common/post-processing-block-model.h b/common/post-processing-block-model.h new file mode 100644 index 0000000000..6053f70e96 --- /dev/null +++ b/common/post-processing-block-model.h @@ -0,0 +1,38 @@ +// License: Apache 2.0. See LICENSE file in root directory. +// Copyright(c) 2019 Intel Corporation. All Rights Reserved. + +#pragma once + +#include "processing-block-model.h" +#include "post-processing-filter.h" + + +namespace rs2 +{ + /* + A less generic processing block that's used specifically for object detection + so that the filters inside are all of type object_detection_filter. + */ + class post_processing_block_model : public processing_block_model + { + public: + post_processing_block_model( + subdevice_model * owner, + std::shared_ptr< post_processing_filter > block, + std::function invoker, + std::string& error_message, + bool enabled = true + ) + : processing_block_model( owner, block->get_name(), block, invoker, error_message, enabled ) + { + } + + protected: + void processing_block_enable_disable( bool actual ) override + { + dynamic_cast(_block.get())->on_processing_block_enable( actual ); + } + }; +} + + diff --git a/common/post-processing-filter.h b/common/post-processing-filter.h new file mode 100644 index 0000000000..c113aac824 --- /dev/null +++ b/common/post-processing-filter.h @@ -0,0 +1,115 @@ +// License: Apache 2.0. See LICENSE file in root directory. +// Copyright(c) 2019 Intel Corporation. All Rights Reserved. + +#pragma once + +#include +#include // for projection utilities +#include +#include + + +/* + Generic base class for any custom post-processing filter that is added after all + the usual supported filters. + + A worker thread does the main job in the background, once start() is called. + See object_detection_filters. +*/ +class post_processing_filter : public rs2::filter +{ + std::string _name; + bool _pb_enabled; + +protected: + // Default ctor to use with post_processing_filters_list::register_filter() cannot take + // anything but a name. This name is what the user will see in the list of post-processors. + post_processing_filter( std::string const & name ) + : rs2::filter( [&]( rs2::frame f, rs2::frame_source & source ) { + // Process the frame, take the result and post it for the next post-processor + source.frame_ready( process_frameset( f.as< rs2::frameset >() ) ); + } ) + , _name( name ) + , _pb_enabled( true ) + { + } + + bool is_pb_enabled() const { return _pb_enabled; } + +public: + // Called from post_processing_filter_model when this particular processing block + // is enabled or disabled + virtual void on_processing_block_enable( bool e ) + { + _pb_enabled = e; + } + + // Returns the name of the filter + std::string const & get_name() const { return _name; } + + virtual void start( rs2::subdevice_model & model ) + { + LOG(INFO) << "Starting " + get_name(); + } + +protected: + // Main handler for each frameset we get + virtual rs2::frameset process_frameset( rs2::frameset fs ) = 0; + + + // Helper fn to get the frame context when logs are written, e.g.: + // LOG(ERROR) << get_context(fs) << "just a dummy error"; + std::string get_context( rs2::frame f = rs2::frame() ) const + { + std::ostringstream ss; + if( f ) + ss << "[#" << f.get_frame_number() << "] "; + return ss.str(); + } + + + // Returns the intrinsics and extrinsics for a depth and color frame. + void get_trinsics( + rs2::video_frame cf, rs2::depth_frame df, + rs2_intrinsics & color_intrin, rs2_intrinsics & depth_intrin, + rs2_extrinsics & color_extrin, rs2_extrinsics & depth_extrin ) + { + rs2::video_stream_profile color_profile = cf.get_profile().as< rs2::video_stream_profile >(); + color_intrin = color_profile.get_intrinsics(); + if( df ) + { + rs2::video_stream_profile depth_profile = df.get_profile().as< rs2::video_stream_profile >(); + depth_intrin = depth_profile.get_intrinsics(); + depth_extrin = depth_profile.get_extrinsics_to( color_profile ); + color_extrin = color_profile.get_extrinsics_to( depth_profile ); + } + } + + + // Returns a bounding box, input in the color-frame coordinates, after converting it to the corresponding + // coordinates in the depth frame. + rs2::rect project_rect_to_depth( + rs2::rect const & bbox, rs2::depth_frame df, + rs2_intrinsics const & color_intrin, rs2_intrinsics const & depth_intrin, + rs2_extrinsics const & color_extrin, rs2_extrinsics const & depth_extrin ) + { + // NOTE: this is a bit expensive as it actually searches along a projected beam from 0.1m to 10 meter + float top_left_depth[2], top_left_color[2] = { bbox.x, bbox.y }; + float bottom_right_depth[2], bottom_right_color[2] = { bbox.x + bbox.w, bbox.y + bbox.h }; + rs2_project_color_pixel_to_depth_pixel( top_left_depth, + reinterpret_cast(df.get_data()), df.get_units(), 0.1f, 10, + &depth_intrin, &color_intrin, + &color_extrin, &depth_extrin, + top_left_color ); + rs2_project_color_pixel_to_depth_pixel( bottom_right_depth, + reinterpret_cast(df.get_data()), df.get_units(), 0.1f, 10, + &depth_intrin, &color_intrin, + &color_extrin, &depth_extrin, + bottom_right_color ); + float left = top_left_depth[0]; + float top = top_left_depth[1]; + float right = bottom_right_depth[0]; + float bottom = bottom_right_depth[1]; + return rs2::rect { left, top, right - left, bottom - top }; + } +}; diff --git a/common/post-processing-filters-list.h b/common/post-processing-filters-list.h new file mode 100644 index 0000000000..4b9758be31 --- /dev/null +++ b/common/post-processing-filters-list.h @@ -0,0 +1,38 @@ +// License: Apache 2.0. See LICENSE file in root directory. +// Copyright(c) 2019 Intel Corporation. All Rights Reserved. + +#pragma once + +#include "post-processing-filter.h" +#include +#include + + +/* + Container for all the filters that are defined. These are added into our list and can be iterated + over, without actual knowledge of any implementation. +*/ +struct post_processing_filters_list +{ + typedef std::function< + std::shared_ptr< post_processing_filter >() + > create_fn; + typedef std::list< create_fn > list; + + static list & get() + { + static list the_filters; + return the_filters; + } + + template< class T > + static list::iterator register_filter( std::string const & name ) + { + auto & filters = get(); + return filters.insert( + filters.end(), + [name]() -> std::shared_ptr< post_processing_filter > { return std::make_shared< T >( name ); } + ); + } +}; + diff --git a/common/post-processing-worker-filter.h b/common/post-processing-worker-filter.h new file mode 100644 index 0000000000..37ef845218 --- /dev/null +++ b/common/post-processing-worker-filter.h @@ -0,0 +1,70 @@ +// License: Apache 2.0. See LICENSE file in root directory. +// Copyright(c) 2019 Intel Corporation. All Rights Reserved. + +#pragma once + +#include "post-processing-filter.h" + + +/* + A post-processing filter that employs a separate thread to do its work. +*/ +class post_processing_worker_filter : public post_processing_filter +{ + std::atomic_bool _alive { true }; + std::thread _worker; + rs2::frame_queue _queue; // size 1! + +protected: + post_processing_worker_filter( std::string const & name ) + : post_processing_filter( name ) + { + } + ~post_processing_worker_filter() + { + _alive = false; + _worker.join(); + } + +public: + void start( rs2::subdevice_model & model ) override + { + post_processing_filter::start( model ); + _worker = std::thread( [&]() + { + try + { + worker_start(); + } + catch( std::exception const & e ) + { + // Most likely file not found, if the user didn't set up his .xml/.bin files right + LOG( ERROR ) << "Cannot start " << get_name() << ": " << e.what(); + return; + } + while( _alive ) + { + rs2::frame f; + if( !_queue.try_wait_for_frame( &f ) ) + continue; + if( !f ) + continue; + worker_body( f.as< rs2::frameset >() ); + } + LOG(DEBUG) << "End of worker loop in " + get_name(); + worker_end(); + } ); + } + +protected: + rs2::frameset process_frameset( rs2::frameset fs ) override + { + _queue.enqueue( fs ); + return fs; + } + + virtual void worker_start() {} + virtual void worker_end() {} + + virtual void worker_body( rs2::frameset fs ) = 0; +}; diff --git a/common/processing-block-model.h b/common/processing-block-model.h new file mode 100644 index 0000000000..51c23ece54 --- /dev/null +++ b/common/processing-block-model.h @@ -0,0 +1,65 @@ +#pragma once + +#include +#include + + +namespace rs2 +{ + class subdevice_model; + class option_model; + + class processing_block_model + { + public: + processing_block_model( subdevice_model* owner, + const std::string& name, + std::shared_ptr block, + std::function invoker, + std::string& error_message, + bool enabled = true ); + virtual ~processing_block_model() = default; + + const std::string& get_name() const { return _name; } + + option_model& get_option( rs2_option opt ); + + rs2::frame invoke( rs2::frame f ) const { return _invoker( f ); } + + void save_to_config_file(); + + std::vector get_option_list() + { + return _block->get_supported_options(); + } + + void populate_options( const std::string& opt_base_label, + subdevice_model* model, + bool* options_invalidated, + std::string& error_message ); + + std::shared_ptr get_block() { return _block; } + + void enable( bool e = true ) + { + processing_block_enable_disable( _enabled = e ); + } + bool is_enabled() const { return _enabled; } + + bool visible = true; + + // Callback when our state changes + // NOTE: actual may not be same as is_enabled()! The latter is this particular pb, + // while the former takes into account global "Post-Processing"... + virtual void processing_block_enable_disable( bool actual ) {} + + protected: + bool _enabled = true; + std::shared_ptr _block; + std::map options_metadata; + std::string _name; + std::string _full_name; + std::function _invoker; + subdevice_model* _owner; + }; +} diff --git a/common/viewer.cpp b/common/viewer.cpp index 9be5e04ead..e3501c11c7 100644 --- a/common/viewer.cpp +++ b/common/viewer.cpp @@ -1583,6 +1583,132 @@ namespace rs2 } } + //switch( stream_mv.profile.stream_type() ) + { + static std::vector< std::pair< ImColor, bool > > colors = + { + { ImColor( 1.f, 1.f, 1.f, 1.f ), false }, // the default color + { ImColor( 1.f, 0.f, 0.f, 1.f ), false }, + { ImColor( 0.f, 1.f, 0.f, 1.f ), false }, + { ImColor( 0.f, 0.f, 1.f, 1.f ), false }, + { ImColor( 1.f, 0.f, 1.f, 1.f ), false }, + { ImColor( 1.f, 1.f, 0.f, 1.f ), false }, + }; + typedef size_t ColorIdx; + static std::map< size_t, ColorIdx > id2color; + + auto get_color = [&]( object_in_frame const & object ) -> ImColor + { + ColorIdx & color = id2color[object.id]; + if( color < int( colors.size() ) ) + { + if( color > 0 ) + return colors[color].first; + size_t x = 0; + for( auto & p : colors ) + { + bool & in_use = p.second; + if( !in_use ) + { + in_use = true; + color = x; + return p.first; + } + ++x; + } + color = 100; + } + return colors[0].first; + }; + + glColor3f( 1, 1, 1 ); + auto p_objects = stream_mv.dev->detected_objects; + std::lock_guard< std::mutex > lock( p_objects->mutex ); + + // Mark colors that are no longer in use + for( auto it = id2color.begin(); it != id2color.end(); ) + { + size_t id = it->first; + bool found = false; + for( object_in_frame & object : *p_objects ) + { + if( object.id == id ) + { + found = true; + break; + } + } + if( !found ) + { + colors[it->second].second = false; // no longer in use + auto it_to_erase = it++; + id2color.erase( it_to_erase ); + } + else + { + ++it; + } + } + + + for( object_in_frame & object : *p_objects ) + { + rect const & normalized_bbox = stream_mv.profile.stream_type() == RS2_STREAM_DEPTH + ? object.normalized_depth_bbox + : object.normalized_color_bbox; + rect bbox = normalized_bbox.unnormalize( stream_rect ); + bbox.grow( 10, 5 ); // Allow more text, and easier identification of the face + + float const max_depth = 2.f; + float const min_depth = 0.8f; + float const depth_range = max_depth - min_depth; + float usable_depth = std::min( object.mean_depth, max_depth ); + float a = 0.75f * (max_depth - usable_depth) / depth_range + 0.25f; + + // Don't draw text in boxes that are too small... + auto h = bbox.h; + ImGui::PushStyleColor( ImGuiCol_Text, ImColor( 1.f, 1.f, 1.f, a ) ); + ImColor bg( dark_sensor_bg.x, dark_sensor_bg.y, dark_sensor_bg.z, dark_sensor_bg.w * a ); + + if( object.mean_depth ) + { + std::string str = to_string() << std::setprecision( 2 ) << object.mean_depth << " m"; + auto size = ImGui::CalcTextSize( str.c_str() ); + if( size.y < h && size.x < bbox.w ) + { + ImGui::GetWindowDrawList()->AddRectFilled( + { bbox.x + 1, bbox.y + 1 }, + { bbox.x + size.x + 20, bbox.y + size.y + 6 }, + bg ); + ImGui::SetCursorScreenPos( { bbox.x + 10, bbox.y + 3 } ); + ImGui::Text( str.c_str() ); + h -= size.y; + } + } + if( ! object.name.empty() ) + { + auto size = ImGui::CalcTextSize( object.name.c_str() ); + if( size.y < h && size.x < bbox.w ) + { + ImGui::GetWindowDrawList()->AddRectFilled( + { bbox.x + bbox.w - size.x - 20, bbox.y + bbox.h - size.y - 6 }, + { bbox.x + bbox.w - 1, bbox.y + bbox.h - 1 }, + bg ); + ImGui::SetCursorScreenPos( { bbox.x + bbox.w - size.x - 10, bbox.y + bbox.h - size.y - 4 } ); + ImGui::Text( object.name.c_str() ); + h -= size.y; + } + } + + ImGui::PopStyleColor(); + + // The rectangle itself is always drawn, in the same color as the text + auto frame_color = get_color( object ); + glColor3f( a * frame_color.Value.x, a * frame_color.Value.y, a * frame_color.Value.z ); + draw_rect( bbox ); + } + } + glColor3f(header_window_bg.x, header_window_bg.y, header_window_bg.z); stream_rect.y -= 32; stream_rect.h += 32; @@ -1784,7 +1910,7 @@ namespace rs2 } { - auto tiles = 12; + float tiles = 12; if (!metric_system) tiles *= 1.f / FEET_TO_METER; // Render "floor" grid @@ -1799,7 +1925,7 @@ namespace rs2 for (int i = 0; i <= ceil(tiles); i++) { - float I = i; + float I = float(i); if (!metric_system) I *= FEET_TO_METER; if (i == tiles / 2) glColor4f(0.7f, 0.7f, 0.7f, 1.f); diff --git a/tools/depth-quality/depth-quality-model.cpp b/tools/depth-quality/depth-quality-model.cpp index 7ba0962447..bb9f03b4fe 100644 --- a/tools/depth-quality/depth-quality-model.cpp +++ b/tools/depth-quality/depth-quality-model.cpp @@ -794,8 +794,8 @@ namespace rs2 _device_model->allow_remove = false; _device_model->show_depth_only = true; _device_model->show_stream_selection = false; - _depth_sensor_model = std::shared_ptr( - new subdevice_model(dev, dpt_sensor, _error_message, _viewer_model)); + std::shared_ptr< atomic_objects_in_frame > no_detected_objects; + _depth_sensor_model = std::make_shared(dev, dpt_sensor, no_detected_objects, _error_message, _viewer_model); _depth_sensor_model->draw_streams_selector = false; _depth_sensor_model->draw_fps_selector = true; diff --git a/tools/realsense-viewer/CMakeLists.txt b/tools/realsense-viewer/CMakeLists.txt index e445028d53..4ccfef4747 100644 --- a/tools/realsense-viewer/CMakeLists.txt +++ b/tools/realsense-viewer/CMakeLists.txt @@ -11,46 +11,108 @@ find_package(Threads REQUIRED) include_directories(${CMAKE_BINARY_DIR}) +set( ELPP_DIR ../../third-party/easyloggingpp/src ) +include_directories( ${ELPP_DIR} ) +set( ELPP_FILES + ${ELPP_DIR}/easylogging++.cc + ${ELPP_DIR}/easylogging++.h +) + + include(../../common/CMakeLists.txt) if(BUILD_GRAPHICAL_EXAMPLES) set(RS_VIEWER_CPP - ${COMMON_SRC} - realsense-viewer.cpp - ../../third-party/imgui/imgui.cpp - ../../third-party/imgui/imgui_draw.cpp - ../../third-party/imgui/imgui_impl_glfw.cpp - ../../third-party/imgui/imgui-fonts-karla.hpp - ../../third-party/imgui/imgui-fonts-fontawesome.hpp - ../../common/realsense-ui-advanced-mode.h - ../../common/rendering.h - ../../common/model-views.h - ../../common/model-views.cpp - ../../common/ux-window.h - ../../common/ux-window.cpp - ../../common/ux-alignment.cpp - ../../common/ux-alignment.h - ../../third-party/glad/glad.c - ../../third-party/tinyfiledialogs/tinyfiledialogs.c - ../../third-party/tinyfiledialogs/tinyfiledialogs.h - ../../common/opengl3.cpp - ../../common/opengl3.h - ../../common/rs-config.h - ../../common/rs-config.cpp - ../../common/os.h - ../../common/os.cpp + ${COMMON_SRC} + realsense-viewer.cpp + ../../third-party/imgui/imgui.cpp + ../../third-party/imgui/imgui_draw.cpp + ../../third-party/imgui/imgui_impl_glfw.cpp + ../../third-party/imgui/imgui-fonts-karla.hpp + ../../third-party/imgui/imgui-fonts-fontawesome.hpp + ../../common/realsense-ui-advanced-mode.h + ../../common/rendering.h + ../../common/model-views.h + ../../common/model-views.cpp + ../../common/ux-window.h + ../../common/ux-window.cpp + ../../common/ux-alignment.cpp + ../../common/ux-alignment.h + ../../common/processing-block-model.h + ../../common/post-processing-block-model.h + ../../common/post-processing-filter.h + ../../common/post-processing-worker-filter.h + ../../common/post-processing-filters-list.h + ../../common/objects-in-frame.h + ../../third-party/glad/glad.c + ../../third-party/tinyfiledialogs/tinyfiledialogs.c + ../../third-party/tinyfiledialogs/tinyfiledialogs.h + ../../common/opengl3.cpp + ../../common/opengl3.h + ../../common/rs-config.h + ../../common/rs-config.cpp + ../../common/os.h + ../../common/os.cpp ) + +if(DEFINED OpenCV_DIR AND IS_DIRECTORY ${OpenCV_DIR}) + + find_package(OpenCV REQUIRED) + + get_property(deps VARIABLE PROPERTY DEPENDENCIES) + set(DEPENDENCIES ${deps} ${OpenCV_LIBS}) + include_directories( ../../wrappers/opencv ) + +endif() +if(DEFINED INTEL_OPENVINO_DIR AND IS_DIRECTORY ${INTEL_OPENVINO_DIR}) + + message( STATUS "Enabling OpenVINO face-detection for realsense-viewer ..." ) + set(IE_ROOT_DIR "${INTEL_OPENVINO_DIR}/inference_engine") + include(${INTEL_OPENVINO_DIR}/inference_engine/share/InferenceEngineConfig.cmake) + + get_property(deps VARIABLE PROPERTY DEPENDENCIES) + set(DEPENDENCIES ${deps} ${InferenceEngine_LIBRARIES} ie_cpu_extension) + include_directories( ../../wrappers/openvino ) + include_directories( ../../wrappers/opencv ) + include_directories( ${InferenceEngine_INCLUDE_DIRS} ) + # We need additional access to ext_list.hpp, for CPU extension support: + include_directories( ${IE_ROOT_DIR}/src/extension ) + + set(OPENVINO_FILES + ../../wrappers/openvino/rs-vino/base-detection.cpp + ../../wrappers/openvino/rs-vino/base-detection.h + ../../wrappers/openvino/rs-vino/object-detection.cpp + ../../wrappers/openvino/rs-vino/object-detection.h + ../../wrappers/openvino/rs-vino/age-gender-detection.cpp + ../../wrappers/openvino/rs-vino/age-gender-detection.h + ../../wrappers/openvino/rs-vino/detected-object.cpp + ../../wrappers/openvino/rs-vino/detected-object.h + ../../wrappers/openvino/rs-vino/openvino-helpers.h + ${IE_ROOT_DIR}/src/extension/ext_list.hpp + ) + + set(RS_VIEWER_CPP + ${RS_VIEWER_CPP} + openvino-face-detection.cpp + ${OPENVINO_FILES} + ) + +endif() + + # config-ui if(WIN32) add_executable(realsense-viewer WIN32 - ${RS_VIEWER_CPP} ${CMAKE_CURRENT_SOURCE_DIR}/res/resource.h - ${CMAKE_CURRENT_SOURCE_DIR}/res/realsense-viewer.rc - ../../common/windows-app-bootstrap.cpp) + ${RS_VIEWER_CPP} + ${ELPP_FILES} + ${CMAKE_CURRENT_SOURCE_DIR}/res/resource.h + ${CMAKE_CURRENT_SOURCE_DIR}/res/realsense-viewer.rc + ../../common/windows-app-bootstrap.cpp) source_group("3rd Party" FILES - ../../third-party/tinyfiledialogs/tinyfiledialogs.c - ../../third-party/tinyfiledialogs/tinyfiledialogs.h + ../../third-party/tinyfiledialogs/tinyfiledialogs.c + ../../third-party/tinyfiledialogs/tinyfiledialogs.h ../../third-party/imgui/imgui.cpp ../../third-party/imgui/imgui_draw.cpp ../../third-party/imgui/imgui_impl_glfw.cpp @@ -59,18 +121,30 @@ if(WIN32) ) source_group("Resources" FILES - ${CMAKE_CURRENT_SOURCE_DIR}/res/resource.h - ${CMAKE_CURRENT_SOURCE_DIR}/res/realsense-viewer.rc) + ${CMAKE_CURRENT_SOURCE_DIR}/res/resource.h + ${CMAKE_CURRENT_SOURCE_DIR}/res/realsense-viewer.rc) + + source_group("OpenVINO" FILES ${OPENVINO_FILES}) - include_directories(realsense-viewer ../../third-party/imgui - ../../third-party/glad ../../common - ../../third-party ${CMAKE_CURRENT_SOURCE_DIR}/res/ ../../third-party/tinyfiledialogs) else() - add_executable(realsense-viewer ${RS_VIEWER_CPP}) - include_directories(realsense-viewer ../../third-party/imgui - ../../third-party/glad ../../common ../../third-party ../../third-party/tinyfiledialogs) + add_executable(realsense-viewer + ${RS_VIEWER_CPP} + ${ELPP_FILES} + ) endif() +source_group("EasyLogging++" FILES ${ELPP_FILES}) + +include_directories( + ../../common + ../../third-party + ../../third-party/tinyfiledialogs + ${GLFW_SOURCE_DIR}/include + ../../third-party/imgui + ../../third-party/glad + ) + + set_property(TARGET realsense-viewer PROPERTY CXX_STANDARD 11) target_link_libraries(realsense-viewer ${DEPENDENCIES} diff --git a/tools/realsense-viewer/openvino-face-detection.cpp b/tools/realsense-viewer/openvino-face-detection.cpp new file mode 100644 index 0000000000..1ad54a9e3b --- /dev/null +++ b/tools/realsense-viewer/openvino-face-detection.cpp @@ -0,0 +1,358 @@ +// License: Apache 2.0. See LICENSE file in root directory. +// Copyright(c) 2020 Intel Corporation. All Rights Reserved. + +// NOTE: This file will be compiled only with INTEL_OPENVINO_DIR pointing to an OpenVINO install! + + +#include "post-processing-filters-list.h" +#include "post-processing-worker-filter.h" +#include +#include +#include +#include + + +/* We need to extend the basic detected_object to include facial characteristics +*/ +class detected_face : public openvino_helpers::detected_object +{ + float _age; + float _male_score, _female_score; // cumulative - see update_gender() + +public: + using ptr = std::shared_ptr< detected_face >; + + explicit detected_face( size_t id, + cv::Rect const& location, + float male_prob, + float age, + cv::Rect const& depth_location = cv::Rect{}, + float intensity = 1, + float depth = 0 ) + : detected_object( id, std::string(), location, depth_location, intensity, depth ) + , _age( age ) + , _male_score( male_prob > 0.5f ? male_prob - 0.5f : 0.f ) + , _female_score( male_prob > 0.5f ? 0.f : 0.5f - male_prob ) + { + } + + void detected_face::update_age( float value ) + { + _age = (_age == -1) ? value : 0.95f * _age + 0.05f * value; + } + + void detected_face::update_gender( float value ) + { + if( value >= 0 ) + { + if( value > 0.5 ) + _male_score += value - 0.5f; + else + _female_score += 0.5f - value; + } + } + + int get_age() const { return static_cast< int >( std::floor( _age + 0.5f )); } + bool is_male() const { return( _male_score > _female_score ); } + bool is_female() const { return !is_male(); } +}; + + +/* Define a filter that will perform facial detection using OpenVINO +*/ +class openvino_face_detection : public post_processing_worker_filter +{ + InferenceEngine::Core _ie; + openvino_helpers::object_detection _face_detector; + openvino_helpers::age_gender_detection _age_detector; + openvino_helpers::detected_objects _faces; + size_t _id = 0; + + std::shared_ptr< atomic_objects_in_frame > _objects; + +public: + openvino_face_detection( std::string const & name ) + : post_processing_worker_filter( name ) + /* + This face detector is from the OpenCV Model Zoo: + https://github.com/opencv/open_model_zoo/blob/master/models/intel/face-detection-adas-0001/description/face-detection-adas-0001.md + */ + , _face_detector( + "face-detection-adas-0001.xml", + 0.5, // Probability threshold + false ) // Not async + /* + */ + , _age_detector( + "age-gender-recognition-retail-0013.xml", + false ) // Not async + { + } + +public: + void start( rs2::subdevice_model & model ) override + { + post_processing_worker_filter::start( model ); + _objects = model.detected_objects; + } + +private: + void worker_start() override + { + LOG(INFO) << "Loading CPU extensions..."; + _ie.AddExtension( std::make_shared< InferenceEngine::Extensions::Cpu::CpuExtensions >(), "CPU" ); + _face_detector.load_into( _ie, "CPU" ); + _age_detector.load_into( _ie, "CPU" ); + } + + /* + Returns the "intensity" of the face in the picture, and calculates the distance to it, ignoring + Invalid depth pixels or those outside a range that would be appropriate for a face. + */ + static float calc_face_attrs( + const rs2::video_frame & cf, + const rs2::depth_frame & df, + cv::Rect const & depth_bbox, + float * p_mean_depth ) + { + uint16_t const * const pdw = reinterpret_cast( df.get_data() ); + uint8_t const * const pcb = reinterpret_cast(const_cast( cf.get_data() )); + float const depth_scale = df.get_units(); + + int const depth_width = df.get_width(); + int const color_width = cf.get_width(); + int const color_bpp = cf.get_bytes_per_pixel(); + + int const top = depth_bbox.y; + int const bot = top + depth_bbox.height; + int const left = depth_bbox.x; + int const right = left + depth_bbox.width; + + // Find a center point that has a depth on it + int center_x = (left + right) / 2; + int center_index = (top + bot) / 2 * depth_width + center_x; + for( int d = 1; !pdw[center_index] && d < 10; ++d ) + { + if( pdw[center_index + d] ) center_index += d; + if( pdw[center_index - d] ) center_index -= d; + if( pdw[center_index + depth_width] ) center_index += depth_width; + if( pdw[center_index - depth_width] ) center_index -= depth_width; + } + if( !pdw[center_index] ) + { + if( p_mean_depth ) + *p_mean_depth = 0; + return 1; + } + float const d_center = pdw[center_index] * depth_scale; + + // Set a "near" and "far" threshold -- anything closer or father, respectively, + // would be deemed not a part of the face and therefore background: + float const d_far_threshold = d_center + 0.2f; + float const d_near_threshold = std::max( d_center - 0.5f, 0.001f ); + // Average human head diameter ~= 7.5" or ~19cm + // Assume that the center point is in the front of the face, so the near threshold + // should be very close to that, while the far farther... + + float total_luminance = 0; + float total_depth = 0; + unsigned pixel_count = 0; +#pragma omp parallel for schedule(dynamic) //Using OpenMP to try to parallelise the loop + for( int y = top; y < bot; ++y ) + { + auto depth_pixel_index = y * depth_width + left; + for( int x = left; x < right; ++x, ++depth_pixel_index ) + { + // Get the depth value of the current pixel + auto d = depth_scale * pdw[depth_pixel_index]; + + // Check if the depth value is invalid (<=0) or greater than the threashold + if( d >= d_near_threshold && d <= d_far_threshold ) + { + // Calculate the offset in other frame's buffer to current pixel + auto const coffset = depth_pixel_index * color_bpp; + auto const pc = &pcb[coffset]; + + // Using RGB... + auto r = pc[0], g = pc[1], b = pc[2]; + total_luminance += 0.2989f * r + 0.5870f * g + 0.1140f * b; // CCIR 601 -- see https://en.wikipedia.org/wiki/Luma_(video) + ++pixel_count; + + // And get a mean depth, too + total_depth += d; + } + } + } + if( p_mean_depth ) + *p_mean_depth = pixel_count ? total_depth / pixel_count : 0; + return pixel_count ? total_luminance / pixel_count : 1; + } + + void worker_body( rs2::frameset fs ) override + { + // A color video frame is the minimum we need for detection + auto cf = fs.get_color_frame(); + if( !cf ) + { + LOG( ERROR ) << get_context( fs ) << "no cf"; + return; + } + if( cf.get_profile().format() != RS2_FORMAT_RGB8 ) + { + LOG(ERROR) << get_context(fs) << "color format is " << cf.get_profile().format(); + return; + } + // A depth frame is optional: if not enabled, we won't get it, and we simply won't provide depth info... + auto df = fs.get_depth_frame(); + if( df ) + { + if( df && df.get_profile().format() != RS2_FORMAT_Z16 ) + { + LOG(ERROR) << get_context(fs) << "depth format is " << df.get_profile().format(); + return; + } + } + + try + { + rs2_intrinsics color_intrin, depth_intrin; + rs2_extrinsics color_extrin, depth_extrin; + get_trinsics( cf, df, color_intrin, depth_intrin, color_extrin, depth_extrin ); + + objects_in_frame objects; + + cv::Mat image( color_intrin.height, color_intrin.width, CV_8UC3, const_cast(cf.get_data()), cv::Mat::AUTO_STEP ); + _face_detector.enqueue( image ); + _face_detector.submit_request(); + auto results = _face_detector.fetch_results(); + + openvino_helpers::detected_objects prev_faces { std::move( _faces ) }; + _faces.clear(); + for( auto && result : results ) + { + cv::Rect rect = result.location & cv::Rect( 0, 0, image.cols, image.rows ); + detected_face::ptr face = std::dynamic_pointer_cast< detected_face >( + openvino_helpers::find_object( rect, prev_faces )); + try + { + // Use a mean of the face intensity to help identify faces -- if the intensity changes too much, + // it's not the same face... + float depth = 0, intensity = 1; + cv::Rect depth_rect; + if( df ) + { + rs2::rect depth_bbox = project_rect_to_depth( + rs2::rect { float( rect.x ), float( rect.y ), float( rect.width ), float( rect.height ) }, + df, + color_intrin, depth_intrin, color_extrin, depth_extrin + ); + // It is possible to get back an invalid rect! + if( depth_bbox == depth_bbox.intersection( rs2::rect { 0.f, 0.f, float( depth_intrin.width ), float( depth_intrin.height) } ) ) + { + depth_rect = cv::Rect( int( depth_bbox.x ), int( depth_bbox.y ), int( depth_bbox.w ), int( depth_bbox.h ) ); + intensity = calc_face_attrs( cf, df, depth_rect, &depth ); + } + else + { + LOG(DEBUG) << get_context(fs) << "depth_bbox is no good!"; + } + } + else + { + intensity = openvino_helpers::calc_intensity( image( rect ) ); + } + float intensity_change = face ? std::abs( intensity - face->get_intensity() ) / face->get_intensity() : 1; + float depth_change = ( face && face->get_depth() ) ? std::abs( depth - face->get_depth() ) / face->get_depth() : 0; + + if( intensity_change > 0.07f || depth_change > 0.2f ) + { + // Figure out the age for this new face + float age = 0, maleProb = 0.5; + // Enlarge the bounding box around the detected face for more robust operation of face analytics networks + cv::Mat face_image = image( + openvino_helpers::adjust_face_bbox( rect, 1.4f ) + & cv::Rect( 0, 0, image.cols, image.rows ) ); + _age_detector.enqueue( face_image ); + _age_detector.submit_request(); + _age_detector.wait(); + auto age_gender = _age_detector[0]; + age = age_gender.age; + maleProb = age_gender.maleProb; + // Note: we may want to update the gender/age for each frame, as it may change... + face = std::make_shared< detected_face >( _id++, rect, maleProb, age, depth_rect, intensity, depth ); + } + else + { + face->move( rect, depth_rect, intensity, depth ); + } + + _faces.push_back( face ); + } + catch( ... ) + { + LOG(ERROR) << get_context(fs) << "Unhandled exception!!!"; + } + } + + for( auto && object : _faces ) + { + auto face = std::dynamic_pointer_cast( object ); + cv::Rect const & loc = face->get_location(); + rs2::rect bbox { float( loc.x ), float( loc.y ), float( loc.width ), float( loc.height ) }; + rs2::rect normalized_color_bbox = bbox.normalize( rs2::rect { 0, 0, float(color_intrin.width), float(color_intrin.height) } ); + rs2::rect normalized_depth_bbox = normalized_color_bbox; + if( df ) + { + cv::Rect const & depth_loc = face->get_depth_location(); + rs2::rect depth_bbox { float( depth_loc.x ), float( depth_loc.y ), float( depth_loc.width ), float( depth_loc.height ) }; + normalized_depth_bbox = depth_bbox.normalize( rs2::rect { 0, 0, float( df.get_width() ), float( df.get_height() ) } ); + } + objects.emplace_back( + face->get_id(), + rs2::to_string() << (face->is_male() ? u8"\uF183" : u8"\uF182") << " " << face->get_age(), + normalized_color_bbox, + normalized_depth_bbox, + face->get_depth() + ); + } + + std::lock_guard< std::mutex > lock( _objects->mutex ); + if( is_pb_enabled() ) + { + if( _objects->sensor_is_on ) + _objects->swap( objects ); + else + LOG(DEBUG) << get_context(fs) << "sensor is off!"; + } + else + { + LOG(DEBUG) << get_context(fs) << "not enabled!"; + _objects->clear(); + } + } + catch( const std::exception & e ) + { + LOG(ERROR) << get_context(fs) << e.what(); + } + catch( ... ) + { + LOG(ERROR) << get_context(fs) << "Unhandled exception caught in openvino_face_detection"; + } + } + + void on_processing_block_enable( bool e ) override + { + post_processing_worker_filter::on_processing_block_enable( e ); + if( !e ) + { + // Make sure all the objects go away! + std::lock_guard< std::mutex > lock( _objects->mutex ); + _objects->clear(); + } + } + +}; + + +static auto it = post_processing_filters_list::register_filter< openvino_face_detection >( "Face Detection : OpenVINO" ); + diff --git a/tools/realsense-viewer/realsense-viewer.cpp b/tools/realsense-viewer/realsense-viewer.cpp index 9e68aec7ed..1e3e13e265 100644 --- a/tools/realsense-viewer/realsense-viewer.cpp +++ b/tools/realsense-viewer/realsense-viewer.cpp @@ -28,6 +28,9 @@ #define FW_SR3XX_FW_IMAGE_VERSION "" #endif // INTERNAL_FW +#include +INITIALIZE_EASYLOGGINGPP + using namespace rs2; using namespace rs400; @@ -242,6 +245,7 @@ bool refresh_devices(std::mutex& m, return true; } + int main(int argc, const char** argv) try { rs2::log_to_console(RS2_LOG_SEVERITY_WARN); @@ -264,6 +268,36 @@ int main(int argc, const char** argv) try std::vector connected_devs; std::mutex m; +#if 1 + // Configure the logger + el::Configurations conf; + conf.set( el::Level::Global, el::ConfigurationType::Format, "[%level] %msg" ); + conf.set( el::Level::Info, el::ConfigurationType::Format, "%msg" ); + conf.set( el::Level::Debug, el::ConfigurationType::Enabled, "false" ); + el::Loggers::reconfigureLogger( "default", conf ); + // Create a dispatch sink which will get any messages logged to EasyLogging, which will then + // post the messages on the viewer's notification window. + class viewer_model_dispatcher : public el::LogDispatchCallback + { + public: + rs2::viewer_model * vm = nullptr; // only the default ctor is available to us...! + protected: + void handle( const el::LogDispatchData* data ) noexcept override + { + vm->not_model.add_log( + data->logMessage()->logger()->logBuilder()->build( + data->logMessage(), + data->dispatchAction() == el::base::DispatchAction::NormalLog + )); + } + }; + el::Helpers::installLogDispatchCallback< viewer_model_dispatcher >( "viewer_model_dispatcher" ); + auto dispatcher = el::Helpers::logDispatchCallback< viewer_model_dispatcher >( "viewer_model_dispatcher" ); + dispatcher->vm = &viewer_model; + // Remove the default logger (which will log to standard out/err) or it'll still be active + el::Helpers::uninstallLogDispatchCallback< el::base::DefaultLogDispatchCallback >( "DefaultLogDispatchCallback" ); +#endif + window.on_file_drop = [&](std::string filename) { std::string error_message{}; diff --git a/wrappers/opencv/cv-helpers.hpp b/wrappers/opencv/cv-helpers.hpp index 93a809b429..89d79fe0c7 100644 --- a/wrappers/opencv/cv-helpers.hpp +++ b/wrappers/opencv/cv-helpers.hpp @@ -8,7 +8,7 @@ #include // Convert rs2::frame to cv::Mat -cv::Mat frame_to_mat(const rs2::frame& f) +static cv::Mat frame_to_mat(const rs2::frame& f) { using namespace cv; using namespace rs2; @@ -45,7 +45,7 @@ cv::Mat frame_to_mat(const rs2::frame& f) } // Converts depth frame to a matrix of doubles with distances in meters -cv::Mat depth_frame_to_meters( const rs2::depth_frame & f ) +static cv::Mat depth_frame_to_meters( const rs2::depth_frame & f ) { cv::Mat dm = frame_to_mat(f); dm.convertTo( dm, CV_64F ); diff --git a/wrappers/openvino/rs-vino/age-gender-detection.cpp b/wrappers/openvino/rs-vino/age-gender-detection.cpp index 027fd9aeb4..901399d8cb 100644 --- a/wrappers/openvino/rs-vino/age-gender-detection.cpp +++ b/wrappers/openvino/rs-vino/age-gender-detection.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2018-2019 Intel Corporation +// Copyright (C) 2020 Intel Corporation // SPDX-License-Identifier: Apache-2.0 // @@ -14,10 +14,11 @@ namespace openvino_helpers { age_gender_detection::age_gender_detection( const std::string &pathToModel, - int maxBatch, bool isBatchDynamic, bool isAsync, + bool isAsync, + int maxBatch, bool isBatchDynamic, bool doRawOutputMessages ) - : base_detection( "Age/Gender", pathToModel, maxBatch, isBatchDynamic, isAsync, doRawOutputMessages) + : base_detection( "age/gender", pathToModel, maxBatch, isBatchDynamic, isAsync, doRawOutputMessages) , _n_enqued_frames(0) { } @@ -27,11 +28,8 @@ namespace openvino_helpers { if( !_n_enqued_frames ) return; - if (isBatchDynamic) { - _request->SetBatch( _n_enqued_frames ); - } - base_detection::submit_request(); _n_enqued_frames = 0; + base_detection::submit_request(); } @@ -39,15 +37,8 @@ namespace openvino_helpers { if( !enabled() ) return; - if( _n_enqued_frames == maxBatch ) - { - LOG(WARNING) << "Number of detected faces more than maximum (" << maxBatch << ") processed by Age/Gender Recognition network"; - return; - } if( !_request ) - { _request = net.CreateInferRequestPtr(); - } Blob::Ptr inputBlob = _request->GetBlob( input ); matU8ToBlob( face, inputBlob, _n_enqued_frames ); @@ -74,20 +65,15 @@ namespace openvino_helpers CNNNetwork age_gender_detection::read_network() { - LOG(INFO) << "Loading network files for Age/Gender Recognition network from: " << pathToModel; + LOG(INFO) << "Loading " << topoName << " model from: " << pathToModel; + CNNNetReader netReader; - // Read network netReader.ReadNetwork(pathToModel); - - // Set maximum batch size to be used. netReader.getNetwork().setBatchSize(maxBatch); - if( doRawOutputMessages ) - LOG(DEBUG) << "Batch size is set to " << netReader.getNetwork().getBatchSize() << " for Age/Gender Recognition network"; - // Extract model name and load its weights - std::string binFileName = fileNameNoExt(pathToModel) + ".bin"; - netReader.ReadWeights(binFileName); + std::string binFileName = remove_ext( pathToModel ) + ".bin"; + netReader.ReadWeights( binFileName ); // Age/Gender Recognition network should have one input and two outputs diff --git a/wrappers/openvino/rs-vino/age-gender-detection.h b/wrappers/openvino/rs-vino/age-gender-detection.h index b2a1d936d8..24d188ba3f 100644 --- a/wrappers/openvino/rs-vino/age-gender-detection.h +++ b/wrappers/openvino/rs-vino/age-gender-detection.h @@ -28,7 +28,8 @@ namespace openvino_helpers public: age_gender_detection( const std::string &pathToModel, - int maxBatch, bool isBatchDynamic, bool isAsync, + bool isAsync = true, + int maxBatch = 1, bool isBatchDynamic = false, bool doRawOutputMessages = false ); InferenceEngine::CNNNetwork read_network() override; diff --git a/wrappers/openvino/rs-vino/detected-object.h b/wrappers/openvino/rs-vino/detected-object.h index 8203fa9ed6..977b1c178d 100644 --- a/wrappers/openvino/rs-vino/detected-object.h +++ b/wrappers/openvino/rs-vino/detected-object.h @@ -30,6 +30,7 @@ namespace openvino_helpers explicit detected_object( size_t id, std::string const & label, cv::Rect const & location, cv::Rect const & depth_location = cv::Rect {}, float intensity = 1, float depth = 0 ); + virtual ~detected_object() {} // Update the location and intensity of the face void move( cv::Rect const & location, cv::Rect const & depth_location = cv::Rect {}, float intensity = 1, float depth = 0 )