Create portable dashboard for OTel .NET runtime metrics (#184842)

## Summary

Create a dedicated "portable dashboard" for OTel .NET.

This uses metrics available in the
[contrib](https://github.com/open-telemetry/opentelemetry-dotnet-contrib)
runtime metrics library. These metrics are opt-in and not enabled by
default in the vanilla SDK. Our Elastic distro brings in the package and
enables them by default. Therefore, the dashboard will only work if a)
the customer uses our distro or b) they enable the metrics themselves
when using the vanilla SDK.

Further, work is ongoing to define [semantic conventions for .NET
runtime
metrics](open-telemetry/semantic-conventions#1035).
Once complete, the metrics will be implemented directly in the .NET
runtime BCL and be available with no additional dependencies. The goal
is to achieve that by .NET 9, which is not guaranteed. At that point,
the metric names will change to align with the semantic conventions.
This is not ideal, but it is our only option if we want to provide some
form of runtime dashboard with the current metrics and OTel distro.

As with #182107, this dashboard uses a table for some of the data and
this table doesn't seem to reflect the correct date filtering. Until
there is a solution, this PR will remain in the draft, or we can
consider dropping the table for the initial dashboard.


![image](https://github.com/elastic/kibana/assets/3669103/0be46495-e09f-4f4e-81e1-5f69361d5781)

packages/kbn-apm-synthtrace-client/src/types/agent_names.ts

@@ -24,6 +24,8 @@ type OpenTelemetryAgentName =
   | 'otlp'
   | 'opentelemetry/cpp'
   | 'opentelemetry/dotnet'
+  | 'opentelemetry/dotnet/opentelemetry-dotnet-instrumentation'
+  | 'opentelemetry/dotnet/elastic'
   | 'opentelemetry/erlang'
   | 'opentelemetry/go'
   | 'opentelemetry/java'

x-pack/plugins/observability_solution/apm/public/components/app/metrics/static_dashboard/dashboards/dashboard_catalog.ts

@@ -12,6 +12,9 @@ export const AGENT_NAME_DASHBOARD_FILE_MAPPING: Record<string, string> = {
   'opentelemetry/java': 'opentelemetry_java',
   'opentelemetry/java/opentelemetry-java-instrumentation': 'opentelemetry_java',
   'opentelemetry/java/elastic': 'opentelemetry_java',
+  'opentelemetry/dotnet': 'opentelemetry_dotnet',
+  'opentelemetry/dotnet/opentelemetry-dotnet-instrumentation': 'opentelemetry_dotnet',
+  'opentelemetry/dotnet/elastic': 'opentelemetry_dotnet',
 };
 
 /**
@@ -44,6 +47,12 @@ export async function loadDashboardFile(filename: string): Promise<any> {
         './opentelemetry_java.json'
       );
     }
+    case 'opentelemetry_dotnet': {
+      return import(
+        /* webpackChunkName: "lazyOtelDotnetDashboard" */
+        './opentelemetry_dotnet.json'
+      );
+    }
     default: {
       break;
     }

x-pack/plugins/observability_solution/apm/public/components/app/metrics/static_dashboard/dashboards/opentelemetry_dotnet.json

@@ -0,0 +1 @@
+{"attributes":{"controlGroupInput":{"chainingSystem":"HIERARCHICAL","controlStyle":"oneLine","ignoreParentSettingsJSON":"{\"ignoreFilters\":false,\"ignoreQuery\":false,\"ignoreTimerange\":false,\"ignoreValidations\":false}","panelsJSON":"{\"2be66584-9de4-4a36-ba54-bfdd1b4ccfb4\":{\"type\":\"optionsListControl\",\"order\":0,\"grow\":true,\"width\":\"medium\",\"explicitInput\":{\"id\":\"2be66584-9de4-4a36-ba54-bfdd1b4ccfb4\",\"fieldName\":\"service.node.name\",\"title\":\"Instance\",\"grow\":true,\"width\":\"medium\",\"searchTechnique\":\"prefix\",\"selectedOptions\":[],\"existsSelected\":true,\"enhancements\":{}}}}"},"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"optionsJSON":"{\"useMargins\":true,\"syncColors\":false,\"syncCursor\":true,\"syncTooltips\":false,\"hidePanelTitles\":false}","panelsJSON":"[{\"type\":\"lens\",\"gridData\":{\"x\":0,\"y\":0,\"w\":48,\"h\":8,\"i\":\"ea9f86f0-ff73-4c92-9b93-41baebdcffab\"},\"panelIndex\":\"ea9f86f0-ff73-4c92-9b93-41baebdcffab\",\"embeddableConfig\":{\"attributes\":{\"title\":\"\",\"visualizationType\":\"lnsDatatable\",\"type\":\"lens\",\"references\":[{\"type\":\"index-pattern\",\"id\":\"APM_STATIC_DATA_VIEW_ID\",\"name\":\"indexpattern-datasource-layer-1ba88117-6e95-46e2-8667-e0bc15145182\"}],\"state\":{\"visualization\":{\"columns\":[{\"columnId\":\"d5553b2b-25e9-4b94-9697-f04885f4a067\",\"isTransposed\":false,\"isMetric\":false,\"alignment\":\"left\",\"summaryRow\":\"none\",\"width\":547.5714285714286},{\"columnId\":\"b3da070f-3463-4990-b257-40ac399bec87\",\"isTransposed\":false,\"isMetric\":true,\"colorMode\":\"none\",\"hidden\":false,\"alignment\":\"left\",\"summaryRow\":\"avg\",\"width\":135.73809523809527},{\"columnId\":\"a7f6a205-1e8f-4b64-b745-efe20c2c6545\",\"isTransposed\":false,\"isMetric\":true,\"alignment\":\"left\",\"summaryRow\":\"sum\",\"width\":143.93809523809523},{\"columnId\":\"2bbcd9e9-15ff-42c1-98a8-65a5b9b4e4c5\",\"isTransposed\":false,\"isMetric\":true,\"alignment\":\"left\",\"summaryRow\":\"sum\",\"width\":142.68809523809523},{\"columnId\":\"9f3d67e3-0513-468e-b9b2-6d8780dac3e0\",\"isTransposed\":false,\"isMetric\":true,\"alignment\":\"left\",\"summaryRow\":\"sum\",\"width\":163.18809523809523},{\"columnId\":\"21bdcb2d-cee4-4dd9-84cb-5031f073bf85\",\"isTransposed\":false,\"isMetric\":true,\"alignment\":\"left\",\"width\":183.68809523809523}],\"layerId\":\"1ba88117-6e95-46e2-8667-e0bc15145182\",\"layerType\":\"data\",\"paging\":{\"size\":10,\"enabled\":true}},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[],\"datasourceStates\":{\"formBased\":{\"layers\":{\"1ba88117-6e95-46e2-8667-e0bc15145182\":{\"columns\":{\"d5553b2b-25e9-4b94-9697-f04885f4a067\":{\"label\":\"Host + Service instance\",\"dataType\":\"string\",\"operationType\":\"terms\",\"scale\":\"ordinal\",\"sourceField\":\"host.name\",\"isBucketed\":true,\"params\":{\"size\":25,\"orderBy\":{\"type\":\"alphabetical\",\"fallback\":false},\"orderDirection\":\"asc\",\"otherBucket\":true,\"missingBucket\":false,\"parentFormat\":{\"id\":\"multi_terms\"},\"include\":[],\"exclude\":[],\"includeIsRegex\":false,\"excludeIsRegex\":false,\"secondaryFields\":[\"service.node.name\"]},\"customLabel\":true},\"b3da070f-3463-4990-b257-40ac399bec87\":{\"label\":\"Memory usage\",\"dataType\":\"number\",\"operationType\":\"last_value\",\"isBucketed\":false,\"scale\":\"ratio\",\"sourceField\":\"process.memory.usage\",\"filter\":{\"query\":\"\\\"process.memory.usage\\\": *\",\"language\":\"kuery\"},\"params\":{\"sortField\":\"@timestamp\",\"format\":{\"id\":\"bytes\",\"params\":{\"decimals\":0}}},\"customLabel\":true},\"a7f6a205-1e8f-4b64-b745-efe20c2c6545\":{\"label\":\"Gen 0 collections\",\"dataType\":\"number\",\"operationType\":\"last_value\",\"isBucketed\":false,\"scale\":\"ratio\",\"sourceField\":\"process.runtime.dotnet.gc.collections.count\",\"filter\":{\"query\":\"\\\"process.runtime.dotnet.gc.collections.count\\\": * AND labels.generation : \\\"gen0\\\" \",\"language\":\"kuery\"},\"params\":{\"sortField\":\"@timestamp\"},\"customLabel\":true},\"2bbcd9e9-15ff-42c1-98a8-65a5b9b4e4c5\":{\"label\":\"Gen 1 collections\",\"dataType\":\"number\",\"operationType\":\"last_value\",\"isBucketed\":false,\"scale\":\"ratio\",\"sourceField\":\"process.runtime.dotnet.gc.collections.count\",\"filter\":{\"query\":\"\\\"process.runtime.dotnet.gc.collections.count\\\": * AND labels.generation : \\\"gen1\\\" \",\"language\":\"kuery\"},\"params\":{\"sortField\":\"@timestamp\"},\"customLabel\":true},\"9f3d67e3-0513-468e-b9b2-6d8780dac3e0\":{\"label\":\"Gen 2 collections\",\"dataType\":\"number\",\"operationType\":\"last_value\",\"isBucketed\":false,\"scale\":\"ratio\",\"sourceField\":\"process.runtime.dotnet.gc.collections.count\",\"filter\":{\"query\":\"\\\"process.runtime.dotnet.gc.collections.count\\\": * AND labels.generation : \\\"gen2\\\" \",\"language\":\"kuery\"},\"params\":{\"sortField\":\"@timestamp\"},\"customLabel\":true},\"21bdcb2d-cee4-4dd9-84cb-5031f073bf85\":{\"label\":\"Managed threads\",\"dataType\":\"number\",\"operationType\":\"last_value\",\"isBucketed\":false,\"scale\":\"ratio\",\"sourceField\":\"process.runtime.dotnet.thread_pool.threads.count\",\"filter\":{\"query\":\"\\\"process.runtime.dotnet.thread_pool.threads.count\\\": *\",\"language\":\"kuery\"},\"params\":{\"sortField\":\"@timestamp\"},\"customLabel\":true}},\"columnOrder\":[\"d5553b2b-25e9-4b94-9697-f04885f4a067\",\"21bdcb2d-cee4-4dd9-84cb-5031f073bf85\",\"b3da070f-3463-4990-b257-40ac399bec87\",\"a7f6a205-1e8f-4b64-b745-efe20c2c6545\",\"2bbcd9e9-15ff-42c1-98a8-65a5b9b4e4c5\",\"9f3d67e3-0513-468e-b9b2-6d8780dac3e0\"],\"sampling\":1,\"ignoreGlobalFilters\":false,\"incompleteColumns\":{},\"indexPatternId\":\"apm_static_data_view_id_default\"}},\"currentIndexPatternId\":\"apm_static_data_view_id_default\"},\"indexpattern\":{\"layers\":{}},\"textBased\":{\"layers\":{}}},\"internalReferences\":[],\"adHocDataViews\":{}}},\"hidePanelTitles\":true,\"enhancements\":{}},\"title\":\"\"},{\"type\":\"lens\",\"gridData\":{\"x\":0,\"y\":8,\"w\":25,\"h\":16,\"i\":\"d0991248-2fad-4f28-bedc-b8723bc45a81\"},\"panelIndex\":\"d0991248-2fad-4f28-bedc-b8723bc45a81\",\"embeddableConfig\":{\"attributes\":{\"title\":\"\",\"description\":\"The amount of physical memory allocated for this process.\",\"visualizationType\":\"lnsXY\",\"type\":\"lens\",\"references\":[{\"type\":\"index-pattern\",\"id\":\"APM_STATIC_DATA_VIEW_ID\",\"name\":\"indexpattern-datasource-layer-961b1efd-6f0d-41e4-a72b-5d66237d212b\"}],\"state\":{\"visualization\":{\"legend\":{\"isVisible\":true,\"position\":\"bottom\",\"showSingleSeries\":true},\"valueLabels\":\"hide\",\"fittingFunction\":\"None\",\"yTitle\":\"Allocated physical memory\",\"axisTitlesVisibilitySettings\":{\"x\":false,\"yLeft\":false,\"yRight\":true},\"tickLabelsVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"labelsOrientation\":{\"x\":0,\"yLeft\":0,\"yRight\":0},\"gridlinesVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"preferredSeriesType\":\"bar_stacked\",\"layers\":[{\"layerId\":\"961b1efd-6f0d-41e4-a72b-5d66237d212b\",\"accessors\":[\"81968f1a-1c2f-46cb-8276-3dca900342e9\",\"54c0cda3-76f3-4b75-9fe1-2265fa68993c\"],\"position\":\"top\",\"seriesType\":\"line\",\"showGridlines\":false,\"layerType\":\"data\",\"colorMapping\":{\"assignments\":[],\"specialAssignments\":[{\"rule\":{\"type\":\"other\"},\"color\":{\"type\":\"loop\"},\"touched\":false}],\"paletteId\":\"eui_amsterdam_color_blind\",\"colorMode\":{\"type\":\"categorical\"}},\"xAccessor\":\"2405efa8-e18d-426f-822f-3a4551bf97d2\",\"yConfig\":[]}]},\"query\":{\"query\":\"agent.name: \\\"opentelemetry/dotnet\\\" \",\"language\":\"kuery\"},\"filters\":[],\"datasourceStates\":{\"formBased\":{\"layers\":{\"961b1efd-6f0d-41e4-a72b-5d66237d212b\":{\"columns\":{\"2405efa8-e18d-426f-822f-3a4551bf97d2\":{\"label\":\"@timestamp\",\"dataType\":\"date\",\"operationType\":\"date_histogram\",\"sourceField\":\"@timestamp\",\"isBucketed\":true,\"scale\":\"interval\",\"params\":{\"interval\":\"auto\",\"includeEmptyRows\":true,\"dropPartials\":false}},\"81968f1a-1c2f-46cb-8276-3dca900342e9\":{\"label\":\"Average\",\"dataType\":\"number\",\"operationType\":\"average\",\"sourceField\":\"process.memory.usage\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"format\":{\"id\":\"bytes\",\"params\":{\"decimals\":0}},\"emptyAsNull\":true},\"customLabel\":true},\"54c0cda3-76f3-4b75-9fe1-2265fa68993c\":{\"label\":\"Max\",\"dataType\":\"number\",\"operationType\":\"max\",\"sourceField\":\"process.memory.usage\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"emptyAsNull\":true,\"format\":{\"id\":\"bytes\",\"params\":{\"decimals\":0}}},\"customLabel\":true}},\"columnOrder\":[\"2405efa8-e18d-426f-822f-3a4551bf97d2\",\"81968f1a-1c2f-46cb-8276-3dca900342e9\",\"54c0cda3-76f3-4b75-9fe1-2265fa68993c\"],\"incompleteColumns\":{},\"sampling\":1,\"indexPatternId\":\"apm_static_data_view_id_default\"}},\"currentIndexPatternId\":\"apm_static_data_view_id_default\"},\"indexpattern\":{\"layers\":{}},\"textBased\":{\"layers\":{}}},\"internalReferences\":[],\"adHocDataViews\":{}}},\"description\":\"The amount of physical memory allocated to the .NET process.\",\"enhancements\":{}},\"title\":\"Allocated physical memory\"},{\"type\":\"lens\",\"gridData\":{\"x\":25,\"y\":8,\"w\":23,\"h\":16,\"i\":\"0bf63f9e-8797-4249-85f7-9407c165f732\"},\"panelIndex\":\"0bf63f9e-8797-4249-85f7-9407c165f732\",\"embeddableConfig\":{\"attributes\":{\"title\":\"\",\"visualizationType\":\"lnsXY\",\"type\":\"lens\",\"references\":[{\"type\":\"index-pattern\",\"id\":\"APM_STATIC_DATA_VIEW_ID\",\"name\":\"indexpattern-datasource-layer-eb4e02de-8962-40fa-9e75-ff25862ca5f3\"}],\"state\":{\"visualization\":{\"legend\":{\"isVisible\":true,\"position\":\"right\",\"maxLines\":1},\"valueLabels\":\"hide\",\"fittingFunction\":\"None\",\"axisTitlesVisibilitySettings\":{\"x\":false,\"yLeft\":false,\"yRight\":true},\"tickLabelsVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"labelsOrientation\":{\"x\":0,\"yLeft\":0,\"yRight\":0},\"gridlinesVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"preferredSeriesType\":\"line\",\"layers\":[{\"layerId\":\"eb4e02de-8962-40fa-9e75-ff25862ca5f3\",\"accessors\":[\"cced3ff5-cfa3-4804-93be-c8d893114e93\",\"211c2cbb-033a-454b-b379-186b8d7b247e\",\"5ac14ba1-f6d4-4015-96d1-aeaa3ed63aec\",\"938252b1-14ec-4a64-b3e5-108e096f116b\",\"1e106c40-c845-4368-baaa-4057e1d29d92\"],\"position\":\"top\",\"seriesType\":\"line\",\"showGridlines\":false,\"layerType\":\"data\",\"colorMapping\":{\"assignments\":[],\"specialAssignments\":[{\"rule\":{\"type\":\"other\"},\"color\":{\"type\":\"loop\"},\"touched\":false}],\"paletteId\":\"eui_amsterdam_color_blind\",\"colorMode\":{\"type\":\"categorical\"}},\"xAccessor\":\"cb7bae9c-fdc5-44a8-8ee8-c0762595511c\"}]},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[],\"datasourceStates\":{\"formBased\":{\"layers\":{\"eb4e02de-8962-40fa-9e75-ff25862ca5f3\":{\"columns\":{\"cb7bae9c-fdc5-44a8-8ee8-c0762595511c\":{\"label\":\"@timestamp\",\"dataType\":\"date\",\"operationType\":\"date_histogram\",\"sourceField\":\"@timestamp\",\"isBucketed\":true,\"scale\":\"interval\",\"params\":{\"interval\":\"auto\",\"includeEmptyRows\":true,\"dropPartials\":false}},\"cced3ff5-cfa3-4804-93be-c8d893114e93\":{\"label\":\"Gen 0\",\"dataType\":\"number\",\"operationType\":\"average\",\"sourceField\":\"process.runtime.dotnet.gc.heap.size\",\"isBucketed\":false,\"scale\":\"ratio\",\"filter\":{\"query\":\"labels.generation : \\\"gen0\\\" \",\"language\":\"kuery\"},\"params\":{\"format\":{\"id\":\"bytes\",\"params\":{\"decimals\":2}},\"emptyAsNull\":true},\"customLabel\":true},\"211c2cbb-033a-454b-b379-186b8d7b247e\":{\"label\":\"Gen 1\",\"dataType\":\"number\",\"operationType\":\"average\",\"sourceField\":\"process.runtime.dotnet.gc.heap.size\",\"isBucketed\":false,\"scale\":\"ratio\",\"filter\":{\"query\":\"labels.generation : \\\"gen1\\\" \",\"language\":\"kuery\"},\"params\":{\"emptyAsNull\":true,\"format\":{\"id\":\"bytes\",\"params\":{\"decimals\":2}}},\"customLabel\":true},\"5ac14ba1-f6d4-4015-96d1-aeaa3ed63aec\":{\"label\":\"Gen 2\",\"dataType\":\"number\",\"operationType\":\"average\",\"sourceField\":\"process.runtime.dotnet.gc.heap.size\",\"isBucketed\":false,\"scale\":\"ratio\",\"filter\":{\"query\":\"labels.generation : \\\"gen2\\\" \",\"language\":\"kuery\"},\"params\":{\"emptyAsNull\":true,\"format\":{\"id\":\"bytes\",\"params\":{\"decimals\":2}}},\"customLabel\":true},\"938252b1-14ec-4a64-b3e5-108e096f116b\":{\"label\":\"LOH\",\"dataType\":\"number\",\"operationType\":\"average\",\"sourceField\":\"process.runtime.dotnet.gc.heap.size\",\"isBucketed\":false,\"scale\":\"ratio\",\"filter\":{\"query\":\"labels.generation:\\\"loh\\\" \",\"language\":\"kuery\"},\"params\":{\"emptyAsNull\":true,\"format\":{\"id\":\"bytes\",\"params\":{\"decimals\":2}}},\"customLabel\":true},\"1e106c40-c845-4368-baaa-4057e1d29d92\":{\"label\":\"POH\",\"dataType\":\"number\",\"operationType\":\"median\",\"sourceField\":\"process.runtime.dotnet.gc.heap.size\",\"isBucketed\":false,\"scale\":\"ratio\",\"filter\":{\"query\":\"labels.generation : \\\"poh\\\" \",\"language\":\"kuery\"},\"params\":{\"emptyAsNull\":true,\"format\":{\"id\":\"bytes\",\"params\":{\"decimals\":2}}},\"customLabel\":true}},\"columnOrder\":[\"cb7bae9c-fdc5-44a8-8ee8-c0762595511c\",\"cced3ff5-cfa3-4804-93be-c8d893114e93\",\"211c2cbb-033a-454b-b379-186b8d7b247e\",\"5ac14ba1-f6d4-4015-96d1-aeaa3ed63aec\",\"938252b1-14ec-4a64-b3e5-108e096f116b\",\"1e106c40-c845-4368-baaa-4057e1d29d92\"],\"incompleteColumns\":{},\"sampling\":1,\"indexPatternId\":\"apm_static_data_view_id_default\"}},\"currentIndexPatternId\":\"apm_static_data_view_id_default\"},\"indexpattern\":{\"layers\":{}},\"textBased\":{\"layers\":{}}},\"internalReferences\":[],\"adHocDataViews\":{}}},\"enhancements\":{}},\"title\":\"Average GC heap size by generation\"}]","timeRestore":false,"title":".NET OpenTelemetry Runtime Metrics","version":1},"coreMigrationVersion":"8.8.0","created_at":"2024-05-17T13:46:01.942Z","id":"c65be603-2c73-4417-972c-033586a56102","managed":false,"references":[{"id":"apm_static_data_view_id_default","name":"ea9f86f0-ff73-4c92-9b93-41baebdcffab:indexpattern-datasource-layer-1ba88117-6e95-46e2-8667-e0bc15145182","type":"index-pattern"},{"id":"APM_STATIC_DATA_VIEW_ID","name":"d0991248-2fad-4f28-bedc-b8723bc45a81:indexpattern-datasource-layer-961b1efd-6f0d-41e4-a72b-5d66237d212b","type":"index-pattern"},{"id":"APM_STATIC_DATA_VIEW_ID","name":"0bf63f9e-8797-4249-85f7-9407c165f732:indexpattern-datasource-layer-eb4e02de-8962-40fa-9e75-ff25862ca5f3","type":"index-pattern"},{"id":"APM_STATIC_DATA_VIEW_ID","name":"controlGroup_2be66584-9de4-4a36-ba54-bfdd1b4ccfb4:optionsListDataView","type":"index-pattern"}],"type":"dashboard","typeMigrationVersion":"8.9.0","updated_at":"2024-05-17T13:46:01.942Z","version":"WzM0NTMsN10="}