Support Jaeger data in trace-analytics (opensearch-project#150) (open…

…search-project#214) Signed-off-by: Derek Ho <[email protected]> (cherry picked from commit f700415) Co-authored-by: Derek Ho <[email protected]> Signed-off-by: Derek Ho <[email protected]>
derek-ho · Feb 23, 2023 · f1d29c1 · f1d29c1
1 parent 5ee3366
commit f1d29c1
Show file tree

Hide file tree

Showing 81 changed files with 19,997 additions and 1,667 deletions.
diff --git a/common/constants/explorer.ts b/common/constants/explorer.ts
@@ -43,6 +43,7 @@ export const REDIRECT_TAB = 'redirect_tab';
 export const PAGE_SIZE = 50;
 export const DEFAULT_COLUMNS = ['', 'Time', '_source'];
 export const OTEL_TRACE_ID = 'traceId';
+export const JAEGER_TRACE_ID = 'traceID';
 export const DATE_PICKER_FORMAT = 'YYYY-MM-DD HH:mm:ss';
 export const TIME_INTERVAL_OPTIONS = [
   {

diff --git a/common/constants/trace_analytics.ts b/common/constants/trace_analytics.ts
@@ -3,6 +3,8 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
+export const JAEGER_INDEX_NAME = '*jaeger-span-*';
+export const JAEGER_SERVICE_INDEX_NAME = '*jaeger-service*';
 export const DATA_PREPPER_INDEX_NAME = 'otel-v1-apm-span-*';
 export const DATA_PREPPER_SERVICE_INDEX_NAME = 'otel-v1-apm-service-map*';
 export const TRACE_ANALYTICS_DATE_FORMAT = 'MM/DD/YYYY HH:mm:ss';
@@ -13,5 +15,6 @@ export const SERVICE_MAP_MAX_EDGES = 1000;
 export const TRACES_MAX_NUM = 3000;
 export const TRACE_ANALYTICS_DOCUMENTATION_LINK = 'https://opensearch.org/docs/latest/observability-plugin/trace/index/';
 
-export const TRACE_ANALYTICS_INDICES_ROUTE = '/api/observability/trace_analytics/indices';
+export const TRACE_ANALYTICS_JAEGER_INDICES_ROUTE = '/api/observability/trace_analytics/jaeger_indices';
+export const TRACE_ANALYTICS_DATA_PREPPER_INDICES_ROUTE = '/api/observability/trace_analytics/data_prepper_indices';
 export const TRACE_ANALYTICS_DSL_ROUTE = '/api/observability/trace_analytics/query';
diff --git a/public/components/application_analytics/__tests__/__snapshots__/service_config.test.tsx.snap b/public/components/application_analytics/__tests__/__snapshots__/service_config.test.tsx.snap
@@ -21,7 +21,7 @@ exports[`Service Config component renders empty service config 1`] = `
   endTime="now"
   filters={Array []}
   http={[MockFunction]}
-  indicesExist={true}
+  mode="data_prepper"
   name=""
   parentBreadcrumb={
     Object {
@@ -1101,7 +1101,7 @@ exports[`Service Config component renders with one service selected 1`] = `
     ]
   }
   http={[MockFunction]}
-  indicesExist={true}
+  mode="data_prepper"
   name=""
   parentBreadcrumb={
     Object {

diff --git a/public/components/application_analytics/__tests__/create.test.tsx b/public/components/application_analytics/__tests__/create.test.tsx
@@ -56,9 +56,9 @@ describe('Create Page', () => {
         setStartTime={setStartTime}
         endTime="now"
         setEndTime={setEndTime}
-        indicesExist={true}
         name=""
         description=""
+        mode='data_prepper'
         setNameWithStorage={setNameWithStorage}
         setDescriptionWithStorage={setDescriptionWithStorage}
         setQueryWithStorage={setQueryWithStorage}
@@ -120,9 +120,9 @@ describe('Create Page', () => {
         setStartTime={setStartTime}
         endTime="now"
         setEndTime={setEndTime}
-        indicesExist={true}
         name="Chic Application"
         description="This is my chic application."
+        mode='data_prepper'   
         setNameWithStorage={setNameWithStorage}
         setDescriptionWithStorage={setDescriptionWithStorage}
         setQueryWithStorage={setQueryWithStorage}
@@ -184,9 +184,9 @@ describe('Create Page', () => {
         setStartTime={setStartTime}
         endTime="now"
         setEndTime={setEndTime}
-        indicesExist={true}
         name=""
         description=""
+        mode='data_prepper'
         setNameWithStorage={setNameWithStorage}
         setDescriptionWithStorage={setDescriptionWithStorage}
         setQueryWithStorage={setQueryWithStorage}
@@ -251,9 +251,9 @@ describe('Create Page', () => {
         setStartTime={setStartTime}
         endTime="now"
         setEndTime={setEndTime}
-        indicesExist={true}
         name=""
         description=""
+        mode='data_prepper'
         setNameWithStorage={setNameWithStorage}
         setDescriptionWithStorage={setDescriptionWithStorage}
         setQueryWithStorage={setQueryWithStorage}
@@ -327,9 +327,9 @@ describe('Create Page', () => {
         setStartTime={setStartTime}
         endTime="now"
         setEndTime={setEndTime}
-        indicesExist={true}
         name=""
         description=""
+        mode='data_prepper'
         setNameWithStorage={setNameWithStorage}
         setDescriptionWithStorage={setDescriptionWithStorage}
         setQueryWithStorage={setQueryWithStorage}
@@ -403,9 +403,9 @@ describe('Create Page', () => {
         setStartTime={setStartTime}
         endTime="now"
         setEndTime={setEndTime}
-        indicesExist={true}
         name=""
         description=""
+        mode='data_prepper'
         setNameWithStorage={setNameWithStorage}
         setDescriptionWithStorage={setDescriptionWithStorage}
         setQueryWithStorage={setQueryWithStorage}
@@ -479,9 +479,9 @@ describe('Create Page', () => {
         setStartTime={setStartTime}
         endTime="now"
         setEndTime={setEndTime}
-        indicesExist={true}
         name=""
         description=""
+        mode='data_prepper'
         setNameWithStorage={setNameWithStorage}
         setDescriptionWithStorage={setDescriptionWithStorage}
         setQueryWithStorage={setQueryWithStorage}
@@ -555,9 +555,9 @@ describe('Create Page', () => {
         setStartTime={setStartTime}
         endTime="now"
         setEndTime={setEndTime}
-        indicesExist={true}
         name=""
         description=""
+        mode='data_prepper'
         setNameWithStorage={setNameWithStorage}
         setDescriptionWithStorage={setDescriptionWithStorage}
         setQueryWithStorage={setQueryWithStorage}

diff --git a/public/components/application_analytics/__tests__/service_config.test.tsx b/public/components/application_analytics/__tests__/service_config.test.tsx
@@ -46,7 +46,7 @@ describe('Service Config component', () => {
         setStartTime={setStartTime}
         endTime="now"
         setEndTime={setEndTime}
-        indicesExist={true}
+        mode='data_prepper'
         dslService={dslService}
         selectedServices={[]}
         setSelectedServices={setSelectedServices}
@@ -108,7 +108,7 @@ describe('Service Config component', () => {
         setStartTime={setStartTime}
         endTime="now"
         setEndTime={setEndTime}
-        indicesExist={true}
+        mode='data_prepper'
         dslService={dslService}
         selectedServices={[]}
         setSelectedServices={setSelectedServices}

diff --git a/public/components/application_analytics/components/application.tsx b/public/components/application_analytics/components/application.tsx
@@ -113,6 +113,7 @@ export function Application(props: AppDetailProps) {
     setFilters,
     callback,
     queryManager,
+    mode,
   } = props;
   const [application, setApplication] = useState<ApplicationType>({
     id: '',
@@ -230,7 +231,7 @@ export function Application(props: AppDetailProps) {
   }, [appId, application.name]);
 
   useEffect(() => {
-    const DSL = filtersToDsl(filters, query, appStartTime, appEndTime, 'app', appConfigs);
+    const DSL = filtersToDsl(mode, filters, query, appStartTime, appEndTime, 'app', appConfigs);
     setSpanDSL(DSL);
   }, [filters, appConfigs, query, appStartTime, appEndTime]);
 
@@ -350,6 +351,7 @@ export function Application(props: AppDetailProps) {
             openFlyout={setSpanFlyoutId}
             DSL={spanDSL}
             setTotal={setTotalSpans}
+            mode='data_prepper'
           />
         </EuiPanel>
       </>
@@ -563,6 +565,7 @@ export function Application(props: AppDetailProps) {
             isFlyoutVisible={!!spanFlyoutId}
             closeFlyout={closeSpanFlyout}
             addSpanFilter={addSpanFilter}
+            mode="data_prepper"
           />
         )}
         {traceFlyoutId && (

diff --git a/public/components/application_analytics/components/config_components/service_config.tsx b/public/components/application_analytics/components/config_components/service_config.tsx
@@ -40,6 +40,7 @@ export const ServiceConfig = (props: ServiceConfigProps) => {
     selectedServices,
     setSelectedServices,
   } = props;
+  const { mode } = props;
   const [servicesOpen, setServicesOpen] = useState(false);
   const [serviceMap, setServiceMap] = useState<ServiceObject>({});
   const [serviceMapIdSelected, setServiceMapIdSelected] = useState<
@@ -49,7 +50,7 @@ export const ServiceConfig = (props: ServiceConfigProps) => {
   const [modalLayout, setModalLayout] = useState(<EuiOverlayMask />);
 
   useEffect(() => {
-    handleServiceMapRequest(http, dslService, setServiceMap);
+    handleServiceMapRequest(http, dslService, mode, setServiceMap);
   }, []);
 
   useEffect(() => {

diff --git a/public/components/application_analytics/components/config_components/trace_config.tsx b/public/components/application_analytics/components/config_components/trace_config.tsx
@@ -17,7 +17,7 @@ import {
 } from '@elastic/eui';
 import DSLService from 'public/services/requests/dsl';
 import React, { useEffect, useState } from 'react';
-import { FilterType } from 'public/components/trace_analytics/components/common/filters/filters';
+import { FilterType } from '../../../../../public/components/trace_analytics/components/common/filters/filters';
 import { OptionType } from '../../../../../common/types/application_analytics';
 import { filtersToDsl } from '../../../trace_analytics/components/common/helper_functions';
 import { handleDashboardRequest } from '../../../trace_analytics/requests/dashboard_request_handler';
@@ -42,6 +42,7 @@ export const TraceConfig = (props: TraceConfigProps) => {
     endTime,
     selectedTraces,
     setSelectedTraces,
+    mode
   } = props;
   const [traceOpen, setTraceOpen] = useState(false);
   const [loading, setLoading] = useState(false);
@@ -54,16 +55,17 @@ export const TraceConfig = (props: TraceConfigProps) => {
 
   useEffect(() => {
     setLoading(true);
-    const timeFilterDSL = filtersToDsl([], '', startTime, endTime);
+    const timeFilterDSL = filtersToDsl(mode, [], '', startTime, endTime);
     const latencyTrendStartTime = dateMath.parse(endTime, { roundUp: true })?.subtract(24, 'hours').toISOString()!;
-    const latencyTrendDSL = filtersToDsl(filters, query, latencyTrendStartTime, endTime);
+    const latencyTrendDSL = filtersToDsl(mode, filters, query, latencyTrendStartTime, endTime);
     handleDashboardRequest(
       http,
       dslService,
       timeFilterDSL,
       latencyTrendDSL,
       traceItems,
       setTraceItems,
+      mode,
       setPercentileMap
     ).then(() => setLoading(false));
     setRedirect(false);

diff --git a/...c/components/application_analytics/components/flyout_components/service_detail_flyout.tsx b/...c/components/application_analytics/components/flyout_components/service_detail_flyout.tsx
@@ -19,7 +19,7 @@ import {
   handleServiceMapRequest,
   handleServiceViewRequest,
 } from '../../../../../public/components/trace_analytics/requests/services_request_handler';
-import { filtersToDsl } from '../../../../../public/components/trace_analytics/components/common/helper_functions';
+import { filtersToDsl, processTimeStamp } from '../../../../../public/components/trace_analytics/components/common/helper_functions';
 import { ServiceMap } from '../../../../../public/components/trace_analytics/components/services';
 import { ServiceObject } from '../../../../../public/components/trace_analytics/components/common/plots/service_map';
 import { SpanDetailTable } from '../../../../../public/components/trace_analytics/components/traces/span_detail_table';
@@ -44,6 +44,7 @@ export function ServiceDetailFlyout(props: ServiceFlyoutProps) {
     query,
     closeServiceFlyout,
     openSpanFlyout,
+    mode,
   } = props;
   const [fields, setFields] = useState<any>({});
   const [serviceMap, setServiceMap] = useState<ServiceObject>({});
@@ -110,16 +111,17 @@ export function ServiceDetailFlyout(props: ServiceFlyoutProps) {
           DSL={DSL}
           openFlyout={openSpanFlyout}
           setTotal={setTotal}
+          mode={mode}
         />
       </>
     );
   }, [serviceName, fields, serviceMap, DSL, serviceMapIdSelected]);
 
   useEffect(() => {
-    const serviceDSL = filtersToDsl(filters, query, startTime, endTime, 'app', appConfigs);
-    handleServiceViewRequest(serviceName, http, serviceDSL, setFields);
-    handleServiceMapRequest(http, serviceDSL, setServiceMap, serviceName);
-    const spanDSL = filtersToDsl(filters, query, startTime, endTime, 'app', appConfigs);
+    const serviceDSL = filtersToDsl(mode, filters, query, processTimeStamp(startTime, mode), processTimeStamp(endTime, mode), 'app', appConfigs);
+    handleServiceViewRequest(serviceName, http, serviceDSL, setFields, mode);
+    handleServiceMapRequest(http, serviceDSL, mode, setServiceMap, serviceName);
+    const spanDSL = filtersToDsl(mode, filters, query, startTime, endTime, 'app', appConfigs);
     spanDSL.query.bool.must.push({
       term: {
         serviceName,

diff --git a/public/components/application_analytics/components/flyout_components/trace_detail_flyout.tsx b/public/components/application_analytics/components/flyout_components/trace_detail_flyout.tsx
@@ -17,7 +17,7 @@ interface TraceFlyoutProps extends TraceAnalyticsComponentDeps {
 export function TraceDetailFlyout(props: TraceFlyoutProps) {
   const { traceId, http, closeTraceFlyout, openSpanFlyout } = props;
   const renderContent = (
-    <TraceDetailRender traceId={traceId} http={http} openSpanFlyout={openSpanFlyout} />
+    <TraceDetailRender traceId={traceId} http={http} openSpanFlyout={openSpanFlyout} mode='data_prepper'/>
   );
   return (
     <EuiFlyout data-test-subj="traceDetailFlyout" onClose={closeTraceFlyout} size="m">

diff --git a/public/components/application_analytics/components/flyout_components/trace_detail_render.tsx b/public/components/application_analytics/components/flyout_components/trace_detail_render.tsx
@@ -14,14 +14,16 @@ import {
 } from '../../../trace_analytics/requests/traces_request_handler';
 import { HttpStart } from '../../../../../../../src/core/public';
 import { getListItem } from '../../helpers/utils';
+import { TraceAnalyticsMode } from '../../../../../public/components/trace_analytics/home';
 
 interface TraceDetailRenderProps {
   traceId: string;
   http: HttpStart;
   openSpanFlyout: (spanId: string) => void;
+  mode : TraceAnalyticsMode
 }
 
-export const TraceDetailRender = ({ traceId, http, openSpanFlyout }: TraceDetailRenderProps) => {
+export const TraceDetailRender = ({ traceId, http, openSpanFlyout, mode }: TraceDetailRenderProps) => {
   const [fields, setFields] = useState<any>({});
   const [serviceBreakdownData, setServiceBreakdownData] = useState([]);
   const [payloadData, setPayloadData] = useState('');
@@ -66,6 +68,7 @@ export const TraceDetailRender = ({ traceId, http, openSpanFlyout }: TraceDetail
           colorMap={colorMap}
           page="app"
           openSpanFlyout={openSpanFlyout}
+          mode={mode}
         />
         <EuiSpacer size="xs" />
         <EuiHorizontalRule margin="s" />
@@ -83,9 +86,9 @@ export const TraceDetailRender = ({ traceId, http, openSpanFlyout }: TraceDetail
   }, [traceId, fields, serviceBreakdownData, colorMap, payloadData]);
 
   useEffect(() => {
-    handleTraceViewRequest(traceId, http, fields, setFields);
-    handleServicesPieChartRequest(traceId, http, setServiceBreakdownData, setColorMap);
-    handlePayloadRequest(traceId, http, payloadData, setPayloadData);
+    handleTraceViewRequest(traceId, http, fields, setFields, mode);
+    handleServicesPieChartRequest(traceId, http, setServiceBreakdownData, setColorMap, mode);
+    handlePayloadRequest(traceId, http, payloadData, setPayloadData, mode);
   }, [traceId]);
 
   return renderContent;

diff --git a/public/components/application_analytics/home.tsx b/public/components/application_analytics/home.tsx
@@ -20,7 +20,7 @@ import { Application } from './components/application';
 import { CreateApp } from './components/create';
 import { TraceAnalyticsComponentDeps, TraceAnalyticsCoreDeps } from '../trace_analytics/home';
 import { FilterType } from '../trace_analytics/components/common/filters/filters';
-import { handleIndicesExistRequest } from '../trace_analytics/requests/request_handler';
+import { handleDataPrepperIndicesExistRequest } from '../trace_analytics/requests/request_handler';
 import { ObservabilitySideBar } from '../common/side_nav';
 import { NotificationsStart } from '../../../../../src/core/public';
 import { APP_ANALYTICS_API_PREFIX } from '../../../common/constants/application_analytics';
@@ -114,7 +114,7 @@ export const Home = (props: HomeProps) => {
   };
 
   useEffect(() => {
-    handleIndicesExistRequest(http, setIndicesExist);
+    handleDataPrepperIndicesExistRequest(http, setIndicesExist);
   }, []);
 
   const commonProps: AppAnalyticsComponentDeps = {
@@ -137,7 +137,8 @@ export const Home = (props: HomeProps) => {
     setStartTime,
     endTime,
     setEndTime,
-    indicesExist,
+    mode: 'data_prepper',
+    dataPrepperIndicesExist:  indicesExist 
   };
 
   const setToast = (title: string, color = 'success', text?: ReactChild) => {

diff --git a/public/components/event_analytics/explorer/events_views/docViewRow.tsx b/public/components/event_analytics/explorer/events_views/docViewRow.tsx
@@ -12,7 +12,7 @@ import { useEffect } from 'react';
 import { IExplorerFields, IField } from '../../../../../common/types/explorer';
 import { DocFlyout } from './doc_flyout';
 import { HttpStart } from '../../../../../../../src/core/public';
-import { OTEL_TRACE_ID, DATE_PICKER_FORMAT } from '../../../../../common/constants/explorer';
+import { OTEL_TRACE_ID, DATE_PICKER_FORMAT, JAEGER_TRACE_ID } from '../../../../../common/constants/explorer';
 import { SurroundingFlyout } from './surrounding_flyout';
 import PPLService from '../../../../services/requests/ppl';
 import { isValidTraceId } from '../../utils';
@@ -77,13 +77,13 @@ export const DocViewRow = forwardRef((props: IDocViewRowProps, ref) => {
         <span>
           <dl className="source truncate-by-height">
             {toPairs(doc).map((entry: string[]) => {
-              const isTraceField = entry[0] === OTEL_TRACE_ID;
+              const isTraceField = (entry[0] === OTEL_TRACE_ID || entry[0] === JAEGER_TRACE_ID);
               return (
                 <span key={uniqueId('grid-desc')}>
                   <dt>{entry[0]}:</dt>
                   <dd>
                     <span>
-                      {isTraceField && isValidTraceId(entry[1]) && !isFlyout ? (
+                      {isTraceField && (isValidTraceId(entry[1]) || entry[0] === JAEGER_TRACE_ID) && !isFlyout ? (
                         <EuiLink onClick={tracesFlyout}>{entry[1]}</EuiLink>
                       ) : (
                         entry[1]

diff --git a/public/components/event_analytics/explorer/events_views/trace_block/trace_block.tsx b/public/components/event_analytics/explorer/events_views/trace_block/trace_block.tsx
@@ -22,7 +22,7 @@ interface props {
 }
 
 export const TraceBlock = ({ http, hit, logTraceId }: props) => {
-  if (logTraceId === '' || !isValidTraceId(logTraceId)) {
+  if ((!hit.traceID || hit.traceID.length === 0) && (logTraceId === '' || !isValidTraceId(logTraceId))){
     return (
       <>
         <EuiCallOut iconType="help" title="No Trace Id found in the event.">
@@ -42,6 +42,7 @@ export const TraceBlock = ({ http, hit, logTraceId }: props) => {
       </>
     );
   }
+  const mode = (!hit.traceID || hit.traceID.length === 0) ? 'data_prepper' : 'jaeger'
 
-  return <TraceDetailRender traceId={logTraceId} http={http} />;
+  return <TraceDetailRender traceId={hit.traceID || logTraceId} http={http} mode={mode}/>;
 };