split request into 2 fetches

x-pack/plugins/maps/public/classes/sources/es_geo_line_source/convert_to_geojson.test.ts

@@ -8,10 +8,9 @@ import { convertToGeoJson } from './convert_to_geojson';
 
 const esResponse = {
   aggregations: {
-    entitySplit: {
-      buckets: [
-        {
-          key: 'ios',
+    tracks: {
+      buckets: {
+        ios: {
           doc_count: 1,
           path: {
             type: 'Feature',
@@ -27,8 +26,7 @@ const esResponse = {
             },
           },
         },
-        {
-          key: 'osx',
+        osx: {
           doc_count: 1,
           path: {
             type: 'Feature',
@@ -44,7 +42,7 @@ const esResponse = {
             },
           },
         },
-      ],
+      },
     },
   },
 };

x-pack/plugins/maps/public/classes/sources/es_geo_line_source/convert_to_geojson.ts

@@ -14,20 +14,23 @@ export function convertToGeoJson(esResponse: any, entitySplitFieldName: string)
   const features: Feature[] = [];
   let numTrimmedTracks = 0;
 
-  const buckets = _.get(esResponse, 'aggregations.entitySplit.buckets', []);
-  buckets.forEach((bucket: any) => {
+  const buckets = _.get(esResponse, 'aggregations.tracks.buckets', {});
+  const entityKeys = Object.keys(buckets);
+  for (let i = 0; i < entityKeys.length; i++) {
+    const entityKey = entityKeys[i];
+    const bucket = buckets[entityKey];
     const feature = bucket.path as Feature;
     if (!feature.properties!.complete) {
       numTrimmedTracks++;
     }
-    feature.id = bucket.key;
+    feature.id = entityKey;
     feature.properties = {
-      [entitySplitFieldName]: bucket.key,
+      [entitySplitFieldName]: entityKey,
       ...feature.properties,
       ...extractPropertiesFromBucket(bucket, KEYS_TO_IGNORE),
     };
     features.push(feature);
-  });
+  }
 
   return {
     featureCollection: {

x-pack/plugins/maps/public/classes/sources/es_geo_line_source/es_geo_line_source.tsx

@@ -29,6 +29,7 @@ import { isValidStringConfig } from '../../util/valid_string_config';
 import { Adapters } from '../../../../../../../src/plugins/inspector/common/adapters';
 import { IField } from '../../fields/field';
 import { ITooltipProperty, TooltipProperty } from '../../tooltips/tooltip_property';
+import { esFilters } from '../../../../../../../src/plugins/data/public';
 
 const MAX_TRACKS = 1000;
 
@@ -159,17 +160,68 @@ export class ESGeoLineSource extends AbstractESAggSource {
     isRequestStillActive: () => boolean
   ): Promise<GeoJsonWithMeta> {
     const indexPattern = await this.getIndexPattern();
-    const searchSource = await this.makeSearchSource(searchFilters, 0);
 
+    // Request is broken into 2 requests
+    // 1) fetch entities: filtered by buffer so that top entities in view are returned
+    // 2) fetch tracks: not filtered by buffer to avoid having invalid tracks
+    //    when the track extends beyond the area of the map buffer.
+
+    //
+    // Fetch entities
+    //
+    const entitySearchSource = await this.makeSearchSource(searchFilters, 0);
     const splitField = getField(indexPattern, this._descriptor.splitField);
     const cardinalityAgg = { precision_threshold: 1 };
     const termsAgg = { size: MAX_TRACKS };
-    searchSource.setField('aggs', {
+    entitySearchSource.setField('aggs', {
       totalEntities: {
         cardinality: addFieldToDSL(cardinalityAgg, splitField),
       },
       entitySplit: {
         terms: addFieldToDSL(termsAgg, splitField),
+      },
+    });
+    const entityResp = await this._runEsQuery({
+      requestId: `${this.getId()}_entities`,
+      requestName: i18n.translate('xpack.maps.source.esGeoLine.entityRequestName', {
+        defaultMessage: '{layerName} entities',
+        values: {
+          layerName,
+        },
+      }),
+      searchSource: entitySearchSource,
+      registerCancelCallback,
+      requestDescription: i18n.translate('xpack.maps.source.esGeoLine.entityRequestDescription', {
+        defaultMessage: 'Elasticsearch terms request to fetch entities within map buffer.',
+      }),
+    });
+    const entityBuckets: Array<{ key: string; doc_count: number }> = _.get(
+      entityResp,
+      'aggregations.entitySplit.buckets',
+      []
+    );
+    const totalEntities = _.get(entityResp, 'aggregations.totalEntities.value', 0);
+    const areEntitiesTrimmed = entityBuckets.length >= MAX_TRACKS;
+
+    //
+    // Fetch tracks
+    //
+    const entityFilters: { [key: string]: unknown } = {};
+    for (let i = 0; i < entityBuckets.length; i++) {
+      entityFilters[entityBuckets[i].key] = esFilters.buildPhraseFilter(
+        splitField,
+        entityBuckets[i].key,
+        indexPattern
+      ).query;
+    }
+    const tracksSearchFilters = { ...searchFilters };
+    delete tracksSearchFilters.buffer;
+    const tracksSearchSource = await this.makeSearchSource(tracksSearchFilters, 0);
+    tracksSearchSource.setField('aggs', {
+      tracks: {
+        filters: {
+          filters: entityFilters,
+        },
         aggs: {
           path: {
             geo_line: {
@@ -185,24 +237,26 @@ export class ESGeoLineSource extends AbstractESAggSource {
         },
       },
     });
-
-    const resp = await this._runEsQuery({
-      requestId: this.getId(),
-      requestName: layerName,
-      searchSource,
+    const tracksResp = await this._runEsQuery({
+      requestId: `${this.getId()}_tracks`,
+      requestName: i18n.translate('xpack.maps.source.esGeoLine.trackRequestName', {
+        defaultMessage: '{layerName} tracks',
+        values: {
+          layerName,
+        },
+      }),
+      searchSource: tracksSearchSource,
       registerCancelCallback,
-      requestDescription: i18n.translate('xpack.maps.source.esGeoLine.requestLabel', {
-        defaultMessage: 'Elasticsearch tracks request',
+      requestDescription: i18n.translate('xpack.maps.source.esGeoLine.trackRequestDescription', {
+        defaultMessage:
+          'Elasticsearch geo_line request to fetch tracks for entities. Tracks are not filtered by map buffer.',
       }),
     });
-
-    const entityBuckets = _.get(resp, 'aggregations.entitySplit.buckets', []);
-    const totalEntities = _.get(resp, 'aggregations.totalEntities.value', 0);
-    const areEntitiesTrimmed = entityBuckets.length >= MAX_TRACKS;
     const { featureCollection, numTrimmedTracks } = convertToGeoJson(
-      resp,
+      tracksResp,
       this._descriptor.splitField
     );
+
     return {
       data: featureCollection,
       meta: {
@@ -285,7 +339,7 @@ export class ESGeoLineSource extends AbstractESAggSource {
         i18n.translate('xpack.maps.source.esGeoLine.isTrackCompleteLabel', {
           defaultMessage: 'track is complete',
         }),
-        properties.complete.toString()
+        properties!.complete.toString()
       )
     );
     return tooltipProperties;