allow adding _meta to ES/OS mapping/template (#3597)

- [x] Add _meta to the mapping/template if provided
- [x] Add tests
- [x] Tested fixMappingRequests under diff es/os versions and made
changes

e2e/package.json

@@ -42,9 +42,9 @@
         "ms": "^2.1.3"
     },
     "devDependencies": {
-        "@terascope/types": "^0.15.0",
+        "@terascope/types": "^0.15.1",
         "bunyan": "^1.8.15",
-        "elasticsearch-store": "^0.81.1",
+        "elasticsearch-store": "^0.82.0",
         "fs-extra": "^11.2.0",
         "ms": "^2.1.3",
         "nanoid": "^3.3.4",

packages/data-mate/package.json

@@ -1,7 +1,7 @@
 {
     "name": "@terascope/data-mate",
     "displayName": "Data-Mate",
-    "version": "0.54.1",
+    "version": "0.54.2",
     "description": "Library of data validations/transformations",
     "homepage": "https://github.com/terascope/teraslice/tree/master/packages/data-mate#readme",
     "repository": {
@@ -29,9 +29,9 @@
         "test:watch": "ts-scripts test --watch . --"
     },
     "dependencies": {
-        "@terascope/data-types": "^0.48.0",
-        "@terascope/types": "^0.15.0",
-        "@terascope/utils": "^0.57.0",
+        "@terascope/data-types": "^0.48.1",
+        "@terascope/types": "^0.15.1",
+        "@terascope/utils": "^0.57.1",
         "@types/validator": "^13.11.9",
         "awesome-phonenumber": "^2.70.0",
         "date-fns": "^2.30.0",
@@ -46,7 +46,7 @@
         "uuid": "^9.0.1",
         "valid-url": "^1.0.9",
         "validator": "^13.11.0",
-        "xlucene-parser": "^0.56.1"
+        "xlucene-parser": "^0.56.2"
     },
     "devDependencies": {
         "@types/ip6addr": "^0.2.6",

packages/data-types/package.json

@@ -1,7 +1,7 @@
 {
     "name": "@terascope/data-types",
     "displayName": "Data Types",
-    "version": "0.48.0",
+    "version": "0.48.1",
     "description": "A library for defining the data structures and mapping",
     "homepage": "https://github.com/terascope/teraslice/tree/master/packages/data-types#readme",
     "bugs": {
@@ -26,8 +26,8 @@
         "test:watch": "ts-scripts test --watch . --"
     },
     "dependencies": {
-        "@terascope/types": "^0.15.0",
-        "@terascope/utils": "^0.57.0",
+        "@terascope/types": "^0.15.1",
+        "@terascope/utils": "^0.57.1",
         "graphql": "^14.7.0",
         "lodash": "^4.17.21",
         "yargs": "^17.7.2"

packages/data-types/src/data-type.ts

@@ -111,13 +111,14 @@ export class DataType {
      */
     toESMapping({
         typeName, overrides, distribution = ElasticsearchDistribution.elasticsearch,
-        majorVersion = 6, minorVersion = 8, version = '6.8.6'
+        majorVersion = 6, minorVersion = 8, version = '6.8.6', _meta
     }: Partial<i.ESMappingOptions> = {}): ESMapping {
         const indexType = typeName || this.name || '_doc';
 
         const mappingSettings: ESTypeMappings = {
             dynamic: false,
             properties: {},
+            ..._meta && { _meta }
         };
 
         if (

packages/data-types/src/interfaces.ts

@@ -106,4 +106,9 @@ export interface ESMappingOptions extends ClientMetadata {
      * uses a deep assignment so nested fields can be overwritten.
      */
     overrides?: Partial<ESMapping>;
+
+    /**
+     * Any metadata to add to the index mapping
+     */
+    _meta?: Record<string, any>;
 }

packages/data-types/test/data-type-es-spec.ts

@@ -43,6 +43,7 @@ describe('DataType (elasticsearch)', () => {
                         _all: {
                             enabled: false,
                         },
+                        _meta: { foo: 'foo' },
                         dynamic: false,
                         properties: {
                             hello: { type: 'text' },
@@ -62,7 +63,8 @@ describe('DataType (elasticsearch)', () => {
                 distribution: ElasticsearchDistribution.elasticsearch,
                 minorVersion: 8,
                 majorVersion: 6,
-                version: '6.8.6'
+                version: '6.8.6',
+                _meta: { foo: 'foo' }
             };
 
             expect(dataType.toESMapping(mappingConfig)).toEqual(results);
@@ -84,6 +86,7 @@ describe('DataType (elasticsearch)', () => {
                         foo: { type: 'keyword' },
                         bar: { type: 'keyword' },
                     },
+                    _meta: { foo: 'foo' },
                 },
                 settings: {},
             };
@@ -93,7 +96,8 @@ describe('DataType (elasticsearch)', () => {
                 distribution: ElasticsearchDistribution.elasticsearch,
                 minorVersion: 3,
                 majorVersion: 7,
-                version: '7.3.1'
+                version: '7.3.1',
+                _meta: { foo: 'foo' },
             };
 
             expect(dataType.toESMapping(mappingConfig)).toEqual(results);

packages/elasticsearch-api/package.json

@@ -1,7 +1,7 @@
 {
     "name": "@terascope/elasticsearch-api",
     "displayName": "Elasticsearch API",
-    "version": "3.18.0",
+    "version": "3.18.1",
     "description": "Elasticsearch client api used across multiple services, handles retries and exponential backoff",
     "homepage": "https://github.com/terascope/teraslice/tree/master/packages/elasticsearch-api#readme",
     "bugs": {
@@ -23,16 +23,16 @@
         "test:watch": "TEST_RESTRAINED_ELASTICSEARCH='true' ts-scripts test --watch . --"
     },
     "dependencies": {
-        "@terascope/types": "^0.15.0",
-        "@terascope/utils": "^0.57.0",
+        "@terascope/types": "^0.15.1",
+        "@terascope/utils": "^0.57.1",
         "bluebird": "^3.7.2",
         "setimmediate": "^1.0.5"
     },
     "devDependencies": {
         "@opensearch-project/opensearch": "^1.2.0",
         "@types/elasticsearch": "^5.0.43",
         "elasticsearch": "^15.4.1",
-        "elasticsearch-store": "^0.81.1",
+        "elasticsearch-store": "^0.82.0",
         "elasticsearch6": "npm:@elastic/elasticsearch@^6.7.0",
         "elasticsearch7": "npm:@elastic/elasticsearch@^7.0.0",
         "elasticsearch8": "npm:@elastic/elasticsearch@^8.0.0"

packages/elasticsearch-store/package.json

@@ -1,7 +1,7 @@
 {
     "name": "elasticsearch-store",
     "displayName": "Elasticsearch Store",
-    "version": "0.81.1",
+    "version": "0.82.0",
     "description": "An API for managing an elasticsearch index, with versioning and migration support.",
     "homepage": "https://github.com/terascope/teraslice/tree/master/packages/elasticsearch-store#readme",
     "bugs": {
@@ -29,10 +29,10 @@
         "test:watch": "ts-scripts test --watch . --"
     },
     "dependencies": {
-        "@terascope/data-mate": "^0.54.1",
-        "@terascope/data-types": "^0.48.0",
-        "@terascope/types": "^0.15.0",
-        "@terascope/utils": "^0.57.0",
+        "@terascope/data-mate": "^0.54.2",
+        "@terascope/data-types": "^0.48.1",
+        "@terascope/types": "^0.15.1",
+        "@terascope/utils": "^0.57.1",
         "ajv": "^6.12.6",
         "elasticsearch6": "npm:@elastic/elasticsearch@^6.7.0",
         "elasticsearch7": "npm:@elastic/elasticsearch@^7.0.0",
@@ -41,7 +41,7 @@
         "opensearch2": "npm:@opensearch-project/opensearch@^2.2.1",
         "setimmediate": "^1.0.5",
         "uuid": "^9.0.1",
-        "xlucene-translator": "^0.42.1"
+        "xlucene-translator": "^0.42.2"
     },
     "devDependencies": {
         "@types/uuid": "^9.0.8"

packages/elasticsearch-store/src/elasticsearch-client/method-helpers/helper-utils.ts

@@ -29,13 +29,17 @@ function validDistributionAndVersion(
 }
 
 export function ensureNoTypeInMapping(mappings: Record<string, any> | undefined) {
+    const parsed: Record<string, any> = {};
     if (mappings != null) {
         for (const [k, v] of Object.entries(mappings)) {
-            if (k === 'properties') return { [k]: v };
+            if (k === 'properties') parsed[k] = v;
+            if (k === '_meta') parsed[k] = v;
 
-            if (v.properties) return { properties: v.properties };
+            if (v.properties) parsed.properties = v.properties;
+            if (v._meta) parsed._meta = v._meta;
         }
     }
+    return parsed;
 }
 
 export function ensureTypeInMapping(body: ESTypes.IndexTemplateProperties | undefined) {

packages/elasticsearch-store/src/elasticsearch-client/method-helpers/indicesPutTemplate.ts

@@ -1,4 +1,5 @@
 import { ElasticsearchDistribution, ClientParams, ClientMetadata } from '@terascope/types';
+import { get, isNumber } from '@terascope/utils';
 
 import { ensureNoTypeInMapping, ensureTypeInMapping } from './helper-utils';
 
@@ -20,11 +21,14 @@ export function convertIndicesPutTemplateParams(
                 ...parsedParams
             } = params;
 
+            const indexSchemaVersion = get(body, 'version');
+
             return {
                 index_patterns: body?.index_patterns,
                 aliases: body?.aliases,
                 mappings: ensureNoTypeInMapping(body?.mappings),
                 settings: body?.settings,
+                ...isNumber(indexSchemaVersion) && { version: indexSchemaVersion },
                 ...parsedParams
             };
         }

packages/elasticsearch-store/src/index-manager.ts

@@ -1,5 +1,7 @@
 import * as ts from '@terascope/utils';
-import { ClientParams, ClientResponse, ClientMetadata } from '@terascope/types';
+import {
+    ClientParams, ClientResponse, ClientMetadata, ESMapping
+} from '@terascope/types';
 import * as utils from './utils';
 import { IndexConfig, MigrateIndexOptions } from './interfaces';
 import { Client } from './elasticsearch-client';
@@ -103,12 +105,13 @@ export class IndexManager {
 
         logger.trace(`Using ${this.clientMetadata.distribution} version ${this.clientMetadata.version}`);
 
-        const body: any = config.data_type.toESMapping({
+        const body = config.data_type.toESMapping({
             typeName: config.name,
             overrides: {
                 settings,
             },
-            ...this.clientMetadata
+            ...this.clientMetadata,
+            ...config._meta && { _meta: config._meta }
         });
 
         const enableMutations = (
@@ -384,13 +387,13 @@ export class IndexManager {
      * Safely create or update a template
      */
     async upsertTemplate(
-        template: Record<string, any>, logger?: ts.Logger
+        template: ESMapping, logger?: ts.Logger
     ): Promise<void> {
         const { template: name, version } = template;
 
         try {
-            const templates = await this.getTemplate(name, true);
-            const latestVersion = templates[name].version;
+            const templates = await this.getTemplate(name || '', true);
+            const latestVersion = templates[name || ''].version;
             if (version === latestVersion) return;
         } catch (err) {
             if (err.statusCode !== 404) {

packages/elasticsearch-store/src/interfaces.ts

@@ -88,6 +88,11 @@ export interface IndexConfig<T extends AnyObject> {
      * The default query access to use
     */
     default_query_access?: QueryAccess<T>;
+
+    /**
+     * Any metadata to add to the index mapping
+     */
+    _meta?: Record<string, any>;
 }
 
 /** Elasticsearch Index Schema, Mapping and Version */

packages/elasticsearch-store/src/utils/elasticsearch.ts

@@ -1,6 +1,7 @@
 import * as ts from '@terascope/utils';
 import {
-    ESFieldType, ESTypeMapping, ClientMetadata, ElasticsearchDistribution
+    ESFieldType, ESTypeMapping, ClientMetadata, ElasticsearchDistribution,
+    ESMapping
 } from '@terascope/types';
 import { Client } from '../elasticsearch-client';
 import { getErrorType } from './errors';
@@ -174,7 +175,7 @@ export function isOpensearch2(client: Client): boolean {
 
 // TODO: move this logic over to datatype
 export function fixMappingRequest(
-    client: Client, _params: Record<string, any>, isTemplate: boolean
+    client: Client, _params: { body: ESMapping, name?: string, index?: string }, isTemplate: boolean
 ): any {
     if (!_params || !_params.body) {
         throw new Error('Invalid mapping request');
@@ -190,24 +191,41 @@ export function fixMappingRequest(
         }
         delete params.body.template;
     }
+
     // we do not support v5 anymore
     if (esVersion !== 6) {
-        const typeMappings: Record<string, any> = ts.get(params.body, 'mappings', {});
-        if (typeMappings.properties) {
-            defaultParams.include_type_name = false;
-        } else {
+        const mappings = params?.body?.mappings || {};
+        if (!mappings.properties && mappings._doc) {
+            // esV8/osV2 seem to convert properly if mapping._doc.properties or mapping.properties
+            // but esV7/osV1 only seem to work w/include_type_name if properties is under "_doc"
+            // along w/metadata fields so set include_type_name if _doc & ensure metadata is in _doc
             defaultParams.include_type_name = true;
-            Object.values(typeMappings).forEach((typeMapping) => {
-                if (typeMapping && typeMapping._all) {
-                    delete typeMapping._all;
-                }
-                return '';
-            });
+
+            if ((esVersion === 7 || esVersion === 1) && defaultParams.include_type_name) {
+                // move any metadata fields to _doc
+                const metadataFields = ['_index', '_id', '_source', '_size', '_doc_count', '_field_names', '_ignored', '_routing', '_meta', '_tier'];
+                metadataFields.forEach((f) => {
+                    if (mappings[f]) {
+                        mappings._doc[f] = { ...mappings._doc[f], ...mappings[f] };
+                        delete mappings[f];
+                    }
+                });
+            }
         }
-    }
 
-    if (isElasticsearch8(client) || isOpensearch(client)) {
-        delete defaultParams.include_type_name;
+        // _all deprecated in esV6, esV8 & osV2 seems to strip automatically but esV7/osV1 don't
+        if (esVersion === 7 || esVersion === 1) {
+            if (mappings.include_type_name) {
+                Object.values(mappings).forEach((typeMapping) => {
+                    if (typeMapping && typeMapping._all) {
+                        delete typeMapping._all;
+                    }
+                });
+            } else if (mappings._all) {
+                // _all might be at root mapping level if not include_type_name
+                delete mappings._all;
+            }
+        }
     }
 
     return Object.assign({}, defaultParams, params);