diff --git a/e2e/package.json b/e2e/package.json index 7e565003e0d..3c94678ee64 100644 --- a/e2e/package.json +++ b/e2e/package.json @@ -42,9 +42,9 @@ "ms": "^2.1.3" }, "devDependencies": { - "@terascope/types": "^0.15.0", + "@terascope/types": "^0.15.1", "bunyan": "^1.8.15", - "elasticsearch-store": "^0.81.1", + "elasticsearch-store": "^0.82.0", "fs-extra": "^11.2.0", "ms": "^2.1.3", "nanoid": "^3.3.4", diff --git a/packages/data-mate/package.json b/packages/data-mate/package.json index 833d8797271..154651cc5d7 100644 --- a/packages/data-mate/package.json +++ b/packages/data-mate/package.json @@ -1,7 +1,7 @@ { "name": "@terascope/data-mate", "displayName": "Data-Mate", - "version": "0.54.1", + "version": "0.54.2", "description": "Library of data validations/transformations", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/data-mate#readme", "repository": { @@ -29,9 +29,9 @@ "test:watch": "ts-scripts test --watch . --" }, "dependencies": { - "@terascope/data-types": "^0.48.0", - "@terascope/types": "^0.15.0", - "@terascope/utils": "^0.57.0", + "@terascope/data-types": "^0.48.1", + "@terascope/types": "^0.15.1", + "@terascope/utils": "^0.57.1", "@types/validator": "^13.11.9", "awesome-phonenumber": "^2.70.0", "date-fns": "^2.30.0", @@ -46,7 +46,7 @@ "uuid": "^9.0.1", "valid-url": "^1.0.9", "validator": "^13.11.0", - "xlucene-parser": "^0.56.1" + "xlucene-parser": "^0.56.2" }, "devDependencies": { "@types/ip6addr": "^0.2.6", diff --git a/packages/data-types/package.json b/packages/data-types/package.json index 0a392d7d359..957b2893654 100644 --- a/packages/data-types/package.json +++ b/packages/data-types/package.json @@ -1,7 +1,7 @@ { "name": "@terascope/data-types", "displayName": "Data Types", - "version": "0.48.0", + "version": "0.48.1", "description": "A library for defining the data structures and mapping", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/data-types#readme", "bugs": { @@ -26,8 +26,8 @@ "test:watch": "ts-scripts test --watch . --" }, "dependencies": { - "@terascope/types": "^0.15.0", - "@terascope/utils": "^0.57.0", + "@terascope/types": "^0.15.1", + "@terascope/utils": "^0.57.1", "graphql": "^14.7.0", "lodash": "^4.17.21", "yargs": "^17.7.2" diff --git a/packages/data-types/src/data-type.ts b/packages/data-types/src/data-type.ts index 09a15281d07..53a5d97347f 100644 --- a/packages/data-types/src/data-type.ts +++ b/packages/data-types/src/data-type.ts @@ -111,13 +111,14 @@ export class DataType { */ toESMapping({ typeName, overrides, distribution = ElasticsearchDistribution.elasticsearch, - majorVersion = 6, minorVersion = 8, version = '6.8.6' + majorVersion = 6, minorVersion = 8, version = '6.8.6', _meta }: Partial = {}): ESMapping { const indexType = typeName || this.name || '_doc'; const mappingSettings: ESTypeMappings = { dynamic: false, properties: {}, + ..._meta && { _meta } }; if ( diff --git a/packages/data-types/src/interfaces.ts b/packages/data-types/src/interfaces.ts index 130bad3dac8..a448927a985 100644 --- a/packages/data-types/src/interfaces.ts +++ b/packages/data-types/src/interfaces.ts @@ -106,4 +106,9 @@ export interface ESMappingOptions extends ClientMetadata { * uses a deep assignment so nested fields can be overwritten. */ overrides?: Partial; + + /** + * Any metadata to add to the index mapping + */ + _meta?: Record; } diff --git a/packages/data-types/test/data-type-es-spec.ts b/packages/data-types/test/data-type-es-spec.ts index ae64b2cc87e..ba255f8c25d 100644 --- a/packages/data-types/test/data-type-es-spec.ts +++ b/packages/data-types/test/data-type-es-spec.ts @@ -43,6 +43,7 @@ describe('DataType (elasticsearch)', () => { _all: { enabled: false, }, + _meta: { foo: 'foo' }, dynamic: false, properties: { hello: { type: 'text' }, @@ -62,7 +63,8 @@ describe('DataType (elasticsearch)', () => { distribution: ElasticsearchDistribution.elasticsearch, minorVersion: 8, majorVersion: 6, - version: '6.8.6' + version: '6.8.6', + _meta: { foo: 'foo' } }; expect(dataType.toESMapping(mappingConfig)).toEqual(results); @@ -84,6 +86,7 @@ describe('DataType (elasticsearch)', () => { foo: { type: 'keyword' }, bar: { type: 'keyword' }, }, + _meta: { foo: 'foo' }, }, settings: {}, }; @@ -93,7 +96,8 @@ describe('DataType (elasticsearch)', () => { distribution: ElasticsearchDistribution.elasticsearch, minorVersion: 3, majorVersion: 7, - version: '7.3.1' + version: '7.3.1', + _meta: { foo: 'foo' }, }; expect(dataType.toESMapping(mappingConfig)).toEqual(results); diff --git a/packages/elasticsearch-api/package.json b/packages/elasticsearch-api/package.json index 8a7b89f280e..717ae2d192b 100644 --- a/packages/elasticsearch-api/package.json +++ b/packages/elasticsearch-api/package.json @@ -1,7 +1,7 @@ { "name": "@terascope/elasticsearch-api", "displayName": "Elasticsearch API", - "version": "3.18.0", + "version": "3.18.1", "description": "Elasticsearch client api used across multiple services, handles retries and exponential backoff", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/elasticsearch-api#readme", "bugs": { @@ -23,8 +23,8 @@ "test:watch": "TEST_RESTRAINED_ELASTICSEARCH='true' ts-scripts test --watch . --" }, "dependencies": { - "@terascope/types": "^0.15.0", - "@terascope/utils": "^0.57.0", + "@terascope/types": "^0.15.1", + "@terascope/utils": "^0.57.1", "bluebird": "^3.7.2", "setimmediate": "^1.0.5" }, @@ -32,7 +32,7 @@ "@opensearch-project/opensearch": "^1.2.0", "@types/elasticsearch": "^5.0.43", "elasticsearch": "^15.4.1", - "elasticsearch-store": "^0.81.1", + "elasticsearch-store": "^0.82.0", "elasticsearch6": "npm:@elastic/elasticsearch@^6.7.0", "elasticsearch7": "npm:@elastic/elasticsearch@^7.0.0", "elasticsearch8": "npm:@elastic/elasticsearch@^8.0.0" diff --git a/packages/elasticsearch-store/package.json b/packages/elasticsearch-store/package.json index 662f341cebd..e46409250bc 100644 --- a/packages/elasticsearch-store/package.json +++ b/packages/elasticsearch-store/package.json @@ -1,7 +1,7 @@ { "name": "elasticsearch-store", "displayName": "Elasticsearch Store", - "version": "0.81.1", + "version": "0.82.0", "description": "An API for managing an elasticsearch index, with versioning and migration support.", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/elasticsearch-store#readme", "bugs": { @@ -29,10 +29,10 @@ "test:watch": "ts-scripts test --watch . --" }, "dependencies": { - "@terascope/data-mate": "^0.54.1", - "@terascope/data-types": "^0.48.0", - "@terascope/types": "^0.15.0", - "@terascope/utils": "^0.57.0", + "@terascope/data-mate": "^0.54.2", + "@terascope/data-types": "^0.48.1", + "@terascope/types": "^0.15.1", + "@terascope/utils": "^0.57.1", "ajv": "^6.12.6", "elasticsearch6": "npm:@elastic/elasticsearch@^6.7.0", "elasticsearch7": "npm:@elastic/elasticsearch@^7.0.0", @@ -41,7 +41,7 @@ "opensearch2": "npm:@opensearch-project/opensearch@^2.2.1", "setimmediate": "^1.0.5", "uuid": "^9.0.1", - "xlucene-translator": "^0.42.1" + "xlucene-translator": "^0.42.2" }, "devDependencies": { "@types/uuid": "^9.0.8" diff --git a/packages/elasticsearch-store/src/elasticsearch-client/method-helpers/helper-utils.ts b/packages/elasticsearch-store/src/elasticsearch-client/method-helpers/helper-utils.ts index 128a3bd9ee4..73a6621abb4 100644 --- a/packages/elasticsearch-store/src/elasticsearch-client/method-helpers/helper-utils.ts +++ b/packages/elasticsearch-store/src/elasticsearch-client/method-helpers/helper-utils.ts @@ -29,13 +29,17 @@ function validDistributionAndVersion( } export function ensureNoTypeInMapping(mappings: Record | undefined) { + const parsed: Record = {}; if (mappings != null) { for (const [k, v] of Object.entries(mappings)) { - if (k === 'properties') return { [k]: v }; + if (k === 'properties') parsed[k] = v; + if (k === '_meta') parsed[k] = v; - if (v.properties) return { properties: v.properties }; + if (v.properties) parsed.properties = v.properties; + if (v._meta) parsed._meta = v._meta; } } + return parsed; } export function ensureTypeInMapping(body: ESTypes.IndexTemplateProperties | undefined) { diff --git a/packages/elasticsearch-store/src/elasticsearch-client/method-helpers/indicesPutTemplate.ts b/packages/elasticsearch-store/src/elasticsearch-client/method-helpers/indicesPutTemplate.ts index d5c45d4244e..9910e8aa6f2 100644 --- a/packages/elasticsearch-store/src/elasticsearch-client/method-helpers/indicesPutTemplate.ts +++ b/packages/elasticsearch-store/src/elasticsearch-client/method-helpers/indicesPutTemplate.ts @@ -1,4 +1,5 @@ import { ElasticsearchDistribution, ClientParams, ClientMetadata } from '@terascope/types'; +import { get, isNumber } from '@terascope/utils'; import { ensureNoTypeInMapping, ensureTypeInMapping } from './helper-utils'; @@ -20,11 +21,14 @@ export function convertIndicesPutTemplateParams( ...parsedParams } = params; + const indexSchemaVersion = get(body, 'version'); + return { index_patterns: body?.index_patterns, aliases: body?.aliases, mappings: ensureNoTypeInMapping(body?.mappings), settings: body?.settings, + ...isNumber(indexSchemaVersion) && { version: indexSchemaVersion }, ...parsedParams }; } diff --git a/packages/elasticsearch-store/src/index-manager.ts b/packages/elasticsearch-store/src/index-manager.ts index fca99d85ff3..1cafb977ea8 100644 --- a/packages/elasticsearch-store/src/index-manager.ts +++ b/packages/elasticsearch-store/src/index-manager.ts @@ -1,5 +1,7 @@ import * as ts from '@terascope/utils'; -import { ClientParams, ClientResponse, ClientMetadata } from '@terascope/types'; +import { + ClientParams, ClientResponse, ClientMetadata, ESMapping +} from '@terascope/types'; import * as utils from './utils'; import { IndexConfig, MigrateIndexOptions } from './interfaces'; import { Client } from './elasticsearch-client'; @@ -103,12 +105,13 @@ export class IndexManager { logger.trace(`Using ${this.clientMetadata.distribution} version ${this.clientMetadata.version}`); - const body: any = config.data_type.toESMapping({ + const body = config.data_type.toESMapping({ typeName: config.name, overrides: { settings, }, - ...this.clientMetadata + ...this.clientMetadata, + ...config._meta && { _meta: config._meta } }); const enableMutations = ( @@ -384,13 +387,13 @@ export class IndexManager { * Safely create or update a template */ async upsertTemplate( - template: Record, logger?: ts.Logger + template: ESMapping, logger?: ts.Logger ): Promise { const { template: name, version } = template; try { - const templates = await this.getTemplate(name, true); - const latestVersion = templates[name].version; + const templates = await this.getTemplate(name || '', true); + const latestVersion = templates[name || ''].version; if (version === latestVersion) return; } catch (err) { if (err.statusCode !== 404) { diff --git a/packages/elasticsearch-store/src/interfaces.ts b/packages/elasticsearch-store/src/interfaces.ts index c39044d7291..a5ddba4d2b0 100644 --- a/packages/elasticsearch-store/src/interfaces.ts +++ b/packages/elasticsearch-store/src/interfaces.ts @@ -88,6 +88,11 @@ export interface IndexConfig { * The default query access to use */ default_query_access?: QueryAccess; + + /** + * Any metadata to add to the index mapping + */ + _meta?: Record; } /** Elasticsearch Index Schema, Mapping and Version */ diff --git a/packages/elasticsearch-store/src/utils/elasticsearch.ts b/packages/elasticsearch-store/src/utils/elasticsearch.ts index c763695f0da..356963068e8 100644 --- a/packages/elasticsearch-store/src/utils/elasticsearch.ts +++ b/packages/elasticsearch-store/src/utils/elasticsearch.ts @@ -1,6 +1,7 @@ import * as ts from '@terascope/utils'; import { - ESFieldType, ESTypeMapping, ClientMetadata, ElasticsearchDistribution + ESFieldType, ESTypeMapping, ClientMetadata, ElasticsearchDistribution, + ESMapping } from '@terascope/types'; import { Client } from '../elasticsearch-client'; import { getErrorType } from './errors'; @@ -174,7 +175,7 @@ export function isOpensearch2(client: Client): boolean { // TODO: move this logic over to datatype export function fixMappingRequest( - client: Client, _params: Record, isTemplate: boolean + client: Client, _params: { body: ESMapping, name?: string, index?: string }, isTemplate: boolean ): any { if (!_params || !_params.body) { throw new Error('Invalid mapping request'); @@ -190,24 +191,41 @@ export function fixMappingRequest( } delete params.body.template; } + // we do not support v5 anymore if (esVersion !== 6) { - const typeMappings: Record = ts.get(params.body, 'mappings', {}); - if (typeMappings.properties) { - defaultParams.include_type_name = false; - } else { + const mappings = params?.body?.mappings || {}; + if (!mappings.properties && mappings._doc) { + // esV8/osV2 seem to convert properly if mapping._doc.properties or mapping.properties + // but esV7/osV1 only seem to work w/include_type_name if properties is under "_doc" + // along w/metadata fields so set include_type_name if _doc & ensure metadata is in _doc defaultParams.include_type_name = true; - Object.values(typeMappings).forEach((typeMapping) => { - if (typeMapping && typeMapping._all) { - delete typeMapping._all; - } - return ''; - }); + + if ((esVersion === 7 || esVersion === 1) && defaultParams.include_type_name) { + // move any metadata fields to _doc + const metadataFields = ['_index', '_id', '_source', '_size', '_doc_count', '_field_names', '_ignored', '_routing', '_meta', '_tier']; + metadataFields.forEach((f) => { + if (mappings[f]) { + mappings._doc[f] = { ...mappings._doc[f], ...mappings[f] }; + delete mappings[f]; + } + }); + } } - } - if (isElasticsearch8(client) || isOpensearch(client)) { - delete defaultParams.include_type_name; + // _all deprecated in esV6, esV8 & osV2 seems to strip automatically but esV7/osV1 don't + if (esVersion === 7 || esVersion === 1) { + if (mappings.include_type_name) { + Object.values(mappings).forEach((typeMapping) => { + if (typeMapping && typeMapping._all) { + delete typeMapping._all; + } + }); + } else if (mappings._all) { + // _all might be at root mapping level if not include_type_name + delete mappings._all; + } + } } return Object.assign({}, defaultParams, params); diff --git a/packages/elasticsearch-store/test/index-manager-index-setup-spec.ts b/packages/elasticsearch-store/test/index-manager-index-setup-spec.ts index 70832a81dd1..c727345d276 100644 --- a/packages/elasticsearch-store/test/index-manager-index-setup-spec.ts +++ b/packages/elasticsearch-store/test/index-manager-index-setup-spec.ts @@ -31,6 +31,7 @@ describe('IndexManager->indexSetup()', () => { 'index.number_of_replicas': 0, }, logger, + _meta: { foo: 'foo' } }; const index = `${config.name}-v1-s1`; @@ -69,6 +70,9 @@ describe('IndexManager->indexSetup()', () => { expect(mapping).toHaveProperty(index); if (esVersion === 6) { expect(mapping[index].mappings).toHaveProperty(config.name); + expect(mapping[index].mappings[config.name]).toHaveProperty('_meta', { foo: 'foo' }); + } else { + expect(mapping[index].mappings).toHaveProperty('_meta', { foo: 'foo' }); } }); @@ -110,6 +114,11 @@ describe('IndexManager->indexSetup()', () => { } }, }); + if (esVersion === 6) { + expect(mapping[index].mappings[config.name]).toHaveProperty('_meta', { foo: 'foo' }); + } else { + expect(mapping[index].mappings).toHaveProperty('_meta', { foo: 'foo' }); + } }); describe('when making a breaking change to the data type', () => { @@ -153,6 +162,11 @@ describe('IndexManager->indexSetup()', () => { } }, }); + if (esVersion === 6) { + expect(mapping[index].mappings[config.name]).toHaveProperty('_meta', { foo: 'foo' }); + } else { + expect(mapping[index].mappings).toHaveProperty('_meta', { foo: 'foo' }); + } }); }); }); @@ -173,6 +187,7 @@ describe('IndexManager->indexSetup()', () => { 'index.number_of_replicas': 0, }, logger, + _meta: { bar: 'bar' } }; const index = `${config.name}-v1-s1`; @@ -214,6 +229,9 @@ describe('IndexManager->indexSetup()', () => { expect(mapping).toHaveProperty(index); if (esVersion === 6) { expect(mapping[index].mappings).toHaveProperty(config.name); + expect(mapping[index].mappings[config.name]).toHaveProperty('_meta', { bar: 'bar' }); + } else { + expect(mapping[index].mappings).toHaveProperty('_meta', { bar: 'bar' }); } }); @@ -225,6 +243,11 @@ describe('IndexManager->indexSetup()', () => { expect(temp[templateName].mappings).toHaveProperty(config.name); } expect(temp[templateName]).toHaveProperty('version', 1); + if (esVersion === 6) { + expect(temp[templateName].mappings?.[config.name]).toHaveProperty('_meta', { bar: 'bar' }); + } else { + expect(temp[templateName].mappings).toHaveProperty('_meta', { bar: 'bar' }); + } }); it('should be able upsert the same template safely', async () => { @@ -246,6 +269,11 @@ describe('IndexManager->indexSetup()', () => { expect(temp).toHaveProperty(templateName); expect(temp[templateName]).toHaveProperty('version', version); + if (esVersion === 6) { + expect(temp[templateName].mappings?.[config.name]).toHaveProperty('_meta', { bar: 'bar' }); + } else { + expect(temp[templateName].mappings).toHaveProperty('_meta', { bar: 'bar' }); + } }); it('should be able to upsert a newer template safely', async () => { @@ -270,6 +298,33 @@ describe('IndexManager->indexSetup()', () => { expect(temp[templateName]).toHaveProperty('version', newVersion); }); + it('should apply _meta to new indices', async () => { + const mapping = get(config, ['index_schema', 'mapping'], {}); + const version = get(config, ['index_schema', 'version'], 1); + mapping._meta = { baz: 'baz' }; + + const mappings = esVersion !== 6 ? mapping : { + [config.name]: mapping + }; + await indexManager.upsertTemplate({ + template: 'foo', + settings: config.index_settings, + index_patterns: ['foo*'], + mappings, + version + }); + + await indexManager.client.indices.create({ index: 'foobar' }); + + const newIdxMapping = await indexManager.getMapping('foobar'); + + if (esVersion === 6) { + expect(newIdxMapping.foobar.mappings[config.name]).toHaveProperty('_meta', { baz: 'baz' }); + } else { + expect(newIdxMapping.foobar.mappings).toHaveProperty('_meta', { baz: 'baz' }); + } + }); + it('should be able to call create again', async () => { const created = await indexManager.indexSetup(config); expect(created).toBeFalse(); diff --git a/packages/elasticsearch-store/test/index-search-compatability-spec.ts b/packages/elasticsearch-store/test/index-search-compatability-spec.ts index 2f3f37bebeb..c0d68198159 100644 --- a/packages/elasticsearch-store/test/index-search-compatability-spec.ts +++ b/packages/elasticsearch-store/test/index-search-compatability-spec.ts @@ -26,7 +26,7 @@ describe('IndexSearchCompatability', () => { } const dataType = makeRecordDataType({ - name: 'SearchCompatability', + name: 'SearchCompatibility', fields: { id: { type: FieldType.Keyword }, foo: { type: FieldType.Keyword }, diff --git a/packages/generator-teraslice/package.json b/packages/generator-teraslice/package.json index 0721e60d436..3e6c7cc7221 100644 --- a/packages/generator-teraslice/package.json +++ b/packages/generator-teraslice/package.json @@ -1,7 +1,7 @@ { "name": "generator-teraslice", "displayName": "Generator Teraslice", - "version": "0.36.0", + "version": "0.36.1", "description": "Generate teraslice related packages and code", "keywords": [ "teraslice", @@ -24,7 +24,7 @@ "test:watch": "ts-scripts test --watch . --" }, "dependencies": { - "@terascope/utils": "^0.57.0", + "@terascope/utils": "^0.57.1", "chalk": "^4.1.2", "lodash": "^4.17.21", "yeoman-generator": "^5.8.0", diff --git a/packages/job-components/package.json b/packages/job-components/package.json index 724069adcaa..624f1bf4c6a 100644 --- a/packages/job-components/package.json +++ b/packages/job-components/package.json @@ -1,7 +1,7 @@ { "name": "@terascope/job-components", "displayName": "Job Components", - "version": "0.71.0", + "version": "0.71.1", "description": "A teraslice library for validating jobs schemas, registering apis, and defining and running new Job APIs", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/job-components#readme", "bugs": { @@ -31,7 +31,7 @@ "test:watch": "ts-scripts test --watch . --" }, "dependencies": { - "@terascope/utils": "^0.57.0", + "@terascope/utils": "^0.57.1", "convict": "^6.2.4", "convict-format-with-moment": "^6.2.0", "convict-format-with-validator": "^6.2.0", diff --git a/packages/scripts/package.json b/packages/scripts/package.json index 9b2588f3d53..3207af7411c 100644 --- a/packages/scripts/package.json +++ b/packages/scripts/package.json @@ -1,7 +1,7 @@ { "name": "@terascope/scripts", "displayName": "Scripts", - "version": "0.75.1", + "version": "0.75.2", "description": "A collection of terascope monorepo scripts", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/scripts#readme", "bugs": { @@ -32,7 +32,7 @@ }, "dependencies": { "@kubernetes/client-node": "^0.20.0", - "@terascope/utils": "^0.57.0", + "@terascope/utils": "^0.57.1", "codecov": "^3.8.3", "execa": "^5.1.0", "fs-extra": "^11.2.0", diff --git a/packages/terafoundation/package.json b/packages/terafoundation/package.json index 2f09ca58c03..717f6fec154 100644 --- a/packages/terafoundation/package.json +++ b/packages/terafoundation/package.json @@ -1,7 +1,7 @@ { "name": "terafoundation", "displayName": "Terafoundation", - "version": "0.60.0", + "version": "0.61.0", "description": "A Clustering and Foundation tool for Terascope Tools", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/terafoundation#readme", "bugs": { @@ -28,15 +28,15 @@ }, "dependencies": { "@terascope/file-asset-apis": "^0.13.0", - "@terascope/types": "^0.15.0", - "@terascope/utils": "^0.57.0", + "@terascope/types": "^0.15.1", + "@terascope/utils": "^0.57.1", "bluebird": "^3.7.2", "bunyan": "^1.8.15", "convict": "^6.2.4", "convict-format-with-moment": "^6.2.0", "convict-format-with-validator": "^6.2.0", "elasticsearch": "^15.4.1", - "elasticsearch-store": "^0.81.1", + "elasticsearch-store": "^0.82.0", "js-yaml": "^4.1.0", "nanoid": "^3.3.4", "node-webhdfs": "^1.0.2", diff --git a/packages/teraslice-cli/package.json b/packages/teraslice-cli/package.json index c610fb940c0..46bd9d4dd62 100644 --- a/packages/teraslice-cli/package.json +++ b/packages/teraslice-cli/package.json @@ -1,7 +1,7 @@ { "name": "teraslice-cli", "displayName": "Teraslice CLI", - "version": "1.2.0", + "version": "1.2.1", "description": "Command line manager for teraslice jobs, assets, and cluster references.", "keywords": [ "teraslice" @@ -38,8 +38,8 @@ }, "dependencies": { "@terascope/fetch-github-release": "^0.8.7", - "@terascope/types": "^0.15.0", - "@terascope/utils": "^0.57.0", + "@terascope/types": "^0.15.1", + "@terascope/utils": "^0.57.1", "chalk": "^4.1.2", "cli-table3": "^0.6.3", "easy-table": "^1.2.0", @@ -54,7 +54,7 @@ "pretty-bytes": "^5.6.0", "prompts": "^2.4.2", "signale": "^1.4.0", - "teraslice-client-js": "^0.58.0", + "teraslice-client-js": "^0.58.1", "tmp": "^0.2.0", "tty-table": "^4.2.3", "yargs": "^17.7.2", diff --git a/packages/teraslice-client-js/package.json b/packages/teraslice-client-js/package.json index a85d4886469..e91549ab538 100644 --- a/packages/teraslice-client-js/package.json +++ b/packages/teraslice-client-js/package.json @@ -1,7 +1,7 @@ { "name": "teraslice-client-js", "displayName": "Teraslice Client (JavaScript)", - "version": "0.58.0", + "version": "0.58.1", "description": "A Node.js client for teraslice jobs, assets, and cluster references.", "keywords": [ "elasticsearch", @@ -31,8 +31,8 @@ "test:watch": "ts-scripts test --watch . --" }, "dependencies": { - "@terascope/types": "^0.15.0", - "@terascope/utils": "^0.57.0", + "@terascope/types": "^0.15.1", + "@terascope/utils": "^0.57.1", "auto-bind": "^4.0.0", "got": "^11.8.3" }, diff --git a/packages/teraslice-messaging/package.json b/packages/teraslice-messaging/package.json index a46b48291c7..1703250c86c 100644 --- a/packages/teraslice-messaging/package.json +++ b/packages/teraslice-messaging/package.json @@ -1,7 +1,7 @@ { "name": "@terascope/teraslice-messaging", "displayName": "Teraslice Messaging", - "version": "0.40.0", + "version": "0.40.1", "description": "An internal teraslice messaging library using socket.io", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/teraslice-messaging#readme", "bugs": { @@ -34,8 +34,8 @@ "ms": "^2.1.3" }, "dependencies": { - "@terascope/types": "^0.15.0", - "@terascope/utils": "^0.57.0", + "@terascope/types": "^0.15.1", + "@terascope/utils": "^0.57.1", "ms": "^2.1.3", "nanoid": "^3.3.4", "p-event": "^4.2.0", diff --git a/packages/teraslice-op-test-harness/package.json b/packages/teraslice-op-test-harness/package.json index a5834c646da..02f3927bb83 100644 --- a/packages/teraslice-op-test-harness/package.json +++ b/packages/teraslice-op-test-harness/package.json @@ -21,10 +21,10 @@ "bluebird": "^3.7.2" }, "devDependencies": { - "@terascope/job-components": "^0.71.0" + "@terascope/job-components": "^0.71.1" }, "peerDependencies": { - "@terascope/job-components": ">=0.71.0" + "@terascope/job-components": ">=0.71.1" }, "engines": { "node": ">=14.17.0", diff --git a/packages/teraslice-state-storage/package.json b/packages/teraslice-state-storage/package.json index 96b94be9de6..08c18517a1f 100644 --- a/packages/teraslice-state-storage/package.json +++ b/packages/teraslice-state-storage/package.json @@ -1,7 +1,7 @@ { "name": "@terascope/teraslice-state-storage", "displayName": "Teraslice State Storage", - "version": "0.51.0", + "version": "0.51.1", "description": "State storage operation api for teraslice", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/teraslice-state-storage#readme", "bugs": { @@ -23,8 +23,8 @@ "test:watch": "ts-scripts test --watch . --" }, "dependencies": { - "@terascope/elasticsearch-api": "^3.18.0", - "@terascope/utils": "^0.57.0" + "@terascope/elasticsearch-api": "^3.18.1", + "@terascope/utils": "^0.57.1" }, "engines": { "node": ">=14.17.0", diff --git a/packages/teraslice-test-harness/package.json b/packages/teraslice-test-harness/package.json index 910cb16de3e..1a04e34bfd1 100644 --- a/packages/teraslice-test-harness/package.json +++ b/packages/teraslice-test-harness/package.json @@ -36,10 +36,10 @@ "fs-extra": "^11.2.0" }, "devDependencies": { - "@terascope/job-components": "^0.71.0" + "@terascope/job-components": "^0.71.1" }, "peerDependencies": { - "@terascope/job-components": ">=0.71.0" + "@terascope/job-components": ">=0.71.1" }, "engines": { "node": ">=14.17.0", diff --git a/packages/teraslice/package.json b/packages/teraslice/package.json index 1337394db9e..4d07ad53c60 100644 --- a/packages/teraslice/package.json +++ b/packages/teraslice/package.json @@ -38,11 +38,11 @@ "ms": "^2.1.3" }, "dependencies": { - "@terascope/elasticsearch-api": "^3.18.0", - "@terascope/job-components": "^0.71.0", - "@terascope/teraslice-messaging": "^0.40.0", - "@terascope/types": "^0.15.0", - "@terascope/utils": "^0.57.0", + "@terascope/elasticsearch-api": "^3.18.1", + "@terascope/job-components": "^0.71.1", + "@terascope/teraslice-messaging": "^0.40.1", + "@terascope/types": "^0.15.1", + "@terascope/utils": "^0.57.1", "async-mutex": "^0.4.0", "barbe": "^3.0.16", "body-parser": "^1.20.2", @@ -64,7 +64,7 @@ "semver": "^7.6.0", "socket.io": "^1.7.4", "socket.io-client": "^1.7.4", - "terafoundation": "^0.60.0", + "terafoundation": "^0.61.0", "uuid": "^9.0.1" }, "devDependencies": { diff --git a/packages/ts-transforms/package.json b/packages/ts-transforms/package.json index bf2a2c74c07..546f2e1dcc8 100644 --- a/packages/ts-transforms/package.json +++ b/packages/ts-transforms/package.json @@ -1,7 +1,7 @@ { "name": "ts-transforms", "displayName": "TS Transforms", - "version": "0.83.1", + "version": "0.83.2", "description": "An ETL framework built upon xlucene-evaluator", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/ts-transforms#readme", "bugs": { @@ -35,9 +35,9 @@ "test:watch": "ts-scripts test --watch . --" }, "dependencies": { - "@terascope/data-mate": "^0.54.1", - "@terascope/types": "^0.15.0", - "@terascope/utils": "^0.57.0", + "@terascope/data-mate": "^0.54.2", + "@terascope/types": "^0.15.1", + "@terascope/utils": "^0.57.1", "awesome-phonenumber": "^2.70.0", "graphlib": "^2.1.8", "is-ip": "^3.1.0", diff --git a/packages/types/package.json b/packages/types/package.json index e9355439d77..da2c18b4970 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -1,7 +1,7 @@ { "name": "@terascope/types", "displayName": "Types", - "version": "0.15.0", + "version": "0.15.1", "description": "A collection of typescript interfaces", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/types#readme", "bugs": { diff --git a/packages/types/src/elasticsearch-interfaces.ts b/packages/types/src/elasticsearch-interfaces.ts index 76bcbf1f2b9..b2c72f1cc87 100644 --- a/packages/types/src/elasticsearch-interfaces.ts +++ b/packages/types/src/elasticsearch-interfaces.ts @@ -219,6 +219,7 @@ export interface ESTypeMappings { enabled?: boolean; [key: string]: any; }; + _meta?: Record; dynamic?: boolean; properties: { [key: string]: ESTypeMapping; @@ -233,7 +234,8 @@ export interface ESMapping { order?: number; aliases?: any; index_patterns?: string[]; - settings: ESIndexSettings; + settings?: ESIndexSettings; + version?: number; } export interface ESIndexSettings { diff --git a/packages/utils/package.json b/packages/utils/package.json index 3f5deaf6ac9..697c3d71137 100644 --- a/packages/utils/package.json +++ b/packages/utils/package.json @@ -1,7 +1,7 @@ { "name": "@terascope/utils", "displayName": "Utils", - "version": "0.57.0", + "version": "0.57.1", "description": "A collection of Teraslice Utilities", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/utils#readme", "bugs": { @@ -28,7 +28,7 @@ "debug": "^4.3.4" }, "dependencies": { - "@terascope/types": "^0.15.0", + "@terascope/types": "^0.15.1", "@turf/bbox": "^6.4.0", "@turf/bbox-polygon": "^6.4.0", "@turf/boolean-contains": "^6.4.0", diff --git a/packages/xlucene-parser/package.json b/packages/xlucene-parser/package.json index 1e6b94ad02a..49cf67efe4f 100644 --- a/packages/xlucene-parser/package.json +++ b/packages/xlucene-parser/package.json @@ -1,7 +1,7 @@ { "name": "xlucene-parser", "displayName": "xLucene Parser", - "version": "0.56.1", + "version": "0.56.2", "description": "Flexible Lucene-like evaluator and language parser", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/xlucene-parser#readme", "repository": { @@ -30,8 +30,8 @@ "test:watch": "ts-scripts test --watch . --" }, "dependencies": { - "@terascope/types": "^0.15.0", - "@terascope/utils": "^0.57.0" + "@terascope/types": "^0.15.1", + "@terascope/utils": "^0.57.1" }, "devDependencies": { "@turf/invariant": "^6.2.0", diff --git a/packages/xlucene-translator/package.json b/packages/xlucene-translator/package.json index 3d29ab70b4f..e4f280711bc 100644 --- a/packages/xlucene-translator/package.json +++ b/packages/xlucene-translator/package.json @@ -1,7 +1,7 @@ { "name": "xlucene-translator", "displayName": "xLucene Translator", - "version": "0.42.1", + "version": "0.42.2", "description": "Translate xlucene query to database queries", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/xlucene-translator#readme", "repository": { @@ -28,10 +28,10 @@ "test:watch": "ts-scripts test --watch . --" }, "dependencies": { - "@terascope/types": "^0.15.0", - "@terascope/utils": "^0.57.0", + "@terascope/types": "^0.15.1", + "@terascope/utils": "^0.57.1", "@types/elasticsearch": "^5.0.43", - "xlucene-parser": "^0.56.1" + "xlucene-parser": "^0.56.2" }, "devDependencies": { "elasticsearch": "^15.4.1" diff --git a/packages/xpressions/package.json b/packages/xpressions/package.json index 669359de990..305c49ffe56 100644 --- a/packages/xpressions/package.json +++ b/packages/xpressions/package.json @@ -1,7 +1,7 @@ { "name": "xpressions", "displayName": "Xpressions", - "version": "0.24.0", + "version": "0.24.1", "description": "Variable expressions with date-math support", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/xpressions#readme", "bugs": { @@ -23,10 +23,10 @@ "test:watch": "ts-scripts test --watch . --" }, "dependencies": { - "@terascope/utils": "^0.57.0" + "@terascope/utils": "^0.57.1" }, "devDependencies": { - "@terascope/types": "^0.15.0" + "@terascope/types": "^0.15.1" }, "engines": { "node": ">=14.17.0",