diff --git a/changelogs/fragments/6613.yml b/changelogs/fragments/6613.yml new file mode 100644 index 000000000000..49cd01f52fdd --- /dev/null +++ b/changelogs/fragments/6613.yml @@ -0,0 +1,2 @@ +feat: +- Support language selector from the data plugin ([#6613](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/6613)) \ No newline at end of file diff --git a/config/opensearch_dashboards.yml b/config/opensearch_dashboards.yml index 0844db34c36c..8caf4ebf6af9 100644 --- a/config/opensearch_dashboards.yml +++ b/config/opensearch_dashboards.yml @@ -331,3 +331,6 @@ # Set the value to true to enable Ui Metric Collectors in Usage Collector # This publishes the Application Usage and UI Metrics into the saved object, which can be accessed by /api/stats?extended=true&legacy=true&exclude_usage=false # usageCollection.uiMetric.enabled: false + +# Set the value to true to enable enhancements for the data plugin +# data.enhancements.enabled: false diff --git a/package.json b/package.json index 2d9a7451af21..e55a565294dd 100644 --- a/package.json +++ b/package.json @@ -154,6 +154,7 @@ "@hapi/vision": "^6.1.0", "@hapi/wreck": "^17.1.0", "@opensearch-project/opensearch": "^2.6.0", + "@opensearch/datemath": "5.0.3", "@osd/ace": "1.0.0", "@osd/analytics": "1.0.0", "@osd/apm-config-loader": "1.0.0", diff --git a/packages/opensearch-datemath/index.d.ts b/packages/opensearch-datemath/index.d.ts index d5a38f0176ea..0706d7d0dccf 100644 --- a/packages/opensearch-datemath/index.d.ts +++ b/packages/opensearch-datemath/index.d.ts @@ -43,6 +43,8 @@ declare const datemath: { unitsAsc: Unit[]; unitsDesc: Unit[]; + isDateTime(input: any): boolean; + /** * Parses a string into a moment object. The string can be something like "now - 15m". * @param options.forceNow If this optional parameter is supplied, "now" will be treated as this diff --git a/packages/opensearch-datemath/index.js b/packages/opensearch-datemath/index.js index 4367949d7cf0..ecbedf482922 100644 --- a/packages/opensearch-datemath/index.js +++ b/packages/opensearch-datemath/index.js @@ -49,6 +49,7 @@ const isDate = (d) => Object.prototype.toString.call(d) === '[object Date]'; const isValidDate = (d) => isDate(d) && !isNaN(d.valueOf()); +const isDateTime = (d, momentInstance = moment) => momentInstance.isMoment(d); /* * This is a simplified version of opensearch's date parser. * If you pass in a momentjs instance as the third parameter the calculation @@ -57,7 +58,7 @@ const isValidDate = (d) => isDate(d) && !isNaN(d.valueOf()); */ function parse(text, { roundUp = false, momentInstance = moment, forceNow } = {}) { if (!text) return undefined; - if (momentInstance.isMoment(text)) return text; + if (isDateTime(text, momentInstance)) return text; if (isDate(text)) return momentInstance(text); if (forceNow !== undefined && !isValidDate(forceNow)) { throw new Error('forceNow must be a valid Date'); @@ -164,6 +165,7 @@ function parseDateMath(mathString, time, roundUp) { module.exports = { parse: parse, + isDateTime: isDateTime, unitsMap: Object.freeze(unitsMap), units: Object.freeze(units), unitsAsc: Object.freeze(unitsAsc), diff --git a/packages/opensearch-datemath/index.test.ts b/packages/opensearch-datemath/index.test.ts index e293da72ac7f..fbd1973fa932 100644 --- a/packages/opensearch-datemath/index.test.ts +++ b/packages/opensearch-datemath/index.test.ts @@ -122,19 +122,19 @@ describe('dateMath', function () { }); it('should return a moment if passed a date', function () { - expect(dateMath.parse(date).format(format)).to.eql(mmnt.format(format)); + expect(dateMath.parse(date)!.format(format)).to.eql(mmnt.format(format)); }); it('should return a moment if passed an ISO8601 string', function () { - expect(dateMath.parse(string).format(format)).to.eql(mmnt.format(format)); + expect(dateMath.parse(string)!.format(format)).to.eql(mmnt.format(format)); }); it('should return the current time when parsing now', function () { - expect(dateMath.parse('now').format(format)).to.eql(now.format(format)); + expect(dateMath.parse('now')!.format(format)).to.eql(now.format(format)); }); it('should use the forceNow parameter when parsing now', function () { - expect(dateMath.parse('now', { forceNow: anchoredDate }).valueOf()).to.eql(unix); + expect(dateMath.parse('now', { forceNow: anchoredDate })!.valueOf()).to.eql(unix); }); }); @@ -158,17 +158,17 @@ describe('dateMath', function () { const thenEx = `${anchor}||-${len}${span}`; it('should return ' + len + span + ' ago', function () { - const parsed = dateMath.parse(nowEx).format(format); + const parsed = dateMath.parse(nowEx)!.format(format); expect(parsed).to.eql(now.subtract(len, span).format(format)); }); it('should return ' + len + span + ' before ' + anchor, function () { - const parsed = dateMath.parse(thenEx).format(format); + const parsed = dateMath.parse(thenEx)!.format(format); expect(parsed).to.eql(anchored.subtract(len, span).format(format)); }); it('should return ' + len + span + ' before forceNow', function () { - const parsed = dateMath.parse(nowEx, { forceNow: anchoredDate }).valueOf(); + const parsed = dateMath.parse(nowEx, { forceNow: anchoredDate })!.valueOf(); expect(parsed).to.eql(anchored.subtract(len, span).valueOf()); }); }); @@ -195,17 +195,17 @@ describe('dateMath', function () { const thenEx = `${anchor}||+${len}${span}`; it('should return ' + len + span + ' from now', function () { - expect(dateMath.parse(nowEx).format(format)).to.eql(now.add(len, span).format(format)); + expect(dateMath.parse(nowEx)!.format(format)).to.eql(now.add(len, span).format(format)); }); it('should return ' + len + span + ' after ' + anchor, function () { - expect(dateMath.parse(thenEx).format(format)).to.eql( + expect(dateMath.parse(thenEx)!.format(format)).to.eql( anchored.add(len, span).format(format) ); }); it('should return ' + len + span + ' after forceNow', function () { - expect(dateMath.parse(nowEx, { forceNow: anchoredDate }).valueOf()).to.eql( + expect(dateMath.parse(nowEx, { forceNow: anchoredDate })!.valueOf()).to.eql( anchored.add(len, span).valueOf() ); }); @@ -229,26 +229,26 @@ describe('dateMath', function () { spans.forEach((span) => { it(`should round now to the beginning of the ${span}`, function () { - expect(dateMath.parse('now/' + span).format(format)).to.eql( + expect(dateMath.parse('now/' + span)!.format(format)).to.eql( now.startOf(span).format(format) ); }); it(`should round now to the beginning of forceNow's ${span}`, function () { - expect(dateMath.parse('now/' + span, { forceNow: anchoredDate }).valueOf()).to.eql( + expect(dateMath.parse('now/' + span, { forceNow: anchoredDate })!.valueOf()).to.eql( anchored.startOf(span).valueOf() ); }); it(`should round now to the end of the ${span}`, function () { - expect(dateMath.parse('now/' + span, { roundUp: true }).format(format)).to.eql( + expect(dateMath.parse('now/' + span, { roundUp: true })!.format(format)).to.eql( now.endOf(span).format(format) ); }); it(`should round now to the end of forceNow's ${span}`, function () { expect( - dateMath.parse('now/' + span, { roundUp: true, forceNow: anchoredDate }).valueOf() + dateMath.parse('now/' + span, { roundUp: true, forceNow: anchoredDate })!.valueOf() ).to.eql(anchored.endOf(span).valueOf()); }); }); @@ -269,28 +269,28 @@ describe('dateMath', function () { }); it('should round to the nearest second with 0 value', function () { - const val = dateMath.parse('now-0s/s').format(format); + const val = dateMath.parse('now-0s/s')!.format(format); expect(val).to.eql(now.startOf('s').format(format)); }); it('should subtract 17s, rounded to the nearest second', function () { - const val = dateMath.parse('now-17s/s').format(format); + const val = dateMath.parse('now-17s/s')!.format(format); expect(val).to.eql(now.startOf('s').subtract(17, 's').format(format)); }); it('should add 555ms, rounded to the nearest millisecond', function () { - const val = dateMath.parse('now+555ms/ms').format(format); + const val = dateMath.parse('now+555ms/ms')!.format(format); expect(val).to.eql(now.add(555, 'ms').startOf('ms').format(format)); }); it('should subtract 555ms, rounded to the nearest second', function () { - const val = dateMath.parse('now-555ms/s').format(format); + const val = dateMath.parse('now-555ms/s')!.format(format); expect(val).to.eql(now.subtract(555, 'ms').startOf('s').format(format)); }); it('should round weeks to Sunday by default', function () { const val = dateMath.parse('now-1w/w'); - expect(val.isoWeekday()).to.eql(7); + expect(val!.isoWeekday()).to.eql(7); }); it('should round weeks based on the passed moment locale start of week setting', function () { @@ -300,7 +300,7 @@ describe('dateMath', function () { week: { dow: 2 }, }); const val = dateMath.parse('now-1w/w', { momentInstance: m }); - expect(val.isoWeekday()).to.eql(2); + expect(val!.isoWeekday()).to.eql(2); }); it('should round up weeks based on the passed moment locale start of week setting', function () { @@ -315,11 +315,11 @@ describe('dateMath', function () { }); // The end of the range (rounding up) should be the last day of the week (so one day before) // our start of the week, that's why 3 - 1 - expect(val.isoWeekday()).to.eql(3 - 1); + expect(val!.isoWeekday()).to.eql(3 - 1); }); it('should round relative to forceNow', function () { - const val = dateMath.parse('now-0s/s', { forceNow: anchoredDate }).valueOf(); + const val = dateMath.parse('now-0s/s', { forceNow: anchoredDate })!.valueOf(); expect(val).to.eql(anchored.startOf('s').valueOf()); }); diff --git a/packages/osd-opensearch/src/cli_commands/snapshot.js b/packages/osd-opensearch/src/cli_commands/snapshot.js index ff21dbe851c8..84d6acee104e 100644 --- a/packages/osd-opensearch/src/cli_commands/snapshot.js +++ b/packages/osd-opensearch/src/cli_commands/snapshot.js @@ -50,6 +50,7 @@ exports.help = (defaults = {}) => { --download-only Download the snapshot but don't actually start it --ssl Sets up SSL on OpenSearch --security Installs and sets up the OpenSearch Security plugin on the cluster + --sql Installs and sets up the required OpenSearch SQL/PPL plugins on the cluster --P OpenSearch plugin artifact URL to install it on the cluster. We can use the flag multiple times to install multiple plugins on the cluster snapshot. The argument value can be url to zip file, maven coordinates of the plugin or for local zip files, use file:. @@ -77,6 +78,8 @@ exports.run = async (defaults = {}) => { boolean: ['security'], + boolean: ['sql'], + default: defaults, }); @@ -98,6 +101,10 @@ exports.run = async (defaults = {}) => { await cluster.setupSecurity(installPath, options.version ?? defaults.version); } + if (options.sql) { + await cluster.setupSql(installPath, options.version ?? defaults.version); + } + options.bundledJDK = true; await cluster.run(installPath, options); diff --git a/packages/osd-opensearch/src/cluster.js b/packages/osd-opensearch/src/cluster.js index 455a1e5f919f..4b1c8b38259d 100644 --- a/packages/osd-opensearch/src/cluster.js +++ b/packages/osd-opensearch/src/cluster.js @@ -70,10 +70,11 @@ const first = (stream, map) => }); exports.Cluster = class Cluster { - constructor({ log = defaultLog, ssl = false, security = false } = {}) { + constructor({ log = defaultLog, ssl = false, security = false, sql = false } = {}) { this._log = log; this._ssl = ssl; this._security = security; + this._sql = sql; this._caCertPromise = ssl ? readFile(CA_CERT_PATH) : undefined; } @@ -224,6 +225,28 @@ exports.Cluster = class Cluster { } } + /** + * Setups cluster with SQL/PPL plugins + * + * @param {string} installPath + * @property {String} version - version of OpenSearch + */ + async setupSql(installPath, version) { + await this.installSqlPlugin(installPath, version, 'opensearch-sql'); + await this.installSqlPlugin(installPath, version, 'opensearch-observability'); + } + + async installSqlPlugin(installPath, version, id) { + this._log.info(`Setting up: ${id}`); + try { + const pluginUrl = generateEnginePluginUrl(version, id); + await this.installOpenSearchPlugins(installPath, pluginUrl); + this._log.info(`Completed setup: ${id}`); + } catch (ex) { + this._log.warning(`Failed to setup: ${id}`); + } + } + /** * Starts OpenSearch and returns resolved promise once started * diff --git a/src/cli/serve/serve.js b/src/cli/serve/serve.js index aed5d74a2c01..c1e489afb4ef 100644 --- a/src/cli/serve/serve.js +++ b/src/cli/serve/serve.js @@ -249,6 +249,7 @@ export default function (program) { .option('--dev', 'Run the server with development mode defaults') .option('--ssl', 'Run the dev server using HTTPS') .option('--security', 'Run the dev server using security defaults') + .option('--sql', 'Run the dev server using SQL/PPL defaults') .option('--dist', 'Use production assets from osd/optimizer') .option( '--no-base-path', diff --git a/src/plugins/dashboard/public/application/components/dashboard_listing/__snapshots__/dashboard_listing.test.tsx.snap b/src/plugins/dashboard/public/application/components/dashboard_listing/__snapshots__/dashboard_listing.test.tsx.snap index db34c4f229bb..4be28c0c4d08 100644 --- a/src/plugins/dashboard/public/application/components/dashboard_listing/__snapshots__/dashboard_listing.test.tsx.snap +++ b/src/plugins/dashboard/public/application/components/dashboard_listing/__snapshots__/dashboard_listing.test.tsx.snap @@ -276,6 +276,24 @@ exports[`dashboard listing hideWriteControls 1`] = ` "getValueSuggestions": [MockFunction], "hasQuerySuggestions": [MockFunction], }, + "dataSources": Object { + "dataSourceFactory": DataSourceFactory { + "dataSourceClasses": Object {}, + }, + "dataSourceService": DataSourceService { + "dataSourceFetchers": Object {}, + "dataSources": Object {}, + "dataSourcesSubject": BehaviorSubject { + "_isScalar": false, + "_value": Object {}, + "closed": false, + "hasError": false, + "isStopped": false, + "observers": Array [], + "thrownError": null, + }, + }, + }, "fieldFormats": Object { "deserialize": [MockFunction], "getByFieldType": [MockFunction], @@ -457,6 +475,7 @@ exports[`dashboard listing hideWriteControls 1`] = ` }, }, "search": Object { + "__enhance": [MockFunction], "aggs": Object { "calculateAutoTimeExpression": [Function], "createAggConfigs": [MockFunction], @@ -465,6 +484,12 @@ exports[`dashboard listing hideWriteControls 1`] = ` "getAll": [Function], }, }, + "df": Object { + "clear": [MockFunction], + "get": [MockFunction], + "set": [MockFunction], + }, + "getDefaultSearchInterceptor": [MockFunction], "search": [MockFunction], "searchSource": Object { "create": [MockFunction], @@ -475,6 +500,9 @@ exports[`dashboard listing hideWriteControls 1`] = ` "ui": Object { "IndexPatternSelect": [MockFunction], "SearchBar": [MockFunction], + "Settings": undefined, + "isEnhancementsEnabled": false, + "queryEnhancements": Map {}, }, }, "docLinks": Object { @@ -1421,6 +1449,24 @@ exports[`dashboard listing render table listing with initial filters from URL 1` "getValueSuggestions": [MockFunction], "hasQuerySuggestions": [MockFunction], }, + "dataSources": Object { + "dataSourceFactory": DataSourceFactory { + "dataSourceClasses": Object {}, + }, + "dataSourceService": DataSourceService { + "dataSourceFetchers": Object {}, + "dataSources": Object {}, + "dataSourcesSubject": BehaviorSubject { + "_isScalar": false, + "_value": Object {}, + "closed": false, + "hasError": false, + "isStopped": false, + "observers": Array [], + "thrownError": null, + }, + }, + }, "fieldFormats": Object { "deserialize": [MockFunction], "getByFieldType": [MockFunction], @@ -1602,6 +1648,7 @@ exports[`dashboard listing render table listing with initial filters from URL 1` }, }, "search": Object { + "__enhance": [MockFunction], "aggs": Object { "calculateAutoTimeExpression": [Function], "createAggConfigs": [MockFunction], @@ -1610,6 +1657,12 @@ exports[`dashboard listing render table listing with initial filters from URL 1` "getAll": [Function], }, }, + "df": Object { + "clear": [MockFunction], + "get": [MockFunction], + "set": [MockFunction], + }, + "getDefaultSearchInterceptor": [MockFunction], "search": [MockFunction], "searchSource": Object { "create": [MockFunction], @@ -1620,6 +1673,9 @@ exports[`dashboard listing render table listing with initial filters from URL 1` "ui": Object { "IndexPatternSelect": [MockFunction], "SearchBar": [MockFunction], + "Settings": undefined, + "isEnhancementsEnabled": false, + "queryEnhancements": Map {}, }, }, "docLinks": Object { @@ -2627,6 +2683,24 @@ exports[`dashboard listing renders call to action when no dashboards exist 1`] = "getValueSuggestions": [MockFunction], "hasQuerySuggestions": [MockFunction], }, + "dataSources": Object { + "dataSourceFactory": DataSourceFactory { + "dataSourceClasses": Object {}, + }, + "dataSourceService": DataSourceService { + "dataSourceFetchers": Object {}, + "dataSources": Object {}, + "dataSourcesSubject": BehaviorSubject { + "_isScalar": false, + "_value": Object {}, + "closed": false, + "hasError": false, + "isStopped": false, + "observers": Array [], + "thrownError": null, + }, + }, + }, "fieldFormats": Object { "deserialize": [MockFunction], "getByFieldType": [MockFunction], @@ -2808,6 +2882,7 @@ exports[`dashboard listing renders call to action when no dashboards exist 1`] = }, }, "search": Object { + "__enhance": [MockFunction], "aggs": Object { "calculateAutoTimeExpression": [Function], "createAggConfigs": [MockFunction], @@ -2816,6 +2891,12 @@ exports[`dashboard listing renders call to action when no dashboards exist 1`] = "getAll": [Function], }, }, + "df": Object { + "clear": [MockFunction], + "get": [MockFunction], + "set": [MockFunction], + }, + "getDefaultSearchInterceptor": [MockFunction], "search": [MockFunction], "searchSource": Object { "create": [MockFunction], @@ -2826,6 +2907,9 @@ exports[`dashboard listing renders call to action when no dashboards exist 1`] = "ui": Object { "IndexPatternSelect": [MockFunction], "SearchBar": [MockFunction], + "Settings": undefined, + "isEnhancementsEnabled": false, + "queryEnhancements": Map {}, }, }, "docLinks": Object { @@ -3833,6 +3917,24 @@ exports[`dashboard listing renders table rows 1`] = ` "getValueSuggestions": [MockFunction], "hasQuerySuggestions": [MockFunction], }, + "dataSources": Object { + "dataSourceFactory": DataSourceFactory { + "dataSourceClasses": Object {}, + }, + "dataSourceService": DataSourceService { + "dataSourceFetchers": Object {}, + "dataSources": Object {}, + "dataSourcesSubject": BehaviorSubject { + "_isScalar": false, + "_value": Object {}, + "closed": false, + "hasError": false, + "isStopped": false, + "observers": Array [], + "thrownError": null, + }, + }, + }, "fieldFormats": Object { "deserialize": [MockFunction], "getByFieldType": [MockFunction], @@ -4014,6 +4116,7 @@ exports[`dashboard listing renders table rows 1`] = ` }, }, "search": Object { + "__enhance": [MockFunction], "aggs": Object { "calculateAutoTimeExpression": [Function], "createAggConfigs": [MockFunction], @@ -4022,6 +4125,12 @@ exports[`dashboard listing renders table rows 1`] = ` "getAll": [Function], }, }, + "df": Object { + "clear": [MockFunction], + "get": [MockFunction], + "set": [MockFunction], + }, + "getDefaultSearchInterceptor": [MockFunction], "search": [MockFunction], "searchSource": Object { "create": [MockFunction], @@ -4032,6 +4141,9 @@ exports[`dashboard listing renders table rows 1`] = ` "ui": Object { "IndexPatternSelect": [MockFunction], "SearchBar": [MockFunction], + "Settings": undefined, + "isEnhancementsEnabled": false, + "queryEnhancements": Map {}, }, }, "docLinks": Object { @@ -5039,6 +5151,24 @@ exports[`dashboard listing renders warning when listingLimit is exceeded 1`] = ` "getValueSuggestions": [MockFunction], "hasQuerySuggestions": [MockFunction], }, + "dataSources": Object { + "dataSourceFactory": DataSourceFactory { + "dataSourceClasses": Object {}, + }, + "dataSourceService": DataSourceService { + "dataSourceFetchers": Object {}, + "dataSources": Object {}, + "dataSourcesSubject": BehaviorSubject { + "_isScalar": false, + "_value": Object {}, + "closed": false, + "hasError": false, + "isStopped": false, + "observers": Array [], + "thrownError": null, + }, + }, + }, "fieldFormats": Object { "deserialize": [MockFunction], "getByFieldType": [MockFunction], @@ -5220,6 +5350,7 @@ exports[`dashboard listing renders warning when listingLimit is exceeded 1`] = ` }, }, "search": Object { + "__enhance": [MockFunction], "aggs": Object { "calculateAutoTimeExpression": [Function], "createAggConfigs": [MockFunction], @@ -5228,6 +5359,12 @@ exports[`dashboard listing renders warning when listingLimit is exceeded 1`] = ` "getAll": [Function], }, }, + "df": Object { + "clear": [MockFunction], + "get": [MockFunction], + "set": [MockFunction], + }, + "getDefaultSearchInterceptor": [MockFunction], "search": [MockFunction], "searchSource": Object { "create": [MockFunction], @@ -5238,6 +5375,9 @@ exports[`dashboard listing renders warning when listingLimit is exceeded 1`] = ` "ui": Object { "IndexPatternSelect": [MockFunction], "SearchBar": [MockFunction], + "Settings": undefined, + "isEnhancementsEnabled": false, + "queryEnhancements": Map {}, }, }, "docLinks": Object { diff --git a/src/plugins/dashboard/public/application/components/dashboard_top_nav/__snapshots__/dashboard_top_nav.test.tsx.snap b/src/plugins/dashboard/public/application/components/dashboard_top_nav/__snapshots__/dashboard_top_nav.test.tsx.snap index 3649379ec583..5f71c7d5d217 100644 --- a/src/plugins/dashboard/public/application/components/dashboard_top_nav/__snapshots__/dashboard_top_nav.test.tsx.snap +++ b/src/plugins/dashboard/public/application/components/dashboard_top_nav/__snapshots__/dashboard_top_nav.test.tsx.snap @@ -260,6 +260,24 @@ exports[`Dashboard top nav render in embed mode 1`] = ` "getValueSuggestions": [MockFunction], "hasQuerySuggestions": [MockFunction], }, + "dataSources": Object { + "dataSourceFactory": DataSourceFactory { + "dataSourceClasses": Object {}, + }, + "dataSourceService": DataSourceService { + "dataSourceFetchers": Object {}, + "dataSources": Object {}, + "dataSourcesSubject": BehaviorSubject { + "_isScalar": false, + "_value": Object {}, + "closed": false, + "hasError": false, + "isStopped": false, + "observers": Array [], + "thrownError": null, + }, + }, + }, "fieldFormats": Object { "deserialize": [MockFunction], "getByFieldType": [MockFunction], @@ -349,6 +367,7 @@ exports[`Dashboard top nav render in embed mode 1`] = ` }, }, "search": Object { + "__enhance": [MockFunction], "aggs": Object { "calculateAutoTimeExpression": [Function], "createAggConfigs": [MockFunction], @@ -357,6 +376,12 @@ exports[`Dashboard top nav render in embed mode 1`] = ` "getAll": [Function], }, }, + "df": Object { + "clear": [MockFunction], + "get": [MockFunction], + "set": [MockFunction], + }, + "getDefaultSearchInterceptor": [MockFunction], "search": [MockFunction], "searchSource": Object { "create": [MockFunction], @@ -367,6 +392,9 @@ exports[`Dashboard top nav render in embed mode 1`] = ` "ui": Object { "IndexPatternSelect": [MockFunction], "SearchBar": [MockFunction], + "Settings": undefined, + "isEnhancementsEnabled": false, + "queryEnhancements": Map {}, }, }, "docLinks": Object { @@ -1231,6 +1259,24 @@ exports[`Dashboard top nav render in embed mode, and force hide filter bar 1`] = "getValueSuggestions": [MockFunction], "hasQuerySuggestions": [MockFunction], }, + "dataSources": Object { + "dataSourceFactory": DataSourceFactory { + "dataSourceClasses": Object {}, + }, + "dataSourceService": DataSourceService { + "dataSourceFetchers": Object {}, + "dataSources": Object {}, + "dataSourcesSubject": BehaviorSubject { + "_isScalar": false, + "_value": Object {}, + "closed": false, + "hasError": false, + "isStopped": false, + "observers": Array [], + "thrownError": null, + }, + }, + }, "fieldFormats": Object { "deserialize": [MockFunction], "getByFieldType": [MockFunction], @@ -1320,6 +1366,7 @@ exports[`Dashboard top nav render in embed mode, and force hide filter bar 1`] = }, }, "search": Object { + "__enhance": [MockFunction], "aggs": Object { "calculateAutoTimeExpression": [Function], "createAggConfigs": [MockFunction], @@ -1328,6 +1375,12 @@ exports[`Dashboard top nav render in embed mode, and force hide filter bar 1`] = "getAll": [Function], }, }, + "df": Object { + "clear": [MockFunction], + "get": [MockFunction], + "set": [MockFunction], + }, + "getDefaultSearchInterceptor": [MockFunction], "search": [MockFunction], "searchSource": Object { "create": [MockFunction], @@ -1338,6 +1391,9 @@ exports[`Dashboard top nav render in embed mode, and force hide filter bar 1`] = "ui": Object { "IndexPatternSelect": [MockFunction], "SearchBar": [MockFunction], + "Settings": undefined, + "isEnhancementsEnabled": false, + "queryEnhancements": Map {}, }, }, "docLinks": Object { @@ -2202,6 +2258,24 @@ exports[`Dashboard top nav render in embed mode, components can be forced show b "getValueSuggestions": [MockFunction], "hasQuerySuggestions": [MockFunction], }, + "dataSources": Object { + "dataSourceFactory": DataSourceFactory { + "dataSourceClasses": Object {}, + }, + "dataSourceService": DataSourceService { + "dataSourceFetchers": Object {}, + "dataSources": Object {}, + "dataSourcesSubject": BehaviorSubject { + "_isScalar": false, + "_value": Object {}, + "closed": false, + "hasError": false, + "isStopped": false, + "observers": Array [], + "thrownError": null, + }, + }, + }, "fieldFormats": Object { "deserialize": [MockFunction], "getByFieldType": [MockFunction], @@ -2291,6 +2365,7 @@ exports[`Dashboard top nav render in embed mode, components can be forced show b }, }, "search": Object { + "__enhance": [MockFunction], "aggs": Object { "calculateAutoTimeExpression": [Function], "createAggConfigs": [MockFunction], @@ -2299,6 +2374,12 @@ exports[`Dashboard top nav render in embed mode, components can be forced show b "getAll": [Function], }, }, + "df": Object { + "clear": [MockFunction], + "get": [MockFunction], + "set": [MockFunction], + }, + "getDefaultSearchInterceptor": [MockFunction], "search": [MockFunction], "searchSource": Object { "create": [MockFunction], @@ -2309,6 +2390,9 @@ exports[`Dashboard top nav render in embed mode, components can be forced show b "ui": Object { "IndexPatternSelect": [MockFunction], "SearchBar": [MockFunction], + "Settings": undefined, + "isEnhancementsEnabled": false, + "queryEnhancements": Map {}, }, }, "docLinks": Object { @@ -3173,6 +3257,24 @@ exports[`Dashboard top nav render in full screen mode with appended URL param bu "getValueSuggestions": [MockFunction], "hasQuerySuggestions": [MockFunction], }, + "dataSources": Object { + "dataSourceFactory": DataSourceFactory { + "dataSourceClasses": Object {}, + }, + "dataSourceService": DataSourceService { + "dataSourceFetchers": Object {}, + "dataSources": Object {}, + "dataSourcesSubject": BehaviorSubject { + "_isScalar": false, + "_value": Object {}, + "closed": false, + "hasError": false, + "isStopped": false, + "observers": Array [], + "thrownError": null, + }, + }, + }, "fieldFormats": Object { "deserialize": [MockFunction], "getByFieldType": [MockFunction], @@ -3262,6 +3364,7 @@ exports[`Dashboard top nav render in full screen mode with appended URL param bu }, }, "search": Object { + "__enhance": [MockFunction], "aggs": Object { "calculateAutoTimeExpression": [Function], "createAggConfigs": [MockFunction], @@ -3270,6 +3373,12 @@ exports[`Dashboard top nav render in full screen mode with appended URL param bu "getAll": [Function], }, }, + "df": Object { + "clear": [MockFunction], + "get": [MockFunction], + "set": [MockFunction], + }, + "getDefaultSearchInterceptor": [MockFunction], "search": [MockFunction], "searchSource": Object { "create": [MockFunction], @@ -3280,6 +3389,9 @@ exports[`Dashboard top nav render in full screen mode with appended URL param bu "ui": Object { "IndexPatternSelect": [MockFunction], "SearchBar": [MockFunction], + "Settings": undefined, + "isEnhancementsEnabled": false, + "queryEnhancements": Map {}, }, }, "docLinks": Object { @@ -4144,6 +4256,24 @@ exports[`Dashboard top nav render in full screen mode, no componenets should be "getValueSuggestions": [MockFunction], "hasQuerySuggestions": [MockFunction], }, + "dataSources": Object { + "dataSourceFactory": DataSourceFactory { + "dataSourceClasses": Object {}, + }, + "dataSourceService": DataSourceService { + "dataSourceFetchers": Object {}, + "dataSources": Object {}, + "dataSourcesSubject": BehaviorSubject { + "_isScalar": false, + "_value": Object {}, + "closed": false, + "hasError": false, + "isStopped": false, + "observers": Array [], + "thrownError": null, + }, + }, + }, "fieldFormats": Object { "deserialize": [MockFunction], "getByFieldType": [MockFunction], @@ -4233,6 +4363,7 @@ exports[`Dashboard top nav render in full screen mode, no componenets should be }, }, "search": Object { + "__enhance": [MockFunction], "aggs": Object { "calculateAutoTimeExpression": [Function], "createAggConfigs": [MockFunction], @@ -4241,6 +4372,12 @@ exports[`Dashboard top nav render in full screen mode, no componenets should be "getAll": [Function], }, }, + "df": Object { + "clear": [MockFunction], + "get": [MockFunction], + "set": [MockFunction], + }, + "getDefaultSearchInterceptor": [MockFunction], "search": [MockFunction], "searchSource": Object { "create": [MockFunction], @@ -4251,6 +4388,9 @@ exports[`Dashboard top nav render in full screen mode, no componenets should be "ui": Object { "IndexPatternSelect": [MockFunction], "SearchBar": [MockFunction], + "Settings": undefined, + "isEnhancementsEnabled": false, + "queryEnhancements": Map {}, }, }, "docLinks": Object { @@ -5115,6 +5255,24 @@ exports[`Dashboard top nav render with all components 1`] = ` "getValueSuggestions": [MockFunction], "hasQuerySuggestions": [MockFunction], }, + "dataSources": Object { + "dataSourceFactory": DataSourceFactory { + "dataSourceClasses": Object {}, + }, + "dataSourceService": DataSourceService { + "dataSourceFetchers": Object {}, + "dataSources": Object {}, + "dataSourcesSubject": BehaviorSubject { + "_isScalar": false, + "_value": Object {}, + "closed": false, + "hasError": false, + "isStopped": false, + "observers": Array [], + "thrownError": null, + }, + }, + }, "fieldFormats": Object { "deserialize": [MockFunction], "getByFieldType": [MockFunction], @@ -5204,6 +5362,7 @@ exports[`Dashboard top nav render with all components 1`] = ` }, }, "search": Object { + "__enhance": [MockFunction], "aggs": Object { "calculateAutoTimeExpression": [Function], "createAggConfigs": [MockFunction], @@ -5212,6 +5371,12 @@ exports[`Dashboard top nav render with all components 1`] = ` "getAll": [Function], }, }, + "df": Object { + "clear": [MockFunction], + "get": [MockFunction], + "set": [MockFunction], + }, + "getDefaultSearchInterceptor": [MockFunction], "search": [MockFunction], "searchSource": Object { "create": [MockFunction], @@ -5222,6 +5387,9 @@ exports[`Dashboard top nav render with all components 1`] = ` "ui": Object { "IndexPatternSelect": [MockFunction], "SearchBar": [MockFunction], + "Settings": undefined, + "isEnhancementsEnabled": false, + "queryEnhancements": Map {}, }, }, "docLinks": Object { @@ -5823,4 +5991,4 @@ exports[`Dashboard top nav render with all components 1`] = ` -`; \ No newline at end of file +`; diff --git a/src/plugins/data/common/constants.ts b/src/plugins/data/common/constants.ts index 43db1fe72b9e..27cfc64cf2f6 100644 --- a/src/plugins/data/common/constants.ts +++ b/src/plugins/data/common/constants.ts @@ -35,6 +35,7 @@ export const UI_SETTINGS = { DOC_HIGHLIGHT: 'doc_table:highlight', QUERY_STRING_OPTIONS: 'query:queryString:options', QUERY_ALLOW_LEADING_WILDCARDS: 'query:allowLeadingWildcards', + QUERY_DATA_SOURCE_READONLY: 'query:dataSourceReadOnly', SEARCH_QUERY_LANGUAGE: 'search:queryLanguage', SORT_OPTIONS: 'sort:options', COURIER_IGNORE_FILTER_IF_FIELD_NOT_IN_INDEX: 'courier:ignoreFilterIfFieldNotInIndex', @@ -60,4 +61,5 @@ export const UI_SETTINGS = { INDEXPATTERN_PLACEHOLDER: 'indexPattern:placeholder', FILTERS_PINNED_BY_DEFAULT: 'filters:pinnedByDefault', FILTERS_EDITOR_SUGGEST_VALUES: 'filterEditor:suggestValues', + DATAFRAME_HYDRATION_STRATEGY: 'dataframe:hydrationStrategy', } as const; diff --git a/src/plugins/data/common/data_frames/_df_cache.ts b/src/plugins/data/common/data_frames/_df_cache.ts new file mode 100644 index 000000000000..177f26840874 --- /dev/null +++ b/src/plugins/data/common/data_frames/_df_cache.ts @@ -0,0 +1,29 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { IDataFrame } from '..'; + +export interface DfCache { + get: () => IDataFrame | undefined; + set: (value: IDataFrame) => IDataFrame; + clear: () => void; +} + +export function createDataFrameCache(): DfCache { + let df: IDataFrame | undefined; + const cache: DfCache = { + get: () => { + return df; + }, + set: (prom: IDataFrame) => { + df = prom; + return prom; + }, + clear: () => { + df = undefined; + }, + }; + return cache; +} diff --git a/src/plugins/data/common/data_frames/fields/index.ts b/src/plugins/data/common/data_frames/fields/index.ts new file mode 100644 index 000000000000..9f269633f307 --- /dev/null +++ b/src/plugins/data/common/data_frames/fields/index.ts @@ -0,0 +1,6 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export * from './types'; diff --git a/src/plugins/data/common/data_frames/fields/types.ts b/src/plugins/data/common/data_frames/fields/types.ts new file mode 100644 index 000000000000..47144c0c0198 --- /dev/null +++ b/src/plugins/data/common/data_frames/fields/types.ts @@ -0,0 +1,18 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export interface IFieldType { + name: string; + type: string; + values: any[]; + count?: number; + aggregatable?: boolean; + filterable?: boolean; + searchable?: boolean; + sortable?: boolean; + visualizable?: boolean; + displayName?: string; + format?: any; +} diff --git a/src/plugins/data/common/data_frames/index.ts b/src/plugins/data/common/data_frames/index.ts new file mode 100644 index 000000000000..8b6a31eaea68 --- /dev/null +++ b/src/plugins/data/common/data_frames/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export * from './types'; +export * from './utils'; diff --git a/src/plugins/data/common/data_frames/types.ts b/src/plugins/data/common/data_frames/types.ts new file mode 100644 index 000000000000..f8dd04193cff --- /dev/null +++ b/src/plugins/data/common/data_frames/types.ts @@ -0,0 +1,103 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { SearchResponse } from 'elasticsearch'; +import { IFieldType } from './fields'; + +export * from './_df_cache'; + +/** @public **/ +export enum DATA_FRAME_TYPES { + DEFAULT = 'data_frame', + POLLING = 'data_frame_polling', +} + +export interface DataFrameService { + get: () => IDataFrame | undefined; + set: (dataFrame: IDataFrame) => Promise; + clear: () => void; +} + +/** + * A data frame is a two-dimensional labeled data structure with columns of potentially different types. + */ +export interface IDataFrame { + type?: DATA_FRAME_TYPES.DEFAULT; + name?: string; + schema?: Array>; + meta?: Record; + fields: IFieldType[]; + size: number; +} + +/** + * An aggregation is a process where the values of multiple rows are grouped together to form a single summary value. + */ +export interface DataFrameAgg { + value: number; +} + +/** + * A bucket aggregation is a type of aggregation that creates buckets or sets of data. + */ +export interface DataFrameBucketAgg extends DataFrameAgg { + key: string; +} + +/** + * This configuration is used to define how the aggregation should be performed. + */ +export interface DataFrameAggConfig { + id: string; + type: string; + field?: string; + order?: Record; + size?: number; + date_histogram?: { + field: string; + fixed_interval?: string; + calendar_interval?: string; + time_zone: string; + min_doc_count: number; + }; + avg?: { + field: string; + }; + cardinality?: { + field: string; + }; + terms?: { + field: string; + size: number; + order: Record; + }; + aggs?: Record; +} + +export interface PartialDataFrame extends Omit { + fields: Array>; +} + +/** + * To be utilize with aggregations and will map to buckets + * Plugins can get the aggregated value by their own logic + * Setting to null will disable the aggregation if plugin wishes + * In future, if the plugin doesn't intentionally set the value to null, + * we can calculate the value based on the fields. + */ +// TODO: handle composite +export interface IDataFrameWithAggs extends IDataFrame { + aggs: Record; +} + +export interface IDataFrameResponse extends SearchResponse { + type: DATA_FRAME_TYPES; + body: IDataFrame | IDataFrameWithAggs | IDataFrameError; + took: number; +} + +export interface IDataFrameError { + error: Error; +} diff --git a/src/plugins/data/common/data_frames/utils.ts b/src/plugins/data/common/data_frames/utils.ts new file mode 100644 index 000000000000..c3c55c5f227c --- /dev/null +++ b/src/plugins/data/common/data_frames/utils.ts @@ -0,0 +1,453 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { SearchResponse } from 'elasticsearch'; +import datemath from '@opensearch/datemath'; +import { + DATA_FRAME_TYPES, + DataFrameAggConfig, + DataFrameBucketAgg, + IDataFrame, + IDataFrameWithAggs, + IDataFrameResponse, + PartialDataFrame, +} from './types'; +import { IFieldType } from './fields'; +import { IndexPatternFieldMap, IndexPatternSpec } from '../index_patterns'; +import { IOpenSearchDashboardsSearchRequest } from '../search'; +import { GetAggTypeFn, GetDataFrameAggQsFn } from '../types'; + +/** + * Returns the raw data frame from the search request. + * + * @param searchRequest - search request object. + * @returns dataframe + */ +export const getRawDataFrame = (searchRequest: IOpenSearchDashboardsSearchRequest) => { + return searchRequest.params?.body?.df; +}; + +/** + * Returns the raw query string from the search request. + * Gets current state query if exists, otherwise gets the initial query. + * + * @param searchRequest - search request object + * @returns query string + */ +export const getRawQueryString = ( + searchRequest: IOpenSearchDashboardsSearchRequest +): string | undefined => { + return ( + searchRequest.params?.body?.query?.queries[1]?.query ?? + searchRequest.params?.body?.query?.queries[0]?.query + ); +}; + +/** + * Returns the raw aggregations from the search request. + * + * @param searchRequest - search request object + * @returns aggregations + */ +export const getRawAggs = (searchRequest: IOpenSearchDashboardsSearchRequest) => { + return searchRequest.params?.body?.aggs; +}; + +/** + * Returns the unique values for raw aggregations. This is used + * with `other-filter` aggregation. To get the values that were not + * included in the aggregation response prior to this request. + * + * @param rawAggs - raw aggregations object + * @returns object containing the field and its unique values + */ +export const getUniqueValuesForRawAggs = (rawAggs: Record) => { + const filters = rawAggs.filters?.filters?.['']?.bool?.must_not; + if (!filters || !Array.isArray(filters)) { + return null; + } + const values: unknown[] = []; + let field: string | undefined; + + filters.forEach((agg: any) => { + Object.values(agg).forEach((aggValue) => { + Object.entries(aggValue as Record).forEach(([key, value]) => { + field = key; + values.push(value); + }); + }); + }); + + return { field, values }; +}; + +/** + * Returns the aggregation configuration for raw aggregations. + * Aggregations are nested objects, so this function recursively + * builds an object that is easier to work with. + * + * @param rawAggs - raw aggregations object + * @returns aggregation configuration + */ +export const getAggConfigForRawAggs = (rawAggs: Record): DataFrameAggConfig | null => { + const aggConfig: DataFrameAggConfig = { id: '', type: '' }; + + Object.entries(rawAggs).forEach(([aggKey, agg]) => { + aggConfig.id = aggKey; + Object.entries(agg as Record).forEach(([name, value]) => { + if (name === 'aggs') { + aggConfig.aggs = {}; + Object.entries(value as Record).forEach(([subAggKey, subRawAgg]) => { + const subAgg = getAggConfigForRawAggs(subRawAgg as Record); + if (subAgg) { + aggConfig.aggs![subAgg.id] = { ...subAgg, id: subAggKey }; + } + }); + } else { + aggConfig.type = name; + Object.assign(aggConfig, { [name]: value }); + } + }); + }); + + return aggConfig; +}; + +/** + * Returns the aggregation configuration. + * + * @param searchRequest - search request object + * @param aggConfig - aggregation configuration object + * @param getAggTypeFn - function to get the aggregation type from the aggsService + * @returns aggregation configuration + */ +export const getAggConfig = ( + searchRequest: IOpenSearchDashboardsSearchRequest, + aggConfig: Partial = {}, + getAggTypeFn: GetAggTypeFn +): DataFrameAggConfig => { + const rawAggs = getRawAggs(searchRequest); + Object.entries(rawAggs).forEach(([aggKey, agg]) => { + aggConfig.id = aggKey; + Object.entries(agg as Record).forEach(([name, value]) => { + if (name === 'aggs' && value) { + aggConfig.aggs = {}; + Object.entries(value as Record).forEach(([subAggKey, subRawAgg]) => { + const subAgg = getAggConfigForRawAggs(subRawAgg as Record); + if (subAgg) { + aggConfig.aggs![subAgg.id] = { ...subAgg, id: subAggKey }; + } + }); + } else { + aggConfig.type = getAggTypeFn(name)?.type ?? name; + Object.assign(aggConfig, { [name]: value }); + } + }); + }); + + return aggConfig as DataFrameAggConfig; +}; + +/** + * Converts the data frame response to a search response. + * This function is used to convert the data frame response to a search response + * to be used by the rest of the application. + * + * @param response - data frame response object + * @returns converted search response + */ +export const convertResult = (response: IDataFrameResponse): SearchResponse => { + const body = response.body; + if (body.hasOwnProperty('error')) { + return response; + } + const data = body as IDataFrame; + const hits: any[] = []; + for (let index = 0; index < data.size; index++) { + const hit: { [key: string]: any } = {}; + data.fields.forEach((field) => { + hit[field.name] = field.values[index]; + }); + hits.push({ + _index: data.name, + _source: hit, + }); + } + const searchResponse: SearchResponse = { + took: response.took, + timed_out: false, + _shards: { + total: 1, + successful: 1, + skipped: 0, + failed: 0, + }, + hits: { + total: 0, + max_score: 0, + hits, + }, + }; + + if (data.hasOwnProperty('aggs')) { + const dataWithAggs = data as IDataFrameWithAggs; + if (!dataWithAggs.aggs) { + // TODO: MQL best guess, get timestamp field and caculate it here + return searchResponse; + } + searchResponse.aggregations = Object.entries(dataWithAggs.aggs).reduce( + (acc: Record, [id, value]) => { + const aggConfig = dataWithAggs.meta?.aggs; + if (id === 'other-filter') { + const buckets = value as DataFrameBucketAgg[]; + buckets.forEach((bucket) => { + const bucketValue = bucket.value; + searchResponse.hits.total += bucketValue; + }); + acc[id] = { + buckets: [{ '': { doc_count: 0 } }], + }; + return acc; + } + if (aggConfig && aggConfig.type === 'buckets') { + const buckets = value as DataFrameBucketAgg[]; + acc[id] = { + buckets: buckets.map((bucket) => { + const bucketValue = bucket.value; + searchResponse.hits.total += bucketValue; + return { + key_as_string: bucket.key, + key: (aggConfig as DataFrameAggConfig).date_histogram + ? new Date(bucket.key).getTime() + : bucket.key, + doc_count: bucketValue, + }; + }), + }; + return acc; + } + acc[id] = Array.isArray(value) ? value[0] : value; + return acc; + }, + {} + ); + } + + return searchResponse; +}; + +/** + * Formats the field value. + * + * @param field - field object + * @param value - value to format + * @returns formatted value + */ +export const formatFieldValue = (field: IFieldType | Partial, value: any): any => { + return field.format && field.format.convert ? field.format.convert(value) : value; +}; + +/** + * Returns the field type. This function is used to determine the field type so that can + * be used by the rest of the application. The field type must map to a OsdFieldType + * to be used by the rest of the application. + * + * @param field - field object + * @returns field type + */ +export const getFieldType = (field: IFieldType | Partial): string | undefined => { + const fieldName = field.name?.toLowerCase(); + if (fieldName?.includes('date') || fieldName?.includes('timestamp')) { + return 'date'; + } + if (field.values?.some((value) => value instanceof Date || datemath.isDateTime(value))) { + return 'date'; + } + if (field.type === 'struct') { + return 'object'; + } + + return field.type; +}; + +/** + * Returns the time field. If there is an aggConfig then we do not have to guess. + * If there is no aggConfig then we will try to guess the time field. + * + * @param data - data frame object. + * @param aggConfig - aggregation configuration object. + * @returns time field. + */ +export const getTimeField = ( + data: IDataFrame, + aggConfig?: DataFrameAggConfig +): Partial | undefined => { + const fields = data.schema || data.fields; + return aggConfig && aggConfig.date_histogram && aggConfig.date_histogram.field + ? fields.find((field) => field.name === aggConfig?.date_histogram?.field) + : fields.find((field) => field.type === 'date'); +}; + +/** + * Checks if the value is a GeoPoint. Expects an object with lat and lon properties. + * + * @param value - value to check + * @returns True if the value is a GeoPoint, false otherwise + */ +export const isGeoPoint = (value: any): boolean => { + return ( + typeof value === 'object' && + value !== null && + 'lat' in value && + 'lon' in value && + typeof value.lat === 'number' && + typeof value.lon === 'number' + ); +}; + +/** + * Creates a data frame. + * + * @param partial - partial data frame object + * @returns data frame. + */ +export const createDataFrame = (partial: PartialDataFrame): IDataFrame | IDataFrameWithAggs => { + let size = 0; + const processField = (field: any) => { + field.type = getFieldType(field); + if (!field.values) { + field.values = new Array(size); + } else if (field.values.length > size) { + size = field.values.length; + } + return field as IFieldType; + }; + + const schema = partial.schema?.map(processField); + const fields = partial.fields?.map(processField); + + return { + ...partial, + schema, + fields, + size, + }; +}; + +/** + * Updates the data frame metadata. Metadata is used to store the aggregation configuration. + * It also stores the query string used to fetch the data frame aggregations. + * + * @param params - { dataFrame, qs, aggConfig, timeField, timeFilter, getAggQsFn } + */ +export const updateDataFrameMeta = ({ + dataFrame, + qs, + aggConfig, + timeField, + timeFilter, + getAggQsFn, +}: { + dataFrame: IDataFrame; + qs: string; + aggConfig: DataFrameAggConfig; + timeField: any; + timeFilter: string; + getAggQsFn: GetDataFrameAggQsFn; +}) => { + dataFrame.meta = { + aggs: aggConfig, + aggsQs: { + [aggConfig.id]: getAggQsFn({ + qs, + aggConfig, + timeField, + timeFilter, + }), + }, + }; + + if (aggConfig.aggs) { + const subAggs = aggConfig.aggs as Record; + for (const [key, subAgg] of Object.entries(subAggs)) { + const subAggConfig: Record = { [key]: subAgg }; + dataFrame.meta.aggsQs[subAgg.id] = getAggQsFn({ + qs, + aggConfig: subAggConfig as DataFrameAggConfig, + timeField, + timeFilter, + }); + } + } +}; + +/** + * Converts a data frame to index pattern spec which can be used to create an index pattern. + * + * @param dataFrame - data frame object + * @param id - index pattern id if it exists + * @returns index pattern spec + */ +export const dataFrameToSpec = (dataFrame: IDataFrame, id?: string): IndexPatternSpec => { + const fields = (dataFrame.schema || dataFrame.fields) as IFieldType[]; + + const toFieldSpec = (field: IFieldType, overrides: Partial) => { + return { + ...field, + ...overrides, + aggregatable: field.aggregatable ?? true, + searchable: field.searchable ?? true, + }; + }; + + const flattenFields = (acc: IndexPatternFieldMap, field: IFieldType): any => { + switch (field.type) { + case 'object': + const dataField = dataFrame.fields.find((f) => f.name === field.name) || field; + if (dataField) { + const subField = dataField.values[0]; + if (!subField) { + acc[field.name] = toFieldSpec(field, {}); + break; + } + Object.entries(subField).forEach(([key, value]) => { + const subFieldName = `${dataField.name}.${key}`; + const subFieldType = typeof value; + if (subFieldType === 'object' && isGeoPoint(value)) { + acc[subFieldName] = toFieldSpec(subField, { + name: subFieldName, + type: 'geo_point', + }); + } else { + acc = flattenFields(acc, { + name: subFieldName, + type: subFieldType, + values: + subFieldType === 'object' + ? Object.entries(value as Record)?.map(([k, v]) => ({ + name: `${subFieldName}.${k}`, + type: typeof v, + })) + : [], + } as IFieldType); + } + }); + } + break; + default: + acc[field.name] = toFieldSpec(field, {}); + break; + } + return acc; + }; + + return { + id: id ?? DATA_FRAME_TYPES.DEFAULT, + title: dataFrame.name, + timeFieldName: getTimeField(dataFrame)?.name, + type: !id ? DATA_FRAME_TYPES.DEFAULT : undefined, + fields: fields.reduce(flattenFields, {} as IndexPatternFieldMap), + }; +}; diff --git a/src/plugins/data/common/index.ts b/src/plugins/data/common/index.ts index 1eefb2383f8b..d7b7e56e2280 100644 --- a/src/plugins/data/common/index.ts +++ b/src/plugins/data/common/index.ts @@ -30,6 +30,7 @@ export * from './constants'; export * from './opensearch_query'; +export * from './data_frames'; export * from './field_formats'; export * from './field_mapping'; export * from './index_patterns'; diff --git a/src/plugins/data/common/index_patterns/errors/index.ts b/src/plugins/data/common/index_patterns/errors/index.ts index 9d2a1175dd3f..d77b7f140afe 100644 --- a/src/plugins/data/common/index_patterns/errors/index.ts +++ b/src/plugins/data/common/index_patterns/errors/index.ts @@ -29,3 +29,4 @@ */ export * from './duplicate_index_pattern'; +export * from './missing_index_pattern'; diff --git a/src/plugins/data/common/index_patterns/errors/missing_index_pattern.ts b/src/plugins/data/common/index_patterns/errors/missing_index_pattern.ts new file mode 100644 index 000000000000..7d5fa3356314 --- /dev/null +++ b/src/plugins/data/common/index_patterns/errors/missing_index_pattern.ts @@ -0,0 +1,11 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export class MissingIndexPatternError extends Error { + constructor(message: string) { + super(message); + this.name = 'MissingIndexPatternError'; + } +} diff --git a/src/plugins/data/common/index_patterns/fields/index.ts b/src/plugins/data/common/index_patterns/fields/index.ts index 351c0d3b7593..58a8612fd8c9 100644 --- a/src/plugins/data/common/index_patterns/fields/index.ts +++ b/src/plugins/data/common/index_patterns/fields/index.ts @@ -29,6 +29,6 @@ */ export * from './types'; -export { isFilterable, isNestedField } from './utils'; +export { isFilterable, isNestedField, setOverrides, getOverrides } from './utils'; export * from './field_list'; export * from './index_pattern_field'; diff --git a/src/plugins/data/common/index_patterns/fields/index_pattern_field.ts b/src/plugins/data/common/index_patterns/fields/index_pattern_field.ts index e1882abb9722..ce9dd5a67eba 100644 --- a/src/plugins/data/common/index_patterns/fields/index_pattern_field.ts +++ b/src/plugins/data/common/index_patterns/fields/index_pattern_field.ts @@ -28,7 +28,7 @@ * under the License. */ -import { OsdFieldType, getOsdFieldType } from '../../osd_field_types'; +import { OsdFieldType, getOsdFieldOverrides, getOsdFieldType } from '../../osd_field_types'; import { OSD_FIELD_TYPES } from '../../osd_field_types/types'; import { IFieldType } from './types'; import { FieldSpec, IndexPattern } from '../..'; @@ -133,6 +133,7 @@ export class IndexPatternField implements IFieldType { } public get filterable() { + if (getOsdFieldOverrides().filterable !== undefined) return !!getOsdFieldOverrides().filterable; return ( this.name === '_id' || this.scripted || @@ -141,6 +142,8 @@ export class IndexPatternField implements IFieldType { } public get visualizable() { + if (getOsdFieldOverrides().visualizable !== undefined) + return !!getOsdFieldOverrides().visualizable; const notVisualizableFieldTypes: string[] = [OSD_FIELD_TYPES.UNKNOWN, OSD_FIELD_TYPES.CONFLICT]; return this.aggregatable && !notVisualizableFieldTypes.includes(this.spec.type); } diff --git a/src/plugins/data/common/index_patterns/fields/utils.ts b/src/plugins/data/common/index_patterns/fields/utils.ts index 5178568533d2..1b96cd3b7063 100644 --- a/src/plugins/data/common/index_patterns/fields/utils.ts +++ b/src/plugins/data/common/index_patterns/fields/utils.ts @@ -28,12 +28,25 @@ * under the License. */ -import { getFilterableOsdTypeNames } from '../../osd_field_types'; +import { + getFilterableOsdTypeNames, + getOsdFieldOverrides, + setOsdFieldOverrides, +} from '../../osd_field_types'; import { IFieldType } from './types'; const filterableTypes = getFilterableOsdTypeNames(); +export function setOverrides(overrides: Record | undefined) { + setOsdFieldOverrides(overrides); +} + +export function getOverrides(): Record { + return getOsdFieldOverrides(); +} + export function isFilterable(field: IFieldType): boolean { + if (getOverrides().filterable !== undefined) return !!getOverrides().filterable; return ( field.name === '_id' || field.scripted || diff --git a/src/plugins/data/common/index_patterns/index_patterns/_pattern_cache.ts b/src/plugins/data/common/index_patterns/index_patterns/_pattern_cache.ts index cc54e7b4cdde..834d21e6c35d 100644 --- a/src/plugins/data/common/index_patterns/index_patterns/_pattern_cache.ts +++ b/src/plugins/data/common/index_patterns/index_patterns/_pattern_cache.ts @@ -32,6 +32,7 @@ import { IndexPattern } from './index_pattern'; export interface PatternCache { get: (id: string) => IndexPattern; + getByTitle: (title: string) => IndexPattern; set: (id: string, value: IndexPattern) => IndexPattern; clear: (id: string) => void; clearAll: () => void; @@ -43,6 +44,9 @@ export function createIndexPatternCache(): PatternCache { get: (id: string) => { return vals[id]; }, + getByTitle: (title: string) => { + return Object.values(vals).find((pattern: IndexPattern) => pattern.title === title); + }, set: (id: string, prom: any) => { vals[id] = prom; return prom; diff --git a/src/plugins/data/common/index_patterns/index_patterns/index_patterns.ts b/src/plugins/data/common/index_patterns/index_patterns/index_patterns.ts index 688605821097..d60c1b6dd901 100644 --- a/src/plugins/data/common/index_patterns/index_patterns/index_patterns.ts +++ b/src/plugins/data/common/index_patterns/index_patterns/index_patterns.ts @@ -55,7 +55,7 @@ import { UI_SETTINGS, SavedObject } from '../../../common'; import { SavedObjectNotFound } from '../../../../opensearch_dashboards_utils/common'; import { IndexPatternMissingIndices } from '../lib'; import { findByTitle, getIndexPatternTitle } from '../utils'; -import { DuplicateIndexPatternError } from '../errors'; +import { DuplicateIndexPatternError, MissingIndexPatternError } from '../errors'; const indexPatternCache = createIndexPatternCache(); const MAX_ATTEMPTS_TO_RESOLVE_CONFLICTS = 3; @@ -192,8 +192,10 @@ export class IndexPatternsService { * Clear index pattern list cache * @param id optionally clear a single id */ - clearCache = (id?: string) => { - this.savedObjectsCache = null; + clearCache = (id?: string, clearSavedObjectsCache: boolean = true) => { + if (clearSavedObjectsCache) { + this.savedObjectsCache = null; + } if (id) { indexPatternCache.clear(id); } else { @@ -208,6 +210,10 @@ export class IndexPatternsService { return this.savedObjectsCache; }; + saveToCache = (id: string, indexPattern: IndexPattern) => { + indexPatternCache.set(id, indexPattern); + }; + /** * Get default index pattern */ @@ -282,9 +288,13 @@ export class IndexPatternsService { * Refresh field list for a given index pattern * @param indexPattern */ - refreshFields = async (indexPattern: IndexPattern) => { + refreshFields = async (indexPattern: IndexPattern, skipType = false) => { try { - const fields = await this.getFieldsForIndexPattern(indexPattern); + const indexPatternCopy = skipType + ? ({ ...indexPattern, type: undefined } as IndexPattern) + : indexPattern; + + const fields = await this.getFieldsForIndexPattern(indexPatternCopy); const scripted = indexPattern.getScriptedFields().map((field) => field.spec); indexPattern.fields.replaceAll([...fields, ...scripted]); } catch (err) { @@ -499,6 +509,19 @@ export class IndexPatternsService { return indexPattern; }; + /** + * Get an index pattern by title if cached + * @param id + */ + + getByTitle = (title: string, ignoreErrors: boolean = false): IndexPattern => { + const indexPattern = indexPatternCache.getByTitle(title); + if (!indexPattern && !ignoreErrors) { + throw new MissingIndexPatternError(`Missing index pattern: ${title}`); + } + return indexPattern; + }; + migrate(indexPattern: IndexPattern, newTitle: string) { return this.savedObjectsClient .update( diff --git a/src/plugins/data/common/opensearch_query/opensearch_query/build_opensearch_query.ts b/src/plugins/data/common/opensearch_query/opensearch_query/build_opensearch_query.ts index 481eae12d121..23295c3bafec 100644 --- a/src/plugins/data/common/opensearch_query/opensearch_query/build_opensearch_query.ts +++ b/src/plugins/data/common/opensearch_query/opensearch_query/build_opensearch_query.ts @@ -66,6 +66,17 @@ export function buildOpenSearchQuery( const validQueries = queries.filter((query) => has(query, 'query')); const queriesByLanguage = groupBy(validQueries, 'language'); + const unsupportedQueries = Object.keys(queriesByLanguage).filter( + (language) => language !== 'kuery' && language.toLowerCase() !== 'lucene' + ); + if (unsupportedQueries.length > 0) { + return { + type: 'unsupported', + queries, + filters, + }; + } + const kueryQuery = buildQueryFromKuery( indexPattern, queriesByLanguage.kuery, diff --git a/src/plugins/data/common/osd_field_types/index.ts b/src/plugins/data/common/osd_field_types/index.ts index 9454adc2d475..345a22094334 100644 --- a/src/plugins/data/common/osd_field_types/index.ts +++ b/src/plugins/data/common/osd_field_types/index.ts @@ -35,4 +35,6 @@ export { getOsdFieldType, getOsdTypeNames, getFilterableOsdTypeNames, + getOsdFieldOverrides, + setOsdFieldOverrides, } from './osd_field_types'; diff --git a/src/plugins/data/common/osd_field_types/osd_field_types.ts b/src/plugins/data/common/osd_field_types/osd_field_types.ts index b9fe14ff1a87..00fb67a8dc5e 100644 --- a/src/plugins/data/common/osd_field_types/osd_field_types.ts +++ b/src/plugins/data/common/osd_field_types/osd_field_types.ts @@ -34,6 +34,7 @@ import { OPENSEARCH_FIELD_TYPES, OSD_FIELD_TYPES } from './types'; /** @private */ const registeredOsdTypes = createOsdFieldTypes(); +let osdFieldOverrides = {}; /** * Get a type object by name @@ -75,3 +76,11 @@ export const castOpenSearchToOsdFieldTypeName = ( */ export const getFilterableOsdTypeNames = (): string[] => registeredOsdTypes.filter((type) => type.filterable).map((type) => type.name); + +export const setOsdFieldOverrides = (newOverrides: { [key: string]: any } | undefined) => { + osdFieldOverrides = newOverrides ? Object.assign({}, osdFieldOverrides, newOverrides) : {}; +}; + +export const getOsdFieldOverrides = (): { [key: string]: any } => { + return osdFieldOverrides; +}; diff --git a/src/plugins/data/common/search/opensearch_search/types.ts b/src/plugins/data/common/search/opensearch_search/types.ts index 3b93177bf201..f90a3f1de245 100644 --- a/src/plugins/data/common/search/opensearch_search/types.ts +++ b/src/plugins/data/common/search/opensearch_search/types.ts @@ -57,6 +57,7 @@ export type ISearchRequestParams> = { export interface IOpenSearchSearchRequest extends IOpenSearchDashboardsSearchRequest { indexType?: string; + language?: string; dataSourceId?: string; } diff --git a/src/plugins/data/common/search/search_source/create_search_source.test.ts b/src/plugins/data/common/search/search_source/create_search_source.test.ts index 467ecec59f5a..68dfa7699e13 100644 --- a/src/plugins/data/common/search/search_source/create_search_source.test.ts +++ b/src/plugins/data/common/search/search_source/create_search_source.test.ts @@ -50,6 +50,11 @@ describe('createSearchSource', () => { callMsearch: jest.fn(), loadingCount$: new BehaviorSubject(0), }, + df: { + get: jest.fn().mockReturnValue({}), + set: jest.fn().mockReturnValue({}), + clear: jest.fn(), + }, }; indexPatternContractMock = ({ diff --git a/src/plugins/data/common/search/search_source/fetch/get_search_params.ts b/src/plugins/data/common/search/search_source/fetch/get_search_params.ts index a25d6e530bad..d9bd7721d6cb 100644 --- a/src/plugins/data/common/search/search_source/fetch/get_search_params.ts +++ b/src/plugins/data/common/search/search_source/fetch/get_search_params.ts @@ -29,7 +29,7 @@ */ import { UI_SETTINGS } from '../../../constants'; -import { GetConfigFn } from '../../../types'; +import { GetConfigFn, GetDataFrameFn, DestroyDataFrameFn } from '../../../types'; import { ISearchRequestParams } from '../../index'; import { SearchRequest } from './types'; @@ -49,16 +49,50 @@ export function getPreference(getConfig: GetConfigFn) { : undefined; } +export function getExternalSearchParamsFromRequest( + searchRequest: SearchRequest, + dependencies: { + getConfig: GetConfigFn; + getDataFrame: GetDataFrameFn; + } +): ISearchRequestParams { + const { getConfig, getDataFrame } = dependencies; + const searchParams = getSearchParams(getConfig); + const dataFrame = getDataFrame(); + const indexTitle = searchRequest.index.title || searchRequest.index; + + return { + index: indexTitle, + body: { + ...searchRequest.body, + ...(dataFrame && dataFrame?.name === indexTitle ? { df: dataFrame } : {}), + }, + ...searchParams, + }; +} + /** @public */ // TODO: Could provide this on runtime contract with dependencies // already wired up. export function getSearchParamsFromRequest( searchRequest: SearchRequest, - dependencies: { getConfig: GetConfigFn } + dependencies: { + getConfig: GetConfigFn; + getDataFrame?: GetDataFrameFn; + destroyDataFrame?: DestroyDataFrameFn; + } ): ISearchRequestParams { - const { getConfig } = dependencies; + const { getConfig, getDataFrame, destroyDataFrame } = dependencies; const searchParams = getSearchParams(getConfig); + if (getDataFrame && destroyDataFrame) { + if (getDataFrame()) { + delete searchRequest.body.df; + delete searchRequest.indexType; + destroyDataFrame(); + } + } + return { index: searchRequest.index.title || searchRequest.index, body: searchRequest.body, diff --git a/src/plugins/data/common/search/search_source/fetch/index.ts b/src/plugins/data/common/search/search_source/fetch/index.ts index bb432ec0d833..f171ca38738e 100644 --- a/src/plugins/data/common/search/search_source/fetch/index.ts +++ b/src/plugins/data/common/search/search_source/fetch/index.ts @@ -28,6 +28,11 @@ * under the License. */ -export { getSearchParams, getSearchParamsFromRequest, getPreference } from './get_search_params'; +export { + getSearchParams, + getExternalSearchParamsFromRequest, + getSearchParamsFromRequest, + getPreference, +} from './get_search_params'; export { RequestFailure } from './request_error'; export * from './types'; diff --git a/src/plugins/data/common/search/search_source/mocks.ts b/src/plugins/data/common/search/search_source/mocks.ts index 959d1aebfe53..83ea582ef99f 100644 --- a/src/plugins/data/common/search/search_source/mocks.ts +++ b/src/plugins/data/common/search/search_source/mocks.ts @@ -47,6 +47,8 @@ export const searchSourceInstanceMock: MockedKeys = { createChild: jest.fn().mockReturnThis(), setParent: jest.fn(), getParent: jest.fn().mockReturnThis(), + setDataFrame: jest.fn(), + getDataFrame: jest.fn().mockReturnThis(), fetch: jest.fn().mockResolvedValue({}), onRequestStart: jest.fn(), getSearchRequestBody: jest.fn(), @@ -54,6 +56,8 @@ export const searchSourceInstanceMock: MockedKeys = { history: [], getSerializedFields: jest.fn(), serialize: jest.fn(), + flatten: jest.fn().mockReturnThis(), + destroyDataFrame: jest.fn(), }; export const searchSourceCommonMock: jest.Mocked = { @@ -70,4 +74,9 @@ export const createSearchSourceMock = (fields?: SearchSourceFields) => callMsearch: jest.fn(), loadingCount$: new BehaviorSubject(0), }, + df: { + get: jest.fn().mockReturnValue({}), + set: jest.fn().mockReturnValue({}), + clear: jest.fn(), + }, }); diff --git a/src/plugins/data/common/search/search_source/search_source.test.ts b/src/plugins/data/common/search/search_source/search_source.test.ts index 92cc0682a136..09adc867d213 100644 --- a/src/plugins/data/common/search/search_source/search_source.test.ts +++ b/src/plugins/data/common/search/search_source/search_source.test.ts @@ -84,6 +84,11 @@ describe('SearchSource', () => { callMsearch: jest.fn(), loadingCount$: new BehaviorSubject(0), }, + df: { + get: jest.fn().mockReturnValue({}), + set: jest.fn().mockReturnValue({}), + clear: jest.fn(), + }, }; }); diff --git a/src/plugins/data/common/search/search_source/search_source.ts b/src/plugins/data/common/search/search_source/search_source.ts index abe6fa1b5cb4..b7dc48b404b6 100644 --- a/src/plugins/data/common/search/search_source/search_source.ts +++ b/src/plugins/data/common/search/search_source/search_source.ts @@ -87,10 +87,17 @@ import { normalizeSortRequest } from './normalize_sort_request'; import { filterDocvalueFields } from './filter_docvalue_fields'; import { fieldWildcardFilter } from '../../../../opensearch_dashboards_utils/common'; import { IIndexPattern } from '../../index_patterns'; +import { DATA_FRAME_TYPES, IDataFrame, IDataFrameResponse, convertResult } from '../../data_frames'; import { IOpenSearchSearchRequest, IOpenSearchSearchResponse, ISearchOptions } from '../..'; import { IOpenSearchDashboardsSearchRequest, IOpenSearchDashboardsSearchResponse } from '../types'; import { ISearchSource, SearchSourceOptions, SearchSourceFields } from './types'; -import { FetchHandlers, RequestFailure, getSearchParamsFromRequest, SearchRequest } from './fetch'; +import { + FetchHandlers, + RequestFailure, + getExternalSearchParamsFromRequest, + getSearchParamsFromRequest, + SearchRequest, +} from './fetch'; import { getOpenSearchQueryConfig, @@ -116,6 +123,7 @@ export const searchSourceRequiredUiSettings = [ UI_SETTINGS.QUERY_STRING_OPTIONS, UI_SETTINGS.SEARCH_INCLUDE_FROZEN, UI_SETTINGS.SORT_OPTIONS, + UI_SETTINGS.DATAFRAME_HYDRATION_STRATEGY, ]; export interface SearchSourceDependencies extends FetchHandlers { @@ -123,11 +131,18 @@ export interface SearchSourceDependencies extends FetchHandlers { // search options required here and returning a promise instead of observable. search: < SearchStrategyRequest extends IOpenSearchDashboardsSearchRequest = IOpenSearchSearchRequest, - SearchStrategyResponse extends IOpenSearchDashboardsSearchResponse = IOpenSearchSearchResponse + SearchStrategyResponse extends + | IOpenSearchDashboardsSearchResponse + | IDataFrameResponse = IOpenSearchSearchResponse >( request: SearchStrategyRequest, options: ISearchOptions ) => Promise; + df: { + get: () => IDataFrame | undefined; + set: (dataFrame: IDataFrame) => Promise; + clear: () => void; + }; } /** @public **/ @@ -267,6 +282,36 @@ export class SearchSource { return this.parent; } + /** + * Get the data frame of this SearchSource + * @return {undefined|IDataFrame} + */ + getDataFrame() { + return this.dependencies.df.get(); + } + + /** + * Set the data frame of this SearchSource + * + * @async + * @return {undefined|IDataFrame} + */ + async setDataFrame(dataFrame: IDataFrame | undefined) { + if (dataFrame) { + await this.dependencies.df.set(dataFrame); + } else { + this.destroyDataFrame(); + } + return this.getDataFrame(); + } + + /** + * Clear the data frame of this SearchSource + */ + destroyDataFrame() { + this.dependencies.df.clear(); + } + /** * Fetch this source and reject the returned Promise on error * @@ -282,6 +327,8 @@ export class SearchSource { let response; if (getConfig(UI_SETTINGS.COURIER_BATCH_SEARCHES)) { response = await this.legacyFetch(searchRequest, options); + } else if (this.isUnsupportedRequest(searchRequest)) { + response = await this.fetchExternalSearch(searchRequest, options); } else { const indexPattern = this.getField('index'); searchRequest.dataSourceId = indexPattern?.dataSourceRef?.id; @@ -337,12 +384,39 @@ export class SearchSource { const params = getSearchParamsFromRequest(searchRequest, { getConfig, + getDataFrame: this.getDataFrame.bind(this), + destroyDataFrame: this.destroyDataFrame.bind(this), }); return search( { params, indexType: searchRequest.indexType, dataSourceId: searchRequest.dataSourceId }, options - ).then(({ rawResponse }) => onResponse(searchRequest, rawResponse)); + ).then((response: any) => onResponse(searchRequest, response.rawResponse)); + } + + /** + * Run a non-native search using the search service + * @return {Promise>} + */ + private async fetchExternalSearch(searchRequest: SearchRequest, options: ISearchOptions) { + const { search, getConfig, onResponse } = this.dependencies; + + const params = getExternalSearchParamsFromRequest(searchRequest, { + getConfig, + getDataFrame: this.getDataFrame.bind(this), + }); + + return search({ params }, options).then(async (response: any) => { + if (response.hasOwnProperty('type')) { + if ((response as IDataFrameResponse).type === DATA_FRAME_TYPES.DEFAULT) { + const dataFrameResponse = response as IDataFrameResponse; + await this.setDataFrame(dataFrameResponse.body as IDataFrame); + return onResponse(searchRequest, convertResult(response as IDataFrameResponse)); + } + // TODO: MQL else if data_frame_polling then poll for the data frame updating the df fields only + } + return onResponse(searchRequest, response.rawResponse); + }); } /** @@ -366,6 +440,10 @@ export class SearchSource { ); } + private isUnsupportedRequest(request: SearchRequest): boolean { + return request.body!.query.hasOwnProperty('type') && request.body!.query.type === 'unsupported'; + } + /** * Called by requests of this search source when they are started * @param options diff --git a/src/plugins/data/common/search/search_source/types.ts b/src/plugins/data/common/search/search_source/types.ts index 13b100aab1a7..9f3fd75e1ce9 100644 --- a/src/plugins/data/common/search/search_source/types.ts +++ b/src/plugins/data/common/search/search_source/types.ts @@ -29,7 +29,7 @@ */ import { NameList } from 'elasticsearch'; -import { Filter, IndexPattern, Query } from '../..'; +import { Filter, IDataFrame, IndexPattern, Query } from '../..'; import { SearchSource } from './search_source'; /** @@ -103,6 +103,7 @@ export interface SearchSourceFields { searchAfter?: OpenSearchQuerySearchAfter; timeout?: string; terminate_after?: number; + df?: IDataFrame; } export interface SearchSourceOptions { diff --git a/src/plugins/data/common/search/types.ts b/src/plugins/data/common/search/types.ts index e05c0adb46f0..d2bcdc0f4d05 100644 --- a/src/plugins/data/common/search/types.ts +++ b/src/plugins/data/common/search/types.ts @@ -34,6 +34,7 @@ import { IOpenSearchSearchResponse, ISearchOptions, } from '../../common/search'; +import { IDataFrameResponse } from '../data_frames'; export type ISearch = ( request: IOpenSearchDashboardsSearchRequest, @@ -42,7 +43,9 @@ export type ISearch = ( export type ISearchGeneric = < SearchStrategyRequest extends IOpenSearchDashboardsSearchRequest = IOpenSearchSearchRequest, - SearchStrategyResponse extends IOpenSearchDashboardsSearchResponse = IOpenSearchSearchResponse + SearchStrategyResponse extends + | IOpenSearchDashboardsSearchResponse + | IDataFrameResponse = IOpenSearchSearchResponse >( request: SearchStrategyRequest, options?: ISearchOptions diff --git a/src/plugins/data/common/types.ts b/src/plugins/data/common/types.ts index 361ba39edfec..6a1f6e5a99d3 100644 --- a/src/plugins/data/common/types.ts +++ b/src/plugins/data/common/types.ts @@ -28,9 +28,13 @@ * under the License. */ +import { DataFrameAggConfig, IDataFrame } from './data_frames'; +import { BucketAggType, MetricAggType } from './search'; + export * from './query/types'; export * from './osd_field_types/types'; export * from './index_patterns/types'; +export * from './data_frames/types'; /** * If a service is being shared on both the client and the server, and @@ -43,3 +47,18 @@ export * from './index_patterns/types'; * not possible. */ export type GetConfigFn = (key: string, defaultOverride?: T) => T; +export type GetDataFrameFn = () => IDataFrame | undefined; +export type GetDataFrameAggQsFn = ({ + qs, + aggConfig, + timeField, + timeFilter, +}: { + qs: string; + aggConfig: DataFrameAggConfig; + timeField: any; + timeFilter: any; +}) => any; + +export type DestroyDataFrameFn = () => void; +export type GetAggTypeFn = (id: string) => BucketAggType | MetricAggType; diff --git a/src/plugins/data/config.ts b/src/plugins/data/config.ts index f8b553f3da1b..f8dcad85fb49 100644 --- a/src/plugins/data/config.ts +++ b/src/plugins/data/config.ts @@ -31,6 +31,9 @@ import { schema, TypeOf } from '@osd/config-schema'; export const configSchema = schema.object({ + enhancements: schema.object({ + enabled: schema.boolean({ defaultValue: false }), + }), autocomplete: schema.object({ querySuggestions: schema.object({ enabled: schema.boolean({ defaultValue: true }), diff --git a/src/plugins/data/public/data_sources/datasource/index.ts b/src/plugins/data/public/data_sources/datasource/index.ts index e45cd6dad22c..7cc4f8e6549a 100644 --- a/src/plugins/data/public/data_sources/datasource/index.ts +++ b/src/plugins/data/public/data_sources/datasource/index.ts @@ -12,5 +12,6 @@ export { IDataSourceQueryResult, DataSourceConnectionStatus, IndexPatternOption, + IDataSourceDataSet, } from './types'; export { DataSourceFactory } from './factory'; diff --git a/src/plugins/data/public/data_sources/datasource_services/datasource_service.test.ts b/src/plugins/data/public/data_sources/datasource_services/datasource_service.test.ts index e1d2077f60e5..1182ffe65d8a 100644 --- a/src/plugins/data/public/data_sources/datasource_services/datasource_service.test.ts +++ b/src/plugins/data/public/data_sources/datasource_services/datasource_service.test.ts @@ -5,7 +5,11 @@ import { waitFor } from '@testing-library/dom'; import { DataSource } from '../datasource'; -import { IndexPatternsService } from '../../index_patterns'; +import { SavedObject } from '../../../../../core/public'; +import { + IndexPatternSavedObjectAttrs, + IndexPatternsService, +} from '../../index_patterns/index_patterns'; import { DataSourceService } from '../datasource_services'; import { LocalDSDataSetParams, @@ -55,8 +59,20 @@ class MockDataSource extends DataSource< } async getDataSet(dataSetParams?: LocalDSDataSetParams): Promise { - await this.indexPattern.ensureDefaultIndexPattern(); - return await this.indexPattern.getCache(); + const savedObjectLst = await this.indexPattern.getCache(); + + if (!Array.isArray(savedObjectLst)) { + return { dataSets: [] }; + } + + return { + dataSets: savedObjectLst.map((savedObject: SavedObject) => { + return { + id: savedObject.id, + title: savedObject.attributes.title, + }; + }), + }; } async testConnection(): Promise { diff --git a/src/plugins/data/public/data_sources/datasource_services/mocks.ts b/src/plugins/data/public/data_sources/datasource_services/mocks.ts new file mode 100644 index 000000000000..1bf4d4d7a9a4 --- /dev/null +++ b/src/plugins/data/public/data_sources/datasource_services/mocks.ts @@ -0,0 +1,19 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { DataSourceService } from './datasource_service'; +import { DataSourceStart } from './types'; +import { DataSourceFactory } from '../datasource'; + +function createStartContract(): jest.Mocked { + return { + dataSourceService: DataSourceService.getInstance(), + dataSourceFactory: DataSourceFactory.getInstance(), + }; +} + +export const dataSourceServiceMock = { + createStartContract, +}; diff --git a/src/plugins/data/public/data_sources/register_default_datasource.ts b/src/plugins/data/public/data_sources/register_default_datasource.ts index ce169955704b..d8ae19114c3c 100644 --- a/src/plugins/data/public/data_sources/register_default_datasource.ts +++ b/src/plugins/data/public/data_sources/register_default_datasource.ts @@ -39,3 +39,5 @@ export const registerDefaultDataSource = (data: Omit>; export type Start = jest.Mocked>; @@ -59,7 +61,7 @@ const createSetupContract = (): Setup => { }; }; -const createStartContract = (): Start => { +const createStartContract = (isEnhancementsEnabled: boolean = false): Start => { const queryStartMock = queryServiceMock.createStartContract(); return { actions: { @@ -70,10 +72,7 @@ const createStartContract = (): Start => { search: searchServiceMock.createStartContract(), fieldFormats: fieldFormatsServiceMock.createStartContract(), query: queryStartMock, - ui: { - IndexPatternSelect: jest.fn(), - SearchBar: jest.fn().mockReturnValue(null), - }, + ui: uiServiceMock.createStartContract(isEnhancementsEnabled), indexPatterns: ({ find: jest.fn((search) => [{ id: search, title: search }]), createField: jest.fn(() => {}), @@ -93,6 +92,7 @@ const createStartContract = (): Start => { ), clearCache: jest.fn(), } as unknown) as IndexPatternsContract, + dataSources: dataSourceServiceMock.createStartContract(), }; }; diff --git a/src/plugins/data/public/plugin.ts b/src/plugins/data/public/plugin.ts index 4917eb9db9e2..f22a61423d1d 100644 --- a/src/plugins/data/public/plugin.ts +++ b/src/plugins/data/public/plugin.ts @@ -46,9 +46,9 @@ import { } from './types'; import { AutocompleteService } from './autocomplete'; import { SearchService } from './search/search_service'; +import { UiService } from './ui/ui_service'; import { FieldFormatsService } from './field_formats'; import { QueryService } from './query'; -import { createIndexPatternSelect } from './ui/index_pattern_select'; import { IndexPatternsService, onRedirectNoIndexPattern, @@ -63,9 +63,9 @@ import { setOverlays, setQueryService, setSearchService, + setUiService, setUiSettings, } from './services'; -import { createSearchBar } from './ui/search_bar/create_search_bar'; import { opensearchaggs } from './search/expressions'; import { SELECT_RANGE_TRIGGER, @@ -112,12 +112,14 @@ export class DataPublicPlugin > { private readonly autocomplete: AutocompleteService; private readonly searchService: SearchService; + private readonly uiService: UiService; private readonly fieldFormatsService: FieldFormatsService; private readonly queryService: QueryService; private readonly storage: IStorageWrapper; constructor(initializerContext: PluginInitializerContext) { this.searchService = new SearchService(initializerContext); + this.uiService = new UiService(initializerContext); this.queryService = new QueryService(); this.fieldFormatsService = new FieldFormatsService(); this.autocomplete = new AutocompleteService(initializerContext); @@ -161,13 +163,17 @@ export class DataPublicPlugin expressions, }); + const uiService = this.uiService.setup(core, {}); + return { + // TODO: MQL autocomplete: this.autocomplete.setup(core), search: searchService, fieldFormats: this.fieldFormatsService.setup(core), query: queryService, __enhance: (enhancements: DataPublicPluginEnhancements) => { - searchService.__enhance(enhancements.search); + if (enhancements.search) searchService.__enhance(enhancements.search); + if (enhancements.ui) uiService.__enhance(enhancements.ui); }, }; } @@ -246,18 +252,12 @@ export class DataPublicPlugin registerDefaultDataSource(dataServices); - const SearchBar = createSearchBar({ - core, - data: dataServices, - storage: this.storage, - }); + const uiService = this.uiService.start(core, { dataServices, storage: this.storage }); + setUiService(uiService); return { ...dataServices, - ui: { - IndexPatternSelect: createIndexPatternSelect(core.savedObjects.client), - SearchBar, - }, + ui: uiService, }; } @@ -265,5 +265,6 @@ export class DataPublicPlugin this.autocomplete.clearProviders(); this.queryService.stop(); this.searchService.stop(); + this.uiService.stop(); } } diff --git a/src/plugins/data/public/query/query_string/query_string_manager.ts b/src/plugins/data/public/query/query_string/query_string_manager.ts index bee5d4c3ded9..3747cabaf9ca 100644 --- a/src/plugins/data/public/query/query_string/query_string_manager.ts +++ b/src/plugins/data/public/query/query_string/query_string_manager.ts @@ -44,6 +44,10 @@ export class QueryStringManager { this.query$ = new BehaviorSubject(this.getDefaultQuery()); } + private getDefaultQueryString() { + return this.storage.get('opensearchDashboards.userQueryString') || ''; + } + private getDefaultLanguage() { return ( this.storage.get('opensearchDashboards.userQueryLanguage') || @@ -53,7 +57,7 @@ export class QueryStringManager { public getDefaultQuery() { return { - query: '', + query: this.getDefaultQueryString(), language: this.getDefaultLanguage(), }; } diff --git a/src/plugins/data/public/search/mocks.ts b/src/plugins/data/public/search/mocks.ts index 32c0ef61aca6..736de838df8d 100644 --- a/src/plugins/data/public/search/mocks.ts +++ b/src/plugins/data/public/search/mocks.ts @@ -45,6 +45,13 @@ function createStartContract(): jest.Mocked { search: jest.fn(), showError: jest.fn(), searchSource: searchSourceMock.createStartContract(), + __enhance: jest.fn(), + getDefaultSearchInterceptor: jest.fn(), + df: { + get: jest.fn().mockReturnValue({}), + set: jest.fn().mockReturnValue({}), + clear: jest.fn(), + }, }; } diff --git a/src/plugins/data/public/search/search_service.ts b/src/plugins/data/public/search/search_service.ts index c73e7881faa6..340c007963af 100644 --- a/src/plugins/data/public/search/search_service.ts +++ b/src/plugins/data/public/search/search_service.ts @@ -57,6 +57,13 @@ import { getShardDelayBucketAgg, } from '../../common/search/aggs/buckets/shard_delay'; import { aggShardDelay } from '../../common/search/aggs/buckets/shard_delay_fn'; +import { + DataFrameService, + IDataFrame, + IDataFrameResponse, + createDataFrameCache, + dataFrameToSpec, +} from '../../common/data_frames'; /** @internal */ export interface SearchServiceSetupDependencies { @@ -73,7 +80,9 @@ export interface SearchServiceStartDependencies { export class SearchService implements Plugin { private readonly aggsService = new AggsService(); private readonly searchSourceService = new SearchSourceService(); + private readonly dfCache = createDataFrameCache(); private searchInterceptor!: ISearchInterceptor; + private defaultSearchInterceptor!: ISearchInterceptor; private usageCollector?: SearchUsageCollector; constructor(private initializerContext: PluginInitializerContext) {} @@ -95,6 +104,7 @@ export class SearchService implements Plugin { startServices: getStartServices(), usageCollector: this.usageCollector!, }); + this.defaultSearchInterceptor = this.searchInterceptor; expressions.registerFunction(opensearchdsl); expressions.registerType(opensearchRawResponse); @@ -129,11 +139,36 @@ export class SearchService implements Plugin { const loadingCount$ = new BehaviorSubject(0); http.addLoadingCountSource(loadingCount$); + const dfService: DataFrameService = { + get: () => this.dfCache.get(), + set: async (dataFrame: IDataFrame) => { + if (this.dfCache.get() && this.dfCache.get()?.name !== dataFrame.name) { + indexPatterns.clearCache(this.dfCache.get()!.name, false); + } + this.dfCache.set(dataFrame); + const existingIndexPattern = indexPatterns.getByTitle(dataFrame.name!, true); + const dataSet = await indexPatterns.create( + dataFrameToSpec(dataFrame, existingIndexPattern?.id), + !existingIndexPattern?.id + ); + // save to cache by title because the id is not unique for temporary index pattern created + indexPatterns.saveToCache(dataSet.title, dataSet); + }, + clear: () => { + if (this.dfCache.get() === undefined) return; + // name because the id is not unique for temporary index pattern created + indexPatterns.clearCache(this.dfCache.get()!.name, false); + this.dfCache.clear(); + }, + }; + const searchSourceDependencies: SearchSourceDependencies = { getConfig: uiSettings.get.bind(uiSettings), search: < SearchStrategyRequest extends IOpenSearchDashboardsSearchRequest = IOpenSearchSearchRequest, - SearchStrategyResponse extends IOpenSearchDashboardsSearchResponse = IOpenSearchSearchResponse + SearchStrategyResponse extends + | IOpenSearchDashboardsSearchResponse + | IDataFrameResponse = IOpenSearchSearchResponse >( request: SearchStrategyRequest, options: ISearchOptions @@ -145,6 +180,7 @@ export class SearchService implements Plugin { callMsearch: getCallMsearch({ http }), loadingCount$, }, + df: dfService, }; return { @@ -154,6 +190,11 @@ export class SearchService implements Plugin { this.searchInterceptor.showError(e); }, searchSource: this.searchSourceService.start(indexPatterns, searchSourceDependencies), + __enhance: (enhancements: SearchEnhancements) => { + this.searchInterceptor = enhancements.searchInterceptor; + }, + getDefaultSearchInterceptor: () => this.defaultSearchInterceptor, + df: dfService, }; } diff --git a/src/plugins/data/public/search/types.ts b/src/plugins/data/public/search/types.ts index 8a0d82b855c8..29dc37b41c91 100644 --- a/src/plugins/data/public/search/types.ts +++ b/src/plugins/data/public/search/types.ts @@ -35,6 +35,7 @@ import { AggsSetup, AggsSetupDependencies, AggsStartDependencies, AggsStart } fr import { ISearchGeneric, ISearchStartSearchSource } from '../../common/search'; import { IndexPatternsContract } from '../../common/index_patterns/index_patterns'; import { UsageCollectionSetup } from '../../../usage_collection/public'; +import { DataFrameService } from '../../common/data_frames'; export { ISearchStartSearchSource }; @@ -78,6 +79,9 @@ export interface ISearchStart { * {@link ISearchStartSearchSource} */ searchSource: ISearchStartSearchSource; + __enhance: (enhancements: SearchEnhancements) => void; + getDefaultSearchInterceptor: () => ISearchInterceptor; + df: DataFrameService; } export { SEARCH_EVENT_TYPE } from './collectors'; diff --git a/src/plugins/data/public/services.ts b/src/plugins/data/public/services.ts index 3bcc9d69a9a4..d75dab2986ca 100644 --- a/src/plugins/data/public/services.ts +++ b/src/plugins/data/public/services.ts @@ -59,3 +59,5 @@ export const [getQueryService, setQueryService] = createGetterSetter< export const [getSearchService, setSearchService] = createGetterSetter< DataPublicPluginStart['search'] >('Search'); + +export const [getUiService, setUiService] = createGetterSetter('Ui'); diff --git a/src/plugins/data/public/types.ts b/src/plugins/data/public/types.ts index 5870ea7def8e..4f7936006a94 100644 --- a/src/plugins/data/public/types.ts +++ b/src/plugins/data/public/types.ts @@ -28,7 +28,6 @@ * under the License. */ -import React from 'react'; import { CoreStart } from 'src/core/public'; import { IStorageWrapper } from 'src/plugins/opensearch_dashboards_utils/public'; import { ExpressionsSetup } from 'src/plugins/expressions/public'; @@ -39,12 +38,13 @@ import { createFiltersFromRangeSelectAction, createFiltersFromValueClickAction } import { ISearchSetup, ISearchStart, SearchEnhancements } from './search'; import { QuerySetup, QueryStart } from './query'; import { IndexPatternsContract } from './index_patterns'; -import { IndexPatternSelectProps, StatefulSearchBarProps } from './ui'; import { UsageCollectionSetup } from '../../usage_collection/public'; import { DataSourceStart } from './data_sources/datasource_services/types'; +import { IUiStart, UiEnhancements } from './ui'; export interface DataPublicPluginEnhancements { - search: SearchEnhancements; + search?: SearchEnhancements; + ui?: UiEnhancements; } export interface DataSetupDependencies { @@ -71,14 +71,6 @@ export interface DataPublicPluginSetup { __enhance: (enhancements: DataPublicPluginEnhancements) => void; } -/** - * Data plugin prewired UI components - */ -export interface DataPublicPluginStartUi { - IndexPatternSelect: React.ComponentType; - SearchBar: React.ComponentType; -} - /** * utilities to generate filters from action context */ @@ -122,10 +114,10 @@ export interface DataPublicPluginStart { */ query: QueryStart; /** - * prewired UI components - * {@link DataPublicPluginStartUi} + * UI components service + * {@link IUiStart} */ - ui: DataPublicPluginStartUi; + ui: IUiStart; /** * multiple datasources * {@link DataSourceStart} diff --git a/src/plugins/data/public/ui/index.ts b/src/plugins/data/public/ui/index.ts index c618df1783b5..98e6d393ce63 100644 --- a/src/plugins/data/public/ui/index.ts +++ b/src/plugins/data/public/ui/index.ts @@ -28,6 +28,7 @@ * under the License. */ +export { UiEnhancements, IUiStart, createSettings, Settings, DataSettings } from './types'; export { IndexPatternSelectProps } from './index_pattern_select'; export { FilterLabel } from './filter_bar'; export { QueryStringInput, QueryStringInputProps } from './query_string_input'; diff --git a/src/plugins/data/public/ui/mocks.ts b/src/plugins/data/public/ui/mocks.ts new file mode 100644 index 000000000000..47d3f059f504 --- /dev/null +++ b/src/plugins/data/public/ui/mocks.ts @@ -0,0 +1,46 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { SettingsMock } from './settings/mocks'; +import { IUiSetup, IUiStart } from './types'; + +const createMockWebStorage = () => ({ + clear: jest.fn(), + getItem: jest.fn(), + key: jest.fn(), + removeItem: jest.fn(), + setItem: jest.fn(), + length: 0, +}); + +const createMockStorage = () => ({ + storage: createMockWebStorage(), + get: jest.fn(), + set: jest.fn(), + remove: jest.fn(), + clear: jest.fn(), +}); + +function createSetupContract(): jest.Mocked { + return { + __enhance: jest.fn(), + }; +} + +function createStartContract(isEnhancementsEnabled: boolean = false): jest.Mocked { + const queryEnhancements = new Map(); + return { + isEnhancementsEnabled, + queryEnhancements, + IndexPatternSelect: jest.fn(), + SearchBar: jest.fn(), + Settings: new SettingsMock(createMockStorage(), queryEnhancements), + }; +} + +export const uiServiceMock = { + createSetupContract, + createStartContract, +}; diff --git a/src/plugins/data/public/ui/query_string_input/_index.scss b/src/plugins/data/public/ui/query_string_input/_index.scss index 8686490016c5..f21b9cbb4327 100644 --- a/src/plugins/data/public/ui/query_string_input/_index.scss +++ b/src/plugins/data/public/ui/query_string_input/_index.scss @@ -1 +1,2 @@ @import "./query_bar"; +@import "./language_switcher" diff --git a/src/plugins/data/public/ui/query_string_input/_language_switcher.scss b/src/plugins/data/public/ui/query_string_input/_language_switcher.scss new file mode 100644 index 000000000000..176d072c102b --- /dev/null +++ b/src/plugins/data/public/ui/query_string_input/_language_switcher.scss @@ -0,0 +1,8 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +.languageSelect { + max-width: 150px; + transform: translateY(-1px) translateX(-0.5px); +} diff --git a/src/plugins/data/public/ui/query_string_input/language_switcher.test.tsx b/src/plugins/data/public/ui/query_string_input/language_switcher.test.tsx index 22ec4e9edd96..dd1a3b4674cd 100644 --- a/src/plugins/data/public/ui/query_string_input/language_switcher.test.tsx +++ b/src/plugins/data/public/ui/query_string_input/language_switcher.test.tsx @@ -33,9 +33,28 @@ import { QueryLanguageSwitcher } from './language_switcher'; import { OpenSearchDashboardsContextProvider } from 'src/plugins/opensearch_dashboards_react/public'; import { coreMock } from '../../../../../core/public/mocks'; import { mountWithIntl } from 'test_utils/enzyme_helpers'; -import { EuiButtonEmpty, EuiPopover } from '@elastic/eui'; +import { EuiComboBox } from '@elastic/eui'; +import { QueryEnhancement } from '../types'; + const startMock = coreMock.createStart(); +jest.mock('../../services', () => ({ + getUiService: () => ({ + isEnhancementsEnabled: true, + queryEnhancements: new Map(), + Settings: { + setUiOverridesByUserQueryLanguage: jest.fn(), + }, + }), + getSearchService: () => ({ + __enhance: jest.fn(), + df: { + clear: jest.fn(), + }, + getDefaultSearchInterceptor: jest.fn(), + }), +})); + describe('LanguageSwitcher', () => { function wrapInContext(testProps: any) { const services = { @@ -50,7 +69,7 @@ describe('LanguageSwitcher', () => { ); } - it('should toggle off if language is lucene', () => { + it('should select lucene if language is lucene', () => { const component = mountWithIntl( wrapInContext({ language: 'lucene', @@ -59,12 +78,17 @@ describe('LanguageSwitcher', () => { }, }) ); - component.find(EuiButtonEmpty).simulate('click'); - expect(component.find(EuiPopover).prop('isOpen')).toBe(true); - expect(component.find('[data-test-subj="languageToggle"]').get(0).props.checked).toBeFalsy(); + const euiComboBox = component.find(EuiComboBox); + expect(euiComboBox.prop('selectedOptions')).toEqual( + expect.arrayContaining([ + { + label: 'Lucene', + }, + ]) + ); }); - it('should toggle on if language is kuery', () => { + it('should select DQL if language is kuery', () => { const component = mountWithIntl( wrapInContext({ language: 'kuery', @@ -73,8 +97,13 @@ describe('LanguageSwitcher', () => { }, }) ); - component.find(EuiButtonEmpty).simulate('click'); - expect(component.find(EuiPopover).prop('isOpen')).toBe(true); - expect(component.find('[data-test-subj="languageToggle"]').get(0).props.checked).toBeTruthy(); + const euiComboBox = component.find(EuiComboBox); + expect(euiComboBox.prop('selectedOptions')).toEqual( + expect.arrayContaining([ + { + label: 'DQL', + }, + ]) + ); }); }); diff --git a/src/plugins/data/public/ui/query_string_input/language_switcher.tsx b/src/plugins/data/public/ui/query_string_input/language_switcher.tsx index 816c21bc0848..a2d1f9ca41ce 100644 --- a/src/plugins/data/public/ui/query_string_input/language_switcher.tsx +++ b/src/plugins/data/public/ui/query_string_input/language_switcher.tsx @@ -28,116 +28,100 @@ * under the License. */ -import { - EuiButtonEmpty, - EuiForm, - EuiFormRow, - EuiLink, - EuiPopover, - EuiPopoverTitle, - EuiSpacer, - EuiSwitch, - EuiText, - PopoverAnchorPosition, -} from '@elastic/eui'; -import { FormattedMessage } from '@osd/i18n/react'; -import React, { useState } from 'react'; -import { useOpenSearchDashboards } from '../../../../opensearch_dashboards_react/public'; +import { EuiComboBox, EuiComboBoxOptionOption, PopoverAnchorPosition } from '@elastic/eui'; +import { i18n } from '@osd/i18n'; +import React from 'react'; +import { getSearchService, getUiService } from '../../services'; interface Props { language: string; onSelectLanguage: (newLanguage: string) => void; anchorPosition?: PopoverAnchorPosition; + appName?: string; +} + +function mapExternalLanguageToOptions(language: string) { + return { + label: language, + value: language, + }; } export function QueryLanguageSwitcher(props: Props) { - const osdDQLDocs = useOpenSearchDashboards().services.docLinks?.links.opensearchDashboards.dql - .base; - const [isPopoverOpen, setIsPopoverOpen] = useState(false); - const luceneLabel = ( - - ); - const dqlLabel = ( - - ); - const dqlFullName = ( - - ); + const dqlLabel = i18n.translate('data.query.queryBar.dqlLanguageName', { + defaultMessage: 'DQL', + }); + const luceneLabel = i18n.translate('data.query.queryBar.luceneLanguageName', { + defaultMessage: 'Lucene', + }); - const button = ( - setIsPopoverOpen(!isPopoverOpen)} - className="euiFormControlLayout__append dqlQueryBar__languageSwitcherButton" - data-test-subj={'switchQueryLanguageButton'} - > - {props.language === 'lucene' ? luceneLabel : dqlLabel} - - ); + const languageOptions: EuiComboBoxOptionOption[] = [ + { + label: dqlLabel, + value: 'kuery', + }, + { + label: luceneLabel, + value: 'lucene', + }, + ]; - return ( - setIsPopoverOpen(false)} - repositionOnScroll - > - - - -
- -

- - {dqlFullName} - - ), - }} - /> -

- + const uiService = getUiService(); + const searchService = getSearchService(); - + const queryEnhancements = uiService.queryEnhancements; + if (uiService.isEnhancementsEnabled) { + queryEnhancements.forEach((enhancement) => { + if ( + enhancement.supportedAppNames && + props.appName && + !enhancement.supportedAppNames.includes(props.appName) + ) + return; + languageOptions.push(mapExternalLanguageToOptions(enhancement.language)); + }); + } - - - - ) : ( - - ) - } - checked={props.language === 'kuery'} - onChange={() => { - const newLanguage = props.language === 'lucene' ? 'kuery' : 'lucene'; - props.onSelectLanguage(newLanguage); - }} - data-test-subj="languageToggle" - /> - - -
- + const selectedLanguage = { + label: + (languageOptions.find( + (option) => (option.value as string).toLowerCase() === props.language.toLowerCase() + )?.label as string) ?? languageOptions[0].label, + }; + + const setSearchEnhance = (queryLanguage: string) => { + if (!uiService.isEnhancementsEnabled) return; + const queryEnhancement = queryEnhancements.get(queryLanguage); + searchService.__enhance({ + searchInterceptor: queryEnhancement + ? queryEnhancement.search + : searchService.getDefaultSearchInterceptor(), + }); + + if (!queryEnhancement) { + searchService.df.clear(); + } + uiService.Settings.setUiOverridesByUserQueryLanguage(queryLanguage); + }; + + const handleLanguageChange = (newLanguage: EuiComboBoxOptionOption[]) => { + const queryLanguage = newLanguage[0].value as string; + props.onSelectLanguage(queryLanguage); + setSearchEnhance(queryLanguage); + }; + + setSearchEnhance(props.language); + + return ( + ); } diff --git a/src/plugins/data/public/ui/query_string_input/legacy_language_switcher.test.tsx b/src/plugins/data/public/ui/query_string_input/legacy_language_switcher.test.tsx new file mode 100644 index 000000000000..7e7e43190bb4 --- /dev/null +++ b/src/plugins/data/public/ui/query_string_input/legacy_language_switcher.test.tsx @@ -0,0 +1,55 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import React from 'react'; +import { LegacyQueryLanguageSwitcher } from './legacy_language_switcher'; +import { OpenSearchDashboardsContextProvider } from 'src/plugins/opensearch_dashboards_react/public'; +import { coreMock } from '../../../../../core/public/mocks'; +import { mountWithIntl } from 'test_utils/enzyme_helpers'; +import { EuiButtonEmpty, EuiPopover } from '@elastic/eui'; +const startMock = coreMock.createStart(); + +describe('LegacyLanguageSwitcher', () => { + function wrapInContext(testProps: any) { + const services = { + uiSettings: startMock.uiSettings, + docLinks: startMock.docLinks, + }; + + return ( + + + + ); + } + + it('should toggle off if language is lucene', () => { + const component = mountWithIntl( + wrapInContext({ + language: 'lucene', + onSelectLanguage: () => { + return; + }, + }) + ); + component.find(EuiButtonEmpty).simulate('click'); + expect(component.find(EuiPopover).prop('isOpen')).toBe(true); + expect(component.find('[data-test-subj="languageToggle"]').get(0).props.checked).toBeFalsy(); + }); + + it('should toggle on if language is kuery', () => { + const component = mountWithIntl( + wrapInContext({ + language: 'kuery', + onSelectLanguage: () => { + return; + }, + }) + ); + component.find(EuiButtonEmpty).simulate('click'); + expect(component.find(EuiPopover).prop('isOpen')).toBe(true); + expect(component.find('[data-test-subj="languageToggle"]').get(0).props.checked).toBeTruthy(); + }); +}); diff --git a/src/plugins/data/public/ui/query_string_input/legacy_language_switcher.tsx b/src/plugins/data/public/ui/query_string_input/legacy_language_switcher.tsx new file mode 100644 index 000000000000..b8128a950d33 --- /dev/null +++ b/src/plugins/data/public/ui/query_string_input/legacy_language_switcher.tsx @@ -0,0 +1,118 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { + EuiButtonEmpty, + EuiForm, + EuiFormRow, + EuiLink, + EuiPopover, + EuiPopoverTitle, + EuiSpacer, + EuiSwitch, + EuiText, + PopoverAnchorPosition, +} from '@elastic/eui'; +import { FormattedMessage } from '@osd/i18n/react'; +import React, { useState } from 'react'; +import { useOpenSearchDashboards } from '../../../../opensearch_dashboards_react/public'; + +interface Props { + language: string; + onSelectLanguage: (newLanguage: string) => void; + anchorPosition?: PopoverAnchorPosition; +} + +export function LegacyQueryLanguageSwitcher(props: Props) { + const osdDQLDocs = useOpenSearchDashboards().services.docLinks?.links.opensearchDashboards.dql + .base; + const [isPopoverOpen, setIsPopoverOpen] = useState(false); + const luceneLabel = ( + + ); + const dqlLabel = ( + + ); + const dqlFullName = ( + + ); + + const button = ( + setIsPopoverOpen(!isPopoverOpen)} + className="euiFormControlLayout__append dqlQueryBar__languageSwitcherButton" + data-test-subj={'switchQueryLanguageButton'} + > + {props.language === 'lucene' ? luceneLabel : dqlLabel} + + ); + + return ( + setIsPopoverOpen(false)} + repositionOnScroll + > + + + +
+ +

+ + {dqlFullName} + + ), + }} + /> +

+ + + + + + + + ) : ( + + ) + } + checked={props.language === 'kuery'} + onChange={() => { + const newLanguage = props.language === 'lucene' ? 'kuery' : 'lucene'; + props.onSelectLanguage(newLanguage); + }} + data-test-subj="languageToggle" + /> + + +
+ + ); +} diff --git a/src/plugins/data/public/ui/query_string_input/query_bar_top_row.test.tsx b/src/plugins/data/public/ui/query_string_input/query_bar_top_row.test.tsx index 3b8c41eb1a39..fa194054930a 100644 --- a/src/plugins/data/public/ui/query_string_input/query_bar_top_row.test.tsx +++ b/src/plugins/data/public/ui/query_string_input/query_bar_top_row.test.tsx @@ -70,6 +70,8 @@ startMock.uiSettings.get.mockImplementation((key: string) => { from: 'now-15m', to: 'now', }; + case UI_SETTINGS.QUERY_DATA_SOURCE_READONLY: + return false; default: throw new Error(`Unexpected config key: ${key}`); } diff --git a/src/plugins/data/public/ui/query_string_input/query_bar_top_row.tsx b/src/plugins/data/public/ui/query_string_input/query_bar_top_row.tsx index 8c509d573e1b..027c90a6c798 100644 --- a/src/plugins/data/public/ui/query_string_input/query_bar_top_row.tsx +++ b/src/plugins/data/public/ui/query_string_input/query_bar_top_row.tsx @@ -46,6 +46,7 @@ import { import { EuiSuperUpdateButton, OnRefreshProps } from '@elastic/eui'; import { FormattedMessage } from '@osd/i18n/react'; import { Toast } from 'src/core/public'; +import { isEqual, compact } from 'lodash'; import { IDataPluginServices, IIndexPattern, TimeRange, TimeHistoryContract, Query } from '../..'; import { useOpenSearchDashboards, @@ -54,14 +55,18 @@ import { } from '../../../../opensearch_dashboards_react/public'; import QueryStringInputUI from './query_string_input'; import { doesKueryExpressionHaveLuceneSyntaxError, UI_SETTINGS } from '../../../common'; -import { PersistedLog, getQueryLog } from '../../query'; +import { PersistedLog, fromUser, getQueryLog } from '../../query'; import { NoDataPopover } from './no_data_popover'; +import { QueryEnhancement, Settings } from '../types'; const QueryStringInput = withOpenSearchDashboards(QueryStringInputUI); // @internal export interface QueryBarTopRowProps { query?: Query; + isEnhancementsEnabled?: boolean; + queryEnhancements?: Map; + settings?: Settings; onSubmit: (payload: { dateRange: TimeRange; query?: Query }) => void; onChange: (payload: { dateRange: TimeRange; query?: Query }) => void; onRefresh?: (payload: { dateRange: TimeRange }) => void; @@ -95,8 +100,22 @@ export default function QueryBarTopRow(props: QueryBarTopRowProps) { const { uiSettings, notifications, storage, appName, docLinks } = opensearchDashboards.services; const osdDQLDocs: string = docLinks!.links.opensearchDashboards.dql.base; + const isDataSourceReadOnly = uiSettings.get('query:dataSourceReadOnly'); const queryLanguage = props.query && props.query.language; + const queryUiEnhancement = + (queryLanguage && + props.queryEnhancements && + props.queryEnhancements.get(queryLanguage)?.searchBar) || + null; + const parsedQuery = + !queryUiEnhancement || isValidQuery(props.query) + ? props.query! + : { query: getQueryStringInitialValue(queryLanguage!), language: queryLanguage! }; + if (!isEqual(parsedQuery?.query, props.query?.query)) { + onQueryChange(parsedQuery); + onSubmit({ query: parsedQuery, dateRange: getDateRange() }); + } const persistedLog: PersistedLog | undefined = React.useMemo( () => queryLanguage && uiSettings && storage && appName @@ -116,15 +135,19 @@ export default function QueryBarTopRow(props: QueryBarTopRowProps) { function getDateRange() { const defaultTimeSetting = uiSettings!.get(UI_SETTINGS.TIMEPICKER_TIME_DEFAULTS); return { - from: props.dateRangeFrom || defaultTimeSetting.from, - to: props.dateRangeTo || defaultTimeSetting.to, + from: + props.dateRangeFrom || + queryUiEnhancement?.dateRange?.initialFrom || + defaultTimeSetting.from, + to: props.dateRangeTo || queryUiEnhancement?.dateRange?.initialTo || defaultTimeSetting.to, }; } - function onQueryChange(query: Query) { + function onQueryChange(query: Query, dateRange?: TimeRange) { + if (queryUiEnhancement && !isValidQuery(query)) return; props.onChange({ query, - dateRange: getDateRange(), + dateRange: dateRange ?? getDateRange(), }); } @@ -181,10 +204,10 @@ export default function QueryBarTopRow(props: QueryBarTopRowProps) { props.onSubmit({ query, dateRange }); } - function onInputSubmit(query: Query) { + function onInputSubmit(query: Query, dateRange?: TimeRange) { onSubmit({ query, - dateRange: getDateRange(), + dateRange: dateRange ?? getDateRange(), }); } @@ -196,6 +219,38 @@ export default function QueryBarTopRow(props: QueryBarTopRowProps) { return valueAsMoment.toISOString(); } + function isValidQuery(query: Query | undefined) { + if (!query || !query.query) return false; + return ( + !Array.isArray(props.indexPatterns!) || + compact(props.indexPatterns!).length === 0 || + !isDataSourceReadOnly || + fromUser(query!.query).includes( + typeof props.indexPatterns[0] === 'string' + ? props.indexPatterns[0] + : props.indexPatterns[0].title + ) + ); + } + + function getQueryStringInitialValue(language: string) { + const { indexPatterns, queryEnhancements } = props; + const input = queryEnhancements?.get(language)?.searchBar?.queryStringInput?.initialValue; + + if ( + !indexPatterns || + (!Array.isArray(indexPatterns) && compact(indexPatterns).length > 0) || + !input + ) + return ''; + + const defaultDataSource = indexPatterns[0]; + const dataSource = + typeof defaultDataSource === 'string' ? defaultDataSource : defaultDataSource.title; + + return input.replace('', dataSource); + } + function renderQueryInput() { if (!shouldRenderQueryInput()) return; return ( @@ -204,11 +259,15 @@ export default function QueryBarTopRow(props: QueryBarTopRowProps) { disableAutoFocus={props.disableAutoFocus} indexPatterns={props.indexPatterns!} prepend={props.prepend} - query={props.query!} + query={parsedQuery} + isEnhancementsEnabled={props.isEnhancementsEnabled} + queryEnhancements={props.queryEnhancements} + settings={props.settings} screenTitle={props.screenTitle} onChange={onQueryChange} onChangeQueryInputFocus={onChangeQueryInputFocus} onSubmit={onInputSubmit} + getQueryStringInitialValue={getQueryStringInitialValue} persistedLog={persistedLog} dataTestSubj={props.dataTestSubj} /> @@ -233,10 +292,15 @@ export default function QueryBarTopRow(props: QueryBarTopRowProps) { } function shouldRenderDatePicker(): boolean { - return Boolean(props.showDatePicker || props.showAutoRefreshOnly); + return Boolean( + (props.showDatePicker && (queryUiEnhancement?.showDatePicker ?? true)) ?? + (props.showAutoRefreshOnly && (queryUiEnhancement?.showAutoRefreshOnly ?? true)) + ); } function shouldRenderQueryInput(): boolean { + // TODO: MQL probably can modify to not care about index patterns + // TODO: call queryUiEnhancement?.showQueryInput return Boolean(props.showQueryInput && props.indexPatterns && props.query && storage); } diff --git a/src/plugins/data/public/ui/query_string_input/query_string_input.test.tsx b/src/plugins/data/public/ui/query_string_input/query_string_input.test.tsx index dfa5d57411d0..da5cc0e017b2 100644 --- a/src/plugins/data/public/ui/query_string_input/query_string_input.test.tsx +++ b/src/plugins/data/public/ui/query_string_input/query_string_input.test.tsx @@ -42,7 +42,7 @@ import { render } from '@testing-library/react'; import { EuiTextArea } from '@elastic/eui'; -import { QueryLanguageSwitcher } from './language_switcher'; +import { LegacyQueryLanguageSwitcher } from './legacy_language_switcher'; import { QueryStringInput } from './'; import type QueryStringInputUI from './query_string_input'; @@ -50,6 +50,7 @@ import { coreMock } from '../../../../../core/public/mocks'; import { dataPluginMock } from '../../mocks'; import { stubIndexPatternWithFields } from '../../stubs'; import { OpenSearchDashboardsContextProvider } from 'src/plugins/opensearch_dashboards_react/public'; +import { SettingsMock } from '../settings/mocks'; const startMock = coreMock.createStart(); @@ -84,6 +85,8 @@ const createMockStorage = () => ({ clear: jest.fn(), }); +const settingsMock = new SettingsMock(createMockStorage(), new Map()); + function wrapQueryStringInputInContext(testProps: any, storage?: any) { const defaultOptions = { screenTitle: 'Another Screen', @@ -132,7 +135,7 @@ describe('QueryStringInput', () => { indexPatterns: [stubIndexPatternWithFields], }) ); - expect(component.find(QueryLanguageSwitcher).prop('language')).toBe(luceneQuery.language); + expect(component.find(LegacyQueryLanguageSwitcher).prop('language')).toBe(luceneQuery.language); }); it('Should disable autoFocus on EuiTextArea when disableAutoFocus prop is true', () => { @@ -173,16 +176,17 @@ describe('QueryStringInput', () => { indexPatterns: [stubIndexPatternWithFields], disableAutoFocus: true, appName: 'discover', + settings: settingsMock, }, mockStorage ) ); - component.find(QueryLanguageSwitcher).props().onSelectLanguage('lucene'); - expect(mockStorage.set).toHaveBeenCalledWith( - 'opensearchDashboards.userQueryLanguage', - 'lucene' - ); + component.find(LegacyQueryLanguageSwitcher).props().onSelectLanguage('lucene'); + expect(settingsMock.updateSettings).toHaveBeenCalledWith({ + userQueryLanguage: 'lucene', + userQueryString: '', + }); expect(mockCallback).toHaveBeenCalledWith({ query: '', language: 'lucene' }); }); diff --git a/src/plugins/data/public/ui/query_string_input/query_string_input.tsx b/src/plugins/data/public/ui/query_string_input/query_string_input.tsx index 5d071748700c..db0d732d1db6 100644 --- a/src/plugins/data/public/ui/query_string_input/query_string_input.tsx +++ b/src/plugins/data/public/ui/query_string_input/query_string_input.tsx @@ -47,7 +47,7 @@ import { import { FormattedMessage } from '@osd/i18n/react'; import { debounce, compact, isEqual, isFunction } from 'lodash'; import { Toast } from 'src/core/public'; -import { IDataPluginServices, IIndexPattern, Query } from '../..'; +import { IDataPluginServices, IIndexPattern, Query, TimeRange } from '../..'; import { QuerySuggestion, QuerySuggestionTypes } from '../../autocomplete'; import { @@ -56,13 +56,18 @@ import { } from '../../../../opensearch_dashboards_react/public'; import { fetchIndexPatterns } from './fetch_index_patterns'; import { QueryLanguageSwitcher } from './language_switcher'; +import { LegacyQueryLanguageSwitcher } from './legacy_language_switcher'; import { PersistedLog, getQueryLog, matchPairs, toUser, fromUser } from '../../query'; import { SuggestionsListSize } from '../typeahead/suggestions_component'; -import { SuggestionsComponent } from '..'; +import { Settings, SuggestionsComponent } from '..'; +import { DataSettings, QueryEnhancement } from '../types'; export interface QueryStringInputProps { indexPatterns: Array; query: Query; + isEnhancementsEnabled?: boolean; + queryEnhancements?: Map; + settings?: Settings; disableAutoFocus?: boolean; screenTitle?: string; prepend?: any; @@ -71,9 +76,10 @@ export interface QueryStringInputProps { placeholder?: string; languageSwitcherPopoverAnchorPosition?: PopoverAnchorPosition; onBlur?: () => void; - onChange?: (query: Query) => void; + onChange?: (query: Query, dateRange?: TimeRange) => void; onChangeQueryInputFocus?: (isFocused: boolean) => void; - onSubmit?: (query: Query) => void; + onSubmit?: (query: Query, dateRange?: TimeRange) => void; + getQueryStringInitialValue?: (language: string) => string; dataTestSubj?: string; size?: SuggestionsListSize; className?: string; @@ -108,6 +114,7 @@ const KEY_CODES = { }; // Needed for React.lazy +// TODO: MQL export this and let people extended this // eslint-disable-next-line import/no-default-export export default class QueryStringInputUI extends Component { public state: State = { @@ -130,9 +137,13 @@ export default class QueryStringInputUI extends Component { private queryBarInputDivRefInstance: RefObject = createRef(); private getQueryString = () => { + if (!this.props.query.query) { + return this.props.getQueryStringInitialValue?.(this.props.query.language) ?? ''; + } return toUser(this.props.query.query); }; + // TODO: MQL don't do this here? || Fetch data sources private fetchIndexPatterns = async () => { const stringPatterns = this.props.indexPatterns.filter( (indexPattern) => typeof indexPattern === 'string' @@ -224,7 +235,7 @@ export default class QueryStringInputUI extends Component { } }, 100); - private onSubmit = (query: Query) => { + private onSubmit = (query: Query, dateRange?: TimeRange) => { if (this.props.onSubmit) { if (this.persistedLog) { this.persistedLog.add(query.query); @@ -234,11 +245,11 @@ export default class QueryStringInputUI extends Component { } }; - private onChange = (query: Query) => { + private onChange = (query: Query, dateRange?: TimeRange) => { this.updateSuggestions(); if (this.props.onChange) { - this.props.onChange({ query: fromUser(query.query), language: query.language }); + this.props.onChange({ query: fromUser(query.query), language: query.language }, dateRange); } }; @@ -457,6 +468,7 @@ export default class QueryStringInputUI extends Component { } }; + // TODO: MQL consider moving language select language of setting search source here private onSelectLanguage = (language: string) => { // Send telemetry info every time the user opts in or out of kuery // As a result it is important this function only ever gets called in the @@ -465,11 +477,28 @@ export default class QueryStringInputUI extends Component { body: JSON.stringify({ opt_in: language === 'kuery' }), }); - this.services.storage.set('opensearchDashboards.userQueryLanguage', language); + const newQuery = { + query: this.props.getQueryStringInitialValue?.(language) ?? '', + language, + }; - const newQuery = { query: '', language }; - this.onChange(newQuery); - this.onSubmit(newQuery); + const fields = this.props.queryEnhancements?.get(newQuery.language)?.fields; + const newSettings: DataSettings = { + userQueryLanguage: newQuery.language, + userQueryString: newQuery.query, + ...(fields && { uiOverrides: { fields } }), + }; + this.props.settings?.updateSettings(newSettings); + + const dateRangeEnhancement = this.props.queryEnhancements?.get(language)?.searchBar?.dateRange; + const dateRange = dateRangeEnhancement + ? { + from: dateRangeEnhancement.initialFrom!, + to: dateRangeEnhancement.initialTo!, + } + : undefined; + this.onChange(newQuery, dateRange); + this.onSubmit(newQuery, dateRange); }; private onOutsideClick = () => { @@ -619,6 +648,14 @@ export default class QueryStringInputUI extends Component { return (
{this.props.prepend} + {!!this.props.isEnhancementsEnabled && ( + + )}
{
- - + {!!!this.props.isEnhancementsEnabled && ( + + )}
); } diff --git a/src/plugins/data/public/ui/search_bar/create_search_bar.tsx b/src/plugins/data/public/ui/search_bar/create_search_bar.tsx index c739b955ff19..b2fdea2b49c9 100644 --- a/src/plugins/data/public/ui/search_bar/create_search_bar.tsx +++ b/src/plugins/data/public/ui/search_bar/create_search_bar.tsx @@ -41,11 +41,15 @@ import { useSavedQuery } from './lib/use_saved_query'; import { DataPublicPluginStart } from '../../types'; import { Filter, Query, TimeRange } from '../../../common'; import { useQueryStringManager } from './lib/use_query_string_manager'; +import { QueryEnhancement, Settings } from '../types'; interface StatefulSearchBarDeps { core: CoreStart; data: Omit; storage: IStorageWrapper; + isEnhancementsEnabled: boolean; + queryEnhancements: Map; + settings: Settings; } export type StatefulSearchBarProps = SearchBarOwnProps & { @@ -130,11 +134,19 @@ const overrideDefaultBehaviors = (props: StatefulSearchBarProps) => { return props.useDefaultBehaviors ? {} : props; }; -export function createSearchBar({ core, storage, data }: StatefulSearchBarDeps) { +export function createSearchBar({ + core, + storage, + data, + isEnhancementsEnabled, + queryEnhancements, + settings, +}: StatefulSearchBarDeps) { // App name should come from the core application service. // Until it's available, we'll ask the user to provide it for the pre-wired component. return (props: StatefulSearchBarProps) => { const { useDefaultBehaviors } = props; + // Handle queries const onQuerySubmitRef = useRef(props.onQuerySubmit); @@ -148,6 +160,7 @@ export function createSearchBar({ core, storage, data }: StatefulSearchBarDeps) query: props.query, queryStringManager: data.query.queryString, }); + const { timeRange, refreshInterval } = useTimefilter({ dateRangeFrom: props.dateRangeFrom, dateRangeTo: props.dateRangeTo, @@ -201,6 +214,9 @@ export function createSearchBar({ core, storage, data }: StatefulSearchBarDeps) isRefreshPaused={refreshInterval.pause} filters={filters} query={query} + isEnhancementsEnabled={isEnhancementsEnabled} + queryEnhancements={queryEnhancements} + settings={settings} onFiltersUpdated={defaultFiltersUpdated(data.query)} onRefreshChange={defaultOnRefreshChange(data.query)} savedQuery={savedQuery} diff --git a/src/plugins/data/public/ui/search_bar/search_bar.tsx b/src/plugins/data/public/ui/search_bar/search_bar.tsx index b05b18b6d64e..937da74914a0 100644 --- a/src/plugins/data/public/ui/search_bar/search_bar.tsx +++ b/src/plugins/data/public/ui/search_bar/search_bar.tsx @@ -47,6 +47,7 @@ import { TimeRange, Query, Filter, IIndexPattern } from '../../../common'; import { FilterBar } from '../filter_bar/filter_bar'; import { SavedQueryMeta, SaveQueryForm } from '../saved_query_form'; import { SavedQueryManagementComponent } from '../saved_query_management'; +import { QueryEnhancement, Settings } from '../types'; interface SearchBarInjectedDeps { opensearchDashboards: OpenSearchDashboardsReactContextValue; @@ -78,6 +79,9 @@ export interface SearchBarOwnProps { dateRangeTo?: string; // Query bar - should be in SearchBarInjectedDeps query?: Query; + isEnhancementsEnabled?: boolean; + queryEnhancements?: Map; + settings?: Settings; // Show when user has privileges to save showSaveQuery?: boolean; savedQuery?: SavedQuery; @@ -96,6 +100,7 @@ export interface SearchBarOwnProps { export type SearchBarProps = SearchBarOwnProps & SearchBarInjectedDeps; +// TODO: MQL: include query enhancement in state in case make adding data sources at runtime? interface State { isFiltersVisible: boolean; showSaveQueryModal: boolean; @@ -202,6 +207,7 @@ class SearchBarUI extends Component { }; private shouldRenderQueryBar() { + // TODO: MQL handle no index patterns? const showDatePicker = this.props.showDatePicker || this.props.showAutoRefreshOnly; const showQueryInput = this.props.showQueryInput && this.props.indexPatterns && this.state.query; @@ -209,11 +215,14 @@ class SearchBarUI extends Component { } private shouldRenderFilterBar() { + // TODO: MQL handle no index patterns? return ( this.props.showFilterBar && this.props.filters && this.props.indexPatterns && - compact(this.props.indexPatterns).length > 0 + compact(this.props.indexPatterns).length > 0 && + (this.props.queryEnhancements?.get(this.state.query?.language!)?.searchBar?.showFilterBar ?? + true) ); } @@ -393,9 +402,13 @@ class SearchBarUI extends Component { let queryBar; if (this.shouldRenderQueryBar()) { + // TODO: MQL make this default query bar top row but this.props.queryEnhancements.get(language) can pass a component queryBar = ( + ) {} + + getUserQueryLanguage() { + return this.storage.get('opensearchDashboards.userQueryLanguage') || 'kuery'; + } + + setUserQueryLanguage(language: string) { + this.storage.set('opensearchDashboards.userQueryLanguage', language); + return true; + } + + getUserQueryString() { + return this.storage.get('opensearchDashboards.userQueryString') || ''; + } + + setUserQueryString(query: string) { + this.storage.set('opensearchDashboards.userQueryString', query); + return true; + } + + getUiOverrides() { + return this.storage.get('opensearchDashboards.uiOverrides') || {}; + } + + setUiOverrides(overrides?: { [key: string]: any }) { + if (!overrides) { + this.storage.remove('opensearchDashboards.uiOverrides'); + setFieldOverrides(undefined); + return true; + } + this.storage.set('opensearchDashboards.uiOverrides', overrides); + setFieldOverrides(overrides.fields); + return true; + } + + setUiOverridesByUserQueryLanguage(language: string) { + const queryEnhancement = this.queryEnhancements.get(language); + if (queryEnhancement) { + const { fields = {}, showDocLinks } = queryEnhancement; + this.setUiOverrides({ fields, showDocLinks }); + } else { + this.setUiOverrides({ fields: undefined, showDocLinks: undefined }); + } + } + + toJSON(): DataSettings { + return { + userQueryLanguage: this.getUserQueryLanguage(), + userQueryString: this.getUserQueryString(), + uiOverrides: this.getUiOverrides(), + }; + } + + updateSettings({ userQueryLanguage, userQueryString, uiOverrides }: DataSettings) { + this.setUserQueryLanguage(userQueryLanguage); + this.setUserQueryString(userQueryString); + this.setUiOverrides(uiOverrides); + } +} + +interface Deps { + storage: IStorageWrapper; + queryEnhancements: Map; +} + +export function createSettings({ storage, queryEnhancements }: Deps) { + return new Settings(storage, queryEnhancements); +} diff --git a/src/plugins/data/public/ui/types.ts b/src/plugins/data/public/ui/types.ts new file mode 100644 index 000000000000..464e6a97afde --- /dev/null +++ b/src/plugins/data/public/ui/types.ts @@ -0,0 +1,65 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { SearchInterceptor } from '../search'; +import { IndexPatternSelectProps } from './index_pattern_select'; +import { StatefulSearchBarProps } from './search_bar'; +import { Settings } from './settings'; + +export * from './settings'; + +export interface QueryEnhancement { + // TODO: MQL do want to default have supported all data_sources? + // or should data connect have a record of query enhancements that are supported + language: string; + search: SearchInterceptor; + // Leave blank to support all data sources + // supportedDataSourceTypes?: Record; + searchBar?: { + showQueryInput?: boolean; + showFilterBar?: boolean; + showDatePicker?: boolean; + showAutoRefreshOnly?: boolean; + queryStringInput?: { + // will replace '' with the data source name + initialValue?: string; + }; + dateRange?: { + initialFrom?: string; + initialTo?: string; + }; + }; + fields?: { + filterable?: boolean; + visualizable?: boolean; + }; + showDocLinks?: boolean; + // List of supported app names that this enhancement should be enabled for, + // if not provided it will be enabled for all apps + supportedAppNames?: string[]; +} + +export interface UiEnhancements { + query?: QueryEnhancement; +} + +/** + * The setup contract exposed by the Search plugin exposes the search strategy extension + * point. + */ +export interface IUiSetup { + __enhance: (enhancements: UiEnhancements) => void; +} + +/** + * Data plugin prewired UI components + */ +export interface IUiStart { + isEnhancementsEnabled: boolean; + queryEnhancements: Map; + IndexPatternSelect: React.ComponentType; + SearchBar: React.ComponentType; + Settings: Settings; +} diff --git a/src/plugins/data/public/ui/ui_service.ts b/src/plugins/data/public/ui/ui_service.ts new file mode 100644 index 000000000000..1ef834b54564 --- /dev/null +++ b/src/plugins/data/public/ui/ui_service.ts @@ -0,0 +1,70 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { Plugin, CoreSetup, CoreStart, PluginInitializerContext } from 'src/core/public'; +import { IUiStart, IUiSetup, QueryEnhancement, UiEnhancements } from './types'; + +import { ConfigSchema } from '../../config'; +import { createIndexPatternSelect } from './index_pattern_select'; +import { createSearchBar } from './search_bar/create_search_bar'; +import { createSettings } from './settings'; +import { DataPublicPluginStart } from '../types'; +import { IStorageWrapper } from '../../../opensearch_dashboards_utils/public'; + +/** @internal */ +// eslint-disable-next-line @typescript-eslint/no-empty-interface +export interface UiServiceSetupDependencies {} + +/** @internal */ +export interface UiServiceStartDependencies { + dataServices: Omit; + storage: IStorageWrapper; +} + +export class UiService implements Plugin { + enhancementsConfig: ConfigSchema['enhancements']; + private queryEnhancements: Map = new Map(); + + constructor(initializerContext: PluginInitializerContext) { + const { enhancements } = initializerContext.config.get(); + + this.enhancementsConfig = enhancements; + } + + public setup(core: CoreSetup, {}: UiServiceSetupDependencies): IUiSetup { + return { + __enhance: (enhancements?: UiEnhancements) => { + if (!enhancements) return; + if (!this.enhancementsConfig.enabled) return; + if (enhancements.query && enhancements.query.language) { + this.queryEnhancements.set(enhancements.query.language, enhancements.query); + } + }, + }; + } + + public start(core: CoreStart, { dataServices, storage }: UiServiceStartDependencies): IUiStart { + const Settings = createSettings({ storage, queryEnhancements: this.queryEnhancements }); + + const SearchBar = createSearchBar({ + core, + data: dataServices, + storage, + isEnhancementsEnabled: this.enhancementsConfig?.enabled, + queryEnhancements: this.queryEnhancements, + settings: Settings, + }); + + return { + isEnhancementsEnabled: this.enhancementsConfig?.enabled, + queryEnhancements: this.queryEnhancements, + IndexPatternSelect: createIndexPatternSelect(core.savedObjects.client), + SearchBar, + Settings, + }; + } + + public stop() {} +} diff --git a/src/plugins/data/server/index.ts b/src/plugins/data/server/index.ts index 4bc3ad62a4ae..5fe531729283 100644 --- a/src/plugins/data/server/index.ts +++ b/src/plugins/data/server/index.ts @@ -133,11 +133,13 @@ export { IFieldFormatsRegistry, FieldFormatsGetConfigFn, FieldFormatConfig } fro * Index patterns: */ -import { isNestedField, isFilterable } from '../common'; +import { isNestedField, isFilterable, setOverrides, getOverrides } from '../common'; export const indexPatterns = { isFilterable, isNestedField, + setOverrides, + getOverrides, }; export { @@ -272,6 +274,8 @@ export const search = { export { // osd field types castOpenSearchToOsdFieldTypeName, + getOsdFieldOverrides, + setOsdFieldOverrides, // query Filter, getTime, @@ -300,6 +304,7 @@ export { export const config: PluginConfigDescriptor = { exposeToBrowser: { + enhancements: true, autocomplete: true, search: true, }, diff --git a/src/plugins/data/server/search/opensearch_search/get_default_search_params.ts b/src/plugins/data/server/search/opensearch_search/get_default_search_params.ts index d7cbd48a6507..172acf9f0e2e 100644 --- a/src/plugins/data/server/search/opensearch_search/get_default_search_params.ts +++ b/src/plugins/data/server/search/opensearch_search/get_default_search_params.ts @@ -45,10 +45,14 @@ export async function getDefaultSearchParams(uiSettingsClient: IUiSettingsClient const maxConcurrentShardRequests = await uiSettingsClient.get( UI_SETTINGS.COURIER_MAX_CONCURRENT_SHARD_REQUESTS ); + const dataFrameHydrationStrategy = await uiSettingsClient.get( + UI_SETTINGS.DATAFRAME_HYDRATION_STRATEGY + ); return { maxConcurrentShardRequests: maxConcurrentShardRequests > 0 ? maxConcurrentShardRequests : undefined, ignoreThrottled, + dataFrameHydrationStrategy, ignoreUnavailable: true, // Don't fail if the index/indices don't exist trackTotalHits: true, }; diff --git a/src/plugins/data/server/search/opensearch_search/opensearch_search_strategy.ts b/src/plugins/data/server/search/opensearch_search/opensearch_search_strategy.ts index fa1b3e4da94c..c5c7602bc4f9 100644 --- a/src/plugins/data/server/search/opensearch_search/opensearch_search_strategy.ts +++ b/src/plugins/data/server/search/opensearch_search/opensearch_search_strategy.ts @@ -64,8 +64,12 @@ export const opensearchSearchStrategyProvider = ( throw new Error(`Unsupported index pattern type ${request.indexType}`); } - // ignoreThrottled is not supported in OSS - const { ignoreThrottled, ...defaultParams } = await getDefaultSearchParams(uiSettingsClient); + // ignoreThrottled & dataFrameHydrationStrategy is not supported by default + const { + ignoreThrottled, + dataFrameHydrationStrategy, + ...defaultParams + } = await getDefaultSearchParams(uiSettingsClient); const params = toSnakeCase({ ...defaultParams, diff --git a/src/plugins/data/server/search/routes/call_msearch.ts b/src/plugins/data/server/search/routes/call_msearch.ts index dc257620fcca..b6bca0875d40 100644 --- a/src/plugins/data/server/search/routes/call_msearch.ts +++ b/src/plugins/data/server/search/routes/call_msearch.ts @@ -80,8 +80,12 @@ export function getCallMsearch(dependencies: CallMsearchDependencies) { const config = await globalConfig$.pipe(first()).toPromise(); const timeout = getShardTimeout(config); - // trackTotalHits is not supported by msearch - const { trackTotalHits, ...defaultParams } = await getDefaultSearchParams(uiSettings); + // trackTotalHits and dataFrameHydrationStrategy is not supported by msearch + const { + trackTotalHits, + dataFrameHydrationStrategy, + ...defaultParams + } = await getDefaultSearchParams(uiSettings); const body = convertRequestBody(params.body, timeout); diff --git a/src/plugins/data/server/search/search_service.ts b/src/plugins/data/server/search/search_service.ts index b955596922a0..2eef461b94da 100644 --- a/src/plugins/data/server/search/search_service.ts +++ b/src/plugins/data/server/search/search_service.ts @@ -73,6 +73,13 @@ import { } from '../../common/search/aggs/buckets/shard_delay'; import { aggShardDelay } from '../../common/search/aggs/buckets/shard_delay_fn'; import { ConfigSchema } from '../../config'; +import { + DataFrameService, + IDataFrame, + IDataFrameResponse, + createDataFrameCache, + dataFrameToSpec, +} from '../../common'; type StrategyMap = Record>; @@ -98,6 +105,7 @@ export interface SearchRouteDependencies { export class SearchService implements Plugin { private readonly aggsService = new AggsService(); private readonly searchSourceService = new SearchSourceService(); + private readonly dfCache = createDataFrameCache(); private defaultSearchStrategyName: string = OPENSEARCH_SEARCH_STRATEGY; private searchStrategies: StrategyMap = {}; @@ -166,7 +174,8 @@ export class SearchService implements Plugin { }); return { - __enhance: (enhancements: SearchEnhancements) => { + __enhance: (enhancements?: SearchEnhancements) => { + if (!enhancements) return; if (this.searchStrategies.hasOwnProperty(enhancements.defaultStrategy)) { this.defaultSearchStrategyName = enhancements.defaultStrategy; } @@ -203,6 +212,29 @@ export class SearchService implements Plugin { searchSourceRequiredUiSettings ); + const dfService: DataFrameService = { + get: () => this.dfCache.get(), + set: async (dataFrame: IDataFrame) => { + if (this.dfCache.get() && this.dfCache.get()?.name !== dataFrame.name) { + scopedIndexPatterns.clearCache(this.dfCache.get()!.name, false); + } + this.dfCache.set(dataFrame); + const existingIndexPattern = scopedIndexPatterns.getByTitle(dataFrame.name!, true); + const dataSet = await scopedIndexPatterns.create( + dataFrameToSpec(dataFrame, existingIndexPattern?.id), + !existingIndexPattern?.id + ); + // save to cache by title because the id is not unique for temporary index pattern created + scopedIndexPatterns.saveToCache(dataSet.title, dataSet); + }, + clear: () => { + if (this.dfCache.get() === undefined) return; + // name because the id is not unique for temporary index pattern created + scopedIndexPatterns.clearCache(this.dfCache.get()!.name, false); + this.dfCache.clear(); + }, + }; + const searchSourceDependencies: SearchSourceDependencies = { getConfig: (key: string): T => uiSettingsCache[key], search: (searchRequest, options) => { @@ -237,6 +269,7 @@ export class SearchService implements Plugin { }), loadingCount$: new BehaviorSubject(0), }, + df: dfService, }; return this.searchSourceService.start(scopedIndexPatterns, searchSourceDependencies); @@ -251,7 +284,9 @@ export class SearchService implements Plugin { private registerSearchStrategy = < SearchStrategyRequest extends IOpenSearchDashboardsSearchRequest = IOpenSearchSearchRequest, - SearchStrategyResponse extends IOpenSearchDashboardsSearchResponse = IOpenSearchSearchResponse + SearchStrategyResponse extends + | IOpenSearchDashboardsSearchResponse + | IDataFrameResponse = IOpenSearchSearchResponse >( name: string, strategy: ISearchStrategy @@ -261,7 +296,9 @@ export class SearchService implements Plugin { private search = < SearchStrategyRequest extends IOpenSearchDashboardsSearchRequest = IOpenSearchSearchRequest, - SearchStrategyResponse extends IOpenSearchDashboardsSearchResponse = IOpenSearchSearchResponse + SearchStrategyResponse extends + | IOpenSearchDashboardsSearchResponse + | IDataFrameResponse = IOpenSearchSearchResponse >( context: RequestHandlerContext, searchRequest: SearchStrategyRequest, @@ -274,7 +311,9 @@ export class SearchService implements Plugin { private getSearchStrategy = < SearchStrategyRequest extends IOpenSearchDashboardsSearchRequest = IOpenSearchSearchRequest, - SearchStrategyResponse extends IOpenSearchDashboardsSearchResponse = IOpenSearchSearchResponse + SearchStrategyResponse extends + | IOpenSearchDashboardsSearchResponse + | IDataFrameResponse = IOpenSearchSearchResponse >( name: string ): ISearchStrategy => { diff --git a/src/plugins/data/server/search/types.ts b/src/plugins/data/server/search/types.ts index 75f21d39c0bf..6927d1289673 100644 --- a/src/plugins/data/server/search/types.ts +++ b/src/plugins/data/server/search/types.ts @@ -38,6 +38,7 @@ import { import { AggsSetup, AggsStart } from './aggs'; import { SearchUsage } from './collectors'; import { IOpenSearchSearchRequest, IOpenSearchSearchResponse } from './opensearch_search'; +import { IDataFrameResponse } from '../../common'; export interface SearchEnhancements { defaultStrategy: string; @@ -51,7 +52,9 @@ export interface ISearchSetup { */ registerSearchStrategy: < SearchStrategyRequest extends IOpenSearchDashboardsSearchRequest = IOpenSearchSearchRequest, - SearchStrategyResponse extends IOpenSearchDashboardsSearchResponse = IOpenSearchSearchResponse + SearchStrategyResponse extends + | IOpenSearchDashboardsSearchResponse + | IDataFrameResponse = IOpenSearchSearchResponse >( name: string, strategy: ISearchStrategy @@ -96,7 +99,9 @@ export interface ISearchStart< */ export interface ISearchStrategy< SearchStrategyRequest extends IOpenSearchDashboardsSearchRequest = IOpenSearchSearchRequest, - SearchStrategyResponse extends IOpenSearchDashboardsSearchResponse = IOpenSearchSearchResponse + SearchStrategyResponse extends + | IOpenSearchDashboardsSearchResponse + | IDataFrameResponse = IOpenSearchSearchResponse > { search: ( context: RequestHandlerContext, diff --git a/src/plugins/data/server/ui_settings.ts b/src/plugins/data/server/ui_settings.ts index 77f4afd11887..e12113b87b29 100644 --- a/src/plugins/data/server/ui_settings.ts +++ b/src/plugins/data/server/ui_settings.ts @@ -55,6 +55,21 @@ const requestPreferenceOptionLabels = { }), }; +const dataFrameHydrationStrategyOptionLabels = { + perSource: i18n.translate('data.advancedSettings.dataFrameHydrationStrategyPerSourceText', { + defaultMessage: 'On data source change', + }), + perQuery: i18n.translate('data.advancedSettings.dataFrameHydrationStrategyPerQueryText', { + defaultMessage: 'Per query', + }), + perResponse: i18n.translate('data.advancedSettings.dataFrameHydrationStrategyPerResponseText', { + defaultMessage: 'Per response', + }), + advanced: i18n.translate('data.advancedSettings.dataFrameHydrationStrategyAdvancedText', { + defaultMessage: 'Advanced', + }), +}; + // We add the `en` key manually here, since that's not a real numeral locale, but the // default fallback in case the locale is not found. const numeralLanguageIds = [ @@ -690,5 +705,47 @@ export function getUiSettings(): Record> { }), schema: schema.boolean(), }, + [UI_SETTINGS.DATAFRAME_HYDRATION_STRATEGY]: { + name: i18n.translate('data.advancedSettings.dataFrameHydrationStrategyTitle', { + defaultMessage: 'Data frame hydration strategy', + }), + value: 'perSource', + options: ['perSource', 'perQuery'], + optionLabels: dataFrameHydrationStrategyOptionLabels, + type: 'select', + description: i18n.translate('data.advancedSettings.dataFrameHydrationStrategyText', { + defaultMessage: `Allows you to set how often the data frame schema is updated. +
    +
  • {perSource}: hydrates the schema when the data source changes. + For example, any time the index pattern is change the data frame schema is hydrated.
    • +
  • {perQuery}: hydrates the schema per query to the data source. + Could be expensive, but ensures the schema of the data frame fits the result set.
    • +
  • {perResponse}: hydrates the schema if the data source returns a schema. + Not Implemented.
    • +
  • {advanced}: hydrates the schema in intervals. If the schema hasn't changed the interval increases. + If the schema has changed the interval resets. Not Implemented.
    • +
`, + values: { + perSource: dataFrameHydrationStrategyOptionLabels.perSource, + perQuery: dataFrameHydrationStrategyOptionLabels.perQuery, + perResponse: dataFrameHydrationStrategyOptionLabels.perResponse, + advanced: dataFrameHydrationStrategyOptionLabels.advanced, + }, + }), + category: ['search'], + schema: schema.string(), + }, + [UI_SETTINGS.QUERY_DATA_SOURCE_READONLY]: { + name: i18n.translate('data.advancedSettings.query.dataSourceReadOnlyTitle', { + defaultMessage: 'Read-only data source in query bar', + }), + value: true, + description: i18n.translate('data.advancedSettings.query.dataSourceReadOnlyText', { + defaultMessage: + 'When enabled, the global search bar prevents modifying the data source in the query input. ' + + '
Experimental: Setting to false enables modifying the data source.', + }), + schema: schema.boolean(), + }, }; } diff --git a/src/plugins/discover/public/application/components/data_grid/data_grid_table_columns.tsx b/src/plugins/discover/public/application/components/data_grid/data_grid_table_columns.tsx index ba80e719491f..496d80703d7c 100644 --- a/src/plugins/discover/public/application/components/data_grid/data_grid_table_columns.tsx +++ b/src/plugins/discover/public/application/components/data_grid/data_grid_table_columns.tsx @@ -80,8 +80,8 @@ export function computeVisibleColumns( const timeFieldName = idxPattern.timeFieldName; let visibleColumnNames = columnNames; - if (displayTimeColumn && !columnNames.includes(timeFieldName)) { - visibleColumnNames = [timeFieldName, ...columnNames]; + if (displayTimeColumn && !columnNames.includes(timeFieldName!)) { + visibleColumnNames = [timeFieldName!, ...columnNames]; } return visibleColumnNames; diff --git a/src/plugins/discover/public/application/components/sidebar/discover_field_data_frame.tsx b/src/plugins/discover/public/application/components/sidebar/discover_field_data_frame.tsx new file mode 100644 index 000000000000..26260635a303 --- /dev/null +++ b/src/plugins/discover/public/application/components/sidebar/discover_field_data_frame.tsx @@ -0,0 +1,44 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import React from 'react'; +import { i18n } from '@osd/i18n'; +import { EuiFlexGroup, EuiFlexItem, EuiButtonEmpty } from '@elastic/eui'; + +export interface Props { + onCreateIndexPattern: () => void; + onNormalizeIndexPattern: () => void; +} + +export function DiscoverFieldDataFrame({ onCreateIndexPattern, onNormalizeIndexPattern }: Props) { + return ( + + + + {i18n.translate('discover.fieldChooser.dataFrame.normalizeIndexPattern', { + defaultMessage: 'Normalize', + })} + + + + + {i18n.translate('discover.fieldChooser.dataFrame.createIndexPattern', { + defaultMessage: 'Create index pattern', + })} + + + + ); +} diff --git a/src/plugins/discover/public/application/components/sidebar/discover_sidebar.test.tsx b/src/plugins/discover/public/application/components/sidebar/discover_sidebar.test.tsx index 6fee8dde6b60..ed5c8b1773f2 100644 --- a/src/plugins/discover/public/application/components/sidebar/discover_sidebar.test.tsx +++ b/src/plugins/discover/public/application/components/sidebar/discover_sidebar.test.tsx @@ -98,6 +98,8 @@ function getCompProps(): DiscoverSidebarProps { onAddFilter: jest.fn(), onAddField: jest.fn(), onRemoveField: jest.fn(), + onNormalize: jest.fn(), + onCreateIndexPattern: jest.fn(), selectedIndexPattern: indexPattern, onReorderFields: jest.fn(), }; diff --git a/src/plugins/discover/public/application/components/sidebar/discover_sidebar.tsx b/src/plugins/discover/public/application/components/sidebar/discover_sidebar.tsx index 9dcb5bf337cb..9d69f756f0e4 100644 --- a/src/plugins/discover/public/application/components/sidebar/discover_sidebar.tsx +++ b/src/plugins/discover/public/application/components/sidebar/discover_sidebar.tsx @@ -43,6 +43,7 @@ import { import { I18nProvider } from '@osd/i18n/react'; import { DiscoverField } from './discover_field'; import { DiscoverFieldSearch } from './discover_field_search'; +import { DiscoverFieldDataFrame } from './discover_field_data_frame'; import { FIELDS_LIMIT_SETTING } from '../../../../common'; import { groupFields } from './lib/group_fields'; import { IndexPatternField, IndexPattern, UI_SETTINGS } from '../../../../../data/public'; @@ -51,6 +52,7 @@ import { getDefaultFieldFilter, setFieldFilterProp } from './lib/field_filter'; import { getIndexPatternFieldList } from './lib/get_index_pattern_field_list'; import { getServices } from '../../../opensearch_dashboards_services'; import { FieldDetails } from './types'; +import { displayIndexPatternCreation } from './lib/display_index_pattern_creation'; export interface DiscoverSidebarProps { /** @@ -82,6 +84,14 @@ export interface DiscoverSidebarProps { * @param fieldName */ onRemoveField: (fieldName: string) => void; + /** + * Callback function to create an index pattern + */ + onCreateIndexPattern: () => void; + /** + * Callback function to normalize the index pattern + */ + onNormalize: () => void; /** * Currently selected index pattern */ @@ -89,7 +99,16 @@ export interface DiscoverSidebarProps { } export function DiscoverSidebar(props: DiscoverSidebarProps) { - const { columns, fieldCounts, hits, onAddField, onReorderFields, selectedIndexPattern } = props; + const { + columns, + fieldCounts, + hits, + onAddField, + onReorderFields, + onNormalize, + onCreateIndexPattern, + selectedIndexPattern, + } = props; const [fields, setFields] = useState(null); const [fieldFilterState, setFieldFilterState] = useState(getDefaultFieldFilter()); const services = useMemo(() => getServices(), []); @@ -99,6 +118,12 @@ export function DiscoverSidebar(props: DiscoverSidebarProps) { setFields(newFields); }, [selectedIndexPattern, fieldCounts, hits, services]); + const onNormalizeIndexPattern = useCallback(async () => { + await onNormalize(); + const newFields = getIndexPatternFieldList(selectedIndexPattern, fieldCounts); + setFields(newFields); + }, [fieldCounts, onNormalize, selectedIndexPattern]); + const onChangeFieldSearch = useCallback( (field: string, value: string | boolean | undefined) => { const newState = setFieldFilterProp(fieldFilterState, field, value); @@ -195,6 +220,14 @@ export function DiscoverSidebar(props: DiscoverSidebarProps) { types={fieldTypes} /> + {displayIndexPatternCreation(selectedIndexPattern) ? ( + + + + ) : null} {fields.length > 0 && ( <> diff --git a/src/plugins/discover/public/application/components/sidebar/lib/display_index_pattern_creation.tsx b/src/plugins/discover/public/application/components/sidebar/lib/display_index_pattern_creation.tsx new file mode 100644 index 000000000000..e7dccc08ef37 --- /dev/null +++ b/src/plugins/discover/public/application/components/sidebar/lib/display_index_pattern_creation.tsx @@ -0,0 +1,17 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { IndexPattern, DATA_FRAME_TYPES } from '../../../../../../data/public'; + +/** + * if we should display index pattern creation in the sidebar + */ +export function displayIndexPatternCreation(indexPattern: IndexPattern | undefined): boolean { + if (!indexPattern || !indexPattern.type || !indexPattern.id) return false; + return ( + Object.values(DATA_FRAME_TYPES).includes(indexPattern.type as DATA_FRAME_TYPES) && + Object.values(DATA_FRAME_TYPES).includes(indexPattern.id as DATA_FRAME_TYPES) + ); +} diff --git a/src/plugins/discover/public/application/helpers/get_data_set.ts b/src/plugins/discover/public/application/helpers/get_data_set.ts new file mode 100644 index 000000000000..b0431ac31c1e --- /dev/null +++ b/src/plugins/discover/public/application/helpers/get_data_set.ts @@ -0,0 +1,25 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { IndexPattern, IndexPatternsContract } from '../../../../data/public'; +import { SearchData } from '../view_components/utils/use_search'; + +function getDataSet( + indexPattern: IndexPattern | undefined, + state: SearchData, + indexPatternsService: IndexPatternsContract +) { + if (!indexPattern) { + return; + } + return ( + (state.title && + state.title !== indexPattern?.title && + indexPatternsService.getByTitle(state.title!, true)) || + indexPattern + ); +} + +export { getDataSet }; diff --git a/src/plugins/discover/public/application/view_components/panel/index.tsx b/src/plugins/discover/public/application/view_components/panel/index.tsx index 6b4cd2a87c91..fbe180f7e316 100644 --- a/src/plugins/discover/public/application/view_components/panel/index.tsx +++ b/src/plugins/discover/public/application/view_components/panel/index.tsx @@ -20,6 +20,7 @@ import { IndexPatternField, opensearchFilters } from '../../../../../data/public import { useOpenSearchDashboards } from '../../../../../opensearch_dashboards_react/public'; import { DiscoverViewServices } from '../../../build_services'; import { popularizeField } from '../../helpers/popularize_field'; +import { getDataSet } from '../../helpers/get_data_set'; import { buildColumns } from '../../utils/columns'; // eslint-disable-next-line import/no-default-export @@ -31,6 +32,7 @@ export default function DiscoverPanel(props: ViewProps) { }, capabilities, indexPatterns, + application, } = services; const { data$, indexPattern } = useDiscoverContext(); const [fetchState, setFetchState] = useState(data$.getValue()); @@ -86,14 +88,30 @@ export default function DiscoverPanel(props: ViewProps) { [filterManager, indexPattern] ); + const onCreateIndexPattern = useCallback(async () => { + if (!fetchState.title) return; + if (fetchState.title === indexPattern?.title) return; + application?.navigateToApp('management', { + path: `opensearch-dashboards/indexPatterns/create?id=${fetchState.title}`, + }); + }, [application, fetchState.title, indexPattern?.title]); + + const onNormalize = useCallback(async () => { + if (!fetchState.title) return; + if (fetchState.title === indexPattern?.title) return; + const dataSet = getDataSet(indexPattern, fetchState, indexPatterns); + await indexPatterns.refreshFields(dataSet!, true); + }, [fetchState, indexPattern, indexPatterns]); + return ( { - if (indexPattern && capabilities.discover?.save) { - popularizeField(indexPattern, fieldName, indexPatterns); + const dataSet = getDataSet(indexPattern, fetchState, indexPatterns); + if (dataSet && capabilities.discover?.save) { + popularizeField(dataSet, fieldName, indexPatterns); } dispatch( @@ -104,8 +122,9 @@ export default function DiscoverPanel(props: ViewProps) { ); }} onRemoveField={(fieldName) => { - if (indexPattern && capabilities.discover?.save) { - popularizeField(indexPattern, fieldName, indexPatterns); + const dataSet = getDataSet(indexPattern, fetchState, indexPatterns); + if (dataSet && capabilities.discover?.save) { + popularizeField(dataSet, fieldName, indexPatterns); } dispatch(removeColumn(fieldName)); @@ -118,7 +137,9 @@ export default function DiscoverPanel(props: ViewProps) { }) ); }} - selectedIndexPattern={indexPattern} + selectedIndexPattern={getDataSet(indexPattern, fetchState, indexPatterns)} + onCreateIndexPattern={onCreateIndexPattern} + onNormalize={onNormalize} onAddFilter={onAddFilter} /> ); diff --git a/src/plugins/discover/public/application/view_components/utils/update_search_source.ts b/src/plugins/discover/public/application/view_components/utils/update_search_source.ts index 1404773eb9d4..a8480fdad18a 100644 --- a/src/plugins/discover/public/application/view_components/utils/update_search_source.ts +++ b/src/plugins/discover/public/application/view_components/utils/update_search_source.ts @@ -30,9 +30,22 @@ export const updateSearchSource = async ({ histogramConfigs, }: Props) => { const { uiSettings, data } = services; + let dataSet = indexPattern; + const dataFrame = searchSource?.getDataFrame(); + if ( + searchSource && + dataFrame && + dataFrame.name && + dataFrame.name !== '' && + dataSet.title !== dataFrame.name + ) { + dataSet = data.indexPatterns.getByTitle(dataFrame.name, true) ?? dataSet; + searchSource.setField('index', dataSet); + } + const sortForSearchSource = getSortForSearchSource( sort, - indexPattern, + dataSet, uiSettings.get(SORT_DEFAULT_ORDER_SETTING) ); const size = uiSettings.get(SAMPLE_SIZE_SETTING); @@ -43,18 +56,18 @@ export const updateSearchSource = async ({ // searchSource which applies time range const timeRangeSearchSource = await data.search.searchSource.create(); const { isDefault } = indexPatternUtils; - if (isDefault(indexPattern)) { + if (isDefault(dataSet)) { const timefilter = data.query.timefilter.timefilter; timeRangeSearchSource.setField('filter', () => { - return timefilter.createFilter(indexPattern); + return timefilter.createFilter(dataSet); }); } searchSourceInstance.setParent(timeRangeSearchSource); searchSourceInstance.setFields({ - index: indexPattern, + index: dataSet, sort: sortForSearchSource, size, query: data.query.queryString.getQuery() || null, diff --git a/src/plugins/discover/public/application/view_components/utils/use_search.ts b/src/plugins/discover/public/application/view_components/utils/use_search.ts index 06eabb1e139f..9da9704f32a7 100644 --- a/src/plugins/discover/public/application/view_components/utils/use_search.ts +++ b/src/plugins/discover/public/application/view_components/utils/use_search.ts @@ -47,6 +47,7 @@ export interface SearchData { rows?: OpenSearchSearchHit[]; bucketInterval?: TimechartHeaderBucketInterval | {}; chartData?: Chart; + title?: string; } export type SearchRefetch = 'refetch' | undefined; @@ -105,7 +106,8 @@ export const useSearch = (services: DiscoverViewServices) => { const refetch$ = useMemo(() => new Subject(), []); const fetch = useCallback(async () => { - if (!indexPattern) { + let dataSet = indexPattern; + if (!dataSet) { data$.next({ status: shouldSearchOnPageLoad() ? ResultStatus.LOADING : ResultStatus.UNINITIALIZED, }); @@ -122,17 +124,19 @@ export const useSearch = (services: DiscoverViewServices) => { // Abort any in-progress requests before fetching again if (fetchStateRef.current.abortController) fetchStateRef.current.abortController.abort(); fetchStateRef.current.abortController = new AbortController(); - const histogramConfigs = indexPattern.timeFieldName - ? createHistogramConfigs(indexPattern, interval || 'auto', data) + const histogramConfigs = dataSet.timeFieldName + ? createHistogramConfigs(dataSet, interval || 'auto', data) : undefined; const searchSource = await updateSearchSource({ - indexPattern, + indexPattern: dataSet, services, sort, searchSource: savedSearch?.searchSource, histogramConfigs, }); + dataSet = searchSource.getField('index'); + try { // Only show loading indicator if we are fetching when the rows are empty if (fetchStateRef.current.rows?.length === 0) { @@ -149,7 +153,7 @@ export const useSearch = (services: DiscoverViewServices) => { }); const inspectorRequest = inspectorAdapters.requests.start(title, { description }); inspectorRequest.stats(getRequestInspectorStats(searchSource)); - searchSource.getSearchRequestBody().then((body) => { + searchSource.getSearchRequestBody().then((body: object) => { inspectorRequest.json(body); }); @@ -167,7 +171,7 @@ export const useSearch = (services: DiscoverViewServices) => { let bucketInterval = {}; let chartData; for (const row of rows) { - const fields = Object.keys(indexPattern.flattenHit(row)); + const fields = Object.keys(dataSet!.flattenHit(row)); for (const fieldName of fields) { fetchStateRef.current.fieldCounts[fieldName] = (fetchStateRef.current.fieldCounts[fieldName] || 0) + 1; @@ -196,6 +200,10 @@ export const useSearch = (services: DiscoverViewServices) => { rows, bucketInterval, chartData, + title: + indexPattern?.title !== searchSource.getDataFrame()?.name + ? searchSource.getDataFrame()?.name + : indexPattern?.title, }); } catch (error) { // If the request was aborted then no need to surface this error in the UI diff --git a/src/plugins/discover/public/plugin.ts b/src/plugins/discover/public/plugin.ts index f8e0f254f925..8b46889a8e36 100644 --- a/src/plugins/discover/public/plugin.ts +++ b/src/plugins/discover/public/plugin.ts @@ -202,6 +202,7 @@ export class DiscoverPlugin generateCb: (renderProps: any) => { const globalFilters: any = getServices().filterManager.getGlobalFilters(); const appFilters: any = getServices().filterManager.getAppFilters(); + const showDocLinks = getServices().data.ui.Settings.getUiOverrides().showDocLinks; const hash = stringify( url.encodeQuery({ @@ -222,7 +223,9 @@ export class DiscoverPlugin return { url: generateDocViewsUrl(contextUrl), - hide: !renderProps.indexPattern.isTimeBased(), + hide: + (showDocLinks !== undefined ? !showDocLinks : false) || + !renderProps.indexPattern.isTimeBased(), }; }, order: 1, @@ -233,11 +236,15 @@ export class DiscoverPlugin defaultMessage: 'View single document', }), generateCb: (renderProps) => { + const showDocLinks = getServices().data.ui.Settings.getUiOverrides().showDocLinks; + const docUrl = `#/doc/${renderProps.indexPattern.id}/${ renderProps.hit._index }?id=${encodeURIComponent(renderProps.hit._id)}`; + return { url: generateDocViewsUrl(docUrl), + hide: showDocLinks !== undefined ? !showDocLinks : false, }; }, order: 2, diff --git a/src/plugins/home/server/services/sample_data/data_sets/index.ts b/src/plugins/home/server/services/sample_data/data_sets/index.ts index a75115950f93..dc9ac8ae3711 100644 --- a/src/plugins/home/server/services/sample_data/data_sets/index.ts +++ b/src/plugins/home/server/services/sample_data/data_sets/index.ts @@ -31,3 +31,4 @@ export { flightsSpecProvider } from './flights'; export { logsSpecProvider } from './logs'; export { ecommerceSpecProvider } from './ecommerce'; +export { appendDataSourceId, getSavedObjectsWithDataSource } from './util'; diff --git a/src/plugins/home/server/services/sample_data/lib/sample_dataset_schema.ts b/src/plugins/home/server/services/sample_data/lib/sample_dataset_schema.ts index 1e2951c596b9..c202290f911a 100644 --- a/src/plugins/home/server/services/sample_data/lib/sample_dataset_schema.ts +++ b/src/plugins/home/server/services/sample_data/lib/sample_dataset_schema.ts @@ -75,13 +75,19 @@ export const sampleDataSchema = { // saved object id of main dashboard for sample data set overviewDashboard: Joi.string().required(), + getDataSourceIntegratedDashboard: Joi.func().required(), appLinks: Joi.array().items(appLinkSchema).default([]), // saved object id of default index-pattern for sample data set defaultIndex: Joi.string().required(), + getDataSourceIntegratedDefaultIndex: Joi.func().required(), // OpenSearch Dashboards saved objects (index patter, visualizations, dashboard, ...) // Should provide a nice demo of OpenSearch Dashboards's functionality with the sample data set savedObjects: Joi.array().items(Joi.object()).required(), + getDataSourceIntegratedSavedObjects: Joi.func().required(), dataIndices: Joi.array().items(dataIndexSchema).required(), + + status: Joi.string(), + statusMsg: Joi.any(), }; diff --git a/src/plugins/vis_type_vega/public/data_model/opensearch_query_parser.ts b/src/plugins/vis_type_vega/public/data_model/opensearch_query_parser.ts index 8f5594863903..2bbd0159316c 100644 --- a/src/plugins/vis_type_vega/public/data_model/opensearch_query_parser.ts +++ b/src/plugins/vis_type_vega/public/data_model/opensearch_query_parser.ts @@ -207,7 +207,7 @@ export class OpenSearchQueryParser { if (context) { // Use dashboard context const newQuery = cloneDeep(this._filters); - if (timefield) { + if (timefield && newQuery.type !== 'unsupported') { newQuery.bool!.must!.push(body.query); } body.query = newQuery;