diff --git a/.ci/Dockerfile b/.ci/Dockerfile index 8568201a2805d..947242ecc0ece 100644 --- a/.ci/Dockerfile +++ b/.ci/Dockerfile @@ -1,7 +1,7 @@ # NOTE: This Dockerfile is ONLY used to run certain tasks in CI. It is not used to run Kibana or as a distributable. # If you're looking for the Kibana Docker image distributable, please see: src/dev/build/tasks/os_packages/docker_generator/templates/dockerfile.template.ts -ARG NODE_VERSION=14.17.3 +ARG NODE_VERSION=14.17.5 FROM node:${NODE_VERSION} AS base diff --git a/.eslintignore b/.eslintignore index f757ed9a1bf98..66684fbcd52e6 100644 --- a/.eslintignore +++ b/.eslintignore @@ -27,6 +27,7 @@ snapshots.js /x-pack/plugins/canvas/shareable_runtime/build /x-pack/plugins/canvas/storybook/build /x-pack/plugins/reporting/server/export_types/printable_pdf/server/lib/pdf/assets/** +/x-pack/plugins/reporting/server/export_types/printable_pdf_v2/server/lib/pdf/assets/** # package overrides /packages/elastic-eslint-config-kibana diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 95c11b05a9783..d98cde7b48c21 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -107,6 +107,8 @@ /x-pack/plugins/observability/public/components/shared/exploratory_view @elastic/uptime /x-pack/test/functional_with_es_ssl/apps/uptime @elastic/uptime /x-pack/test/functional/apps/uptime @elastic/uptime +/x-pack/test/functional/es_archives/uptime @elastic/uptime +/x-pack/test/functional/services/uptime @elastic/uptime /x-pack/test/api_integration/apis/uptime @elastic/uptime # Client Side Monitoring / Uptime (lives in APM directories but owned by Uptime) @@ -123,6 +125,12 @@ # Presentation /src/plugins/dashboard/ @elastic/kibana-presentation +/src/plugins/expression_error/ @elastic/kibana-presentation +/src/plugins/expression_image/ @elastic/kibana-presentation +/src/plugins/expression_metric/ @elastic/kibana-presentation +/src/plugins/expression_repeat_image/ @elastic/kibana-presentation +/src/plugins/expression_reveal_image/ @elastic/kibana-presentation +/src/plugins/expression_shape/ @elastic/kibana-presentation /src/plugins/input_control_vis/ @elastic/kibana-presentation /src/plugins/vis_type_markdown/ @elastic/kibana-presentation /src/plugins/presentation_util/ @elastic/kibana-presentation diff --git a/.node-version b/.node-version index c6244cda0441f..18711d290eac4 100644 --- a/.node-version +++ b/.node-version @@ -1 +1 @@ -14.17.3 +14.17.5 diff --git a/.nvmrc b/.nvmrc index c6244cda0441f..18711d290eac4 100644 --- a/.nvmrc +++ b/.nvmrc @@ -1 +1 @@ -14.17.3 +14.17.5 diff --git a/WORKSPACE.bazel b/WORKSPACE.bazel index e26c2ec09acf7..384277822709c 100644 --- a/WORKSPACE.bazel +++ b/WORKSPACE.bazel @@ -10,15 +10,15 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") # Fetch Node.js rules http_archive( name = "build_bazel_rules_nodejs", - sha256 = "8f5f192ba02319254aaf2cdcca00ec12eaafeb979a80a1e946773c520ae0a2c9", - urls = ["https://github.com/bazelbuild/rules_nodejs/releases/download/3.7.0/rules_nodejs-3.7.0.tar.gz"], + sha256 = "e79c08a488cc5ac40981987d862c7320cee8741122a2649e9b08e850b6f20442", + urls = ["https://github.com/bazelbuild/rules_nodejs/releases/download/3.8.0/rules_nodejs-3.8.0.tar.gz"], ) # Now that we have the rules let's import from them to complete the work load("@build_bazel_rules_nodejs//:index.bzl", "check_rules_nodejs_version", "node_repositories", "yarn_install") # Assure we have at least a given rules_nodejs version -check_rules_nodejs_version(minimum_version_string = "3.7.0") +check_rules_nodejs_version(minimum_version_string = "3.8.0") # Setup the Node.js toolchain for the architectures we want to support # @@ -27,13 +27,13 @@ check_rules_nodejs_version(minimum_version_string = "3.7.0") # we can update that rule. node_repositories( node_repositories = { - "14.17.3-darwin_amd64": ("node-v14.17.3-darwin-x64.tar.gz", "node-v14.17.3-darwin-x64", "522f85db1d1fe798cba5f601d1bba7b5203ca8797b2bc934ff6f24263f0b7fb2"), - "14.17.3-linux_arm64": ("node-v14.17.3-linux-arm64.tar.xz", "node-v14.17.3-linux-arm64", "80f4143d3c2d4cf3c4420eea3202c7bf16788b0a72fd512e60bfc8066a08a51c"), - "14.17.3-linux_s390x": ("node-v14.17.3-linux-s390x.tar.xz", "node-v14.17.3-linux-s390x", "4f69c30732f94189b9ab98f3100b17f1e4db2000848d56064e887be1c28e81ae"), - "14.17.3-linux_amd64": ("node-v14.17.3-linux-x64.tar.xz", "node-v14.17.3-linux-x64", "d659d78144042a1801f35dd611d0fab137e841cde902b2c6a821163a5e36f105"), - "14.17.3-windows_amd64": ("node-v14.17.3-win-x64.zip", "node-v14.17.3-win-x64", "170fb4f95539d1d7e1295fb2556cb72bee352cdf81a02ffb16cf6d50ad2fefbf"), + "14.17.5-darwin_amd64": ("node-v14.17.5-darwin-x64.tar.gz", "node-v14.17.5-darwin-x64", "2e40ab625b45b9bdfcb963ddd4d65d87ddf1dd37a86b6f8b075cf3d77fe9dc09"), + "14.17.5-linux_arm64": ("node-v14.17.5-linux-arm64.tar.xz", "node-v14.17.5-linux-arm64", "3a2e674b6db50dfde767c427e8f077235bbf6f9236e1b12a4cc3496b12f94bae"), + "14.17.5-linux_s390x": ("node-v14.17.5-linux-s390x.tar.xz", "node-v14.17.5-linux-s390x", "7d40eee3d54241403db12fb3bc420cd776e2b02e89100c45cf5e74a73942e7f6"), + "14.17.5-linux_amd64": ("node-v14.17.5-linux-x64.tar.xz", "node-v14.17.5-linux-x64", "2d759de07a50cd7f75bd73d67e97b0d0e095ee3c413efac7d1b3d1e84ed76fff"), + "14.17.5-windows_amd64": ("node-v14.17.5-win-x64.zip", "node-v14.17.5-win-x64", "a99b7ee08e846e5d1f4e70c4396265542819d79ed9cebcc27760b89571f03cbf"), }, - node_version = "14.17.3", + node_version = "14.17.5", node_urls = [ "https://nodejs.org/dist/v{version}/{filename}", ], diff --git a/dev_docs/assets/data_view_diagram.png b/dev_docs/assets/data_view_diagram.png new file mode 100644 index 0000000000000..7a0a47abc195d Binary files /dev/null and b/dev_docs/assets/data_view_diagram.png differ diff --git a/dev_docs/assets/kibana_template_no_data_config.png b/dev_docs/assets/kibana_template_no_data_config.png new file mode 100644 index 0000000000000..5e54bfdce1938 Binary files /dev/null and b/dev_docs/assets/kibana_template_no_data_config.png differ diff --git a/dev_docs/best_practices.mdx b/dev_docs/best_practices.mdx index 0bc86da6998dd..767e525c0afa7 100644 --- a/dev_docs/best_practices.mdx +++ b/dev_docs/best_practices.mdx @@ -22,8 +22,10 @@ Refer to [divio documentation](https://documentation.divio.com/) for guidance on and sections are both _explanation_ oriented, - covers both _tutorials_ and _How to_, and -the section covers _reference_ material. + covers both _tutorials_ and _How to_, and the section covers _reference_ material. #### Location @@ -256,17 +258,17 @@ links](https://elastic.github.io/eui/#/navigation/link#link-validation), and a r **Best practices** -* Check for dangerous functions or assignments that can result in unescaped user input in the browser DOM. Avoid using: - * **React:** [`dangerouslySetInnerHtml`](https://reactjs.org/docs/dom-elements.html#dangerouslysetinnerhtml). - * **Browser DOM:** `Element.innerHTML` and `Element.outerHTML`. -* If using the aforementioned unsafe functions or assignments is absolutely necessary, follow [these XSS prevention -rules](https://cheatsheetseries.owasp.org/cheatsheets/Cross_Site_Scripting_Prevention_Cheat_Sheet.html#xss-prevention-rules) to ensure that -user input is not inserted into unsafe locations and that it is escaped properly. -* Use EUI components to build your UI, particularly when rendering `href` links. Otherwise, sanitize user input before rendering links to -ensure that they do not use the `javascript:` protocol. -* Don't use the `eval`, `Function`, and `_.template` functions -- these are restricted by ESLint rules. -* Be careful when using `setTimeout` and `setInterval` in client-side code. If an attacker can manipulate the arguments and pass a string to -one of these, it is evaluated dynamically, which is equivalent to the dangerous `eval` function. +- Check for dangerous functions or assignments that can result in unescaped user input in the browser DOM. Avoid using: + - **React:** [`dangerouslySetInnerHtml`](https://reactjs.org/docs/dom-elements.html#dangerouslysetinnerhtml). + - **Browser DOM:** `Element.innerHTML` and `Element.outerHTML`. +- If using the aforementioned unsafe functions or assignments is absolutely necessary, follow [these XSS prevention + rules](https://cheatsheetseries.owasp.org/cheatsheets/Cross_Site_Scripting_Prevention_Cheat_Sheet.html#xss-prevention-rules) to ensure that + user input is not inserted into unsafe locations and that it is escaped properly. +- Use EUI components to build your UI, particularly when rendering `href` links. Otherwise, sanitize user input before rendering links to + ensure that they do not use the `javascript:` protocol. +- Don't use the `eval`, `Function`, and `_.template` functions -- these are restricted by ESLint rules. +- Be careful when using `setTimeout` and `setInterval` in client-side code. If an attacker can manipulate the arguments and pass a string to + one of these, it is evaluated dynamically, which is equivalent to the dangerous `eval` function. ### Cross-Site Request Forgery (CSRF/XSRF) @@ -280,10 +282,10 @@ Headers](https://www.elastic.co/guide/en/kibana/master/api.html#api-request-head **Best practices** -* Ensure all HTTP routes are registered with the [Kibana HTTP service](https://www.elastic.co/guide/en/kibana/master/http-service.html) to -take advantage of the custom request header security control. - * Note that HTTP GET requests do **not** require the custom request header; any routes that change data should [adhere to the HTTP -specification and use a different method (PUT, POST, etc.)](https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods) +- Ensure all HTTP routes are registered with the [Kibana HTTP service](https://www.elastic.co/guide/en/kibana/master/http-service.html) to + take advantage of the custom request header security control. + - Note that HTTP GET requests do **not** require the custom request header; any routes that change data should [adhere to the HTTP + specification and use a different method (PUT, POST, etc.)](https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods) ### Remote Code Execution (RCE) @@ -295,11 +297,11 @@ ESLint rules to restrict vulnerable functions, and by hooking into or hardening **Best practices** -* Don't use the `eval`, `Function`, and `_.template` functions -- these are restricted by ESLint rules. -* Don't use dynamic `require`. -* Check for usages of templating libraries. Ensure that user-provided input doesn't influence the template and is used only as data for -rendering the template. -* Take extra caution when spawning child processes with any user input or parameters that are user-controlled. +- Don't use the `eval`, `Function`, and `_.template` functions -- these are restricted by ESLint rules. +- Don't use dynamic `require`. +- Check for usages of templating libraries. Ensure that user-provided input doesn't influence the template and is used only as data for + rendering the template. +- Take extra caution when spawning child processes with any user input or parameters that are user-controlled. ### Prototype Pollution @@ -309,26 +311,26 @@ hardening sensitive functions (such as those exposed by `child_process`), and by **Best practices** -* Check for instances of `anObject[a][b] = c` where `a`, `b`, and `c` are controlled by user input. This includes code paths where the -following logical code steps could be performed in separate files by completely different operations, or by recursively using dynamic -operations. -* Validate all user input, including API URL parameters, query parameters, and payloads. Preferably, use a schema that only allows specific -keys and values. At a minimum, implement a deny-list that prevents `__proto__` and `prototype.constructor` from being used within object -keys. -* When calling APIs that spawn new processes or perform code generation from strings, protect against Prototype Pollution by checking if -`Object.hasOwnProperty` has arguments to the APIs that originate from an Object. An example is the defunct Code app's -[`spawnProcess`](https://github.com/elastic/kibana/blob/b49192626a8528af5d888545fb14cd1ce66a72e7/x-pack/legacy/plugins/code/server/lsp/workspace_command.ts#L40-L44) -function. - * Common Node.js offenders: `child_process.spawn`, `child_process.exec`, `eval`, `Function('some string')`, `vm.runInContext(x)`, -`vm.runInNewContext(x)`, `vm.runInThisContext()` - * Common client-side offenders: `eval`, `Function('some string')`, `setTimeout('some string', num)`, `setInterval('some string', num)` +- Check for instances of `anObject[a][b] = c` where `a`, `b`, and `c` are controlled by user input. This includes code paths where the + following logical code steps could be performed in separate files by completely different operations, or by recursively using dynamic + operations. +- Validate all user input, including API URL parameters, query parameters, and payloads. Preferably, use a schema that only allows specific + keys and values. At a minimum, implement a deny-list that prevents `__proto__` and `prototype.constructor` from being used within object + keys. +- When calling APIs that spawn new processes or perform code generation from strings, protect against Prototype Pollution by checking if + `Object.hasOwnProperty` has arguments to the APIs that originate from an Object. An example is the defunct Code app's + [`spawnProcess`](https://github.com/elastic/kibana/blob/b49192626a8528af5d888545fb14cd1ce66a72e7/x-pack/legacy/plugins/code/server/lsp/workspace_command.ts#L40-L44) + function. + - Common Node.js offenders: `child_process.spawn`, `child_process.exec`, `eval`, `Function('some string')`, `vm.runInContext(x)`, + `vm.runInNewContext(x)`, `vm.runInThisContext()` + - Common client-side offenders: `eval`, `Function('some string')`, `setTimeout('some string', num)`, `setInterval('some string', num)` See also: -* [Prototype pollution: The dangerous and underrated vulnerability impacting JavaScript applications | -portswigger.net](https://portswigger.net/daily-swig/prototype-pollution-the-dangerous-and-underrated-vulnerability-impacting-javascript-applications) -* [Prototype pollution attack in NodeJS application | Olivier -Arteau](https://github.com/HoLyVieR/prototype-pollution-nsec18/blob/master/paper/JavaScript_prototype_pollution_attack_in_NodeJS.pdf) +- [Prototype pollution: The dangerous and underrated vulnerability impacting JavaScript applications | + portswigger.net](https://portswigger.net/daily-swig/prototype-pollution-the-dangerous-and-underrated-vulnerability-impacting-javascript-applications) +- [Prototype pollution attack in NodeJS application | Olivier + Arteau](https://github.com/HoLyVieR/prototype-pollution-nsec18/blob/master/paper/JavaScript_prototype_pollution_attack_in_NodeJS.pdf) ### Server-Side Request Forgery (SSRF) @@ -339,12 +341,12 @@ a vector for information disclosure or injection attacks. **Best practices** -* Ensure that all outbound requests from the Kibana server use hard-coded URLs. -* If user input is used to construct a URL for an outbound request, ensure that an allow-list is used to validate the endpoints and that -user input is escaped properly. Ideally, the allow-list should be set in `kibana.yml`, so only server administrators can change it. - * This is particularly relevant when using `transport.request` with the Elasticsearch client, as no automatic escaping is performed. - * Note that URLs are very hard to validate properly; exact match validation for user input is most preferable, while URL parsing or RegEx -validation should only be used if absolutely necessary. +- Ensure that all outbound requests from the Kibana server use hard-coded URLs. +- If user input is used to construct a URL for an outbound request, ensure that an allow-list is used to validate the endpoints and that + user input is escaped properly. Ideally, the allow-list should be set in `kibana.yml`, so only server administrators can change it. + - This is particularly relevant when using `transport.request` with the Elasticsearch client, as no automatic escaping is performed. + - Note that URLs are very hard to validate properly; exact match validation for user input is most preferable, while URL parsing or RegEx + validation should only be used if absolutely necessary. ### Reverse tabnabbing @@ -356,10 +358,10 @@ buttons, and other vulnerable DOM elements. **Best practices** -* Use EUI components to build your UI whenever possible. Otherwise, ensure that any DOM elements that have an `href` attribute also have the -`rel="noreferrer noopener"` attribute specified. For more information, refer to the [OWASP HTML5 Security Cheat -Sheet](https://github.com/OWASP/CheatSheetSeries/blob/master/cheatsheets/HTML5_Security_Cheat_Sheet.md#tabnabbing). -* If using a non-EUI markdown renderer, use a custom link renderer for rendered links. +- Use EUI components to build your UI whenever possible. Otherwise, ensure that any DOM elements that have an `href` attribute also have the + `rel="noreferrer noopener"` attribute specified. For more information, refer to the [OWASP HTML5 Security Cheat + Sheet](https://github.com/OWASP/CheatSheetSeries/blob/master/cheatsheets/HTML5_Security_Cheat_Sheet.md#tabnabbing). +- If using a non-EUI markdown renderer, use a custom link renderer for rendered links. ### Information disclosure @@ -370,7 +372,7 @@ control, but at a high level, Kibana relies on the hapi framework to automatical **Best practices** -* Look for instances where sensitive information might accidentally be revealed, particularly in error messages, in the UI, and URL -parameters that are exposed to users. -* Make sure that sensitive request data is not forwarded to external resources. For example, copying client request headers and using them -to make an another request could accidentally expose the user's credentials. +- Look for instances where sensitive information might accidentally be revealed, particularly in error messages, in the UI, and URL + parameters that are exposed to users. +- Make sure that sensitive request data is not forwarded to external resources. For example, copying client request headers and using them + to make an another request could accidentally expose the user's credentials. diff --git a/dev_docs/getting_started/add_data.mdx b/dev_docs/getting_started/add_data.mdx new file mode 100644 index 0000000000000..b09e3f6262e77 --- /dev/null +++ b/dev_docs/getting_started/add_data.mdx @@ -0,0 +1,37 @@ +--- +id: kibDevAddData +slug: /kibana-dev-docs/tutorial/sample-data +title: Add data +summary: Learn how to add data to Kibana +date: 2021-08-11 +tags: ['kibana', 'onboarding', 'dev', 'architecture', 'tutorials'] +--- + +Building a feature and need an easy way to test it out with some data? Below are three options. + +## 1. Add Sample Data from the UI + +Kibana ships with sample data that you can install at the click of the button. If you are building a feature and need some data to test it out with, sample data is a great option. The only limitation is that this data will not work for Security or Observability solutions (see [#62962](https://github.com/elastic/kibana/issues/62962)). + +1. Navigate to the home page. +2. Click **Add data**. +3. Click on the **Sample data** tab. +4. Select a dataset by clicking on the **Add data** button. + +![Sample Data](../assets/sample_data.png) + +## CSV Upload + +1. If you don't have any data, navigate to Stack Management > Index Patterns and click the link to the uploader. If you do have data, navigate to the **Machine Learning** application. +2. Click on the **Data Visualizer** tab. +3. Click on **Select file** in the **Import data** container. + +![CSV Upload](../assets/ml_csv_upload.png) + +## makelogs + +The makelogs script generates sample web server logs. Make sure Elasticsearch is running before running the script. + +```sh +node scripts/makelogs --auth : +``` diff --git a/dev_docs/getting_started/dev_welcome.mdx b/dev_docs/getting_started/dev_welcome.mdx index 3d645b4e54d66..5e569bd377ee0 100644 --- a/dev_docs/getting_started/dev_welcome.mdx +++ b/dev_docs/getting_started/dev_welcome.mdx @@ -14,7 +14,8 @@ Kibana ships with many out-of-the-box capabilities that can be extended and enha Recommended next reading: 1. -2. Create a simple . +2. Create a . +3. . Check out our to dig into the nitty gritty details of every public plugin API. diff --git a/dev_docs/getting_started/hello_world_plugin.mdx b/dev_docs/getting_started/hello_world_plugin.mdx index d3b30b240dedc..7c02d2807472c 100644 --- a/dev_docs/getting_started/hello_world_plugin.mdx +++ b/dev_docs/getting_started/hello_world_plugin.mdx @@ -27,7 +27,7 @@ $ mkdir hello_world $ cd hello_world ``` -2. Create the . +2. Create the . ``` $ touch kibana.json @@ -44,7 +44,7 @@ and add the following: } ``` -3. Create a `tsconfig.json` file. +3. Create a . ``` $ touch tsconfig.json @@ -56,8 +56,7 @@ And add the following to it: { "extends": "../../tsconfig.json", "compilerOptions": { - "outDir": "./target", - "skipLibCheck": true + "outDir": "./target/types", }, "include": [ "index.ts", @@ -67,11 +66,14 @@ And add the following to it: "server/**/*.ts", "../../typings/**/*" ], - "exclude": [] + "exclude": [], + "references": [ + { "path": "../../src/core/tsconfig.json" } + ] } ``` -4. Create a . +4. Create a . ``` $ mkdir public @@ -104,7 +106,7 @@ export class HelloWorldPlugin implements Plugin { } ``` -5. Create a . +5. Create a . ``` $ touch index.ts diff --git a/dev_docs/key_concepts/anatomy_of_a_plugin.mdx b/dev_docs/key_concepts/anatomy_of_a_plugin.mdx index 4ff5e403ff851..fa0aae2299bb0 100644 --- a/dev_docs/key_concepts/anatomy_of_a_plugin.mdx +++ b/dev_docs/key_concepts/anatomy_of_a_plugin.mdx @@ -1,6 +1,6 @@ --- -id: kibDevTutorialBuildAPlugin -slug: /kibana-dev-docs/tutorials/anatomy-of-a-plugin +id: kibDevAnatomyOfAPlugin +slug: /kibana-dev-docs/anatomy-of-a-plugin title: Anatomy of a plugin summary: Anatomy of a Kibana plugin. date: 2021-08-03 @@ -22,22 +22,23 @@ The basic file structure of a Kibana plugin named demo that has both client-side ``` plugins/ demo - kibana.json [1] + kibana.json + tsconfig.json public - index.ts [2] - plugin.ts [3] + index.ts + plugin.ts server - index.ts [4] - plugin.ts [5] + index.ts + plugin.ts common - index.ts [6] + index.ts ``` -### [1] kibana.json +### kibana.json `kibana.json` is a static manifest file that is used to identify the plugin and to specify if this plugin has server-side code, browser-side code, or both: -``` +```json { "id": "examplePluginId", "version": "1.0.0", @@ -88,12 +89,38 @@ plugins/ You don't need to declare a dependency on a plugin if you only wish to access its types. -### [2] public/index.ts +### tsconfig.json + +If you are developing in TypeScript (which we recommend), you will need to add a `tsconfig.json` file. Here is an example file that you would use if adding a plugin into the `examples` directory. + +```json +{ + "extends": "../../tsconfig.json", // Extend kibana/tsconfig.json + "compilerOptions": { + "outDir": "./target/types" + }, + "include": [ + "index.ts", + "../../typings/**/*", + // The following paths are optional, based on whether you have common code, + // or are building a client-side-only or server-side-only plugin. + "common/**/*.ts", + "public/**/*.ts", + "public/**/*.tsx", + "server/**/*.ts" + ], + "exclude": [], + // If you import another plugin's types, point to their `tsconfig.json` file. + "references": [{ "path": "../../src/core/tsconfig.json" }] +} +``` + +### public/index.ts `public/index.ts` is the entry point into the client-side code of this plugin. Everything exported from this file will be a part of the plugins . If the plugin only exists to export static utilities, consider using a package. Otherwise, this file must export a function named plugin, which will receive a standard set of core capabilities as an argument. It should return an instance of its plugin class for Kibana to load. -``` +```ts import type { PluginInitializerContext } from 'kibana/server'; import { DemoPlugin } from './plugin'; @@ -122,7 +149,7 @@ Using the non-`type` variation will increase the bundle size unnecessarily and m -### [3] public/plugin.ts +### public/plugin.ts `public/plugin.ts` is the client-side plugin definition itself. Technically speaking, it does not need to be a class or even a separate file from the entry point, but all plugins at Elastic should be consistent in this way. @@ -147,11 +174,11 @@ export class DemoPlugin implements Plugin { } ``` -### [4] server/index.ts +### server/index.ts `server/index.ts` is the entry-point into the server-side code of this plugin. It is identical in almost every way to the client-side entry-point: -### [5] server/plugin.ts +### server/plugin.ts `server/plugin.ts` is the server-side plugin definition. The shape of this plugin is the same as it’s client-side counter-part: @@ -178,7 +205,7 @@ export class DemoPlugin implements Plugin { Kibana does not impose any technical restrictions on how the the internals of a plugin are architected, though there are certain considerations related to how plugins integrate with core APIs and APIs exposed by other plugins that may greatly impact how they are built. -### [6] common/index.ts +### common/index.ts `common/index.ts` is the entry-point into code that can be used both server-side or client side. @@ -208,13 +235,15 @@ dependency in it’s kibana.json manifest file. ** foobar plugin.ts: ** -``` +```ts import type { Plugin } from 'kibana/server'; -export interface FoobarPluginSetup { [1] +// [1] +export interface FoobarPluginSetup { getFoo(): string; } -export interface FoobarPluginStart { [1] +// [1] +export interface FoobarPluginStart { getBar(): string; } @@ -256,7 +285,8 @@ With that specified in the plugin manifest, the appropriate interfaces are then import type { CoreSetup, CoreStart } from 'kibana/server'; import type { FoobarPluginSetup, FoobarPluginStart } from '../../foobar/server'; -interface DemoSetupPlugins { [1]; +// [1] +interface DemoSetupPlugins { foobar: FoobarPluginSetup; } @@ -265,13 +295,15 @@ interface DemoStartPlugins { } export class DemoPlugin { - public setup(core: CoreSetup, plugins: DemoSetupPlugins) { [2]; + // [2] + public setup(core: CoreSetup, plugins: DemoSetupPlugins) { const { foobar } = plugins; foobar.getFoo(); // 'foo' foobar.getBar(); // throws because getBar does not exist } - public start(core: CoreStart, plugins: DemoStartPlugins) { [3]; + //[3] + public start(core: CoreStart, plugins: DemoStartPlugins) { const { foobar } = plugins; foobar.getFoo(); // throws because getFoo does not exist foobar.getBar(); // 'bar' diff --git a/dev_docs/key_concepts/data_views.mdx b/dev_docs/key_concepts/data_views.mdx new file mode 100644 index 0000000000000..e2b64c8705c48 --- /dev/null +++ b/dev_docs/key_concepts/data_views.mdx @@ -0,0 +1,32 @@ +--- +id: kibDataViewsKeyConcepts +slug: /kibana-dev-docs/data-view-intro +title: Data Views +summary: Data views are the central method of defining queryable data sets in Kibana +date: 2021-08-11 +tags: ['kibana','dev', 'contributor', 'api docs'] +--- + +*Note: Kibana index patterns are currently being renamed to data views. There will be some naming inconsistencies until the transition is complete.* + +Data views (formerly Kibana index patterns or KIPs) are the central method of describing sets of indices for queries. Usage is strongly recommended +as a number of high level rely on them. Further, they provide a consistent view of data across +a variety Kibana apps. + +Data views are defined by a wildcard string (an index pattern) which matches indices, data streams, and index aliases, optionally specify a +timestamp field for time series data, and are stored as a . They have a field list which comprises all the fields in matching indices plus fields defined specifically +on the data view via runtime fields. Schema-on-read functionality is provided by data view defined runtime fields. + +![image](../assets/data_view_diagram.png) + + + +The data view API is made available via the data plugin (`data.indexPatterns`, soon to be renamed) and most commonly used with +(`data.search.search.SearchSource`) to perform queries. SearchSource will apply existing filters and queries from the search bar UI. + +Users can create data views via [Data view management](https://www.elastic.co/guide/en/kibana/current/index-patterns.html). +Additionally, they can be created through the data view API. + +Data views also allow formatters and custom labels to be defined for fields. + diff --git a/dev_docs/kibana_platform_plugin_intro.mdx b/dev_docs/kibana_platform_plugin_intro.mdx index f70c42cb520cc..252a6dcd9cd8e 100644 --- a/dev_docs/kibana_platform_plugin_intro.mdx +++ b/dev_docs/kibana_platform_plugin_intro.mdx @@ -4,7 +4,7 @@ slug: /kibana-dev-docs/platform-intro title: Plugins and the Kibana platform summary: An introduction to the Kibana platform and how to use it to build a plugin. date: 2021-01-06 -tags: ['kibana','onboarding', 'dev', 'architecture'] +tags: ['kibana', 'onboarding', 'dev', 'architecture'] --- From an end user perspective, Kibana is a tool for interacting with Elasticsearch, providing an easy way @@ -23,11 +23,11 @@ developer tools. The Kibana platform is a blank canvas, just waiting for a devel Plugins have access to three kinds of public services: - - Platform services provided by `core` () - - Platform services provided by plugins () - - Shared services provided by plugins, that are only relevant for only a few, specific plugins (e.g. "presentation utils"). +- Platform services provided by `core` () +- Platform services provided by plugins () +- Shared services provided by plugins, that are only relevant for only a few, specific plugins (e.g. "presentation utils"). - The first two items are what make up "Platform services". +The first two items are what make up "Platform services". @@ -46,24 +46,24 @@ In reality, we ended up with many platform-like services living outside of core, them, so we consider them part of platform services. When we built our platform system, we also thought we'd end up with only a handful of large plugins outside core. Users could turn certain plugins off, to minimize the code - footprint and speed up Kibana. +footprint and speed up Kibana. In reality, our plugin model ended up being used like micro-services. Plugins are the only form of encapsulation we provide developers, and they liked it! However, we ended - up with a ton of small plugins, that developers never intended to be uninstallable, nor tested in this manner. We are considering ways to provide developers the ability to build services - with the encapsulation - they desire, without the need to build a plugin. +up with a ton of small plugins, that developers never intended to be uninstallable, nor tested in this manner. We are considering ways to provide developers the ability to build services +with the encapsulation +they desire, without the need to build a plugin. Another side effect of having many small plugins is that common code often ends up extracted into another plugin. Use case specific utilities are exported, - that are not meant to be used in a general manner. This makes our definition of "platform code" a bit trickier to define. We'd like to say "The platform is made up of - every publically exposed service", but in today's world, that wouldn't be a very accurate picture. +that are not meant to be used in a general manner. This makes our definition of "platform code" a bit trickier to define. We'd like to say "The platform is made up of +every publically exposed service", but in today's world, that wouldn't be a very accurate picture. We recognize the need to better clarify the relationship between core functionality, platform-like plugin functionality, and functionality exposed by other plugins. - It's something we will be working on! +It's something we will be working on! + We will continue to focus on adding clarity around these types of services and what developers can expect from each. - ### Core services @@ -75,8 +75,10 @@ there are some plugins that provide platform functionality. We call these plugin. There is no official way to tell if a plugin is a platform plugin or not. -Platform plugins are _usually_ plugins that are managed by the Platform Group, but we are starting to see some exceptions. + + plugin. There is no official way to tell if a plugin is a +platform plugin or not. Platform plugins are _usually_ plugins that are managed by the Platform Group, +but we are starting to see some exceptions. ## Plugins @@ -110,19 +112,19 @@ Any plugin that exports something from those files, or from the lifecycle method ## Lifecycle methods Core, and plugins, expose different features at different parts of their lifecycle. We describe the lifecycle of core services and plugins with - specifically-named functions on the service definition. +specifically-named functions on the service definition. Kibana has three lifecycles: setup, start, and stop. Each plugin’s setup function is called sequentially while Kibana is setting up - on the server or when it is being loaded in the browser. The start functions are called sequentially after setup has been completed for all plugins. - The stop functions are called sequentially while Kibana is gracefully shutting down the server or when the browser tab or window is being closed. +on the server or when it is being loaded in the browser. The start functions are called sequentially after setup has been completed for all plugins. +The stop functions are called sequentially while Kibana is gracefully shutting down the server or when the browser tab or window is being closed. The table below explains how each lifecycle relates to the state of Kibana. -| lifecycle | purpose | server | browser | -| ---------- | ------ | ------- | ----- | -| setup | perform "registration" work to setup environment for runtime |configure REST API endpoint, register saved object types, etc. | configure application routes in SPA, register custom UI elements in extension points, etc. | -| start | bootstrap runtime logic | respond to an incoming request, request Elasticsearch server, etc. | start polling Kibana server, update DOM tree in response to user interactions, etc.| -| stop | cleanup runtime | dispose of active handles before the server shutdown. | store session data in the LocalStorage when the user navigates away from Kibana, etc. | +| lifecycle | purpose | server | browser | +| --------- | ------------------------------------------------------------ | ------------------------------------------------------------------ | ------------------------------------------------------------------------------------------ | +| setup | perform "registration" work to setup environment for runtime | configure REST API endpoint, register saved object types, etc. | configure application routes in SPA, register custom UI elements in extension points, etc. | +| start | bootstrap runtime logic | respond to an incoming request, request Elasticsearch server, etc. | start polling Kibana server, update DOM tree in response to user interactions, etc. | +| stop | cleanup runtime | dispose of active handles before the server shutdown. | store session data in the LocalStorage when the user navigates away from Kibana, etc. | Different service interfaces can and will be passed to setup, start, and stop because certain functionality makes sense in the context of a running plugin while other types of functionality may have restrictions or may only make sense in the context of a plugin that is stopping. @@ -141,4 +143,4 @@ plugins to customize the Kibana experience. Examples of extension points are: ## Follow up material -Learn how to build your own plugin by following . +Learn how to build your own plugin by following . diff --git a/dev_docs/tutorials/data_views.mdx b/dev_docs/tutorials/data_views.mdx new file mode 100644 index 0000000000000..453146d81c4d7 --- /dev/null +++ b/dev_docs/tutorials/data_views.mdx @@ -0,0 +1,86 @@ +--- +id: kibDevTutorialDataViews +slug: /kibana-dev-docs/tutorials/data-views +title: Data views API +summary: Data views API +date: 2021-08-11 +tags: ['kibana', 'onboarding', 'dev', 'architecture'] +--- + +*Note: Kibana index patterns are currently being renamed to data views. There will be some naming inconsistencies until the transition is complete.* + +### Data views API + +- Get list of data views +- Get default data view and examine fields +- Get data view by id +- Find data view by title +- Create data view +- Create data view and save it +- Modify data view and save it +- Delete data view + +#### Get list of data view titles and ids + +``` +const idsAndTitles = await data.indexPatterns.getIdsWithTitle(); +idsAndTitles.forEach(({id, title}) => console.log(`Data view id: ${id} title: ${title}`)); +``` + +#### Get default data view and examine fields + +``` +const defaultDataView = await data.indexPatterns.getDefault(); +defaultDataView.fields.forEach(({name}) => { console.log(name); }) +``` + +#### Get data view by id + +``` +const id = 'xxxxxx-xxx-xxxxxx'; +const dataView = await data.indexPatterns.get(id); +``` + +#### Find data view by title + +``` +const title = 'kibana-*'; +const [dataView] = await data.indexPatterns.find(title); +``` + +#### Create data view + +``` +const dataView = await data.indexPatterns.create({ title: 'kibana-*' }); +``` + +#### Create data view and save it immediately + +``` +const dataView = await data.indexPatterns.createAndSave({ title: 'kibana-*' }); +``` + +#### Create data view, modify, and save + +``` +const dataView = await data.indexPatterns.create({ title: 'kibana-*' }); +dataView.setFieldCustomLabel('customer_name', 'Customer Name'); +data.indexPatterns.createSavedObject(dataView); +``` + +#### Modify data view and save it + +``` +dataView.setFieldCustomLabel('customer_name', 'Customer Name'); +await data.indexPatterns.updateSavedObject(dataView); +``` + +#### Delete index pattern + +``` +await data.indexPatterns.delete(dataViewId); +``` + +### Data view HTTP API + +Rest-like HTTP CRUD+ API - [docs](https://www.elastic.co/guide/en/kibana/master/index-patterns-api.html) diff --git a/dev_docs/tutorials/kibana_page_template.mdx b/dev_docs/tutorials/kibana_page_template.mdx index d9605ac5643ba..eab5b2eb3ce8e 100644 --- a/dev_docs/tutorials/kibana_page_template.mdx +++ b/dev_docs/tutorials/kibana_page_template.mdx @@ -1,7 +1,7 @@ --- id: kibDevDocsKPTTutorial slug: /kibana-dev-docs/tutorials/kibana-page-template -title: KibanaPageTemplate component +title: Kibana Page Template summary: Learn how to create pages in Kibana date: 2021-03-20 tags: ['kibana', 'dev', 'ui', 'tutorials'] @@ -117,3 +117,54 @@ When using `EuiSideNav`, root level items should not be linked but provide secti ![Screenshot of Stack Management empty state with a provided solution navigation shown on the left, outlined in pink.](../assets/kibana_template_solution_nav.png) ![Screenshots of Stack Management page in mobile view. Menu closed on the left, menu open on the right.](../assets/kibana_template_solution_nav_mobile.png) + +## `noDataConfig` + +Increases the consistency in messaging across all the solutions during the getting started process when no data exists. Each solution/template instance decides when is the most appropriate time to show this configuration, but is messaged specifically towards having no indices or index patterns at all or that match the particular solution. + +This is a built-in configuration that displays a very specific UI and requires very specific keys. It will also ignore all other configurations of the template including `pageHeader` and `children`, with the exception of continuing to show `solutionNav`. + +The `noDataConfig` is of type [`NoDataPagProps`](https://github.com/elastic/kibana/blob/master/src/plugins/kibana_react/public/page_template/no_data_page/no_data_page.tsx): + +1. `solution: string`: Single name for the current solution, used to auto-generate the title, logo, description, and button label *(required)* +2. `docsLink: string`: Required to set the docs link for the whole solution *(required)* +3. `logo?: string`: Optionally replace the auto-generated logo +4. `pageTitle?: string`: Optionally replace the auto-generated page title (h1) +5. `actions: NoDataPageActionsProps`: An object of `NoDataPageActions` configurations with unique primary keys *(required)* + +### `NoDataPageActions` + +There are two main actions for adding data that we promote throughout Kibana, Elastic Agent and Beats. They are added to the cards that are displayed by using the keys `elasticAgent` and `beats` respectively. For consistent messaging, these two cards are pre-configured but require specific `href`s and/or `onClick` handlers for directing the user to the right location for that solution. + +It also accepts a `recommended` prop as a boolean to promote one or more of the cards through visuals added to the UI. It will also place the `recommended` ones first in the list. By default, the configuration will recommend `elasticAgent`. Optionally you can also replace the `button` label by passing a string, or the whole component by passing a `ReactNode`. + + +```tsx +// Perform your own check +const hasData = checkForData(); + +// No data configuration +const noDataConfig: KibanaPageTemplateProps['noDataConfig'] = { + solution: 'Observability', + docsLink: '#', + actions: { + elasticAgent: { + href: '#', + }, + beats: { + href: '#', + }, + }, +}; + + + {/* Children will be ignored */} + +``` + + +![Screenshot of and example in Observability using the no data configuration and using the corresponding list numbers to point out the UI elements that they adjust.](../assets/kibana_template_no_data_config.png) diff --git a/dev_docs/tutorials/sample_data.mdx b/dev_docs/tutorials/sample_data.mdx deleted file mode 100644 index 75afaaaea6f32..0000000000000 --- a/dev_docs/tutorials/sample_data.mdx +++ /dev/null @@ -1,33 +0,0 @@ ---- -id: kibDevTutorialSampleData -slug: /kibana-dev-docs/tutorial/sample-data -title: Add sample data -summary: Learn how to add sample data to Kibana -date: 2021-04-26 -tags: ['kibana', 'onboarding', 'dev', 'architecture', 'tutorials'] ---- - -## Installation from the UI - -1. Navigate to the home page. -2. Click **Add data**. -3. Click on the **Sample data** tab. -4. Select a dataset by clicking on the **Add data** button. - -![Sample Data](../assets/sample_data.png) - -## CSV Upload - -1. Navigate to the **Machine Learning** application. -2. Click on the **Data Visualizer** tab. -3. Click on **Select file** in the **Import data** container. - -![CSV Upload](../assets/ml_csv_upload.png) - -## makelogs - -The makelogs script generates sample web server logs. Make sure Elasticsearch is running before running the script. - -```sh -node scripts/makelogs --auth : -``` \ No newline at end of file diff --git a/docs/apm/agent-configuration.asciidoc b/docs/apm/agent-configuration.asciidoc index f2e07412c4a38..4e4a37067ea10 100644 --- a/docs/apm/agent-configuration.asciidoc +++ b/docs/apm/agent-configuration.asciidoc @@ -27,6 +27,8 @@ For this reason, it is still essential to set custom default configurations loca ==== APM Server setup This feature requires {apm-server-ref}/setup-kibana-endpoint.html[Kibana endpoint configuration] in APM Server. +In addition, if an APM agent is using {apm-server-ref}/configuration-anonymous.html[anonymous authentication] to communicate with the APM Server, +the agent's service name must be included in the `apm-server.auth.anonymous.allow_service` list. APM Server acts as a proxy between the agents and Kibana. Kibana communicates any changed settings to APM Server so that your agents only need to poll APM Server to determine which settings have changed. diff --git a/docs/developer/advanced/running-elasticsearch.asciidoc b/docs/developer/advanced/running-elasticsearch.asciidoc index 36f9ee420d41d..324d2af2ed3af 100644 --- a/docs/developer/advanced/running-elasticsearch.asciidoc +++ b/docs/developer/advanced/running-elasticsearch.asciidoc @@ -76,6 +76,7 @@ If many other users will be interacting with your remote cluster, you'll want to [source,bash] ---- kibana.index: '.{YourGitHubHandle}-kibana' +xpack.task_manager.index: '.{YourGitHubHandle}-task-manager-kibana' ---- ==== Running remote clusters diff --git a/docs/development/core/public/kibana-plugin-core-public.overlayflyoutopenoptions._aria-label_.md b/docs/development/core/public/kibana-plugin-core-public.overlayflyoutopenoptions._aria-label_.md new file mode 100644 index 0000000000000..f135fa9618958 --- /dev/null +++ b/docs/development/core/public/kibana-plugin-core-public.overlayflyoutopenoptions._aria-label_.md @@ -0,0 +1,11 @@ + + +[Home](./index.md) > [kibana-plugin-core-public](./kibana-plugin-core-public.md) > [OverlayFlyoutOpenOptions](./kibana-plugin-core-public.overlayflyoutopenoptions.md) > ["aria-label"](./kibana-plugin-core-public.overlayflyoutopenoptions._aria-label_.md) + +## OverlayFlyoutOpenOptions."aria-label" property + +Signature: + +```typescript +'aria-label'?: string; +``` diff --git a/docs/development/core/public/kibana-plugin-core-public.overlayflyoutopenoptions.md b/docs/development/core/public/kibana-plugin-core-public.overlayflyoutopenoptions.md index fc4959b87a987..dcecdeb840869 100644 --- a/docs/development/core/public/kibana-plugin-core-public.overlayflyoutopenoptions.md +++ b/docs/development/core/public/kibana-plugin-core-public.overlayflyoutopenoptions.md @@ -15,11 +15,13 @@ export interface OverlayFlyoutOpenOptions | Property | Type | Description | | --- | --- | --- | +| ["aria-label"](./kibana-plugin-core-public.overlayflyoutopenoptions._aria-label_.md) | string | | | ["data-test-subj"](./kibana-plugin-core-public.overlayflyoutopenoptions._data-test-subj_.md) | string | | | [className](./kibana-plugin-core-public.overlayflyoutopenoptions.classname.md) | string | | | [closeButtonAriaLabel](./kibana-plugin-core-public.overlayflyoutopenoptions.closebuttonarialabel.md) | string | | | [hideCloseButton](./kibana-plugin-core-public.overlayflyoutopenoptions.hideclosebutton.md) | boolean | | | [maxWidth](./kibana-plugin-core-public.overlayflyoutopenoptions.maxwidth.md) | boolean | number | string | | +| [onClose](./kibana-plugin-core-public.overlayflyoutopenoptions.onclose.md) | (flyout: OverlayRef) => void | EuiFlyout onClose handler. If provided the consumer is responsible for calling flyout.close() to close the flyout; | | [ownFocus](./kibana-plugin-core-public.overlayflyoutopenoptions.ownfocus.md) | boolean | | | [size](./kibana-plugin-core-public.overlayflyoutopenoptions.size.md) | EuiFlyoutSize | | diff --git a/docs/development/core/public/kibana-plugin-core-public.overlayflyoutopenoptions.onclose.md b/docs/development/core/public/kibana-plugin-core-public.overlayflyoutopenoptions.onclose.md new file mode 100644 index 0000000000000..5cfbba4c84a36 --- /dev/null +++ b/docs/development/core/public/kibana-plugin-core-public.overlayflyoutopenoptions.onclose.md @@ -0,0 +1,13 @@ + + +[Home](./index.md) > [kibana-plugin-core-public](./kibana-plugin-core-public.md) > [OverlayFlyoutOpenOptions](./kibana-plugin-core-public.overlayflyoutopenoptions.md) > [onClose](./kibana-plugin-core-public.overlayflyoutopenoptions.onclose.md) + +## OverlayFlyoutOpenOptions.onClose property + +EuiFlyout onClose handler. If provided the consumer is responsible for calling flyout.close() to close the flyout; + +Signature: + +```typescript +onClose?: (flyout: OverlayRef) => void; +``` diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs._constructor_.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs._constructor_.md index c4d09001087de..9111941b368ee 100644 --- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs._constructor_.md +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigs._constructor_.md @@ -13,7 +13,7 @@ constructor(indexPattern: IndexPattern, configStates: Pick & Pick<{ type: string | IAggType; @@ -27,6 +27,6 @@ constructor(indexPattern: IndexPattern, configStates: PickIndexPattern | | -| configStates | Pick<Pick<{
type: string;
enabled?: boolean | undefined;
id?: string | undefined;
params?: {} | import("@kbn/common-utils").SerializableRecord | undefined;
schema?: string | undefined;
}, "schema" | "enabled" | "id" | "params"> & Pick<{
type: string | IAggType;
}, "type"> & Pick<{
type: string | IAggType;
}, never>, "schema" | "type" | "enabled" | "id" | "params">[] | undefined | | +| configStates | Pick<Pick<{
type: string;
enabled?: boolean | undefined;
id?: string | undefined;
params?: {} | import("@kbn/utility-types").SerializableRecord | undefined;
schema?: string | undefined;
}, "schema" | "enabled" | "id" | "params"> & Pick<{
type: string | IAggType;
}, "type"> & Pick<{
type: string | IAggType;
}, never>, "schema" | "type" | "enabled" | "id" | "params">[] | undefined | | | opts | AggConfigsOptions | | diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigserialized.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigserialized.md index 9660a15d94a69..631569464e176 100644 --- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigserialized.md +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.aggconfigserialized.md @@ -13,7 +13,7 @@ export declare type AggConfigSerialized = Ensure<{ type: string; enabled?: boolean; id?: string; - params?: {} | SerializableState; + params?: {} | SerializableRecord; schema?: string; -}, SerializableState>; +}, SerializableRecord>; ``` diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.eskuery.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.eskuery.md index 3b768404aab95..c25cd70e99b4f 100644 --- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.eskuery.md +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.eskuery.md @@ -14,7 +14,7 @@ ```typescript esKuery: { nodeTypes: import("@kbn/es-query/target_types/kuery/node_types").NodeTypes; - fromKueryExpression: (expression: any, parseOptions?: Partial | undefined) => import("@kbn/es-query").KueryNode; - toElasticsearchQuery: (node: import("@kbn/es-query").KueryNode, indexPattern?: import("@kbn/es-query").IndexPatternBase | undefined, config?: Record | undefined, context?: Record | undefined) => import("@kbn/common-utils").JsonObject; + fromKueryExpression: (expression: string | import("@elastic/elasticsearch/api/types").QueryDslQueryContainer, parseOptions?: Partial | undefined) => import("@kbn/es-query").KueryNode; + toElasticsearchQuery: (node: import("@kbn/es-query").KueryNode, indexPattern?: import("@kbn/es-query").IndexPatternBase | undefined, config?: Record | undefined, context?: Record | undefined) => import("@kbn/utility-types").JsonObject; } ``` diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.esquery.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.esquery.md index 2a26b009d7447..0ffdf8c98b920 100644 --- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.esquery.md +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.esquery.md @@ -15,12 +15,7 @@ esQuery: { buildEsQuery: typeof import("@kbn/es-query").buildEsQuery; getEsQueryConfig: typeof getEsQueryConfig; - buildQueryFromFilters: (filters: import("@kbn/es-query").Filter[] | undefined, indexPattern: import("@kbn/es-query").IndexPatternBase | undefined, ignoreFilterIfFieldNotInIndex?: boolean | undefined) => { - must: never[]; - filter: import("@kbn/es-query").Filter[]; - should: never[]; - must_not: import("@kbn/es-query").Filter[]; - }; + buildQueryFromFilters: (filters: import("@kbn/es-query").Filter[] | undefined, indexPattern: import("@kbn/es-query").IndexPatternBase | undefined, ignoreFilterIfFieldNotInIndex?: boolean | undefined) => import("@kbn/es-query").BoolQuery; luceneStringToDsl: typeof import("@kbn/es-query").luceneStringToDsl; decorateQuery: typeof import("@kbn/es-query").decorateQuery; } diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.filtermanager.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.filtermanager.md index 7baa23fffe0d3..7cfc8c4e48805 100644 --- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.filtermanager.md +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.filtermanager.md @@ -24,7 +24,7 @@ export declare class FilterManager implements PersistableStateService | [getAllMigrations](./kibana-plugin-plugins-data-public.filtermanager.getallmigrations.md) | | () => {} | | | [inject](./kibana-plugin-plugins-data-public.filtermanager.inject.md) | | any | | | [migrateToLatest](./kibana-plugin-plugins-data-public.filtermanager.migratetolatest.md) | | any | | -| [telemetry](./kibana-plugin-plugins-data-public.filtermanager.telemetry.md) | | (filters: import("@kbn/common-utils").SerializableRecord, collector: unknown) => {} | | +| [telemetry](./kibana-plugin-plugins-data-public.filtermanager.telemetry.md) | | (filters: import("@kbn/utility-types").SerializableRecord, collector: unknown) => {} | | ## Methods diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.filtermanager.telemetry.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.filtermanager.telemetry.md index df5b4ea0a26c8..0eeb026abf2e1 100644 --- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.filtermanager.telemetry.md +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.filtermanager.telemetry.md @@ -7,5 +7,5 @@ Signature: ```typescript -telemetry: (filters: import("@kbn/common-utils").SerializableRecord, collector: unknown) => {}; +telemetry: (filters: import("@kbn/utility-types").SerializableRecord, collector: unknown) => {}; ``` diff --git a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.eskuery.md b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.eskuery.md index f0261648e32ab..620a547d30245 100644 --- a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.eskuery.md +++ b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.eskuery.md @@ -9,7 +9,7 @@ ```typescript esKuery: { nodeTypes: import("@kbn/es-query/target_types/kuery/node_types").NodeTypes; - fromKueryExpression: (expression: any, parseOptions?: Partial | undefined) => import("@kbn/es-query").KueryNode; + fromKueryExpression: (expression: string | import("@elastic/elasticsearch/api/types").QueryDslQueryContainer, parseOptions?: Partial | undefined) => import("@kbn/es-query").KueryNode; toElasticsearchQuery: (node: import("@kbn/es-query").KueryNode, indexPattern?: import("@kbn/es-query").IndexPatternBase | undefined, config?: Record | undefined, context?: Record | undefined) => import("@kbn/utility-types").JsonObject; } ``` diff --git a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.esquery.md b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.esquery.md index 8dfea00081d89..38cad914e72d0 100644 --- a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.esquery.md +++ b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.esquery.md @@ -8,12 +8,7 @@ ```typescript esQuery: { - buildQueryFromFilters: (filters: import("@kbn/es-query").Filter[] | undefined, indexPattern: import("@kbn/es-query").IndexPatternBase | undefined, ignoreFilterIfFieldNotInIndex?: boolean | undefined) => { - must: never[]; - filter: import("@kbn/es-query").Filter[]; - should: never[]; - must_not: import("@kbn/es-query").Filter[]; - }; + buildQueryFromFilters: (filters: import("@kbn/es-query").Filter[] | undefined, indexPattern: import("@kbn/es-query").IndexPatternBase | undefined, ignoreFilterIfFieldNotInIndex?: boolean | undefined) => import("@kbn/es-query").BoolQuery; getEsQueryConfig: typeof getEsQueryConfig; buildEsQuery: typeof import("@kbn/es-query").buildEsQuery; } diff --git a/docs/development/plugins/embeddable/public/kibana-plugin-plugins-embeddable-public.container.md b/docs/development/plugins/embeddable/public/kibana-plugin-plugins-embeddable-public.container.md index d14adc31e4123..00f75470900d3 100644 --- a/docs/development/plugins/embeddable/public/kibana-plugin-plugins-embeddable-public.container.md +++ b/docs/development/plugins/embeddable/public/kibana-plugin-plugins-embeddable-public.container.md @@ -38,6 +38,7 @@ export declare abstract class Container + +[Home](./index.md) > [kibana-plugin-plugins-embeddable-public](./kibana-plugin-plugins-embeddable-public.md) > [Container](./kibana-plugin-plugins-embeddable-public.container.md) > [setChildLoaded](./kibana-plugin-plugins-embeddable-public.container.setchildloaded.md) + +## Container.setChildLoaded() method + +Signature: + +```typescript +setChildLoaded(embeddable: IEmbeddable): void; +``` + +## Parameters + +| Parameter | Type | Description | +| --- | --- | --- | +| embeddable | IEmbeddable | | + +Returns: + +`void` + diff --git a/docs/development/plugins/embeddable/public/kibana-plugin-plugins-embeddable-public.embeddable.deferembeddableload.md b/docs/development/plugins/embeddable/public/kibana-plugin-plugins-embeddable-public.embeddable.deferembeddableload.md new file mode 100644 index 0000000000000..86ef74ef312ec --- /dev/null +++ b/docs/development/plugins/embeddable/public/kibana-plugin-plugins-embeddable-public.embeddable.deferembeddableload.md @@ -0,0 +1,11 @@ + + +[Home](./index.md) > [kibana-plugin-plugins-embeddable-public](./kibana-plugin-plugins-embeddable-public.md) > [Embeddable](./kibana-plugin-plugins-embeddable-public.embeddable.md) > [deferEmbeddableLoad](./kibana-plugin-plugins-embeddable-public.embeddable.deferembeddableload.md) + +## Embeddable.deferEmbeddableLoad property + +Signature: + +```typescript +readonly deferEmbeddableLoad: boolean; +``` diff --git a/docs/development/plugins/embeddable/public/kibana-plugin-plugins-embeddable-public.embeddable.md b/docs/development/plugins/embeddable/public/kibana-plugin-plugins-embeddable-public.embeddable.md index fe64bcf7c1177..ef9e0ee72862f 100644 --- a/docs/development/plugins/embeddable/public/kibana-plugin-plugins-embeddable-public.embeddable.md +++ b/docs/development/plugins/embeddable/public/kibana-plugin-plugins-embeddable-public.embeddable.md @@ -20,6 +20,7 @@ export declare abstract class Embeddableboolean | | | [fatalError](./kibana-plugin-plugins-embeddable-public.embeddable.fatalerror.md) | | Error | | | [id](./kibana-plugin-plugins-embeddable-public.embeddable.id.md) | | string | | | [input](./kibana-plugin-plugins-embeddable-public.embeddable.input.md) | | TEmbeddableInput | | @@ -48,6 +49,7 @@ export declare abstract class EmbeddableIn case if input data did change and reload is requested input$ and output$ would still emit before reload is calledThe order would be as follows: input$ output$ reload() \-\-\-- updated$ | | [render(el)](./kibana-plugin-plugins-embeddable-public.embeddable.render.md) | | | +| [setInitializationFinished()](./kibana-plugin-plugins-embeddable-public.embeddable.setinitializationfinished.md) | | communicate to the parent embeddable that this embeddable's initialization is finished. This only applies to embeddables which defer their loading state with deferEmbeddableLoad. | | [supportedTriggers()](./kibana-plugin-plugins-embeddable-public.embeddable.supportedtriggers.md) | | | | [updateInput(changes)](./kibana-plugin-plugins-embeddable-public.embeddable.updateinput.md) | | | | [updateOutput(outputChanges)](./kibana-plugin-plugins-embeddable-public.embeddable.updateoutput.md) | | | diff --git a/docs/development/plugins/embeddable/public/kibana-plugin-plugins-embeddable-public.embeddable.setinitializationfinished.md b/docs/development/plugins/embeddable/public/kibana-plugin-plugins-embeddable-public.embeddable.setinitializationfinished.md new file mode 100644 index 0000000000000..d407c7b820454 --- /dev/null +++ b/docs/development/plugins/embeddable/public/kibana-plugin-plugins-embeddable-public.embeddable.setinitializationfinished.md @@ -0,0 +1,17 @@ + + +[Home](./index.md) > [kibana-plugin-plugins-embeddable-public](./kibana-plugin-plugins-embeddable-public.md) > [Embeddable](./kibana-plugin-plugins-embeddable-public.embeddable.md) > [setInitializationFinished](./kibana-plugin-plugins-embeddable-public.embeddable.setinitializationfinished.md) + +## Embeddable.setInitializationFinished() method + +communicate to the parent embeddable that this embeddable's initialization is finished. This only applies to embeddables which defer their loading state with deferEmbeddableLoad. + +Signature: + +```typescript +protected setInitializationFinished(): void; +``` +Returns: + +`void` + diff --git a/docs/development/plugins/embeddable/public/kibana-plugin-plugins-embeddable-public.icontainer.md b/docs/development/plugins/embeddable/public/kibana-plugin-plugins-embeddable-public.icontainer.md index cba430069c7a4..701948bbba4d9 100644 --- a/docs/development/plugins/embeddable/public/kibana-plugin-plugins-embeddable-public.icontainer.md +++ b/docs/development/plugins/embeddable/public/kibana-plugin-plugins-embeddable-public.icontainer.md @@ -18,6 +18,7 @@ export interface IContainer + +[Home](./index.md) > [kibana-plugin-plugins-embeddable-public](./kibana-plugin-plugins-embeddable-public.md) > [IContainer](./kibana-plugin-plugins-embeddable-public.icontainer.md) > [setChildLoaded](./kibana-plugin-plugins-embeddable-public.icontainer.setchildloaded.md) + +## IContainer.setChildLoaded() method + +Embeddables which have deferEmbeddableLoad set to true need to manually call setChildLoaded on their parent container to communicate when they have finished loading. + +Signature: + +```typescript +setChildLoaded(embeddable: E): void; +``` + +## Parameters + +| Parameter | Type | Description | +| --- | --- | --- | +| embeddable | E | the embeddable to set | + +Returns: + +`void` + diff --git a/docs/development/plugins/embeddable/public/kibana-plugin-plugins-embeddable-public.iembeddable.deferembeddableload.md b/docs/development/plugins/embeddable/public/kibana-plugin-plugins-embeddable-public.iembeddable.deferembeddableload.md new file mode 100644 index 0000000000000..638c66690a4ae --- /dev/null +++ b/docs/development/plugins/embeddable/public/kibana-plugin-plugins-embeddable-public.iembeddable.deferembeddableload.md @@ -0,0 +1,13 @@ + + +[Home](./index.md) > [kibana-plugin-plugins-embeddable-public](./kibana-plugin-plugins-embeddable-public.md) > [IEmbeddable](./kibana-plugin-plugins-embeddable-public.iembeddable.md) > [deferEmbeddableLoad](./kibana-plugin-plugins-embeddable-public.iembeddable.deferembeddableload.md) + +## IEmbeddable.deferEmbeddableLoad property + +If set to true, defer embeddable load tells the container that this embeddable type isn't completely loaded when the constructor returns. This embeddable will have to manually call setChildLoaded on its parent when all of its initial output is finalized. For instance, after loading a saved object. + +Signature: + +```typescript +readonly deferEmbeddableLoad: boolean; +``` diff --git a/docs/development/plugins/embeddable/public/kibana-plugin-plugins-embeddable-public.iembeddable.md b/docs/development/plugins/embeddable/public/kibana-plugin-plugins-embeddable-public.iembeddable.md index f96477ed65a04..dd0227b1ef72b 100644 --- a/docs/development/plugins/embeddable/public/kibana-plugin-plugins-embeddable-public.iembeddable.md +++ b/docs/development/plugins/embeddable/public/kibana-plugin-plugins-embeddable-public.iembeddable.md @@ -14,6 +14,7 @@ export interface IEmbeddableboolean | If set to true, defer embeddable load tells the container that this embeddable type isn't completely loaded when the constructor returns. This embeddable will have to manually call setChildLoaded on its parent when all of its initial output is finalized. For instance, after loading a saved object. | | [enhancements](./kibana-plugin-plugins-embeddable-public.iembeddable.enhancements.md) | object | Extra abilities added to Embeddable by *_enhanced plugins. | | [fatalError](./kibana-plugin-plugins-embeddable-public.iembeddable.fatalerror.md) | Error | If this embeddable has encountered a fatal error, that error will be stored here | | [id](./kibana-plugin-plugins-embeddable-public.iembeddable.id.md) | string | A unique identifier for this embeddable. Mainly only used by containers to map their Panel States to a child embeddable instance. | diff --git a/docs/maps/images/app_gis_icon.png b/docs/maps/images/app_gis_icon.png new file mode 100644 index 0000000000000..5fbc032584130 Binary files /dev/null and b/docs/maps/images/app_gis_icon.png differ diff --git a/docs/maps/maps-getting-started.asciidoc b/docs/maps/maps-getting-started.asciidoc index 97a5fc7ddaef4..548a574293403 100644 --- a/docs/maps/maps-getting-started.asciidoc +++ b/docs/maps/maps-getting-started.asciidoc @@ -34,8 +34,7 @@ refer to <>. . Open the main menu, and then click *Dashboard*. . Click **Create dashboard**. . Set the time range to *Last 7 days*. -. Click **Create panel**. -. Click **Maps**. +. Click the **Create new Maps** icon image:maps/images/app_gis_icon.png[] [float] [[maps-add-choropleth-layer]] @@ -62,14 +61,15 @@ and lighter shades will symbolize countries with less traffic. . Add a Tooltip field: -** Select **ISO 3166-1 alpha-2 code** and **name**. -** Click **Add**. +** **ISO 3166-1 alpha-2 code** is added by default. +** Click **+ Add** to open field select. +** Select **name** and click *Add*. -. In **Layer style**, set: +. In **Layer style**: -** **Fill color: As number** to the grey color ramp -** **Border color** to white -** **Label** to symbol label +** Set **Fill color: As number** to the grey color ramp. +** Set **Border color** to white. +** Under **Label**, change **By value** to **Fixed**. . Click **Save & close**. + @@ -135,9 +135,10 @@ grids with less bytes transferred. ** **Name** to `Total Requests and Bytes` ** **Visibility** to the range [0, 9] ** **Opacity** to 100% -. In **Metrics**, use: -** **Agregation** set to **Count**, and -** **Aggregation** set to **Sum** with **Field** set to **bytes** +. In **Metrics**: +** Set **Agregation** to **Count**. +** Click **Add metric**. +** Set **Aggregation** to **Sum** with **Field** set to **bytes**. . In **Layer style**, change **Symbol size**: ** Set the field select to *sum bytes*. ** Set the min size to 7 and the max size to 25 px. diff --git a/docs/maps/vector-layer.asciidoc b/docs/maps/vector-layer.asciidoc index 5017ecf91dffd..7191197c27dbe 100644 --- a/docs/maps/vector-layer.asciidoc +++ b/docs/maps/vector-layer.asciidoc @@ -14,6 +14,8 @@ To add a vector layer to your map, click *Add layer*, then select one of the fol *Clusters and grids*:: Geospatial data grouped in grids with metrics for each gridded cell. The index must contain at least one field mapped as {ref}/geo-point.html[geo_point] or {ref}/geo-shape.html[geo_shape]. +*Create index*:: Draw shapes on the map and index in Elasticsearch. + *Documents*:: Points, lines, and polyons from Elasticsearch. The index must contain at least one field mapped as {ref}/geo-point.html[geo_point] or {ref}/geo-shape.html[geo_shape]. + diff --git a/docs/settings/alert-action-settings.asciidoc b/docs/settings/alert-action-settings.asciidoc index c9847effd5f49..f168195e10ef5 100644 --- a/docs/settings/alert-action-settings.asciidoc +++ b/docs/settings/alert-action-settings.asciidoc @@ -41,7 +41,7 @@ You can configure the following settings in the `kibana.yml` file. [cols="2*<"] |=== | `xpack.actions.enabled` - | Feature toggle that enables Actions in {kib}. + | Deprecated. This will be removed in 8.0. Feature toggle that enables Actions in {kib}. If `false`, all features dependent on Actions are disabled, including the *Observability* and *Security* apps. Default: `true`. | `xpack.actions.allowedHosts` {ess-icon} diff --git a/docs/settings/task-manager-settings.asciidoc b/docs/settings/task-manager-settings.asciidoc index 387d2308aa5e8..fa89b7780e475 100644 --- a/docs/settings/task-manager-settings.asciidoc +++ b/docs/settings/task-manager-settings.asciidoc @@ -22,6 +22,9 @@ Task Manager runs background tasks by polling for work on an interval. You can | `xpack.task_manager.request_capacity` | How many requests can Task Manager buffer before it rejects new requests. Defaults to 1000. +| `xpack.task_manager.index` + | The name of the index used to store task information. Defaults to `.kibana_task_manager`. + | `xpack.task_manager.max_workers` | The maximum number of tasks that this Kibana instance will run simultaneously. Defaults to 10. Starting in 8.0, it will not be possible to set the value greater than 100. diff --git a/docs/user/management.asciidoc b/docs/user/management.asciidoc index 2f9f1fe371dc3..4e5f70db9aef6 100644 --- a/docs/user/management.asciidoc +++ b/docs/user/management.asciidoc @@ -83,7 +83,7 @@ connectors>> for triggering actions. A report can contain a dashboard, visualization, saved search, or Canvas workpad. | Machine Learning Jobs -| View your <> and +| View, export, and import your <> and <> jobs. Open the Single Metric Viewer or Anomaly Explorer to see your {anomaly-detect} results. diff --git a/docs/user/production-considerations/task-manager-production-considerations.asciidoc b/docs/user/production-considerations/task-manager-production-considerations.asciidoc index 36745b913544b..17eae59ff2f9c 100644 --- a/docs/user/production-considerations/task-manager-production-considerations.asciidoc +++ b/docs/user/production-considerations/task-manager-production-considerations.asciidoc @@ -12,7 +12,7 @@ This has three major benefits: [IMPORTANT] ============================================== -Task definitions for alerts and actions are stored in the index `.kibana_task_manager`. +Task definitions for alerts and actions are stored in the index specified by <>. The default is `.kibana_task_manager`. You must have at least one replica of this index for production deployments. diff --git a/examples/hello_world/tsconfig.json b/examples/hello_world/tsconfig.json index 06d3953e4a6bf..b494fba903415 100644 --- a/examples/hello_world/tsconfig.json +++ b/examples/hello_world/tsconfig.json @@ -1,8 +1,7 @@ { "extends": "../../tsconfig.json", "compilerOptions": { - "outDir": "./target", - "skipLibCheck": true + "outDir": "./target/types" }, "include": [ "index.ts", @@ -15,6 +14,6 @@ "exclude": [], "references": [ { "path": "../../src/core/tsconfig.json" }, - { "path": "../developer_examples/tsconfig.json" }, + { "path": "../developer_examples/tsconfig.json" } ] } diff --git a/examples/search_examples/public/search_sessions/app.tsx b/examples/search_examples/public/search_sessions/app.tsx index 7fdf91537c977..173cf91cd9c71 100644 --- a/examples/search_examples/public/search_sessions/app.tsx +++ b/examples/search_examples/public/search_sessions/app.tsx @@ -702,7 +702,6 @@ function doSearch( const startTs = performance.now(); // Submit the search request using the `data.search` service. - // @ts-expect-error request.params is incompatible. Filter is not assignable to QueryDslQueryContainer return data.search .search(req, { sessionId }) .pipe( diff --git a/package.json b/package.json index d7a072d1caef0..98b2252cc0dba 100644 --- a/package.json +++ b/package.json @@ -89,7 +89,7 @@ "**/underscore": "^1.13.1" }, "engines": { - "node": "14.17.3", + "node": "14.17.5", "yarn": "^1.21.1" }, "dependencies": { @@ -309,6 +309,7 @@ "nock": "12.0.3", "node-fetch": "^2.6.1", "node-forge": "^0.10.0", + "node-sql-parser": "^3.6.1", "nodemailer": "^6.6.2", "normalize-path": "^3.0.0", "object-hash": "^1.3.1", @@ -336,7 +337,7 @@ "re-resizable": "^6.1.1", "re2": "^1.15.4", "react": "^16.12.0", - "react-ace": "^5.9.0", + "react-ace": "^7.0.5", "react-beautiful-dnd": "^13.0.0", "react-color": "^2.13.8", "react-dom": "^16.12.0", @@ -446,7 +447,7 @@ "@babel/traverse": "^7.12.12", "@babel/types": "^7.12.12", "@bazel/ibazel": "^0.15.10", - "@bazel/typescript": "^3.7.0", + "@bazel/typescript": "^3.8.0", "@cypress/snapshot": "^2.1.7", "@cypress/webpack-preprocessor": "^5.6.0", "@elastic/eslint-config-kibana": "link:bazel-bin/packages/elastic-eslint-config-kibana", diff --git a/packages/kbn-es-archiver/.babelrc b/packages/kbn-es-archiver/.babelrc new file mode 100644 index 0000000000000..7da72d1779128 --- /dev/null +++ b/packages/kbn-es-archiver/.babelrc @@ -0,0 +1,3 @@ +{ + "presets": ["@kbn/babel-preset/node_preset"] +} diff --git a/packages/kbn-es-archiver/BUILD.bazel b/packages/kbn-es-archiver/BUILD.bazel index b7040b584a318..90c63f82b72fa 100644 --- a/packages/kbn-es-archiver/BUILD.bazel +++ b/packages/kbn-es-archiver/BUILD.bazel @@ -1,5 +1,6 @@ load("@npm//@bazel/typescript:index.bzl", "ts_config", "ts_project") load("@build_bazel_rules_nodejs//:index.bzl", "js_library", "pkg_npm") +load("//src/dev/bazel:index.bzl", "jsts_transpiler") PKG_BASE_NAME = "kbn-es-archiver" PKG_REQUIRE_NAME = "@kbn/es-archiver" @@ -27,7 +28,7 @@ NPM_MODULE_EXTRA_FILES = [ "package.json", ] -SRC_DEPS = [ +RUNTIME_DEPS = [ "//packages/kbn-dev-utils", "//packages/kbn-test", "//packages/kbn-utils", @@ -43,6 +44,13 @@ SRC_DEPS = [ ] TYPES_DEPS = [ + "//packages/kbn-dev-utils", + "//packages/kbn-test", + "//packages/kbn-utils", + "@npm//@elastic/elasticsearch", + "@npm//aggregate-error", + "@npm//globby", + "@npm//zlib", "@npm//@types/bluebird", "@npm//@types/chance", "@npm//@types/jest", @@ -52,7 +60,11 @@ TYPES_DEPS = [ "@npm//@types/sinon", ] -DEPS = SRC_DEPS + TYPES_DEPS +jsts_transpiler( + name = "target_node", + srcs = SRCS, + build_pkg_name = package_name(), +) ts_config( name = "tsconfig", @@ -64,13 +76,14 @@ ts_config( ) ts_project( - name = "tsc", + name = "tsc_types", args = ['--pretty'], srcs = SRCS, - deps = DEPS, + deps = TYPES_DEPS, declaration = True, declaration_map = True, - out_dir = "target", + emit_declaration_only = True, + out_dir = "target_types", source_map = True, root_dir = "src", tsconfig = ":tsconfig", @@ -79,7 +92,7 @@ ts_project( js_library( name = PKG_BASE_NAME, srcs = NPM_MODULE_EXTRA_FILES, - deps = DEPS + [":tsc"], + deps = RUNTIME_DEPS + [":target_node", ":tsc_types"], package_name = PKG_REQUIRE_NAME, visibility = ["//visibility:public"], ) diff --git a/packages/kbn-es-archiver/package.json b/packages/kbn-es-archiver/package.json index e8eb7b5f8f1c9..0cce08eaf0352 100644 --- a/packages/kbn-es-archiver/package.json +++ b/packages/kbn-es-archiver/package.json @@ -3,8 +3,8 @@ "version": "1.0.0", "license": "SSPL-1.0 OR Elastic License 2.0", "private": "true", - "main": "target/index.js", - "types": "target/index.d.ts", + "main": "target_node/index.js", + "types": "target_types/index.d.ts", "kibana": { "devOnly": true } diff --git a/packages/kbn-es-archiver/tsconfig.json b/packages/kbn-es-archiver/tsconfig.json index dce71fd6cd4a1..15c846f052b47 100644 --- a/packages/kbn-es-archiver/tsconfig.json +++ b/packages/kbn-es-archiver/tsconfig.json @@ -1,9 +1,10 @@ { "extends": "../../tsconfig.bazel.json", "compilerOptions": { - "outDir": "./target/types", "declaration": true, "declarationMap": true, + "emitDeclarationOnly": true, + "outDir": "./target_types", "sourceMap": true, "sourceRoot": "../../../../packages/kbn-es-archiver/src", "types": [ diff --git a/packages/kbn-es-query/src/es_query/build_es_query.ts b/packages/kbn-es-query/src/es_query/build_es_query.ts index c01b11f580ba6..42fa26ac50a95 100644 --- a/packages/kbn-es-query/src/es_query/build_es_query.ts +++ b/packages/kbn-es-query/src/es_query/build_es_query.ts @@ -7,11 +7,12 @@ */ import { groupBy, has, isEqual } from 'lodash'; +import { SerializableRecord } from '@kbn/utility-types'; import { buildQueryFromKuery } from './from_kuery'; import { buildQueryFromFilters } from './from_filters'; import { buildQueryFromLucene } from './from_lucene'; import { Filter, Query } from '../filters'; -import { IndexPatternBase } from './types'; +import { BoolQuery, IndexPatternBase } from './types'; import { KueryQueryOptions } from '../kuery'; /** @@ -20,7 +21,7 @@ import { KueryQueryOptions } from '../kuery'; */ export type EsQueryConfig = KueryQueryOptions & { allowLeadingWildcards: boolean; - queryStringOptions: Record; + queryStringOptions: SerializableRecord; ignoreFilterIfFieldNotInIndex: boolean; }; @@ -49,11 +50,11 @@ export function buildEsQuery( queryStringOptions: {}, ignoreFilterIfFieldNotInIndex: false, } -) { +): { bool: BoolQuery } { queries = Array.isArray(queries) ? queries : [queries]; filters = Array.isArray(filters) ? filters : [filters]; - const validQueries = queries.filter((query: any) => has(query, 'query')); + const validQueries = queries.filter((query) => has(query, 'query')); const queriesByLanguage = groupBy(validQueries, 'language'); const kueryQuery = buildQueryFromKuery( indexPattern, diff --git a/packages/kbn-es-query/src/es_query/decorate_query.ts b/packages/kbn-es-query/src/es_query/decorate_query.ts index b6623b9b1946c..e5bcf01a45915 100644 --- a/packages/kbn-es-query/src/es_query/decorate_query.ts +++ b/packages/kbn-es-query/src/es_query/decorate_query.ts @@ -6,9 +6,11 @@ * Side Public License, v 1. */ +import { estypes } from '@elastic/elasticsearch'; +import { SerializableRecord } from '@kbn/utility-types'; import { extend, defaults } from 'lodash'; import { getTimeZoneFromSettings } from '../utils'; -import { DslQuery, isEsQueryString } from './es_query_dsl'; +import { isEsQueryString } from './es_query_dsl'; /** * Decorate queries with default parameters @@ -21,8 +23,8 @@ import { DslQuery, isEsQueryString } from './es_query_dsl'; */ export function decorateQuery( - query: DslQuery, - queryStringOptions: Record | string, + query: estypes.QueryDslQueryContainer, + queryStringOptions: SerializableRecord | string, dateFormatTZ?: string ) { if (isEsQueryString(query)) { diff --git a/packages/kbn-es-query/src/es_query/es_query_dsl.ts b/packages/kbn-es-query/src/es_query/es_query_dsl.ts index 6cff8b0ff47c7..90b234c0e29d7 100644 --- a/packages/kbn-es-query/src/es_query/es_query_dsl.ts +++ b/packages/kbn-es-query/src/es_query/es_query_dsl.ts @@ -8,26 +8,6 @@ import { has } from 'lodash'; -export interface DslRangeQuery { - range: { - [name: string]: { - gte: number; - lte: number; - format: string; - }; - }; -} - -export interface DslMatchQuery { - match: { - [name: string]: { - query: string; - operator: string; - zero_terms_query: string; - }; - }; -} - export interface DslQueryStringQuery { query_string: { query: string; @@ -35,24 +15,6 @@ export interface DslQueryStringQuery { }; } -export interface DslMatchAllQuery { - match_all: Record; -} - -export interface DslTermQuery { - term: Record; -} - -/** - * @public - */ -export type DslQuery = - | DslRangeQuery - | DslMatchQuery - | DslQueryStringQuery - | DslMatchAllQuery - | DslTermQuery; - /** @internal */ export const isEsQueryString = (query: any): query is DslQueryStringQuery => has(query, 'query_string.query'); diff --git a/packages/kbn-es-query/src/es_query/from_filters.ts b/packages/kbn-es-query/src/es_query/from_filters.ts index 94def4008a2bc..ea2ee18442703 100644 --- a/packages/kbn-es-query/src/es_query/from_filters.ts +++ b/packages/kbn-es-query/src/es_query/from_filters.ts @@ -7,10 +7,11 @@ */ import { isUndefined } from 'lodash'; +import { estypes } from '@elastic/elasticsearch'; import { migrateFilter } from './migrate_filter'; import { filterMatchesIndex } from './filter_matches_index'; import { Filter, cleanFilter, isFilterDisabled } from '../filters'; -import { IndexPatternBase } from './types'; +import { BoolQuery, IndexPatternBase } from './types'; import { handleNestedFilter } from './handle_nested_filter'; /** @@ -33,20 +34,19 @@ const filterNegate = (reverse: boolean) => (filter: Filter) => { * @param {Object} filter - The filter to translate * @return {Object} the query version of that filter */ -const translateToQuery = (filter: Filter) => { - if (!filter) return; - +const translateToQuery = (filter: Partial): estypes.QueryDslQueryContainer => { if (filter.query) { return filter.query; } - return filter; + // TODO: investigate what's going on here! What does this mean for filters that don't have a query! + return filter as estypes.QueryDslQueryContainer; }; /** * @param filters * @param indexPattern - * @param ignoreFilterIfFieldNotInIndex by default filters that use fields that can't be found in the specified index pattern are not applied. Set this to true if you want to apply them any way. + * @param ignoreFilterIfFieldNotInIndex by default filters that use fields that can't be found in the specified index pattern are not applied. Set this to true if you want to apply them anyway. * @returns An EQL query * * @public @@ -55,11 +55,12 @@ export const buildQueryFromFilters = ( filters: Filter[] = [], indexPattern: IndexPatternBase | undefined, ignoreFilterIfFieldNotInIndex: boolean = false -) => { +): BoolQuery => { filters = filters.filter((filter) => filter && !isFilterDisabled(filter)); const filtersToESQueries = (negate: boolean) => { return filters + .filter((f) => !!f) .filter(filterNegate(negate)) .filter( (filter) => !ignoreFilterIfFieldNotInIndex || filterMatchesIndex(filter, indexPattern) @@ -68,8 +69,8 @@ export const buildQueryFromFilters = ( return migrateFilter(filter, indexPattern); }) .map((filter) => handleNestedFilter(filter, indexPattern)) - .map(translateToQuery) - .map(cleanFilter); + .map(cleanFilter) + .map(translateToQuery); }; return { diff --git a/packages/kbn-es-query/src/es_query/from_kuery.ts b/packages/kbn-es-query/src/es_query/from_kuery.ts index bf66057e49327..949f9691e9e6d 100644 --- a/packages/kbn-es-query/src/es_query/from_kuery.ts +++ b/packages/kbn-es-query/src/es_query/from_kuery.ts @@ -6,9 +6,10 @@ * Side Public License, v 1. */ +import { SerializableRecord } from '@kbn/utility-types'; import { Query } from '../filters'; import { fromKueryExpression, toElasticsearchQuery, nodeTypes, KueryNode } from '../kuery'; -import { IndexPatternBase } from './types'; +import { BoolQuery, IndexPatternBase } from './types'; /** @internal */ export function buildQueryFromKuery( @@ -17,7 +18,7 @@ export function buildQueryFromKuery( allowLeadingWildcards: boolean = false, dateFormatTZ?: string, filtersInMustClause: boolean = false -) { +): BoolQuery { const queryASTs = queries.map((query) => { return fromKueryExpression(query.query, { allowLeadingWildcards }); }); @@ -28,7 +29,7 @@ export function buildQueryFromKuery( function buildQuery( indexPattern: IndexPatternBase | undefined, queryASTs: KueryNode[], - config: Record = {} + config: SerializableRecord = {} ) { const compoundQueryAST = nodeTypes.function.buildNode('and', queryASTs); const kueryQuery = toElasticsearchQuery(compoundQueryAST, indexPattern, config); diff --git a/packages/kbn-es-query/src/es_query/from_lucene.ts b/packages/kbn-es-query/src/es_query/from_lucene.ts index ef4becd1d1584..d00614b31347f 100644 --- a/packages/kbn-es-query/src/es_query/from_lucene.ts +++ b/packages/kbn-es-query/src/es_query/from_lucene.ts @@ -6,16 +6,18 @@ * Side Public License, v 1. */ +import { SerializableRecord } from '@kbn/utility-types'; import { Query } from '..'; import { decorateQuery } from './decorate_query'; import { luceneStringToDsl } from './lucene_string_to_dsl'; +import { BoolQuery } from './types'; /** @internal */ export function buildQueryFromLucene( queries: Query[], - queryStringOptions: Record, + queryStringOptions: SerializableRecord, dateFormatTZ?: string -) { +): BoolQuery { const combinedQueries = (queries || []).map((query) => { const queryDsl = luceneStringToDsl(query.query); diff --git a/packages/kbn-es-query/src/es_query/index.ts b/packages/kbn-es-query/src/es_query/index.ts index beba50f50dd81..6e4a58fbe96c3 100644 --- a/packages/kbn-es-query/src/es_query/index.ts +++ b/packages/kbn-es-query/src/es_query/index.ts @@ -10,4 +10,4 @@ export { buildEsQuery, EsQueryConfig } from './build_es_query'; export { buildQueryFromFilters } from './from_filters'; export { luceneStringToDsl } from './lucene_string_to_dsl'; export { decorateQuery } from './decorate_query'; -export { IndexPatternBase, IndexPatternFieldBase, IFieldSubType } from './types'; +export { IndexPatternBase, IndexPatternFieldBase, IFieldSubType, BoolQuery } from './types'; diff --git a/packages/kbn-es-query/src/es_query/lucene_string_to_dsl.ts b/packages/kbn-es-query/src/es_query/lucene_string_to_dsl.ts index 2e4eb5ab7f7c4..91a912a5da0e3 100644 --- a/packages/kbn-es-query/src/es_query/lucene_string_to_dsl.ts +++ b/packages/kbn-es-query/src/es_query/lucene_string_to_dsl.ts @@ -6,8 +6,8 @@ * Side Public License, v 1. */ +import { estypes } from '@elastic/elasticsearch'; import { isString } from 'lodash'; -import { DslQuery } from './es_query_dsl'; /** * @@ -16,7 +16,9 @@ import { DslQuery } from './es_query_dsl'; * * @public */ -export function luceneStringToDsl(query: string | any): DslQuery { +export function luceneStringToDsl( + query: string | estypes.QueryDslQueryContainer +): estypes.QueryDslQueryContainer { if (isString(query)) { if (query.trim() === '') { return { match_all: {} }; diff --git a/packages/kbn-es-query/src/es_query/migrate_filter.ts b/packages/kbn-es-query/src/es_query/migrate_filter.ts index 5edab3e042f5c..8fc0278433645 100644 --- a/packages/kbn-es-query/src/es_query/migrate_filter.ts +++ b/packages/kbn-es-query/src/es_query/migrate_filter.ts @@ -23,7 +23,7 @@ export interface DeprecatedMatchPhraseFilter extends Filter { }; } -function isDeprecatedMatchPhraseFilter(filter: any): filter is DeprecatedMatchPhraseFilter { +function isDeprecatedMatchPhraseFilter(filter: Filter): filter is DeprecatedMatchPhraseFilter { const fieldName = filter.query && filter.query.match && Object.keys(filter.query.match)[0]; return Boolean(fieldName && get(filter, ['query', 'match', fieldName, 'type']) === 'phrase'); diff --git a/packages/kbn-es-query/src/es_query/types.ts b/packages/kbn-es-query/src/es_query/types.ts index d68d9e4a4da22..333536a5f3ecd 100644 --- a/packages/kbn-es-query/src/es_query/types.ts +++ b/packages/kbn-es-query/src/es_query/types.ts @@ -49,3 +49,10 @@ export interface IndexPatternBase { id?: string; title?: string; } + +export interface BoolQuery { + must: estypes.QueryDslQueryContainer[]; + must_not: estypes.QueryDslQueryContainer[]; + filter: estypes.QueryDslQueryContainer[]; + should: estypes.QueryDslQueryContainer[]; +} diff --git a/packages/kbn-es-query/src/filters/helpers/meta_filter.ts b/packages/kbn-es-query/src/filters/helpers/meta_filter.ts index 61b89d45d1962..1406c979bc549 100644 --- a/packages/kbn-es-query/src/filters/helpers/meta_filter.ts +++ b/packages/kbn-es-query/src/filters/helpers/meta_filter.ts @@ -135,4 +135,4 @@ export const isFilters = (x: unknown): x is Filter[] => * * @public */ -export const cleanFilter = (filter: Filter): Filter => omit(filter, ['meta', '$state']) as Filter; +export const cleanFilter = (filter: Filter): Partial => omit(filter, ['meta', '$state']); diff --git a/packages/kbn-es-query/src/kuery/ast/ast.ts b/packages/kbn-es-query/src/kuery/ast/ast.ts index 030b5a8f1c29a..826fa194f1b30 100644 --- a/packages/kbn-es-query/src/kuery/ast/ast.ts +++ b/packages/kbn-es-query/src/kuery/ast/ast.ts @@ -7,15 +7,16 @@ */ import { JsonObject } from '@kbn/utility-types'; +import { estypes } from '@elastic/elasticsearch'; import { nodeTypes } from '../node_types/index'; import { KQLSyntaxError } from '../kuery_syntax_error'; -import { KueryNode, DslQuery, KueryParseOptions } from '../types'; +import { KueryNode, KueryParseOptions } from '../types'; import { parse as parseKuery } from '../grammar'; import { IndexPatternBase } from '../..'; const fromExpression = ( - expression: string | DslQuery, + expression: string | estypes.QueryDslQueryContainer, parseOptions: Partial = {}, parse: Function = parseKuery ): KueryNode => { @@ -27,7 +28,7 @@ const fromExpression = ( }; export const fromLiteralExpression = ( - expression: string | DslQuery, + expression: string | estypes.QueryDslQueryContainer, parseOptions: Partial = {} ): KueryNode => { return fromExpression( @@ -41,7 +42,7 @@ export const fromLiteralExpression = ( }; export const fromKueryExpression = ( - expression: string | DslQuery, + expression: string | estypes.QueryDslQueryContainer, parseOptions: Partial = {} ): KueryNode => { try { diff --git a/packages/kbn-es-query/src/kuery/types.ts b/packages/kbn-es-query/src/kuery/types.ts index 59c48f21425bc..656e06e712079 100644 --- a/packages/kbn-es-query/src/kuery/types.ts +++ b/packages/kbn-es-query/src/kuery/types.ts @@ -6,6 +6,7 @@ * Side Public License, v 1. */ +import { estypes } from '@elastic/elasticsearch'; import { NodeTypes } from './node_types'; /** @public */ @@ -15,10 +16,9 @@ export interface KueryNode { } /** - * TODO: Replace with real type * @public */ -export type DslQuery = any; +export type DslQuery = estypes.QueryDslQueryContainer; /** @internal */ export interface KueryParseOptions { diff --git a/packages/kbn-io-ts-utils/.babelrc b/packages/kbn-io-ts-utils/.babelrc new file mode 100644 index 0000000000000..7da72d1779128 --- /dev/null +++ b/packages/kbn-io-ts-utils/.babelrc @@ -0,0 +1,3 @@ +{ + "presets": ["@kbn/babel-preset/node_preset"] +} diff --git a/packages/kbn-io-ts-utils/BUILD.bazel b/packages/kbn-io-ts-utils/BUILD.bazel index 0e5210bcc38a5..474fa2c2bb121 100644 --- a/packages/kbn-io-ts-utils/BUILD.bazel +++ b/packages/kbn-io-ts-utils/BUILD.bazel @@ -1,5 +1,6 @@ load("@npm//@bazel/typescript:index.bzl", "ts_config", "ts_project") load("@build_bazel_rules_nodejs//:index.bzl", "js_library", "pkg_npm") +load("//src/dev/bazel:index.bzl", "jsts_transpiler") PKG_BASE_NAME = "kbn-io-ts-utils" PKG_REQUIRE_NAME = "@kbn/io-ts-utils" @@ -24,7 +25,7 @@ NPM_MODULE_EXTRA_FILES = [ "package.json", ] -SRC_DEPS = [ +RUNTIME_DEPS = [ "//packages/kbn-config-schema", "@npm//fp-ts", "@npm//io-ts", @@ -33,12 +34,20 @@ SRC_DEPS = [ ] TYPES_DEPS = [ + "//packages/kbn-config-schema", + "@npm//fp-ts", + "@npm//io-ts", + "@npm//tslib", "@npm//@types/jest", "@npm//@types/lodash", "@npm//@types/node", ] -DEPS = SRC_DEPS + TYPES_DEPS +jsts_transpiler( + name = "target_node", + srcs = SRCS, + build_pkg_name = package_name(), +) ts_config( name = "tsconfig", @@ -50,13 +59,14 @@ ts_config( ) ts_project( - name = "tsc", + name = "tsc_types", args = ['--pretty'], srcs = SRCS, - deps = DEPS, + deps = TYPES_DEPS, declaration = True, declaration_map = True, - out_dir = "target", + emit_declaration_only = True, + out_dir = "target_types", source_map = True, root_dir = "src", tsconfig = ":tsconfig", @@ -65,7 +75,7 @@ ts_project( js_library( name = PKG_BASE_NAME, srcs = NPM_MODULE_EXTRA_FILES, - deps = DEPS + [":tsc"], + deps = RUNTIME_DEPS + [":target_node", ":tsc_types"], package_name = PKG_REQUIRE_NAME, visibility = ["//visibility:public"], ) diff --git a/packages/kbn-io-ts-utils/package.json b/packages/kbn-io-ts-utils/package.json index 9d22277f27c01..fb1179b06bf45 100644 --- a/packages/kbn-io-ts-utils/package.json +++ b/packages/kbn-io-ts-utils/package.json @@ -1,7 +1,7 @@ { "name": "@kbn/io-ts-utils", - "main": "./target/index.js", - "types": "./target/index.d.ts", + "main": "./target_node/index.js", + "types": "./target_types/index.d.ts", "version": "1.0.0", "license": "SSPL-1.0 OR Elastic License 2.0", "private": true diff --git a/packages/kbn-io-ts-utils/src/index.ts b/packages/kbn-io-ts-utils/src/index.ts index a60bc2086fa3a..88cfc063f738a 100644 --- a/packages/kbn-io-ts-utils/src/index.ts +++ b/packages/kbn-io-ts-utils/src/index.ts @@ -6,6 +6,7 @@ * Side Public License, v 1. */ +export { deepExactRt } from './deep_exact_rt'; export { jsonRt } from './json_rt'; export { mergeRt } from './merge_rt'; export { strictKeysRt } from './strict_keys_rt'; @@ -13,3 +14,4 @@ export { isoToEpochRt } from './iso_to_epoch_rt'; export { toNumberRt } from './to_number_rt'; export { toBooleanRt } from './to_boolean_rt'; export { toJsonSchema } from './to_json_schema'; +export { nonEmptyStringRt } from './non_empty_string_rt'; diff --git a/packages/kbn-io-ts-utils/src/non_empty_string_rt/index.test.ts b/packages/kbn-io-ts-utils/src/non_empty_string_rt/index.test.ts new file mode 100644 index 0000000000000..85b58ef76622f --- /dev/null +++ b/packages/kbn-io-ts-utils/src/non_empty_string_rt/index.test.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +import { nonEmptyStringRt } from './'; +import { isLeft, isRight } from 'fp-ts/lib/Either'; + +describe('nonEmptyStringRt', () => { + it('fails on empty strings', () => { + expect(isLeft(nonEmptyStringRt.decode(''))).toBe(true); + }); + + it('passes non-empty strings', () => { + expect(isRight(nonEmptyStringRt.decode('foo'))).toBe(true); + }); +}); diff --git a/packages/kbn-io-ts-utils/src/non_empty_string_rt/index.ts b/packages/kbn-io-ts-utils/src/non_empty_string_rt/index.ts new file mode 100644 index 0000000000000..740fcfe3f2f40 --- /dev/null +++ b/packages/kbn-io-ts-utils/src/non_empty_string_rt/index.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +import * as t from 'io-ts'; + +// from https://github.com/gcanti/io-ts-types/blob/master/src/NonEmptyString.ts + +export interface NonEmptyStringBrand { + readonly NonEmptyString: unique symbol; +} + +export type NonEmptyString = t.Branded; + +export const nonEmptyStringRt = t.brand( + t.string, + (str): str is NonEmptyString => str.length > 0, + 'NonEmptyString' +); diff --git a/packages/kbn-io-ts-utils/tsconfig.json b/packages/kbn-io-ts-utils/tsconfig.json index 3ee769739dfc7..72d1479621345 100644 --- a/packages/kbn-io-ts-utils/tsconfig.json +++ b/packages/kbn-io-ts-utils/tsconfig.json @@ -1,12 +1,14 @@ { "extends": "../../tsconfig.bazel.json", "compilerOptions": { - "outDir": "./target/types", - "stripInternal": false, "declaration": true, "declarationMap": true, + "emitDeclarationOnly": true, + "outDir": "./target_types", + "rootDir": "src", "sourceMap": true, "sourceRoot": "../../../../packages/kbn-io-ts-utils/src", + "stripInternal": false, "types": [ "jest", "node" diff --git a/packages/kbn-optimizer/limits.yml b/packages/kbn-optimizer/limits.yml index 4772e00d56450..48130a7bfcf5b 100644 --- a/packages/kbn-optimizer/limits.yml +++ b/packages/kbn-optimizer/limits.yml @@ -42,7 +42,7 @@ pageLoadAssetSize: inspector: 148711 kibanaLegacy: 107711 kibanaOverview: 56279 - kibanaReact: 161921 + kibanaReact: 188705 kibanaUtils: 198829 lens: 96624 licenseManagement: 41817 diff --git a/packages/kbn-rule-data-utils/src/alerts_as_data_rbac.ts b/packages/kbn-rule-data-utils/src/alerts_as_data_rbac.ts index 795d194ee8a92..476425487df1b 100644 --- a/packages/kbn-rule-data-utils/src/alerts_as_data_rbac.ts +++ b/packages/kbn-rule-data-utils/src/alerts_as_data_rbac.ts @@ -27,7 +27,7 @@ export const AlertConsumers = { SYNTHETICS: 'synthetics', } as const; export type AlertConsumers = typeof AlertConsumers[keyof typeof AlertConsumers]; -export type STATUS_VALUES = 'open' | 'acknowledged' | 'closed'; +export type STATUS_VALUES = 'open' | 'acknowledged' | 'closed' | 'in-progress'; // TODO: remove 'in-progress' after migration to 'acknowledged' export const mapConsumerToIndexName: Record = { apm: '.alerts-observability-apm', diff --git a/packages/kbn-securitysolution-t-grid/src/mock/mock_event_details.ts b/packages/kbn-securitysolution-t-grid/src/mock/mock_event_details.ts index 167fc9dd17a2a..8708aa4c07603 100644 --- a/packages/kbn-securitysolution-t-grid/src/mock/mock_event_details.ts +++ b/packages/kbn-securitysolution-t-grid/src/mock/mock_event_details.ts @@ -39,12 +39,12 @@ export const eventHit = { '/var/lib/jenkins/workspace/Beats_beats_PR-22624/.gvm/versions/go1.14.7.linux.amd64/bin/go', ], 'source.geo.location': [{ coordinates: [118.7778, 32.0617], type: 'Point' }], - 'threat.indicator': [ + 'threat.enrichments': [ { 'matched.field': ['matched_field', 'other_matched_field'], - first_seen: ['2021-02-22T17:29:25.195Z'], - provider: ['yourself'], - type: ['custom'], + 'indicator.first_seen': ['2021-02-22T17:29:25.195Z'], + 'indicator.provider': ['yourself'], + 'indicator.type': ['custom'], 'matched.atomic': ['matched_atomic'], lazer: [ { @@ -57,9 +57,9 @@ export const eventHit = { }, { 'matched.field': ['matched_field_2'], - first_seen: ['2021-02-22T17:29:25.195Z'], - provider: ['other_you'], - type: ['custom'], + 'indicator.first_seen': ['2021-02-22T17:29:25.195Z'], + 'indicator.provider': ['other_you'], + 'indicator.type': ['custom'], 'matched.atomic': ['matched_atomic_2'], lazer: [ { @@ -259,70 +259,70 @@ export const eventDetailsFormattedFields = [ }, { category: 'threat', - field: 'threat.indicator.matched.field', + field: 'threat.enrichments.matched.field', values: ['matched_field', 'other_matched_field', 'matched_field_2'], originalValue: ['matched_field', 'other_matched_field', 'matched_field_2'], isObjectArray: false, }, { category: 'threat', - field: 'threat.indicator.first_seen', + field: 'threat.enrichments.indicator.first_seen', values: ['2021-02-22T17:29:25.195Z'], originalValue: ['2021-02-22T17:29:25.195Z'], isObjectArray: false, }, { category: 'threat', - field: 'threat.indicator.provider', + field: 'threat.enrichments.indicator.provider', values: ['yourself', 'other_you'], originalValue: ['yourself', 'other_you'], isObjectArray: false, }, { category: 'threat', - field: 'threat.indicator.type', + field: 'threat.enrichments.indicator.type', values: ['custom'], originalValue: ['custom'], isObjectArray: false, }, { category: 'threat', - field: 'threat.indicator.matched.atomic', + field: 'threat.enrichments.matched.atomic', values: ['matched_atomic', 'matched_atomic_2'], originalValue: ['matched_atomic', 'matched_atomic_2'], isObjectArray: false, }, { category: 'threat', - field: 'threat.indicator.lazer.great.field', + field: 'threat.enrichments.lazer.great.field', values: ['grrrrr', 'grrrrr_2'], originalValue: ['grrrrr', 'grrrrr_2'], isObjectArray: false, }, { category: 'threat', - field: 'threat.indicator.lazer.great.field.wowoe.fooooo', + field: 'threat.enrichments.lazer.great.field.wowoe.fooooo', values: ['grrrrr'], originalValue: ['grrrrr'], isObjectArray: false, }, { category: 'threat', - field: 'threat.indicator.lazer.great.field.astring', + field: 'threat.enrichments.lazer.great.field.astring', values: ['cool'], originalValue: ['cool'], isObjectArray: false, }, { category: 'threat', - field: 'threat.indicator.lazer.great.field.aNumber', + field: 'threat.enrichments.lazer.great.field.aNumber', values: ['1'], originalValue: ['1'], isObjectArray: false, }, { category: 'threat', - field: 'threat.indicator.lazer.great.field.neat', + field: 'threat.enrichments.lazer.great.field.neat', values: ['true'], originalValue: ['true'], isObjectArray: false, diff --git a/packages/kbn-typed-react-router-config/src/create_router.test.tsx b/packages/kbn-typed-react-router-config/src/create_router.test.tsx index 4de4b44196ddd..d8f42c8714e8b 100644 --- a/packages/kbn-typed-react-router-config/src/create_router.test.tsx +++ b/packages/kbn-typed-react-router-config/src/create_router.test.tsx @@ -7,7 +7,7 @@ */ import React from 'react'; import * as t from 'io-ts'; -import { toNumberRt } from '@kbn/io-ts-utils/target/to_number_rt'; +import { toNumberRt } from '@kbn/io-ts-utils'; import { createRouter } from './create_router'; import { createMemoryHistory } from 'history'; import { route } from './route'; @@ -61,6 +61,7 @@ describe('createRouter', () => { params: t.type({ query: t.type({ aggregationType: t.string, + kuery: t.string, }), }), }, @@ -112,7 +113,7 @@ describe('createRouter', () => { }, }); - history.push('/traces?rangeFrom=now-15m&rangeTo=now&aggregationType=avg'); + history.push('/traces?rangeFrom=now-15m&rangeTo=now&aggregationType=avg&kuery='); const topTracesParams = router.getParams('/traces', history.location); @@ -122,6 +123,7 @@ describe('createRouter', () => { rangeFrom: 'now-15m', rangeTo: 'now', aggregationType: 'avg', + kuery: '', }, }); @@ -156,6 +158,22 @@ describe('createRouter', () => { maxNumNodes: 3, }, }); + + history.push( + '/traces?rangeFrom=now-15m&rangeTo=now&aggregationType=avg&kuery=service.name%3A%22metricbeat%22' + ); + + const topTracesParams = router.getParams('/traces', history.location); + + expect(topTracesParams).toEqual({ + path: {}, + query: { + rangeFrom: 'now-15m', + rangeTo: 'now', + aggregationType: 'avg', + kuery: 'service.name:"metricbeat"', + }, + }); }); it('throws an error if the given path does not match any routes', () => { @@ -280,10 +298,26 @@ describe('createRouter', () => { query: { rangeTo: 'now', aggregationType: 'avg', + kuery: '', + }, + }); + + expect(href).toEqual('/traces?aggregationType=avg&kuery=&rangeFrom=now-30m&rangeTo=now'); + }); + + it('encodes query parameters', () => { + const href = router.link('/traces', { + // @ts-ignore + query: { + rangeTo: 'now', + aggregationType: 'avg', + kuery: 'service.name:"metricbeat"', }, }); - expect(href).toEqual('/traces?aggregationType=avg&rangeFrom=now-30m&rangeTo=now'); + expect(href).toEqual( + '/traces?aggregationType=avg&kuery=service.name%3A%22metricbeat%22&rangeFrom=now-30m&rangeTo=now' + ); }); }); }); diff --git a/packages/kbn-typed-react-router-config/src/create_router.ts b/packages/kbn-typed-react-router-config/src/create_router.ts index 846808cb798f1..28f9e2774eb74 100644 --- a/packages/kbn-typed-react-router-config/src/create_router.ts +++ b/packages/kbn-typed-react-router-config/src/create_router.ts @@ -14,10 +14,16 @@ import { } from 'react-router-config'; import qs from 'query-string'; import { findLastIndex, merge, compact } from 'lodash'; -import { deepExactRt } from '@kbn/io-ts-utils/target/deep_exact_rt'; -import { mergeRt } from '@kbn/io-ts-utils/target/merge_rt'; +import type { deepExactRt as deepExactRtTyped, mergeRt as mergeRtTyped } from '@kbn/io-ts-utils'; +// @ts-expect-error +import { deepExactRt as deepExactRtNonTyped } from '@kbn/io-ts-utils/target_node/deep_exact_rt'; +// @ts-expect-error +import { mergeRt as mergeRtNonTyped } from '@kbn/io-ts-utils/target_node/merge_rt'; import { Route, Router } from './types'; +const deepExactRt: typeof deepExactRtTyped = deepExactRtNonTyped; +const mergeRt: typeof mergeRtTyped = mergeRtNonTyped; + export function createRouter(routes: TRoutes): Router { const routesByReactRouterConfig = new Map(); const reactRouterConfigsByRoute = new Map(); @@ -79,7 +85,7 @@ export function createRouter(routes: TRoutes): Router(routes: TRoutes): Router { params: TRoute extends { params: t.Type; } - ? t.OutputOf + ? t.TypeOf : {}; }; } @@ -107,9 +107,11 @@ type TypeOfMatches = TRouteMatches extends [ (TNextRouteMatches extends RouteMatch[] ? TypeOfMatches : {}) : {}; -export type TypeOf> = TypeOfMatches< - Match ->; +export type TypeOf< + TRoutes extends Route[], + TPath extends PathsOf, + TWithDefaultOutput extends boolean = true +> = TypeOfMatches> & (TWithDefaultOutput extends true ? DefaultOutput : {}); export type TypeAsArgs = keyof TObject extends never ? [] @@ -126,15 +128,15 @@ export interface Router { getParams>( path: TPath, location: Location - ): OutputOf; + ): TypeOf; getParams, TOptional extends boolean>( path: TPath, location: Location, optional: TOptional - ): TOptional extends true ? OutputOf | undefined : OutputOf; + ): TOptional extends true ? TypeOf | undefined : TypeOf; link>( path: TPath, - ...args: TypeAsArgs> + ...args: TypeAsArgs> ): string; getRoutePath(route: Route): string; } diff --git a/src/core/public/overlays/flyout/flyout_service.tsx b/src/core/public/overlays/flyout/flyout_service.tsx index b41b85e5f429f..6e986cc8ecb48 100644 --- a/src/core/public/overlays/flyout/flyout_service.tsx +++ b/src/core/public/overlays/flyout/flyout_service.tsx @@ -82,9 +82,15 @@ export interface OverlayFlyoutOpenOptions { closeButtonAriaLabel?: string; ownFocus?: boolean; 'data-test-subj'?: string; + 'aria-label'?: string; size?: EuiFlyoutSize; maxWidth?: boolean | number | string; hideCloseButton?: boolean; + /** + * EuiFlyout onClose handler. + * If provided the consumer is responsible for calling flyout.close() to close the flyout; + */ + onClose?: (flyout: OverlayRef) => void; } interface StartDeps { @@ -119,9 +125,17 @@ export class FlyoutService { this.activeFlyout = flyout; + const onCloseFlyout = () => { + if (options.onClose) { + options.onClose(flyout); + } else { + flyout.close(); + } + }; + render( - flyout.close()}> + , diff --git a/src/core/public/public.api.md b/src/core/public/public.api.md index 5ce12a1889c26..d3f9ce71379b7 100644 --- a/src/core/public/public.api.md +++ b/src/core/public/public.api.md @@ -1006,6 +1006,8 @@ export interface OverlayBannersStart { // @public (undocumented) export interface OverlayFlyoutOpenOptions { + // (undocumented) + 'aria-label'?: string; // (undocumented) 'data-test-subj'?: string; // (undocumented) @@ -1016,6 +1018,7 @@ export interface OverlayFlyoutOpenOptions { hideCloseButton?: boolean; // (undocumented) maxWidth?: boolean | number | string; + onClose?: (flyout: OverlayRef) => void; // (undocumented) ownFocus?: boolean; // (undocumented) diff --git a/src/core/server/elasticsearch/client/configure_client.ts b/src/core/server/elasticsearch/client/configure_client.ts index 3c32dd2cfd4f4..631e20ac238f1 100644 --- a/src/core/server/elasticsearch/client/configure_client.ts +++ b/src/core/server/elasticsearch/client/configure_client.ts @@ -72,11 +72,13 @@ function ensureString(body: RequestBody): string { return JSON.stringify(body); } -function getErrorMessage(error: ApiError, event: RequestEvent): string { +/** + * Returns a debug message from an Elasticsearch error in the following format: + * [error type] error reason + */ +export function getErrorMessage(error: ApiError): string { if (error instanceof errors.ResponseError) { - return `${getResponseMessage(event)} [${event.body?.error?.type}]: ${ - event.body?.error?.reason ?? error.message - }`; + return `[${error.meta.body?.error?.type}]: ${error.meta.body?.error?.reason ?? error.message}`; } return `[${error.name}]: ${error.message}`; } @@ -85,19 +87,33 @@ function getErrorMessage(error: ApiError, event: RequestEvent): string { * returns a string in format: * * status code - * URL + * method URL * request body * * so it could be copy-pasted into the Dev console */ function getResponseMessage(event: RequestEvent): string { - const params = event.meta.request.params; + const errorMeta = getRequestDebugMeta(event); + const body = errorMeta.body ? `\n${errorMeta.body}` : ''; + return `${errorMeta.statusCode}\n${errorMeta.method} ${errorMeta.url}${body}`; +} +/** + * Returns stringified debug information from an Elasticsearch request event + * useful for logging in case of an unexpected failure. + */ +export function getRequestDebugMeta( + event: RequestEvent +): { url: string; body: string; statusCode: number | null; method: string } { + const params = event.meta.request.params; // definition is wrong, `params.querystring` can be either a string or an object const querystring = convertQueryString(params.querystring); - const url = `${params.path}${querystring ? `?${querystring}` : ''}`; - const body = params.body ? `\n${ensureString(params.body)}` : ''; - return `${event.statusCode}\n${params.method} ${url}${body}`; + return { + url: `${params.path}${querystring ? `?${querystring}` : ''}`, + body: params.body ? `${ensureString(params.body)}` : '', + method: params.method, + statusCode: event.statusCode, + }; } const addLogging = (client: Client, logger: Logger) => { @@ -110,7 +126,11 @@ const addLogging = (client: Client, logger: Logger) => { } : undefined; // do not clutter logs if opaqueId is not present if (error) { - logger.debug(getErrorMessage(error, event), meta); + if (error instanceof errors.ResponseError) { + logger.debug(`${getResponseMessage(event)} ${getErrorMessage(error)}`, meta); + } else { + logger.debug(getErrorMessage(error), meta); + } } else { logger.debug(getResponseMessage(event), meta); } diff --git a/src/core/server/elasticsearch/client/index.ts b/src/core/server/elasticsearch/client/index.ts index c7600b723ade0..29f8b85695190 100644 --- a/src/core/server/elasticsearch/client/index.ts +++ b/src/core/server/elasticsearch/client/index.ts @@ -20,5 +20,5 @@ export type { IScopedClusterClient } from './scoped_cluster_client'; export type { ElasticsearchClientConfig } from './client_config'; export { ClusterClient } from './cluster_client'; export type { IClusterClient, ICustomClusterClient } from './cluster_client'; -export { configureClient } from './configure_client'; +export { configureClient, getRequestDebugMeta, getErrorMessage } from './configure_client'; export { retryCallCluster, migrationRetryCallCluster } from './retry_call_cluster'; diff --git a/src/core/server/elasticsearch/index.ts b/src/core/server/elasticsearch/index.ts index 8bcc841669fc9..62bb30452bb98 100644 --- a/src/core/server/elasticsearch/index.ts +++ b/src/core/server/elasticsearch/index.ts @@ -37,4 +37,5 @@ export type { GetResponse, DeleteDocumentResponse, } from './client'; +export { getRequestDebugMeta, getErrorMessage } from './client'; export { isSupportedEsServer } from './supported_server_response_check'; diff --git a/src/core/server/saved_objects/import/lib/check_origin_conflicts.test.ts b/src/core/server/saved_objects/import/lib/check_origin_conflicts.test.ts index 90693f29836d7..011e5500b8d9c 100644 --- a/src/core/server/saved_objects/import/lib/check_origin_conflicts.test.ts +++ b/src/core/server/saved_objects/import/lib/check_origin_conflicts.test.ts @@ -23,14 +23,20 @@ type SavedObjectType = SavedObject<{ title?: string }>; type CheckOriginConflictsParams = Parameters[0]; /** - * Function to create a realistic-looking import object given a type, ID, and optional originId + * Function to create a realistic-looking import object given a type, ID, optional originId, and optional updated_at */ -const createObject = (type: string, id: string, originId?: string): SavedObjectType => ({ +const createObject = ( + type: string, + id: string, + originId?: string, + updatedAt?: string +): SavedObjectType => ({ type, id, attributes: { title: `Title for ${type}:${id}` }, references: (Symbol() as unknown) as SavedObjectReference[], ...(originId && { originId }), + ...(updatedAt && { updated_at: updatedAt }), }); const MULTI_NS_TYPE = 'multi'; @@ -389,21 +395,21 @@ describe('#checkOriginConflicts', () => { // try to import obj1 and obj2 const obj1 = createObject(MULTI_NS_TYPE, 'id-1'); const obj2 = createObject(MULTI_NS_TYPE, 'id-2', 'originId-foo'); - const objA = createObject(MULTI_NS_TYPE, 'id-A', obj1.id); - const objB = createObject(MULTI_NS_TYPE, 'id-B', obj1.id); + const objA = createObject(MULTI_NS_TYPE, 'id-A', obj1.id, '2017-09-21T18:59:16.270Z'); + const objB = createObject(MULTI_NS_TYPE, 'id-B', obj1.id, '2021-08-10T13:21:44.135Z'); const objC = createObject(MULTI_NS_TYPE, 'id-C', obj2.originId); const objD = createObject(MULTI_NS_TYPE, 'id-D', obj2.originId); const objects = [obj1, obj2]; const params = setupParams({ objects }); mockFindResult(objA, objB); // find for obj1: the result is an inexact match with two destinations - mockFindResult(objC, objD); // find for obj2: the result is an inexact match with two destinations + mockFindResult(objD, objC); // find for obj2: the result is an inexact match with two destinations const checkOriginConflictsResult = await checkOriginConflicts(params); const expectedResult = { importIdMap: new Map(), errors: [ - createAmbiguousConflictError(obj1, [objA, objB]), - createAmbiguousConflictError(obj2, [objC, objD]), + createAmbiguousConflictError(obj1, [objB, objA]), // Assert that these have been sorted by updatedAt in descending order + createAmbiguousConflictError(obj2, [objC, objD]), // Assert that these have been sorted by ID in ascending order (since their updatedAt values are the same) ], pendingOverwrites: new Set(), }; diff --git a/src/core/server/saved_objects/import/lib/check_origin_conflicts.ts b/src/core/server/saved_objects/import/lib/check_origin_conflicts.ts index 1952a04ab815c..d689f37f5ad26 100644 --- a/src/core/server/saved_objects/import/lib/check_origin_conflicts.ts +++ b/src/core/server/saved_objects/import/lib/check_origin_conflicts.ts @@ -58,11 +58,21 @@ const createQuery = (type: string, id: string, rawIdPrefix: string) => const transformObjectsToAmbiguousConflictFields = ( objects: Array> ) => - objects.map(({ id, attributes, updated_at: updatedAt }) => ({ - id, - title: attributes?.title, - updatedAt, - })); + objects + .map(({ id, attributes, updated_at: updatedAt }) => ({ + id, + title: attributes?.title, + updatedAt, + })) + // Sort to ensure that integration tests are not flaky + .sort((a, b) => { + const aUpdatedAt = a.updatedAt ?? ''; + const bUpdatedAt = b.updatedAt ?? ''; + if (aUpdatedAt !== bUpdatedAt) { + return aUpdatedAt < bUpdatedAt ? 1 : -1; // descending + } + return a.id < b.id ? -1 : 1; // ascending + }); const getAmbiguousConflictSourceKey = ({ object }: InexactMatch) => `${object.type}:${object.originId || object.id}`; diff --git a/src/core/server/saved_objects/migrationsv2/actions/bulk_overwrite_transformed_documents.ts b/src/core/server/saved_objects/migrationsv2/actions/bulk_overwrite_transformed_documents.ts index 4c0f8717576ac..d0259f8f21ca4 100644 --- a/src/core/server/saved_objects/migrationsv2/actions/bulk_overwrite_transformed_documents.ts +++ b/src/core/server/saved_objects/migrationsv2/actions/bulk_overwrite_transformed_documents.ts @@ -8,7 +8,7 @@ import * as Either from 'fp-ts/lib/Either'; import * as TaskEither from 'fp-ts/lib/TaskEither'; -import type { estypes } from '@elastic/elasticsearch'; +import { errors as esErrors, estypes } from '@elastic/elasticsearch'; import { ElasticsearchClient } from '../../../elasticsearch'; import type { SavedObjectsRawDoc } from '../../serialization'; import { @@ -17,7 +17,7 @@ import { } from './catch_retryable_es_client_errors'; import { isWriteBlockException } from './es_errors'; import { WAIT_FOR_ALL_SHARDS_TO_BE_ACTIVE } from './constants'; -import type { TargetIndexHadWriteBlock } from './index'; +import type { TargetIndexHadWriteBlock, RequestEntityTooLargeException } from './index'; /** @internal */ export interface BulkOverwriteTransformedDocumentsParams { @@ -37,7 +37,7 @@ export const bulkOverwriteTransformedDocuments = ({ transformedDocs, refresh = false, }: BulkOverwriteTransformedDocumentsParams): TaskEither.TaskEither< - RetryableEsClientError | TargetIndexHadWriteBlock, + RetryableEsClientError | TargetIndexHadWriteBlock | RequestEntityTooLargeException, 'bulk_index_succeeded' > => () => { return client @@ -90,5 +90,12 @@ export const bulkOverwriteTransformedDocuments = ({ throw new Error(JSON.stringify(errors)); } }) + .catch((error) => { + if (error instanceof esErrors.ResponseError && error.statusCode === 413) { + return Either.left({ type: 'request_entity_too_large_exception' as const }); + } else { + throw error; + } + }) .catch(catchRetryableEsClientErrors); }; diff --git a/src/core/server/saved_objects/migrationsv2/actions/index.ts b/src/core/server/saved_objects/migrationsv2/actions/index.ts index 97ffcff25dd76..158d97f3b7c27 100644 --- a/src/core/server/saved_objects/migrationsv2/actions/index.ts +++ b/src/core/server/saved_objects/migrationsv2/actions/index.ts @@ -120,6 +120,10 @@ export interface TargetIndexHadWriteBlock { type: 'target_index_had_write_block'; } +export interface RequestEntityTooLargeException { + type: 'request_entity_too_large_exception'; +} + /** @internal */ export interface AcknowledgeResponse { acknowledged: boolean; @@ -136,6 +140,7 @@ export interface ActionErrorTypeMap { alias_not_found_exception: AliasNotFound; remove_index_not_a_concrete_index: RemoveIndexNotAConcreteIndex; documents_transform_failed: DocumentsTransformFailed; + request_entity_too_large_exception: RequestEntityTooLargeException; } /** diff --git a/src/core/server/saved_objects/migrationsv2/actions/integration_tests/actions.test.ts b/src/core/server/saved_objects/migrationsv2/actions/integration_tests/actions.test.ts index 052316e7944ce..65be45e49190d 100644 --- a/src/core/server/saved_objects/migrationsv2/actions/integration_tests/actions.test.ts +++ b/src/core/server/saved_objects/migrationsv2/actions/integration_tests/actions.test.ts @@ -46,6 +46,12 @@ import { TaskEither } from 'fp-ts/lib/TaskEither'; const { startES } = kbnTestServer.createTestServers({ adjustTimeout: (t: number) => jest.setTimeout(t), + settings: { + es: { + license: 'basic', + esArgs: ['http.max_content_length=10Kb'], + }, + }, }); let esServer: kbnTestServer.TestElasticsearchUtils; @@ -1472,11 +1478,11 @@ describe('migration actions', () => { }); await expect(task()).resolves.toMatchInlineSnapshot(` - Object { - "_tag": "Right", - "right": "bulk_index_succeeded", - } - `); + Object { + "_tag": "Right", + "right": "bulk_index_succeeded", + } + `); }); it('resolves right even if there were some version_conflict_engine_exception', async () => { const existingDocs = ((await searchForOutdatedDocuments(client, { @@ -1501,7 +1507,7 @@ describe('migration actions', () => { } `); }); - it('resolves left if there are write_block errors', async () => { + it('resolves left target_index_had_write_block if there are write_block errors', async () => { const newDocs = ([ { _source: { title: 'doc 5' } }, { _source: { title: 'doc 6' } }, @@ -1515,13 +1521,34 @@ describe('migration actions', () => { refresh: 'wait_for', })() ).resolves.toMatchInlineSnapshot(` - Object { - "_tag": "Left", - "left": Object { - "type": "target_index_had_write_block", - }, - } - `); + Object { + "_tag": "Left", + "left": Object { + "type": "target_index_had_write_block", + }, + } + `); + }); + it('resolves left request_entity_too_large_exception when the payload is too large', async () => { + const newDocs = new Array(10000).fill({ + _source: { + title: + 'how do I create a document thats large enoug to exceed the limits without typing long sentences', + }, + }) as SavedObjectsRawDoc[]; + const task = bulkOverwriteTransformedDocuments({ + client, + index: 'existing_index_with_docs', + transformedDocs: newDocs, + }); + await expect(task()).resolves.toMatchInlineSnapshot(` + Object { + "_tag": "Left", + "left": Object { + "type": "request_entity_too_large_exception", + }, + } + `); }); }); }); diff --git a/src/core/server/saved_objects/migrationsv2/migrations_state_action_machine.test.ts b/src/core/server/saved_objects/migrationsv2/migrations_state_action_machine.test.ts index d4862cddf2666..773a0af469bd4 100644 --- a/src/core/server/saved_objects/migrationsv2/migrations_state_action_machine.test.ts +++ b/src/core/server/saved_objects/migrationsv2/migrations_state_action_machine.test.ts @@ -353,6 +353,9 @@ describe('migrationsStateActionMachine', () => { next: () => { throw new ResponseError( elasticsearchClientMock.createApiResponse({ + meta: { + request: { options: {}, id: '', params: { method: 'POST', path: '/mock' } }, + } as any, body: { error: { type: 'snapshot_in_progress_exception', @@ -365,14 +368,14 @@ describe('migrationsStateActionMachine', () => { client: esClient, }) ).rejects.toMatchInlineSnapshot( - `[Error: Unable to complete saved object migrations for the [.my-so-index] index. Please check the health of your Elasticsearch cluster and try again. Error: [snapshot_in_progress_exception]: Cannot delete indices that are being snapshotted]` + `[Error: Unable to complete saved object migrations for the [.my-so-index] index. Please check the health of your Elasticsearch cluster and try again. Unexpected Elasticsearch ResponseError: statusCode: 200, method: POST, url: /mock error: [snapshot_in_progress_exception]: Cannot delete indices that are being snapshotted,]` ); expect(loggingSystemMock.collect(mockLogger)).toMatchInlineSnapshot(` Object { "debug": Array [], "error": Array [ Array [ - "[.my-so-index] [snapshot_in_progress_exception]: Cannot delete indices that are being snapshotted", + "[.my-so-index] Unexpected Elasticsearch ResponseError: statusCode: 200, method: POST, url: /mock error: [snapshot_in_progress_exception]: Cannot delete indices that are being snapshotted,", ], Array [ "[.my-so-index] migration failed, dumping execution log:", diff --git a/src/core/server/saved_objects/migrationsv2/migrations_state_action_machine.ts b/src/core/server/saved_objects/migrationsv2/migrations_state_action_machine.ts index cd42d4077695e..8e3b8ee4ab556 100644 --- a/src/core/server/saved_objects/migrationsv2/migrations_state_action_machine.ts +++ b/src/core/server/saved_objects/migrationsv2/migrations_state_action_machine.ts @@ -10,6 +10,7 @@ import { errors as EsErrors } from '@elastic/elasticsearch'; import * as Option from 'fp-ts/lib/Option'; import { Logger, LogMeta } from '../../logging'; import type { ElasticsearchClient } from '../../elasticsearch'; +import { getErrorMessage, getRequestDebugMeta } from '../../elasticsearch'; import { Model, Next, stateActionMachine } from './state_action_machine'; import { cleanup } from './migrations_state_machine_cleanup'; import { State } from './types'; @@ -196,16 +197,19 @@ export async function migrationStateActionMachine({ } catch (e) { await cleanup(client, executionLog, lastState); if (e instanceof EsErrors.ResponseError) { - logger.error( - logMessagePrefix + `[${e.body?.error?.type}]: ${e.body?.error?.reason ?? e.message}` - ); + // Log the failed request. This is very similar to the + // elasticsearch-service's debug logs, but we log everything in single + // line until we have sub-ms resolution in our cloud logs. Because this + // is error level logs, we're also more careful and don't log the request + // body since this can very likely have sensitive saved objects. + const req = getRequestDebugMeta(e.meta); + const failedRequestMessage = `Unexpected Elasticsearch ResponseError: statusCode: ${ + req.statusCode + }, method: ${req.method}, url: ${req.url} error: ${getErrorMessage(e)},`; + logger.error(logMessagePrefix + failedRequestMessage); dumpExecutionLog(logger, logMessagePrefix, executionLog); throw new Error( - `Unable to complete saved object migrations for the [${ - initialState.indexPrefix - }] index. Please check the health of your Elasticsearch cluster and try again. Error: [${ - e.body?.error?.type - }]: ${e.body?.error?.reason ?? e.message}` + `Unable to complete saved object migrations for the [${initialState.indexPrefix}] index. Please check the health of your Elasticsearch cluster and try again. ${failedRequestMessage}` ); } else { logger.error(e); diff --git a/src/core/server/saved_objects/migrationsv2/model/model.test.ts b/src/core/server/saved_objects/migrationsv2/model/model.test.ts index 0837183b2a116..30612b82d58aa 100644 --- a/src/core/server/saved_objects/migrationsv2/model/model.test.ts +++ b/src/core/server/saved_objects/migrationsv2/model/model.test.ts @@ -1154,6 +1154,16 @@ describe('migrations v2 model', () => { expect(newState.retryCount).toEqual(0); expect(newState.retryDelay).toEqual(0); }); + test('REINDEX_SOURCE_TO_TEMP_INDEX_BULK -> FATAL if action returns left request_entity_too_large_exception', () => { + const res: ResponseType<'REINDEX_SOURCE_TO_TEMP_INDEX_BULK'> = Either.left({ + type: 'request_entity_too_large_exception', + }); + const newState = model(reindexSourceToTempIndexBulkState, res) as FatalState; + expect(newState.controlState).toEqual('FATAL'); + expect(newState.reason).toMatchInlineSnapshot( + `"While indexing a batch of saved objects, Elasticsearch returned a 413 Request Entity Too Large exception. Try to use smaller batches by changing the Kibana 'migrations.batchSize' configuration option and restarting Kibana."` + ); + }); test('REINDEX_SOURCE_TO_TEMP_INDEX_BULK should throw a throwBadResponse error if action failed', () => { const res: ResponseType<'REINDEX_SOURCE_TO_TEMP_INDEX_BULK'> = Either.left({ type: 'retryable_es_client_error', @@ -1532,6 +1542,17 @@ describe('migrations v2 model', () => { expect(newState.retryCount).toEqual(1); expect(newState.retryDelay).toEqual(2000); }); + + test('TRANSFORMED_DOCUMENTS_BULK_INDEX -> FATAL if action returns left request_entity_too_large_exception', () => { + const res: ResponseType<'TRANSFORMED_DOCUMENTS_BULK_INDEX'> = Either.left({ + type: 'request_entity_too_large_exception', + }); + const newState = model(transformedDocumentsBulkIndexState, res) as FatalState; + expect(newState.controlState).toEqual('FATAL'); + expect(newState.reason).toMatchInlineSnapshot( + `"While indexing a batch of saved objects, Elasticsearch returned a 413 Request Entity Too Large exception. Try to use smaller batches by changing the Kibana 'migrations.batchSize' configuration option and restarting Kibana."` + ); + }); }); describe('UPDATE_TARGET_MAPPINGS', () => { diff --git a/src/core/server/saved_objects/migrationsv2/model/model.ts b/src/core/server/saved_objects/migrationsv2/model/model.ts index 474fe56ed64a4..01c1893154c6c 100644 --- a/src/core/server/saved_objects/migrationsv2/model/model.ts +++ b/src/core/server/saved_objects/migrationsv2/model/model.ts @@ -540,6 +540,12 @@ export const model = (currentState: State, resW: ResponseType): ...stateP, controlState: 'REINDEX_SOURCE_TO_TEMP_CLOSE_PIT', }; + } else if (isLeftTypeof(res.left, 'request_entity_too_large_exception')) { + return { + ...stateP, + controlState: 'FATAL', + reason: `While indexing a batch of saved objects, Elasticsearch returned a 413 Request Entity Too Large exception. Try to use smaller batches by changing the Kibana 'migrations.batchSize' configuration option and restarting Kibana.`, + }; } throwBadResponse(stateP, res.left); } @@ -709,7 +715,19 @@ export const model = (currentState: State, resW: ResponseType): hasTransformedDocs: true, }; } else { - throwBadResponse(stateP, res as never); + if (isLeftTypeof(res.left, 'request_entity_too_large_exception')) { + return { + ...stateP, + controlState: 'FATAL', + reason: `While indexing a batch of saved objects, Elasticsearch returned a 413 Request Entity Too Large exception. Try to use smaller batches by changing the Kibana 'migrations.batchSize' configuration option and restarting Kibana.`, + }; + } else if (isLeftTypeof(res.left, 'target_index_had_write_block')) { + // we fail on this error since the target index will only have a write + // block if a newer version of Kibana started an upgrade + throwBadResponse(stateP, res.left as never); + } else { + throwBadResponse(stateP, res.left); + } } } else if (stateP.controlState === 'UPDATE_TARGET_MAPPINGS') { const res = resW as ExcludeRetryableEsError>; diff --git a/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker b/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker index e65c5542cce7e..e2d81c5ae1752 100755 --- a/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker +++ b/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker @@ -395,6 +395,7 @@ kibana_vars=( xpack.spaces.enabled xpack.spaces.maxSpaces xpack.task_manager.enabled + xpack.task_manager.index xpack.task_manager.max_attempts xpack.task_manager.max_poll_inactivity_cycles xpack.task_manager.max_workers diff --git a/src/dev/license_checker/config.ts b/src/dev/license_checker/config.ts index 9225659c6ca20..e1900019a5b30 100644 --- a/src/dev/license_checker/config.ts +++ b/src/dev/license_checker/config.ts @@ -73,6 +73,7 @@ export const DEV_ONLY_LICENSE_ALLOWED = ['MPL-2.0']; export const LICENSE_OVERRIDES = { 'jsts@1.6.2': ['Eclipse Distribution License - v 1.0'], // cf. https://github.com/bjornharrtell/jsts '@mapbox/jsonlint-lines-primitives@2.0.2': ['MIT'], // license in readme https://github.com/tmcw/jsonlint + 'node-sql-parser@3.6.1': ['(GPL-2.0 OR MIT)'], // GPL-2.0* https://github.com/taozhi8833998/node-sql-parser '@elastic/ems-client@7.14.0': ['Elastic License 2.0'], '@elastic/eui@36.0.0': ['SSPL-1.0 OR Elastic License 2.0'], diff --git a/src/plugins/dashboard/kibana.json b/src/plugins/dashboard/kibana.json index 54eaf461b73d7..d270b7dad3c7c 100644 --- a/src/plugins/dashboard/kibana.json +++ b/src/plugins/dashboard/kibana.json @@ -1,5 +1,10 @@ { "id": "dashboard", + "owner": { + "name": "Kibana Presentation", + "githubTeam": "kibana-presentation" + }, + "description": "Adds the Dashboard app to Kibana", "version": "kibana", "requiredPlugins": [ "data", @@ -14,17 +19,8 @@ "presentationUtil", "visualizations" ], - "optionalPlugins": [ - "home", - "spacesOss", - "savedObjectsTaggingOss", - "usageCollection"], + "optionalPlugins": ["home", "spacesOss", "savedObjectsTaggingOss", "usageCollection"], "server": true, "ui": true, - "requiredBundles": [ - "home", - "kibanaReact", - "kibanaUtils", - "presentationUtil" - ] + "requiredBundles": ["home", "kibanaReact", "kibanaUtils", "presentationUtil"] } diff --git a/src/plugins/dashboard/public/application/lib/sync_dashboard_index_patterns.ts b/src/plugins/dashboard/public/application/lib/sync_dashboard_index_patterns.ts index 7360725e39cc1..3a1d60696331a 100644 --- a/src/plugins/dashboard/public/application/lib/sync_dashboard_index_patterns.ts +++ b/src/plugins/dashboard/public/application/lib/sync_dashboard_index_patterns.ts @@ -8,16 +8,8 @@ import { uniqBy } from 'lodash'; import deepEqual from 'fast-deep-equal'; -import { merge, Observable, pipe, EMPTY } from 'rxjs'; -import { - distinctUntilChanged, - catchError, - switchMap, - startWith, - filter, - mapTo, - map, -} from 'rxjs/operators'; +import { Observable, pipe } from 'rxjs'; +import { distinctUntilChanged, switchMap, filter, mapTo, map } from 'rxjs/operators'; import { DashboardContainer } from '..'; import { isErrorEmbeddable } from '../../services/embeddable'; @@ -36,7 +28,7 @@ export const syncDashboardIndexPatterns = ({ }: SyncDashboardIndexPatternsProps) => { const updateIndexPatternsOperator = pipe( filter((container: DashboardContainer) => !!container && !isErrorEmbeddable(container)), - map((container: DashboardContainer): IndexPattern[] => { + map((container: DashboardContainer): IndexPattern[] | undefined => { let panelIndexPatterns: IndexPattern[] = []; Object.values(container.getChildIds()).forEach((id) => { const embeddableInstance = container.getChild(id); @@ -46,18 +38,31 @@ export const syncDashboardIndexPatterns = ({ panelIndexPatterns.push(...embeddableIndexPatterns); }); panelIndexPatterns = uniqBy(panelIndexPatterns, 'id'); + + /** + * If no index patterns have been returned yet, and there is at least one embeddable which + * hasn't yet loaded, defer the loading of the default index pattern by returning undefined. + */ + if ( + panelIndexPatterns.length === 0 && + Object.keys(container.getOutput().embeddableLoaded).length > 0 && + Object.values(container.getOutput().embeddableLoaded).some((value) => value === false) + ) { + return; + } return panelIndexPatterns; }), distinctUntilChanged((a, b) => deepEqual( - a.map((ip) => ip && ip.id), - b.map((ip) => ip && ip.id) + a?.map((ip) => ip && ip.id), + b?.map((ip) => ip && ip.id) ) ), // using switchMap for previous task cancellation - switchMap((panelIndexPatterns: IndexPattern[]) => { + switchMap((panelIndexPatterns?: IndexPattern[]) => { return new Observable((observer) => { - if (panelIndexPatterns && panelIndexPatterns.length > 0) { + if (!panelIndexPatterns) return; + if (panelIndexPatterns.length > 0) { if (observer.closed) return; onUpdateIndexPatterns(panelIndexPatterns); observer.complete(); @@ -72,32 +77,8 @@ export const syncDashboardIndexPatterns = ({ }) ); - return merge( - // output of dashboard container itself - dashboardContainer.getOutput$(), - // plus output of dashboard container children, - // children may change, so make sure we subscribe/unsubscribe with switchMap - dashboardContainer.getOutput$().pipe( - map(() => dashboardContainer!.getChildIds()), - distinctUntilChanged(deepEqual), - switchMap((newChildIds: string[]) => - merge( - ...newChildIds.map((childId) => - dashboardContainer! - .getChild(childId) - .getOutput$() - // Embeddables often throw errors into their output streams. - // This should not affect dashboard loading - .pipe(catchError(() => EMPTY)) - ) - ) - ) - ) - ) - .pipe( - mapTo(dashboardContainer), - startWith(dashboardContainer), // to trigger initial index pattern update - updateIndexPatternsOperator - ) + return dashboardContainer + .getOutput$() + .pipe(mapTo(dashboardContainer), updateIndexPatternsOperator) .subscribe(); }; diff --git a/src/plugins/dashboard/public/application/top_nav/dashboard_top_nav.tsx b/src/plugins/dashboard/public/application/top_nav/dashboard_top_nav.tsx index dab74373efef5..acc656a7871b8 100644 --- a/src/plugins/dashboard/public/application/top_nav/dashboard_top_nav.tsx +++ b/src/plugins/dashboard/public/application/top_nav/dashboard_top_nav.tsx @@ -64,11 +64,7 @@ export interface DashboardTopNavState { type CompleteDashboardAppState = Required< DashboardAppState, - | 'getLatestDashboardState' - | 'dashboardContainer' - | 'savedDashboard' - | 'indexPatterns' - | 'applyFilters' + 'getLatestDashboardState' | 'dashboardContainer' | 'savedDashboard' | 'applyFilters' >; export const isCompleteDashboardAppState = ( @@ -78,7 +74,6 @@ export const isCompleteDashboardAppState = ( Boolean(state.getLatestDashboardState) && Boolean(state.dashboardContainer) && Boolean(state.savedDashboard) && - Boolean(state.indexPatterns) && Boolean(state.applyFilters) ); }; diff --git a/src/plugins/dashboard/public/dashboard_strings.ts b/src/plugins/dashboard/public/dashboard_strings.ts index 34a3317289f11..a32acf8d3bdf7 100644 --- a/src/plugins/dashboard/public/dashboard_strings.ts +++ b/src/plugins/dashboard/public/dashboard_strings.ts @@ -92,7 +92,7 @@ export const dashboardCopyToDashboardAction = { }), getDescription: () => i18n.translate('dashboard.panel.copyToDashboard.description', { - defaultMessage: "Select where to copy the panel. You're navigated to destination dashboard.", + defaultMessage: 'Choose the destination dashboard.', }), }; diff --git a/src/plugins/data/README.mdx b/src/plugins/data/README.mdx index c2da9a686cacb..40e82d3034ee2 100644 --- a/src/plugins/data/README.mdx +++ b/src/plugins/data/README.mdx @@ -49,471 +49,14 @@ This is helpful when you want to provide a user with options, for example when c ``` -## Index Patterns +## Data Views -The Index Patterns API provides a consistent method of structuring and formatting documents +The data views API provides a consistent method of structuring and formatting documents and field lists across the various Kibana apps. Its typically used in conjunction with -SearchSource for composing queries. + for composing queries. -### Index Patterns API +*Note: Kibana index patterns are currently being renamed to data views. There will be some naming inconsistencies until the transition is complete.* -- Get list of index patterns -- Get default index pattern and examine fields -- Get index pattern by id -- Find index pattern by title -- Create index pattern -- Create index pattern and save it -- Modify index pattern and save it -- Delete index pattern - -#### Get list of index pattern titles and ids - -``` -const idsAndTitles = await data.indexPatterns.getIdsWithTitle(); -idsAndTitles.forEach(({id, title}) => console.log(`Index pattern id: ${id} title: ${title}`)); -``` - -#### Get default index pattern and examine fields - -``` -const defaultIndexPattern = await data.indexPatterns.getDefault(); -defaultIndexPattern.fields.forEach(({name}) => { console.log(name); }) -``` - -#### Get index pattern by id - -``` -const id = 'xxxxxx-xxx-xxxxxx'; -const indexPattern = await data.indexPatterns.get(id); -``` - -#### Find index pattern by title - -``` -const title = 'kibana-*'; -const [indexPattern] = await data.indexPatterns.find(title); -``` - -#### Create index pattern - -``` -const indexPattern = await data.indexPatterns.create({ title: 'kibana-*' }); -``` - -#### Create index pattern and save it immediately - -``` -const indexPattern = await data.indexPatterns.createAndSave({ title: 'kibana-*' }); -``` - -#### Create index pattern, modify, and save - -``` -const indexPattern = await data.indexPatterns.create({ title: 'kibana-*' }); -indexPattern.setFieldCustomLabel('customer_name', 'Customer Name'); -data.indexPatterns.createSavedObject(indexPattern); -``` - -#### Modify index pattern and save it - -``` -indexPattern.setFieldCustomLabel('customer_name', 'Customer Name'); -await data.indexPatterns.updateSavedObject(indexPattern); -``` - -#### Delete index pattern - -``` -await data.indexPatterns.delete(indexPatternId); -``` - -### Index Patterns HTTP API - -Index patterns provide Rest-like HTTP CRUD+ API with the following endpoints: - -- Index Patterns API - - Create an index pattern — `POST /api/index_patterns/index_pattern` - - Fetch an index pattern by `{id}` — `GET /api/index_patterns/index_pattern/{id}` - - Delete an index pattern by `{id}` — `DELETE /api/index_patterns/index_pattern/{id}` - - Partially update an index pattern by `{id}` — `POST /api/index_patterns/index_pattern/{id}` -- Fields API - - Update field — `POST /api/index_patterns/index_pattern/{id}/fields` -- Scripted Fields API - - Create a scripted field — `POST /api/index_patterns/index_pattern/{id}/scripted_field` - - Upsert a scripted field — `PUT /api/index_patterns/index_pattern/{id}/scripted_field` - - Fetch a scripted field — `GET /api/index_patterns/index_pattern/{id}/scripted_field/{name}` - - Remove a scripted field — `DELETE /api/index_patterns/index_pattern/{id}/scripted_field/{name}` - - Update a scripted field — `POST /api/index_patterns/index_pattern/{id}/scripted_field/{name}` - -### Index Patterns API - -Index Patterns REST API allows you to create, retrieve and delete index patterns. I also -exposes an update endpoint which allows you to update specific fields without changing -the rest of the index pattern object. - -#### Create an index pattern - -Create an index pattern with a custom title. - -``` -POST /api/index_patterns/index_pattern -{ - "index_pattern": { - "title": "hello" - } -} -``` - -Customize creation behavior with: - -- `override` — if set to `true`, replaces an existing index pattern if an - index pattern with the provided title already exists. Defaults to `false`. -- `refresh_fields` — if set to `true` reloads index pattern fields after - the index pattern is stored. Defaults to `false`. - -``` -POST /api/index_patterns/index_pattern -{ - "override": false, - "refresh_fields": true, - "index_pattern": { - "title": "hello" - } -} -``` - -At creation all index pattern fields are option and you can provide them. - -``` -POST /api/index_patterns/index_pattern -{ - "index_pattern": { - "id": "...", - "version": "...", - "title": "...", - "type": "...", - "intervalName": "...", - "timeFieldName": "...", - "sourceFilters": [], - "fields": {}, - "typeMeta": {}, - "fieldFormats": {}, - "fieldAttrs": {} - } -} -``` - -The endpoint returns the created index pattern object. - -```json -{ - "index_pattern": {} -} -``` - -#### Fetch an index pattern by ID - -Retrieve an index pattern by its ID. - -``` -GET /api/index_patterns/index_pattern/xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx -``` - -Returns an index pattern object. - -```json -{ - "index_pattern": { - "id": "...", - "version": "...", - "title": "...", - "type": "...", - "intervalName": "...", - "timeFieldName": "...", - "sourceFilters": [], - "fields": {}, - "typeMeta": {}, - "fieldFormats": {}, - "fieldAttrs": {} - } -} -``` - -#### Delete an index pattern by ID - -Delete and index pattern by its ID. - -``` -DELETE /api/index_patterns/index_pattern/xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx -``` - -Returns an '200 OK` response with empty body on success. - -#### Partially update an index pattern by ID - -Update part of an index pattern. Only provided fields will be updated on the -index pattern, missing fields will stay as they are persisted. - -These fields can be update partially: - -- `title` -- `timeFieldName` -- `intervalName` -- `fields` (optionally refresh fields) -- `sourceFilters` -- `fieldFormatMap` -- `type` -- `typeMeta` - -Update a title of an index pattern. - -``` -POST /api/index_patterns/index_pattern/xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx -{ - "index_pattern": { - "title": "new_title" - } -} -``` - -All update fields are optional, you can specify the following fields. - -``` -POST /api/index_patterns/index_pattern -{ - "index_pattern": { - "title": "...", - "timeFieldName": "...", - "intervalName": "...", - "sourceFilters": [], - "fieldFormats": {}, - "type": "...", - "typeMeta": {}, - "fields": {} - } -} -``` - -- `refresh_fields` — if set to `true` reloads index pattern fields after - the index pattern is stored. Defaults to `false`. - -``` -POST /api/index_patterns/index_pattern -{ - "refresh_fields": true, - "index_pattern": { - "fields": {} - } -} -``` - -This endpoint returns the updated index pattern object. - -```json -{ - "index_pattern": {} -} -``` - -### Fields API - -Fields API allows to change field metadata, such as `count`, `customLabel`, and `format`. - -#### Update fields - -Update endpoint allows you to update fields presentation metadata, such as `count`, -`customLabel`, and `format`. You can update multiple fields in one request. Updates -are merges with persisted metadata. To remove existing metadata specify `null` as value. - -Set popularity `count` for field `foo`: - -``` -POST /api/index_patterns/index_pattern/xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx/fields -{ - "fields": { - "foo": { - "count": 123 - } - } -} -``` - -Update multiple metadata values and fields in one request: - -``` -POST /api/index_patterns/index_pattern/xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx/fields -{ - "fields": { - "foo": { - "count": 123, - "customLabel": "Foo" - }, - "bar": { - "customLabel": "Bar" - } - } -} -``` - -Use `null` value to delete metadata: - -``` -POST /api/index_patterns/index_pattern/xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx/fields -{ - "fields": { - "foo": { - "customLabel": null - } - } -} -``` - -This endpoint returns the updated index pattern object. - -```json -{ - "index_pattern": {} -} -``` - -### Scripted Fields API - -Scripted Fields API provides CRUD API for scripted fields of an index pattern. - -#### Create a scripted field - -Create a field by simply specifying its name, will default to `string` type. Returns -an error if a field with the provided name already exists. - -``` -POST /api/index_patterns/index_pattern/xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx/scripted_field -{ - "field": { - "name": "my_field" - } -} -``` - -Create a field by specifying all field properties. - -``` -POST /api/index_patterns/index_pattern/xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx/scripted_field -{ - "field": { - "name": "", - "type": "", - "searchable": false, - "aggregatable": false, - "count": 0, - "script": "", - "scripted": false, - "lang": "", - "conflictDescriptions": {}, - "format": {}, - "esTypes": [], - "readFromDocValues": false, - "subType": {}, - "indexed": false, - "customLabel": "", - "shortDotsEnable": false - } -} -``` - -#### Upsert a scripted field - -Creates a new field or updates an existing one, if one already exists with the same name. - -Create a field by simply specifying its name. - -``` -PUT /api/index_patterns/index_pattern/xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx/scripted_field -{ - "field": { - "name": "my_field" - } -} -``` - -Create a field by specifying all field properties. - -``` -PUT /api/index_patterns/index_pattern/xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx/scripted_field -{ - "field": { - "name": "", - "type": "", - "searchable": false, - "aggregatable": false, - "count": 0, - "script": "", - "scripted": false, - "lang": "", - "conflictDescriptions": {}, - "format": {}, - "esTypes": [], - "readFromDocValues": false, - "subType": {}, - "indexed": false, - "customLabel": "", - "shortDotsEnable": false - } -} -``` - -#### Fetch a scripted field - -Fetch an existing index pattern field by field name. - -``` -GET /api/index_patterns/index_pattern/xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx/scripted_field/ -``` - -Returns the field object. - -```json -{ - "field": {} -} -``` - -#### Delete a scripted field - -Delete a field of an index pattern. - -``` -DELETE /api/index_patterns/index_pattern/xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx/scripted_field/ -``` - -#### Update a an existing scripted field - -Updates an exiting field by mergin provided properties with the existing ones. If -there is no existing field with the specified name, returns a `404 Not Found` error. - -You can specify any field properties, except `name` which is specified in the URL path. - -``` -POST /api/index_patterns/index_pattern/xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx/scripted_field/ -{ - "field": { - "type": "", - "searchable": false, - "aggregatable": false, - "count": 0, - "script": "", - "scripted": false, - "lang": "", - "conflictDescriptions": {}, - "format": {}, - "esTypes": [], - "readFromDocValues": false, - "subType": {}, - "indexed": false, - "customLabel": "", - "shortDotsEnable": false - } -} -``` ## Query diff --git a/src/plugins/data/common/index_patterns/constants.ts b/src/plugins/data/common/index_patterns/constants.ts index 88309447a8a29..d508a62422fc7 100644 --- a/src/plugins/data/common/index_patterns/constants.ts +++ b/src/plugins/data/common/index_patterns/constants.ts @@ -6,4 +6,12 @@ * Side Public License, v 1. */ -export const RUNTIME_FIELD_TYPES = ['keyword', 'long', 'double', 'date', 'ip', 'boolean'] as const; +export const RUNTIME_FIELD_TYPES = [ + 'keyword', + 'long', + 'double', + 'date', + 'ip', + 'boolean', + 'geo_point', +] as const; diff --git a/src/plugins/data/public/public.api.md b/src/plugins/data/public/public.api.md index 2fee05760186b..995a1e7a908c5 100644 --- a/src/plugins/data/public/public.api.md +++ b/src/plugins/data/public/public.api.md @@ -791,7 +791,7 @@ export const esFilters: { // @public @deprecated (undocumented) export const esKuery: { nodeTypes: import("@kbn/es-query/target_types/kuery/node_types").NodeTypes; - fromKueryExpression: (expression: any, parseOptions?: Partial | undefined) => import("@kbn/es-query").KueryNode; + fromKueryExpression: (expression: string | import("@elastic/elasticsearch/api/types").QueryDslQueryContainer, parseOptions?: Partial | undefined) => import("@kbn/es-query").KueryNode; toElasticsearchQuery: (node: import("@kbn/es-query").KueryNode, indexPattern?: import("@kbn/es-query").IndexPatternBase | undefined, config?: Record | undefined, context?: Record | undefined) => import("@kbn/utility-types").JsonObject; }; @@ -801,12 +801,7 @@ export const esKuery: { export const esQuery: { buildEsQuery: typeof import("@kbn/es-query").buildEsQuery; getEsQueryConfig: typeof getEsQueryConfig; - buildQueryFromFilters: (filters: import("@kbn/es-query").Filter[] | undefined, indexPattern: import("@kbn/es-query").IndexPatternBase | undefined, ignoreFilterIfFieldNotInIndex?: boolean | undefined) => { - must: never[]; - filter: import("@kbn/es-query").Filter[]; - should: never[]; - must_not: import("@kbn/es-query").Filter[]; - }; + buildQueryFromFilters: (filters: import("@kbn/es-query").Filter[] | undefined, indexPattern: import("@kbn/es-query").IndexPatternBase | undefined, ignoreFilterIfFieldNotInIndex?: boolean | undefined) => import("@kbn/es-query").BoolQuery; luceneStringToDsl: typeof import("@kbn/es-query").luceneStringToDsl; decorateQuery: typeof import("@kbn/es-query").decorateQuery; }; diff --git a/src/plugins/data/public/query/filter_manager/lib/map_and_flatten_filters.ts b/src/plugins/data/public/query/filter_manager/lib/map_and_flatten_filters.ts index 4e871ad7263f6..73894a4b9ab63 100644 --- a/src/plugins/data/public/query/filter_manager/lib/map_and_flatten_filters.ts +++ b/src/plugins/data/public/query/filter_manager/lib/map_and_flatten_filters.ts @@ -7,8 +7,8 @@ */ import { compact, flatten } from 'lodash'; +import { Filter } from '@kbn/es-query'; import { mapFilter } from './map_filter'; -import { Filter } from '../../../../common'; export const mapAndFlattenFilters = (filters: Filter[]) => { return compact(flatten(filters)).map((item: Filter) => mapFilter(item)); diff --git a/src/plugins/data/public/query/filter_manager/lib/map_filter.ts b/src/plugins/data/public/query/filter_manager/lib/map_filter.ts index e017775003ec9..249c7bf47b8fb 100644 --- a/src/plugins/data/public/query/filter_manager/lib/map_filter.ts +++ b/src/plugins/data/public/query/filter_manager/lib/map_filter.ts @@ -8,6 +8,7 @@ import { reduceRight } from 'lodash'; +import { Filter } from '@kbn/es-query'; import { mapSpatialFilter } from './mappers/map_spatial_filter'; import { mapMatchAll } from './mappers/map_match_all'; import { mapPhrase } from './mappers/map_phrase'; @@ -20,7 +21,6 @@ import { mapGeoBoundingBox } from './mappers/map_geo_bounding_box'; import { mapGeoPolygon } from './mappers/map_geo_polygon'; import { mapDefault } from './mappers/map_default'; import { generateMappingChain } from './generate_mapping_chain'; -import { Filter } from '../../../../common'; export function mapFilter(filter: Filter) { /** Mappers **/ diff --git a/src/plugins/data/server/server.api.md b/src/plugins/data/server/server.api.md index 121cd8ebc0af7..c1f22d1be1d01 100644 --- a/src/plugins/data/server/server.api.md +++ b/src/plugins/data/server/server.api.md @@ -124,7 +124,7 @@ export const esFilters: { // @public (undocumented) export const esKuery: { nodeTypes: import("@kbn/es-query/target_types/kuery/node_types").NodeTypes; - fromKueryExpression: (expression: any, parseOptions?: Partial | undefined) => import("@kbn/es-query").KueryNode; + fromKueryExpression: (expression: string | import("@elastic/elasticsearch/api/types").QueryDslQueryContainer, parseOptions?: Partial | undefined) => import("@kbn/es-query").KueryNode; toElasticsearchQuery: (node: import("@kbn/es-query").KueryNode, indexPattern?: import("@kbn/es-query").IndexPatternBase | undefined, config?: Record | undefined, context?: Record | undefined) => import("@kbn/utility-types").JsonObject; }; @@ -132,12 +132,7 @@ export const esKuery: { // // @public (undocumented) export const esQuery: { - buildQueryFromFilters: (filters: import("@kbn/es-query").Filter[] | undefined, indexPattern: import("@kbn/es-query").IndexPatternBase | undefined, ignoreFilterIfFieldNotInIndex?: boolean | undefined) => { - must: never[]; - filter: import("@kbn/es-query").Filter[]; - should: never[]; - must_not: import("@kbn/es-query").Filter[]; - }; + buildQueryFromFilters: (filters: import("@kbn/es-query").Filter[] | undefined, indexPattern: import("@kbn/es-query").IndexPatternBase | undefined, ignoreFilterIfFieldNotInIndex?: boolean | undefined) => import("@kbn/es-query").BoolQuery; getEsQueryConfig: typeof getEsQueryConfig; buildEsQuery: typeof import("@kbn/es-query").buildEsQuery; }; diff --git a/src/plugins/discover/public/application/apps/main/utils/fetch_chart.ts b/src/plugins/discover/public/application/apps/main/utils/fetch_chart.ts index 25a2a964a778f..67f34c7503c59 100644 --- a/src/plugins/discover/public/application/apps/main/utils/fetch_chart.ts +++ b/src/plugins/discover/public/application/apps/main/utils/fetch_chart.ts @@ -50,6 +50,14 @@ export function fetchChart( sendLoadingMsg(charts$); sendLoadingMsg(totalHits$); + const executionContext = { + type: 'application', + name: 'discover', + description: 'fetch chart data and total hits', + url: window.location.pathname, + id: '', + }; + const fetch$ = searchSource .fetch$({ abortSignal: abortController.signal, @@ -64,6 +72,7 @@ export function fetchChart( 'This request queries Elasticsearch to fetch the aggregation data for the chart.', }), }, + executionContext, }) .pipe(filter((res) => isCompleteResponse(res))); diff --git a/src/plugins/discover/public/application/apps/main/utils/fetch_documents.ts b/src/plugins/discover/public/application/apps/main/utils/fetch_documents.ts index edaf86cef6874..2f06a9dbbb3db 100644 --- a/src/plugins/discover/public/application/apps/main/utils/fetch_documents.ts +++ b/src/plugins/discover/public/application/apps/main/utils/fetch_documents.ts @@ -41,6 +41,14 @@ export const fetchDocuments = ( sendLoadingMsg(documents$); + const executionContext = { + type: 'application', + name: 'discover', + description: 'fetch documents', + url: window.location.pathname, + id: '', + }; + const fetch$ = searchSource .fetch$({ abortSignal: abortController.signal, @@ -54,6 +62,7 @@ export const fetchDocuments = ( defaultMessage: 'This request queries Elasticsearch to fetch the documents.', }), }, + executionContext, }) .pipe(filter((res) => isCompleteResponse(res))); diff --git a/src/plugins/discover/public/application/apps/main/utils/fetch_total_hits.ts b/src/plugins/discover/public/application/apps/main/utils/fetch_total_hits.ts index 4fb43652f28c3..9688f5ddd614d 100644 --- a/src/plugins/discover/public/application/apps/main/utils/fetch_total_hits.ts +++ b/src/plugins/discover/public/application/apps/main/utils/fetch_total_hits.ts @@ -46,6 +46,14 @@ export function fetchTotalHits( sendLoadingMsg(totalHits$); + const executionContext = { + type: 'application', + name: 'discover', + description: 'fetch total hits', + url: window.location.pathname, + id: '', + }; + const fetch$ = searchSource .fetch$({ inspector: { @@ -59,6 +67,7 @@ export function fetchTotalHits( }, abortSignal: abortController.signal, sessionId: searchSessionId, + executionContext, }) .pipe(filter((res) => isCompleteResponse(res))); diff --git a/src/plugins/discover/public/application/embeddable/saved_search_embeddable.tsx b/src/plugins/discover/public/application/embeddable/saved_search_embeddable.tsx index 3fd7b2f50d319..1981f0228d2c7 100644 --- a/src/plugins/discover/public/application/embeddable/saved_search_embeddable.tsx +++ b/src/plugins/discover/public/application/embeddable/saved_search_embeddable.tsx @@ -170,6 +170,14 @@ export class SavedSearchEmbeddable this.searchProps!.isLoading = true; this.updateOutput({ loading: true, error: undefined }); + const executionContext = { + type: this.type, + name: 'discover', + id: this.savedSearch.id, + description: this.output.title || this.output.defaultTitle || '', + url: this.output.editUrl, + parent: this.input.executionContext, + }; try { // Make the request @@ -187,6 +195,7 @@ export class SavedSearchEmbeddable 'This request queries Elasticsearch to fetch the data for the search.', }), }, + executionContext, }) .toPromise(); this.updateOutput({ loading: false, error: undefined }); diff --git a/src/plugins/embeddable/public/lib/containers/container.ts b/src/plugins/embeddable/public/lib/containers/container.ts index fabe3cd32f34b..5d51b0e172937 100644 --- a/src/plugins/embeddable/public/lib/containers/container.ts +++ b/src/plugins/embeddable/public/lib/containers/container.ts @@ -53,6 +53,22 @@ export abstract class Container< }); } + public setChildLoaded(embeddable: IEmbeddable) { + // make sure the panel wasn't removed in the mean time, since the embeddable creation is async + if (!this.input.panels[embeddable.id]) { + embeddable.destroy(); + return; + } + + this.children[embeddable.id] = embeddable; + this.updateOutput({ + embeddableLoaded: { + ...this.output.embeddableLoaded, + [embeddable.id]: true, + }, + } as Partial); + } + public updateInputForChild( id: string, changes: Partial @@ -307,19 +323,9 @@ export abstract class Container< // switch over to inline creation we can probably clean this up, and force EmbeddableFactory.create to always // return an embeddable, or throw an error. if (embeddable) { - // make sure the panel wasn't removed in the mean time, since the embeddable creation is async - if (!this.input.panels[panel.explicitInput.id]) { - embeddable.destroy(); - return; + if (!embeddable.deferEmbeddableLoad) { + this.setChildLoaded(embeddable); } - - this.children[embeddable.id] = embeddable; - this.updateOutput({ - embeddableLoaded: { - ...this.output.embeddableLoaded, - [panel.explicitInput.id]: true, - }, - } as Partial); } else if (embeddable === undefined) { this.removeEmbeddable(panel.explicitInput.id); } diff --git a/src/plugins/embeddable/public/lib/containers/i_container.ts b/src/plugins/embeddable/public/lib/containers/i_container.ts index fd36993d93415..aec1b09270943 100644 --- a/src/plugins/embeddable/public/lib/containers/i_container.ts +++ b/src/plugins/embeddable/public/lib/containers/i_container.ts @@ -62,6 +62,13 @@ export interface IContainer< */ getChild = Embeddable>(id: string): E; + /** + * Embeddables which have deferEmbeddableLoad set to true need to manually call setChildLoaded + * on their parent container to communicate when they have finished loading. + * @param embeddable - the embeddable to set + */ + setChildLoaded(embeddable: E): void; + /** * Removes the embeddable with the given id. * @param embeddableId diff --git a/src/plugins/embeddable/public/lib/embeddables/embeddable.tsx b/src/plugins/embeddable/public/lib/embeddables/embeddable.tsx index a0cd213b7bf24..d8c133890a669 100644 --- a/src/plugins/embeddable/public/lib/embeddables/embeddable.tsx +++ b/src/plugins/embeddable/public/lib/embeddables/embeddable.tsx @@ -30,6 +30,7 @@ export abstract class Embeddable< public readonly parent?: IContainer; public readonly isContainer: boolean = false; + public readonly deferEmbeddableLoad: boolean = false; public abstract readonly type: string; public readonly id: string; public fatalError?: Error; @@ -196,6 +197,16 @@ export abstract class Embeddable< return; } + /** + * communicate to the parent embeddable that this embeddable's initialization is finished. + * This only applies to embeddables which defer their loading state with deferEmbeddableLoad. + */ + protected setInitializationFinished() { + if (this.deferEmbeddableLoad && this.parent?.isContainer) { + this.parent.setChildLoaded(this); + } + } + protected updateOutput(outputChanges: Partial): void { const newOutput = { ...this.output, @@ -210,6 +221,11 @@ export abstract class Embeddable< protected onFatalError(e: Error) { this.fatalError = e; this.output$.error(e); + // if the container is waiting for this embeddable to complete loading, + // a fatal error counts as complete. + if (this.deferEmbeddableLoad && this.parent?.isContainer) { + this.parent.setChildLoaded(this); + } } private onResetInput(newInput: TEmbeddableInput) { diff --git a/src/plugins/embeddable/public/lib/embeddables/i_embeddable.ts b/src/plugins/embeddable/public/lib/embeddables/i_embeddable.ts index 60a88133efcce..bbac617d41590 100644 --- a/src/plugins/embeddable/public/lib/embeddables/i_embeddable.ts +++ b/src/plugins/embeddable/public/lib/embeddables/i_embeddable.ts @@ -60,6 +60,14 @@ export interface IEmbeddable< **/ readonly id: string; + /** + * If set to true, defer embeddable load tells the container that this embeddable + * type isn't completely loaded when the constructor returns. This embeddable + * will have to manually call setChildLoaded on its parent when all of its initial + * output is finalized. For instance, after loading a saved object. + */ + readonly deferEmbeddableLoad: boolean; + /** * Unique ID an embeddable is assigned each time it is initialized. This ID * is different for different instances of the same embeddable. For example, diff --git a/src/plugins/embeddable/public/public.api.md b/src/plugins/embeddable/public/public.api.md index cf28f65ceaa79..2e46cb82dc592 100644 --- a/src/plugins/embeddable/public/public.api.md +++ b/src/plugins/embeddable/public/public.api.md @@ -185,6 +185,8 @@ export abstract class Container = { // (undocumented) removeEmbeddable(embeddableId: string): void; // (undocumented) + setChildLoaded(embeddable: IEmbeddable): void; + // (undocumented) untilEmbeddableLoaded(id: string): Promise; // (undocumented) updateInputForChild(id: string, changes: Partial): void; @@ -265,6 +267,8 @@ export class EditPanelAction implements Action_3 { // @public (undocumented) export abstract class Embeddable implements IEmbeddable { constructor(input: TEmbeddableInput, output: TEmbeddableOutput, parent?: IContainer); + // (undocumented) + readonly deferEmbeddableLoad: boolean; destroy(): void; // (undocumented) fatalError?: Error; @@ -306,6 +310,7 @@ export abstract class Embeddable = Embeddable>(id: string): E; getInputForChild(id: string): EEI; removeEmbeddable(embeddableId: string): void; + setChildLoaded(embeddable: E): void; untilEmbeddableLoaded(id: string): Promise; updateInputForChild(id: string, changes: Partial): void; } @@ -659,6 +665,7 @@ export interface IContainer { + readonly deferEmbeddableLoad: boolean; destroy(): void; enhancements?: object; fatalError?: Error; diff --git a/src/plugins/es_ui_shared/public/components/code_editor/__snapshots__/code_editor.test.tsx.snap b/src/plugins/es_ui_shared/public/components/code_editor/__snapshots__/code_editor.test.tsx.snap new file mode 100644 index 0000000000000..aeab9a66c7694 --- /dev/null +++ b/src/plugins/es_ui_shared/public/components/code_editor/__snapshots__/code_editor.test.tsx.snap @@ -0,0 +1,627 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`EuiCodeEditor behavior hint element should be disabled when the ui ace box gains focus 1`] = ` + +`; + +exports[`EuiCodeEditor behavior hint element should be enabled when the ui ace box loses focus 1`] = ` + +`; + +exports[`EuiCodeEditor behavior hint element should be tabable 1`] = ` + +`; + +exports[`EuiCodeEditor is rendered 1`] = ` +
+ +
+