From 8b85491944d3cf87ecb5106fa497b7991567a8fd Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Mon, 12 Jun 2023 15:28:28 +0200 Subject: [PATCH 01/27] Add other file spec for routing rules --- .../internal/validator/semantic/types_test.go | 5 ++ .../data_stream/routing-rules.spec.yml | 52 +++++++++++++++++++ spec/integration/data_stream/spec.yml | 7 +++ .../routing_rules/agent/stream/stream.yml.hbs | 7 +++ .../elasticsearch/ingest_pipeline/default.yml | 10 ++++ .../routing_rules/fields/base-fields.yml | 12 +++++ .../data_stream/routing_rules/manifest.yml | 13 +++++ .../routing_rules/routing_rules.yml | 34 ++++++++++++ 8 files changed, 140 insertions(+) create mode 100644 spec/integration/data_stream/routing-rules.spec.yml create mode 100644 test/packages/good_v2/data_stream/routing_rules/agent/stream/stream.yml.hbs create mode 100644 test/packages/good_v2/data_stream/routing_rules/elasticsearch/ingest_pipeline/default.yml create mode 100644 test/packages/good_v2/data_stream/routing_rules/fields/base-fields.yml create mode 100644 test/packages/good_v2/data_stream/routing_rules/manifest.yml create mode 100644 test/packages/good_v2/data_stream/routing_rules/routing_rules.yml diff --git a/code/go/internal/validator/semantic/types_test.go b/code/go/internal/validator/semantic/types_test.go index 598a8d822..ed8f94e35 100644 --- a/code/go/internal/validator/semantic/types_test.go +++ b/code/go/internal/validator/semantic/types_test.go @@ -79,6 +79,11 @@ func TestListFieldsFiles(t *testing.T) { fullFilePath: "../../../../../test/packages/good_v2/data_stream/pe/fields/some_fields.yml", dataStream: "pe", }, + fieldFileMetadata{ + filePath: "data_stream/routing_rules/fields/base-fields.yml", + fullFilePath: "../../../../../test/packages/good_v2/data_stream/routing_rules/fields/base-fields.yml", + dataStream: "routing_rules", + }, fieldFileMetadata{ filePath: "data_stream/skipped_tests/fields/base-fields.yml", fullFilePath: "../../../../../test/packages/good_v2/data_stream/skipped_tests/fields/base-fields.yml", diff --git a/spec/integration/data_stream/routing-rules.spec.yml b/spec/integration/data_stream/routing-rules.spec.yml new file mode 100644 index 000000000..cbcf14393 --- /dev/null +++ b/spec/integration/data_stream/routing-rules.spec.yml @@ -0,0 +1,52 @@ +## +## Describes the specification for a data stream's manifest.yml file +## +spec: + # Everything under here follows JSON schema (https://json-schema.org/), written as YAML for readability + type: object + additionalProperties: false + definitions: + route: + type: object + properties: + dataset: + type: string + examples: + - nginx.error + - nginx + if: + type: string + examples: + - "ctx?.file?.path?.contains('/var/log/nginx/error')" + - "ctx?.container?.image?.name == 'nginx'" + namespace: + type: array + items: + type: string + examples: + - default + - "{{ labels.dasta_stream.namespace}}" + required: + - dataset + - if + - namespace + properties: + routing_rules: + description: Routing rules set. + type: array + items: + type: object + additionalProperties: false + properties: + name: + type: string + rules: + type: array + items: + $ref: "#/definitions/route" + required: + - name + - rules + + required: + - routing_rules diff --git a/spec/integration/data_stream/spec.yml b/spec/integration/data_stream/spec.yml index 3d8078397..347ac096d 100644 --- a/spec/integration/data_stream/spec.yml +++ b/spec/integration/data_stream/spec.yml @@ -78,3 +78,10 @@ spec: required: false visibility: private $ref: "./_dev/spec.yml" + - description: Routing rules definitions file + type: file + contentMediaType: "application/x-yaml" + sizeLimit: 5MB + name: "routing_rules.yml" + required: false + $ref: "./routing-rules.spec.yml" diff --git a/test/packages/good_v2/data_stream/routing_rules/agent/stream/stream.yml.hbs b/test/packages/good_v2/data_stream/routing_rules/agent/stream/stream.yml.hbs new file mode 100644 index 000000000..5845510de --- /dev/null +++ b/test/packages/good_v2/data_stream/routing_rules/agent/stream/stream.yml.hbs @@ -0,0 +1,7 @@ +paths: +{{#each paths as |path i|}} + - {{path}} +{{/each}} +exclude_files: [".gz$"] +processors: + - add_locale: ~ diff --git a/test/packages/good_v2/data_stream/routing_rules/elasticsearch/ingest_pipeline/default.yml b/test/packages/good_v2/data_stream/routing_rules/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 000000000..81221adf3 --- /dev/null +++ b/test/packages/good_v2/data_stream/routing_rules/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,10 @@ +--- +description: Pipeline for processing sample logs +processors: +- set: + field: sample_field + value: "1" +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' \ No newline at end of file diff --git a/test/packages/good_v2/data_stream/routing_rules/fields/base-fields.yml b/test/packages/good_v2/data_stream/routing_rules/fields/base-fields.yml new file mode 100644 index 000000000..7c798f453 --- /dev/null +++ b/test/packages/good_v2/data_stream/routing_rules/fields/base-fields.yml @@ -0,0 +1,12 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. diff --git a/test/packages/good_v2/data_stream/routing_rules/manifest.yml b/test/packages/good_v2/data_stream/routing_rules/manifest.yml new file mode 100644 index 000000000..0e561e2e9 --- /dev/null +++ b/test/packages/good_v2/data_stream/routing_rules/manifest.yml @@ -0,0 +1,13 @@ +title: "This data stream contains routing rules definitions" +type: logs +streams: + - input: logfile + title: Sample logs + description: Collect sample logs + vars: + - name: paths + type: text + title: Paths + multi: true + default: + - /var/log/*.log diff --git a/test/packages/good_v2/data_stream/routing_rules/routing_rules.yml b/test/packages/good_v2/data_stream/routing_rules/routing_rules.yml new file mode 100644 index 000000000..8d883b059 --- /dev/null +++ b/test/packages/good_v2/data_stream/routing_rules/routing_rules.yml @@ -0,0 +1,34 @@ +routing_rules: + # "Local" routing rules are included under this current dataset, not a special case + - name: nginx + rules: # Route error logs to `nginx.error` when they're sourced from an error logfile + - dataset: nginx.error + if: "ctx?.file?.path?.contains('/var/log/nginx/error')" + namespace: + - "{{labels.data_stream.namespace}}" + - default + + # Route access logs to `nginx.access` when they're sourced from an access logfile + - dataset: nginx.access + if: "ctx?.file?.path?.contains('/var/log/nginx/access')" + namespace: + - "{{labels.data_stream.namespace}}" + - default + + # Route K8's container logs to this catch-all dataset for further routing + - name: k8s.router + rules: + - dataset: nginx + if: "ctx?.container?.image?.name == 'nginx'" + namespace: + - "{{labels.data_stream.namespace}}" + - default + + # Route syslog entries tagged with nginx to this catch-all dataset + - name: syslog + rules: + - dataset: nginx + if: "ctx?.tags?.contains('nginx')" + namespace: + - "{{labels.data_stream.namespace}}" + - default From 513c6a781c9419bf4890a9123fb21758cf10c9d1 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Mon, 12 Jun 2023 15:37:19 +0200 Subject: [PATCH 02/27] Add changelog entry Increase following version to 2.9.0 --- spec/changelog.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/spec/changelog.yml b/spec/changelog.yml index 342019a67..1c4053ed8 100644 --- a/spec/changelog.yml +++ b/spec/changelog.yml @@ -2,17 +2,17 @@ ## This file documents changes in the package specification. It is NOT a package specification file. ## Newer entries go at the bottom of each in-development version. ## -- version: 2.8.1-next +- version: 2.9.0-next changes: - - description: Prepare for next version - type: enhancement - link: https://github.com/elastic/package-spec/pull/522 - description: Add validation for data types of metrics type: enhancement link: https://github.com/elastic/package-spec/pull/525 - description: Extend benchmarks spec to add rally scenario type: enhancement link: https://github.com/elastic/package-spec/pull/524 + - description: Add support to define routing rules in data streams + type: enhancement + link: https://github.com/elastic/package-spec/pull/535 - version: 2.8.0 changes: - description: Add new subcategory "advanced_analytics_ueba" From 8b01b292f2b5374eb2d34e260ca13acdd49b6749 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Mon, 12 Jun 2023 18:14:52 +0200 Subject: [PATCH 03/27] Move routing_rules definition to datastream manifest --- code/go/pkg/validator/validator_test.go | 6 ++ .../integration/data_stream/manifest.spec.yml | 52 ++++++++++ .../data_stream/routing-rules.spec.yml | 52 ---------- spec/integration/data_stream/spec.yml | 14 +-- test/packages/bad_routing_rules/LICENSE.txt | 93 ++++++++++++++++++ test/packages/bad_routing_rules/changelog.yml | 6 ++ .../rules/agent/stream/stream.yml.hbs | 7 ++ .../elasticsearch/ingest_pipeline/default.yml | 10 ++ .../data_stream/rules/fields/base-fields.yml | 12 +++ .../data_stream/rules/manifest.yml} | 27 +++-- .../packages/bad_routing_rules/docs/README.md | 84 ++++++++++++++++ .../bad_routing_rules/img/sample-logo.svg | 1 + .../img/sample-screenshot.png | Bin 0 -> 18849 bytes test/packages/bad_routing_rules/manifest.yml | 33 +++++++ .../data_stream/routing_rules/manifest.yml | 36 +++++++ test/packages/good_v2/manifest.yml | 2 +- 16 files changed, 368 insertions(+), 67 deletions(-) delete mode 100644 spec/integration/data_stream/routing-rules.spec.yml create mode 100644 test/packages/bad_routing_rules/LICENSE.txt create mode 100644 test/packages/bad_routing_rules/changelog.yml create mode 100644 test/packages/bad_routing_rules/data_stream/rules/agent/stream/stream.yml.hbs create mode 100644 test/packages/bad_routing_rules/data_stream/rules/elasticsearch/ingest_pipeline/default.yml create mode 100644 test/packages/bad_routing_rules/data_stream/rules/fields/base-fields.yml rename test/packages/{good_v2/data_stream/routing_rules/routing_rules.yml => bad_routing_rules/data_stream/rules/manifest.yml} (66%) create mode 100644 test/packages/bad_routing_rules/docs/README.md create mode 100644 test/packages/bad_routing_rules/img/sample-logo.svg create mode 100644 test/packages/bad_routing_rules/img/sample-screenshot.png create mode 100644 test/packages/bad_routing_rules/manifest.yml diff --git a/code/go/pkg/validator/validator_test.go b/code/go/pkg/validator/validator_test.go index f56b941a4..db2606701 100644 --- a/code/go/pkg/validator/validator_test.go +++ b/code/go/pkg/validator/validator_test.go @@ -160,6 +160,12 @@ func TestValidateFile(t *testing.T) { `field 3: Must not be present`, }, }, + "bad_routing_rules": { + "data_stream/rules/manifest.yml", + []string{ + `field (root): dataset is required`, + }, + }, } for pkgName, test := range tests { diff --git a/spec/integration/data_stream/manifest.spec.yml b/spec/integration/data_stream/manifest.spec.yml index 00e901f99..ca271ce86 100644 --- a/spec/integration/data_stream/manifest.spec.yml +++ b/spec/integration/data_stream/manifest.spec.yml @@ -169,6 +169,31 @@ spec: hidden: description: Makes the data stream hidden type: boolean + route: + description: Routing rule definition + type: object + properties: + dataset: + type: string + examples: + - nginx.error + - nginx + if: + type: string + examples: + - "ctx?.file?.path?.contains('/var/log/nginx/error')" + - "ctx?.container?.image?.name == 'nginx'" + namespace: + type: array + items: + type: string + examples: + - default + - "{{ labels.dasta_stream.namespace}}" + required: + - dataset + - if + - namespace properties: dataset: description: Name of data set. @@ -307,6 +332,29 @@ spec: dynamic_namespace: description: When set to true, agents running this integration are granted data stream privileges for all namespaces of its type type: boolean + routing_rules: + description: Routing rules set. + type: array + items: + type: object + additionalProperties: false + properties: + dataset: + type: string + rules: + type: array + items: + $ref: "#/definitions/route" + required: + - dataset + - rules + allOf: + - if: + required: + - routing_rules + then: + required: + - dataset required: - title versions: @@ -318,3 +366,7 @@ versions: patch: - op: remove path: /properties/type/enum/4 + - before: 2.9.0 + patch: + - op: remove + path: /properties/routing_rules diff --git a/spec/integration/data_stream/routing-rules.spec.yml b/spec/integration/data_stream/routing-rules.spec.yml deleted file mode 100644 index cbcf14393..000000000 --- a/spec/integration/data_stream/routing-rules.spec.yml +++ /dev/null @@ -1,52 +0,0 @@ -## -## Describes the specification for a data stream's manifest.yml file -## -spec: - # Everything under here follows JSON schema (https://json-schema.org/), written as YAML for readability - type: object - additionalProperties: false - definitions: - route: - type: object - properties: - dataset: - type: string - examples: - - nginx.error - - nginx - if: - type: string - examples: - - "ctx?.file?.path?.contains('/var/log/nginx/error')" - - "ctx?.container?.image?.name == 'nginx'" - namespace: - type: array - items: - type: string - examples: - - default - - "{{ labels.dasta_stream.namespace}}" - required: - - dataset - - if - - namespace - properties: - routing_rules: - description: Routing rules set. - type: array - items: - type: object - additionalProperties: false - properties: - name: - type: string - rules: - type: array - items: - $ref: "#/definitions/route" - required: - - name - - rules - - required: - - routing_rules diff --git a/spec/integration/data_stream/spec.yml b/spec/integration/data_stream/spec.yml index 347ac096d..1ff45a663 100644 --- a/spec/integration/data_stream/spec.yml +++ b/spec/integration/data_stream/spec.yml @@ -78,10 +78,10 @@ spec: required: false visibility: private $ref: "./_dev/spec.yml" - - description: Routing rules definitions file - type: file - contentMediaType: "application/x-yaml" - sizeLimit: 5MB - name: "routing_rules.yml" - required: false - $ref: "./routing-rules.spec.yml" + # - description: Routing rules definitions file + # type: file + # contentMediaType: "application/x-yaml" + # sizeLimit: 5MB + # name: "routing_rules.yml" + # required: false + # $ref: "./routing-rules.spec.yml" diff --git a/test/packages/bad_routing_rules/LICENSE.txt b/test/packages/bad_routing_rules/LICENSE.txt new file mode 100644 index 000000000..809108b85 --- /dev/null +++ b/test/packages/bad_routing_rules/LICENSE.txt @@ -0,0 +1,93 @@ +Elastic License 2.0 + +URL: https://www.elastic.co/licensing/elastic-license + +## Acceptance + +By using the software, you agree to all of the terms and conditions below. + +## Copyright License + +The licensor grants you a non-exclusive, royalty-free, worldwide, +non-sublicensable, non-transferable license to use, copy, distribute, make +available, and prepare derivative works of the software, in each case subject to +the limitations and conditions below. + +## Limitations + +You may not provide the software to third parties as a hosted or managed +service, where the service provides users with access to any substantial set of +the features or functionality of the software. + +You may not move, change, disable, or circumvent the license key functionality +in the software, and you may not remove or obscure any functionality in the +software that is protected by the license key. + +You may not alter, remove, or obscure any licensing, copyright, or other notices +of the licensor in the software. Any use of the licensor’s trademarks is subject +to applicable law. + +## Patents + +The licensor grants you a license, under any patent claims the licensor can +license, or becomes able to license, to make, have made, use, sell, offer for +sale, import and have imported the software, in each case subject to the +limitations and conditions in this license. This license does not cover any +patent claims that you cause to be infringed by modifications or additions to +the software. If you or your company make any written claim that the software +infringes or contributes to infringement of any patent, your patent license for +the software granted under these terms ends immediately. If your company makes +such a claim, your patent license ends immediately for work on behalf of your +company. + +## Notices + +You must ensure that anyone who gets a copy of any part of the software from you +also gets a copy of these terms. + +If you modify the software, you must include in any modified copies of the +software prominent notices stating that you have modified the software. + +## No Other Rights + +These terms do not imply any licenses other than those expressly granted in +these terms. + +## Termination + +If you use the software in violation of these terms, such use is not licensed, +and your licenses will automatically terminate. If the licensor provides you +with a notice of your violation, and you cease all violation of this license no +later than 30 days after you receive that notice, your licenses will be +reinstated retroactively. However, if you violate these terms after such +reinstatement, any additional violation of these terms will cause your licenses +to terminate automatically and permanently. + +## No Liability + +*As far as the law allows, the software comes as is, without any warranty or +condition, and the licensor will not be liable to you for any damages arising +out of these terms or the use or nature of the software, under any kind of +legal claim.* + +## Definitions + +The **licensor** is the entity offering these terms, and the **software** is the +software the licensor makes available under these terms, including any portion +of it. + +**you** refers to the individual or entity agreeing to these terms. + +**your company** is any legal entity, sole proprietorship, or other kind of +organization that you work for, plus all organizations that have control over, +are under the control of, or are under common control with that +organization. **control** means ownership of substantially all the assets of an +entity, or the power to direct its management and policies by vote, contract, or +otherwise. Control can be direct or indirect. + +**your licenses** are all the licenses granted to you for the software under +these terms. + +**use** means anything you do with the software requiring one of your licenses. + +**trademark** means trademarks, service marks, and similar rights. diff --git a/test/packages/bad_routing_rules/changelog.yml b/test/packages/bad_routing_rules/changelog.yml new file mode 100644 index 000000000..bb0320a52 --- /dev/null +++ b/test/packages/bad_routing_rules/changelog.yml @@ -0,0 +1,6 @@ +# newer versions go on top +- version: "0.0.1" + changes: + - description: Initial draft of the package + type: enhancement + link: https://github.com/elastic/integrations/pull/1 # FIXME Replace with the real PR link diff --git a/test/packages/bad_routing_rules/data_stream/rules/agent/stream/stream.yml.hbs b/test/packages/bad_routing_rules/data_stream/rules/agent/stream/stream.yml.hbs new file mode 100644 index 000000000..5845510de --- /dev/null +++ b/test/packages/bad_routing_rules/data_stream/rules/agent/stream/stream.yml.hbs @@ -0,0 +1,7 @@ +paths: +{{#each paths as |path i|}} + - {{path}} +{{/each}} +exclude_files: [".gz$"] +processors: + - add_locale: ~ diff --git a/test/packages/bad_routing_rules/data_stream/rules/elasticsearch/ingest_pipeline/default.yml b/test/packages/bad_routing_rules/data_stream/rules/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 000000000..81221adf3 --- /dev/null +++ b/test/packages/bad_routing_rules/data_stream/rules/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,10 @@ +--- +description: Pipeline for processing sample logs +processors: +- set: + field: sample_field + value: "1" +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' \ No newline at end of file diff --git a/test/packages/bad_routing_rules/data_stream/rules/fields/base-fields.yml b/test/packages/bad_routing_rules/data_stream/rules/fields/base-fields.yml new file mode 100644 index 000000000..7c798f453 --- /dev/null +++ b/test/packages/bad_routing_rules/data_stream/rules/fields/base-fields.yml @@ -0,0 +1,12 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. diff --git a/test/packages/good_v2/data_stream/routing_rules/routing_rules.yml b/test/packages/bad_routing_rules/data_stream/rules/manifest.yml similarity index 66% rename from test/packages/good_v2/data_stream/routing_rules/routing_rules.yml rename to test/packages/bad_routing_rules/data_stream/rules/manifest.yml index 8d883b059..ae8de2599 100644 --- a/test/packages/good_v2/data_stream/routing_rules/routing_rules.yml +++ b/test/packages/bad_routing_rules/data_stream/rules/manifest.yml @@ -1,6 +1,19 @@ +title: "Rules" +type: logs +streams: + - input: logfile + title: Sample logs + description: Collect sample logs + vars: + - name: paths + type: text + title: Paths + multi: true + default: + - /var/log/*.log routing_rules: # "Local" routing rules are included under this current dataset, not a special case - - name: nginx + - dataset: routing_rules rules: # Route error logs to `nginx.error` when they're sourced from an error logfile - dataset: nginx.error if: "ctx?.file?.path?.contains('/var/log/nginx/error')" @@ -16,19 +29,19 @@ routing_rules: - default # Route K8's container logs to this catch-all dataset for further routing - - name: k8s.router + - dataset: k8s.router rules: - - dataset: nginx - if: "ctx?.container?.image?.name == 'nginx'" + - dataset: routing_rules + if: "ctx?.container?.image?.name == 'routing_rules'" namespace: - "{{labels.data_stream.namespace}}" - default # Route syslog entries tagged with nginx to this catch-all dataset - - name: syslog + - dataset: syslog rules: - - dataset: nginx - if: "ctx?.tags?.contains('nginx')" + - dataset: routing_rules + if: "ctx?.tags?.contains('routing_rules')" namespace: - "{{labels.data_stream.namespace}}" - default diff --git a/test/packages/bad_routing_rules/docs/README.md b/test/packages/bad_routing_rules/docs/README.md new file mode 100644 index 000000000..0d5cbcf21 --- /dev/null +++ b/test/packages/bad_routing_rules/docs/README.md @@ -0,0 +1,84 @@ + + + +# Bad Routing Rules + + + +## Data streams + + + + + + + + + + + +## Requirements + +You need Elasticsearch for storing and searching your data and Kibana for visualizing and managing it. +You can use our hosted Elasticsearch Service on Elastic Cloud, which is recommended, or self-manage the Elastic Stack on your own hardware. + + + +## Setup + + + +For step-by-step instructions on how to set up an integration, see the +[Getting started](https://www.elastic.co/guide/en/welcome-to-elastic/current/getting-started-observability.html) guide. + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/test/packages/bad_routing_rules/img/sample-logo.svg b/test/packages/bad_routing_rules/img/sample-logo.svg new file mode 100644 index 000000000..6268dd88f --- /dev/null +++ b/test/packages/bad_routing_rules/img/sample-logo.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/test/packages/bad_routing_rules/img/sample-screenshot.png b/test/packages/bad_routing_rules/img/sample-screenshot.png new file mode 100644 index 0000000000000000000000000000000000000000..d7a56a3ecc078c38636698cefba33f86291dd178 GIT binary patch literal 18849 zcmeEu^S~#!E#4Tq;}?6chqwB{?k=6jc5D4>l%v(rleJ2Y%tW zDj9g7px}|*e;{M?LDwiK3@FNS(lDRTd-MJYIyUJCN948~OJk1M(DrJyI#iV;P4k~& zFZo35IfQt0RwlUN`48^6(1dv_wm(y1xhEdMld=Y?!%u=fPT_*{3( zwBwz3#qR}_)t>C*jp5@U)Ti~B)Y;qq*TRxZJ7ZRN_^A3TDAEM*@7Ve%(Ro7=1%1B< zVj6GBUTxXev>_^SFA zgKZ=g4aTS}9>Ofj7cSB0WO?gQ)x=+!hs_)b$6#>ScFZ>XAoIX)%Bc|BDC~JFBk0f0 z0NY}6gb)&!qx^FWC(!ji+Kl$V$2|ocA=vN0TM0Y`U?tX+T)c*C zA!IL(T2Vm%MCLa85^if@J@Kkprx8QN5!6eCR@4Oa5S?4-4|ou?90mFCM8D!;n(5xz zO}-*t!TntN>|a$s(kGQg1P-U?hqvGF2_fGvd&~yZ_l3Qf&j~XWa=;>N3#-~#zjzcc z*m18L`A-K2o!d@J>a8SRbm4P&-q1(H>|JgIymDbnJF&@008`=X!P?4DGgZb>voUl^ zNJKgPR4S={)3vuk_{n@=M8q;;aJL>q+VLdTnO=}`&x;1DKjJA3*f*idS{jP5?+;!W zn-^7021Z4zv`Aq`hmX1aid997RNh3fa-@PG(W7TzKa1W&5^y3|lPeETP7j9qXpo4)7%(W0_2 z^Nmq;t@rb1eP3?%kOkH`P%!zTC7ZHjSfNN3*Sb#=3#jB*KpNGNfnRZ{N(6DrW(;B2Bwom<%m?VQP%K+ zsFeF1-(DY}oP@)w^Kw~gPg03q?N;)Ec6^|nikA34T~RynX*z}H>R~qgT$`Zbhn8wzZs$j2fsGN&rOK-mIBBvzD@a8FgbLpL!h5N^u&0wG} zq!#md3MHITv?3@$37J?lc_5*LWJTTjel;IiU-Yq;(g9I^D&KN_NKVS0O~GvB~FzPM6}=4d%fG4Nw4pZshcyLqK@`b8?RhD38haIyr@+8+0r5TC1*C7^WleJ zZN3_ngTD#RQvNL*;qD2H@cBWJbCC#d!}=oKfod5SE9a?!?j%DVt1z@inN}Iy$r+96 zM@P?AC+(`cM;z6J94BYGJ;+P-N#yj$?`G26ydS&OVH?~JY(N4l()Fh+x+DoJ@r<+i zhm^ck@QP`=fLApr62@KyOef~}zuG;(VbDQmw|Wb+oSHSw=%w9R)=et0cY*~ytX)#M zEXlK^p;zM@vTnXn+C1vwP)~TJv|TvDE2($;;EzC5_5IL#H;u z)#CO8)TSzbt8)wHB8$I8KcIojx&GoE)3QNu{CQ+_xBmQ&`mL5-u=BX(hs^hMY^ zae!!*Q;Tr$@(0~GoBJAohGw*d{l8~!aXop87aaSUb2jm)Tk>#$1*cdo5Sl+?oD!l4Og~yX+soottl4 zp4OartUuAN(dD~yLJ}`A1*!D4-|L^hM;`_DM^1KYs-VF(}h(BjRO``b+xV~%O=-)?p z7ciJH7Fnl?V&=ay_AB{oQoa2iR;6$^tiE|-eRCFy|3F@%j#6gUxkZX@?K`F$u#;T< z4IZORpUthmB?U`;zrOkp?P(Rvd5TFRWrBJmVg;KEZvJ+;Q}FRY%QZ?c^&$oPXW+C5 zdN#c>v%U?QuE+hMQdzxS1Q(BT90;29qu#^A?a^)Ui;{TJ;%`nLgm2ew$J4NvREjCJ z$`C7&?tH$CrVG@M3J1-KJw_*9BKeL*JX{ zN+Vg_TXb9^jJO$ZGkXO6BBFDjt~w5`w2TB*z$&1W5Il3IiDs=ZMDt|9iRtKET*wF6 z0Z+|N87p-5Fh)^(*l>OVr5^aY5LW(@PuM>Qo@&)yj6XRkPm1>eTF#Y_c*aRF^ZY5A z9FAU7lKEHG@i{wJMPg;n6z2|69d-)q9@<7t()d-zPy&X zdXG7{Uw{k23)CzzQAXw#iqj<1u~W@K_Ljc#?ukh;fRKHeJ2l~Z+52b2n^bGiDF2oX zm25FLx|4AP8>rAi@koY03lrtS#X?zK591c?2iZ_jjc>0y>q9>fU<08o6zG%z9WK+S zDwZMW4~28wu#ye#V*@#5t^S@NiAA`3{SF$xINmc_WW^u-C9M=H>RQ1>WM=|R!660{ z6E6%DwX`eu<3pkmz7Z=FCRd$(vhDkc3yMnSr)5C*aho)DZ<12$`$TXj<8Z70)|rK7 zXFD8QzksfWZU`qL2K8X{C~TcF{KVW`3Y{IMb&)T9%1V`tv(HY1 z+LXkLyM|3mtLD{x-#hOw-U?sr-iLeHFA|=-sGZ4#hX)atL!a91(tWJc+og&5W}VfZ zpgE7`{5D`~?yGR++y7~xA&eU0N*ZezDjF$> zUeK&1aTFQRg*?v^Z2e7u<`lk$czR6}b6Cl-qA9%A`#A6q0*zyTu)X`3rhjR86NK3= zLdw{+-F}+b2gxd-qF7>Rla}dFkj|L#c|pg5Ni+MRA|BZH(@ME*o<1ijKcoXb%PVfJ ztp_uf=G%kvU((pHcw90Xut=}atA!giM-5By)f40nKp zv7Wdb{;^<}VRvruH~rYr~wEuYY2ov-5Q|p@u3Da9+z7PeIpBAwi?RxnxN3Kt+N9L(LUS%wxY` z>e&1VV;{CYw8DNRlvBH)>!I49SU4R!t3I4=y;mCevPZh!-}~G+F>6hcL_Rli4r zC4(WN)`j$>^S=~GMGR=^)A6wrqi(-x{xK37&Vx!OS6t=KQ2JVZo#GrSODtTe=TVh%*qfF%91nqsMNLNL^Gp|_ zz%I*HUkMQGqb!1eh{{bp|0GSCDbkG_D_d)8<(0r<6-%Qi7qDa7xZjcdZ$?Rth9L!f z$erCcs3<~mtupywbaT8NWZF#v?iZkvqSz3@p`RiXs7P!GUa~-U9hEG(NgI#3BzO-# z!9JWf(;r!*A=@g$f}>wi|6Q@9z8AmYf~x8G%sp>C5cfuJY;hs1o3Ozu^{pH0AFbs%yU)Xy5>Cf?qXiHn*-PAfKDRiy`U0sFSKFsgEZ6_ z9#ma!<#Izr^}_z*>PRSt564u6We*XmZUx^jv*dK; z4zyFZ*ZFSE!00<6!|+#33&R)@RA8V9YRjp$HS9?CGq*xDSDRbX#i;}mateEF{fqTI zt?X}Efkq_Ap*_ETgaikOBbQ|;47}hwX44K`(DUI@C)QiG&6UJ1UmRn*Q@6%e`+x(gpQp74O{;yli8YLCV}qD z4gIyZd_(8ED~WWaeXOb0^r=9=AiDT}by~+$KVF~M{ywbQl zng-h?a_E;yX?DCr4|_h7JMc7>xgWf7Ek-VmH^hCYunVp3{(d{---&%-GZ=rK#V5Jo zJvP8b!2AA5?9)G8gwzB6ze3TU<5*Pqms^Q-?C9-CN~4hb-`U0D@kAkTWn23``cao^ z8IWAp8h7`%ZA+eI?w$sJktq5m>e&0@mQn>2BdpKAxbj1$m$8Z;`!iFvl9($Lb9Ff? zT^6cTZ~HgIeR6R*;G(rzpgsJP41Fx9Df;G6{;k6T(i}&8hX(jHSC@~#X@70h#)g(( z*9vUC+a*b%oAdf1$}Z3NR;|c5nY4^Z51pfqk(tmJbB;Q#ka#tf5eae;-kq$I{xO3<(TI$0lSe-JQzJ*es;il=Kn_?&?E zfLbs{qErPqm)-*ZfwbA*D-shgb|1;X;cH*yA|q8gS=HiosF=-kbdk6--SR+`F^H_` z0*i`J==@XSe=HT;_``G}ulE=H@*3GU*?gVd@h*`eT^GKjI;C@8+h~;(u3bA#b&bN{ zYw>dJ$(;RfHDLlndS`CWOE=g0jOocCc&;w(dOzrLf4-DK*MD@P_;u&CbfMw=#Q-B` zDq8hGwKN-O7(hQA_bP3f5XrZH+@*FGw~ppmDgNWcf|Lf*Pc%e5dw1DcJ1BWm!z7z3 zr^toEU*P(>G#;_1X}Rz(5lbDtCui%hY^d3lm)kw0vyk zX~K4$AG#7cG`6s2%9g9zsaQ9o?;3yzW4Pt!;NlS zzI#G7tiq&@eV&}qDtY(e$1JwscAfle%Al{3>Nr%``n?`Jac^CdOXUbFgI3;m{RkA~ zokl+lxuw9=%W&MmzA+G%ZdFMMP&N2^6BWjG2Lt|xKx)lMCR@b0n+xgw<)&Dwi?}>- z+$_e|@M;uW@3z6)q&L7bYitZ%huzGqH_qHOr&G5o!?(8TJv_MN1ka|&c6_!Q>#PgHSFoPWiLg|k_{ zQd#Zy&BPkU(0OE5S35!B5qb6%T3Wd#J(zBl8dw6I#xIDDF-LBPi-jXv1E?!gE|1OIdTejK)+U3ooC^otSIRsWZf-`&K}6}s!407Y58zH zK(oYx*7sN1O|Z_1YIJS_H$E@DH(hB4QKNCGQT3PTvwYoe2&8WKi5`5tU-r4!>_V3XUT}N)>8V;+z-!@-IGCKiD>E9RC(K`NMx=;Qp zf$2g^t?)zpU0L!BZi(oE#)^Z_biT*Svh>r#%1=O+Wo37G`Q)4@k#Pe?^mgBIugC)8 zyEICH=`{A~^x#X&%tr-$j|(nXrIrGQYNY+C3M+LO;yUU4-|v>a5#P)XYp>_|C0f0n{_p0mvwWmghfd%!Cm}$qBDxOqA3htLs~ghSA1>6^dVgd~ zVHHBBy6;Pp=El;dkTE=ttp~BoOJ$L@EB3Z37T1kTNG3tm4PY5O-7hP5DA$-k=vV&6 z?RiAm;W~*o)R7!x9>u$&@|&D4xMmJ*y+^-6t!F0u8G~78t&Bs#W>w_NbW>W9M3tXWXRf zI86FWVx%iXXh6MJ>dg#?lNu{K@S#nzMIG4PXQd%!Bvc*H0c7F_Y=adptJr*cHevMQ z%?Xu~q8CFw>^L*S_83kVhq=)hf0%_Lq}SE*g(Da_A{kXVZfAd*YCwp~bG32wi&SNM z#QZ7}Ug5-=+s^uqAh_|}gzya<(&E?XAZ%0ybd9nraj?|z1YfPr*{N?Q{ji}YG`T#| z=uwJZHIMlsmevnenT#-)t$L*=2wh|1EYXW?_36TR?L!sUItJVxaC0$Gb|gq4{|4gA z(v0ODFj!T)jc5>65ys)* z7$aBHfbKdz@QJq1b`NT`344*g()$>5*Ey`TPB7WI;|_8o8t9-_4ikFub|I{66>ge> zHA+6onzFKY*eaiA!77SD*^&LyumAR6gSvxY6Q?;!AvI{rZ##!G$%ZfIgce4F`aF;e z?jVh%+B-vj69ei~bh_zA9w}S4B4rzRKQ1~u$gwVu_x5PlRKDXX2(_2Mm7fs%6{SS7Qh1gWT8xaxc=f8`mW38ukIZxwU;lmHABwFSg50*o zrj%f%j~IKR?N5Dxwrq|sTa?!pd{b3sFM&~{4~_^YH4$bI^Fq2W4-y`))^|7fS?i0) zJ&Z9wY!8%l7@gAr`2{fqA;L;ptQR*X2|xUtrT47KK%XN+dydN$*M?65LuXTRabgERR{n>;E;(&vS0_@COY!p<%5LsRqGpER%~YjkSK zwBo9-2|-ZFiU3TT&S+@}3gDT35t0IXTzX@yHA(v>Y8;-mZNySQ&fE7RJ1^tzJfvdApX& z*!+tE)Y{oR%jk8A)3EiI3i*(TOwP!;B3hAOj?KQ6^h-q~1V^166uYS~mH*2Hh*0}r z`R3u1#^LG9IW|^QT^|61H(T1Jz?n;(Z>52lU0BO>Q6*zgpP*gTFk2Uw)!3zt>3F~_ ztil4!R*-j}wjh%&(kSB%}X=u4RbFRp@^l+$SmM@nW9B;yGbf@nasjFMEE{m9Oe

}qal5$moSACwfNXLXG5|3R0AtBcN` z?%yS)&>O>sqxU64U~C3&Q^>z-Zt}WuX4Wh3dKj9EO zfSbV!c3e;EOeKHQmWEw#NM4;*tw-2o@x&kKT?rsmy-F|$jw-F>WgA7?C@{O1qPg*J zf92|RTBMh&ptHADFc{T+cB?+mOj>h2HKgwkxq6w&XBxPc?>=JKvU2K9aU93@vp-R% z{5T=P$9U}AYZ5QU{3%7}YZ+ACWXw#-U zWyxU(OP#Q9-2AeGmCwcp`zWghf2hvsOjWjDQbU?U`v0&a--f1`v0Bd8HLiLmo)PKz5!A1|XVO+89 zm3h2~6yI~cpWor!_yt-?Lt>z`c0a7cJAW)#d8N8nNIf0H<+v;s4{0guDD(?T7Z<~$ zd`$vpZ_QQgFaMT0_d5&+(jwGU?M1FqUu6wjA-9z?mRM}(CmSdK;2e$Na}F-8jbhgN z9)@AIQeghf{xCC^{9P%VdYW1PP#}2BJwWt z0Hd8%st1NK5%h+)UB^mVwh{e#8TIm$xxgGo6I5;e{~VUeeMGRpM_Z%=eH5$X1}?Z5 z`|*_Vp~K&ziz45-Ih9y>EOr(Buy0&n$dbQ4$5eSr=Ti z#~7^n8dmem;$0D4+6eV7&G2D~d@ z+R#u8+nw_N%7_U_1e53P?~&10^m|ZUXrZhVp04lQLsGos%0fRDhS=@>8TOAAxK;Cy z9GZw_1pfSxD5~xoR!INI?tU0wrKDd6^Tv{jL>`Xb49kBaNPlhMaIfh_nq_)zB7NcX z05XeQKz`@BDUx7*i!V~%dc8XQ#ngBw0A2tSr(npSCrNy5Z7>48v&Zz?0{%FRElh_h zN2|?#EhJL5HQMIu6m1=ypTR?tVymHK)xQvS9ir7FzMp?CjlND39PK`od#GytVhZWp zQ1@>MTE1*Ip>hnXSWa?XbMH#708@j12yPbm`JfcqIgmJepn$5YgkJn_%5I)mr`Q(k z-a0yFR3A`houhvf&|wNpIsV{2p%MqhR@`@R(l6`}iufEgI*UxWq~26?WTpZCV{JtG zYL?&#I98fyf_;2S0?_V{=Aa4t^x%vy$pF$_Lh7W2f*~5uPvGYh;vZhMv|u+Z?2t0~ zcYPXdxbg6OS*LUjR_=jLDt)ab6;?g1IuySLG@UE;jLpt-wjLX&RlY>fnd@f&?0NyT zht5vhP^};k6`U76$%&I)iWPNxG6KPjdh`S6>g9GN@;KObQsLG zKyjfrPR0PU1B0a0=)3@9eCDl?mB9rFdlTMtTAeZv2}F*|@JWleq2+H1bt>>x!^wTk z+I)cgsZwzCMwoRpW_*!3IySTQu!`HWugAXe(Ai(a9Rsu;*0#o6torxwNMxPzEAjt` z>70Vw;HCQ?AnP`RKQ;2R8h%;LI#tx^(MO*lMWJe4_?)Q571P`kTmN#(ez21V!<6+S z@Uap+y%#8&cGgdf+E@y$dUx3g#)=#5k31Vqv0p!%L`*=-PiQAiSg-d9lKRZQDuJ-| zA96zwwomG+4}X$vR*IU=NC!vL<`rUTbf_uRJC4FS;k&HtV<=<)p(qymH)=MDV^aqK z#%sid7K|~!H`J!7hRr~Z!emxgWq6#GpQs%c#BM+scvNGz|Gi4G`;8Z~dP8)+51iB8 zw)0fazNz5(iK$LJeC_4e^8&@wT(DZ~~>SStz3P(>V8CLNlZqgv=2K-|Lu~si@XFwMN>QE^k zVS2U_A?Q$?M`NkU}^!M8m%O&T=kW>dG}1s2I~hxp9Y=a=1XX-(fB5) zej3`e5Et~R^r%?CZK0)UZsF_+tSOGIBMdrtMf#oJjGF9U`*P8t>i*TWed$Z2WNUZ* z_1Qw4Yr+Q0@bD?hD0P-^v}?FpPBg~zz5~g@J#J76C695|P>1l;OS8%~hZh5&-9Ji# z50%&56ZK4FC9}{jHL0!=qo9Yd(GGHCEX2|-F(f}q6@NMT4P3rQd{Q!=bz-8N(Z^!N;;ZzAWRf@C?X>mG=_NgyQX_?Jv$m(9$W>P;+e}O|&w&DjbsJPdWp0A2$yLr*!BY73Z z5d*BCaTI)w=sTlofc>n}@v_tSXIK?8(g`G_06u>SD*fOZJ~visq3lBVS2+cf-r$UQ zZ(8A0g&5M$IV7w5nqL(m$VS0X?=yy-e6>S>Ca3wZNT)b{GF39_gJdONflqc-j$b~o z2l@@h{$KVfC)V?#We*)@xYC;L^<@cHo>8axRMbSzw|eYTl|8pkabsQJ(3`z{>5H}c z`psz_Y6t)hvzL^=}P#++XUl6v`-j)SuXd6BynjNZ!&c2hnyE&4*K$nXn31Zk)cm+lx;> zya{T?{MRtSu?^3Y9bS&O$*mW^vRUpv!J3Tz12?3&Y62b_oiZ$24O(75Z)JWb+Rj)ACbK`f<&tSwtT$|Sy z$41kRPiM-jnPY9PKrLyI`pHm6LusMsrO*HpmE){Kp1^u2t%6nW^;GB|!4k!Ik8oav zjM?DBKh9G@W0gEwiU-M}0B)}olvoM71RccgiZBCs)L?q_GX&JDhegx4k2&cNatr5w zU)1#2USb8&`etO5Vk z?0}K+*2*@a5yt*X{qg0@8jEz~jcylVj>-042p1PBnabI#xUiCRD!ouw3?u-wwsqwF z8(@m8-Lk7q@v154g6yvx_tRDa>}oqpVda)wfI9(;ZVGt1v^{<|X?vC_(i@IJC+2I_lusrT=$h zF1lPc*Neb`;Xgrdf`p$w)~MzQW0M3_FYRKu{2$VU82J^B=X1#^<&P$_`=S$Ey04WU zTxG;hrFNLhWC*p+sH3x=JVcBJ9*7>eO20)n671SxQhZQlHMRP8FyO}yai~OTsbms0 zQ3b$C1Cn!>jMHDq{VX1ab^~_Q!z+f75+_AuwiN0*wA_#M#0|rU{+NlB%>Y+TNT0Gj z`3^LKMSJjz2(?lwg~ixDl_5%rzzZ}o_6Fj9e)T7gpH4=BgT1zmwJpC@g(f%&0`}8B z%7Y&qlP3aFmI#nmT`|R3+Lwzp+PLXt|5g%vlY_$fvse7zjus0D0fA##r+i4G4K-2Y zC#H95NGoYfWP#ZF_v$^Li{PZpm}fc&)aL?5doPcb835Cr6`T+EzzcEvLtmXcbAb<^ zw!_Zgk6Az7YA@*vb)(G{_W-B|zrf76z^`X%jOgqIIaqi~5nUup3vugzzg&rA^w(zR z+qCzvIV~nGR=47pDOcNTzuBw#5a=<=DMvGa)g zPw$^pmq9Fg&b#BZrPSoml(149rZS!fioV*Dy$z440U3MXDJmI?RZqLy0}IKSxN)o( z8+8wIZs#q(|KTg6y;Z(=96>xfpUsr@SP}I^v zN^R;ZVrDaWmNrM5-<X@k6JyjvA3;jHhma|Y|7!Vk& zgf(UK_6~cC;!|b!YTjke=nBiUqQdb#I9TY}!s5P)H+^c;9cW(QO8O%n5J^8Xfktd*qrn)+?-gP`m%B&q zi^}7jKm`yMW8ITFOMN#!QIB6$SWx*75tnCMaNg*_J*WuwBh~AT>0($nS8%&zmFQDp z$dL65niDtTV%!Kg1`6epWoQGNG`$`doy;Zjaa`keyL0F6iJMae6FIgnhAfzU%m@V+ zm5rQihLwS~b6{-bVR1ZSzBI7(Yj+V6T-8V*7I`ptWArGdy~8pnV>fALpi~NQLZ7;^ zpaj35=md<~-(tNmF69UX3?ua}A7UIn)q5i1iPYEGlhYSbkfeX`5epkxtzk3Qbu| zlgA`7ts%IvF4HJ}-98akyRnjCo{u-`A4&b+r?s|o`4wdYAHs-yh91p$7C_|+EdYH5 z10`!*=n+W9g>V&dfU1H!J}ASZi&-?`2IlDOAHnu306rD`y>jT)4^@S(X4XhN2{g9i zj-ym98+RT|d0ejIFJCM5>S{mT-8uGmRRqkJ3sMO_AQDrv77Q zv$t>zaVpVF6eBguE%9M2u?E-Oleft8z5+~W`G}KXD(Yc;7m4{Op>Le(k`g1UK7(1# zt6g}$n=Tdn{T4pu>v!c;xRCd_WI$Ali13x=U_0T!Ga-U~9W88q-lU+RLn2`N8Ouho z^0@SvC>$DguHWx)?^*ms-{PVq%dn(U3vrLj9zITDqQZ`H>Wsp@Gf%}SG=m)Vh}F$ztQAbwVGdDgd!28j&yX9wLW&s! zNR~6`nYg;ULAq8zi<;gUchAV5ib67Y##l2 zy+%gaD(|~G4@||{A;TYDSoS>q2o{t23t-^!NDSDEm8j3ao7Ei>KYLEpb$jz}7ciAM zD}trDN+AVVT_lXW<++~>8>Cj8fzJo@R;>%nGq)6+w?(#mNc#1J4W+!hA}?g$0Xqo? zn67qJmss)e%k(xO*&K@z6+}nHA(lCkb6n-|{pSztys$8HiOWTVR)tCO*Q9~if%3n7`uxGzE+OCu zwcVV|tgQdq60952$>85-GHk$lwM(uI+CU1?i{sVnKd0+UNq#eSSKjUKfDDgLnBG1y z^v?f#MRFkph~TgkoKBvM`L_~we8__xpLcjh`GwV|87q`vazJq?SX=mXhdvK>VqUf~ z4sYoTIpt5S)KrE-?>&=cRoBumD7;b5pq!Y07)#I$`)<@U+mo*dE*P~773p*u^6waO z2#thJahX_ySlYMpjx%h<)i43ao~Is`^Ya zMNZkuChEA7+ZJe6$>-C*dzTYf3#1SY82yFG?S&Q)5rTbKS-XLjckTLEc7>^sFcntQ zBeNXCSg&q1N3Bi^4zlQ%mcEBQ%2ab$?(;t-$HYd2%cnX$uuwU#I_6D3($m zR(>gHzM9ODf;r8b0l5LuEIQVZiQ0-|3Y_xzJkZc*CD=bPJ+&J+>>se%D4uTq?Ny{l z0Z5~og*Wa1O&anlcRWu_%o)(x?IZ0CfUNk_R-ik>GyvdFmpu1wHZaKTDGhL zqxsji)n<+)VKbV0_BRq9E;Kb`f=&vn(BK0Ba-gL?ZN;^^b3YFg6R=!q#zM;tcX0dM zdy5PPx@6pJPXHzH7$dGjM|6@6777nXPWV;CIQdNf(*Znv)sMy&Xcq> zhCq+6h6&v8<0}vd2(sKqU3j>fr7&#Xy%qZHcMU3m{wld^Nstkz8GagB?Y=SI&H z&{&BSA-|(i35$9(l6LpFyLm$0M0fK`Dz!~ezL?yEInsXAFR!bHe;ZL>Gd(#Hv?<$%`^b)oi?x%(jkylCPb=juPlF znMo&o961=NZ_$gd{xp1ZY2dNDOS!=XVj!M^A z+$z`EK4v=m{Bs{&I4W)({`&<5*^BV#z{IBAI_d+9Qx;~ zby?2zEjzUUeZWBDo5cz>%;z||z)<+6UtC)y60yD5J5`oo_zSM;l21@CY<0_|)NME5 zs)kHCMBa5YzB#N=W2aR?y9((~WuYwwf+HAc2mvU>NYlxOTvGf^Ye3za?*f-qUs^`a zT3>RPh9*Jf%3*bf|kqtnD_Buxv!<9N>BbuD#uYv-q^ z%RDnd7a3O4M9Y~TNISS@9K}JDkdg@>x8E6@n8jF=6qiDV+}{!V)(o?ykcr0sxBGEx zo!X;pc=r{H^vw6ztV5VZXBa4~(ujB$rZQ|AaGN@J7#q%2nU9gJ)g6dcj}zYB1& z@iFE0vMQVxa|v7tDHS$gwX$Ihc#M^DXRC>J@Zk?dC(3uB_s~*W&m-01DFMQGWjj5x z5po1@1gPl!v1Yra@qPG{D;$bYLM3qOwpl~7f~l)#n< zP+6`!NYe3EE~4RFR#_e=7YctPRBt6$He@`%e5m}f$M%yzC2S0<1}hRPjO>HJY~ z*dx(nbMbjv*;o&k{qzBdF|lS;UNVKziV=gbLq}UOCwr8GT5E9oRYQ}+>DhbQ1R=lj zgcNJN8|D)$Mx3#c+t@lhqcDUnHGVt0&EyQ{b5)=52B(VTzw=pQ^ba3`JB@BU^lS`_ zJEiLzgU#Acd_!}FMxCWC**FP^i#P}bYzNs78)#uSejEtYLbG>JJ7Igtho2oKQ;XW~ z4eMGO+t!_;G^V6c&R`5Tg+Pz2ToN(aybq4Q0ssie_{`t*DO%V7FaZ`{MBobFc9|pV z70o5ayHGJo9$$&Pgbs)pWNzduAcbh?~U?_P)(ve0S*3H%eNF&a5XR=!J#4c z;t992n7ZJr{*%`^dU1d-ALE8!3i#v;3r4r%j+JFCe=%3Vj=8{aXe zs)jrcUBZ=;LudcTUXj2ub>K5!{HHFHJ}Trx(PYugbQ8yK7&sqX;(;|UWjk3tGs3zuceeX)i4i_jA8Qz2Bc%DxN8 zXw!$+9jBtEHd1y90bYG4f8DcJM)Ab!M39tH5zz94*MAvnhA377@buNupSOUU3j8~> zd6&hk^ENRCp9T?_QUHk<=(&9Q^MJ^pi;nKOYNR@?L=RCSmKMJ5UQJQ`X!i~(gD*P! zs`RobzJG3Ra_Pg+WZUXUmMU$ilpwfcEti6)mw(~MZ0q!^sza>#jv!-+7B6F3QuMWg zVO!rXwD+lF1BBTito?ml-CV3vxuek~TKuOX^N6sol$v*{_%nAuD7i81eXm^Lz(Z~I z2Xj_Dts#G0&C;PV_Wkq*1QvB7+Post4={v;gk7b9u%#DC_bh(iJm$rqog^{JEx6NE zrs5^2SEL$|98#2WV#iG@L6cq|)SuTMSfGocPl65wUd^|5Lbpnb(;t>-Qu2jvANLgv zdte0vED-3C@^BdyHWLL(7{G$WA02z@JG!T-U^Q7HZ(7Bs&vchkh(p&}KvnS{MG^i6 z4r){gJp9p7WyWOEiKA2Cm6EXIn&&gk|Fc6^78OpPrX4ExCFE=SD$xcH;C2eB^{XTI zaxz_Cef*Yj==w_i_BTGXP;8C&f? z*QEM>={jFM8)lWAR870pG4XEWsl%%K|82S5b=9hVz7p_6i-d(Iyvq76&a#PV zR;VbQV|n?mg}&(ehClg%tK%IjgtnTR-u)lxH06XxXqH0soAZbB_Rm)XX=6Nge1uoG7 z9vQM_S~2h53n|W`y{{R9+=08rv~MohI_v4-BU^7fZ0-A}#b5{AOSTJm+(J;9yw%pD zX6u62GJ&@HKX5zQwq~j8T!Hrv-Mk^QSB5cu09L03{ToDO7jikM0WAcsjW>D}^jqCF zT0DEZ@K^KO_MD*%M!+V)lGVU6?LpX)eQVXEmq}R`NIJv;kBitJ!nW?0OxTVlu2ADf zE{A!*0g3%nwVcBD+AgT5bGx@WOnQk{zRpiZ4HhP`3BF%N|HdqPbbiV5)7x)kzC3ID zZ;27>0^mrMgWc7evsbQY`l`l})wr+e;=8U_!2&B77;1qL!N8y)eTJ2lf#CvhR~!Qa mc;sM|90DP5A*JW%f2r=u1xt!e4gwD_V(@hJb6Mw<&;$SznOm^{ literal 0 HcmV?d00001 diff --git a/test/packages/bad_routing_rules/manifest.yml b/test/packages/bad_routing_rules/manifest.yml new file mode 100644 index 000000000..a1696fe21 --- /dev/null +++ b/test/packages/bad_routing_rules/manifest.yml @@ -0,0 +1,33 @@ +format_version: 2.9.0 +name: bad_routing_rules +title: "Bad Routing Rules" +version: 0.0.1 +source: + license: "Elastic-2.0" +description: "This package contains routing rules with errors" +type: integration +categories: + - custom +conditions: + kibana.version: "^8.7.1" + elastic.subscription: "basic" +screenshots: + - src: /img/sample-screenshot.png + title: Sample screenshot + size: 600x600 + type: image/png +icons: + - src: /img/sample-logo.svg + title: Sample logo + size: 32x32 + type: image/svg+xml +policy_templates: + - name: sample + title: Sample logs + description: Collect sample logs + inputs: + - type: logfile + title: Collect sample logs from instances + description: Collecting sample logs +owner: + github: elastic/integrations diff --git a/test/packages/good_v2/data_stream/routing_rules/manifest.yml b/test/packages/good_v2/data_stream/routing_rules/manifest.yml index 0e561e2e9..ca071e4b9 100644 --- a/test/packages/good_v2/data_stream/routing_rules/manifest.yml +++ b/test/packages/good_v2/data_stream/routing_rules/manifest.yml @@ -1,5 +1,6 @@ title: "This data stream contains routing rules definitions" type: logs +dataset: routing_rules streams: - input: logfile title: Sample logs @@ -11,3 +12,38 @@ streams: multi: true default: - /var/log/*.log + +routing_rules: + # "Local" routing rules are included under this current dataset, not a special case + - dataset: routing_rules + rules: # Route error logs to `nginx.error` when they're sourced from an error logfile + - dataset: nginx.error + if: "ctx?.file?.path?.contains('/var/log/nginx/error')" + namespace: + - "{{labels.data_stream.namespace}}" + - default + + # Route access logs to `nginx.access` when they're sourced from an access logfile + - dataset: nginx.access + if: "ctx?.file?.path?.contains('/var/log/nginx/access')" + namespace: + - "{{labels.data_stream.namespace}}" + - default + + # Route K8's container logs to this catch-all dataset for further routing + - dataset: k8s.router + rules: + - dataset: routing_rules + if: "ctx?.container?.image?.name == 'routing_rules'" + namespace: + - "{{labels.data_stream.namespace}}" + - default + + # Route syslog entries tagged with nginx to this catch-all dataset + - dataset: syslog + rules: + - dataset: routing_rules + if: "ctx?.tags?.contains('routing_rules')" + namespace: + - "{{labels.data_stream.namespace}}" + - default diff --git a/test/packages/good_v2/manifest.yml b/test/packages/good_v2/manifest.yml index 22275938d..b3a5a72db 100644 --- a/test/packages/good_v2/manifest.yml +++ b/test/packages/good_v2/manifest.yml @@ -1,4 +1,4 @@ -format_version: 2.8.0 +format_version: 2.9.0 name: good_v2 title: Good package description: This package is good for format version 2 From 921b9e845f5e8466af24bafb2c921874d9419863 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 13 Jun 2023 12:51:46 +0200 Subject: [PATCH 04/27] update test packages --- .../data_stream/rules/manifest.yml | 14 +++++++------- .../good_v2/data_stream/routing_rules/manifest.yml | 14 +++++++------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/test/packages/bad_routing_rules/data_stream/rules/manifest.yml b/test/packages/bad_routing_rules/data_stream/rules/manifest.yml index ae8de2599..dec213891 100644 --- a/test/packages/bad_routing_rules/data_stream/rules/manifest.yml +++ b/test/packages/bad_routing_rules/data_stream/rules/manifest.yml @@ -14,16 +14,16 @@ streams: routing_rules: # "Local" routing rules are included under this current dataset, not a special case - dataset: routing_rules - rules: # Route error logs to `nginx.error` when they're sourced from an error logfile - - dataset: nginx.error - if: "ctx?.file?.path?.contains('/var/log/nginx/error')" + rules: # Route error logs to `routing_rules.error` when they're sourced from an error logfile + - dataset: routing_rules.error + if: "ctx?.file?.path?.contains('/var/log/routing_rules/error')" namespace: - "{{labels.data_stream.namespace}}" - default - # Route access logs to `nginx.access` when they're sourced from an access logfile - - dataset: nginx.access - if: "ctx?.file?.path?.contains('/var/log/nginx/access')" + # Route access logs to `routing_rules.access` when they're sourced from an access logfile + - dataset: routing_rules.access + if: "ctx?.file?.path?.contains('/var/log/routing_rules/access')" namespace: - "{{labels.data_stream.namespace}}" - default @@ -37,7 +37,7 @@ routing_rules: - "{{labels.data_stream.namespace}}" - default - # Route syslog entries tagged with nginx to this catch-all dataset + # Route syslog entries tagged with routing_rules to this catch-all dataset - dataset: syslog rules: - dataset: routing_rules diff --git a/test/packages/good_v2/data_stream/routing_rules/manifest.yml b/test/packages/good_v2/data_stream/routing_rules/manifest.yml index ca071e4b9..adb88a0e8 100644 --- a/test/packages/good_v2/data_stream/routing_rules/manifest.yml +++ b/test/packages/good_v2/data_stream/routing_rules/manifest.yml @@ -16,16 +16,16 @@ streams: routing_rules: # "Local" routing rules are included under this current dataset, not a special case - dataset: routing_rules - rules: # Route error logs to `nginx.error` when they're sourced from an error logfile - - dataset: nginx.error - if: "ctx?.file?.path?.contains('/var/log/nginx/error')" + rules: # Route error logs to `routing_rules.error` when they're sourced from an error logfile + - dataset: routing_rules.error + if: "ctx?.file?.path?.contains('/var/log/routing_rules/error')" namespace: - "{{labels.data_stream.namespace}}" - default - # Route access logs to `nginx.access` when they're sourced from an access logfile - - dataset: nginx.access - if: "ctx?.file?.path?.contains('/var/log/nginx/access')" + # Route access logs to `routing_rules.access` when they're sourced from an access logfile + - dataset: routing_rules.access + if: "ctx?.file?.path?.contains('/var/log/routing_rules/access')" namespace: - "{{labels.data_stream.namespace}}" - default @@ -39,7 +39,7 @@ routing_rules: - "{{labels.data_stream.namespace}}" - default - # Route syslog entries tagged with nginx to this catch-all dataset + # Route syslog entries tagged with routing_rules to this catch-all dataset - dataset: syslog rules: - dataset: routing_rules From a2dc265485a14a77ab121d71095ce5facaafd4f4 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 13 Jun 2023 12:52:06 +0200 Subject: [PATCH 05/27] Mark feature as technical preview --- spec/changelog.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/changelog.yml b/spec/changelog.yml index 1c4053ed8..02db632e7 100644 --- a/spec/changelog.yml +++ b/spec/changelog.yml @@ -10,7 +10,7 @@ - description: Extend benchmarks spec to add rally scenario type: enhancement link: https://github.com/elastic/package-spec/pull/524 - - description: Add support to define routing rules in data streams + - description: Add support to define routing rules in data streams (technical preview) type: enhancement link: https://github.com/elastic/package-spec/pull/535 - version: 2.8.0 From 76f5a021c1e82b0f4dd68d943f0b7c1ab73f53d7 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 13 Jun 2023 15:59:39 +0200 Subject: [PATCH 06/27] Add comment --- spec/integration/data_stream/manifest.spec.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/integration/data_stream/manifest.spec.yml b/spec/integration/data_stream/manifest.spec.yml index ca271ce86..fa857c8a6 100644 --- a/spec/integration/data_stream/manifest.spec.yml +++ b/spec/integration/data_stream/manifest.spec.yml @@ -332,7 +332,7 @@ spec: dynamic_namespace: description: When set to true, agents running this integration are granted data stream privileges for all namespaces of its type type: boolean - routing_rules: + routing_rules: # technical preview description: Routing rules set. type: array items: From 4c8a7e2425320d0ff8da3457f363e5229cc1c890 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 13 Jun 2023 17:13:15 +0200 Subject: [PATCH 07/27] Remove unnecessary file definition --- spec/integration/data_stream/spec.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/spec/integration/data_stream/spec.yml b/spec/integration/data_stream/spec.yml index 1ff45a663..3d8078397 100644 --- a/spec/integration/data_stream/spec.yml +++ b/spec/integration/data_stream/spec.yml @@ -78,10 +78,3 @@ spec: required: false visibility: private $ref: "./_dev/spec.yml" - # - description: Routing rules definitions file - # type: file - # contentMediaType: "application/x-yaml" - # sizeLimit: 5MB - # name: "routing_rules.yml" - # required: false - # $ref: "./routing-rules.spec.yml" From 558772347407d431b47630eb1a10666fe3bb0b32 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 14 Jun 2023 12:32:08 +0200 Subject: [PATCH 08/27] Rename --- spec/integration/data_stream/manifest.spec.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spec/integration/data_stream/manifest.spec.yml b/spec/integration/data_stream/manifest.spec.yml index fa857c8a6..fc3a4e7e6 100644 --- a/spec/integration/data_stream/manifest.spec.yml +++ b/spec/integration/data_stream/manifest.spec.yml @@ -169,7 +169,7 @@ spec: hidden: description: Makes the data stream hidden type: boolean - route: + routing_rule: description: Routing rule definition type: object properties: @@ -344,7 +344,7 @@ spec: rules: type: array items: - $ref: "#/definitions/route" + $ref: "#/definitions/routing_rule" required: - dataset - rules From 88cb7cb76efd03b0d83a87a452056954d40834f3 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 14 Jun 2023 12:38:31 +0200 Subject: [PATCH 09/27] Add description --- spec/integration/data_stream/manifest.spec.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/spec/integration/data_stream/manifest.spec.yml b/spec/integration/data_stream/manifest.spec.yml index fc3a4e7e6..2d0501609 100644 --- a/spec/integration/data_stream/manifest.spec.yml +++ b/spec/integration/data_stream/manifest.spec.yml @@ -174,16 +174,32 @@ spec: type: object properties: dataset: + description: > + Field references or a static value for the dataset part of the data stream name. + In addition to the criteria for index names, cannot contain - and must be no longer than 100 characters. + Example values are nginx.access and nginx.error. + + Supports field references with a mustache-like syntax (denoted as {{double}} or {{{triple}}} curly braces). + When resolving field references, the processor replaces invalid characters with _. + Uses the part of the index name as a fallback if all field references resolve to a null, missing, or non-string value type: string examples: - nginx.error - nginx if: + description: Conditionally execute the processor type: string examples: - "ctx?.file?.path?.contains('/var/log/nginx/error')" - "ctx?.container?.image?.name == 'nginx'" namespace: + description: > + Field references or a static value for the namespace part of the data stream name. + See the criteria for index names for allowed characters. Must be no longer than 100 characters. + + Supports field references with a mustache-like syntax (denoted as {{double}} or {{{triple}}} curly braces). + When resolving field references, the processor replaces invalid characters with _. + Uses the part of the index name as a fallback if all field references resolve to a null, missing, or non-string value. type: array items: type: string From fae5b7a8edc6fabf9e61c32bbe40e8bb546c1696 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 14 Jun 2023 12:41:38 +0200 Subject: [PATCH 10/27] Add string and array for dataset and namespace --- spec/integration/data_stream/manifest.spec.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/spec/integration/data_stream/manifest.spec.yml b/spec/integration/data_stream/manifest.spec.yml index 2d0501609..8e6f8ac6f 100644 --- a/spec/integration/data_stream/manifest.spec.yml +++ b/spec/integration/data_stream/manifest.spec.yml @@ -182,7 +182,11 @@ spec: Supports field references with a mustache-like syntax (denoted as {{double}} or {{{triple}}} curly braces). When resolving field references, the processor replaces invalid characters with _. Uses the part of the index name as a fallback if all field references resolve to a null, missing, or non-string value - type: string + anyOf: + - type: string + - type: array + items: + type: string examples: - nginx.error - nginx @@ -200,7 +204,11 @@ spec: Supports field references with a mustache-like syntax (denoted as {{double}} or {{{triple}}} curly braces). When resolving field references, the processor replaces invalid characters with _. Uses the part of the index name as a fallback if all field references resolve to a null, missing, or non-string value. - type: array + anyOf: + - type: string + - type: array + items: + type: string items: type: string examples: From e1c83a531a5826737e6d94a1bd2c0159bb17a07f Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 15 Jun 2023 18:55:07 +0200 Subject: [PATCH 11/27] Move routing_rules as a new file in datastream --- .../validate_routing_rules_and_dataset.go | 81 +++++++++++++++++++ code/go/internal/validator/spec.go | 1 + code/go/pkg/validator/validator_test.go | 34 ++++++-- .../data_stream/routing_rules.spec.yml | 76 +++++++++++++++++ spec/integration/data_stream/spec.yml | 14 ++++ .../data_stream/rules/manifest.yml | 34 -------- .../data_stream/rules/routing_rules.yml | 35 ++++++++ .../data_stream/routing_rules/manifest.yml | 35 -------- .../routing_rules/routing_rules.yml | 35 ++++++++ 9 files changed, 270 insertions(+), 75 deletions(-) create mode 100644 code/go/internal/validator/semantic/validate_routing_rules_and_dataset.go create mode 100644 spec/integration/data_stream/routing_rules.spec.yml create mode 100644 test/packages/bad_routing_rules/data_stream/rules/routing_rules.yml create mode 100644 test/packages/good_v2/data_stream/routing_rules/routing_rules.yml diff --git a/code/go/internal/validator/semantic/validate_routing_rules_and_dataset.go b/code/go/internal/validator/semantic/validate_routing_rules_and_dataset.go new file mode 100644 index 000000000..f40c9ea48 --- /dev/null +++ b/code/go/internal/validator/semantic/validate_routing_rules_and_dataset.go @@ -0,0 +1,81 @@ +package semantic + +import ( + "fmt" + "io/fs" + "path" + + "gopkg.in/yaml.v3" + + ve "github.com/elastic/package-spec/v2/code/go/internal/errors" + "github.com/elastic/package-spec/v2/code/go/internal/fspath" + "github.com/elastic/package-spec/v2/code/go/internal/pkgpath" +) + +// ValidateRoutingRulesAndDataset returns validation errors if there are routing rules defined in any dataStream +// but that dataStream does not defines "dataset" field. +func ValidateRoutingRulesAndDataset(fsys fspath.FS) ve.ValidationErrors { + dataStreams, err := listDataStreams(fsys) + if err != nil { + return ve.ValidationErrors{err} + } + + var errs ve.ValidationErrors + for _, dataStream := range dataStreams { + anyRoutingRules, err := anyRoutingRulesInDataStream(fsys, dataStream) + if !anyRoutingRules { + continue + } + err = validateDatasetInDataStream(fsys, dataStream) + if err != nil { + errs.Append(ve.ValidationErrors{fmt.Errorf("routing rules defined in data stream %q but dataset field is missing: %w", dataStream, err)}) + } + } + return errs +} + +func validateDatasetInDataStream(fsys fspath.FS, dataStream string) error { + manifestPath := path.Join("data_stream", dataStream, "manifest.yml") + d, err := fs.ReadFile(fsys, manifestPath) + if err != nil { + return fmt.Errorf("failed to read data stream manifest in %q: %w", fsys.Path(manifestPath), err) + } + + var manifest struct { + Dataset string `yaml:"dataset,omitempty"` + } + err = yaml.Unmarshal(d, &manifest) + if err != nil { + return fmt.Errorf("failed to parse data stream manifest in %q: %w", fsys.Path(manifestPath), err) + } + + if manifest.Dataset == "" { + return fmt.Errorf("dataset field is required in data stream %q", dataStream) + } + return nil +} + +func anyRoutingRulesInDataStream(fsys fspath.FS, dataStream string) (bool, error) { + routingRulesPath := path.Join("data_stream", dataStream, "routing_rules.yml") + f, err := pkgpath.Files(fsys, routingRulesPath) + if err != nil { + return false, nil + } + if len(f) != 1 { + return false, fmt.Errorf("single routing rules expected") + } + + vals, err := f[0].Values("$.routing_rules") + if err != nil { + return false, fmt.Errorf("can't read routing_rules: %w", err) + } + + rules, ok := vals.([]interface{}) + if !ok { + return false, fmt.Errorf("routing rules conversion error") + } + if len(rules) > 0 { + return true, nil + } + return false, nil +} diff --git a/code/go/internal/validator/spec.go b/code/go/internal/validator/spec.go index f46124224..e759637f4 100644 --- a/code/go/internal/validator/spec.go +++ b/code/go/internal/validator/spec.go @@ -131,6 +131,7 @@ func (s Spec) rules(pkgType string, rootSpec spectypes.ItemSpec) validationRules {fn: semantic.ValidateILMPolicyPresent, since: semver.MustParse("2.0.0"), types: []string{"integration"}}, {fn: semantic.ValidateProfilingNonGA, types: []string{"integration"}}, {fn: semantic.ValidateKibanaObjectIDs, types: []string{"integration"}}, + {fn: semantic.ValidateRoutingRulesAndDataset, types: []string{"integration"}, since: semver.MustParse("2.9.0")}, } var validationRules validationRules diff --git a/code/go/pkg/validator/validator_test.go b/code/go/pkg/validator/validator_test.go index db2606701..4e4fd8581 100644 --- a/code/go/pkg/validator/validator_test.go +++ b/code/go/pkg/validator/validator_test.go @@ -160,12 +160,6 @@ func TestValidateFile(t *testing.T) { `field 3: Must not be present`, }, }, - "bad_routing_rules": { - "data_stream/rules/manifest.yml", - []string{ - `field (root): dataset is required`, - }, - }, } for pkgName, test := range tests { @@ -578,6 +572,34 @@ func TestValidateExternalFieldsWithoutDevFolder(t *testing.T) { } }) } +} + +func TestValidateRoutingRules(t *testing.T) { + tests := map[string][]string{ + "good_v2": []string{}, + "bad_routing_rules": []string{ + `routing rules defined in data stream "rules" but dataset field is missing: dataset field is required in data stream "rules"`, + }, + } + + for pkgName, expectedErrorMessages := range tests { + t.Run(pkgName, func(t *testing.T) { + err := ValidateFromPath(path.Join("..", "..", "..", "..", "test", "packages", pkgName)) + if len(expectedErrorMessages) == 0 { + assert.NoError(t, err) + return + } + assert.Error(t, err) + + errs, ok := err.(errors.ValidationErrors) + require.True(t, ok) + assert.Len(t, errs, len(expectedErrorMessages)) + + for _, foundError := range errs { + require.Contains(t, expectedErrorMessages, foundError.Error()) + } + }) + } } diff --git a/spec/integration/data_stream/routing_rules.spec.yml b/spec/integration/data_stream/routing_rules.spec.yml new file mode 100644 index 000000000..42705fe05 --- /dev/null +++ b/spec/integration/data_stream/routing_rules.spec.yml @@ -0,0 +1,76 @@ +## +## Describes the specification for a routing rules yml file +## +spec: + # Everything under here follows JSON schema (https://json-schema.org/), written as YAML for readability + type: object + additionalProperties: false + definitions: + routing_rule: + description: Routing rule definition + type: object + properties: + dataset: + description: > + Field references or a static value for the dataset part of the data stream name. + In addition to the criteria for index names, cannot contain - and must be no longer than 100 characters. + Example values are nginx.access and nginx.error. + + Supports field references with a mustache-like syntax (denoted as {{double}} or {{{triple}}} curly braces). + When resolving field references, the processor replaces invalid characters with _. + Uses the part of the index name as a fallback if all field references resolve to a null, missing, or non-string value + anyOf: + - type: string + - type: array + items: + type: string + examples: + - nginx.error + - nginx + if: + description: Conditionally execute the processor + type: string + examples: + - "ctx?.file?.path?.contains('/var/log/nginx/error')" + - "ctx?.container?.image?.name == 'nginx'" + namespace: + description: > + Field references or a static value for the namespace part of the data stream name. + See the criteria for index names for allowed characters. Must be no longer than 100 characters. + + Supports field references with a mustache-like syntax (denoted as {{double}} or {{{triple}}} curly braces). + When resolving field references, the processor replaces invalid characters with _. + Uses the part of the index name as a fallback if all field references resolve to a null, missing, or non-string value. + anyOf: + - type: string + - type: array + items: + type: string + items: + type: string + examples: + - default + - "{{ labels.dasta_stream.namespace}}" + required: + - dataset + - if + - namespace + properties: + routing_rules: # technical preview + description: Routing rules set. + type: array + items: + type: object + additionalProperties: false + properties: + dataset: + type: string + rules: + type: array + items: + $ref: "#/definitions/routing_rule" + required: + - dataset + - rules + required: + - routing_rules diff --git a/spec/integration/data_stream/spec.yml b/spec/integration/data_stream/spec.yml index 3d8078397..1c296b329 100644 --- a/spec/integration/data_stream/spec.yml +++ b/spec/integration/data_stream/spec.yml @@ -78,3 +78,17 @@ spec: required: false visibility: private $ref: "./_dev/spec.yml" + - description: File containing routing rules definitions + type: file + contentMediaType: "application/x-yaml" + sizeLimit: 5MB + name: "routing_rules.yml" + required: false + $ref: "./routing_rules.spec.yml" + +# TODO add JSON patch to remove routing_rules.yml entry +# versions: +# - before: 2.9.0 +# patch: +# - op: remove +# path: "/contents/0/contents/7" diff --git a/test/packages/bad_routing_rules/data_stream/rules/manifest.yml b/test/packages/bad_routing_rules/data_stream/rules/manifest.yml index dec213891..6e433eb1a 100644 --- a/test/packages/bad_routing_rules/data_stream/rules/manifest.yml +++ b/test/packages/bad_routing_rules/data_stream/rules/manifest.yml @@ -11,37 +11,3 @@ streams: multi: true default: - /var/log/*.log -routing_rules: - # "Local" routing rules are included under this current dataset, not a special case - - dataset: routing_rules - rules: # Route error logs to `routing_rules.error` when they're sourced from an error logfile - - dataset: routing_rules.error - if: "ctx?.file?.path?.contains('/var/log/routing_rules/error')" - namespace: - - "{{labels.data_stream.namespace}}" - - default - - # Route access logs to `routing_rules.access` when they're sourced from an access logfile - - dataset: routing_rules.access - if: "ctx?.file?.path?.contains('/var/log/routing_rules/access')" - namespace: - - "{{labels.data_stream.namespace}}" - - default - - # Route K8's container logs to this catch-all dataset for further routing - - dataset: k8s.router - rules: - - dataset: routing_rules - if: "ctx?.container?.image?.name == 'routing_rules'" - namespace: - - "{{labels.data_stream.namespace}}" - - default - - # Route syslog entries tagged with routing_rules to this catch-all dataset - - dataset: syslog - rules: - - dataset: routing_rules - if: "ctx?.tags?.contains('routing_rules')" - namespace: - - "{{labels.data_stream.namespace}}" - - default diff --git a/test/packages/bad_routing_rules/data_stream/rules/routing_rules.yml b/test/packages/bad_routing_rules/data_stream/rules/routing_rules.yml new file mode 100644 index 000000000..5b68ea747 --- /dev/null +++ b/test/packages/bad_routing_rules/data_stream/rules/routing_rules.yml @@ -0,0 +1,35 @@ +routing_rules: + # "Local" routing rules are included under this current dataset, not a special case + - dataset: routing_rules + rules: # Route error logs to `routing_rules.error` when they're sourced from an error logfile + - dataset: routing_rules.error + if: "ctx?.file?.path?.contains('/var/log/routing_rules/error')" + namespace: + - "{{labels.data_stream.namespace}}" + - default + + # Route access logs to `routing_rules.access` when they're sourced from an access logfile + - dataset: routing_rules.access + if: "ctx?.file?.path?.contains('/var/log/routing_rules/access')" + namespace: + - "{{labels.data_stream.namespace}}" + - default + + # Route K8's container logs to this catch-all dataset for further routing + - dataset: k8s.router + rules: + - dataset: routing_rules + if: "ctx?.container?.image?.name == 'routing_rules'" + namespace: + - "{{labels.data_stream.namespace}}" + - default + + # Route syslog entries tagged with routing_rules to this catch-all dataset + - dataset: syslog + rules: + - dataset: routing_rules + if: "ctx?.tags?.contains('routing_rules')" + namespace: + - "{{labels.data_stream.namespace}}" + - default + diff --git a/test/packages/good_v2/data_stream/routing_rules/manifest.yml b/test/packages/good_v2/data_stream/routing_rules/manifest.yml index adb88a0e8..1f4d7dae5 100644 --- a/test/packages/good_v2/data_stream/routing_rules/manifest.yml +++ b/test/packages/good_v2/data_stream/routing_rules/manifest.yml @@ -12,38 +12,3 @@ streams: multi: true default: - /var/log/*.log - -routing_rules: - # "Local" routing rules are included under this current dataset, not a special case - - dataset: routing_rules - rules: # Route error logs to `routing_rules.error` when they're sourced from an error logfile - - dataset: routing_rules.error - if: "ctx?.file?.path?.contains('/var/log/routing_rules/error')" - namespace: - - "{{labels.data_stream.namespace}}" - - default - - # Route access logs to `routing_rules.access` when they're sourced from an access logfile - - dataset: routing_rules.access - if: "ctx?.file?.path?.contains('/var/log/routing_rules/access')" - namespace: - - "{{labels.data_stream.namespace}}" - - default - - # Route K8's container logs to this catch-all dataset for further routing - - dataset: k8s.router - rules: - - dataset: routing_rules - if: "ctx?.container?.image?.name == 'routing_rules'" - namespace: - - "{{labels.data_stream.namespace}}" - - default - - # Route syslog entries tagged with routing_rules to this catch-all dataset - - dataset: syslog - rules: - - dataset: routing_rules - if: "ctx?.tags?.contains('routing_rules')" - namespace: - - "{{labels.data_stream.namespace}}" - - default diff --git a/test/packages/good_v2/data_stream/routing_rules/routing_rules.yml b/test/packages/good_v2/data_stream/routing_rules/routing_rules.yml new file mode 100644 index 000000000..5b68ea747 --- /dev/null +++ b/test/packages/good_v2/data_stream/routing_rules/routing_rules.yml @@ -0,0 +1,35 @@ +routing_rules: + # "Local" routing rules are included under this current dataset, not a special case + - dataset: routing_rules + rules: # Route error logs to `routing_rules.error` when they're sourced from an error logfile + - dataset: routing_rules.error + if: "ctx?.file?.path?.contains('/var/log/routing_rules/error')" + namespace: + - "{{labels.data_stream.namespace}}" + - default + + # Route access logs to `routing_rules.access` when they're sourced from an access logfile + - dataset: routing_rules.access + if: "ctx?.file?.path?.contains('/var/log/routing_rules/access')" + namespace: + - "{{labels.data_stream.namespace}}" + - default + + # Route K8's container logs to this catch-all dataset for further routing + - dataset: k8s.router + rules: + - dataset: routing_rules + if: "ctx?.container?.image?.name == 'routing_rules'" + namespace: + - "{{labels.data_stream.namespace}}" + - default + + # Route syslog entries tagged with routing_rules to this catch-all dataset + - dataset: syslog + rules: + - dataset: routing_rules + if: "ctx?.tags?.contains('routing_rules')" + namespace: + - "{{labels.data_stream.namespace}}" + - default + From 8cb7bd740a3623fecb8ada5671dec98d5977be9b Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 15 Jun 2023 19:02:06 +0200 Subject: [PATCH 12/27] Add license header --- .../validator/semantic/validate_routing_rules_and_dataset.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/code/go/internal/validator/semantic/validate_routing_rules_and_dataset.go b/code/go/internal/validator/semantic/validate_routing_rules_and_dataset.go index f40c9ea48..3d952e7a3 100644 --- a/code/go/internal/validator/semantic/validate_routing_rules_and_dataset.go +++ b/code/go/internal/validator/semantic/validate_routing_rules_and_dataset.go @@ -1,3 +1,7 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + package semantic import ( From 989b8c1f353d1eda673d04ee1cc6d44840824235 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Mon, 19 Jun 2023 17:27:08 +0200 Subject: [PATCH 13/27] Add descriptions --- spec/integration/data_stream/manifest.spec.yml | 2 ++ spec/integration/data_stream/routing_rules.spec.yml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/spec/integration/data_stream/manifest.spec.yml b/spec/integration/data_stream/manifest.spec.yml index 8e6f8ac6f..f657745f0 100644 --- a/spec/integration/data_stream/manifest.spec.yml +++ b/spec/integration/data_stream/manifest.spec.yml @@ -364,8 +364,10 @@ spec: additionalProperties: false properties: dataset: + description: Source dataset to be used for this reroute processsor type: string rules: + description: List of routing rules type: array items: $ref: "#/definitions/routing_rule" diff --git a/spec/integration/data_stream/routing_rules.spec.yml b/spec/integration/data_stream/routing_rules.spec.yml index 42705fe05..fdac6690f 100644 --- a/spec/integration/data_stream/routing_rules.spec.yml +++ b/spec/integration/data_stream/routing_rules.spec.yml @@ -64,8 +64,10 @@ spec: additionalProperties: false properties: dataset: + description: Source dataset to be used for this reroute processsor type: string rules: + description: List of routing rules type: array items: $ref: "#/definitions/routing_rule" From 71ba32082aed28efa9bdad3ff75f22c5f2596592 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Mon, 19 Jun 2023 17:33:10 +0200 Subject: [PATCH 14/27] Add comment for JSON patch --- spec/integration/data_stream/manifest.spec.yml | 2 -- spec/integration/data_stream/spec.yml | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/spec/integration/data_stream/manifest.spec.yml b/spec/integration/data_stream/manifest.spec.yml index f657745f0..8e6f8ac6f 100644 --- a/spec/integration/data_stream/manifest.spec.yml +++ b/spec/integration/data_stream/manifest.spec.yml @@ -364,10 +364,8 @@ spec: additionalProperties: false properties: dataset: - description: Source dataset to be used for this reroute processsor type: string rules: - description: List of routing rules type: array items: $ref: "#/definitions/routing_rule" diff --git a/spec/integration/data_stream/spec.yml b/spec/integration/data_stream/spec.yml index 1c296b329..0bfac7092 100644 --- a/spec/integration/data_stream/spec.yml +++ b/spec/integration/data_stream/spec.yml @@ -91,4 +91,4 @@ spec: # - before: 2.9.0 # patch: # - op: remove -# path: "/contents/0/contents/7" +# path: "/contents/0/contents/7" # remove routing_rules file definition From 79fe3adffbd4ddae1e92b4b3c2241e323d59091a Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Mon, 19 Jun 2023 17:36:31 +0200 Subject: [PATCH 15/27] Remove routing_rules definition from manifest.spec.yml --- .../integration/data_stream/manifest.spec.yml | 76 ------------------- 1 file changed, 76 deletions(-) diff --git a/spec/integration/data_stream/manifest.spec.yml b/spec/integration/data_stream/manifest.spec.yml index 8e6f8ac6f..00e901f99 100644 --- a/spec/integration/data_stream/manifest.spec.yml +++ b/spec/integration/data_stream/manifest.spec.yml @@ -169,55 +169,6 @@ spec: hidden: description: Makes the data stream hidden type: boolean - routing_rule: - description: Routing rule definition - type: object - properties: - dataset: - description: > - Field references or a static value for the dataset part of the data stream name. - In addition to the criteria for index names, cannot contain - and must be no longer than 100 characters. - Example values are nginx.access and nginx.error. - - Supports field references with a mustache-like syntax (denoted as {{double}} or {{{triple}}} curly braces). - When resolving field references, the processor replaces invalid characters with _. - Uses the part of the index name as a fallback if all field references resolve to a null, missing, or non-string value - anyOf: - - type: string - - type: array - items: - type: string - examples: - - nginx.error - - nginx - if: - description: Conditionally execute the processor - type: string - examples: - - "ctx?.file?.path?.contains('/var/log/nginx/error')" - - "ctx?.container?.image?.name == 'nginx'" - namespace: - description: > - Field references or a static value for the namespace part of the data stream name. - See the criteria for index names for allowed characters. Must be no longer than 100 characters. - - Supports field references with a mustache-like syntax (denoted as {{double}} or {{{triple}}} curly braces). - When resolving field references, the processor replaces invalid characters with _. - Uses the part of the index name as a fallback if all field references resolve to a null, missing, or non-string value. - anyOf: - - type: string - - type: array - items: - type: string - items: - type: string - examples: - - default - - "{{ labels.dasta_stream.namespace}}" - required: - - dataset - - if - - namespace properties: dataset: description: Name of data set. @@ -356,29 +307,6 @@ spec: dynamic_namespace: description: When set to true, agents running this integration are granted data stream privileges for all namespaces of its type type: boolean - routing_rules: # technical preview - description: Routing rules set. - type: array - items: - type: object - additionalProperties: false - properties: - dataset: - type: string - rules: - type: array - items: - $ref: "#/definitions/routing_rule" - required: - - dataset - - rules - allOf: - - if: - required: - - routing_rules - then: - required: - - dataset required: - title versions: @@ -390,7 +318,3 @@ versions: patch: - op: remove path: /properties/type/enum/4 - - before: 2.9.0 - patch: - - op: remove - path: /properties/routing_rules From a07d8ee83644431f51f8a5d389d89ec8211bc76b Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 20 Jun 2023 10:50:37 +0200 Subject: [PATCH 16/27] Add JSON patch to remove routing_rules --- spec/integration/data_stream/spec.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/spec/integration/data_stream/spec.yml b/spec/integration/data_stream/spec.yml index 0bfac7092..1f7e49f95 100644 --- a/spec/integration/data_stream/spec.yml +++ b/spec/integration/data_stream/spec.yml @@ -87,8 +87,8 @@ spec: $ref: "./routing_rules.spec.yml" # TODO add JSON patch to remove routing_rules.yml entry -# versions: -# - before: 2.9.0 -# patch: -# - op: remove -# path: "/contents/0/contents/7" # remove routing_rules file definition +versions: + - before: 2.9.0 + patch: + - op: remove + path: "/contents/0/contents/7" # remove routing_rules file definition From 6fd616ecbbd2d30e5cfe82305a45942a1ba4f208 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 20 Jun 2023 10:50:48 +0200 Subject: [PATCH 17/27] Add new test package --- code/go/pkg/validator/validator_test.go | 4 + .../bad_routing_rules_wrong_spec/LICENSE.txt | 93 ++++++++++++++++++ .../changelog.yml | 6 ++ .../rules/agent/stream/stream.yml.hbs | 7 ++ .../elasticsearch/ingest_pipeline/default.yml | 10 ++ .../data_stream/rules/fields/base-fields.yml | 12 +++ .../data_stream/rules/manifest.yml | 14 +++ .../data_stream/rules/routing_rules.yml | 35 +++++++ .../docs/README.md | 84 ++++++++++++++++ .../img/sample-logo.svg | 1 + .../img/sample-screenshot.png | Bin 0 -> 18849 bytes .../bad_routing_rules_wrong_spec/manifest.yml | 33 +++++++ 12 files changed, 299 insertions(+) create mode 100644 test/packages/bad_routing_rules_wrong_spec/LICENSE.txt create mode 100644 test/packages/bad_routing_rules_wrong_spec/changelog.yml create mode 100644 test/packages/bad_routing_rules_wrong_spec/data_stream/rules/agent/stream/stream.yml.hbs create mode 100644 test/packages/bad_routing_rules_wrong_spec/data_stream/rules/elasticsearch/ingest_pipeline/default.yml create mode 100644 test/packages/bad_routing_rules_wrong_spec/data_stream/rules/fields/base-fields.yml create mode 100644 test/packages/bad_routing_rules_wrong_spec/data_stream/rules/manifest.yml create mode 100644 test/packages/bad_routing_rules_wrong_spec/data_stream/rules/routing_rules.yml create mode 100644 test/packages/bad_routing_rules_wrong_spec/docs/README.md create mode 100644 test/packages/bad_routing_rules_wrong_spec/img/sample-logo.svg create mode 100644 test/packages/bad_routing_rules_wrong_spec/img/sample-screenshot.png create mode 100644 test/packages/bad_routing_rules_wrong_spec/manifest.yml diff --git a/code/go/pkg/validator/validator_test.go b/code/go/pkg/validator/validator_test.go index 1ac63d4f2..740fd1b76 100644 --- a/code/go/pkg/validator/validator_test.go +++ b/code/go/pkg/validator/validator_test.go @@ -576,10 +576,14 @@ func TestValidateExternalFieldsWithoutDevFolder(t *testing.T) { func TestValidateRoutingRules(t *testing.T) { tests := map[string][]string{ + "good": []string{}, "good_v2": []string{}, "bad_routing_rules": []string{ `routing rules defined in data stream "rules" but dataset field is missing: dataset field is required in data stream "rules"`, }, + "bad_routing_rules_wrong_spec": []string{ + `item [routing_rules.yml] is not allowed in folder [../../../../test/packages/bad_routing_rules_wrong_spec/data_stream/rules]`, + }, } for pkgName, expectedErrorMessages := range tests { diff --git a/test/packages/bad_routing_rules_wrong_spec/LICENSE.txt b/test/packages/bad_routing_rules_wrong_spec/LICENSE.txt new file mode 100644 index 000000000..809108b85 --- /dev/null +++ b/test/packages/bad_routing_rules_wrong_spec/LICENSE.txt @@ -0,0 +1,93 @@ +Elastic License 2.0 + +URL: https://www.elastic.co/licensing/elastic-license + +## Acceptance + +By using the software, you agree to all of the terms and conditions below. + +## Copyright License + +The licensor grants you a non-exclusive, royalty-free, worldwide, +non-sublicensable, non-transferable license to use, copy, distribute, make +available, and prepare derivative works of the software, in each case subject to +the limitations and conditions below. + +## Limitations + +You may not provide the software to third parties as a hosted or managed +service, where the service provides users with access to any substantial set of +the features or functionality of the software. + +You may not move, change, disable, or circumvent the license key functionality +in the software, and you may not remove or obscure any functionality in the +software that is protected by the license key. + +You may not alter, remove, or obscure any licensing, copyright, or other notices +of the licensor in the software. Any use of the licensor’s trademarks is subject +to applicable law. + +## Patents + +The licensor grants you a license, under any patent claims the licensor can +license, or becomes able to license, to make, have made, use, sell, offer for +sale, import and have imported the software, in each case subject to the +limitations and conditions in this license. This license does not cover any +patent claims that you cause to be infringed by modifications or additions to +the software. If you or your company make any written claim that the software +infringes or contributes to infringement of any patent, your patent license for +the software granted under these terms ends immediately. If your company makes +such a claim, your patent license ends immediately for work on behalf of your +company. + +## Notices + +You must ensure that anyone who gets a copy of any part of the software from you +also gets a copy of these terms. + +If you modify the software, you must include in any modified copies of the +software prominent notices stating that you have modified the software. + +## No Other Rights + +These terms do not imply any licenses other than those expressly granted in +these terms. + +## Termination + +If you use the software in violation of these terms, such use is not licensed, +and your licenses will automatically terminate. If the licensor provides you +with a notice of your violation, and you cease all violation of this license no +later than 30 days after you receive that notice, your licenses will be +reinstated retroactively. However, if you violate these terms after such +reinstatement, any additional violation of these terms will cause your licenses +to terminate automatically and permanently. + +## No Liability + +*As far as the law allows, the software comes as is, without any warranty or +condition, and the licensor will not be liable to you for any damages arising +out of these terms or the use or nature of the software, under any kind of +legal claim.* + +## Definitions + +The **licensor** is the entity offering these terms, and the **software** is the +software the licensor makes available under these terms, including any portion +of it. + +**you** refers to the individual or entity agreeing to these terms. + +**your company** is any legal entity, sole proprietorship, or other kind of +organization that you work for, plus all organizations that have control over, +are under the control of, or are under common control with that +organization. **control** means ownership of substantially all the assets of an +entity, or the power to direct its management and policies by vote, contract, or +otherwise. Control can be direct or indirect. + +**your licenses** are all the licenses granted to you for the software under +these terms. + +**use** means anything you do with the software requiring one of your licenses. + +**trademark** means trademarks, service marks, and similar rights. diff --git a/test/packages/bad_routing_rules_wrong_spec/changelog.yml b/test/packages/bad_routing_rules_wrong_spec/changelog.yml new file mode 100644 index 000000000..bb0320a52 --- /dev/null +++ b/test/packages/bad_routing_rules_wrong_spec/changelog.yml @@ -0,0 +1,6 @@ +# newer versions go on top +- version: "0.0.1" + changes: + - description: Initial draft of the package + type: enhancement + link: https://github.com/elastic/integrations/pull/1 # FIXME Replace with the real PR link diff --git a/test/packages/bad_routing_rules_wrong_spec/data_stream/rules/agent/stream/stream.yml.hbs b/test/packages/bad_routing_rules_wrong_spec/data_stream/rules/agent/stream/stream.yml.hbs new file mode 100644 index 000000000..5845510de --- /dev/null +++ b/test/packages/bad_routing_rules_wrong_spec/data_stream/rules/agent/stream/stream.yml.hbs @@ -0,0 +1,7 @@ +paths: +{{#each paths as |path i|}} + - {{path}} +{{/each}} +exclude_files: [".gz$"] +processors: + - add_locale: ~ diff --git a/test/packages/bad_routing_rules_wrong_spec/data_stream/rules/elasticsearch/ingest_pipeline/default.yml b/test/packages/bad_routing_rules_wrong_spec/data_stream/rules/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 000000000..81221adf3 --- /dev/null +++ b/test/packages/bad_routing_rules_wrong_spec/data_stream/rules/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,10 @@ +--- +description: Pipeline for processing sample logs +processors: +- set: + field: sample_field + value: "1" +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' \ No newline at end of file diff --git a/test/packages/bad_routing_rules_wrong_spec/data_stream/rules/fields/base-fields.yml b/test/packages/bad_routing_rules_wrong_spec/data_stream/rules/fields/base-fields.yml new file mode 100644 index 000000000..7c798f453 --- /dev/null +++ b/test/packages/bad_routing_rules_wrong_spec/data_stream/rules/fields/base-fields.yml @@ -0,0 +1,12 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. diff --git a/test/packages/bad_routing_rules_wrong_spec/data_stream/rules/manifest.yml b/test/packages/bad_routing_rules_wrong_spec/data_stream/rules/manifest.yml new file mode 100644 index 000000000..bd09ad2ac --- /dev/null +++ b/test/packages/bad_routing_rules_wrong_spec/data_stream/rules/manifest.yml @@ -0,0 +1,14 @@ +title: "Rules" +dataset: routing_rules +type: logs +streams: + - input: logfile + title: Sample logs + description: Collect sample logs + vars: + - name: paths + type: text + title: Paths + multi: true + default: + - /var/log/*.log diff --git a/test/packages/bad_routing_rules_wrong_spec/data_stream/rules/routing_rules.yml b/test/packages/bad_routing_rules_wrong_spec/data_stream/rules/routing_rules.yml new file mode 100644 index 000000000..5b68ea747 --- /dev/null +++ b/test/packages/bad_routing_rules_wrong_spec/data_stream/rules/routing_rules.yml @@ -0,0 +1,35 @@ +routing_rules: + # "Local" routing rules are included under this current dataset, not a special case + - dataset: routing_rules + rules: # Route error logs to `routing_rules.error` when they're sourced from an error logfile + - dataset: routing_rules.error + if: "ctx?.file?.path?.contains('/var/log/routing_rules/error')" + namespace: + - "{{labels.data_stream.namespace}}" + - default + + # Route access logs to `routing_rules.access` when they're sourced from an access logfile + - dataset: routing_rules.access + if: "ctx?.file?.path?.contains('/var/log/routing_rules/access')" + namespace: + - "{{labels.data_stream.namespace}}" + - default + + # Route K8's container logs to this catch-all dataset for further routing + - dataset: k8s.router + rules: + - dataset: routing_rules + if: "ctx?.container?.image?.name == 'routing_rules'" + namespace: + - "{{labels.data_stream.namespace}}" + - default + + # Route syslog entries tagged with routing_rules to this catch-all dataset + - dataset: syslog + rules: + - dataset: routing_rules + if: "ctx?.tags?.contains('routing_rules')" + namespace: + - "{{labels.data_stream.namespace}}" + - default + diff --git a/test/packages/bad_routing_rules_wrong_spec/docs/README.md b/test/packages/bad_routing_rules_wrong_spec/docs/README.md new file mode 100644 index 000000000..90ff67864 --- /dev/null +++ b/test/packages/bad_routing_rules_wrong_spec/docs/README.md @@ -0,0 +1,84 @@ + + + +# Bad Routing Rules Wrong Spec + + + +## Data streams + + + + + + + + + + + +## Requirements + +You need Elasticsearch for storing and searching your data and Kibana for visualizing and managing it. +You can use our hosted Elasticsearch Service on Elastic Cloud, which is recommended, or self-manage the Elastic Stack on your own hardware. + + + +## Setup + + + +For step-by-step instructions on how to set up an integration, see the +[Getting started](https://www.elastic.co/guide/en/welcome-to-elastic/current/getting-started-observability.html) guide. + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/test/packages/bad_routing_rules_wrong_spec/img/sample-logo.svg b/test/packages/bad_routing_rules_wrong_spec/img/sample-logo.svg new file mode 100644 index 000000000..6268dd88f --- /dev/null +++ b/test/packages/bad_routing_rules_wrong_spec/img/sample-logo.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/test/packages/bad_routing_rules_wrong_spec/img/sample-screenshot.png b/test/packages/bad_routing_rules_wrong_spec/img/sample-screenshot.png new file mode 100644 index 0000000000000000000000000000000000000000..d7a56a3ecc078c38636698cefba33f86291dd178 GIT binary patch literal 18849 zcmeEu^S~#!E#4Tq;}?6chqwB{?k=6jc5D4>l%v(rleJ2Y%tW zDj9g7px}|*e;{M?LDwiK3@FNS(lDRTd-MJYIyUJCN948~OJk1M(DrJyI#iV;P4k~& zFZo35IfQt0RwlUN`48^6(1dv_wm(y1xhEdMld=Y?!%u=fPT_*{3( zwBwz3#qR}_)t>C*jp5@U)Ti~B)Y;qq*TRxZJ7ZRN_^A3TDAEM*@7Ve%(Ro7=1%1B< zVj6GBUTxXev>_^SFA zgKZ=g4aTS}9>Ofj7cSB0WO?gQ)x=+!hs_)b$6#>ScFZ>XAoIX)%Bc|BDC~JFBk0f0 z0NY}6gb)&!qx^FWC(!ji+Kl$V$2|ocA=vN0TM0Y`U?tX+T)c*C zA!IL(T2Vm%MCLa85^if@J@Kkprx8QN5!6eCR@4Oa5S?4-4|ou?90mFCM8D!;n(5xz zO}-*t!TntN>|a$s(kGQg1P-U?hqvGF2_fGvd&~yZ_l3Qf&j~XWa=;>N3#-~#zjzcc z*m18L`A-K2o!d@J>a8SRbm4P&-q1(H>|JgIymDbnJF&@008`=X!P?4DGgZb>voUl^ zNJKgPR4S={)3vuk_{n@=M8q;;aJL>q+VLdTnO=}`&x;1DKjJA3*f*idS{jP5?+;!W zn-^7021Z4zv`Aq`hmX1aid997RNh3fa-@PG(W7TzKa1W&5^y3|lPeETP7j9qXpo4)7%(W0_2 z^Nmq;t@rb1eP3?%kOkH`P%!zTC7ZHjSfNN3*Sb#=3#jB*KpNGNfnRZ{N(6DrW(;B2Bwom<%m?VQP%K+ zsFeF1-(DY}oP@)w^Kw~gPg03q?N;)Ec6^|nikA34T~RynX*z}H>R~qgT$`Zbhn8wzZs$j2fsGN&rOK-mIBBvzD@a8FgbLpL!h5N^u&0wG} zq!#md3MHITv?3@$37J?lc_5*LWJTTjel;IiU-Yq;(g9I^D&KN_NKVS0O~GvB~FzPM6}=4d%fG4Nw4pZshcyLqK@`b8?RhD38haIyr@+8+0r5TC1*C7^WleJ zZN3_ngTD#RQvNL*;qD2H@cBWJbCC#d!}=oKfod5SE9a?!?j%DVt1z@inN}Iy$r+96 zM@P?AC+(`cM;z6J94BYGJ;+P-N#yj$?`G26ydS&OVH?~JY(N4l()Fh+x+DoJ@r<+i zhm^ck@QP`=fLApr62@KyOef~}zuG;(VbDQmw|Wb+oSHSw=%w9R)=et0cY*~ytX)#M zEXlK^p;zM@vTnXn+C1vwP)~TJv|TvDE2($;;EzC5_5IL#H;u z)#CO8)TSzbt8)wHB8$I8KcIojx&GoE)3QNu{CQ+_xBmQ&`mL5-u=BX(hs^hMY^ zae!!*Q;Tr$@(0~GoBJAohGw*d{l8~!aXop87aaSUb2jm)Tk>#$1*cdo5Sl+?oD!l4Og~yX+soottl4 zp4OartUuAN(dD~yLJ}`A1*!D4-|L^hM;`_DM^1KYs-VF(}h(BjRO``b+xV~%O=-)?p z7ciJH7Fnl?V&=ay_AB{oQoa2iR;6$^tiE|-eRCFy|3F@%j#6gUxkZX@?K`F$u#;T< z4IZORpUthmB?U`;zrOkp?P(Rvd5TFRWrBJmVg;KEZvJ+;Q}FRY%QZ?c^&$oPXW+C5 zdN#c>v%U?QuE+hMQdzxS1Q(BT90;29qu#^A?a^)Ui;{TJ;%`nLgm2ew$J4NvREjCJ z$`C7&?tH$CrVG@M3J1-KJw_*9BKeL*JX{ zN+Vg_TXb9^jJO$ZGkXO6BBFDjt~w5`w2TB*z$&1W5Il3IiDs=ZMDt|9iRtKET*wF6 z0Z+|N87p-5Fh)^(*l>OVr5^aY5LW(@PuM>Qo@&)yj6XRkPm1>eTF#Y_c*aRF^ZY5A z9FAU7lKEHG@i{wJMPg;n6z2|69d-)q9@<7t()d-zPy&X zdXG7{Uw{k23)CzzQAXw#iqj<1u~W@K_Ljc#?ukh;fRKHeJ2l~Z+52b2n^bGiDF2oX zm25FLx|4AP8>rAi@koY03lrtS#X?zK591c?2iZ_jjc>0y>q9>fU<08o6zG%z9WK+S zDwZMW4~28wu#ye#V*@#5t^S@NiAA`3{SF$xINmc_WW^u-C9M=H>RQ1>WM=|R!660{ z6E6%DwX`eu<3pkmz7Z=FCRd$(vhDkc3yMnSr)5C*aho)DZ<12$`$TXj<8Z70)|rK7 zXFD8QzksfWZU`qL2K8X{C~TcF{KVW`3Y{IMb&)T9%1V`tv(HY1 z+LXkLyM|3mtLD{x-#hOw-U?sr-iLeHFA|=-sGZ4#hX)atL!a91(tWJc+og&5W}VfZ zpgE7`{5D`~?yGR++y7~xA&eU0N*ZezDjF$> zUeK&1aTFQRg*?v^Z2e7u<`lk$czR6}b6Cl-qA9%A`#A6q0*zyTu)X`3rhjR86NK3= zLdw{+-F}+b2gxd-qF7>Rla}dFkj|L#c|pg5Ni+MRA|BZH(@ME*o<1ijKcoXb%PVfJ ztp_uf=G%kvU((pHcw90Xut=}atA!giM-5By)f40nKp zv7Wdb{;^<}VRvruH~rYr~wEuYY2ov-5Q|p@u3Da9+z7PeIpBAwi?RxnxN3Kt+N9L(LUS%wxY` z>e&1VV;{CYw8DNRlvBH)>!I49SU4R!t3I4=y;mCevPZh!-}~G+F>6hcL_Rli4r zC4(WN)`j$>^S=~GMGR=^)A6wrqi(-x{xK37&Vx!OS6t=KQ2JVZo#GrSODtTe=TVh%*qfF%91nqsMNLNL^Gp|_ zz%I*HUkMQGqb!1eh{{bp|0GSCDbkG_D_d)8<(0r<6-%Qi7qDa7xZjcdZ$?Rth9L!f z$erCcs3<~mtupywbaT8NWZF#v?iZkvqSz3@p`RiXs7P!GUa~-U9hEG(NgI#3BzO-# z!9JWf(;r!*A=@g$f}>wi|6Q@9z8AmYf~x8G%sp>C5cfuJY;hs1o3Ozu^{pH0AFbs%yU)Xy5>Cf?qXiHn*-PAfKDRiy`U0sFSKFsgEZ6_ z9#ma!<#Izr^}_z*>PRSt564u6We*XmZUx^jv*dK; z4zyFZ*ZFSE!00<6!|+#33&R)@RA8V9YRjp$HS9?CGq*xDSDRbX#i;}mateEF{fqTI zt?X}Efkq_Ap*_ETgaikOBbQ|;47}hwX44K`(DUI@C)QiG&6UJ1UmRn*Q@6%e`+x(gpQp74O{;yli8YLCV}qD z4gIyZd_(8ED~WWaeXOb0^r=9=AiDT}by~+$KVF~M{ywbQl zng-h?a_E;yX?DCr4|_h7JMc7>xgWf7Ek-VmH^hCYunVp3{(d{---&%-GZ=rK#V5Jo zJvP8b!2AA5?9)G8gwzB6ze3TU<5*Pqms^Q-?C9-CN~4hb-`U0D@kAkTWn23``cao^ z8IWAp8h7`%ZA+eI?w$sJktq5m>e&0@mQn>2BdpKAxbj1$m$8Z;`!iFvl9($Lb9Ff? zT^6cTZ~HgIeR6R*;G(rzpgsJP41Fx9Df;G6{;k6T(i}&8hX(jHSC@~#X@70h#)g(( z*9vUC+a*b%oAdf1$}Z3NR;|c5nY4^Z51pfqk(tmJbB;Q#ka#tf5eae;-kq$I{xO3<(TI$0lSe-JQzJ*es;il=Kn_?&?E zfLbs{qErPqm)-*ZfwbA*D-shgb|1;X;cH*yA|q8gS=HiosF=-kbdk6--SR+`F^H_` z0*i`J==@XSe=HT;_``G}ulE=H@*3GU*?gVd@h*`eT^GKjI;C@8+h~;(u3bA#b&bN{ zYw>dJ$(;RfHDLlndS`CWOE=g0jOocCc&;w(dOzrLf4-DK*MD@P_;u&CbfMw=#Q-B` zDq8hGwKN-O7(hQA_bP3f5XrZH+@*FGw~ppmDgNWcf|Lf*Pc%e5dw1DcJ1BWm!z7z3 zr^toEU*P(>G#;_1X}Rz(5lbDtCui%hY^d3lm)kw0vyk zX~K4$AG#7cG`6s2%9g9zsaQ9o?;3yzW4Pt!;NlS zzI#G7tiq&@eV&}qDtY(e$1JwscAfle%Al{3>Nr%``n?`Jac^CdOXUbFgI3;m{RkA~ zokl+lxuw9=%W&MmzA+G%ZdFMMP&N2^6BWjG2Lt|xKx)lMCR@b0n+xgw<)&Dwi?}>- z+$_e|@M;uW@3z6)q&L7bYitZ%huzGqH_qHOr&G5o!?(8TJv_MN1ka|&c6_!Q>#PgHSFoPWiLg|k_{ zQd#Zy&BPkU(0OE5S35!B5qb6%T3Wd#J(zBl8dw6I#xIDDF-LBPi-jXv1E?!gE|1OIdTejK)+U3ooC^otSIRsWZf-`&K}6}s!407Y58zH zK(oYx*7sN1O|Z_1YIJS_H$E@DH(hB4QKNCGQT3PTvwYoe2&8WKi5`5tU-r4!>_V3XUT}N)>8V;+z-!@-IGCKiD>E9RC(K`NMx=;Qp zf$2g^t?)zpU0L!BZi(oE#)^Z_biT*Svh>r#%1=O+Wo37G`Q)4@k#Pe?^mgBIugC)8 zyEICH=`{A~^x#X&%tr-$j|(nXrIrGQYNY+C3M+LO;yUU4-|v>a5#P)XYp>_|C0f0n{_p0mvwWmghfd%!Cm}$qBDxOqA3htLs~ghSA1>6^dVgd~ zVHHBBy6;Pp=El;dkTE=ttp~BoOJ$L@EB3Z37T1kTNG3tm4PY5O-7hP5DA$-k=vV&6 z?RiAm;W~*o)R7!x9>u$&@|&D4xMmJ*y+^-6t!F0u8G~78t&Bs#W>w_NbW>W9M3tXWXRf zI86FWVx%iXXh6MJ>dg#?lNu{K@S#nzMIG4PXQd%!Bvc*H0c7F_Y=adptJr*cHevMQ z%?Xu~q8CFw>^L*S_83kVhq=)hf0%_Lq}SE*g(Da_A{kXVZfAd*YCwp~bG32wi&SNM z#QZ7}Ug5-=+s^uqAh_|}gzya<(&E?XAZ%0ybd9nraj?|z1YfPr*{N?Q{ji}YG`T#| z=uwJZHIMlsmevnenT#-)t$L*=2wh|1EYXW?_36TR?L!sUItJVxaC0$Gb|gq4{|4gA z(v0ODFj!T)jc5>65ys)* z7$aBHfbKdz@QJq1b`NT`344*g()$>5*Ey`TPB7WI;|_8o8t9-_4ikFub|I{66>ge> zHA+6onzFKY*eaiA!77SD*^&LyumAR6gSvxY6Q?;!AvI{rZ##!G$%ZfIgce4F`aF;e z?jVh%+B-vj69ei~bh_zA9w}S4B4rzRKQ1~u$gwVu_x5PlRKDXX2(_2Mm7fs%6{SS7Qh1gWT8xaxc=f8`mW38ukIZxwU;lmHABwFSg50*o zrj%f%j~IKR?N5Dxwrq|sTa?!pd{b3sFM&~{4~_^YH4$bI^Fq2W4-y`))^|7fS?i0) zJ&Z9wY!8%l7@gAr`2{fqA;L;ptQR*X2|xUtrT47KK%XN+dydN$*M?65LuXTRabgERR{n>;E;(&vS0_@COY!p<%5LsRqGpER%~YjkSK zwBo9-2|-ZFiU3TT&S+@}3gDT35t0IXTzX@yHA(v>Y8;-mZNySQ&fE7RJ1^tzJfvdApX& z*!+tE)Y{oR%jk8A)3EiI3i*(TOwP!;B3hAOj?KQ6^h-q~1V^166uYS~mH*2Hh*0}r z`R3u1#^LG9IW|^QT^|61H(T1Jz?n;(Z>52lU0BO>Q6*zgpP*gTFk2Uw)!3zt>3F~_ ztil4!R*-j}wjh%&(kSB%}X=u4RbFRp@^l+$SmM@nW9B;yGbf@nasjFMEE{m9Oe

}qal5$moSACwfNXLXG5|3R0AtBcN` z?%yS)&>O>sqxU64U~C3&Q^>z-Zt}WuX4Wh3dKj9EO zfSbV!c3e;EOeKHQmWEw#NM4;*tw-2o@x&kKT?rsmy-F|$jw-F>WgA7?C@{O1qPg*J zf92|RTBMh&ptHADFc{T+cB?+mOj>h2HKgwkxq6w&XBxPc?>=JKvU2K9aU93@vp-R% z{5T=P$9U}AYZ5QU{3%7}YZ+ACWXw#-U zWyxU(OP#Q9-2AeGmCwcp`zWghf2hvsOjWjDQbU?U`v0&a--f1`v0Bd8HLiLmo)PKz5!A1|XVO+89 zm3h2~6yI~cpWor!_yt-?Lt>z`c0a7cJAW)#d8N8nNIf0H<+v;s4{0guDD(?T7Z<~$ zd`$vpZ_QQgFaMT0_d5&+(jwGU?M1FqUu6wjA-9z?mRM}(CmSdK;2e$Na}F-8jbhgN z9)@AIQeghf{xCC^{9P%VdYW1PP#}2BJwWt z0Hd8%st1NK5%h+)UB^mVwh{e#8TIm$xxgGo6I5;e{~VUeeMGRpM_Z%=eH5$X1}?Z5 z`|*_Vp~K&ziz45-Ih9y>EOr(Buy0&n$dbQ4$5eSr=Ti z#~7^n8dmem;$0D4+6eV7&G2D~d@ z+R#u8+nw_N%7_U_1e53P?~&10^m|ZUXrZhVp04lQLsGos%0fRDhS=@>8TOAAxK;Cy z9GZw_1pfSxD5~xoR!INI?tU0wrKDd6^Tv{jL>`Xb49kBaNPlhMaIfh_nq_)zB7NcX z05XeQKz`@BDUx7*i!V~%dc8XQ#ngBw0A2tSr(npSCrNy5Z7>48v&Zz?0{%FRElh_h zN2|?#EhJL5HQMIu6m1=ypTR?tVymHK)xQvS9ir7FzMp?CjlND39PK`od#GytVhZWp zQ1@>MTE1*Ip>hnXSWa?XbMH#708@j12yPbm`JfcqIgmJepn$5YgkJn_%5I)mr`Q(k z-a0yFR3A`houhvf&|wNpIsV{2p%MqhR@`@R(l6`}iufEgI*UxWq~26?WTpZCV{JtG zYL?&#I98fyf_;2S0?_V{=Aa4t^x%vy$pF$_Lh7W2f*~5uPvGYh;vZhMv|u+Z?2t0~ zcYPXdxbg6OS*LUjR_=jLDt)ab6;?g1IuySLG@UE;jLpt-wjLX&RlY>fnd@f&?0NyT zht5vhP^};k6`U76$%&I)iWPNxG6KPjdh`S6>g9GN@;KObQsLG zKyjfrPR0PU1B0a0=)3@9eCDl?mB9rFdlTMtTAeZv2}F*|@JWleq2+H1bt>>x!^wTk z+I)cgsZwzCMwoRpW_*!3IySTQu!`HWugAXe(Ai(a9Rsu;*0#o6torxwNMxPzEAjt` z>70Vw;HCQ?AnP`RKQ;2R8h%;LI#tx^(MO*lMWJe4_?)Q571P`kTmN#(ez21V!<6+S z@Uap+y%#8&cGgdf+E@y$dUx3g#)=#5k31Vqv0p!%L`*=-PiQAiSg-d9lKRZQDuJ-| zA96zwwomG+4}X$vR*IU=NC!vL<`rUTbf_uRJC4FS;k&HtV<=<)p(qymH)=MDV^aqK z#%sid7K|~!H`J!7hRr~Z!emxgWq6#GpQs%c#BM+scvNGz|Gi4G`;8Z~dP8)+51iB8 zw)0fazNz5(iK$LJeC_4e^8&@wT(DZ~~>SStz3P(>V8CLNlZqgv=2K-|Lu~si@XFwMN>QE^k zVS2U_A?Q$?M`NkU}^!M8m%O&T=kW>dG}1s2I~hxp9Y=a=1XX-(fB5) zej3`e5Et~R^r%?CZK0)UZsF_+tSOGIBMdrtMf#oJjGF9U`*P8t>i*TWed$Z2WNUZ* z_1Qw4Yr+Q0@bD?hD0P-^v}?FpPBg~zz5~g@J#J76C695|P>1l;OS8%~hZh5&-9Ji# z50%&56ZK4FC9}{jHL0!=qo9Yd(GGHCEX2|-F(f}q6@NMT4P3rQd{Q!=bz-8N(Z^!N;;ZzAWRf@C?X>mG=_NgyQX_?Jv$m(9$W>P;+e}O|&w&DjbsJPdWp0A2$yLr*!BY73Z z5d*BCaTI)w=sTlofc>n}@v_tSXIK?8(g`G_06u>SD*fOZJ~visq3lBVS2+cf-r$UQ zZ(8A0g&5M$IV7w5nqL(m$VS0X?=yy-e6>S>Ca3wZNT)b{GF39_gJdONflqc-j$b~o z2l@@h{$KVfC)V?#We*)@xYC;L^<@cHo>8axRMbSzw|eYTl|8pkabsQJ(3`z{>5H}c z`psz_Y6t)hvzL^=}P#++XUl6v`-j)SuXd6BynjNZ!&c2hnyE&4*K$nXn31Zk)cm+lx;> zya{T?{MRtSu?^3Y9bS&O$*mW^vRUpv!J3Tz12?3&Y62b_oiZ$24O(75Z)JWb+Rj)ACbK`f<&tSwtT$|Sy z$41kRPiM-jnPY9PKrLyI`pHm6LusMsrO*HpmE){Kp1^u2t%6nW^;GB|!4k!Ik8oav zjM?DBKh9G@W0gEwiU-M}0B)}olvoM71RccgiZBCs)L?q_GX&JDhegx4k2&cNatr5w zU)1#2USb8&`etO5Vk z?0}K+*2*@a5yt*X{qg0@8jEz~jcylVj>-042p1PBnabI#xUiCRD!ouw3?u-wwsqwF z8(@m8-Lk7q@v154g6yvx_tRDa>}oqpVda)wfI9(;ZVGt1v^{<|X?vC_(i@IJC+2I_lusrT=$h zF1lPc*Neb`;Xgrdf`p$w)~MzQW0M3_FYRKu{2$VU82J^B=X1#^<&P$_`=S$Ey04WU zTxG;hrFNLhWC*p+sH3x=JVcBJ9*7>eO20)n671SxQhZQlHMRP8FyO}yai~OTsbms0 zQ3b$C1Cn!>jMHDq{VX1ab^~_Q!z+f75+_AuwiN0*wA_#M#0|rU{+NlB%>Y+TNT0Gj z`3^LKMSJjz2(?lwg~ixDl_5%rzzZ}o_6Fj9e)T7gpH4=BgT1zmwJpC@g(f%&0`}8B z%7Y&qlP3aFmI#nmT`|R3+Lwzp+PLXt|5g%vlY_$fvse7zjus0D0fA##r+i4G4K-2Y zC#H95NGoYfWP#ZF_v$^Li{PZpm}fc&)aL?5doPcb835Cr6`T+EzzcEvLtmXcbAb<^ zw!_Zgk6Az7YA@*vb)(G{_W-B|zrf76z^`X%jOgqIIaqi~5nUup3vugzzg&rA^w(zR z+qCzvIV~nGR=47pDOcNTzuBw#5a=<=DMvGa)g zPw$^pmq9Fg&b#BZrPSoml(149rZS!fioV*Dy$z440U3MXDJmI?RZqLy0}IKSxN)o( z8+8wIZs#q(|KTg6y;Z(=96>xfpUsr@SP}I^v zN^R;ZVrDaWmNrM5-<X@k6JyjvA3;jHhma|Y|7!Vk& zgf(UK_6~cC;!|b!YTjke=nBiUqQdb#I9TY}!s5P)H+^c;9cW(QO8O%n5J^8Xfktd*qrn)+?-gP`m%B&q zi^}7jKm`yMW8ITFOMN#!QIB6$SWx*75tnCMaNg*_J*WuwBh~AT>0($nS8%&zmFQDp z$dL65niDtTV%!Kg1`6epWoQGNG`$`doy;Zjaa`keyL0F6iJMae6FIgnhAfzU%m@V+ zm5rQihLwS~b6{-bVR1ZSzBI7(Yj+V6T-8V*7I`ptWArGdy~8pnV>fALpi~NQLZ7;^ zpaj35=md<~-(tNmF69UX3?ua}A7UIn)q5i1iPYEGlhYSbkfeX`5epkxtzk3Qbu| zlgA`7ts%IvF4HJ}-98akyRnjCo{u-`A4&b+r?s|o`4wdYAHs-yh91p$7C_|+EdYH5 z10`!*=n+W9g>V&dfU1H!J}ASZi&-?`2IlDOAHnu306rD`y>jT)4^@S(X4XhN2{g9i zj-ym98+RT|d0ejIFJCM5>S{mT-8uGmRRqkJ3sMO_AQDrv77Q zv$t>zaVpVF6eBguE%9M2u?E-Oleft8z5+~W`G}KXD(Yc;7m4{Op>Le(k`g1UK7(1# zt6g}$n=Tdn{T4pu>v!c;xRCd_WI$Ali13x=U_0T!Ga-U~9W88q-lU+RLn2`N8Ouho z^0@SvC>$DguHWx)?^*ms-{PVq%dn(U3vrLj9zITDqQZ`H>Wsp@Gf%}SG=m)Vh}F$ztQAbwVGdDgd!28j&yX9wLW&s! zNR~6`nYg;ULAq8zi<;gUchAV5ib67Y##l2 zy+%gaD(|~G4@||{A;TYDSoS>q2o{t23t-^!NDSDEm8j3ao7Ei>KYLEpb$jz}7ciAM zD}trDN+AVVT_lXW<++~>8>Cj8fzJo@R;>%nGq)6+w?(#mNc#1J4W+!hA}?g$0Xqo? zn67qJmss)e%k(xO*&K@z6+}nHA(lCkb6n-|{pSztys$8HiOWTVR)tCO*Q9~if%3n7`uxGzE+OCu zwcVV|tgQdq60952$>85-GHk$lwM(uI+CU1?i{sVnKd0+UNq#eSSKjUKfDDgLnBG1y z^v?f#MRFkph~TgkoKBvM`L_~we8__xpLcjh`GwV|87q`vazJq?SX=mXhdvK>VqUf~ z4sYoTIpt5S)KrE-?>&=cRoBumD7;b5pq!Y07)#I$`)<@U+mo*dE*P~773p*u^6waO z2#thJahX_ySlYMpjx%h<)i43ao~Is`^Ya zMNZkuChEA7+ZJe6$>-C*dzTYf3#1SY82yFG?S&Q)5rTbKS-XLjckTLEc7>^sFcntQ zBeNXCSg&q1N3Bi^4zlQ%mcEBQ%2ab$?(;t-$HYd2%cnX$uuwU#I_6D3($m zR(>gHzM9ODf;r8b0l5LuEIQVZiQ0-|3Y_xzJkZc*CD=bPJ+&J+>>se%D4uTq?Ny{l z0Z5~og*Wa1O&anlcRWu_%o)(x?IZ0CfUNk_R-ik>GyvdFmpu1wHZaKTDGhL zqxsji)n<+)VKbV0_BRq9E;Kb`f=&vn(BK0Ba-gL?ZN;^^b3YFg6R=!q#zM;tcX0dM zdy5PPx@6pJPXHzH7$dGjM|6@6777nXPWV;CIQdNf(*Znv)sMy&Xcq> zhCq+6h6&v8<0}vd2(sKqU3j>fr7&#Xy%qZHcMU3m{wld^Nstkz8GagB?Y=SI&H z&{&BSA-|(i35$9(l6LpFyLm$0M0fK`Dz!~ezL?yEInsXAFR!bHe;ZL>Gd(#Hv?<$%`^b)oi?x%(jkylCPb=juPlF znMo&o961=NZ_$gd{xp1ZY2dNDOS!=XVj!M^A z+$z`EK4v=m{Bs{&I4W)({`&<5*^BV#z{IBAI_d+9Qx;~ zby?2zEjzUUeZWBDo5cz>%;z||z)<+6UtC)y60yD5J5`oo_zSM;l21@CY<0_|)NME5 zs)kHCMBa5YzB#N=W2aR?y9((~WuYwwf+HAc2mvU>NYlxOTvGf^Ye3za?*f-qUs^`a zT3>RPh9*Jf%3*bf|kqtnD_Buxv!<9N>BbuD#uYv-q^ z%RDnd7a3O4M9Y~TNISS@9K}JDkdg@>x8E6@n8jF=6qiDV+}{!V)(o?ykcr0sxBGEx zo!X;pc=r{H^vw6ztV5VZXBa4~(ujB$rZQ|AaGN@J7#q%2nU9gJ)g6dcj}zYB1& z@iFE0vMQVxa|v7tDHS$gwX$Ihc#M^DXRC>J@Zk?dC(3uB_s~*W&m-01DFMQGWjj5x z5po1@1gPl!v1Yra@qPG{D;$bYLM3qOwpl~7f~l)#n< zP+6`!NYe3EE~4RFR#_e=7YctPRBt6$He@`%e5m}f$M%yzC2S0<1}hRPjO>HJY~ z*dx(nbMbjv*;o&k{qzBdF|lS;UNVKziV=gbLq}UOCwr8GT5E9oRYQ}+>DhbQ1R=lj zgcNJN8|D)$Mx3#c+t@lhqcDUnHGVt0&EyQ{b5)=52B(VTzw=pQ^ba3`JB@BU^lS`_ zJEiLzgU#Acd_!}FMxCWC**FP^i#P}bYzNs78)#uSejEtYLbG>JJ7Igtho2oKQ;XW~ z4eMGO+t!_;G^V6c&R`5Tg+Pz2ToN(aybq4Q0ssie_{`t*DO%V7FaZ`{MBobFc9|pV z70o5ayHGJo9$$&Pgbs)pWNzduAcbh?~U?_P)(ve0S*3H%eNF&a5XR=!J#4c z;t992n7ZJr{*%`^dU1d-ALE8!3i#v;3r4r%j+JFCe=%3Vj=8{aXe zs)jrcUBZ=;LudcTUXj2ub>K5!{HHFHJ}Trx(PYugbQ8yK7&sqX;(;|UWjk3tGs3zuceeX)i4i_jA8Qz2Bc%DxN8 zXw!$+9jBtEHd1y90bYG4f8DcJM)Ab!M39tH5zz94*MAvnhA377@buNupSOUU3j8~> zd6&hk^ENRCp9T?_QUHk<=(&9Q^MJ^pi;nKOYNR@?L=RCSmKMJ5UQJQ`X!i~(gD*P! zs`RobzJG3Ra_Pg+WZUXUmMU$ilpwfcEti6)mw(~MZ0q!^sza>#jv!-+7B6F3QuMWg zVO!rXwD+lF1BBTito?ml-CV3vxuek~TKuOX^N6sol$v*{_%nAuD7i81eXm^Lz(Z~I z2Xj_Dts#G0&C;PV_Wkq*1QvB7+Post4={v;gk7b9u%#DC_bh(iJm$rqog^{JEx6NE zrs5^2SEL$|98#2WV#iG@L6cq|)SuTMSfGocPl65wUd^|5Lbpnb(;t>-Qu2jvANLgv zdte0vED-3C@^BdyHWLL(7{G$WA02z@JG!T-U^Q7HZ(7Bs&vchkh(p&}KvnS{MG^i6 z4r){gJp9p7WyWOEiKA2Cm6EXIn&&gk|Fc6^78OpPrX4ExCFE=SD$xcH;C2eB^{XTI zaxz_Cef*Yj==w_i_BTGXP;8C&f? z*QEM>={jFM8)lWAR870pG4XEWsl%%K|82S5b=9hVz7p_6i-d(Iyvq76&a#PV zR;VbQV|n?mg}&(ehClg%tK%IjgtnTR-u)lxH06XxXqH0soAZbB_Rm)XX=6Nge1uoG7 z9vQM_S~2h53n|W`y{{R9+=08rv~MohI_v4-BU^7fZ0-A}#b5{AOSTJm+(J;9yw%pD zX6u62GJ&@HKX5zQwq~j8T!Hrv-Mk^QSB5cu09L03{ToDO7jikM0WAcsjW>D}^jqCF zT0DEZ@K^KO_MD*%M!+V)lGVU6?LpX)eQVXEmq}R`NIJv;kBitJ!nW?0OxTVlu2ADf zE{A!*0g3%nwVcBD+AgT5bGx@WOnQk{zRpiZ4HhP`3BF%N|HdqPbbiV5)7x)kzC3ID zZ;27>0^mrMgWc7evsbQY`l`l})wr+e;=8U_!2&B77;1qL!N8y)eTJ2lf#CvhR~!Qa mc;sM|90DP5A*JW%f2r=u1xt!e4gwD_V(@hJb6Mw<&;$SznOm^{ literal 0 HcmV?d00001 diff --git a/test/packages/bad_routing_rules_wrong_spec/manifest.yml b/test/packages/bad_routing_rules_wrong_spec/manifest.yml new file mode 100644 index 000000000..fb4cf73ef --- /dev/null +++ b/test/packages/bad_routing_rules_wrong_spec/manifest.yml @@ -0,0 +1,33 @@ +format_version: 2.8.0 +name: bad_routing_rules_wrong_spec +title: "Bad Routing Rules Wrong Spec" +version: 0.0.1 +source: + license: "Elastic-2.0" +description: "This package contains routing rules with the wrong spec version" +type: integration +categories: + - custom +conditions: + kibana.version: "^8.7.1" + elastic.subscription: "basic" +screenshots: + - src: /img/sample-screenshot.png + title: Sample screenshot + size: 600x600 + type: image/png +icons: + - src: /img/sample-logo.svg + title: Sample logo + size: 32x32 + type: image/svg+xml +policy_templates: + - name: sample + title: Sample logs + description: Collect sample logs + inputs: + - type: logfile + title: Collect sample logs from instances + description: Collecting sample logs +owner: + github: elastic/integrations From b5fd187fbefc7fe4ce10f8a328fec7978fea2e5b Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 20 Jun 2023 13:01:44 +0200 Subject: [PATCH 18/27] Change routing rules definition to be an array As routing rules are not defined in its own file, its definition has been updated to be an array directly. No need to have a "routing_rules" key. Updated description for source dataset. --- .../data_stream/routing_rules.spec.yml | 42 +++++++++---------- 1 file changed, 19 insertions(+), 23 deletions(-) diff --git a/spec/integration/data_stream/routing_rules.spec.yml b/spec/integration/data_stream/routing_rules.spec.yml index fdac6690f..c729d6ba4 100644 --- a/spec/integration/data_stream/routing_rules.spec.yml +++ b/spec/integration/data_stream/routing_rules.spec.yml @@ -3,8 +3,6 @@ ## spec: # Everything under here follows JSON schema (https://json-schema.org/), written as YAML for readability - type: object - additionalProperties: false definitions: routing_rule: description: Routing rule definition @@ -55,24 +53,22 @@ spec: - dataset - if - namespace - properties: - routing_rules: # technical preview - description: Routing rules set. - type: array - items: - type: object - additionalProperties: false - properties: - dataset: - description: Source dataset to be used for this reroute processsor - type: string - rules: - description: List of routing rules - type: array - items: - $ref: "#/definitions/routing_rule" - required: - - dataset - - rules - required: - - routing_rules + type: array + description: Routing rules set. + items: + type: object + additionalProperties: false + properties: + dataset: + description: > + Source dataset to be used by this reroute processsor. + If applicable, documents from this dataset will be routed according to the rules defined. + type: string + rules: + description: List of routing rules + type: array + items: + $ref: "#/definitions/routing_rule" + required: + - dataset + - rules From 56671f02c79bfd4a2382f5a74bd680a3dd2ab190 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 20 Jun 2023 13:07:34 +0200 Subject: [PATCH 19/27] Update validation rule and test packages --- .../validate_routing_rules_and_dataset.go | 2 +- .../data_stream/rules/routing_rules.yml | 62 +++++++++---------- .../data_stream/rules/routing_rules.yml | 62 +++++++++---------- .../routing_rules/routing_rules.yml | 62 +++++++++---------- 4 files changed, 91 insertions(+), 97 deletions(-) diff --git a/code/go/internal/validator/semantic/validate_routing_rules_and_dataset.go b/code/go/internal/validator/semantic/validate_routing_rules_and_dataset.go index 3d952e7a3..b8b6e571d 100644 --- a/code/go/internal/validator/semantic/validate_routing_rules_and_dataset.go +++ b/code/go/internal/validator/semantic/validate_routing_rules_and_dataset.go @@ -69,7 +69,7 @@ func anyRoutingRulesInDataStream(fsys fspath.FS, dataStream string) (bool, error return false, fmt.Errorf("single routing rules expected") } - vals, err := f[0].Values("$.routing_rules") + vals, err := f[0].Values("$.") if err != nil { return false, fmt.Errorf("can't read routing_rules: %w", err) } diff --git a/test/packages/bad_routing_rules/data_stream/rules/routing_rules.yml b/test/packages/bad_routing_rules/data_stream/rules/routing_rules.yml index 5b68ea747..07fd0e18b 100644 --- a/test/packages/bad_routing_rules/data_stream/rules/routing_rules.yml +++ b/test/packages/bad_routing_rules/data_stream/rules/routing_rules.yml @@ -1,35 +1,33 @@ -routing_rules: - # "Local" routing rules are included under this current dataset, not a special case - - dataset: routing_rules - rules: # Route error logs to `routing_rules.error` when they're sourced from an error logfile - - dataset: routing_rules.error - if: "ctx?.file?.path?.contains('/var/log/routing_rules/error')" - namespace: - - "{{labels.data_stream.namespace}}" - - default - - # Route access logs to `routing_rules.access` when they're sourced from an access logfile - - dataset: routing_rules.access - if: "ctx?.file?.path?.contains('/var/log/routing_rules/access')" - namespace: - - "{{labels.data_stream.namespace}}" - - default +# "Local" routing rules are included under this current dataset, not a special case +- dataset: routing_rules + rules: # Route error logs to `routing_rules.error` when they're sourced from an error logfile + - dataset: routing_rules.error + if: "ctx?.file?.path?.contains('/var/log/routing_rules/error')" + namespace: + - "{{labels.data_stream.namespace}}" + - default - # Route K8's container logs to this catch-all dataset for further routing - - dataset: k8s.router - rules: - - dataset: routing_rules - if: "ctx?.container?.image?.name == 'routing_rules'" - namespace: - - "{{labels.data_stream.namespace}}" - - default + # Route access logs to `routing_rules.access` when they're sourced from an access logfile + - dataset: routing_rules.access + if: "ctx?.file?.path?.contains('/var/log/routing_rules/access')" + namespace: + - "{{labels.data_stream.namespace}}" + - default - # Route syslog entries tagged with routing_rules to this catch-all dataset - - dataset: syslog - rules: - - dataset: routing_rules - if: "ctx?.tags?.contains('routing_rules')" - namespace: - - "{{labels.data_stream.namespace}}" - - default +# Route K8's container logs to this catch-all dataset for further routing +- dataset: k8s.router + rules: + - dataset: routing_rules + if: "ctx?.container?.image?.name == 'routing_rules'" + namespace: + - "{{labels.data_stream.namespace}}" + - default +# Route syslog entries tagged with routing_rules to this catch-all dataset +- dataset: syslog + rules: + - dataset: routing_rules + if: "ctx?.tags?.contains('routing_rules')" + namespace: + - "{{labels.data_stream.namespace}}" + - default diff --git a/test/packages/bad_routing_rules_wrong_spec/data_stream/rules/routing_rules.yml b/test/packages/bad_routing_rules_wrong_spec/data_stream/rules/routing_rules.yml index 5b68ea747..07fd0e18b 100644 --- a/test/packages/bad_routing_rules_wrong_spec/data_stream/rules/routing_rules.yml +++ b/test/packages/bad_routing_rules_wrong_spec/data_stream/rules/routing_rules.yml @@ -1,35 +1,33 @@ -routing_rules: - # "Local" routing rules are included under this current dataset, not a special case - - dataset: routing_rules - rules: # Route error logs to `routing_rules.error` when they're sourced from an error logfile - - dataset: routing_rules.error - if: "ctx?.file?.path?.contains('/var/log/routing_rules/error')" - namespace: - - "{{labels.data_stream.namespace}}" - - default - - # Route access logs to `routing_rules.access` when they're sourced from an access logfile - - dataset: routing_rules.access - if: "ctx?.file?.path?.contains('/var/log/routing_rules/access')" - namespace: - - "{{labels.data_stream.namespace}}" - - default +# "Local" routing rules are included under this current dataset, not a special case +- dataset: routing_rules + rules: # Route error logs to `routing_rules.error` when they're sourced from an error logfile + - dataset: routing_rules.error + if: "ctx?.file?.path?.contains('/var/log/routing_rules/error')" + namespace: + - "{{labels.data_stream.namespace}}" + - default - # Route K8's container logs to this catch-all dataset for further routing - - dataset: k8s.router - rules: - - dataset: routing_rules - if: "ctx?.container?.image?.name == 'routing_rules'" - namespace: - - "{{labels.data_stream.namespace}}" - - default + # Route access logs to `routing_rules.access` when they're sourced from an access logfile + - dataset: routing_rules.access + if: "ctx?.file?.path?.contains('/var/log/routing_rules/access')" + namespace: + - "{{labels.data_stream.namespace}}" + - default - # Route syslog entries tagged with routing_rules to this catch-all dataset - - dataset: syslog - rules: - - dataset: routing_rules - if: "ctx?.tags?.contains('routing_rules')" - namespace: - - "{{labels.data_stream.namespace}}" - - default +# Route K8's container logs to this catch-all dataset for further routing +- dataset: k8s.router + rules: + - dataset: routing_rules + if: "ctx?.container?.image?.name == 'routing_rules'" + namespace: + - "{{labels.data_stream.namespace}}" + - default +# Route syslog entries tagged with routing_rules to this catch-all dataset +- dataset: syslog + rules: + - dataset: routing_rules + if: "ctx?.tags?.contains('routing_rules')" + namespace: + - "{{labels.data_stream.namespace}}" + - default diff --git a/test/packages/good_v2/data_stream/routing_rules/routing_rules.yml b/test/packages/good_v2/data_stream/routing_rules/routing_rules.yml index 5b68ea747..07fd0e18b 100644 --- a/test/packages/good_v2/data_stream/routing_rules/routing_rules.yml +++ b/test/packages/good_v2/data_stream/routing_rules/routing_rules.yml @@ -1,35 +1,33 @@ -routing_rules: - # "Local" routing rules are included under this current dataset, not a special case - - dataset: routing_rules - rules: # Route error logs to `routing_rules.error` when they're sourced from an error logfile - - dataset: routing_rules.error - if: "ctx?.file?.path?.contains('/var/log/routing_rules/error')" - namespace: - - "{{labels.data_stream.namespace}}" - - default - - # Route access logs to `routing_rules.access` when they're sourced from an access logfile - - dataset: routing_rules.access - if: "ctx?.file?.path?.contains('/var/log/routing_rules/access')" - namespace: - - "{{labels.data_stream.namespace}}" - - default +# "Local" routing rules are included under this current dataset, not a special case +- dataset: routing_rules + rules: # Route error logs to `routing_rules.error` when they're sourced from an error logfile + - dataset: routing_rules.error + if: "ctx?.file?.path?.contains('/var/log/routing_rules/error')" + namespace: + - "{{labels.data_stream.namespace}}" + - default - # Route K8's container logs to this catch-all dataset for further routing - - dataset: k8s.router - rules: - - dataset: routing_rules - if: "ctx?.container?.image?.name == 'routing_rules'" - namespace: - - "{{labels.data_stream.namespace}}" - - default + # Route access logs to `routing_rules.access` when they're sourced from an access logfile + - dataset: routing_rules.access + if: "ctx?.file?.path?.contains('/var/log/routing_rules/access')" + namespace: + - "{{labels.data_stream.namespace}}" + - default - # Route syslog entries tagged with routing_rules to this catch-all dataset - - dataset: syslog - rules: - - dataset: routing_rules - if: "ctx?.tags?.contains('routing_rules')" - namespace: - - "{{labels.data_stream.namespace}}" - - default +# Route K8's container logs to this catch-all dataset for further routing +- dataset: k8s.router + rules: + - dataset: routing_rules + if: "ctx?.container?.image?.name == 'routing_rules'" + namespace: + - "{{labels.data_stream.namespace}}" + - default +# Route syslog entries tagged with routing_rules to this catch-all dataset +- dataset: syslog + rules: + - dataset: routing_rules + if: "ctx?.tags?.contains('routing_rules')" + namespace: + - "{{labels.data_stream.namespace}}" + - default From 45141b0ac7f6eb9dbdceddeeb3aa178f6b2432a8 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 20 Jun 2023 14:18:15 +0200 Subject: [PATCH 20/27] Add comment about technical preview --- spec/integration/data_stream/spec.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/integration/data_stream/spec.yml b/spec/integration/data_stream/spec.yml index 1f7e49f95..4f59ee677 100644 --- a/spec/integration/data_stream/spec.yml +++ b/spec/integration/data_stream/spec.yml @@ -78,7 +78,7 @@ spec: required: false visibility: private $ref: "./_dev/spec.yml" - - description: File containing routing rules definitions + - description: File containing routing rules definitions # technical preview type: file contentMediaType: "application/x-yaml" sizeLimit: 5MB From 8eb07813b3a64c40dc6b81195c7bf0392209218e Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 20 Jun 2023 16:58:46 +0200 Subject: [PATCH 21/27] Update JSON path to read routing rules array --- .../validator/semantic/validate_routing_rules_and_dataset.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/go/internal/validator/semantic/validate_routing_rules_and_dataset.go b/code/go/internal/validator/semantic/validate_routing_rules_and_dataset.go index b8b6e571d..bee8db6e1 100644 --- a/code/go/internal/validator/semantic/validate_routing_rules_and_dataset.go +++ b/code/go/internal/validator/semantic/validate_routing_rules_and_dataset.go @@ -69,7 +69,7 @@ func anyRoutingRulesInDataStream(fsys fspath.FS, dataStream string) (bool, error return false, fmt.Errorf("single routing rules expected") } - vals, err := f[0].Values("$.") + vals, err := f[0].Values("$[*]") if err != nil { return false, fmt.Errorf("can't read routing_rules: %w", err) } From 2450b6340ec0f2ddbd234e92d2cf75c361b87abd Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 21 Jun 2023 10:24:14 +0200 Subject: [PATCH 22/27] Rename dataset fields in routing_rules file --- .../integration/data_stream/routing_rules.spec.yml | 8 ++++---- .../data_stream/rules/routing_rules.yml | 14 +++++++------- .../data_stream/rules/routing_rules.yml | 14 +++++++------- .../data_stream/routing_rules/routing_rules.yml | 14 +++++++------- 4 files changed, 25 insertions(+), 25 deletions(-) diff --git a/spec/integration/data_stream/routing_rules.spec.yml b/spec/integration/data_stream/routing_rules.spec.yml index c729d6ba4..becff0502 100644 --- a/spec/integration/data_stream/routing_rules.spec.yml +++ b/spec/integration/data_stream/routing_rules.spec.yml @@ -8,7 +8,7 @@ spec: description: Routing rule definition type: object properties: - dataset: + target_dataset: description: > Field references or a static value for the dataset part of the data stream name. In addition to the criteria for index names, cannot contain - and must be no longer than 100 characters. @@ -50,7 +50,7 @@ spec: - default - "{{ labels.dasta_stream.namespace}}" required: - - dataset + - target_dataset - if - namespace type: array @@ -59,7 +59,7 @@ spec: type: object additionalProperties: false properties: - dataset: + source_dataset: description: > Source dataset to be used by this reroute processsor. If applicable, documents from this dataset will be routed according to the rules defined. @@ -70,5 +70,5 @@ spec: items: $ref: "#/definitions/routing_rule" required: - - dataset + - source_dataset - rules diff --git a/test/packages/bad_routing_rules/data_stream/rules/routing_rules.yml b/test/packages/bad_routing_rules/data_stream/rules/routing_rules.yml index 07fd0e18b..f626e8b07 100644 --- a/test/packages/bad_routing_rules/data_stream/rules/routing_rules.yml +++ b/test/packages/bad_routing_rules/data_stream/rules/routing_rules.yml @@ -1,32 +1,32 @@ # "Local" routing rules are included under this current dataset, not a special case -- dataset: routing_rules +- source_dataset: routing_rules rules: # Route error logs to `routing_rules.error` when they're sourced from an error logfile - - dataset: routing_rules.error + - target_dataset: routing_rules.error if: "ctx?.file?.path?.contains('/var/log/routing_rules/error')" namespace: - "{{labels.data_stream.namespace}}" - default # Route access logs to `routing_rules.access` when they're sourced from an access logfile - - dataset: routing_rules.access + - target_dataset: routing_rules.access if: "ctx?.file?.path?.contains('/var/log/routing_rules/access')" namespace: - "{{labels.data_stream.namespace}}" - default # Route K8's container logs to this catch-all dataset for further routing -- dataset: k8s.router +- source_dataset: k8s.router rules: - - dataset: routing_rules + - target_dataset: routing_rules if: "ctx?.container?.image?.name == 'routing_rules'" namespace: - "{{labels.data_stream.namespace}}" - default # Route syslog entries tagged with routing_rules to this catch-all dataset -- dataset: syslog +- source_dataset: syslog rules: - - dataset: routing_rules + - target_dataset: routing_rules if: "ctx?.tags?.contains('routing_rules')" namespace: - "{{labels.data_stream.namespace}}" diff --git a/test/packages/bad_routing_rules_wrong_spec/data_stream/rules/routing_rules.yml b/test/packages/bad_routing_rules_wrong_spec/data_stream/rules/routing_rules.yml index 07fd0e18b..f626e8b07 100644 --- a/test/packages/bad_routing_rules_wrong_spec/data_stream/rules/routing_rules.yml +++ b/test/packages/bad_routing_rules_wrong_spec/data_stream/rules/routing_rules.yml @@ -1,32 +1,32 @@ # "Local" routing rules are included under this current dataset, not a special case -- dataset: routing_rules +- source_dataset: routing_rules rules: # Route error logs to `routing_rules.error` when they're sourced from an error logfile - - dataset: routing_rules.error + - target_dataset: routing_rules.error if: "ctx?.file?.path?.contains('/var/log/routing_rules/error')" namespace: - "{{labels.data_stream.namespace}}" - default # Route access logs to `routing_rules.access` when they're sourced from an access logfile - - dataset: routing_rules.access + - target_dataset: routing_rules.access if: "ctx?.file?.path?.contains('/var/log/routing_rules/access')" namespace: - "{{labels.data_stream.namespace}}" - default # Route K8's container logs to this catch-all dataset for further routing -- dataset: k8s.router +- source_dataset: k8s.router rules: - - dataset: routing_rules + - target_dataset: routing_rules if: "ctx?.container?.image?.name == 'routing_rules'" namespace: - "{{labels.data_stream.namespace}}" - default # Route syslog entries tagged with routing_rules to this catch-all dataset -- dataset: syslog +- source_dataset: syslog rules: - - dataset: routing_rules + - target_dataset: routing_rules if: "ctx?.tags?.contains('routing_rules')" namespace: - "{{labels.data_stream.namespace}}" diff --git a/test/packages/good_v2/data_stream/routing_rules/routing_rules.yml b/test/packages/good_v2/data_stream/routing_rules/routing_rules.yml index 07fd0e18b..f626e8b07 100644 --- a/test/packages/good_v2/data_stream/routing_rules/routing_rules.yml +++ b/test/packages/good_v2/data_stream/routing_rules/routing_rules.yml @@ -1,32 +1,32 @@ # "Local" routing rules are included under this current dataset, not a special case -- dataset: routing_rules +- source_dataset: routing_rules rules: # Route error logs to `routing_rules.error` when they're sourced from an error logfile - - dataset: routing_rules.error + - target_dataset: routing_rules.error if: "ctx?.file?.path?.contains('/var/log/routing_rules/error')" namespace: - "{{labels.data_stream.namespace}}" - default # Route access logs to `routing_rules.access` when they're sourced from an access logfile - - dataset: routing_rules.access + - target_dataset: routing_rules.access if: "ctx?.file?.path?.contains('/var/log/routing_rules/access')" namespace: - "{{labels.data_stream.namespace}}" - default # Route K8's container logs to this catch-all dataset for further routing -- dataset: k8s.router +- source_dataset: k8s.router rules: - - dataset: routing_rules + - target_dataset: routing_rules if: "ctx?.container?.image?.name == 'routing_rules'" namespace: - "{{labels.data_stream.namespace}}" - default # Route syslog entries tagged with routing_rules to this catch-all dataset -- dataset: syslog +- source_dataset: syslog rules: - - dataset: routing_rules + - target_dataset: routing_rules if: "ctx?.tags?.contains('routing_rules')" namespace: - "{{labels.data_stream.namespace}}" From f8abd1237ac31549ea6b0ff28468a6934f168793 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 21 Jun 2023 13:53:57 +0200 Subject: [PATCH 23/27] Add check if there is no routing rule file --- .../validator/semantic/validate_routing_rules_and_dataset.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/code/go/internal/validator/semantic/validate_routing_rules_and_dataset.go b/code/go/internal/validator/semantic/validate_routing_rules_and_dataset.go index bee8db6e1..839c78c09 100644 --- a/code/go/internal/validator/semantic/validate_routing_rules_and_dataset.go +++ b/code/go/internal/validator/semantic/validate_routing_rules_and_dataset.go @@ -65,6 +65,11 @@ func anyRoutingRulesInDataStream(fsys fspath.FS, dataStream string) (bool, error if err != nil { return false, nil } + + if len(f) == 0 { + return false, nil + } + if len(f) != 1 { return false, fmt.Errorf("single routing rules expected") } From 6081bf8e111c8cbfd44c8d67d88b8fa47e690e9e Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 21 Jun 2023 16:17:50 +0200 Subject: [PATCH 24/27] Add technical preview annotations --- spec/integration/data_stream/routing_rules.spec.yml | 2 +- spec/integration/data_stream/spec.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/spec/integration/data_stream/routing_rules.spec.yml b/spec/integration/data_stream/routing_rules.spec.yml index becff0502..4b0cf0dac 100644 --- a/spec/integration/data_stream/routing_rules.spec.yml +++ b/spec/integration/data_stream/routing_rules.spec.yml @@ -5,7 +5,7 @@ spec: # Everything under here follows JSON schema (https://json-schema.org/), written as YAML for readability definitions: routing_rule: - description: Routing rule definition + description: Routing rule definition (technical preview) type: object properties: target_dataset: diff --git a/spec/integration/data_stream/spec.yml b/spec/integration/data_stream/spec.yml index 4f59ee677..0df58332c 100644 --- a/spec/integration/data_stream/spec.yml +++ b/spec/integration/data_stream/spec.yml @@ -78,7 +78,7 @@ spec: required: false visibility: private $ref: "./_dev/spec.yml" - - description: File containing routing rules definitions # technical preview + - description: File containing routing rules definitions (technical preview) type: file contentMediaType: "application/x-yaml" sizeLimit: 5MB From 420ccba9a09f00b3ec8a84a3eefa015910febd60 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 21 Jun 2023 16:52:24 +0200 Subject: [PATCH 25/27] Apply suggestions from code review Co-authored-by: Jaime Soriano Pastor --- spec/integration/data_stream/routing_rules.spec.yml | 1 + spec/integration/data_stream/spec.yml | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/integration/data_stream/routing_rules.spec.yml b/spec/integration/data_stream/routing_rules.spec.yml index 4b0cf0dac..48fba89df 100644 --- a/spec/integration/data_stream/routing_rules.spec.yml +++ b/spec/integration/data_stream/routing_rules.spec.yml @@ -67,6 +67,7 @@ spec: rules: description: List of routing rules type: array + minItems: 1 items: $ref: "#/definitions/routing_rule" required: diff --git a/spec/integration/data_stream/spec.yml b/spec/integration/data_stream/spec.yml index 0df58332c..79fc0f613 100644 --- a/spec/integration/data_stream/spec.yml +++ b/spec/integration/data_stream/spec.yml @@ -81,7 +81,6 @@ spec: - description: File containing routing rules definitions (technical preview) type: file contentMediaType: "application/x-yaml" - sizeLimit: 5MB name: "routing_rules.yml" required: false $ref: "./routing_rules.spec.yml" From 4a36c597f449775fed136f7f14e89c629286156e Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 21 Jun 2023 17:02:53 +0200 Subject: [PATCH 26/27] Add comment why routing rules is an array --- spec/integration/data_stream/routing_rules.spec.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/spec/integration/data_stream/routing_rules.spec.yml b/spec/integration/data_stream/routing_rules.spec.yml index 48fba89df..25a567fa1 100644 --- a/spec/integration/data_stream/routing_rules.spec.yml +++ b/spec/integration/data_stream/routing_rules.spec.yml @@ -53,6 +53,8 @@ spec: - target_dataset - if - namespace + # this is not an object because using the source dataset as key would require to support keys with dots. + # keys with dots are expanded here https://github.com/elastic/package-spec/blob/66abf8992f3ab7e9dd0b833e4ab9b43fc8b16471/code/go/internal/yamlschema/loader.go#L92 type: array description: Routing rules set. items: From 327705921fc718b75264d2d50891b0fbe19c24af Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 21 Jun 2023 17:07:17 +0200 Subject: [PATCH 27/27] Remove minItems from routingRules Adding minItems key results in an error: minItems must be of an integer, that could be related to the conversions performed in package-spec between YAML and JSON formats. --- spec/integration/data_stream/routing_rules.spec.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/spec/integration/data_stream/routing_rules.spec.yml b/spec/integration/data_stream/routing_rules.spec.yml index 25a567fa1..339ed5340 100644 --- a/spec/integration/data_stream/routing_rules.spec.yml +++ b/spec/integration/data_stream/routing_rules.spec.yml @@ -69,7 +69,6 @@ spec: rules: description: List of routing rules type: array - minItems: 1 items: $ref: "#/definitions/routing_rule" required: