Skip to content

Commit

Permalink
Enhance the ingest node simulate verbose output (elastic#60433)
Browse files Browse the repository at this point in the history 
This commit enhances the verbose output for the
`_ingest/pipeline/_simulate?verbose` api. Specifically
this adds the following:

* the pipeline processor is now included in the output
* the conditional (if) and result is now included in the output iff it was defined
* a status field is always displayed. the possible values of status are
  * `success` - if the processor ran with out errors
  * `error` - if the processor ran but threw an error that was not ingored
  * `error_ignored` - if the processor ran but threw an error that was ingored
  * `skipped` - if the process did not run (currently only possible if the if condition evaluates to false)
  * `dropped` - if the the `drop` processor ran and dropped the document
* a `processor_type` field for the type of processor (e.g. set, rename, etc.)
* throw a better error if trying to simulate with a pipeline that does not exist

closes elastic#56004
# Conflicts:
#	docs/reference/ingest/apis/simulate-pipeline.asciidoc
  • Loading branch information
@jakelandis
jakelandis committed Aug 4, 2020
1 parent 0587199 commit 0edc03b
Show file tree
Hide file tree
Showing 10 changed files with 507 additions and 217 deletions.
144 changes: 72 additions & 72 deletions docs/reference/ingest/apis/simulate-pipeline.asciidoc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -342,84 +342,84 @@ The API returns the following response:
[source,console-result]
----
{
"docs": [
{
"processor_results": [
{
"doc": {
"_id": "id",
"_index": "index",
"_type": "_doc",
"_source": {
"field2": "_value2",
"foo": "bar"
},
"_ingest": {
"timestamp": "2017-05-04T22:46:09.674Z",
"pipeline": "_simulate_pipeline"
}
}
"docs" : [
{
"processor_results" : [
{
"processor_type" : "set",
"status" : "success",
"doc" : {
"_index" : "index",
"_source" : {
"field2" : "_value2",
"foo" : "bar"
},
{
"doc": {
"_id": "id",
"_index": "index",
"_type": "_doc",
"_source": {
"field3": "_value3",
"field2": "_value2",
"foo": "bar"
},
"_ingest": {
"timestamp": "2017-05-04T22:46:09.675Z",
"pipeline": "_simulate_pipeline"
}
}
"_ingest" : {
"pipeline" : "_simulate_pipeline",
"timestamp" : "2020-07-30T01:21:24.251836Z"
}
]
},
{
"processor_results": [
{
"doc": {
"_id": "id",
"_index": "index",
"_type": "_doc",
"_source": {
"field2": "_value2",
"foo": "rab"
},
"_ingest": {
"timestamp": "2017-05-04T22:46:09.676Z",
"pipeline": "_simulate_pipeline"
}
}
}
},
{
"processor_type" : "set",
"status" : "success",
"doc" : {
"_index" : "index",
"_source" : {
"field3" : "_value3",
"field2" : "_value2",
"foo" : "bar"
},
{
"doc": {
"_id": "id",
"_index": "index",
"_type": "_doc",
"_source": {
"field3": "_value3",
"field2": "_value2",
"foo": "rab"
},
"_ingest": {
"timestamp": "2017-05-04T22:46:09.677Z",
"pipeline": "_simulate_pipeline"
}
}
"_ingest" : {
"pipeline" : "_simulate_pipeline",
"timestamp" : "2020-07-30T01:21:24.251836Z"
}
]
}
]
}
}
]
},
{
"processor_results" : [
{
"processor_type" : "set",
"status" : "success",
"doc" : {
"_index" : "index",
"_source" : {
"field2" : "_value2",
"foo" : "rab"
},
"_ingest" : {
"pipeline" : "_simulate_pipeline",
"timestamp" : "2020-07-30T01:21:24.251863Z"
}
}
},
{
"processor_type" : "set",
"status" : "success",
"doc" : {
"_index" : "index",
"_source" : {
"field3" : "_value3",
"field2" : "_value2",
"foo" : "rab"
},
"_ingest" : {
"pipeline" : "_simulate_pipeline",
"timestamp" : "2020-07-30T01:21:24.251863Z"
}
}
}
]
}
]
}
----
// TESTRESPONSE[s/"2017-05-04T22:46:09.674Z"/$body.docs.0.processor_results.0.doc._ingest.timestamp/]
// TESTRESPONSE[s/"2017-05-04T22:46:09.675Z"/$body.docs.0.processor_results.1.doc._ingest.timestamp/]
// TESTRESPONSE[s/"2017-05-04T22:46:09.676Z"/$body.docs.1.processor_results.0.doc._ingest.timestamp/]
// TESTRESPONSE[s/"2017-05-04T22:46:09.677Z"/$body.docs.1.processor_results.1.doc._ingest.timestamp/]
// TESTRESPONSE[s/"2020-07-30T01:21:24.251836Z"/$body.docs.0.processor_results.0.doc._ingest.timestamp/]
// TESTRESPONSE[s/"2020-07-30T01:21:24.251836Z"/$body.docs.0.processor_results.1.doc._ingest.timestamp/]
// TESTRESPONSE[s/"2020-07-30T01:21:24.251863Z"/$body.docs.1.processor_results.0.doc._ingest.timestamp/]
// TESTRESPONSE[s/"2020-07-30T01:21:24.251863Z"/$body.docs.1.processor_results.1.doc._ingest.timestamp/]

////
[source,console]
Expand Down
91 changes: 82 additions & 9 deletions modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/90_simulate.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -559,11 +559,15 @@ teardown:
- match: { docs.0.processor_results.0.tag: "setstatus-1" }
- match: { docs.0.processor_results.0.doc._source.field1: "123.42 400 <foo>" }
- match: { docs.0.processor_results.0.doc._source.status: 200 }
- match: { docs.0.processor_results.0.status: "success" }
- match: { docs.0.processor_results.0.processor_type: "set" }
- match: { docs.0.processor_results.1.tag: "rename-1" }
- match: { docs.0.processor_results.1.ignored_error.error.type: "illegal_argument_exception" }
- match: { docs.0.processor_results.1.ignored_error.error.reason: "field [foofield] doesn't exist" }
- match: { docs.0.processor_results.1.doc._source.field1: "123.42 400 <foo>" }
- match: { docs.0.processor_results.1.doc._source.status: 200 }
- match: { docs.0.processor_results.1.status: "error_ignored" }
- match: { docs.0.processor_results.1.processor_type: "rename" }

---
"Test verbose simulate with ignore_failure and no exception thrown":
Expand Down Expand Up @@ -605,11 +609,16 @@ teardown:
}
- length: { docs: 1 }
- length: { docs.0.processor_results: 2 }
- length: { docs.0.processor_results.0: 4 }
- match: { docs.0.processor_results.0.tag: "setstatus-1" }
- match: { docs.0.processor_results.0.status: "success" }
- match: { docs.0.processor_results.0.processor_type: "set" }
- match: { docs.0.processor_results.0.doc._source.field1: "123.42 400 <foo>" }
- match: { docs.0.processor_results.0.doc._source.status: 200 }
- length: { docs.0.processor_results.1: 2 }
- length: { docs.0.processor_results.1: 4 }
- match: { docs.0.processor_results.1.tag: "rename-1" }
- match: { docs.0.processor_results.1.status: "success" }
- match: { docs.0.processor_results.1.processor_type: "rename" }
- match: { docs.0.processor_results.1.doc._source.new_status: 200 }

---
Expand Down Expand Up @@ -725,7 +734,8 @@ teardown:
{
"set": {
"field": "pipeline0",
"value": true
"value": true,
"description" : "first_set"
}
},
{
Expand All @@ -747,16 +757,25 @@ teardown:
]
}
- length: { docs: 1 }
- length: { docs.0.processor_results: 3 }
- length: { docs.0.processor_results: 5 }
- match: { docs.0.processor_results.0.doc._source.pipeline0: true }
- match: { docs.0.processor_results.0.status: "success" }
- match: { docs.0.processor_results.0.processor_type: "set" }
- match: { docs.0.processor_results.0.description: "first_set" }
- is_false: docs.0.processor_results.0.doc._source.pipeline1
- is_false: docs.0.processor_results.0.doc._source.pipeline2
- match: { docs.0.processor_results.1.doc._source.pipeline0: true }
- match: { docs.0.processor_results.1.doc._source.pipeline1: true }
- is_false: docs.0.processor_results.1.doc._source.pipeline2
- match: { docs.0.processor_results.1.doc: null }
- match: { docs.0.processor_results.1.status: "success" }
- match: { docs.0.processor_results.1.processor_type: "pipeline" }
- match: { docs.0.processor_results.2.doc._source.pipeline0: true }
- match: { docs.0.processor_results.2.doc._source.pipeline1: true }
- match: { docs.0.processor_results.2.doc._source.pipeline2: true }
- is_false: docs.0.processor_results.2.doc._source.pipeline2
- match: { docs.0.processor_results.3.doc: null }
- match: { docs.0.processor_results.3.status: "success" }
- match: { docs.0.processor_results.3.processor_type: "pipeline" }
- match: { docs.0.processor_results.4.doc._source.pipeline0: true }
- match: { docs.0.processor_results.4.doc._source.pipeline1: true }
- match: { docs.0.processor_results.4.doc._source.pipeline2: true }

---
"Test verbose simulate with true conditional and on failure":
Expand Down Expand Up @@ -817,19 +836,27 @@ teardown:
- length: { docs.0.processor_results: 4 }
- match: { docs.0.processor_results.0.tag: "gunna_fail" }
- match: { docs.0.processor_results.0.error.reason: "field [foo1] doesn't exist" }
- match: { docs.0.processor_results.0.status: "error" }
- match: { docs.0.processor_results.0.processor_type: "rename" }
- match: { docs.0.processor_results.1.tag: "failed1" }
- match: { docs.0.processor_results.1.doc._source.failed1: "failed1" }
- match: { docs.0.processor_results.1.doc._ingest.on_failure_processor_tag: "gunna_fail" }
- match: { docs.0.processor_results.1.status: "success" }
- match: { docs.0.processor_results.1.processor_type: "set" }
- match: { docs.0.processor_results.2.tag: "gunna_fail_again" }
- match: { docs.0.processor_results.2.error.reason: "field [foo2] doesn't exist" }
- match: { docs.0.processor_results.2.status: "error" }
- match: { docs.0.processor_results.2.processor_type: "rename" }
- match: { docs.0.processor_results.3.tag: "failed2" }
- match: { docs.0.processor_results.3.doc._source.failed1: "failed1" }
- match: { docs.0.processor_results.3.doc._source.failed2: "failed2" }
- match: { docs.0.processor_results.3.doc._ingest.on_failure_processor_tag: "gunna_fail_again" }
- match: { docs.0.processor_results.3.status: "success" }
- match: { docs.0.processor_results.3.processor_type: "set" }


---
"Test simulate with provided pipeline definition with tag and description in processors":
"Test simulate with pipeline with conditional and skipped and dropped":
- do:
ingest.simulate:
verbose: true
Expand All @@ -845,6 +872,16 @@ teardown:
"field" : "field2",
"value" : "_value"
}
},
{
"drop" : {
"if": "false"
}
},
{
"drop" : {
"if": "true"
}
}
]
},
Expand All @@ -859,7 +896,43 @@ teardown:
]
}
- length: { docs: 1 }
- length: { docs.0.processor_results: 1 }
- length: { docs.0.processor_results: 3 }
- match: { docs.0.processor_results.0.doc._source.field2: "_value" }
- match: { docs.0.processor_results.0.description: "processor_description" }
- match: { docs.0.processor_results.0.tag: "processor_tag" }
- match: { docs.0.processor_results.0.status: "success" }
- match: { docs.0.processor_results.0.processor_type: "set" }
- match: { docs.0.processor_results.1.status: "skipped" }
- match: { docs.0.processor_results.1.processor_type: "drop" }
- match: { docs.0.processor_results.1.if.condition: "false" }
- match: { docs.0.processor_results.1.if.result: false }
- match: { docs.0.processor_results.2.status: "dropped" }
- match: { docs.0.processor_results.2.processor_type: "drop" }
- match: { docs.0.processor_results.2.if.condition: "true" }
- match: { docs.0.processor_results.2.if.result: true }
---
"Test simulate with provided pipeline that does not exist":
- do:
catch: bad_request
ingest.simulate:
verbose: true
body: >
{
"pipeline": {
"description": "_description",
"processors": [
{
"pipeline": {
"name": "____pipeline_doesnot_exist___"
}
}
]
},
"docs": [
{
"_source": {}
}
]
}
- match: { error.root_cause.0.type: "illegal_argument_exception" }
- match: { error.root_cause.0.reason: "Pipeline processor configured for non-existent pipeline [____pipeline_doesnot_exist___]" }
Loading

0 comments on commit 0edc03b

Please sign in to comment.