diff --git a/packages/http_endpoint/_dev/build/docs/README.md b/packages/http_endpoint/_dev/build/docs/README.md deleted file mode 100644 index b0fa7e8b34f..00000000000 --- a/packages/http_endpoint/_dev/build/docs/README.md +++ /dev/null @@ -1,18 +0,0 @@ -# Custom HTTP Endpoint Log integration - -The custom HTTP Endpoint Log integration initializes a listening HTTP server that collects incoming HTTP POST requests containing a JSON body. The body must be either an object or an array of objects. Any other data types will result in an HTTP 400 (Bad Request) response. For arrays, one document is created for each object in the array. - -These are the possible response codes from the server. - -| HTTP Response Code | Name | Reason | -|-------------------- |------------------------ |-------------------------------------------------------------------- | -| 200 | OK | Returned on success. | -| 400 | Bad Request | Returned if JSON body decoding fails. | -| 401 | Unauthorized | Returned when basic auth, secret header, or HMAC validation fails. | -| 405 | Method Not Allowed | Returned if methods other than POST are used. | -| 406 | Not Acceptable | Returned if the POST request does not contain a body. | -| 415 | Unsupported Media Type | Returned if the Content-Type is not application/json. | -| 500 | Internal Server Error | Returned if an I/O error occurs reading the request. | - - -Custom ingest pipelines may be added by adding the name to the pipeline configuration option, creating custom ingest pipelines can be done either through the API or the [Ingest Node Pipeline UI](/app/management/ingest/ingest_pipelines/). diff --git a/packages/http_endpoint/_dev/test/system/test-http-config.yml b/packages/http_endpoint/_dev/test/system/test-http-config.yml new file mode 100644 index 00000000000..2ce887e221e --- /dev/null +++ b/packages/http_endpoint/_dev/test/system/test-http-config.yml @@ -0,0 +1,13 @@ +service: test-webhook-http +service_notify_signal: SIGHUP +input: http_endpoint +vars: + data_stream.dataset: http_endpoint.http_endpoint + preserve_original_event: true + listen_address: 0.0.0.0 + url: /json + prefix: event + listen_port: 9515 + basic_auth: true + username: abc123 + password: abc123 diff --git a/packages/http_endpoint/data_stream/generic/agent/stream/http.yml.hbs b/packages/http_endpoint/agent/input/http.yml.hbs similarity index 94% rename from packages/http_endpoint/data_stream/generic/agent/stream/http.yml.hbs rename to packages/http_endpoint/agent/input/http.yml.hbs index 0e41a572137..58b6712bc77 100644 --- a/packages/http_endpoint/data_stream/generic/agent/stream/http.yml.hbs +++ b/packages/http_endpoint/agent/input/http.yml.hbs @@ -9,9 +9,6 @@ prefix: {{prefix}} {{#if pipeline}} pipeline: {{pipeline}} {{/if}} -{{#if preserve_original_event}} -preserve_original_event: {{preserve_original_event}} -{{/if}} {{#if basic_auth}} basic_auth: {{basic_auth}} {{/if}} @@ -61,12 +58,13 @@ include_headers: ssl: {{ssl}} {{/if}} -{{#if tags}} tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} {{#each tags as |tag|}} - {{tag}} {{/each}} -{{/if}} {{#contains "forwarded" tags}} publisher_pipeline.disable_host: true {{/contains}} diff --git a/packages/http_endpoint/changelog.yml b/packages/http_endpoint/changelog.yml index 569e32f48a6..a836ac38062 100644 --- a/packages/http_endpoint/changelog.yml +++ b/packages/http_endpoint/changelog.yml @@ -1,3 +1,8 @@ +- version: "2.0.0" + changes: + - description: Converted package to input type. + type: enhancement + link: https://github.com/elastic/integrations/pull/9732 - version: "1.16.0" changes: - description: Allow user selection of HTTP method. diff --git a/packages/http_endpoint/data_stream/generic/_dev/test/system/test-http-config.yml b/packages/http_endpoint/data_stream/generic/_dev/test/system/test-http-config.yml deleted file mode 100644 index d571a65ba42..00000000000 --- a/packages/http_endpoint/data_stream/generic/_dev/test/system/test-http-config.yml +++ /dev/null @@ -1,12 +0,0 @@ -service: test-webhook-http -service_notify_signal: SIGHUP -input: http_endpoint -data_stream: - vars: - listen_address: 0.0.0.0 - url: /json - prefix: event - listen_port: 9515 - basic_auth: true - username: abc123 - password: abc123 diff --git a/packages/http_endpoint/data_stream/generic/manifest.yml b/packages/http_endpoint/data_stream/generic/manifest.yml deleted file mode 100644 index 1fbca609a99..00000000000 --- a/packages/http_endpoint/data_stream/generic/manifest.yml +++ /dev/null @@ -1,182 +0,0 @@ -title: Custom HTTP Endpoint Logs -type: logs -streams: - - input: http_endpoint - description: Collect JSON data from listening HTTP port with Elastic Agent. - title: Custom HTTP Endpoint Logs - template_path: http.yml.hbs - vars: - - name: method - title: HTTP Method - description: This options specifies which HTTP method to accept. - type: select - required: true - show_user: false - options: - - value: POST - text: POST - - value: PUT - text: PUT - - value: PATCH - text: PATCH - default: POST - - name: listen_address - type: text - title: Listen Address - description: | - Bind address for the HTTP listener. Use 0.0.0.0 to listen on all interfaces. - required: true - show_user: true - default: localhost - - name: listen_port - type: text - title: Listen port - description: | - Bind port for the listener. - required: true - show_user: true - default: 8080 - - name: url - type: text - title: URL - description: This options specific which URL path to accept requests on. Defaults to /. - required: false - show_user: true - - name: data_stream.dataset - type: text - title: Dataset name - description: | - Dataset to write data to. Changing the dataset will send the data to a different index. You can't use `-` in the name of a dataset and only valid characters for [Elasticsearch index names](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-index_.html). - default: http_endpoint.generic - required: true - show_user: true - - name: pipeline - type: text - title: Ingest Pipeline - description: | - The Ingest Node pipeline ID to be used by the integration. - required: false - show_user: true - - name: preserve_original_event - type: bool - title: Preserve Original Event - description: This option copies the raw unmodified body of the incoming request to the event.original field as a string before sending the event to Elasticsearch. - required: false - show_user: true - - name: prefix - type: text - title: Prefix - description: This option specifies which prefix field the incoming request will be mapped to. - required: false - show_user: false - - name: basic_auth - type: bool - title: Basic Auth - description: Enables or disables HTTP basic auth for each incoming request. If enabled then username and password will also need to be configured. - required: false - show_user: false - - name: username - type: text - title: Username - description: If basic_auth is enabled, this is the username used for authentication against the HTTP listener. Requires password to also be set. - required: false - show_user: false - - name: password - type: password - title: Password - description: If basic_auth is enabled, this is the password used for authentication against the HTTP listener. Requires username to also be set. - required: false - show_user: false - secret: true - - name: secret_header - type: text - title: Secret Header - description: The header to check for a specific value specified by secret.value. Certain webhooks provide the possibility to include a special header and secret to identify the source. - required: false - show_user: false - secret: false - - name: secret_value - type: password - title: Secret Value - description: The secret stored in the header name specified by secret.header. Certain webhooks provide the possibility to include a special header and secret to identify the source. - required: false - show_user: false - secret: true - - name: hmac_header - type: text - title: HMAC Header - description: The name of the header that contains the HMAC signature, for example X-Dropbox-Signature, X-Hub-Signature-256, etc. - required: false - show_user: false - - name: hmac_key - type: password - title: HMAC Key - description: The secret key used to calculate the HMAC signature. Typically, the webhook sender provides this value. - required: false - show_user: false - secret: true - - name: hmac_type - type: text - title: HMAC Type - description: The hash algorithm to use for the HMAC comparison. At this time the only valid values are sha256 or sha1. - required: false - show_user: false - - name: hmac_prefix - type: text - title: HMAC Prefix - description: The prefix for the signature. Certain webhooks prefix the HMAC signature with a value, for example sha256=. - required: false - show_user: false - - name: content_type - type: text - title: Content Type - description: By default the input expects the incoming POST to include a Content-Type of application/json to try to enforce the incoming data to be valid JSON. In certain scenarios when the source of the request is not able to do that, it can be overwritten with another value or set to null. - required: false - show_user: false - - name: response_code - type: text - title: Response Code - description: The HTTP response code returned upon success. Should be in the 2XX range. - required: false - show_user: false - - name: response_body - type: text - title: Response Body - description: The response body returned upon success. Should be a single line JSON string. - required: false - show_user: false - - name: include_headers - type: text - title: Include Headers - description: This options specifies a list of HTTP headers that should be copied from the incoming request and included in the document. All configured headers will always be canonicalized to match the headers of the incoming request. For example, ["content-type"] will become ["Content-Type"] when the filebeat is running. - multi: true - required: false - show_user: false - - name: ssl - type: yaml - title: TLS - description: Options for enabling TLS for the listening webhook endpoint. See the [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/configuration-ssl.html) for a list of all options. - multi: false - required: false - show_user: false - default: | - enabled: false - certificate: "/etc/pki/client/cert.pem" - key: "/etc/pki/client/cert.key" - - name: processors - type: yaml - title: Processors - multi: false - required: false - show_user: false - description: | - Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. - - name: tags - type: text - title: Tags - description: Tags to include in the published event - required: false - default: - - forwarded - multi: true - show_user: true diff --git a/packages/http_endpoint/fields/agent.yml b/packages/http_endpoint/fields/agent.yml new file mode 100644 index 00000000000..230f7bc911d --- /dev/null +++ b/packages/http_endpoint/fields/agent.yml @@ -0,0 +1,35 @@ +- name: cloud + title: Cloud + group: 2 + description: Fields related to the cloud or infrastructure the events are coming from. + footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' + type: group + fields: + - name: project.id + type: keyword + description: Name of the project in Google Cloud. + - name: image.id + type: keyword + description: Image ID for the cloud instance. +- name: host + title: Host + group: 2 + description: 'A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + type: group + fields: + - name: containerized + type: boolean + description: > + If the host is a container. + + - name: os.build + type: keyword + example: "18D109" + description: > + OS build information. + + - name: os.codename + type: keyword + example: "stretch" + description: >- + OS codename, if any. diff --git a/packages/http_endpoint/data_stream/generic/fields/base-fields.yml b/packages/http_endpoint/fields/base-fields.yml similarity index 94% rename from packages/http_endpoint/data_stream/generic/fields/base-fields.yml rename to packages/http_endpoint/fields/base-fields.yml index 25671cec132..430aaf54343 100644 --- a/packages/http_endpoint/data_stream/generic/fields/base-fields.yml +++ b/packages/http_endpoint/fields/base-fields.yml @@ -14,7 +14,6 @@ - name: event.dataset type: constant_keyword description: Event dataset - value: http_endpoint.generic - name: "@timestamp" type: date description: Event timestamp. diff --git a/packages/http_endpoint/data_stream/generic/fields/beats.yml b/packages/http_endpoint/fields/beats.yml similarity index 100% rename from packages/http_endpoint/data_stream/generic/fields/beats.yml rename to packages/http_endpoint/fields/beats.yml diff --git a/packages/http_endpoint/data_stream/generic/fields/ecs.yml b/packages/http_endpoint/fields/ecs.yml similarity index 65% rename from packages/http_endpoint/data_stream/generic/fields/ecs.yml rename to packages/http_endpoint/fields/ecs.yml index fd8179d53d8..21845b26f5a 100644 --- a/packages/http_endpoint/data_stream/generic/fields/ecs.yml +++ b/packages/http_endpoint/fields/ecs.yml @@ -1,5 +1,9 @@ - name: ecs.version external: ecs +- name: event.code + external: ecs +- name: event.created + external: ecs - name: log.level external: ecs - name: message diff --git a/packages/http_endpoint/manifest.yml b/packages/http_endpoint/manifest.yml index 4a6f9e94894..dff840316c5 100644 --- a/packages/http_endpoint/manifest.yml +++ b/packages/http_endpoint/manifest.yml @@ -2,8 +2,8 @@ format_version: "3.0.2" name: http_endpoint title: Custom HTTP Endpoint Logs description: Collect JSON data from listening HTTP port with Elastic Agent. -type: integration -version: "1.16.0" +type: input +version: "2.0.0" conditions: kibana: version: "^8.12.0" @@ -12,12 +12,187 @@ categories: - observability policy_templates: - name: http_endpoint + type: logs title: Custom HTTP Endpoint Logs description: Collect JSON data from listening HTTP port with Elastic Agent. - inputs: - - type: http_endpoint - title: Custom HTTP Endpoint Logs - description: Collect JSON data from listening HTTP port with Elastic Agent. + input: http_endpoint + template_path: http.yml.hbs + vars: + - name: method + title: HTTP Method + description: This options specifies which HTTP method to accept. + type: select + required: true + show_user: false + options: + - value: POST + text: POST + - value: PUT + text: PUT + - value: PATCH + text: PATCH + default: POST + - name: listen_address + type: text + title: Listen Address + description: | + Bind address for the HTTP listener. Use 0.0.0.0 to listen on all interfaces. + required: true + show_user: true + default: localhost + - name: listen_port + type: text + title: Listen port + description: | + Bind port for the listener. + required: true + show_user: true + default: 8080 + - name: url + type: text + title: URL + description: This options specific which URL path to accept requests on. Defaults to /. + required: false + show_user: true + - name: data_stream.dataset + type: text + title: Dataset name + description: | + Dataset to write data to. Changing the dataset will send the data to a different index. You can't use `-` in the name of a dataset and only valid characters for [Elasticsearch index names](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-index_.html). + default: http_endpoint.generic + required: true + show_user: true + - name: pipeline + type: text + title: Ingest Pipeline + description: | + The Ingest Node pipeline ID to be used by the integration. + required: false + show_user: true + - name: preserve_original_event + type: bool + title: Preserve Original Event + description: This option copies the raw unmodified body of the incoming request to the event.original field as a string before sending the event to Elasticsearch. + required: false + show_user: true + default: false + - name: prefix + type: text + title: Prefix + description: This option specifies which prefix field the incoming request will be mapped to. + required: false + show_user: false + - name: basic_auth + type: bool + title: Basic Auth + description: Enables or disables HTTP basic auth for each incoming request. If enabled then username and password will also need to be configured. + required: false + show_user: false + - name: username + type: text + title: Username + description: If basic_auth is enabled, this is the username used for authentication against the HTTP listener. Requires password to also be set. + required: false + show_user: false + - name: password + type: password + title: Password + description: If basic_auth is enabled, this is the password used for authentication against the HTTP listener. Requires username to also be set. + required: false + show_user: false + secret: true + - name: secret_header + type: text + title: Secret Header + description: The header to check for a specific value specified by secret.value. Certain webhooks provide the possibility to include a special header and secret to identify the source. + required: false + show_user: false + secret: false + - name: secret_value + type: password + title: Secret Value + description: The secret stored in the header name specified by secret.header. Certain webhooks provide the possibility to include a special header and secret to identify the source. + required: false + show_user: false + secret: true + - name: hmac_header + type: text + title: HMAC Header + description: The name of the header that contains the HMAC signature, for example X-Dropbox-Signature, X-Hub-Signature-256, etc. + required: false + show_user: false + - name: hmac_key + type: password + title: HMAC Key + description: The secret key used to calculate the HMAC signature. Typically, the webhook sender provides this value. + required: false + show_user: false + secret: true + - name: hmac_type + type: text + title: HMAC Type + description: The hash algorithm to use for the HMAC comparison. At this time the only valid values are sha256 or sha1. + required: false + show_user: false + - name: hmac_prefix + type: text + title: HMAC Prefix + description: The prefix for the signature. Certain webhooks prefix the HMAC signature with a value, for example sha256=. + required: false + show_user: false + - name: content_type + type: text + title: Content Type + description: By default the input expects the incoming POST to include a Content-Type of application/json to try to enforce the incoming data to be valid JSON. In certain scenarios when the source of the request is not able to do that, it can be overwritten with another value or set to null. + required: false + show_user: false + - name: response_code + type: text + title: Response Code + description: The HTTP response code returned upon success. Should be in the 2XX range. + required: false + show_user: false + - name: response_body + type: text + title: Response Body + description: The response body returned upon success. Should be a single line JSON string. + required: false + show_user: false + - name: include_headers + type: text + title: Include Headers + description: This options specifies a list of HTTP headers that should be copied from the incoming request and included in the document. All configured headers will always be canonicalized to match the headers of the incoming request. For example, ["content-type"] will become ["Content-Type"] when the filebeat is running. + multi: true + required: false + show_user: false + - name: ssl + type: yaml + title: TLS + description: Options for enabling TLS for the listening webhook endpoint. See the [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/configuration-ssl.html) for a list of all options. + multi: false + required: false + show_user: false + default: | + enabled: false + certificate: "/etc/pki/client/cert.pem" + key: "/etc/pki/client/cert.key" + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: | + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + - name: tags + type: text + title: Tags + description: Tags to include in the published event + required: false + default: + - forwarded + multi: true + show_user: true icons: - src: "/img/icon.svg" type: "image/svg+xml" diff --git a/packages/http_endpoint/data_stream/generic/sample_event.json b/packages/http_endpoint/sample_event.json similarity index 50% rename from packages/http_endpoint/data_stream/generic/sample_event.json rename to packages/http_endpoint/sample_event.json index dbe815e5acb..a5e5c3ebf27 100644 --- a/packages/http_endpoint/data_stream/generic/sample_event.json +++ b/packages/http_endpoint/sample_event.json @@ -1,35 +1,36 @@ { - "@timestamp": "2022-02-21T20:47:45.698Z", + "@timestamp": "2024-04-30T07:17:15.883Z", "agent": { - "ephemeral_id": "1b28b196-5728-4dcb-aba9-13d6c4182de9", - "id": "d5d45ca4-6db0-4931-bf28-7de9e83c2223", + "ephemeral_id": "96009d3d-576a-4f1b-b10d-0c17a0fc276d", + "id": "11a5b254-bd1f-402d-9d5c-593cbebda407", "name": "docker-fleet-agent", "type": "filebeat", - "version": "8.0.0" + "version": "8.12.1" }, "data_stream": { - "dataset": "http_endpoint.generic", + "dataset": "http_endpoint.http_endpoint", "namespace": "ep", "type": "logs" }, "ecs": { - "version": "8.11.0" + "version": "8.0.0" }, "elastic_agent": { - "id": "d5d45ca4-6db0-4931-bf28-7de9e83c2223", + "id": "11a5b254-bd1f-402d-9d5c-593cbebda407", "snapshot": false, - "version": "8.0.0" + "version": "8.12.1" }, "event": { "agent_id_status": "verified", - "dataset": "http_endpoint.generic", - "ingested": "2022-02-21T20:47:46Z", + "dataset": "http_endpoint.http_endpoint", + "ingested": "2024-04-30T07:17:25Z", "original": "message" }, "input": { "type": "http_endpoint" }, "tags": [ + "preserve_original_event", "forwarded" ] -} +} \ No newline at end of file