Add extra headers in rest config and move rest parameters into rest c…

…onfig section (#4198) * append compatible elastic header * code format * Review PR on es header * Update config with additional headers. (#4221) * Update config with additional headers. * Move rest_listen_port into rest config. * Update docs/configuration/node-config.md Co-authored-by: Adrien Guillo <[email protected]> * Update docs/configuration/node-config.md Co-authored-by: Adrien Guillo <[email protected]> * Update docs/configuration/node-config.md Co-authored-by: Adrien Guillo <[email protected]> * Clean. --------- Co-authored-by: Adrien Guillo <[email protected]> * Please clippy. * Apply suggestions from code review Co-authored-by: Adrien Guillo <[email protected]> * Log extra headers if any and take review's comments into account. --------- Co-authored-by: fmassot <[email protected]> Co-authored-by: Adrien Guillo <[email protected]>
quickwit-oss · Dec 4, 2023 · 1faf703 · 1faf703
1 parent 73fde83
commit 1faf703
Show file tree

Hide file tree

Showing 22 changed files with 419 additions and 129 deletions.
diff --git a/config/quickwit.yaml b/config/quickwit.yaml
@@ -32,7 +32,14 @@ version: 0.6
 #   2. pass `0.0.0.0` and let Quickwit do its best to discover the node's IP (see `advertise_address`)
 #
 # listen_address: 127.0.0.1
-# rest_listen_port: 7280
+#
+# rest:
+#   listen_port: 7280
+#   cors_allow_origins:
+#     - "http://localhost:3000"
+#   extra_headers:
+#     x-header-1: header-value-1
+#     x-header-2: header-value-2
 #
 # IP address advertised by the node, i.e. the IP address that peer nodes should use to connect to the node for RPCs.
 # The environment variable `QW_ADVERTISE_ADDRESS` can also be used to override this value.

diff --git a/config/tutorials/hdfs-logs/searcher-1.yaml b/config/tutorials/hdfs-logs/searcher-1.yaml
@@ -1,7 +1,11 @@
 version: 0.6
 node_id: searcher-1
 listen_address: 127.0.0.1
-rest_listen_port: 7280
+rest:
+  listen_port: 7280
+ingest_api:
+  max_queue_memory_usage: 4GiB
+  max_queue_disk_usage: 8GiB
 peer_seeds:
   - 127.0.0.1:7290 # searcher-2
   - 127.0.0.1:7300 # searcher-3
diff --git a/config/tutorials/hdfs-logs/searcher-2.yaml b/config/tutorials/hdfs-logs/searcher-2.yaml
@@ -1,7 +1,8 @@
 version: 0.6
 node_id: searcher-2
 listen_address: 127.0.0.1
-rest_listen_port: 7290
+rest:
+  listen_port: 7290
 peer_seeds:
   - 127.0.0.1:7280 # searcher-1
   - 127.0.0.1:7300 # searcher-3
diff --git a/config/tutorials/hdfs-logs/searcher-3.yaml b/config/tutorials/hdfs-logs/searcher-3.yaml
@@ -1,7 +1,8 @@
 version: 0.6
 node_id: searcher-3
 listen_address: 127.0.0.1
-rest_listen_port: 7300
+rest:
+  listen_port: 7300
 peer_seeds:
   - 127.0.0.1:7280 # searcher-1
   - 127.0.0.1:7290 # searcher-2

diff --git a/docs/configuration/node-config.md b/docs/configuration/node-config.md
@@ -25,14 +25,48 @@ A commented example is available here: [quickwit.yaml](https://github.com/quickw
 | `enabled_services` | Enabled services (control_plane, indexer, janitor, metastore, searcher) | `QW_ENABLED_SERVICES` | all services |
 | `listen_address` | The IP address or hostname that Quickwit service binds to for starting REST and GRPC server and connecting this node to other nodes. By default, Quickwit binds itself to 127.0.0.1 (localhost). This default is not valid when trying to form a cluster. | `QW_LISTEN_ADDRESS` | `127.0.0.1` |
 | `advertise_address` | IP address advertised by the node, i.e. the IP address that peer nodes should use to connect to the node for RPCs. | `QW_ADVERTISE_ADDRESS` | `listen_address` |
-| `rest_listen_port` | The port which to listen for HTTP REST API. | `QW_REST_LISTEN_PORT` | `7280` |
-| `gossip_listen_port` | The port which to listen for the Gossip cluster membership service (UDP). | `QW_GOSSIP_LISTEN_PORT` | `rest_listen_port` |
-| `grpc_listen_port` | The port which to listen for the gRPC service.| `QW_GRPC_LISTEN_PORT` | `rest_listen_port + 1` |
+| `gossip_listen_port` | The port which to listen for the Gossip cluster membership service (UDP). | `QW_GOSSIP_LISTEN_PORT` | `rest.listen_port` |
+| `grpc_listen_port` | The port on which gRPC services listen for traffic. | `QW_GRPC_LISTEN_PORT` | `rest.listen_port + 1` |
 | `peer_seeds` | List of IP addresses or hostnames used to bootstrap the cluster and discover the complete set of nodes. This list may contain the current node address and does not need to be exhaustive. | `QW_PEER_SEEDS` | |
 | `data_dir` | Path to directory where data (tmp data, splits kept for caching purpose) is persisted. This is mostly used in indexing. | `QW_DATA_DIR` | `./qwdata` |
 | `metastore_uri` | Metastore URI. Can be a local directory or `s3://my-bucket/indexes` or `postgres://username:password@localhost:5432/metastore`. [Learn more about the metastore configuration](metastore-config.md). | `QW_METASTORE_URI` | `{data_dir}/indexes` |
 | `default_index_root_uri` | Default index root URI that defines the location where index data (splits) is stored. The index URI is built following the scheme: `{default_index_root_uri}/{index-id}` | `QW_DEFAULT_INDEX_ROOT_URI` | `{data_dir}/indexes` |
-| `rest_cors_allow_origins` | Configure the CORS origins which are allowed to access the API. [Read more](#configuring-cors-cross-origin-resource-sharing) | |
+
+## REST configuration
+
+This section contains the REST API configuration options.
+
+| Property | Description | Env variable | Default value |
+| --- | --- | --- | --- |
+| `listen_port` | The port on which the REST API listens for HTTP traffic. | `QW_REST_LISTEN_PORT` | `7280` |
+| `cors_allow_origins` | Configure the CORS origins which are allowed to access the API. [Read more](#configuring-cors-cross-origin-resource-sharing) | |
+| `extra_headers` | List of header names and values | | |
+
+### Configuring CORS (Cross-origin resource sharing)
+
+CORS (Cross-origin resource sharing) describes which address or origins can access the REST API from the browser.
+By default, sharing resources cross-origin is not allowed.
+
+A wildcard, single origin, or multiple origins can be specified as part of the `cors_allow_origins` parameter:
+
+
+Example of a REST configuration:
+
+```yaml
+
+rest:
+  listen_port: 1789
+  extra_headers:
+    x-header-1: header-value-1
+    x-header-2: header-value-2
+  cors_allow_origins: '*'
+
+#   cors_allow_origins: https://my-hdfs-logs.domain.com   # Optionally we can specify one domain
+#   cors_allow_origins:                                   # Or allow multiple origins
+#     - https://my-hdfs-logs.domain.com
+#     - https://my-hdfs.other-domain.com
+
+```
 
 ## Storage configuration
 
@@ -203,7 +237,8 @@ version: 0.6
 cluster_id: quickwit-cluster
 node_id: my-unique-node-id
 listen_address: ${QW_LISTEN_ADDRESS}
-rest_listen_port: ${QW_LISTEN_PORT:-1111}
+rest:
+  listen_port: ${QW_LISTEN_PORT:-1111}
 ```
 
 Will be interpreted by Quickwit as:
@@ -213,23 +248,6 @@ version: 0.6
 cluster_id: quickwit-cluster
 node_id: my-unique-node-id
 listen_address: 0.0.0.0
-rest_listen_port: 1111
-```
-
-## Configuring CORS (Cross-origin resource sharing)
-
-CORS (Cross-origin resource sharing) describes which address or origins can access the REST API from the browser.
-By default, sharing resources cross-origin is not allowed.
-
-A wildcard, single origin, or multiple origins can be specified as part of the `rest_cors_allow_origins` parameter:
-
-```yaml
-version: 0.6
-index_id: hdfs
-
-rest_cors_allow_origins: '*'                                 # Allow all origins
-# rest_cors_allow_origins: https://my-hdfs-logs.domain.com   # Optionally we can specify one domain
-# rest_cors_allow_origins:                                   # Or allow multiple origins
-#   - https://my-hdfs-logs.domain.com
-#   - https://my-hdfs.other-domain.com
+rest:
+  listen_port: 1111
 ```
diff --git a/docs/configuration/ports-config.md b/docs/configuration/ports-config.md
@@ -4,18 +4,18 @@ sidebar_position: 6
 ---
 
 When starting a quickwit search server, one important parameter that can be configured is
-the `rest_listen_port` (defaults to :7280).
+the `rest.listen_port` (defaults to :7280).
 
 Internally, Quickwit will, in fact, use three sockets. The ports of these three sockets
 cannot be configured independently at the moment.
-The ports used are computed relative to the `rest_listen_port` port, as follows.
+The ports used are computed relative to the `rest.listen_port` port, as follows.
 
 
 | Service                       | Port used                 | Protocol |  Default  |
 |-------------------------------|---------------------------|----------|-----------|
-| Http server with the rest api | `${rest_listen_port}`     |   TCP    | 7280      |
-| Cluster membership            | `${rest_listen_port}`     |   UDP    | 7280      |
-| GRPC service                  | `${rest_listen_port} + 1` |   TCP    | 7281      |
+| Http server with the rest api | `${rest.listen_port}`     |   TCP    | 7280      |
+| Cluster membership            | `${rest.listen_port}`     |   UDP    | 7280      |
+| GRPC service                  | `${rest.listen_port} + 1` |   TCP    | 7281      |
 
 It is not possible for the moment to configure these ports independently.
 

diff --git a/docs/get-started/tutorials/tutorial-hdfs-logs-distributed-search-aws-s3.md b/docs/get-started/tutorials/tutorial-hdfs-logs-distributed-search-aws-s3.md
@@ -197,8 +197,8 @@ Now that we have indexed the logs and can search from one instance, it's time to
 
 ## Start two more instances
 
-Quickwit needs a port `rest_listen_port` for serving the HTTP rest API via TCP as well as maintaining the cluster formation via UDP. 
-Also, it needs `{rest_listen_port} + 1` for gRPC communication between instances.
+Quickwit needs a port `rest.listen_port` for serving the HTTP rest API via TCP as well as maintaining the cluster formation via UDP. 
+Also, it needs `{rest.listen_port} + 1` for gRPC communication between instances.
 
 In AWS, you can create a security group to group these inbound rules. Check out the [network section](../../guides/aws-setup) of our AWS setup guide.
 

diff --git a/docs/guides/cluster-with-docker-compose.md b/docs/guides/cluster-with-docker-compose.md
diff --git a/quickwit/Cargo.lock b/quickwit/Cargo.lock
diff --git a/quickwit/quickwit-cli/tests/helpers.rs b/quickwit/quickwit-cli/tests/helpers.rs
@@ -84,7 +84,8 @@ const DEFAULT_QUICKWIT_CONFIG: &str = r#"
     version: 0.6
     metastore_uri: #metastore_uri
     data_dir: #data_dir
-    rest_listen_port: #rest_listen_port
+    rest:
+        listen_port: #rest_listen_port
     grpc_listen_port: #grpc_listen_port
 "#;
 

diff --git a/quickwit/quickwit-config/Cargo.toml b/quickwit/quickwit-config/Cargo.toml
@@ -16,6 +16,8 @@ bytesize = { workspace = true }
 chrono = { workspace = true }
 cron = { workspace = true }
 enum-iterator = { workspace = true }
+http = { workspace = true }
+http-serde = { workspace = true }
 humantime = { workspace = true }
 itertools = { workspace = true }
 json_comments = { workspace = true }

diff --git a/quickwit/quickwit-config/resources/tests/config/quickwit.json b/quickwit/quickwit-config/resources/tests/config/quickwit.json
@@ -9,7 +9,6 @@
     ],
     "listen_address": "0.0.0.0",
     "advertise_address": "172.0.0.12",
-    "rest_listen_port": 1111,
     "gossip_listen_port": 2222,
     "grpc_listen_port": 3333,
     "peer_seeds": [
@@ -19,6 +18,13 @@
     "data_dir": "/opt/quickwit/data",
     "metastore_uri": "postgres://username:password@host:port/db",
     "default_index_root_uri": "s3://quickwit-indexes",
+    "rest": {
+        "listen_port": 1111,
+        "extra_headers": {
+            "x-header-1": "header-value-1",
+            "x-header-2": "header-value-2"
+        }
+    },
     "storage": {
         "azure": {
             "account": "quickwit-dev"

diff --git a/quickwit/quickwit-config/resources/tests/config/quickwit.toml b/quickwit/quickwit-config/resources/tests/config/quickwit.toml
@@ -5,14 +5,20 @@ node_id = "my-unique-node-id"
 enabled_services = [ "janitor", "metastore" ]
 listen_address = "0.0.0.0"
 advertise_address = "172.0.0.12"
-rest_listen_port = 1111
 gossip_listen_port = 2222
 grpc_listen_port = 3333
 peer_seeds = [ "quickwit-searcher-0.local", "quickwit-searcher-1.local" ]
 data_dir = "/opt/quickwit/data"
 metastore_uri = "postgres://username:password@host:port/db"
 default_index_root_uri = "s3://quickwit-indexes"
 
+[rest]
+listen_port = 1111
+
+[rest.extra_headers]
+x-header-1 = "header-value-1"
+x-header-2 = "header-value-2"
+
 [storage.azure]
 account = "quickwit-dev"
 

diff --git a/quickwit/quickwit-config/resources/tests/config/quickwit.yaml b/quickwit/quickwit-config/resources/tests/config/quickwit.yaml
@@ -7,7 +7,6 @@ enabled_services:
   - metastore
 listen_address: 0.0.0.0
 advertise_address: 172.0.0.12
-rest_listen_port: 1111
 gossip_listen_port: 2222
 grpc_listen_port: 3333
 peer_seeds:
@@ -17,6 +16,12 @@ data_dir: /opt/quickwit/data
 metastore_uri: postgres://username:password@host:port/db
 default_index_root_uri: s3://quickwit-indexes
 
+rest:
+  listen_port: 1111
+  extra_headers:
+    x-header-1: header-value-1
+    x-header-2: header-value-2
+
 storage:
   azure:
     account: quickwit-dev

diff --git a/quickwit/quickwit-config/src/node_config/mod.rs b/quickwit/quickwit-config/src/node_config/mod.rs
@@ -28,6 +28,7 @@ use std::time::Duration;
 
 use anyhow::{bail, ensure};
 use bytesize::ByteSize;
+use http::HeaderMap;
 use quickwit_common::net::HostAddr;
 use quickwit_common::uri::Uri;
 use quickwit_proto::indexing::CpuCapacity;
@@ -41,6 +42,15 @@ use crate::{ConfigFormat, MetastoreConfigs};
 
 pub const DEFAULT_QW_CONFIG_PATH: &str = "config/quickwit.yaml";
 
+#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct RestConfig {
+    pub listen_addr: SocketAddr,
+    pub cors_allow_origins: Vec<String>,
+    #[serde(with = "http_serde::header_map")]
+    pub extra_headers: HeaderMap,
+}
+
 #[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
 #[serde(deny_unknown_fields)]
 pub struct IndexerConfig {
@@ -304,7 +314,6 @@ pub struct NodeConfig {
     pub cluster_id: String,
     pub node_id: String,
     pub enabled_services: HashSet<QuickwitService>,
-    pub rest_listen_addr: SocketAddr,
     pub gossip_listen_addr: SocketAddr,
     pub grpc_listen_addr: SocketAddr,
     pub gossip_advertise_addr: SocketAddr,
@@ -313,7 +322,7 @@ pub struct NodeConfig {
     pub data_dir_path: PathBuf,
     pub metastore_uri: Uri,
     pub default_index_root_uri: Uri,
-    pub rest_cors_allow_origins: Vec<String>,
+    pub rest_config: RestConfig,
     pub storage_configs: StorageConfigs,
     pub metastore_configs: MetastoreConfigs,
     pub indexer_config: IndexerConfig,