From b389eb7d48b6494c80dcf8581edfc1698b382ce8 Mon Sep 17 00:00:00 2001 From: dblock Date: Wed, 1 Feb 2023 17:32:47 -0500 Subject: [PATCH] Added support for Amazon OpenSearch Serverless. Signed-off-by: dblock --- CHANGELOG.md | 1 + USER_GUIDE.md | 8 +-- opensearch/examples/cat_indices.rs | 4 +- opensearch/src/http/aws_auth.rs | 21 +++++--- opensearch/src/http/transport.rs | 28 ++++++++-- opensearch/src/lib.rs | 11 ++-- opensearch/tests/auth.rs | 2 - opensearch/tests/aws_auth.rs | 86 ++++++++++++++++++++++++++++++ opensearch/tests/cert.rs | 5 +- opensearch/tests/common/client.rs | 4 +- opensearch/tests/error.rs | 2 +- yaml_test_runner/Cargo.toml | 2 +- 12 files changed, 148 insertions(+), 26 deletions(-) create mode 100644 opensearch/tests/aws_auth.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index c6be0de3..91e61fd6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) ### Added - Adds Github workflow for changelog verification ([#89](https://github.com/opensearch-project/opensearch-rs/pull/89)) - Adds Github workflow for unit tests ([#112](https://github.com/opensearch-project/opensearch-rs/pull/112)) +- Adds support for OpenSearch Serverless ([#96](https://github.com/opensearch-project/opensearch-rs/pull/96)) ### Dependencies - Bumps `simple_logger` from 2.3.0 to 4.0.0 diff --git a/USER_GUIDE.md b/USER_GUIDE.md index f2864a9a..e6820848 100644 --- a/USER_GUIDE.md +++ b/USER_GUIDE.md @@ -6,7 +6,7 @@ - [Add a Document to the Index](#add-a-document-to-the-index) - [Search for a Document](#search-for-a-document) - [Delete the Index](#delete-the-index) - - [Amazon OpenSearch Service](#amazon-opensearch-service) + - [Amazon OpenSearch and OpenSearch Serverless](#amazon-opensearch-and-opensearch-serverless) - [Create a Client](#create-a-client-1) # User Guide @@ -103,9 +103,9 @@ client .await?; ``` -## Amazon OpenSearch Service +## Amazon OpenSearch and OpenSearch Serverless -This library supports [Amazon OpenSearch Service](https://aws.amazon.com/opensearch-service/). +This library supports [Amazon OpenSearch Service](https://aws.amazon.com/opensearch-service/) and [OpenSearch Serverless](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless.html). ### Create a Client @@ -113,11 +113,13 @@ Create a client with AWS credentials as follows. Make sure to specify the correc ```rust let url = Url::parse("https://..."); +let service_name = "es"; // use "aoss" for OpenSearch Serverless let conn_pool = SingleNodeConnectionPool::new(url?); let region_provider = RegionProviderChain::default_provider().or_else("us-east-1"); let aws_config = aws_config::from_env().region(region_provider).load().await.clone(); let transport = TransportBuilder::new(conn_pool) .auth(aws_config.clone().try_into()?) + .service_name(service_name) .build()?; let client = OpenSearch::new(transport); ``` diff --git a/opensearch/examples/cat_indices.rs b/opensearch/examples/cat_indices.rs index 3551704d..761472ae 100644 --- a/opensearch/examples/cat_indices.rs +++ b/opensearch/examples/cat_indices.rs @@ -65,7 +65,9 @@ fn create_client() -> Result { /// Determines if Fiddler.exe proxy process is running fn running_proxy() -> bool { - let system = System::new_with_specifics(RefreshKind::new().with_processes(ProcessRefreshKind::default())); + let system = System::new_with_specifics( + RefreshKind::new().with_processes(ProcessRefreshKind::default()), + ); let has_fiddler = system.processes_by_name("Fiddler").next().is_some(); has_fiddler } diff --git a/opensearch/src/http/aws_auth.rs b/opensearch/src/http/aws_auth.rs index 6977327b..ffa33c76 100644 --- a/opensearch/src/http/aws_auth.rs +++ b/opensearch/src/http/aws_auth.rs @@ -12,29 +12,33 @@ use std::time::SystemTime; use aws_credential_types::{ - Credentials, - provider::{ProvideCredentials, SharedCredentialsProvider} + provider::{ProvideCredentials, SharedCredentialsProvider}, + Credentials, }; use aws_sigv4::{ - http_request::{sign, SignableBody, SignableRequest, SigningParams, SigningSettings}, + http_request::{ + sign, PayloadChecksumKind, SignableBody, SignableRequest, SigningParams, SigningSettings, + }, signing_params::BuildError, }; use aws_types::region::Region; use reqwest::Request; -const SERVICE_NAME: &str = "es"; - fn get_signing_params<'a>( credentials: &'a Credentials, + service_name: &'a str, region: &'a Region, ) -> Result, BuildError> { + let mut signing_settings = SigningSettings::default(); + signing_settings.payload_checksum_kind = PayloadChecksumKind::XAmzSha256; // required for OpenSearch Serverless + let mut builder = SigningParams::builder() .access_key(credentials.access_key_id()) .secret_key(credentials.secret_access_key()) - .service_name(SERVICE_NAME) + .service_name(service_name) .region(region.as_ref()) .time(SystemTime::now()) - .settings(SigningSettings::default()); + .settings(signing_settings); builder.set_security_token(credentials.session_token()); @@ -44,11 +48,12 @@ fn get_signing_params<'a>( pub async fn sign_request( request: &mut Request, credentials_provider: &SharedCredentialsProvider, + service_name: &str, region: &Region, ) -> Result<(), Box> { let credentials = credentials_provider.provide_credentials().await?; - let params = get_signing_params(&credentials, region)?; + let params = get_signing_params(&credentials, service_name, region)?; let uri = request.url().as_str().parse()?; diff --git a/opensearch/src/http/transport.rs b/opensearch/src/http/transport.rs index 406e34ba..f70c6eb1 100644 --- a/opensearch/src/http/transport.rs +++ b/opensearch/src/http/transport.rs @@ -154,6 +154,8 @@ pub struct TransportBuilder { disable_proxy: bool, headers: HeaderMap, timeout: Option, + #[cfg(feature = "aws-auth")] + service_name: String, } impl TransportBuilder { @@ -174,6 +176,8 @@ impl TransportBuilder { disable_proxy: false, headers: HeaderMap::new(), timeout: None, + #[cfg(feature = "aws-auth")] + service_name: "es".to_string(), } } @@ -241,6 +245,15 @@ impl TransportBuilder { self } + /// Sets a global AWS service name. + /// + /// Default is "es". Other supported services are "aoss" for OpenSearch Serverless. + #[cfg(feature = "aws-auth")] + pub fn service_name(mut self, service_name: &str) -> Self { + self.service_name = service_name.to_string(); + self + } + /// Builds a [Transport] to use to send API calls to Elasticsearch. pub fn build(self) -> Result { let mut client_builder = self.client_builder; @@ -313,6 +326,8 @@ impl TransportBuilder { client, conn_pool: self.conn_pool, credentials: self.credentials, + #[cfg(feature = "aws-auth")] + service_name: self.service_name, }) } } @@ -352,6 +367,8 @@ pub struct Transport { client: reqwest::Client, credentials: Option, conn_pool: Box, + #[cfg(feature = "aws-auth")] + service_name: String, } impl Transport { @@ -460,9 +477,14 @@ impl Transport { #[cfg(feature = "aws-auth")] if let Some(Credentials::AwsSigV4(credentials_provider, region)) = &self.credentials { - super::aws_auth::sign_request(&mut request, credentials_provider, region) - .await - .map_err(|e| crate::error::lib(format!("AWSV4 Signing Failed: {}", e)))?; + super::aws_auth::sign_request( + &mut request, + credentials_provider, + &self.service_name, + region, + ) + .await + .map_err(|e| crate::error::lib(format!("AWSV4 Signing Failed: {}", e)))?; } let response = self.client.execute(request).await; diff --git a/opensearch/src/lib.rs b/opensearch/src/lib.rs index 0cd3abec..4e858eaa 100644 --- a/opensearch/src/lib.rs +++ b/opensearch/src/lib.rs @@ -58,7 +58,7 @@ //! - **experimental-apis**: Enables experimental APIs. Experimental APIs are just that - an experiment. An experimental //! API might have breaking changes in any future version, or it might even be removed entirely. This feature also //! enables `beta-apis`. -//! - **aws-auth**: Enables authentication with Amazon OpenSearch. +//! - **aws-auth**: Enables authentication with Amazon OpenSearch and OpenSearch Serverless. //! Performs AWS SigV4 signing using credential types from `aws-types`. //! //! # Getting started @@ -327,9 +327,9 @@ //! # } //! ``` //! -//! ## Amazon OpenSearch +//! ## Amazon OpenSearch and OpenSearch Serverless //! -//! For authenticating against an Amazon OpenSearch endpoint using AWS SigV4 request signing, +//! For authenticating against an Amazon OpenSearch or OpenSearch Serverless endpoint using AWS SigV4 request signing, //! you must enable the `aws-auth` feature, then pass the AWS credentials to the [TransportBuilder](http::transport::TransportBuilder). //! The easiest way to retrieve AWS credentials in the required format is to use [aws-config](https://docs.rs/aws-config/latest/aws_config/). //! @@ -349,14 +349,15 @@ //! # use std::convert::TryInto; //! # #[tokio::main] //! # async fn main() -> Result<(), Box> { +//! # #[cfg(feature = "aws-auth")] { //! let creds = aws_config::load_from_env().await; -//! let url = Url::parse("https://example.com")?; +//! let url = Url::parse("https://...")?; //! let region_provider = RegionProviderChain::default_provider().or_else("us-east-1"); //! let aws_config = aws_config::from_env().region(region_provider).load().await.clone(); //! let conn_pool = SingleNodeConnectionPool::new(url); -//! # #[cfg(feature = "aws-auth")] { //! let transport = TransportBuilder::new(conn_pool) //! .auth(aws_config.clone().try_into()?) +//! .service_name("es") // use "aoss" for OpenSearch Serverless //! .build()?; //! let client = OpenSearch::new(transport); //! # } diff --git a/opensearch/tests/auth.rs b/opensearch/tests/auth.rs index d9a943f3..bda8371d 100644 --- a/opensearch/tests/auth.rs +++ b/opensearch/tests/auth.rs @@ -34,8 +34,6 @@ use common::*; use opensearch::auth::Credentials; use base64::{prelude::BASE64_STANDARD, write::EncoderWriter as Base64Encoder}; -// use std::fs::File; -// use std::io::Read; use std::io::Write; #[tokio::test] diff --git a/opensearch/tests/aws_auth.rs b/opensearch/tests/aws_auth.rs new file mode 100644 index 00000000..ed500c6b --- /dev/null +++ b/opensearch/tests/aws_auth.rs @@ -0,0 +1,86 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +#![cfg(feature = "aws-auth")] + +pub mod common; +use common::*; +use opensearch::OpenSearch; +use regex::Regex; + +use aws_config::SdkConfig; +use aws_credential_types::provider::SharedCredentialsProvider; +use aws_credential_types::Credentials; +use aws_types::region::Region; +use std::convert::TryInto; + +#[tokio::test] +async fn aws_auth_get() -> Result<(), failure::Error> { + let server = server::http(move |req| async move { + let authorization_header = req.headers()["authorization"].to_str().unwrap(); + let re = Regex::new(r"^AWS4-HMAC-SHA256 Credential=id/\d*/us-west-1/custom/aws4_request, SignedHeaders=accept;content-type;host;x-amz-content-sha256;x-amz-date, Signature=[a-f,0-9].*$").unwrap(); + assert!( + re.is_match(authorization_header), + "{}", + authorization_header + ); + let amz_content_sha256_header = req.headers()["x-amz-content-sha256"].to_str().unwrap(); + assert_eq!( + amz_content_sha256_header, + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + ); // SHA of empty string + http::Response::default() + }); + + let client = create_aws_client(format!("http://{}", server.addr()).as_ref())?; + let _response = client.ping().send().await?; + + Ok(()) +} + +#[tokio::test] +async fn aws_auth_post() -> Result<(), failure::Error> { + let server = server::http(move |req| async move { + let amz_content_sha256_header = req.headers()["x-amz-content-sha256"].to_str().unwrap(); + assert_eq!( + amz_content_sha256_header, + "f3a842f988a653a734ebe4e57c45f19293a002241a72f0b3abbff71e4f5297b9" + ); // SHA of the JSON + http::Response::default() + }); + + let client = create_aws_client(format!("http://{}", server.addr()).as_ref())?; + client + .index(opensearch::IndexParts::Index("movies")) + .body(serde_json::json!({ + "title": "Moneyball", + "director": "Bennett Miller", + "year": 2011 + } + )) + .send() + .await?; + + Ok(()) +} + +fn create_aws_client(addr: &str) -> Result { + let aws_creds = Credentials::new("id", "secret", None, None, "token"); + let creds_provider = SharedCredentialsProvider::new(aws_creds); + let aws_config = SdkConfig::builder() + .credentials_provider(creds_provider) + .region(Region::new("us-west-1")) + .build(); + let builder = client::create_builder(addr) + .auth(aws_config.clone().try_into()?) + .service_name("custom"); + Ok(client::create(builder)) +} diff --git a/opensearch/tests/cert.rs b/opensearch/tests/cert.rs index 04326f17..50af1162 100644 --- a/opensearch/tests/cert.rs +++ b/opensearch/tests/cert.rs @@ -21,7 +21,10 @@ //! //! //! DETACH=true .ci/run-opensearch.sh -#![cfg(all(feature = "_integration", any(feature = "native-tls", feature = "rustls-tls")))] +#![cfg(all( + feature = "_integration", + any(feature = "native-tls", feature = "rustls-tls") +))] pub mod common; use common::*; diff --git a/opensearch/tests/common/client.rs b/opensearch/tests/common/client.rs index d6e86c00..f974bf8e 100644 --- a/opensearch/tests/common/client.rs +++ b/opensearch/tests/common/client.rs @@ -56,7 +56,9 @@ pub fn cluster_addr() -> String { /// Checks if Fiddler proxy process is running fn running_proxy() -> bool { - let system = System::new_with_specifics(RefreshKind::new().with_processes(ProcessRefreshKind::default())); + let system = System::new_with_specifics( + RefreshKind::new().with_processes(ProcessRefreshKind::default()), + ); let has_fiddler = system.processes_by_name("Fiddler").next().is_some(); has_fiddler } diff --git a/opensearch/tests/error.rs b/opensearch/tests/error.rs index c692196b..ebb73cf2 100644 --- a/opensearch/tests/error.rs +++ b/opensearch/tests/error.rs @@ -28,7 +28,7 @@ * GitHub history for details. */ - #![cfg(feature = "_integration")] +#![cfg(feature = "_integration")] pub mod common; use common::*; diff --git a/yaml_test_runner/Cargo.toml b/yaml_test_runner/Cargo.toml index de1bc8e3..2c23f984 100644 --- a/yaml_test_runner/Cargo.toml +++ b/yaml_test_runner/Cargo.toml @@ -31,7 +31,7 @@ serde_yaml = "0.9" serde_json = { version = "~1", features = ["arbitrary_precision"] } simple_logger = "4.0.0" syn = { version = "~1.0", features = ["full"] } -sysinfo = "0.26.4" +sysinfo = "0.26" url = "2.1.1" yaml-rust = "0.4.3" tar = "~0.4"