From ca0d2878560c2ec2bd12facb3379d975a9e416b1 Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Fri, 16 Jun 2023 14:51:12 -0400 Subject: [PATCH] Document testing of discovery-ec2 and repository-s3. (#7868) Signed-off-by: dblock Signed-off-by: Rishab Nahata --- plugins/discovery-ec2/README.md | 94 +++++++++++++++++++ .../discovery/ec2/Ec2NameResolver.java | 2 +- plugins/repository-s3/README.md | 25 +++++ 3 files changed, 120 insertions(+), 1 deletion(-) create mode 100644 plugins/discovery-ec2/README.md create mode 100644 plugins/repository-s3/README.md diff --git a/plugins/discovery-ec2/README.md b/plugins/discovery-ec2/README.md new file mode 100644 index 0000000000000..6daeb6305b988 --- /dev/null +++ b/plugins/discovery-ec2/README.md @@ -0,0 +1,94 @@ +# discovery-ec2 + +The discovery-ec2 plugin allows OpenSearch to find the master-eligible nodes in a cluster running on AWS EC2 by querying the AWS Instance Metadata Service ([IMDS](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html)) API for the addresses of the EC2 instances running these nodes. + +This is typically configured as follows in `opensearch.yml`. + +```yml +cluster.initial_cluster_manager_nodes: ["seed"] +discovery.seed_providers: ec2 +discovery.ec2.tag.role: my-cluster-manager +``` + +The implementation in the discovery-ec2 plugin queries for instances of a given type (e.g. private or public IP), with a tag, or running in a specific availability zone. + +## Testing + +### Unit and Integration Tests + +``` +./gradlew :plugins:discovery-ec2:check +``` + +### Manual Testing + +The following instructions manually exercise the plugin. + +Setup a 5-node (Ubuntu) cluster on EC2, with 3 of them tagged with `role: my-cluster-manager`, and a custom TCP rule to expose ports 9200 to 9300 to allow TCP traffic. Default EC2 configuration only allows SSH TCP traffic and hence only exposes port 22. + +Set the following properties. + +- `network.host`: Set this to the IP of the eth0 network adapter, can be fetched by `ipconfig` or by running `hostname -I` on Linux hosts. +- `discovery.seed_hosts`: List of all nodes in the cluster. +- `cluster.initial_cluster_manager_nodes`: List of all initial cluster manager nodes. +- `discovery.seed_providers`: Set this to `ec2` for EC2 IMDS based discovery. +- `discovery.ec2.tag.role: my-cluster-manager`: Filter out only those nodes with value `my-cluster-manager` for the `role` tag. +- `discovery.ec2.region`: Optionally set to your region, e.g. `us-east-1`, by default the plugin uses the region of the current EC2 instance. + +While sending the request to IMDS, specified discovery settings for finding master-eligible nodes are being set. You will see the following in the logs. + +``` +[2023-05-31T15:22:39,274][DEBUG][o.o.d.e.AwsEc2SeedHostsProvider] [ip-172-31-73-184] using host_type [private_ip], tags [{role=[my-cluster-manager]}], groups [[]] with any_group [true], availability_zones [[]] +``` + +The nodes getting added as eligible masters are ones with the role tag set to `my-cluster-manager`. + +``` +[2023-05-31T15:23:03,676][TRACE][o.o.d.e.AwsEc2SeedHostsProvider] [ip-172-31-73-184] finding seed nodes... +[2023-05-31T15:23:03,677][TRACE][o.o.d.e.AwsEc2SeedHostsProvider] [ip-172-31-73-184] adding i-01fa1736e8566c693, address 172.31.73.184, transport_address 172.31.73.184:9300 +[2023-05-31T15:23:03,677][TRACE][o.o.d.e.AwsEc2SeedHostsProvider] [ip-172-31-73-184] adding i-03d70a4521045cc3b, address 172.31.74.169, transport_address 172.31.74.169:9300 +[2023-05-31T15:23:03,677][TRACE][o.o.d.e.AwsEc2SeedHostsProvider] [ip-172-31-73-184] adding i-0c6ffdd10ebd3c2f1, address 172.31.74.156, transport_address 172.31.74.156:9300 +``` + +## Troubleshooting + +### Trace Level Logs + +Enable `TRACE`-level logging in `opensearch.yml` to see more output. + +```yml +logger.org.opensearch.discovery.ec2: trace +``` + +### Sample IMDS Query + +You may need to query IMDS without running OpenSearch. Use [this sample](https://github.com/dblock/aws-imds-sample) or copy the following code that makes a query to IMDS that looks for `running`, `pending` or `stopped` instances. + +```java +DefaultCredentialsProvider credentialsProviderChain = DefaultCredentialsProvider.create(); + +RetryPolicy retryPolicy = RetryPolicy.builder() + .numRetries(10) + .build(); + +Ec2Client client = Ec2Client.builder() + .httpClientBuilder(ApacheHttpClient.builder()) + .credentialsProvider(credentialsProviderChain) + .build(); + +DescribeInstancesRequest describeInstancesRequest = DescribeInstancesRequest.builder() + .filters( + Filter.builder() + .name("instance-state-name") + .values("running", "pending", "stopped") + .build() + ).build(); + +DescribeInstancesResponse describeInstancesResponse = client.describeInstances(describeInstancesRequest); +for (final Reservation reservation : describeInstancesResponse.reservations()) { + System.out.println(reservation.reservationId()); + for (final Instance instance : reservation.instances()) { + System.out.println("\t" + instance.publicDnsName()); + } +} +``` diff --git a/plugins/discovery-ec2/src/main/java/org/opensearch/discovery/ec2/Ec2NameResolver.java b/plugins/discovery-ec2/src/main/java/org/opensearch/discovery/ec2/Ec2NameResolver.java index 4a78572e2e18f..3f7a8e1e03353 100644 --- a/plugins/discovery-ec2/src/main/java/org/opensearch/discovery/ec2/Ec2NameResolver.java +++ b/plugins/discovery-ec2/src/main/java/org/opensearch/discovery/ec2/Ec2NameResolver.java @@ -117,7 +117,7 @@ public InetAddress[] resolve(Ec2HostnameType type) throws IOException { String metadataResult = urlReader.readLine(); if (metadataResult == null || metadataResult.length() == 0) { - throw new IOException("no gce metadata returned from [" + url + "] for [" + type.configName + "]"); + throw new IOException("no ec2 metadata returned from [" + url + "] for [" + type.configName + "]"); } logger.debug("obtained ec2 hostname from ec2 meta-data url {}: {}", url, metadataResult); // only one address: because we explicitly ask for only one via the Ec2HostnameType diff --git a/plugins/repository-s3/README.md b/plugins/repository-s3/README.md new file mode 100644 index 0000000000000..03007e03b633e --- /dev/null +++ b/plugins/repository-s3/README.md @@ -0,0 +1,25 @@ +# repository-s3 + +The repository-s3 plugin enables the use of S3 as a place to store snapshots. + +## Testing + +### Unit Tests + +``` +./gradlew :plugins:repository-s3:test +``` + +### Integration Tests + +Integration tests require several environment variables. + +- `amazon_s3_bucket`: Name of the S3 bucket to use. +- `amazon_s3_access_key`: The access key ID (`AWS_ACCESS_KEY_ID`) with r/w access to the S3 bucket. +- `amazon_s3_secret_key`: The secret access key (`AWS_SECRET_ACCESS_KEY`). +- `amazon_s3_base_path`: A relative path inside the S3 bucket, e.g. `opensearch`. +- `AWS_REGION`: The region in which the S3 bucket was created. While S3 buckets are global, credentials must scoped to a specific region and cross-region access is not allowed. (TODO: rename this to `amazon_s3_region` in https://github.com/opensearch-project/opensearch-build/issues/3615 and https://github.com/opensearch-project/OpenSearch/pull/7974.) + +``` +AWS_REGION=us-west-2 amazon_s3_access_key=$AWS_ACCESS_KEY_ID amazon_s3_secret_key=$AWS_SECRET_ACCESS_KEY amazon_s3_base_path=path amazon_s3_bucket=dblock-opensearch ./gradlew :plugins:repository-s3:s3ThirdPartyTest +```