diff --git a/core/trino-server/src/main/provisio/trino.xml b/core/trino-server/src/main/provisio/trino.xml index 1afed3b2b4fc..873be68f74b8 100644 --- a/core/trino-server/src/main/provisio/trino.xml +++ b/core/trino-server/src/main/provisio/trino.xml @@ -183,12 +183,6 @@ - - - - - - diff --git a/docs/release-template.md b/docs/release-template.md index a091f1a4c4fc..e6f58ac57231 100644 --- a/docs/release-template.md +++ b/docs/release-template.md @@ -44,8 +44,6 @@ ## Kafka connector -## Kinesis connector - ## Kudu connector ## MariaDB connector diff --git a/docs/src/main/sphinx/connector.md b/docs/src/main/sphinx/connector.md index 0ccb69b40fa7..d4ab8d924d29 100644 --- a/docs/src/main/sphinx/connector.md +++ b/docs/src/main/sphinx/connector.md @@ -23,7 +23,6 @@ Iceberg Ignite JMX Kafka -Kinesis Kudu MariaDB Memory diff --git a/docs/src/main/sphinx/connector/kinesis.md b/docs/src/main/sphinx/connector/kinesis.md deleted file mode 100644 index bd4640bb3934..000000000000 --- a/docs/src/main/sphinx/connector/kinesis.md +++ /dev/null @@ -1,274 +0,0 @@ -# Kinesis connector - -```{raw} html - -``` - -[Kinesis](https://aws.amazon.com/kinesis/) is Amazon's fully managed cloud-based service for real-time processing of large, distributed data streams. - -This connector allows the use of Kinesis streams as tables in Trino, such that each data-blob/message -in a Kinesis stream is presented as a row in Trino. A flexible table mapping approach lets us -treat fields of the messages as columns in the table. - -Under the hood, a Kinesis -[shard iterator](https://docs.aws.amazon.com/kinesis/latest/APIReference/API_GetShardIterator.html) -is used to retrieve the records, along with a series of -[GetRecords](https://docs.aws.amazon.com/kinesis/latest/APIReference/API_GetRecords.html) calls. -The shard iterator starts by default 24 hours before the current time, and works its way forward. -To be able to query a stream, table mappings are needed. These table definitions can be -stored on Amazon S3 (preferred), or stored in a local directory on each Trino node. - -This connector is a **read-only** connector. It can only fetch data from Kinesis streams, -but cannot create streams or push data into existing streams. - -To configure the Kinesis connector, create a catalog properties file -`etc/catalog/example.properties` with the following contents, replacing the -properties as appropriate: - -```text -connector.name=kinesis -kinesis.access-key=XXXXXX -kinesis.secret-key=XXXXXX -``` - -## Configuration properties - -The following configuration properties are available: - -| Property name | Description | -| -------------------------------------------- | --------------------------------------------------------------------- | -| `kinesis.access-key` | Access key to AWS account or blank to use default provider chain | -| `kinesis.secret-key` | Secret key to AWS account or blank to use default provider chain | -| `kinesis.aws-region` | AWS region to be used to read kinesis stream from | -| `kinesis.default-schema` | Default schema name for tables | -| `kinesis.table-description-location` | Directory containing table description files | -| `kinesis.table-description-refresh-interval` | How often to get the table description from S3 | -| `kinesis.hide-internal-columns` | Controls whether internal columns are part of the table schema or not | -| `kinesis.batch-size` | Maximum number of records to return in one batch | -| `kinesis.fetch-attempts` | Read attempts made when no records returned and not caught up | -| `kinesis.max-batches` | Maximum batches to read from Kinesis in one single query | -| `kinesis.sleep-time` | Time for thread to sleep waiting to make next attempt to fetch batch | -| `kinesis.iterator-from-timestamp` | Begin iterating from a given timestamp instead of the trim horizon | -| `kinesis.iterator-offset-seconds` | Number of seconds before current time to start iterating | - -### `kinesis.access-key` - -Defines the access key ID for AWS root account or IAM roles, which is used to sign programmatic requests to AWS Kinesis. - -This property is optional; if not defined, the connector tries to follow `Default-Credential-Provider-Chain` provided by AWS in the following order: - -- Environment Variable: Load credentials from environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`. -- Java System Variable: Load from java system as `aws.accessKeyId` and `aws.secretKey`. -- Profile Credentials File: Load from file typically located at `~/.aws/credentials`. -- Instance profile credentials: These credentials can be used on EC2 instances, and are delivered through the Amazon EC2 metadata service. - -### `kinesis.secret-key` - -Defines the secret key for AWS root account or IAM roles, which together with Access Key ID, is used to sign programmatic requests to AWS Kinesis. - -This property is optional; if not defined, connector will try to follow `Default-Credential-Provider-Chain` same as above. - -### `kinesis.aws-region` - -Defines AWS Kinesis regional endpoint. Selecting appropriate region may reduce latency in fetching data. - -This field is optional; The default region is `us-east-1` referring to end point 'kinesis.us-east-1.amazonaws.com'. - -See [Kinesis Data Streams regions](https://docs.aws.amazon.com/general/latest/gr/rande.html#ak_region) -for a current list of available regions. - -### `kinesis.default-schema` - -Defines the schema which contains all tables that were defined without a qualifying schema name. - -This property is optional; the default is `default`. - -### `kinesis.table-description-location` - -References an S3 URL or a folder within Trino deployment that holds one or more JSON files ending with `.json`, which contain table description files. -The S3 bucket and folder will be checked every 10 minutes for updates and changed files. - -This property is optional; the default is `etc/kinesis`. - -### `kinesis.table-description-refresh-interval` - -This property controls how often the table description is refreshed from S3. - -This property is optional; the default is `10m`. - -### `kinesis.batch-size` - -Defines the maximum number of records to return in one request to Kinesis Streams. Maximum limit is `10000` records. - -This field is optional; the default value is `10000`. - -### `kinesis.max-batches` - -The maximum number of batches to read in a single query. The default value is `1000`. - -### `kinesis.fetch-attempts` - -Defines the number of attempts made to read a batch from Kinesis Streams, when no records are returned and the *millis behind latest* -parameter shows we are not yet caught up. When records are returned no additional attempts are necessary. -`GetRecords` has been observed to return no records even though the shard is not empty. -That is why multiple attempts need to be made. - -This field is optional; the default value is `2`. - -### `kinesis.sleep-time` - -Defines the duration for which a thread needs to sleep between `kinesis.fetch-attempts` made to fetch data. - -This field is optional; the default value is `1000ms`. - -### `kinesis.iterator-from-timestamp` - -Use an initial shard iterator type of `AT_TIMESTAMP` starting `kinesis.iterator-offset-seconds` before the current time. -When this is false, an iterator type of `TRIM_HORIZON` is used, meaning it starts from the oldest record in the stream. - -The default is true. - -### `kinesis.iterator-offset-seconds` - -When `kinesis.iterator-from-timestamp` is true, the shard iterator starts at `kinesis.iterator-offset-seconds` before the current time. - -The default is `86400` seconds (24 hours). - -### `kinesis.hide-internal-columns` - -In addition to the data columns defined in a table description file, the connector maintains a number of additional columns for each table. -If these columns are hidden, they can still be used in queries, but they do not show up in `DESCRIBE ` or `SELECT *`. - -This property is optional; the default is true. - -## Internal columns - -For each defined table, the connector maintains the following columns: - -| Column name | Type | Description | -| -------------------- | ----------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `_shard_id` | `VARCHAR` | ID of the Kinesis stream shard which contains this row. | -| `_shard_sequence_id` | `VARCHAR` | Sequence id within the Kinesis shard for this row. | -| `_segment_start` | `BIGINT` | Lowest offset in the segment (inclusive) which contains this row. This offset is partition specific. | -| `_segment_end` | `BIGINT` | Highest offset in the segment (exclusive) which contains this row. The offset is partition specific. This is the same value as `_segment_start` of the next segment (if it exists). | -| `_segment_count` | `BIGINT` | Running count for the current row within the segment. For an uncompacted topic, `_segment_start + _segment_count` is equal to `_partition_offset`. | -| `_message_valid` | `BOOLEAN` | True if the decoder could decode the message successfully for this row. When false, data columns mapped from the message should be treated as invalid. | -| `_message` | `VARCHAR` | Message bytes as an UTF-8 encoded string. This is only useful for a text topic. | -| `_message_length` | `BIGINT` | Number of bytes in the message. | -| `_message_timestamp` | `TIMESTAMP` | Approximate arrival time of the message (milliseconds granularity). | -| `_key` | `VARCHAR` | Key bytes as an UTF-8 encoded string. This is only useful for textual keys. | -| `_partition_key` | `VARCHAR` | Partition Key bytes as a UTF-8 encoded string. | - -For tables without a table definition file, the `_message_valid` column is always `true`. - -## Table definition - -A table definition file consists of a JSON definition for a table, which corresponds to one stream in Kinesis. -The name of the file can be arbitrary but must end in `.json`. The structure of the table definition is as follows: - -```text -{ - "tableName": ..., - "schemaName": ..., - "streamName": ..., - "message": { - "dataFormat": ..., - "fields": [ - ... - ] - } - } -``` - -| Field | Required | Type | Description | -| ------------ | -------- | ----------- | ----------------------------------------------------------------------------- | -| `tableName` | required | string | Trino table name defined by this file. | -| `schemaName` | optional | string | Schema which contains the table. If omitted, the default schema name is used. | -| `streamName` | required | string | Name of the Kinesis Stream that is mapped | -| `message` | optional | JSON object | Field definitions for data columns mapped to the message itself. | - -Every message in a Kinesis stream can be decoded using the definition provided in the message object. -The JSON object message in the table definition contains two fields: - -| Field | Required | Type | Description | -| ------------ | -------- | ---------- | ------------------------------------------------------------------------------------------- | -| `dataFormat` | required | string | Selects the decoder for this group of fields. | -| `fields` | required | JSON array | A list of field definitions. Each field definition creates a new column in the Trino table. | - -Each field definition is a JSON object. At a minimum, a name, type, and mapping must be provided. -The overall structure looks like this: - -```text -{ - "name": ..., - "type": ..., - "dataFormat": ..., - "mapping": ..., - "formatHint": ..., - "hidden": ..., - "comment": ... -} -``` - -| Field | Required | Type | Description | -| ------------ | -------- | ------- | -------------------------------------------------------------------------------------------------------------------- | -| `name` | required | string | Name of the column in the Trino table. | -| `type` | required | string | Trino type of the column. | -| `dataFormat` | optional | string | Selects the column decoder for this field. Defaults to the default decoder for this row data format and column type. | -| `mapping` | optional | string | Mapping information for the column. This is decoder specific -- see below. | -| `formatHint` | optional | string | Sets a column specific format hint to the column decoder. | -| `hidden` | optional | boolean | Hides the column from `DESCRIBE ` and `SELECT *`. Defaults to `false`. | -| `comment` | optional | string | Adds a column comment which is shown with `DESCRIBE
`. | - -The name field is exposed to Trino as the column name, while the mapping field is the portion of the message that gets -mapped to that column. For JSON object messages, this refers to the field name of an object, and can be a path that drills -into the object structure of the message. Additionally, you can map a field of the JSON object to a string column type, -and if it is a more complex type (JSON array or JSON object) then the JSON itself becomes the field value. - -There is no limit on field descriptions for either key or message. - -(kinesis-type-mapping)= -## Type mapping - -Because Trino and Kinesis each support types that the other does not, this -connector {ref}`maps some types ` when reading data. Type -mapping depends on the RAW, CSV, JSON, and AVRO file formats. - -### Row decoding - -A decoder is used to map data to table columns. - -The connector contains the following decoders: - -- `raw`: Message is not interpreted; ranges of raw message bytes are mapped - to table columns. -- `csv`: Message is interpreted as comma separated message, and fields are - mapped to table columns. -- `json`: Message is parsed as JSON, and JSON fields are mapped to table - columns. -- `avro`: Message is parsed based on an Avro schema, and Avro fields are - mapped to table columns. - -:::{note} -If no table definition file exists for a table, the `dummy` decoder is -used, which does not expose any columns. -::: - -```{include} raw-decoder.fragment -``` - -```{include} csv-decoder.fragment -``` - -```{include} json-decoder.fragment -``` - -```{include} avro-decoder.fragment -``` - -(kinesis-sql-support)= -## SQL support - -The connector provides {ref}`globally available ` and -{ref}`read operation ` statements to access data and -metadata from Kinesis streams. diff --git a/docs/src/main/sphinx/release/release-320.md b/docs/src/main/sphinx/release/release-320.md index c57887a46036..ec13b501889b 100644 --- a/docs/src/main/sphinx/release/release-320.md +++ b/docs/src/main/sphinx/release/release-320.md @@ -10,7 +10,7 @@ - Avoid unnecessary evaluation of redundant filters. ({issue}`1516`) - Improve performance of certain window functions when using bounded window frames (e.g., `ROWS BETWEEN ... PRECEDING AND ... FOLLOWING`). ({issue}`464`) -- Add {doc}`/connector/kinesis`. ({issue}`476`) +- Add Kinesis connector. ({issue}`476`) - Add {func}`geometry_from_hadoop_shape`. ({issue}`1593`) - Add {func}`at_timezone`. ({issue}`1612`) - Add {func}`with_timezone`. ({issue}`1612`) diff --git a/docs/src/main/sphinx/static/img/kinesis.png b/docs/src/main/sphinx/static/img/kinesis.png deleted file mode 100644 index 7f5619032211..000000000000 Binary files a/docs/src/main/sphinx/static/img/kinesis.png and /dev/null differ diff --git a/plugin/trino-kinesis/pom.xml b/plugin/trino-kinesis/pom.xml deleted file mode 100644 index bfa7282d39a8..000000000000 --- a/plugin/trino-kinesis/pom.xml +++ /dev/null @@ -1,245 +0,0 @@ - - - 4.0.0 - - io.trino - trino-root - 464-SNAPSHOT - ../../pom.xml - - - trino-kinesis - trino-plugin - Trino - Kinesis connector - - - - com.amazonaws - amazon-kinesis-client - - - - com.amazonaws - aws-java-sdk-core - - - commons-logging - commons-logging - - - - - - com.amazonaws - aws-java-sdk-dynamodb - - - - com.amazonaws - aws-java-sdk-kinesis - - - - com.amazonaws - aws-java-sdk-s3 - ${dep.aws-sdk.version} - - - commons-logging - commons-logging - - - joda-time - joda-time - - - - - - com.google.guava - guava - - - - com.google.inject - guice - - - - io.airlift - bootstrap - - - - io.airlift - concurrent - - - - io.airlift - configuration - - - - io.airlift - json - - - - io.airlift - log - - - - io.airlift - units - - - - io.trino - trino-plugin-toolkit - - - - io.trino - trino-record-decoder - - - - jakarta.annotation - jakarta.annotation-api - - - - jakarta.validation - jakarta.validation-api - - - - com.fasterxml.jackson.core - jackson-annotations - provided - - - - io.airlift - slice - provided - - - - io.opentelemetry - opentelemetry-api - provided - - - - io.opentelemetry - opentelemetry-context - provided - - - - io.trino - trino-spi - provided - - - - org.openjdk.jol - jol-core - provided - - - - io.airlift - junit-extensions - test - - - - io.trino - trino-main - test - - - - io.trino - trino-main - test-jar - test - - - - io.trino - trino-testing - test - - - - org.assertj - assertj-core - test - - - - org.junit.jupiter - junit-jupiter-api - test - - - - org.junit.jupiter - junit-jupiter-engine - test - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - - - **/TestMinimalFunctionality.java - **/TestS3TableConfigClient.java - - - ACCESS-KEY - SECRET-KEY - s3://S3-LOC - - - - - org.basepom.maven - duplicate-finder-maven-plugin - - - - google/protobuf/.*\.proto$ - - - - - - - - - test-kinesis - - - - org.apache.maven.plugins - maven-surefire-plugin - - - - - - - - - diff --git a/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/KinesisClientManager.java b/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/KinesisClientManager.java deleted file mode 100644 index 591775001a1b..000000000000 --- a/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/KinesisClientManager.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.kinesis; - -import com.amazonaws.auth.BasicAWSCredentials; -import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; -import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient; -import com.amazonaws.services.kinesis.AmazonKinesisClient; -import com.amazonaws.services.s3.AmazonS3Client; -import com.google.inject.Inject; - -import static com.google.common.base.Strings.isNullOrEmpty; - -/** - * Creates and manages AWS clients for this connector. - *

- * Note: credentials can be supplied explicitly through the configuration. However when these are - * omitted, the default AWS provider chain is used (which includes instance profile credentials). - */ -public class KinesisClientManager - implements KinesisClientProvider -{ - private final AmazonKinesisClient client; - private final AmazonS3Client amazonS3Client; - private final AmazonDynamoDBClient dynamoDbClient; // for Checkpointing - - @Inject - public KinesisClientManager(KinesisConfig config) - { - if (!isNullOrEmpty(config.getAccessKey()) && !isNullOrEmpty(config.getSecretKey())) { - BasicAWSCredentials awsCredentials = new BasicAWSCredentials(config.getAccessKey(), config.getSecretKey()); - this.client = new AmazonKinesisClient(awsCredentials); - this.amazonS3Client = new AmazonS3Client(awsCredentials); - this.dynamoDbClient = new AmazonDynamoDBClient(awsCredentials); - } - else { - DefaultAWSCredentialsProviderChain defaultChain = new DefaultAWSCredentialsProviderChain(); - this.client = new AmazonKinesisClient(defaultChain); - this.amazonS3Client = new AmazonS3Client(defaultChain); - this.dynamoDbClient = new AmazonDynamoDBClient(defaultChain); - } - - this.client.setEndpoint("kinesis." + config.getAwsRegion() + ".amazonaws.com"); - this.dynamoDbClient.setEndpoint("dynamodb." + config.getAwsRegion() + ".amazonaws.com"); - } - - @Override - public AmazonKinesisClient getClient() - { - return client; - } - - @Override - public AmazonDynamoDBClient getDynamoDbClient() - { - return dynamoDbClient; - } - - @Override - public AmazonS3Client getS3Client() - { - return amazonS3Client; - } -} diff --git a/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/KinesisClientProvider.java b/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/KinesisClientProvider.java deleted file mode 100644 index 67d2609a2fbb..000000000000 --- a/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/KinesisClientProvider.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.kinesis; - -import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient; -import com.amazonaws.services.kinesis.AmazonKinesisClient; -import com.amazonaws.services.s3.AmazonS3Client; - -/** - * Interface to a client manager that provides the AWS clients needed. - */ -//TODO: This interface needs to be removed and abstraction in unneccesary -public interface KinesisClientProvider -{ - AmazonKinesisClient getClient(); - - AmazonDynamoDBClient getDynamoDbClient(); - - AmazonS3Client getS3Client(); -} diff --git a/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/KinesisColumnHandle.java b/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/KinesisColumnHandle.java deleted file mode 100644 index 6d77f9e6c91d..000000000000 --- a/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/KinesisColumnHandle.java +++ /dev/null @@ -1,170 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.kinesis; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import io.trino.decoder.DecoderColumnHandle; -import io.trino.spi.connector.ColumnMetadata; -import io.trino.spi.type.Type; -import jakarta.annotation.Nullable; - -import java.util.Objects; - -import static com.google.common.base.MoreObjects.toStringHelper; -import static java.util.Objects.requireNonNull; - -//TODO: Use Optional for nullable fields, changes to be done across Kafka and Redis too -public class KinesisColumnHandle - implements DecoderColumnHandle -{ - private final int ordinalPosition; - private final String name; - private final Type type; - private final String mapping; - - private final String dataFormat; // Data format to use (selects the decoder). Can be null. - private final String formatHint; // Additional format hint for the selected decoder. Selects a decoder subtype (e.g. which timestamp decoder). - private final boolean hidden; - private final boolean internal; - - //TODO: use Optional and check that Optional wrapper passed in is not null, across Kafka and Redis too - @JsonCreator - public KinesisColumnHandle( - @JsonProperty("ordinalPosition") int ordinalPosition, - @JsonProperty("name") String name, - @JsonProperty("type") Type type, - @JsonProperty("mapping") String mapping, - @JsonProperty("dataFormat") String dataFormat, - @JsonProperty("formatHint") String formatHint, - @JsonProperty("hidden") boolean hidden, - @JsonProperty("internal") boolean internal) - { - this.ordinalPosition = ordinalPosition; - this.name = requireNonNull(name, "name is null"); - this.type = requireNonNull(type, "type is null"); - this.mapping = mapping; - this.dataFormat = dataFormat; - this.formatHint = formatHint; - this.hidden = hidden; - this.internal = internal; - } - - @JsonProperty - public int getOrdinalPosition() - { - return ordinalPosition; - } - - @Override - @JsonProperty - public String getName() - { - return name; - } - - @Override - @JsonProperty - public Type getType() - { - return type; - } - - @Override - @Nullable - @JsonProperty - public String getMapping() - { - return mapping; - } - - @Override - @Nullable - @JsonProperty - public String getDataFormat() - { - return dataFormat; - } - - @Override - @JsonProperty - public String getFormatHint() - { - return formatHint; - } - - @JsonProperty - public boolean isHidden() - { - return hidden; - } - - @Override - @JsonProperty - public boolean isInternal() - { - return internal; - } - - ColumnMetadata getColumnMetadata() - { - return ColumnMetadata.builder() - .setName(name) - .setType(type) - .setHidden(hidden) - .build(); - } - - @Override - public int hashCode() - { - return Objects.hash(ordinalPosition, name, type, mapping, dataFormat, formatHint, hidden, internal); - } - - @Override - public boolean equals(Object obj) - { - if (this == obj) { - return true; - } - if (obj == null || getClass() != obj.getClass()) { - return false; - } - - KinesisColumnHandle other = (KinesisColumnHandle) obj; - return this.ordinalPosition == other.ordinalPosition && - Objects.equals(this.name, other.name) && - Objects.equals(this.type, other.type) && - Objects.equals(this.mapping, other.mapping) && - Objects.equals(this.dataFormat, other.dataFormat) && - Objects.equals(this.formatHint, other.formatHint) && - this.hidden == other.hidden && - this.internal == other.internal; - } - - @Override - public String toString() - { - return toStringHelper(this) - .add("ordinalPosition", ordinalPosition) - .add("name", name) - .add("type", type) - .add("mapping", mapping) - .add("dataFormat", dataFormat) - .add("formatHint", formatHint) - .add("hidden", hidden) - .add("internal", internal) - .toString(); - } -} diff --git a/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/KinesisCompressionCodec.java b/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/KinesisCompressionCodec.java deleted file mode 100644 index 4c1c00990cfa..000000000000 --- a/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/KinesisCompressionCodec.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.kinesis; - -public enum KinesisCompressionCodec -{ - UNCOMPRESSED, - GZIP, - AUTOMATIC; - - public static boolean canUseGzip(KinesisCompressionCodec compressionCodec) - { - return compressionCodec == GZIP || compressionCodec == AUTOMATIC; - } -} diff --git a/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/KinesisConfig.java b/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/KinesisConfig.java deleted file mode 100644 index 9d0da37e7c14..000000000000 --- a/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/KinesisConfig.java +++ /dev/null @@ -1,308 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.kinesis; - -import io.airlift.configuration.Config; -import io.airlift.configuration.ConfigDescription; -import io.airlift.configuration.ConfigSecuritySensitive; -import io.airlift.configuration.DefunctConfig; -import io.airlift.units.Duration; -import io.airlift.units.MinDuration; -import jakarta.validation.constraints.Max; -import jakarta.validation.constraints.Min; -import jakarta.validation.constraints.NotNull; - -import java.util.concurrent.TimeUnit; - -@DefunctConfig("kinesis.checkpoint-interval") -public class KinesisConfig -{ - private String defaultSchema = "default"; - private String tableDescriptionLocation = "etc/kinesis/"; - private Duration tableDescriptionRefreshInterval = new Duration(10, TimeUnit.MINUTES); - private boolean hideInternalColumns = true; - private String awsRegion = "us-east-1"; - private int batchSize = 10000; - private int maxBatches = 600; - private int fetchAttempts = 2; - private Duration sleepTime = new Duration(1000, TimeUnit.MILLISECONDS); - private boolean isIteratorFromTimestamp = true; - private long iteratorOffsetSeconds = 86400; - private String accessKey; - private String secretKey; - private boolean logKinesisBatches = true; - private boolean checkpointEnabled; - private long dynamoReadCapacity = 50L; - private long dynamoWriteCapacity = 10L; - private String logicalProcessName = "process1"; - private int iteratorNumber; - - @NotNull - public String getTableDescriptionLocation() - { - return tableDescriptionLocation; - } - - @Config("kinesis.table-description-location") - @ConfigDescription("S3 or local filesystem directory location where table schema descriptions are present") - public KinesisConfig setTableDescriptionLocation(String tableDescriptionLocation) - { - this.tableDescriptionLocation = tableDescriptionLocation; - return this; - } - - @NotNull - @MinDuration("1ms") - public Duration getTableDescriptionRefreshInterval() - { - return tableDescriptionRefreshInterval; - } - - @Config("kinesis.table-description-refresh-interval") - @ConfigDescription("How often to get the table description from S3") - public KinesisConfig setTableDescriptionRefreshInterval(Duration tableDescriptionRefreshInterval) - { - this.tableDescriptionRefreshInterval = tableDescriptionRefreshInterval; - return this; - } - - public boolean isHideInternalColumns() - { - return hideInternalColumns; - } - - @Config("kinesis.hide-internal-columns") - @ConfigDescription("Toggle to decide whether to show Kinesis internal columns or not") - public KinesisConfig setHideInternalColumns(boolean hideInternalColumns) - { - this.hideInternalColumns = hideInternalColumns; - return this; - } - - @NotNull - public String getDefaultSchema() - { - return defaultSchema; - } - - @Config("kinesis.default-schema") - @ConfigDescription("Sets default schema for kinesis catalogs") - public KinesisConfig setDefaultSchema(String defaultSchema) - { - this.defaultSchema = defaultSchema; - return this; - } - - public String getAccessKey() - { - return this.accessKey; - } - - @Config("kinesis.access-key") - @ConfigDescription("S3 Access Key to access s3 locations") - public KinesisConfig setAccessKey(String accessKey) - { - this.accessKey = accessKey; - return this; - } - - public String getSecretKey() - { - return this.secretKey; - } - - @Config("kinesis.secret-key") - @ConfigDescription("S3 Secret Key to access s3 locations") - @ConfigSecuritySensitive - public KinesisConfig setSecretKey(String secretKey) - { - this.secretKey = secretKey; - return this; - } - - public String getAwsRegion() - { - return awsRegion; - } - - @Config("kinesis.aws-region") - @ConfigDescription("Region to set while creating S3 client") - public KinesisConfig setAwsRegion(String awsRegion) - { - this.awsRegion = awsRegion; - return this; - } - - @Min(1) - @Max(Integer.MAX_VALUE) - public int getBatchSize() - { - return this.batchSize; - } - - @Config("kinesis.batch-size") - @ConfigDescription("Limit maximum number of rows to return in a batch") - public KinesisConfig setBatchSize(int batchSize) - { - this.batchSize = batchSize; - return this; - } - - @Min(1) - public int getMaxBatches() - { - return this.maxBatches; - } - - @Config("kinesis.max-batches") - @ConfigDescription("Maximum number of calls to Kinesis per query") - public KinesisConfig setMaxBatches(int maxBatches) - { - this.maxBatches = maxBatches; - return this; - } - - @Min(1) - @Max(1000) - public int getFetchAttempts() - { - return this.fetchAttempts; - } - - @Config("kinesis.fetch-attempts") - @ConfigDescription("Maximum number of attempts to fetch the next batch from a shard iterator") - public KinesisConfig setFetchAttempts(int fetchAttempts) - { - this.fetchAttempts = fetchAttempts; - return this; - } - - public Duration getSleepTime() - { - return this.sleepTime; - } - - @Config("kinesis.sleep-time") - @ConfigDescription("Sleep time between fetch attempt retries") - public KinesisConfig setSleepTime(Duration sleepTime) - { - this.sleepTime = sleepTime; - return this; - } - - public boolean isLogBatches() - { - return logKinesisBatches; - } - - @Config("kinesis.log-batches") - @ConfigDescription("Decides whether to log batch fetch details") - public KinesisConfig setLogBatches(boolean logBatches) - { - this.logKinesisBatches = logBatches; - return this; - } - - public boolean isIteratorFromTimestamp() - { - return isIteratorFromTimestamp; - } - - @Config("kinesis.iterator-from-timestamp") - @ConfigDescription("Whether to use start timestamp from shard iterator") - public KinesisConfig setIteratorFromTimestamp(boolean isIteratorFromTimestamp) - { - this.isIteratorFromTimestamp = isIteratorFromTimestamp; - return this; - } - - public long getIteratorOffsetSeconds() - { - return iteratorOffsetSeconds; - } - - @Config("kinesis.iterator-offset-seconds") - @ConfigDescription("Seconds before current time to start fetching records from") - public KinesisConfig setIteratorOffsetSeconds(long iteratorOffsetSeconds) - { - this.iteratorOffsetSeconds = iteratorOffsetSeconds; - return this; - } - - public boolean isCheckpointEnabled() - { - return checkpointEnabled; - } - - @Config("kinesis.checkpoint-enabled") - @ConfigDescription("Whether to remember last read sequence number and use it in later requests") - public KinesisConfig setCheckpointEnabled(boolean checkpointEnabled) - { - this.checkpointEnabled = checkpointEnabled; - return this; - } - - public long getDynamoReadCapacity() - { - return dynamoReadCapacity; - } - - @Config("kinesis.dynamo-read-capacity") - @ConfigDescription("DynamoDB read capacity to be set in client") - public KinesisConfig setDynamoReadCapacity(long dynamoReadCapacity) - { - this.dynamoReadCapacity = dynamoReadCapacity; - return this; - } - - public long getDynamoWriteCapacity() - { - return dynamoWriteCapacity; - } - - @Config("kinesis.dynamo-write-capacity") - @ConfigDescription("DynamoDB read capacity to be set in client") - public KinesisConfig setDynamoWriteCapacity(long dynamoWriteCapacity) - { - this.dynamoWriteCapacity = dynamoWriteCapacity; - return this; - } - - public String getLogicalProcessName() - { - return logicalProcessName; - } - - @Config("kinesis.checkpoint-logical-name") - @ConfigDescription("Prefix to the checkpoint name") - public KinesisConfig setLogicalProcessName(String logicalPrcessName) - { - this.logicalProcessName = logicalPrcessName; - return this; - } - - @Min(0) - public int getIteratorNumber() - { - return iteratorNumber; - } - - @Config("kinesis.iterator-number") - @ConfigDescription("Checkpoint iteration number") - public KinesisConfig setIteratorNumber(int iteratorNumber) - { - this.iteratorNumber = iteratorNumber; - return this; - } -} diff --git a/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/KinesisConnector.java b/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/KinesisConnector.java deleted file mode 100644 index 336c122669de..000000000000 --- a/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/KinesisConnector.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.kinesis; - -import com.google.common.collect.ImmutableList; -import com.google.inject.Inject; -import io.airlift.bootstrap.LifeCycleManager; -import io.trino.spi.connector.Connector; -import io.trino.spi.connector.ConnectorMetadata; -import io.trino.spi.connector.ConnectorRecordSetProvider; -import io.trino.spi.connector.ConnectorSession; -import io.trino.spi.connector.ConnectorSplitManager; -import io.trino.spi.connector.ConnectorTransactionHandle; -import io.trino.spi.session.PropertyMetadata; -import io.trino.spi.transaction.IsolationLevel; - -import java.util.List; - -import static io.trino.spi.transaction.IsolationLevel.READ_COMMITTED; -import static io.trino.spi.transaction.IsolationLevel.checkConnectorSupports; -import static java.util.Objects.requireNonNull; - -public class KinesisConnector - implements Connector -{ - private final LifeCycleManager lifeCycleManager; - private final KinesisMetadata metadata; - private final KinesisSplitManager splitManager; - private final KinesisRecordSetProvider recordSetProvider; - - private final List> propertyList; - - @Inject - public KinesisConnector( - LifeCycleManager lifeCycleManager, - KinesisMetadata metadata, - KinesisSplitManager splitManager, - KinesisRecordSetProvider recordSetProvider, - KinesisSessionProperties properties) - { - this.lifeCycleManager = requireNonNull(lifeCycleManager, "lifeCycleManager is null"); - this.metadata = requireNonNull(metadata, "metadata is null"); - this.splitManager = requireNonNull(splitManager, "splitManager is null"); - this.recordSetProvider = requireNonNull(recordSetProvider, "recordSetProvider is null"); - this.propertyList = ImmutableList.copyOf(properties.getSessionProperties()); - } - - @Override - public ConnectorMetadata getMetadata(ConnectorSession session, ConnectorTransactionHandle transactionHandle) - { - return metadata; - } - - @Override - public ConnectorTransactionHandle beginTransaction(IsolationLevel isolationLevel, boolean readOnly, boolean autoCommit) - { - checkConnectorSupports(READ_COMMITTED, isolationLevel); - return KinesisTransactionHandle.INSTANCE; - } - - @Override - public ConnectorSplitManager getSplitManager() - { - return splitManager; - } - - @Override - public ConnectorRecordSetProvider getRecordSetProvider() - { - return recordSetProvider; - } - - @Override - public List> getSessionProperties() - { - return propertyList; - } - - @Override - public void shutdown() - { - lifeCycleManager.stop(); - } -} diff --git a/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/KinesisConnectorFactory.java b/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/KinesisConnectorFactory.java deleted file mode 100644 index 1f0b4f230da7..000000000000 --- a/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/KinesisConnectorFactory.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.kinesis; - -import com.google.inject.Injector; -import com.google.inject.Scopes; -import com.google.inject.TypeLiteral; -import io.airlift.bootstrap.Bootstrap; -import io.airlift.json.JsonModule; -import io.trino.plugin.base.TypeDeserializerModule; -import io.trino.spi.NodeManager; -import io.trino.spi.connector.Connector; -import io.trino.spi.connector.ConnectorContext; -import io.trino.spi.connector.ConnectorFactory; -import io.trino.spi.connector.SchemaTableName; - -import java.util.Map; -import java.util.function.Supplier; - -import static com.google.common.base.Throwables.throwIfUnchecked; -import static io.trino.plugin.base.Versions.checkStrictSpiVersionMatch; -import static java.util.Objects.requireNonNull; - -public class KinesisConnectorFactory - implements ConnectorFactory -{ - @Override - public String getName() - { - return "kinesis"; - } - - @Override - public Connector create(String catalogName, Map config, ConnectorContext context) - { - requireNonNull(catalogName, "catalogName is null"); - requireNonNull(config, "config is null"); - checkStrictSpiVersionMatch(context, this); - - try { - Bootstrap app = new Bootstrap( - new JsonModule(), - new TypeDeserializerModule(context.getTypeManager()), - new KinesisModule(), - binder -> { - binder.bind(NodeManager.class).toInstance(context.getNodeManager()); - binder.bind(KinesisClientProvider.class).to(KinesisClientManager.class).in(Scopes.SINGLETON); - binder.bind(new TypeLiteral>>() {}).to(KinesisTableDescriptionSupplier.class).in(Scopes.SINGLETON); - }); - - Injector injector = app - .doNotInitializeLogging() - .setRequiredConfigurationProperties(config) - .initialize(); - - return injector.getInstance(KinesisConnector.class); - } - catch (Exception e) { - throwIfUnchecked(e); - throw new RuntimeException(e); - } - } -} diff --git a/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/KinesisErrorCode.java b/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/KinesisErrorCode.java deleted file mode 100644 index a53fa7a0c91b..000000000000 --- a/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/KinesisErrorCode.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.kinesis; - -import io.trino.spi.ErrorCode; -import io.trino.spi.ErrorCodeSupplier; -import io.trino.spi.ErrorType; - -import static io.trino.spi.ErrorType.EXTERNAL; -import static io.trino.spi.ErrorType.INTERNAL_ERROR; - -/** - * Kinesis connector specific error codes. - */ -public enum KinesisErrorCode - implements ErrorCodeSupplier -{ - KINESIS_CONVERSION_NOT_SUPPORTED(0, EXTERNAL), - KINESIS_SPLIT_ERROR(1, INTERNAL_ERROR), - KINESIS_METADATA_EXCEPTION(2, INTERNAL_ERROR); - - // Connectors can use error codes starting at EXTERNAL - public static final int StartingErrorCode = 0x0200_0000; - - private final ErrorCode errorCode; - - KinesisErrorCode(int code, ErrorType errorType) - { - errorCode = new ErrorCode(code + StartingErrorCode, name(), errorType); - } - - @Override - public ErrorCode toErrorCode() - { - return errorCode; - } -} diff --git a/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/KinesisInternalFieldDescription.java b/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/KinesisInternalFieldDescription.java deleted file mode 100644 index d8ea1a96ff16..000000000000 --- a/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/KinesisInternalFieldDescription.java +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.kinesis; - -import io.trino.spi.connector.ColumnMetadata; -import io.trino.spi.type.BigintType; -import io.trino.spi.type.BooleanType; -import io.trino.spi.type.TimestampType; -import io.trino.spi.type.Type; -import io.trino.spi.type.VarcharType; - -import java.util.Map; -import java.util.Optional; - -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Strings.isNullOrEmpty; -import static com.google.common.collect.ImmutableMap.toImmutableMap; -import static java.util.Arrays.stream; -import static java.util.Objects.requireNonNull; -import static java.util.function.Function.identity; - -public enum KinesisInternalFieldDescription -{ - SHARD_ID_FIELD("_shard_id", VarcharType.VARCHAR, "Shard Id"), - SEGMENT_START_FIELD("_segment_start", VarcharType.VARCHAR, "Segment start sequence id"), - SEGMENT_END_FIELD("_shard_sequence_id", VarcharType.VARCHAR, "Segment end sequence id"), - SHARD_SEQUENCE_ID_FIELD("_shard_sequence_id_field", BigintType.BIGINT, "Segment start offset"), - SEGMENT_COUNT_FIELD("_segment_count", BigintType.BIGINT, "Running message count per segment"), - MESSAGE_VALID_FIELD("_message_valid", BooleanType.BOOLEAN, "Message data is valid"), - MESSAGE_FIELD("_message", VarcharType.VARCHAR, "Message text"), - MESSAGE_TIMESTAMP("_message_timestamp", TimestampType.TIMESTAMP_MILLIS, "Approximate message arrival timestamp"), - MESSAGE_LENGTH_FIELD("_message_length", BigintType.BIGINT, "Total number of message bytes"), - PARTITION_KEY_FIELD("_partition_key", VarcharType.VARCHAR, "Key text"); - - private static final Map BY_COLUMN_NAME = stream(KinesisInternalFieldDescription.values()) - .collect(toImmutableMap(KinesisInternalFieldDescription::getColumnName, identity())); - - public static KinesisInternalFieldDescription forColumnName(String columnName) - { - KinesisInternalFieldDescription description = BY_COLUMN_NAME.get(columnName); - checkArgument(description != null, "Unknown internal column name %s", columnName); - return description; - } - - private final String columnName; - private final Type type; - private final String comment; - - KinesisInternalFieldDescription( - String columnName, - Type type, - String comment) - { - checkArgument(!isNullOrEmpty(columnName), "name is null or is empty"); - this.columnName = columnName; - this.type = requireNonNull(type, "type is null"); - this.comment = requireNonNull(comment, "comment is null"); - } - - public String getColumnName() - { - return columnName; - } - - public Type getType() - { - return type; - } - - KinesisColumnHandle getColumnHandle(int index, boolean hidden) - { - return new KinesisColumnHandle( - index, - getColumnName(), - getType(), - null, - null, - null, - false, - hidden); - } - - ColumnMetadata getColumnMetadata(boolean hidden) - { - return ColumnMetadata.builder() - .setName(columnName) - .setType(type) - .setComment(Optional.ofNullable(comment)) - .setHidden(hidden) - .build(); - } -} diff --git a/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/KinesisMetadata.java b/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/KinesisMetadata.java deleted file mode 100644 index 4a177c28c711..000000000000 --- a/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/KinesisMetadata.java +++ /dev/null @@ -1,200 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.kinesis; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.inject.Inject; -import io.trino.decoder.dummy.DummyRowDecoder; -import io.trino.spi.TrinoException; -import io.trino.spi.connector.ColumnHandle; -import io.trino.spi.connector.ColumnMetadata; -import io.trino.spi.connector.ConnectorMetadata; -import io.trino.spi.connector.ConnectorSession; -import io.trino.spi.connector.ConnectorTableHandle; -import io.trino.spi.connector.ConnectorTableMetadata; -import io.trino.spi.connector.ConnectorTableVersion; -import io.trino.spi.connector.SchemaTableName; -import io.trino.spi.connector.SchemaTablePrefix; -import io.trino.spi.connector.TableNotFoundException; - -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.function.Supplier; - -import static com.google.common.collect.ImmutableList.toImmutableList; -import static io.trino.plugin.kinesis.KinesisCompressionCodec.UNCOMPRESSED; -import static io.trino.spi.StandardErrorCode.NOT_SUPPORTED; -import static java.util.Objects.requireNonNull; - -public class KinesisMetadata - implements ConnectorMetadata -{ - private final Supplier> tableDescriptionSupplier; - private final Set internalFieldDescriptions; - private final boolean isHideInternalColumns; - - @Inject - public KinesisMetadata( - KinesisConfig kinesisConfig, - Supplier> tableDescriptionSupplier, - Set internalFieldDescriptions) - { - isHideInternalColumns = kinesisConfig.isHideInternalColumns(); - this.tableDescriptionSupplier = requireNonNull(tableDescriptionSupplier, "tableDescriptionSupplier is null"); - this.internalFieldDescriptions = requireNonNull(internalFieldDescriptions, "internalFieldDescriptions is null"); - } - - @Override - public List listSchemaNames(ConnectorSession session) - { - return tableDescriptionSupplier.get().keySet().stream() - .map(SchemaTableName::getSchemaName) - .collect(toImmutableList()); - } - - @Override - public KinesisTableHandle getTableHandle(ConnectorSession session, SchemaTableName schemaTableName, Optional startVersion, Optional endVersion) - { - if (startVersion.isPresent() || endVersion.isPresent()) { - throw new TrinoException(NOT_SUPPORTED, "This connector does not support versioned tables"); - } - - KinesisStreamDescription table = tableDescriptionSupplier.get().get(schemaTableName); - if (table == null) { - throw new TableNotFoundException(schemaTableName); - } - - return new KinesisTableHandle( - schemaTableName.getSchemaName(), - schemaTableName.getTableName(), - table.streamName(), - getDataFormat(table.message()), - table.message().compressionCodec().orElse(UNCOMPRESSED)); - } - - @Override - public ConnectorTableMetadata getTableMetadata(ConnectorSession connectorSession, ConnectorTableHandle tableHandle) - { - return getTableMetadata(((KinesisTableHandle) tableHandle).schemaTableName()); - } - - @Override - public List listTables(ConnectorSession session, Optional schemaName) - { - ImmutableList.Builder builder = ImmutableList.builder(); - for (SchemaTableName tableName : tableDescriptionSupplier.get().keySet()) { - if (schemaName.isEmpty() || tableName.getSchemaName().equals(schemaName.get())) { - builder.add(tableName); - } - } - - return builder.build(); - } - - @Override - public Map getColumnHandles(ConnectorSession connectorSession, ConnectorTableHandle tableHandle) - { - KinesisTableHandle kinesisTableHandle = (KinesisTableHandle) tableHandle; - - KinesisStreamDescription kinesisStreamDescription = tableDescriptionSupplier.get().get(kinesisTableHandle.schemaTableName()); - if (kinesisStreamDescription == null) { - throw new TableNotFoundException(kinesisTableHandle.schemaTableName()); - } - - ImmutableMap.Builder columnHandles = ImmutableMap.builder(); - - int index = 0; - // Note: partition key and related fields are handled by internalFieldDescriptions below - KinesisStreamFieldGroup message = kinesisStreamDescription.message(); - if (message != null) { - List fields = message.fields(); - if (fields != null) { - for (KinesisStreamFieldDescription kinesisStreamFieldDescription : fields) { - columnHandles.put(kinesisStreamFieldDescription.name(), kinesisStreamFieldDescription.columnHandle(index++)); - } - } - } - - for (KinesisInternalFieldDescription kinesisInternalFieldDescription : internalFieldDescriptions) { - columnHandles.put(kinesisInternalFieldDescription.getColumnName(), kinesisInternalFieldDescription.getColumnHandle(index++, isHideInternalColumns)); - } - - return columnHandles.buildOrThrow(); - } - - @Override - public ColumnMetadata getColumnMetadata(ConnectorSession connectorSession, ConnectorTableHandle tableHandle, ColumnHandle columnHandle) - { - KinesisColumnHandle kinesisColumnHandle = (KinesisColumnHandle) columnHandle; - return kinesisColumnHandle.getColumnMetadata(); - } - - @Override - public Map> listTableColumns(ConnectorSession session, SchemaTablePrefix prefix) - { - requireNonNull(prefix, "prefix is null"); - ImmutableMap.Builder> columns = ImmutableMap.builder(); - - // NOTE: prefix.getTableName or prefix.getSchemaName can be null - List tableNames; - if (prefix.getSchema().isPresent() && prefix.getTable().isPresent()) { - tableNames = ImmutableList.of(new SchemaTableName(prefix.getSchema().get(), prefix.getTable().get())); - } - else { - tableNames = listTables(session, Optional.empty()); - } - - for (SchemaTableName tableName : tableNames) { - ConnectorTableMetadata tableMetadata = getTableMetadata(tableName); - if (tableMetadata != null) { - columns.put(tableName, tableMetadata.getColumns()); - } - } - return columns.buildOrThrow(); - } - - private static String getDataFormat(KinesisStreamFieldGroup fieldGroup) - { - return (fieldGroup == null) ? DummyRowDecoder.NAME : fieldGroup.dataFormat(); - } - - private ConnectorTableMetadata getTableMetadata(SchemaTableName schemaTableName) - { - KinesisStreamDescription kinesisStreamDescription = tableDescriptionSupplier.get().get(schemaTableName); - if (kinesisStreamDescription == null) { - throw new TableNotFoundException(schemaTableName); - } - - ImmutableList.Builder builder = ImmutableList.builder(); - - KinesisStreamFieldGroup message = kinesisStreamDescription.message(); - if (message != null) { - List fields = message.fields(); - if (fields != null) { - for (KinesisStreamFieldDescription fieldDescription : fields) { - builder.add(fieldDescription.columnMetadata()); - } - } - } - - for (KinesisInternalFieldDescription fieldDescription : internalFieldDescriptions) { - builder.add(fieldDescription.getColumnMetadata(isHideInternalColumns)); - } - - return new ConnectorTableMetadata(schemaTableName, builder.build()); - } -} diff --git a/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/KinesisModule.java b/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/KinesisModule.java deleted file mode 100644 index 01fd79cae363..000000000000 --- a/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/KinesisModule.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.kinesis; - -import com.google.inject.Binder; -import com.google.inject.Module; -import com.google.inject.Scopes; -import com.google.inject.multibindings.Multibinder; -import io.trino.decoder.DecoderModule; -import io.trino.plugin.kinesis.s3config.S3TableConfigClient; - -import static io.airlift.configuration.ConfigBinder.configBinder; -import static io.airlift.json.JsonCodecBinder.jsonCodecBinder; - -public class KinesisModule - implements Module -{ - @Override - public void configure(Binder binder) - { - // Note: handle resolver handled separately, along with several other classes. - binder.bind(KinesisConnector.class).in(Scopes.SINGLETON); - - binder.bind(KinesisMetadata.class).in(Scopes.SINGLETON); - binder.bind(KinesisSplitManager.class).in(Scopes.SINGLETON); - binder.bind(KinesisRecordSetProvider.class).in(Scopes.SINGLETON); - binder.bind(S3TableConfigClient.class).in(Scopes.SINGLETON); - binder.bind(KinesisSessionProperties.class).in(Scopes.SINGLETON); - - configBinder(binder).bindConfig(KinesisConfig.class); - - jsonCodecBinder(binder).bindJsonCodec(KinesisStreamDescription.class); - - binder.install(new DecoderModule()); - - for (KinesisInternalFieldDescription internalFieldDescription : KinesisInternalFieldDescription.values()) { - bindInternalColumn(binder, internalFieldDescription); - } - } - - private static void bindInternalColumn(Binder binder, KinesisInternalFieldDescription fieldDescription) - { - Multibinder fieldDescriptionBinder = Multibinder.newSetBinder(binder, KinesisInternalFieldDescription.class); - fieldDescriptionBinder.addBinding().toInstance(fieldDescription); - } -} diff --git a/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/KinesisPlugin.java b/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/KinesisPlugin.java deleted file mode 100644 index b137175f0d51..000000000000 --- a/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/KinesisPlugin.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.kinesis; - -import com.google.common.collect.ImmutableList; -import io.trino.spi.Plugin; -import io.trino.spi.connector.ConnectorFactory; - -public class KinesisPlugin - implements Plugin -{ - private KinesisConnectorFactory factory; - - public KinesisPlugin() {} - - public KinesisPlugin(KinesisConnectorFactory factory) - { - this.factory = factory; - } - - @Override - public synchronized Iterable getConnectorFactories() - { - if (factory == null) { - this.factory = new KinesisConnectorFactory(); - } - return ImmutableList.of(this.factory); - } -} diff --git a/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/KinesisRecordSet.java b/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/KinesisRecordSet.java deleted file mode 100644 index 6275929e65c7..000000000000 --- a/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/KinesisRecordSet.java +++ /dev/null @@ -1,525 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.kinesis; - -import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient; -import com.amazonaws.services.kinesis.model.GetRecordsRequest; -import com.amazonaws.services.kinesis.model.GetRecordsResult; -import com.amazonaws.services.kinesis.model.GetShardIteratorRequest; -import com.amazonaws.services.kinesis.model.GetShardIteratorResult; -import com.amazonaws.services.kinesis.model.Record; -import com.amazonaws.services.kinesis.model.ResourceNotFoundException; -import com.google.common.collect.ImmutableList; -import io.airlift.log.Logger; -import io.airlift.slice.Slice; -import io.airlift.units.Duration; -import io.trino.decoder.DecoderColumnHandle; -import io.trino.decoder.FieldValueProvider; -import io.trino.decoder.RowDecoder; -import io.trino.spi.TrinoException; -import io.trino.spi.connector.ColumnHandle; -import io.trino.spi.connector.ConnectorSession; -import io.trino.spi.connector.RecordCursor; -import io.trino.spi.connector.RecordSet; -import io.trino.spi.type.Type; - -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.Date; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.zip.GZIPInputStream; - -import static com.google.common.base.Preconditions.checkArgument; -import static io.airlift.units.Duration.nanosSince; -import static io.trino.decoder.FieldValueProviders.booleanValueProvider; -import static io.trino.decoder.FieldValueProviders.bytesValueProvider; -import static io.trino.decoder.FieldValueProviders.longValueProvider; -import static io.trino.plugin.kinesis.KinesisCompressionCodec.AUTOMATIC; -import static io.trino.plugin.kinesis.KinesisCompressionCodec.GZIP; -import static io.trino.plugin.kinesis.KinesisCompressionCodec.UNCOMPRESSED; -import static io.trino.plugin.kinesis.KinesisCompressionCodec.canUseGzip; -import static io.trino.plugin.kinesis.KinesisSessionProperties.getBatchSize; -import static io.trino.plugin.kinesis.KinesisSessionProperties.getCheckpointLogicalName; -import static io.trino.plugin.kinesis.KinesisSessionProperties.getIterationNumber; -import static io.trino.plugin.kinesis.KinesisSessionProperties.getIteratorOffsetSeconds; -import static io.trino.plugin.kinesis.KinesisSessionProperties.getIteratorStartTimestamp; -import static io.trino.plugin.kinesis.KinesisSessionProperties.getMaxBatches; -import static io.trino.plugin.kinesis.KinesisSessionProperties.isCheckpointEnabled; -import static io.trino.plugin.kinesis.KinesisSessionProperties.isIteratorFromTimestamp; -import static io.trino.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; -import static java.lang.String.format; -import static java.nio.charset.StandardCharsets.UTF_8; -import static java.util.Objects.requireNonNull; -import static java.util.zip.GZIPInputStream.GZIP_MAGIC; - -public class KinesisRecordSet - implements RecordSet -{ - /** - * Indicates how close to current we want to be before stopping the fetch of records in a query. - */ - private static final int MILLIS_BEHIND_LIMIT = 10000; - - private static final Logger log = Logger.get(KinesisRecordSet.class); - - private static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; - - private final KinesisSplit split; - private final ConnectorSession session; - private final KinesisClientProvider clientManager; - - private final RowDecoder messageDecoder; - - private final List columnHandles; - private final List columnTypes; - - private final int batchSize; - private final int maxBatches; - private final int fetchAttempts; - private final long sleepTime; - - private final boolean isLogBatches; - - // for checkpointing - private final boolean checkpointEnabled; - private final KinesisShardCheckpointer shardCheckpointer; - private String lastReadSequenceNumber; - - KinesisRecordSet( - KinesisSplit split, - ConnectorSession session, - KinesisClientProvider clientManager, - List columnHandles, - RowDecoder messageDecoder, - long dynamoReadCapacity, - long dynamoWriteCapacity, - boolean isLogBatches, - int fetchAttempts, - Duration sleepTime) - { - this.split = requireNonNull(split, "split is null"); - this.session = requireNonNull(session, "session is null"); - this.isLogBatches = isLogBatches; - - this.clientManager = requireNonNull(clientManager, "clientManager is null"); - - this.columnHandles = requireNonNull(columnHandles, "columnHandles is null"); - this.messageDecoder = messageDecoder; - - ImmutableList.Builder typeBuilder = ImmutableList.builder(); - - for (KinesisColumnHandle handle : columnHandles) { - typeBuilder.add(handle.getType()); - } - - this.columnTypes = typeBuilder.build(); - - // Note: these default to what is in the configuration if not given in the session - this.batchSize = getBatchSize(session); - this.maxBatches = getMaxBatches(this.session); - - this.fetchAttempts = fetchAttempts; - this.sleepTime = sleepTime.toMillis(); - - this.checkpointEnabled = isCheckpointEnabled(session); - this.lastReadSequenceNumber = null; - - // Initialize checkpoint related code - if (checkpointEnabled) { - AmazonDynamoDBClient dynamoDBClient = clientManager.getDynamoDbClient(); - String dynamoDBTable = split.getStreamName(); - int curIterationNumber = getIterationNumber(session); - String sessionLogicalName = getCheckpointLogicalName(session); - - String logicalProcessName = null; - if (sessionLogicalName != null) { - logicalProcessName = sessionLogicalName; - } - - shardCheckpointer = new KinesisShardCheckpointer( - dynamoDBClient, - dynamoDBTable, - split, - logicalProcessName, - curIterationNumber, - dynamoReadCapacity, - dynamoWriteCapacity); - - lastReadSequenceNumber = shardCheckpointer.getLastReadSeqNumber(); - } - else { - shardCheckpointer = null; - } - } - - @Override - public List getColumnTypes() - { - return columnTypes; - } - - @Override - public RecordCursor cursor() - { - return new KinesisRecordCursor(); - } - - public class KinesisRecordCursor - implements RecordCursor - { - private final FieldValueProvider[] currentRowValues = new FieldValueProvider[columnHandles.size()]; - // TODO: total bytes here only includes records we iterate through, not total read from Kinesis. - // This may not be an issue, but if total vs. completed is an important signal to Trino then - // the implementation below could be a problem. Need to investigate. - private long batchesRead; - private long messagesRead; - private long totalBytes; - private long totalMessages; - private long lastReadTime; - private String shardIterator; - private List kinesisRecords; - private Iterator listIterator; - private GetRecordsRequest getRecordsRequest; - private GetRecordsResult getRecordsResult; - - @Override - public long getCompletedBytes() - { - return totalBytes; - } - - @Override - public long getReadTimeNanos() - { - return 0; - } - - @Override - public Type getType(int field) - { - checkArgument(field < columnHandles.size(), "Invalid field index"); - return columnHandles.get(field).getType(); - } - - /** - * Advances the cursor by one position, retrieving more records from Kinesis if needed. - *

- * We retrieve records from Kinesis in batches, using the getRecordsRequest. After a - * getRecordsRequest we keep iterating through that list of records until we run out. Then - * we will get another batch unless we've hit the limit or have caught up. - */ - @Override - public boolean advanceNextPosition() - { - if (shardIterator == null && getRecordsRequest == null) { - getIterator(); // first shard iterator - log.debug("(%s:%s) Starting read. Retrieved first shard iterator from AWS Kinesis.", - split.getStreamName(), - split.getShardId()); - } - - if (getRecordsRequest == null || (!listIterator.hasNext() && shouldGetMoreRecords())) { - getKinesisRecords(); - } - - if (listIterator.hasNext()) { - return nextRow(); - } - log.debug("(%s:%s) Read all of the records from the shard: %d batches and %d messages and %d total bytes.", - split.getStreamName(), - split.getShardId(), - batchesRead, - totalMessages, - totalBytes); - return false; - } - - private boolean shouldGetMoreRecords() - { - return shardIterator != null && batchesRead < maxBatches && getMillisBehindLatest() > MILLIS_BEHIND_LIMIT; - } - - /** - * Retrieves the next batch of records from Kinesis using the shard iterator. - *

- * Most of the time this results in one getRecords call. However we allow for - * a call to return an empty list, and we'll try again if we are far enough - * away from the latest record. - */ - private void getKinesisRecords() - throws ResourceNotFoundException - { - // Normally this loop will execute once, but we have to allow for the odd Kinesis - // behavior, per the docs: - // A single call to getRecords might return an empty record list, even when the shard contains - // more records at later sequence numbers - boolean fetchedRecords = false; - int attempts = 0; - while (!fetchedRecords && attempts < fetchAttempts) { - Duration duration = nanosSince(lastReadTime); - if (duration.toMillis() <= sleepTime) { - try { - Thread.sleep(duration.toMillis()); - } - catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new RuntimeException("thread interrupted"); - } - } - getRecordsRequest = new GetRecordsRequest(); - getRecordsRequest.setShardIterator(shardIterator); - getRecordsRequest.setLimit(batchSize); - - getRecordsResult = clientManager.getClient().getRecords(getRecordsRequest); - lastReadTime = System.nanoTime(); - - shardIterator = getRecordsResult.getNextShardIterator(); - kinesisRecords = getRecordsResult.getRecords(); - if (isLogBatches) { - log.info("(%s:%s) Fetched %d records from Kinesis. MillisBehindLatest=%d Attempt=%d", - split.getStreamName(), - split.getShardId(), - kinesisRecords.size(), - getRecordsResult.getMillisBehindLatest(), - attempts); - } - - fetchedRecords = (kinesisRecords.size() > 0 || getMillisBehindLatest() <= MILLIS_BEHIND_LIMIT); - attempts++; - } - - listIterator = kinesisRecords.iterator(); - batchesRead++; - messagesRead += kinesisRecords.size(); - } - - private boolean nextRow() - { - Record currentRecord = listIterator.next(); - String partitionKey = currentRecord.getPartitionKey(); - log.debug("(%s:%s) Reading record with partition key %s", - split.getStreamName(), - split.getShardId(), - partitionKey); - - byte[] messageData = EMPTY_BYTE_ARRAY; - ByteBuffer message = currentRecord.getData(); - if (message != null) { - messageData = new byte[message.remaining()]; - message.get(messageData); - } - totalBytes += messageData.length; - totalMessages++; - - log.debug("(%s:%s) Fetching %d bytes from current record. %d messages read so far", - split.getStreamName(), - split.getShardId(), - messageData.length, - totalMessages); - - Optional> decodedValue = decodeMessage(messageData); - - Map currentRowValuesMap = new HashMap<>(); - for (DecoderColumnHandle columnHandle : columnHandles) { - if (columnHandle.isInternal()) { - KinesisInternalFieldDescription fieldDescription = KinesisInternalFieldDescription.forColumnName(columnHandle.getName()); - switch (fieldDescription) { - case SHARD_ID_FIELD: - currentRowValuesMap.put(columnHandle, bytesValueProvider(split.getShardId().getBytes(UTF_8))); - break; - case SEGMENT_START_FIELD: - currentRowValuesMap.put(columnHandle, bytesValueProvider(split.getStart().getBytes(UTF_8))); - break; - case SEGMENT_COUNT_FIELD: - currentRowValuesMap.put(columnHandle, longValueProvider(totalMessages)); - break; - case SHARD_SEQUENCE_ID_FIELD: - currentRowValuesMap.put(columnHandle, bytesValueProvider(currentRecord.getSequenceNumber().getBytes(UTF_8))); - break; - case MESSAGE_FIELD: - currentRowValuesMap.put(columnHandle, bytesValueProvider(messageData)); - break; - case MESSAGE_LENGTH_FIELD: - currentRowValuesMap.put(columnHandle, longValueProvider(messageData.length)); - break; - case MESSAGE_VALID_FIELD: - currentRowValuesMap.put(columnHandle, booleanValueProvider(decodedValue.isEmpty())); - break; - default: - throw new IllegalArgumentException("unknown internal field " + fieldDescription); - } - } - } - - decodedValue.ifPresent(currentRowValuesMap::putAll); - for (int i = 0; i < columnHandles.size(); i++) { - ColumnHandle columnHandle = columnHandles.get(i); - currentRowValues[i] = currentRowValuesMap.get(columnHandle); - } - - return true; - } - - private Optional> decodeMessage(byte[] messageData) - { - KinesisCompressionCodec kinesisCompressionCodec = split.getCompressionCodec(); - if (kinesisCompressionCodec == UNCOMPRESSED) { - return messageDecoder.decodeRow(messageData); - } - if (isGZipped(messageData)) { - if (!canUseGzip(kinesisCompressionCodec)) { - throw new TrinoException(GENERIC_INTERNAL_ERROR, format("A %s message was found that did not match the required %s compression codec. Consider using %s or %s compressionCodec in table description", GZIP, kinesisCompressionCodec, GZIP, AUTOMATIC)); - } - try ( - ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(messageData); - GZIPInputStream gZIPInputStream = new GZIPInputStream(byteArrayInputStream)) { - return messageDecoder.decodeRow(gZIPInputStream.readAllBytes()); - } - catch (IOException e) { - throw new TrinoException(GENERIC_INTERNAL_ERROR, e); - } - } - if (kinesisCompressionCodec == AUTOMATIC) { - return messageDecoder.decodeRow(messageData); - } - throw new TrinoException(GENERIC_INTERNAL_ERROR, format("A message was found that did not match the required %s compression codec. Consider using %s or %s compressionCodec in table description", kinesisCompressionCodec, UNCOMPRESSED, AUTOMATIC)); - } - - /** - * Protect against possibly null values if this isn't set (not expected) - */ - private long getMillisBehindLatest() - { - if (getRecordsResult != null && getRecordsResult.getMillisBehindLatest() != null) { - return getRecordsResult.getMillisBehindLatest(); - } - return MILLIS_BEHIND_LIMIT + 1; - } - - @Override - public boolean getBoolean(int field) - { - return getFieldValueProvider(field, boolean.class).getBoolean(); - } - - @Override - public long getLong(int field) - { - return getFieldValueProvider(field, long.class).getLong(); - } - - @Override - public double getDouble(int field) - { - return getFieldValueProvider(field, double.class).getDouble(); - } - - @Override - public Slice getSlice(int field) - { - return getFieldValueProvider(field, Slice.class).getSlice(); - } - - @Override - public Object getObject(int field) - { - return getFieldValueProvider(field, Object.class).getObject(); - } - - @Override - public boolean isNull(int field) - { - checkArgument(field < columnHandles.size(), "Invalid field index"); - return currentRowValues[field] == null || currentRowValues[field].isNull(); - } - - private FieldValueProvider getFieldValueProvider(int field, Class expectedType) - { - checkArgument(field < columnHandles.size(), "Invalid field index"); - checkFieldType(field, expectedType); - return currentRowValues[field]; - } - - @Override - public void close() - { - log.info("(%s:%s) Closing cursor - read complete. Total read: %d batches %d messages, processed: %d messages and %d bytes.", - split.getStreamName(), - split.getShardId(), - batchesRead, - messagesRead, - totalMessages, - totalBytes); - if (checkpointEnabled && lastReadSequenceNumber != null) { - shardCheckpointer.checkpoint(lastReadSequenceNumber); - } - } - - private void checkFieldType(int field, Class expected) - { - Class actual = getType(field).getJavaType(); - checkArgument(actual == expected, "Expected field %s to be type %s but is %s", field, expected, actual); - } - - private void getIterator() - throws ResourceNotFoundException - { - GetShardIteratorRequest getShardIteratorRequest = new GetShardIteratorRequest(); - getShardIteratorRequest.setStreamName(split.getStreamName()); - getShardIteratorRequest.setShardId(split.getShardId()); - - // Explanation: when we have a sequence number from a prior read or checkpoint, always use it. - // Otherwise, decide if starting at a timestamp or the trim horizon based on configuration. - // If starting at a timestamp, use the session variable STARTING_TIMESTAMP when given, otherwise - // fallback on starting at STARTING_OFFSET_SECONDS from timestamp. - if (lastReadSequenceNumber == null) { - if (isIteratorFromTimestamp(session)) { - getShardIteratorRequest.setShardIteratorType("AT_TIMESTAMP"); - long iteratorStartTimestamp = getIteratorStartTimestamp(session); - if (iteratorStartTimestamp == 0) { - long startTimestamp = System.currentTimeMillis() - (getIteratorOffsetSeconds(session) * 1000); - getShardIteratorRequest.setTimestamp(new Date(startTimestamp)); - } - else { - getShardIteratorRequest.setTimestamp(new Date(iteratorStartTimestamp)); - } - } - else { - getShardIteratorRequest.setShardIteratorType("TRIM_HORIZON"); - } - } - else { - getShardIteratorRequest.setShardIteratorType("AFTER_SEQUENCE_NUMBER"); - getShardIteratorRequest.setStartingSequenceNumber(lastReadSequenceNumber); - } - - GetShardIteratorResult getShardIteratorResult = clientManager.getClient().getShardIterator(getShardIteratorRequest); - shardIterator = getShardIteratorResult.getShardIterator(); - } - } - - private static boolean isGZipped(byte[] data) - { - if (data == null || data.length < 2) { - return false; - } - int magic = data[0] & 0xff | ((data[1] << 8) & 0xff00); - return magic == GZIP_MAGIC; - } -} diff --git a/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/KinesisRecordSetProvider.java b/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/KinesisRecordSetProvider.java deleted file mode 100644 index 2758c9065843..000000000000 --- a/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/KinesisRecordSetProvider.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.kinesis; - -import com.google.common.collect.ImmutableList; -import com.google.inject.Inject; -import io.airlift.units.Duration; -import io.trino.decoder.DispatchingRowDecoderFactory; -import io.trino.decoder.RowDecoder; -import io.trino.decoder.RowDecoderSpec; -import io.trino.spi.connector.ColumnHandle; -import io.trino.spi.connector.ConnectorRecordSetProvider; -import io.trino.spi.connector.ConnectorSession; -import io.trino.spi.connector.ConnectorSplit; -import io.trino.spi.connector.ConnectorTableHandle; -import io.trino.spi.connector.ConnectorTransactionHandle; -import io.trino.spi.connector.RecordSet; - -import java.util.List; - -import static com.google.common.collect.ImmutableList.toImmutableList; -import static com.google.common.collect.ImmutableSet.toImmutableSet; -import static java.util.Collections.emptyMap; -import static java.util.Objects.requireNonNull; - -public class KinesisRecordSetProvider - implements ConnectorRecordSetProvider -{ - private final KinesisClientProvider clientManager; - private final long dynamoReadCapacity; - private final long dynamoWriteCapacity; - private final boolean isLogBatches; - private final int fetchAttempts; - private final Duration sleepTime; - private final DispatchingRowDecoderFactory decoderFactory; - - @Inject - public KinesisRecordSetProvider( - DispatchingRowDecoderFactory decoderFactory, - KinesisClientProvider clientManager, - KinesisConfig kinesisConfig) - { - this.decoderFactory = requireNonNull(decoderFactory, "decoderFactory is null"); - this.clientManager = requireNonNull(clientManager, "clientManager is null"); - requireNonNull(kinesisConfig, "kinesisConfig is null"); - this.dynamoReadCapacity = kinesisConfig.getDynamoReadCapacity(); - this.dynamoWriteCapacity = kinesisConfig.getDynamoWriteCapacity(); - this.isLogBatches = kinesisConfig.isLogBatches(); - this.fetchAttempts = kinesisConfig.getFetchAttempts(); - this.sleepTime = kinesisConfig.getSleepTime(); - } - - @Override - public RecordSet getRecordSet( - ConnectorTransactionHandle transactionHandle, - ConnectorSession session, - ConnectorSplit split, - ConnectorTableHandle table, - List columns) - { - KinesisSplit kinesisSplit = (KinesisSplit) split; - List kinesisColumns = columns.stream() - .map(x -> (KinesisColumnHandle) x) - .collect(toImmutableList()); - - ImmutableList.Builder handleBuilder = ImmutableList.builder(); - - RowDecoder messageDecoder = decoderFactory.create( - session, - new RowDecoderSpec( - kinesisSplit.getMessageDataFormat(), - emptyMap(), - kinesisColumns.stream() - .filter(column -> !column.isInternal()) - .collect(toImmutableSet()))); - - for (ColumnHandle handle : columns) { - KinesisColumnHandle columnHandle = (KinesisColumnHandle) handle; - handleBuilder.add(columnHandle); - } - return new KinesisRecordSet( - kinesisSplit, - session, - clientManager, - handleBuilder.build(), - messageDecoder, - dynamoReadCapacity, - dynamoWriteCapacity, - isLogBatches, - fetchAttempts, - sleepTime); - } -} diff --git a/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/KinesisSessionProperties.java b/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/KinesisSessionProperties.java deleted file mode 100644 index 5b4266cd3102..000000000000 --- a/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/KinesisSessionProperties.java +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.kinesis; - -import com.google.common.collect.ImmutableList; -import com.google.inject.Inject; -import io.trino.spi.connector.ConnectorSession; -import io.trino.spi.session.PropertyMetadata; - -import java.text.ParsePosition; -import java.text.SimpleDateFormat; -import java.util.Date; -import java.util.List; -import java.util.TimeZone; - -import static io.trino.spi.session.PropertyMetadata.booleanProperty; -import static io.trino.spi.session.PropertyMetadata.integerProperty; -import static io.trino.spi.session.PropertyMetadata.stringProperty; - -public final class KinesisSessionProperties -{ - private static final String TRINO_TIMESTAMP_FORMAT = "yyyy-MM-dd HH:mm:ss.SSS"; - private static final String UNSET_TIMESTAMP = "2000-01-01 00:00:00.000"; - - private static final String CHECKPOINT_ENABLED = "checkpoint_enabled"; - private static final String ITERATION_NUMBER = "iteration_number"; - private static final String CHECKPOINT_LOGICAL_NAME = "checkpoint_logical_name"; - private static final String MAX_BATCHES = "max_batches"; - private static final String BATCH_SIZE = "batch_size"; - private static final String START_FROM_TIMESTAMP = "start_from_timestamp"; - private static final String STARTING_OFFSET_SECONDS = "starting_offset_seconds"; - private static final String STARTING_TIMESTAMP = "starting_timestamp"; - - private final List> sessionProperties; - - @Inject - public KinesisSessionProperties(KinesisConfig config) - { - sessionProperties = ImmutableList.>builder() - .add(booleanProperty( - CHECKPOINT_ENABLED, - "Are checkpoints used in this session?", - config.isCheckpointEnabled(), - false)) - .add(integerProperty( - ITERATION_NUMBER, - "Checkpoint iteration number", - config.getIteratorNumber(), - false)) - .add(stringProperty( - CHECKPOINT_LOGICAL_NAME, - "checkpoint logical name", - config.getLogicalProcessName(), - false)) - .add(PropertyMetadata.integerProperty( - MAX_BATCHES, - "max number of calls to Kinesis per query", - config.getMaxBatches(), - false)) - .add(PropertyMetadata.integerProperty( - BATCH_SIZE, - "Record limit in calls to Kinesis", - config.getBatchSize(), - false)) - .add(PropertyMetadata.booleanProperty( - START_FROM_TIMESTAMP, - "Start from timestamp not trim horizon", - config.isIteratorFromTimestamp(), - false)) - .add(PropertyMetadata.longProperty( - STARTING_OFFSET_SECONDS, - "Seconds before current time to start iterator", - config.getIteratorOffsetSeconds(), - false)) - .add(PropertyMetadata.stringProperty( - STARTING_TIMESTAMP, - "Timestamp in Trino format to start iterator", - UNSET_TIMESTAMP, - false)) - .build(); - } - - public List> getSessionProperties() - { - return sessionProperties; - } - - public static boolean isCheckpointEnabled(ConnectorSession session) - { - return session.getProperty(CHECKPOINT_ENABLED, Boolean.class); - } - - public static int getIterationNumber(ConnectorSession session) - { - return session.getProperty(ITERATION_NUMBER, Integer.class); - } - - public static String getCheckpointLogicalName(ConnectorSession session) - { - return session.getProperty(CHECKPOINT_LOGICAL_NAME, String.class); - } - - public static int getMaxBatches(ConnectorSession session) - { - return session.getProperty(MAX_BATCHES, Integer.class); - } - - public static int getBatchSize(ConnectorSession session) - { - return session.getProperty(BATCH_SIZE, Integer.class); - } - - public static boolean isIteratorFromTimestamp(ConnectorSession session) - { - return session.getProperty(START_FROM_TIMESTAMP, Boolean.class); - } - - public static long getIteratorOffsetSeconds(ConnectorSession session) - { - return session.getProperty(STARTING_OFFSET_SECONDS, Long.class); - } - - public static long getIteratorStartTimestamp(ConnectorSession session) - { - String value = session.getProperty(STARTING_TIMESTAMP, String.class); - if (value.equals(UNSET_TIMESTAMP)) { - return 0; - } - return getTimestampAsMillis(value, session); - } - - public static long getTimestampAsMillis(String timestampValue, ConnectorSession session) - { - // Parse this as a date and return the long timestamp value (2016-07-10 17:03:56.124). - // They will be entering timestamps in their session's timezone. Use session.getTimeZoneKey(). - SimpleDateFormat format = new SimpleDateFormat(TRINO_TIMESTAMP_FORMAT); - - if (!session.getTimeZoneKey().getId().equals(TimeZone.getDefault().getID())) { - TimeZone sessionTimeZone = TimeZone.getTimeZone(session.getTimeZoneKey().getZoneId()); - format.setTimeZone(sessionTimeZone); - } - - Date result = format.parse(timestampValue, new ParsePosition(0)); - return result.getTime(); - } -} diff --git a/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/KinesisShardCheckpointer.java b/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/KinesisShardCheckpointer.java deleted file mode 100644 index d3ccc2a5b987..000000000000 --- a/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/KinesisShardCheckpointer.java +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.kinesis; - -import com.amazonaws.services.dynamodbv2.AmazonDynamoDB; -import com.amazonaws.services.kinesis.clientlibrary.types.ExtendedSequenceNumber; -import com.amazonaws.services.kinesis.leases.exceptions.DependencyException; -import com.amazonaws.services.kinesis.leases.exceptions.InvalidStateException; -import com.amazonaws.services.kinesis.leases.exceptions.ProvisionedThroughputException; -import com.amazonaws.services.kinesis.leases.impl.KinesisClientLease; -import com.amazonaws.services.kinesis.leases.impl.KinesisClientLeaseManager; -import io.airlift.log.Logger; - -public class KinesisShardCheckpointer -{ - private static final Logger log = Logger.get(KinesisShardCheckpointer.class); - private final KinesisClientLeaseManager leaseManager; - private final KinesisSplit kinesisSplit; - private final String logicalProcessName; - private final int currentIterationNumber; - private final KinesisClientLease kinesisClientLease; - - public KinesisShardCheckpointer( - AmazonDynamoDB dynamoDBClient, - String dynamoDBTable, - KinesisSplit kinesisSplit, - String logicalProcessName, - int currentIterationNumber, - long dynamoReadCapacity, - long dynamoWriteCapacity) - { - this(new KinesisClientLeaseManager(dynamoDBTable, dynamoDBClient), - kinesisSplit, - logicalProcessName, - currentIterationNumber, - dynamoReadCapacity, - dynamoWriteCapacity); - } - - public KinesisShardCheckpointer( - KinesisClientLeaseManager leaseManager, - KinesisSplit kinesisSplit, - String logicalProcessName, - int currentIterationNumber, - long dynamoReadCapacity, - long dynamoWriteCapacity) - { - this.leaseManager = leaseManager; - this.kinesisSplit = kinesisSplit; - this.logicalProcessName = logicalProcessName; - this.currentIterationNumber = currentIterationNumber; - - try { - this.leaseManager.createLeaseTableIfNotExists(dynamoReadCapacity, dynamoWriteCapacity); - - KinesisClientLease oldLease = this.leaseManager.getLease(createCheckpointKey(currentIterationNumber)); - if (oldLease != null) { - this.kinesisClientLease = oldLease; - } - else { - this.kinesisClientLease = new KinesisClientLease(); - this.kinesisClientLease.setLeaseKey(createCheckpointKey(currentIterationNumber)); - } - } - catch (ProvisionedThroughputException | InvalidStateException | DependencyException e) { - throw new RuntimeException(e); - } - } - - private String createCheckpointKey(int iterationNo) - { - return "%s_%s_%s_%d".formatted( - this.logicalProcessName, - this.kinesisSplit.getStreamName(), - this.kinesisSplit.getShardId(), - iterationNo); - } - - // storing last read sequence no. in dynamodb table - public void checkpoint(String lastReadSequenceNumber) - { - log.info("Trying to checkpoint at %s", lastReadSequenceNumber); - try { - ExtendedSequenceNumber esn = new ExtendedSequenceNumber(lastReadSequenceNumber); - kinesisClientLease.setCheckpoint(esn); - leaseManager.createLeaseIfNotExists(kinesisClientLease); - if (!leaseManager.updateLease(kinesisClientLease)) { - log.warn("Checkpointing unsuccessful"); - } - } - catch (DependencyException | InvalidStateException | ProvisionedThroughputException e) { - throw new RuntimeException(e); - } - } - - //return checkpoint of previous iteration if found - public String getLastReadSeqNumber() - { - String lastReadSeqNumber = null; - if (currentIterationNumber > 0) { - KinesisClientLease oldLease; - try { - oldLease = leaseManager.getLease(createCheckpointKey(currentIterationNumber - 1)); - } - catch (DependencyException | InvalidStateException | ProvisionedThroughputException e) { - throw new RuntimeException(e); - } - if (oldLease != null) { - // ExtendedSequenceNumber type in latest API: - lastReadSeqNumber = oldLease.getCheckpoint().toString(); - } - } - if (lastReadSeqNumber == null) { - log.info("Previous checkpoint not found. Starting from beginning of shard"); - } - else { - log.info("Resuming from %s", lastReadSeqNumber); - } - return lastReadSeqNumber; - } -} diff --git a/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/KinesisSplit.java b/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/KinesisSplit.java deleted file mode 100644 index eb3b5277edba..000000000000 --- a/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/KinesisSplit.java +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.kinesis; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.collect.ImmutableMap; -import io.trino.spi.connector.ConnectorSplit; - -import java.util.Map; - -import static com.google.common.base.MoreObjects.toStringHelper; -import static io.airlift.slice.SizeOf.estimatedSizeOf; -import static io.airlift.slice.SizeOf.instanceSize; -import static java.util.Objects.requireNonNull; - -/** - * Kinesis vertion of ConnectorSplit. KinesisConnector fetch the data from kinesis stream and splits the big chunk to multiple split. - * By default, one shard data is one KinesisSplit. - */ -public class KinesisSplit - implements ConnectorSplit -{ - private static final int INSTANCE_SIZE = instanceSize(KinesisSplit.class); - - private final String streamName; - private final String messageDataFormat; - private final KinesisCompressionCodec compressionCodec; - private final String shardId; - private final String start; - private final String end; - - @JsonCreator - public KinesisSplit( - @JsonProperty("streamName") String streamName, - @JsonProperty("messageDataFormat") String messageDataFormat, - @JsonProperty("compressionCodec") KinesisCompressionCodec compressionCodec, - @JsonProperty("shardId") String shardId, - @JsonProperty("start") String start, - @JsonProperty("end") String end) - { - this.streamName = requireNonNull(streamName, "streamName is null"); - this.messageDataFormat = requireNonNull(messageDataFormat, "messageDataFormat is null"); - this.compressionCodec = requireNonNull(compressionCodec, "compressionCodec is null"); - this.shardId = shardId; - this.start = start; - this.end = end; - } - - @JsonProperty - public String getStart() - { - return start; - } - - @JsonProperty - public String getEnd() - { - return end; - } - - @JsonProperty - public String getStreamName() - { - return streamName; - } - - @JsonProperty - public String getMessageDataFormat() - { - return messageDataFormat; - } - - @JsonProperty - public KinesisCompressionCodec getCompressionCodec() - { - return compressionCodec; - } - - @JsonProperty - public String getShardId() - { - return shardId; - } - - @Override - public Map getSplitInfo() - { - return ImmutableMap.of( - "streamName", streamName, - "shardId", shardId, - "messageDataFormat", messageDataFormat, - "compressionCodec", compressionCodec.name(), - "start", start, - "end", end); - } - - @Override - public long getRetainedSizeInBytes() - { - return INSTANCE_SIZE - + estimatedSizeOf(streamName) - + estimatedSizeOf(messageDataFormat) - + estimatedSizeOf(shardId) - + estimatedSizeOf(start) - + estimatedSizeOf(end); - } - - @Override - public String toString() - { - return toStringHelper(this) - .add("streamName", streamName) - .add("messageDataFormat", messageDataFormat) - .add("compressionCodec", compressionCodec) - .add("shardId", shardId) - .add("start", start) - .add("end", end) - .toString(); - } -} diff --git a/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/KinesisSplitManager.java b/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/KinesisSplitManager.java deleted file mode 100644 index 21e7115ff723..000000000000 --- a/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/KinesisSplitManager.java +++ /dev/null @@ -1,167 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.kinesis; - -import com.amazonaws.services.kinesis.model.DescribeStreamRequest; -import com.amazonaws.services.kinesis.model.DescribeStreamResult; -import com.amazonaws.services.kinesis.model.ResourceNotFoundException; -import com.amazonaws.services.kinesis.model.Shard; -import com.google.common.collect.ImmutableList; -import com.google.inject.Inject; -import io.airlift.units.Duration; -import io.trino.spi.connector.ConnectorSession; -import io.trino.spi.connector.ConnectorSplit; -import io.trino.spi.connector.ConnectorSplitManager; -import io.trino.spi.connector.ConnectorSplitSource; -import io.trino.spi.connector.ConnectorTableHandle; -import io.trino.spi.connector.ConnectorTransactionHandle; -import io.trino.spi.connector.Constraint; -import io.trino.spi.connector.DynamicFilter; -import io.trino.spi.connector.FixedSplitSource; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static java.util.Objects.requireNonNull; -import static java.util.concurrent.TimeUnit.DAYS; - -/** - * Split data chunk from kinesis Stream to multiple small chunks for parallelization and distribution to multiple Trino workers. - * By default, each shard of Kinesis Stream forms one Kinesis Split - */ -public class KinesisSplitManager - implements ConnectorSplitManager -{ - public static final long MAX_CACHE_AGE_MILLIS = new Duration(1, DAYS).toMillis(); - - private final KinesisClientProvider clientManager; - - private final Map streamMap = Collections.synchronizedMap(new HashMap<>()); - - /** - * Cache the result of a Kinesis describe stream call so we don't need to retrieve - * the shards for every single query. - */ - public static class InternalStreamDescription - { - private final String streamName; - private final List shards = new ArrayList<>(); - private final long createTimeStamp; - - public InternalStreamDescription(String streamName) - { - this.streamName = requireNonNull(streamName); - this.createTimeStamp = System.currentTimeMillis(); - } - - public long getCreateTimeStamp() - { - return createTimeStamp; - } - - public String getStreamName() - { - return streamName; - } - - public List getShards() - { - return shards; - } - - public void addAllShards(List shards) - { - this.shards.addAll(shards); - } - } - - @Inject - public KinesisSplitManager(KinesisClientProvider clientManager) - { - this.clientManager = clientManager; - } - - @Override - public ConnectorSplitSource getSplits( - ConnectorTransactionHandle transactionHandle, - ConnectorSession session, - ConnectorTableHandle table, - DynamicFilter dynamicFilter, - Constraint constraint) - { - KinesisTableHandle kinesisTableHandle = (KinesisTableHandle) table; - - InternalStreamDescription description = this.getStreamDescription(kinesisTableHandle.streamName()); - - ImmutableList.Builder builder = ImmutableList.builder(); - for (Shard shard : description.getShards()) { - KinesisSplit split = new KinesisSplit( - kinesisTableHandle.streamName(), - kinesisTableHandle.messageDataFormat(), - kinesisTableHandle.compressionCodec(), - shard.getShardId(), - shard.getSequenceNumberRange().getStartingSequenceNumber(), - shard.getSequenceNumberRange().getEndingSequenceNumber()); - builder.add(split); - } - - return new FixedSplitSource(builder.build()); - } - - /** - * Internal method to retrieve the stream description and get the shards from AWS. - *

- * Gets from the internal cache unless not yet created or too old. - */ - protected InternalStreamDescription getStreamDescription(String streamName) - { - InternalStreamDescription internalStreamDescription = this.streamMap.get(streamName); - if (internalStreamDescription == null || System.currentTimeMillis() - internalStreamDescription.getCreateTimeStamp() >= MAX_CACHE_AGE_MILLIS) { - internalStreamDescription = new InternalStreamDescription(streamName); - - DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest(); - describeStreamRequest.setStreamName(streamName); - - // Collect shards from Kinesis - String exclusiveStartShardId = null; - List shards = new ArrayList<>(); - do { - describeStreamRequest.setExclusiveStartShardId(exclusiveStartShardId); - DescribeStreamResult describeStreamResult = clientManager.getClient().describeStream(describeStreamRequest); - - String streamStatus = describeStreamResult.getStreamDescription().getStreamStatus(); - if (!streamStatus.equals("ACTIVE") && !streamStatus.equals("UPDATING")) { - throw new ResourceNotFoundException("Stream not Active"); - } - - internalStreamDescription.addAllShards(describeStreamResult.getStreamDescription().getShards()); - - if (describeStreamResult.getStreamDescription().getHasMoreShards() && (shards.size() > 0)) { - exclusiveStartShardId = shards.getLast().getShardId(); - } - else { - exclusiveStartShardId = null; - } - } - while (exclusiveStartShardId != null); - - this.streamMap.put(streamName, internalStreamDescription); - } - - return internalStreamDescription; - } -} diff --git a/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/KinesisStreamDescription.java b/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/KinesisStreamDescription.java deleted file mode 100644 index e5fb427b2692..000000000000 --- a/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/KinesisStreamDescription.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.kinesis; - -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Strings.isNullOrEmpty; -import static java.util.Objects.requireNonNull; - -/** - * This Class maintains all the details of Kinesis stream like name, fields of data, Trino table stream is mapping to, tables's schema name - */ -public record KinesisStreamDescription( - String tableName, - String schemaName, - String streamName, - KinesisStreamFieldGroup message) -{ - public KinesisStreamDescription - { - checkArgument(!isNullOrEmpty(tableName), "tableName is null or is empty"); - requireNonNull(streamName, "streamName is null"); - } -} diff --git a/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/KinesisStreamFieldDescription.java b/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/KinesisStreamFieldDescription.java deleted file mode 100644 index 8b7106c83193..000000000000 --- a/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/KinesisStreamFieldDescription.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.kinesis; - -import io.trino.spi.connector.ColumnMetadata; -import io.trino.spi.type.Type; - -import java.util.Optional; - -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Strings.isNullOrEmpty; -import static java.util.Objects.requireNonNull; - -public record KinesisStreamFieldDescription( - String name, - Type type, - String mapping, - String comment, - String dataFormat, - String formatHint, - boolean hidden) -{ - public KinesisStreamFieldDescription - { - checkArgument(!isNullOrEmpty(name), "name is null or is empty"); - requireNonNull(type, "type is null"); - } - - KinesisColumnHandle columnHandle(int index) - { - return new KinesisColumnHandle( - index, - name(), - type(), - mapping(), - dataFormat(), - formatHint(), - hidden(), - false); - } - - ColumnMetadata columnMetadata() - { - return ColumnMetadata.builder() - .setName(name()) - .setType(type()) - .setComment(Optional.ofNullable(comment())) - .setHidden(hidden()) - .build(); - } -} diff --git a/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/KinesisStreamFieldGroup.java b/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/KinesisStreamFieldGroup.java deleted file mode 100644 index b7bed503c1f4..000000000000 --- a/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/KinesisStreamFieldGroup.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.kinesis; - -import com.google.common.collect.ImmutableList; - -import java.util.List; -import java.util.Optional; - -import static java.util.Objects.requireNonNull; - -public record KinesisStreamFieldGroup( - String dataFormat, - Optional compressionCodec, - List fields) -{ - public KinesisStreamFieldGroup - { - requireNonNull(dataFormat, "dataFormat is null"); - requireNonNull(compressionCodec, "compressionCodec is null"); - fields = ImmutableList.copyOf(requireNonNull(fields, "fields is null")); - } -} diff --git a/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/KinesisTableDescriptionSupplier.java b/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/KinesisTableDescriptionSupplier.java deleted file mode 100644 index c5cb26052c8f..000000000000 --- a/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/KinesisTableDescriptionSupplier.java +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.kinesis; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.inject.Inject; -import io.airlift.json.JsonCodec; -import io.airlift.log.Logger; -import io.trino.plugin.kinesis.s3config.S3TableConfigClient; -import io.trino.spi.TrinoException; -import io.trino.spi.connector.SchemaTableName; -import jakarta.annotation.PreDestroy; - -import java.io.IOException; -import java.nio.file.DirectoryIteratorException; -import java.nio.file.DirectoryStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.List; -import java.util.Map; -import java.util.function.Supplier; - -import static com.google.common.base.MoreObjects.firstNonNull; -import static com.google.common.base.Throwables.throwIfUnchecked; -import static java.util.Objects.requireNonNull; - -/** - * This class get() method reads the table description file stored in Kinesis directory - * and then creates user defined field for Trino Table. - */ -public class KinesisTableDescriptionSupplier - implements Supplier> -{ - private static final Logger log = Logger.get(KinesisTableDescriptionSupplier.class); - - private final String tableDescriptionLocation; - private final String defaultSchema; - private final JsonCodec streamDescriptionCodec; - private final S3TableConfigClient s3TableConfigClient; - - @Inject - public KinesisTableDescriptionSupplier( - KinesisConfig kinesisConfig, - JsonCodec streamDescriptionCodec, - S3TableConfigClient s3TableConfigClient) - { - requireNonNull(kinesisConfig, "kinesisConfig is null"); - this.tableDescriptionLocation = kinesisConfig.getTableDescriptionLocation(); - this.defaultSchema = kinesisConfig.getDefaultSchema(); - this.streamDescriptionCodec = requireNonNull(streamDescriptionCodec, "streamDescriptionCodec is null"); - this.s3TableConfigClient = requireNonNull(s3TableConfigClient, "s3TableConfigClient is null"); - } - - @Override - public Map get() - { - if (s3TableConfigClient.isUsingS3()) { - return s3TableConfigClient.getTablesFromS3(); - } - - return getTablesFromPath(); - } - - public Map getTablesFromPath() - { - ImmutableMap.Builder builder = ImmutableMap.builder(); - try { - for (Path file : listFiles(Paths.get(tableDescriptionLocation))) { - if (Files.isRegularFile(file) && file.getFileName().toString().endsWith("json")) { - KinesisStreamDescription table = streamDescriptionCodec.fromJson(Files.readAllBytes(file)); - String schemaName = firstNonNull(table.schemaName(), defaultSchema); - log.debug("Kinesis table %s %s %s", schemaName, table.tableName(), table); - builder.put(new SchemaTableName(schemaName, table.tableName()), table); - } - } - - Map tableDefinitions = builder.buildOrThrow(); - log.debug("Loaded table definitions: %s", tableDefinitions.keySet()); - - return tableDefinitions; - } - catch (IOException e) { - throw new RuntimeException(e); - } - } - - /** - * Shutdown any periodic update jobs. - */ - @PreDestroy - public void shutdown() - { - this.s3TableConfigClient.run(); - } - - private static List listFiles(Path path) - { - if (path == null || !Files.isDirectory(path)) { - throw new TrinoException(KinesisErrorCode.KINESIS_METADATA_EXCEPTION, "Table description location does not exist or is not a directory"); - } - try (DirectoryStream stream = Files.newDirectoryStream(path)) { - ImmutableList.Builder builder = ImmutableList.builder(); - for (Path file : stream) { - builder.add(file); - } - - return builder.build(); - } - catch (IOException | DirectoryIteratorException e) { - throwIfUnchecked(e); - throw new RuntimeException(e); - } - } -} diff --git a/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/KinesisTableHandle.java b/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/KinesisTableHandle.java deleted file mode 100644 index 829ece850f12..000000000000 --- a/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/KinesisTableHandle.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.kinesis; - -import io.trino.spi.connector.ConnectorTableHandle; -import io.trino.spi.connector.SchemaTableName; - -import static java.util.Objects.requireNonNull; - -/** - * Class maintains all the properties of Trino Table - * - * @param schemaName The schema name for this table. Is set through configuration and read - * using {@link KinesisConfig#getDefaultSchema()}. Usually 'default'. - * @param tableName The table name used by Trino. - * @param streamName The stream name that is read from Kinesis - */ -public record KinesisTableHandle( - String schemaName, - String tableName, - String streamName, - String messageDataFormat, - KinesisCompressionCodec compressionCodec) - implements ConnectorTableHandle -{ - public KinesisTableHandle - { - requireNonNull(schemaName, "schemaName is null"); - requireNonNull(tableName, "tableName is null"); - requireNonNull(streamName, "streamName is null"); - requireNonNull(messageDataFormat, "messageDataFormat is null"); - requireNonNull(compressionCodec, "compressionCodec is null"); - } - - public SchemaTableName schemaTableName() - { - return new SchemaTableName(schemaName, tableName); - } -} diff --git a/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/KinesisTransactionHandle.java b/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/KinesisTransactionHandle.java deleted file mode 100644 index 528ee8e8a3ba..000000000000 --- a/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/KinesisTransactionHandle.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.kinesis; - -import io.trino.spi.connector.ConnectorTransactionHandle; - -public enum KinesisTransactionHandle - implements ConnectorTransactionHandle -{ - INSTANCE -} diff --git a/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/s3config/S3TableConfigClient.java b/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/s3config/S3TableConfigClient.java deleted file mode 100644 index 565d6f17ce43..000000000000 --- a/plugin/trino-kinesis/src/main/java/io/trino/plugin/kinesis/s3config/S3TableConfigClient.java +++ /dev/null @@ -1,220 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.kinesis.s3config; - -import com.amazonaws.AmazonClientException; -import com.amazonaws.services.s3.AmazonS3Client; -import com.amazonaws.services.s3.AmazonS3URI; -import com.amazonaws.services.s3.model.GetObjectRequest; -import com.amazonaws.services.s3.model.ListObjectsRequest; -import com.amazonaws.services.s3.model.ObjectListing; -import com.amazonaws.services.s3.model.S3Object; -import com.amazonaws.services.s3.model.S3ObjectSummary; -import com.google.common.collect.ImmutableMap; -import com.google.common.io.CharStreams; -import com.google.inject.Inject; -import io.airlift.json.JsonCodec; -import io.airlift.log.Logger; -import io.airlift.units.Duration; -import io.trino.plugin.kinesis.KinesisClientProvider; -import io.trino.plugin.kinesis.KinesisConfig; -import io.trino.plugin.kinesis.KinesisStreamDescription; -import io.trino.spi.connector.SchemaTableName; -import jakarta.annotation.PostConstruct; -import jakarta.annotation.PreDestroy; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.TimeUnit; - -import static io.airlift.concurrent.Threads.daemonThreadsNamed; -import static java.nio.charset.StandardCharsets.UTF_8; -import static java.util.Objects.requireNonNull; -import static java.util.concurrent.Executors.newSingleThreadScheduledExecutor; - -/** - * Utility class to retrieve table definitions from a common place on Amazon S3. - *

- * This is so that we can add new tables in a central "metastore" location without - * having to update every single node with the files. - *

- * This makes calls to Amazon AWS using the S3 client. - */ -public class S3TableConfigClient - implements Runnable -{ - private static final Logger log = Logger.get(S3TableConfigClient.class); - - private final KinesisClientProvider clientManager; - private final JsonCodec streamDescriptionCodec; - private final Duration tableDescriptionRefreshInterval; - private final Optional bucketUrl; - private volatile long lastCheck; - private volatile ScheduledFuture updateTaskHandle; - - private final Map descriptors = Collections.synchronizedMap(new HashMap<>()); - - private ScheduledExecutorService scheduler; - - @Inject - public S3TableConfigClient( - KinesisConfig connectorConfig, - KinesisClientProvider clientManager, - JsonCodec jsonCodec) - { - this.tableDescriptionRefreshInterval = connectorConfig.getTableDescriptionRefreshInterval(); - this.clientManager = requireNonNull(clientManager, "clientManager is null"); - this.streamDescriptionCodec = requireNonNull(jsonCodec, "jsonCodec is null"); - - // If using S3 start thread that periodically looks for updates - if (connectorConfig.getTableDescriptionLocation().startsWith("s3://")) { - this.bucketUrl = Optional.of(connectorConfig.getTableDescriptionLocation()); - } - else { - this.bucketUrl = Optional.empty(); - } - } - - @PostConstruct - protected void startS3Updates() - { - if (this.bucketUrl.isPresent()) { - scheduler = newSingleThreadScheduledExecutor(daemonThreadsNamed("s3-table-config-client-%s")); - this.updateTaskHandle = scheduler.scheduleAtFixedRate(this::updateTablesFromS3, 5_000, tableDescriptionRefreshInterval.toMillis(), TimeUnit.MILLISECONDS); - } - } - - @PreDestroy - public void destroy() - { - if (scheduler != null) { - scheduler.shutdownNow(); - } - } - - /** - * Indicates this class is being used and actively reading table definitions from S3. - */ - public boolean isUsingS3() - { - return bucketUrl.isPresent() && bucketUrl.get().startsWith("s3://"); - } - - /** - * Main entry point to get table definitions from S3 using bucket and object directory - * given in the configuration. - *

- * For safety, an immutable copy built from the internal map is returned. - */ - public Map getTablesFromS3() - { - updateTablesFromS3(); - Collection streamValues = this.descriptors.values(); - ImmutableMap.Builder builder = ImmutableMap.builder(); - for (KinesisStreamDescription stream : streamValues) { - builder.put(new SchemaTableName(stream.schemaName(), stream.tableName()), stream); - } - return builder.buildOrThrow(); - } - - @Override - public void run() - { - if (isUsingS3() && updateTaskHandle != null) { - updateTaskHandle.cancel(true); - } - } - - /** - * Call S3 to get the most recent object list. - *

- * This is an object list request to AWS in the given "directory". - */ - private List getObjectSummaries() - { - AmazonS3Client s3client = clientManager.getS3Client(); - AmazonS3URI directoryURI = new AmazonS3URI(bucketUrl.get()); - - List result = new ArrayList<>(); - try { - log.info("Getting the listing of objects in the S3 table config directory: bucket %s prefix %s :", directoryURI.getBucket(), directoryURI.getKey()); - ListObjectsRequest request = new ListObjectsRequest() - .withBucketName(directoryURI.getBucket()) - .withPrefix(directoryURI.getKey() + "/") - .withDelimiter("/") - .withMaxKeys(25); - ObjectListing response; - - do { - response = s3client.listObjects(request); - - result.addAll(response.getObjectSummaries()); - request.setMarker(response.getNextMarker()); - } - while (response.isTruncated()); - - log.info("Completed getting S3 object listing."); - } - catch (AmazonClientException e) { - log.error("Skipping update as faced error fetching table descriptions from S3 %s", e); - } - return result; - } - - /** - * Connect to S3 directory to look for new or updated table definitions and then - * update the map. - */ - private void updateTablesFromS3() - { - long now = System.currentTimeMillis(); - - AmazonS3Client s3client = clientManager.getS3Client(); - - for (S3ObjectSummary summary : getObjectSummaries()) { - if (!descriptors.containsKey(summary.getKey()) || summary.getLastModified().getTime() >= lastCheck) { - // New or updated file, so we must read from AWS - if (summary.getKey().endsWith("/")) { - continue; - } - - log.info("Getting : %s - %s", summary.getBucketName(), summary.getKey()); - S3Object object = s3client.getObject(new GetObjectRequest(summary.getBucketName(), summary.getKey())); - - try (BufferedReader reader = new BufferedReader(new InputStreamReader(object.getObjectContent(), UTF_8))) { - KinesisStreamDescription table = streamDescriptionCodec.fromJson(CharStreams.toString(reader)); - descriptors.put(summary.getKey(), table); - log.info("Put table description into the map from %s", summary.getKey()); - } - catch (IOException iox) { - log.error(iox, "Problem reading input stream from object."); - throw new RuntimeException(iox); - } - } - } - - log.info("Completed updating table definitions from S3."); - lastCheck = now; - } -} diff --git a/plugin/trino-kinesis/src/test/java/io/trino/plugin/kinesis/TestKinesisConfig.java b/plugin/trino-kinesis/src/test/java/io/trino/plugin/kinesis/TestKinesisConfig.java deleted file mode 100644 index 87fbc88338c0..000000000000 --- a/plugin/trino-kinesis/src/test/java/io/trino/plugin/kinesis/TestKinesisConfig.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.kinesis; - -import com.google.common.collect.ImmutableMap; -import io.airlift.configuration.testing.ConfigAssertions; -import io.airlift.units.Duration; -import org.junit.jupiter.api.Test; - -import java.util.Map; -import java.util.concurrent.TimeUnit; - -public class TestKinesisConfig -{ - @Test - public void testDefaults() - { - ConfigAssertions.assertRecordedDefaults(ConfigAssertions.recordDefaults(KinesisConfig.class) - .setDefaultSchema("default") - .setHideInternalColumns(true) - .setTableDescriptionLocation("etc/kinesis/") - .setAccessKey(null) - .setSecretKey(null) - .setAwsRegion("us-east-1") - .setTableDescriptionRefreshInterval(new Duration(10, TimeUnit.MINUTES)) - .setSleepTime(new Duration(1000, TimeUnit.MILLISECONDS)) - .setFetchAttempts(2) - .setMaxBatches(600) - .setBatchSize(10000) - .setLogBatches(true) - .setIteratorFromTimestamp(true) - .setIteratorOffsetSeconds(86400) - .setCheckpointEnabled(false) - .setDynamoReadCapacity(50) - .setDynamoWriteCapacity(10) - .setLogicalProcessName("process1") - .setIteratorNumber(0)); - } - - @Test - public void testExplicitPropertyMappings() - { - Map properties = ImmutableMap.builder() - .put("kinesis.table-description-location", "/var/lib/kinesis") - .put("kinesis.default-schema", "kinesis") - .put("kinesis.hide-internal-columns", "false") - .put("kinesis.access-key", "kinesis.accessKey") - .put("kinesis.secret-key", "kinesis.secretKey") - .put("kinesis.fetch-attempts", "3") - .put("kinesis.max-batches", "500") - .put("kinesis.aws-region", "us-west-1") - .put("kinesis.table-description-refresh-interval", "200ms") - .put("kinesis.sleep-time", "100ms") - .put("kinesis.batch-size", "9000") - .put("kinesis.log-batches", "false") - .put("kinesis.iterator-from-timestamp", "false") - .put("kinesis.iterator-offset-seconds", "36000") - .put("kinesis.checkpoint-enabled", "true") - .put("kinesis.dynamo-read-capacity", "100") - .put("kinesis.dynamo-write-capacity", "20") - .put("kinesis.checkpoint-logical-name", "process") - .put("kinesis.iterator-number", "1") - .buildOrThrow(); - - KinesisConfig expected = new KinesisConfig() - .setTableDescriptionLocation("/var/lib/kinesis") - .setDefaultSchema("kinesis") - .setHideInternalColumns(false) - .setAccessKey("kinesis.accessKey") - .setSecretKey("kinesis.secretKey") - .setAwsRegion("us-west-1") - .setTableDescriptionRefreshInterval(new Duration(200, TimeUnit.MILLISECONDS)) - .setFetchAttempts(3) - .setMaxBatches(500) - .setSleepTime(new Duration(100, TimeUnit.MILLISECONDS)) - .setBatchSize(9000) - .setLogBatches(false) - .setIteratorFromTimestamp(false) - .setIteratorOffsetSeconds(36000) - .setCheckpointEnabled(true) - .setDynamoReadCapacity(100) - .setDynamoWriteCapacity(20) - .setLogicalProcessName("process") - .setIteratorNumber(1); - - ConfigAssertions.assertFullMapping(properties, expected); - } -} diff --git a/plugin/trino-kinesis/src/test/java/io/trino/plugin/kinesis/TestKinesisPlugin.java b/plugin/trino-kinesis/src/test/java/io/trino/plugin/kinesis/TestKinesisPlugin.java deleted file mode 100644 index ad1f35530873..000000000000 --- a/plugin/trino-kinesis/src/test/java/io/trino/plugin/kinesis/TestKinesisPlugin.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.kinesis; - -import com.google.common.collect.ImmutableMap; -import io.trino.plugin.kinesis.util.TestUtils; -import io.trino.spi.connector.Connector; -import io.trino.spi.connector.ConnectorFactory; -import io.trino.spi.connector.ConnectorMetadata; -import io.trino.spi.connector.ConnectorTransactionHandle; -import io.trino.testing.TestingConnectorContext; -import org.junit.jupiter.api.Test; - -import static com.google.common.collect.Iterables.getOnlyElement; -import static io.trino.spi.transaction.IsolationLevel.READ_COMMITTED; -import static io.trino.testing.TestingConnectorSession.SESSION; -import static org.assertj.core.api.Assertions.assertThat; - -/** - * Test that the plug in API is satisfied and all of the required objects can be created. - *

- * This will not make any calls to AWS, it merely checks that all of the Plug in SPI - * objects are in place. - */ -public class TestKinesisPlugin -{ - @Test - public void testCreateConnector() - { - KinesisPlugin plugin = new KinesisPlugin(); - ConnectorFactory factory = getOnlyElement(plugin.getConnectorFactories()); - - String accessKey = "kinesis.accessKey"; - String secretKey = "kinesis.secretKey"; - - Connector c = factory.create("kinesis.test-connector", ImmutableMap.builder() - .put("kinesis.hide-internal-columns", "false") - .put("kinesis.access-key", TestUtils.noneToBlank(accessKey)) - .put("kinesis.secret-key", TestUtils.noneToBlank(secretKey)) - .put("bootstrap.quiet", "true") - .buildOrThrow(), new TestingConnectorContext()); - assertThat(c).isNotNull(); - - // Verify that the key objects have been created on the connector - assertThat(c.getRecordSetProvider()).isNotNull(); - assertThat(c.getSplitManager()).isNotNull(); - ConnectorMetadata md = c.getMetadata(SESSION, KinesisTransactionHandle.INSTANCE); - assertThat(md).isNotNull(); - - ConnectorTransactionHandle handle = c.beginTransaction(READ_COMMITTED, true, true); - assertThat(handle).isInstanceOf(KinesisTransactionHandle.class); - - c.shutdown(); - } -} diff --git a/plugin/trino-kinesis/src/test/java/io/trino/plugin/kinesis/TestKinesisTableDescriptionSupplier.java b/plugin/trino-kinesis/src/test/java/io/trino/plugin/kinesis/TestKinesisTableDescriptionSupplier.java deleted file mode 100644 index a001f6fabdc7..000000000000 --- a/plugin/trino-kinesis/src/test/java/io/trino/plugin/kinesis/TestKinesisTableDescriptionSupplier.java +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.kinesis; - -import com.google.common.collect.ImmutableMap; -import com.google.common.io.Resources; -import io.trino.plugin.kinesis.util.KinesisTestClientManager; -import io.trino.plugin.kinesis.util.MockKinesisClient; -import io.trino.plugin.kinesis.util.TestUtils; -import io.trino.spi.connector.ColumnHandle; -import io.trino.spi.connector.ColumnMetadata; -import io.trino.spi.connector.ConnectorTableMetadata; -import io.trino.spi.connector.ConnectorTransactionHandle; -import io.trino.spi.connector.SchemaTableName; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; -import org.junit.jupiter.api.parallel.Execution; - -import java.util.List; -import java.util.Map; -import java.util.Optional; - -import static io.trino.testing.TestingConnectorSession.SESSION; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; -import static org.junit.jupiter.api.parallel.ExecutionMode.CONCURRENT; - -@TestInstance(PER_CLASS) -@Execution(CONCURRENT) -public class TestKinesisTableDescriptionSupplier -{ - private KinesisConnector connector; - - @BeforeAll - public void start() - { - // Create dependent objects, including the minimal config needed for this test - Map properties = ImmutableMap.builder() - .put("kinesis.table-description-location", Resources.getResource("etc/kinesis").getPath()) - .put("kinesis.default-schema", "kinesis") - .put("kinesis.hide-internal-columns", "true") - .put("bootstrap.quiet", "true") - .buildOrThrow(); - - KinesisTestClientManager kinesisTestClientManager = new KinesisTestClientManager(); - MockKinesisClient mockClient = (MockKinesisClient) kinesisTestClientManager.getClient(); - mockClient.createStream("test123", 2); - mockClient.createStream("sampleTable", 2); - KinesisConnectorFactory kinesisConnectorFactory = new TestingKinesisConnectorFactory(kinesisTestClientManager); - - KinesisPlugin kinesisPlugin = new KinesisPlugin(kinesisConnectorFactory); - connector = TestUtils.createConnector(kinesisPlugin, properties, true); - } - - @Test - public void testTableDefinition() - { - KinesisMetadata metadata = (KinesisMetadata) connector.getMetadata(SESSION, new ConnectorTransactionHandle() {}); - SchemaTableName tblName = new SchemaTableName("prod", "test_table"); - KinesisTableHandle tableHandle = metadata.getTableHandle(SESSION, tblName, Optional.empty(), Optional.empty()); - assertThat(metadata).isNotNull(); - SchemaTableName tableSchemaName = tableHandle.schemaTableName(); - assertThat(tableSchemaName.getSchemaName()).isEqualTo("prod"); - assertThat(tableSchemaName.getTableName()).isEqualTo("test_table"); - assertThat(tableHandle.streamName()).isEqualTo("test_kinesis_stream"); - assertThat(tableHandle.messageDataFormat()).isEqualTo("json"); - Map columnHandles = metadata.getColumnHandles(SESSION, tableHandle); - assertThat(columnHandles.size()).isEqualTo(14); - assertThat(columnHandles.values().stream().filter(x -> ((KinesisColumnHandle) x).isInternal()).count()).isEqualTo(10); - } - - @Test - public void testRelatedObjects() - { - KinesisMetadata metadata = (KinesisMetadata) connector.getMetadata(SESSION, new ConnectorTransactionHandle() {}); - assertThat(metadata).isNotNull(); - - SchemaTableName tblName = new SchemaTableName("prod", "test_table"); - List schemas = metadata.listSchemaNames(null); - assertThat(schemas.size()).isEqualTo(1); - assertThat(schemas.get(0)).isEqualTo("prod"); - - KinesisTableHandle tblHandle = metadata.getTableHandle(null, tblName, Optional.empty(), Optional.empty()); - assertThat(tblHandle).isNotNull(); - assertThat(tblHandle.schemaName()).isEqualTo("prod"); - assertThat(tblHandle.tableName()).isEqualTo("test_table"); - assertThat(tblHandle.streamName()).isEqualTo("test_kinesis_stream"); - assertThat(tblHandle.messageDataFormat()).isEqualTo("json"); - - ConnectorTableMetadata tblMeta = metadata.getTableMetadata(null, tblHandle); - assertThat(tblMeta).isNotNull(); - assertThat(tblMeta.getTable().getSchemaName()).isEqualTo("prod"); - assertThat(tblMeta.getTable().getTableName()).isEqualTo("test_table"); - List columnList = tblMeta.getColumns(); - assertThat(columnList).isNotNull(); - - boolean foundServiceType = false; - boolean foundPartitionKey = false; - for (ColumnMetadata column : columnList) { - if (column.getName().equals("service_type")) { - foundServiceType = true; - assertThat(column.getType().getDisplayName()).isEqualTo("varchar(20)"); - } - if (column.getName().equals("_partition_key")) { - foundPartitionKey = true; - assertThat(column.getType().getDisplayName()).isEqualTo("varchar"); - } - } - assertThat(foundServiceType).isTrue(); - assertThat(foundPartitionKey).isTrue(); - } -} diff --git a/plugin/trino-kinesis/src/test/java/io/trino/plugin/kinesis/TestMinimalFunctionality.java b/plugin/trino-kinesis/src/test/java/io/trino/plugin/kinesis/TestMinimalFunctionality.java deleted file mode 100644 index 07cb2d3ceab5..000000000000 --- a/plugin/trino-kinesis/src/test/java/io/trino/plugin/kinesis/TestMinimalFunctionality.java +++ /dev/null @@ -1,172 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.kinesis; - -import com.amazonaws.services.kinesis.model.PutRecordsRequest; -import com.amazonaws.services.kinesis.model.PutRecordsRequestEntry; -import io.trino.Session; -import io.trino.metadata.Metadata; -import io.trino.metadata.QualifiedObjectName; -import io.trino.metadata.SessionPropertyManager; -import io.trino.metadata.TableHandle; -import io.trino.plugin.kinesis.util.EmbeddedKinesisStream; -import io.trino.plugin.kinesis.util.TestUtils; -import io.trino.security.AllowAllAccessControl; -import io.trino.spi.QueryId; -import io.trino.spi.security.Identity; -import io.trino.sql.query.QueryAssertions; -import io.trino.testing.QueryRunner; -import io.trino.testing.StandaloneQueryRunner; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; -import org.junit.jupiter.api.parallel.Execution; - -import java.io.File; -import java.nio.ByteBuffer; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import java.util.UUID; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static io.trino.spi.type.TimeZoneKey.UTC_KEY; -import static io.trino.testing.TransactionBuilder.transaction; -import static java.nio.charset.StandardCharsets.UTF_8; -import static java.util.Locale.ENGLISH; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; -import static org.junit.jupiter.api.parallel.ExecutionMode.SAME_THREAD; - -/** - * Note: this is an integration test that connects to AWS Kinesis. - *

- * Only run if you have an account setup where you can create streams and put/get records. - * You may incur AWS charges if you run this test. You probably want to setup an IAM - * user for your CI server to use. - */ -@TestInstance(PER_CLASS) -@Execution(SAME_THREAD) -public class TestMinimalFunctionality -{ - public static final Session SESSION = Session.builder(new SessionPropertyManager()) - .setIdentity(Identity.ofUser("user")) - .setSource("source") - .setCatalog("kinesis") - .setSchema("default") - .setTimeZoneKey(UTC_KEY) - .setLocale(ENGLISH) - .setQueryId(new QueryId("dummy")) - .build(); - private final String accessKey; - private final String secretKey; - - private final EmbeddedKinesisStream embeddedKinesisStream; - private String streamName; - private QueryRunner queryRunner; - private QueryAssertions assertions; - - public TestMinimalFunctionality() - { - accessKey = System.getProperty("kinesis.awsAccessKey"); - secretKey = System.getProperty("kinesis.awsSecretKey"); - embeddedKinesisStream = new EmbeddedKinesisStream(TestUtils.noneToBlank(accessKey), TestUtils.noneToBlank(secretKey)); - } - - @AfterAll - public void stop() - { - embeddedKinesisStream.close(); - } - - @BeforeEach - public void spinUp() - throws Exception - { - streamName = "test_" + UUID.randomUUID().toString().replaceAll("-", "_"); - - embeddedKinesisStream.createStream(2, streamName); - this.queryRunner = new StandaloneQueryRunner(SESSION); - assertions = new QueryAssertions(queryRunner); - Path tempDir = Files.createTempDirectory("tempdir"); - File baseFile = new File("src/test/resources/tableDescriptions/EmptyTable.json"); - File file = new File(tempDir.toAbsolutePath().toString() + "/" + streamName + ".json"); - - try (Stream lines = Files.lines(baseFile.toPath())) { - List replaced = lines - .map(line -> line.replaceAll("TABLE_NAME", streamName)) - .map(line -> line.replaceAll("STREAM_NAME", streamName)) - .collect(Collectors.toList()); - Files.write(file.toPath(), replaced); - } - TestUtils.installKinesisPlugin(queryRunner, tempDir.toAbsolutePath().toString(), - TestUtils.noneToBlank(accessKey), TestUtils.noneToBlank(secretKey)); - } - - private void createMessages(String streamName, long count) - { - PutRecordsRequest putRecordsRequest = new PutRecordsRequest(); - putRecordsRequest.setStreamName(streamName); - List putRecordsRequestEntryList = new ArrayList<>(); - for (int i = 0; i < count; i++) { - PutRecordsRequestEntry putRecordsRequestEntry = new PutRecordsRequestEntry(); - putRecordsRequestEntry.setData(ByteBuffer.wrap(UUID.randomUUID().toString().getBytes(UTF_8))); - putRecordsRequestEntry.setPartitionKey(Long.toString(i)); - putRecordsRequestEntryList.add(putRecordsRequestEntry); - } - - putRecordsRequest.setRecords(putRecordsRequestEntryList); - embeddedKinesisStream.getKinesisClient().putRecords(putRecordsRequest); - } - - @Test - public void testStreamExists() - { - QualifiedObjectName name = new QualifiedObjectName("kinesis", "default", streamName); - - Metadata metadata = queryRunner.getPlannerContext().getMetadata(); - transaction(queryRunner.getTransactionManager(), metadata, new AllowAllAccessControl()) - .singleStatement() - .execute(SESSION, session -> { - Optional handle = metadata.getTableHandle(session, name); - assertThat(handle).isPresent(); - }); - } - - @Test - public void testStreamHasData() - { - assertThat(assertions.query("SELECT COUNT(1) FROM " + streamName)) - .matches("VALUES 0"); - - long count = 500L; - createMessages(streamName, count); - - assertThat(assertions.query("SELECT COUNT(1) FROM " + streamName)) - .matches("VALUES %s".formatted(count)); - } - - @AfterEach - public void tearDown() - { - embeddedKinesisStream.deleteStream(streamName); - queryRunner.close(); - queryRunner = null; - } -} diff --git a/plugin/trino-kinesis/src/test/java/io/trino/plugin/kinesis/TestRecordAccess.java b/plugin/trino-kinesis/src/test/java/io/trino/plugin/kinesis/TestRecordAccess.java deleted file mode 100644 index 223f0c3c0966..000000000000 --- a/plugin/trino-kinesis/src/test/java/io/trino/plugin/kinesis/TestRecordAccess.java +++ /dev/null @@ -1,226 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.kinesis; - -import com.amazonaws.services.kinesis.model.PutRecordsRequest; -import com.amazonaws.services.kinesis.model.PutRecordsRequestEntry; -import io.airlift.log.Logger; -import io.trino.Session; -import io.trino.metadata.Metadata; -import io.trino.metadata.QualifiedObjectName; -import io.trino.metadata.TableHandle; -import io.trino.plugin.kinesis.util.MockKinesisClient; -import io.trino.plugin.kinesis.util.TestUtils; -import io.trino.security.AllowAllAccessControl; -import io.trino.spi.type.BigintType; -import io.trino.spi.type.Type; -import io.trino.testing.MaterializedResult; -import io.trino.testing.MaterializedRow; -import io.trino.testing.QueryRunner; -import io.trino.testing.StandaloneQueryRunner; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; -import org.junit.jupiter.api.parallel.Execution; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import java.util.UUID; -import java.util.zip.GZIPOutputStream; - -import static io.trino.testing.TestingSession.testSessionBuilder; -import static io.trino.testing.TransactionBuilder.transaction; -import static java.lang.String.format; -import static java.nio.charset.StandardCharsets.UTF_8; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; -import static org.junit.jupiter.api.parallel.ExecutionMode.SAME_THREAD; - -/** - * Test record access and querying along with all associated setup. - *

- * This is a lighter weight integration test that exercises more parts of - * the plug in without requiring an actual Kinesis connection. It uses the mock - * kinesis client so no AWS activity will occur. - */ -@TestInstance(PER_CLASS) -@Execution(SAME_THREAD) -public class TestRecordAccess -{ - private static final Logger log = Logger.get(TestRecordAccess.class); - - private static final Session SESSION = testSessionBuilder() - .setCatalog("kinesis") - .setSchema("default") - .build(); - - private String dummyStreamName; - private String jsonStreamName; - private String jsonGzipCompressStreamName; - private String jsonAutomaticCompressStreamName; - private QueryRunner queryRunner; - private MockKinesisClient mockClient; - - @BeforeAll - public void start() - { - dummyStreamName = "test123"; - jsonStreamName = "sampleTable"; - jsonGzipCompressStreamName = "sampleGzipCompressTable"; - jsonAutomaticCompressStreamName = "sampleAutomaticCompressTable"; - queryRunner = new StandaloneQueryRunner(SESSION); - mockClient = TestUtils.installKinesisPlugin(queryRunner); - } - - @AfterAll - public void stop() - { - queryRunner.close(); - queryRunner = null; - } - - private void createDummyMessages(String streamName, int count) - { - PutRecordsRequest putRecordsRequest = new PutRecordsRequest(); - putRecordsRequest.setStreamName(streamName); - List putRecordsRequestEntryList = new ArrayList<>(); - for (int i = 0; i < count; i++) { - PutRecordsRequestEntry putRecordsRequestEntry = new PutRecordsRequestEntry(); - putRecordsRequestEntry.setData(ByteBuffer.wrap(UUID.randomUUID().toString().getBytes(UTF_8))); - putRecordsRequestEntry.setPartitionKey(Long.toString(i)); - putRecordsRequestEntryList.add(putRecordsRequestEntry); - } - - putRecordsRequest.setRecords(putRecordsRequestEntryList); - mockClient.putRecords(putRecordsRequest); - } - - private void createJsonMessages(String streamName, int count, int idStart, boolean compress) - { - String jsonFormat = "{\"id\" : %d, \"name\" : \"%s\"}"; - PutRecordsRequest putRecordsRequest = new PutRecordsRequest(); - putRecordsRequest.setStreamName(streamName); - List putRecordsRequestEntryList = new ArrayList<>(); - for (int i = 0; i < count; i++) { - PutRecordsRequestEntry putRecordsRequestEntry = new PutRecordsRequestEntry(); - long id = idStart + i; - String name = UUID.randomUUID().toString(); - String jsonVal = format(jsonFormat, id, name); - - // ? with StandardCharsets.UTF_8 - if (compress) { - putRecordsRequestEntry.setData(ByteBuffer.wrap(compressMessage(jsonVal.getBytes(UTF_8)))); - } - else { - putRecordsRequestEntry.setData(ByteBuffer.wrap(jsonVal.getBytes(UTF_8))); - } - putRecordsRequestEntry.setPartitionKey(Long.toString(id)); - putRecordsRequestEntryList.add(putRecordsRequestEntry); - } - - putRecordsRequest.setRecords(putRecordsRequestEntryList); - mockClient.putRecords(putRecordsRequest); - } - - private static byte[] compressMessage(byte[] data) - { - try (ByteArrayOutputStream byteOS = new ByteArrayOutputStream()) { - GZIPOutputStream gzipOS = new GZIPOutputStream(byteOS); - gzipOS.write(data); - gzipOS.close(); //Explict close is needed to end the stream - return byteOS.toByteArray(); - } - catch (IOException e) { - throw new RuntimeException(e); - } - } - - @Test - public void testStreamExists() - { - QualifiedObjectName name = new QualifiedObjectName("kinesis", "default", dummyStreamName); - - Metadata metadata = queryRunner.getPlannerContext().getMetadata(); - transaction(queryRunner.getTransactionManager(), metadata, new AllowAllAccessControl()) - .singleStatement() - .execute(SESSION, session -> { - Optional handle = metadata.getTableHandle(session, name); - assertThat(handle).isPresent(); - }); - log.info("Completed first test (access table handle)"); - } - - @Test - public void testStreamHasData() - { - MaterializedResult result = queryRunner.execute("Select count(1) from " + dummyStreamName); - MaterializedResult expected = MaterializedResult.resultBuilder(SESSION, BigintType.BIGINT) - .row(0) - .build(); - - assertThat(result.getRowCount()).isEqualTo(expected.getRowCount()); - - int count = 500; - createDummyMessages(dummyStreamName, count); - - result = queryRunner.execute("SELECT count(1) from " + dummyStreamName); - - expected = MaterializedResult.resultBuilder(SESSION, BigintType.BIGINT) - .row(count) - .build(); - - assertThat(result.getRowCount()).isEqualTo(expected.getRowCount()); - log.info("Completed second test (select counts)"); - } - - @Test - public void testJsonStream() - { - testJsonStream(4, 0, jsonStreamName); - testJsonStream(0, 4, jsonGzipCompressStreamName); - testJsonStream(2, 2, jsonAutomaticCompressStreamName); - } - - private void testJsonStream(int uncompressedMessages, int compressedMessages, String streamName) - { - // Simple case: add a few specific items, query object and internal fields: - if (uncompressedMessages > 0) { - createJsonMessages(streamName, uncompressedMessages, 100, false); - } - if (compressedMessages > 0) { - createJsonMessages(streamName, compressedMessages, 100 + uncompressedMessages, true); - } - MaterializedResult result = queryRunner.execute("Select id, name, _shard_id, _message_length, _message from " + streamName + " where _message_length >= 1"); - assertThat(result.getRowCount()).isEqualTo(uncompressedMessages + compressedMessages); - - List types = result.getTypes(); - assertThat(types.size()).isEqualTo(5); - assertThat(types.get(0).toString()).isEqualTo("bigint"); - assertThat(types.get(1).toString()).isEqualTo("varchar"); - log.info("Types : %s", types); - - List rows = result.getMaterializedRows(); - assertThat(rows.size()).isEqualTo(uncompressedMessages + compressedMessages); - for (MaterializedRow row : rows) { - assertThat(row.getFieldCount()).isEqualTo(5); - assertThat((long) row.getFields().get(0) >= 100).isTrue(); - log.info("ROW: %s", row); - } - } -} diff --git a/plugin/trino-kinesis/src/test/java/io/trino/plugin/kinesis/TestingKinesisConnectorFactory.java b/plugin/trino-kinesis/src/test/java/io/trino/plugin/kinesis/TestingKinesisConnectorFactory.java deleted file mode 100644 index 2adc56ae4765..000000000000 --- a/plugin/trino-kinesis/src/test/java/io/trino/plugin/kinesis/TestingKinesisConnectorFactory.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.kinesis; - -import com.google.inject.Injector; -import com.google.inject.Scopes; -import com.google.inject.TypeLiteral; -import io.airlift.bootstrap.Bootstrap; -import io.airlift.json.JsonModule; -import io.trino.plugin.base.TypeDeserializerModule; -import io.trino.spi.NodeManager; -import io.trino.spi.connector.Connector; -import io.trino.spi.connector.ConnectorContext; -import io.trino.spi.connector.SchemaTableName; - -import java.util.Map; -import java.util.function.Supplier; - -import static com.google.common.base.Throwables.throwIfUnchecked; -import static java.util.Objects.requireNonNull; - -public class TestingKinesisConnectorFactory - extends KinesisConnectorFactory -{ - private KinesisClientProvider kinesisClientProvider; - - public TestingKinesisConnectorFactory(KinesisClientProvider kinesisClientProvider) - { - super(); - this.kinesisClientProvider = kinesisClientProvider; - } - - @Override - public Connector create(String catalogName, Map config, ConnectorContext context) - { - requireNonNull(catalogName, "catalogName is null"); - requireNonNull(config, "config is null"); - - try { - Bootstrap app = new Bootstrap( - new JsonModule(), - new TypeDeserializerModule(context.getTypeManager()), - new KinesisModule(), - binder -> { - binder.bind(NodeManager.class).toInstance(context.getNodeManager()); - binder.bind(KinesisClientProvider.class).toInstance(kinesisClientProvider); - binder.bind(new TypeLiteral>>() {}).to(KinesisTableDescriptionSupplier.class).in(Scopes.SINGLETON); - }); - - Injector injector = app - .doNotInitializeLogging() - .setRequiredConfigurationProperties(config) - .initialize(); - - KinesisConnector connector = injector.getInstance(KinesisConnector.class); - return connector; - } - catch (Exception e) { - throwIfUnchecked(e); - throw new RuntimeException(e); - } - } -} diff --git a/plugin/trino-kinesis/src/test/java/io/trino/plugin/kinesis/s3config/TestS3TableConfigClient.java b/plugin/trino-kinesis/src/test/java/io/trino/plugin/kinesis/s3config/TestS3TableConfigClient.java deleted file mode 100644 index 7d17de5f6d14..000000000000 --- a/plugin/trino-kinesis/src/test/java/io/trino/plugin/kinesis/s3config/TestS3TableConfigClient.java +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.kinesis.s3config; - -import com.amazonaws.services.s3.AmazonS3URI; -import com.google.common.collect.ImmutableMap; -import io.airlift.log.Logger; -import io.trino.plugin.kinesis.KinesisConnector; -import io.trino.plugin.kinesis.KinesisMetadata; -import io.trino.plugin.kinesis.KinesisPlugin; -import io.trino.plugin.kinesis.KinesisTableHandle; -import io.trino.plugin.kinesis.util.TestUtils; -import io.trino.spi.connector.ColumnHandle; -import io.trino.spi.connector.ConnectorTransactionHandle; -import io.trino.spi.connector.SchemaTableName; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; -import org.junit.jupiter.api.parallel.Execution; - -import java.util.Map; -import java.util.Optional; - -import static io.trino.testing.TestingConnectorSession.SESSION; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; -import static org.junit.jupiter.api.parallel.ExecutionMode.SAME_THREAD; - -@TestInstance(PER_CLASS) -@Execution(SAME_THREAD) -public class TestS3TableConfigClient -{ - private static final Logger log = Logger.get(TestS3TableConfigClient.class); - - private final String tableDescriptionS3; - private final String accessKey; - private final String secretKey; - - public TestS3TableConfigClient() - { - tableDescriptionS3 = System.getProperty("kinesis.test-table-description-location"); - accessKey = System.getProperty("kinesis.awsAccessKey"); - secretKey = System.getProperty("kinesis.awsSecretKey"); - } - - @Test - public void testS3URIValues() - { - // Verify that S3URI values will work: - AmazonS3URI uri1 = new AmazonS3URI("s3://our.data.warehouse/prod/client_actions"); - assertThat(uri1.getKey()).isNotNull(); - assertThat(uri1.getBucket()).isNotNull(); - - assertThat(uri1.toString()).isEqualTo("s3://our.data.warehouse/prod/client_actions"); - assertThat(uri1.getBucket()).isEqualTo("our.data.warehouse"); - assertThat(uri1.getKey()).isEqualTo("prod/client_actions"); - assertThat(uri1.getRegion()).isNull(); - - // show info: - log.info("Tested out URI1 : %s", uri1); - - AmazonS3URI uri2 = new AmazonS3URI("s3://some.big.bucket/long/complex/path"); - assertThat(uri2.getKey()).isNotNull(); - assertThat(uri2.getBucket()).isNotNull(); - - assertThat(uri2.toString()).isEqualTo("s3://some.big.bucket/long/complex/path"); - assertThat(uri2.getBucket()).isEqualTo("some.big.bucket"); - assertThat(uri2.getKey()).isEqualTo("long/complex/path"); - assertThat(uri2.getRegion()).isNull(); - - // info: - log.info("Tested out URI2 : %s", uri2); - - AmazonS3URI uri3 = new AmazonS3URI("s3://trino.kinesis.config/unit-test/trino-kinesis"); - assertThat(uri3.getKey()).isNotNull(); - assertThat(uri3.getBucket()).isNotNull(); - - assertThat(uri3.toString()).isEqualTo("s3://trino.kinesis.config/unit-test/trino-kinesis"); - assertThat(uri3.getBucket()).isEqualTo("trino.kinesis.config"); - assertThat(uri3.getKey()).isEqualTo("unit-test/trino-kinesis"); - } - - @Test - public void testTableReading() - { - // To run this test: setup an S3 bucket with a folder for unit testing, and put - // MinimalTable.json in that folder. - - // Create dependent objects, including the minimal config needed for this test - Map properties = ImmutableMap.builder() - .put("kinesis.table-description-location", tableDescriptionS3) - .put("kinesis.default-schema", "kinesis") - .put("kinesis.hide-internal-columns", "false") - .put("kinesis.access-key", TestUtils.noneToBlank(accessKey)) - .put("kinesis.secret-key", TestUtils.noneToBlank(secretKey)) - .put("bootstrap.quiet", "true") - .buildOrThrow(); - - KinesisPlugin kinesisPlugin = new KinesisPlugin(); - KinesisConnector kinesisConnector = TestUtils.createConnector(kinesisPlugin, properties, false); - - // Sleep for 10 seconds to ensure that we've loaded the tables: - try { - Thread.sleep(10000); - log.info("done sleeping, will now try to read the tables."); - } - catch (InterruptedException e) { - log.error("interrupted ..."); - } - - KinesisMetadata metadata = (KinesisMetadata) kinesisConnector.getMetadata(SESSION, new ConnectorTransactionHandle() {}); - SchemaTableName tblName = new SchemaTableName("default", "test123"); - KinesisTableHandle tableHandle = metadata.getTableHandle(SESSION, tblName, Optional.empty(), Optional.empty()); - assertThat(metadata).isNotNull(); - SchemaTableName tableSchemaName = tableHandle.schemaTableName(); - assertThat(tableSchemaName.getSchemaName()).isEqualTo("default"); - assertThat(tableSchemaName.getTableName()).isEqualTo("test123"); - assertThat(tableHandle.streamName()).isEqualTo("test123"); - assertThat(tableHandle.messageDataFormat()).isEqualTo("json"); - Map columnHandles = metadata.getColumnHandles(SESSION, tableHandle); - assertThat(columnHandles.size()).isEqualTo(12); - } -} diff --git a/plugin/trino-kinesis/src/test/java/io/trino/plugin/kinesis/util/EmbeddedKinesisStream.java b/plugin/trino-kinesis/src/test/java/io/trino/plugin/kinesis/util/EmbeddedKinesisStream.java deleted file mode 100644 index 4c888df7ba25..000000000000 --- a/plugin/trino-kinesis/src/test/java/io/trino/plugin/kinesis/util/EmbeddedKinesisStream.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.kinesis.util; - -import com.amazonaws.auth.BasicAWSCredentials; -import com.amazonaws.services.kinesis.AmazonKinesisClient; -import com.amazonaws.services.kinesis.model.CreateStreamRequest; -import com.amazonaws.services.kinesis.model.DeleteStreamRequest; -import com.amazonaws.services.kinesis.model.DescribeStreamRequest; -import com.amazonaws.services.kinesis.model.StreamDescription; - -import java.io.Closeable; - -import static java.util.concurrent.TimeUnit.MILLISECONDS; - -public class EmbeddedKinesisStream - implements Closeable -{ - private final AmazonKinesisClient amazonKinesisClient; - - public EmbeddedKinesisStream(String accessKey, String secretKey) - { - this.amazonKinesisClient = new AmazonKinesisClient(new BasicAWSCredentials(accessKey, secretKey)); - } - - @Override - public void close() {} - - private String checkStreamStatus(String streamName) - { - DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest(); - describeStreamRequest.setStreamName(streamName); - - StreamDescription streamDescription = amazonKinesisClient.describeStream(describeStreamRequest).getStreamDescription(); - return streamDescription.getStreamStatus(); - } - - public void createStream(int shardCount, String streamName) - { - CreateStreamRequest createStreamRequest = new CreateStreamRequest(); - createStreamRequest.setStreamName(streamName); - createStreamRequest.setShardCount(shardCount); - - amazonKinesisClient.createStream(createStreamRequest); - try { - while (!checkStreamStatus(streamName).equals("ACTIVE")) { - MILLISECONDS.sleep(1000); - } - } - catch (Exception e) { - } - } - - public AmazonKinesisClient getKinesisClient() - { - return amazonKinesisClient; - } - - public void deleteStream(String streamName) - { - DeleteStreamRequest deleteStreamRequest = new DeleteStreamRequest(); - deleteStreamRequest.setStreamName(streamName); - amazonKinesisClient.deleteStream(deleteStreamRequest); - } -} diff --git a/plugin/trino-kinesis/src/test/java/io/trino/plugin/kinesis/util/KinesisTestClientManager.java b/plugin/trino-kinesis/src/test/java/io/trino/plugin/kinesis/util/KinesisTestClientManager.java deleted file mode 100644 index e361865efb62..000000000000 --- a/plugin/trino-kinesis/src/test/java/io/trino/plugin/kinesis/util/KinesisTestClientManager.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.kinesis.util; - -import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient; -import com.amazonaws.services.kinesis.AmazonKinesisClient; -import com.amazonaws.services.s3.AmazonS3Client; -import io.trino.plugin.kinesis.KinesisClientProvider; - -/** - * Test implementation of KinesisClientProvider that incorporates a mock Kinesis client. - */ -public class KinesisTestClientManager - implements KinesisClientProvider -{ - private final AmazonKinesisClient client = new MockKinesisClient(); - private final AmazonDynamoDBClient dynamoDBClient; - private final AmazonS3Client amazonS3Client; - - public KinesisTestClientManager() - { - this.dynamoDBClient = new AmazonDynamoDBClient(); - this.amazonS3Client = new AmazonS3Client(); - } - - @Override - public AmazonKinesisClient getClient() - { - return client; - } - - @Override - public AmazonDynamoDBClient getDynamoDbClient() - { - return this.dynamoDBClient; - } - - @Override - public AmazonS3Client getS3Client() - { - return amazonS3Client; - } -} diff --git a/plugin/trino-kinesis/src/test/java/io/trino/plugin/kinesis/util/MockKinesisClient.java b/plugin/trino-kinesis/src/test/java/io/trino/plugin/kinesis/util/MockKinesisClient.java deleted file mode 100644 index 31d64fd6bad4..000000000000 --- a/plugin/trino-kinesis/src/test/java/io/trino/plugin/kinesis/util/MockKinesisClient.java +++ /dev/null @@ -1,529 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.kinesis.util; - -import com.amazonaws.AmazonClientException; -import com.amazonaws.AmazonServiceException; -import com.amazonaws.AmazonWebServiceRequest; -import com.amazonaws.ResponseMetadata; -import com.amazonaws.services.kinesis.AmazonKinesisClient; -import com.amazonaws.services.kinesis.model.CreateStreamRequest; -import com.amazonaws.services.kinesis.model.CreateStreamResult; -import com.amazonaws.services.kinesis.model.DescribeStreamRequest; -import com.amazonaws.services.kinesis.model.DescribeStreamResult; -import com.amazonaws.services.kinesis.model.GetRecordsRequest; -import com.amazonaws.services.kinesis.model.GetRecordsResult; -import com.amazonaws.services.kinesis.model.GetShardIteratorRequest; -import com.amazonaws.services.kinesis.model.GetShardIteratorResult; -import com.amazonaws.services.kinesis.model.ListStreamsRequest; -import com.amazonaws.services.kinesis.model.ListStreamsResult; -import com.amazonaws.services.kinesis.model.ListTagsForStreamRequest; -import com.amazonaws.services.kinesis.model.ListTagsForStreamResult; -import com.amazonaws.services.kinesis.model.PutRecordRequest; -import com.amazonaws.services.kinesis.model.PutRecordResult; -import com.amazonaws.services.kinesis.model.PutRecordsRequest; -import com.amazonaws.services.kinesis.model.PutRecordsRequestEntry; -import com.amazonaws.services.kinesis.model.PutRecordsResult; -import com.amazonaws.services.kinesis.model.PutRecordsResultEntry; -import com.amazonaws.services.kinesis.model.Record; -import com.amazonaws.services.kinesis.model.SequenceNumberRange; -import com.amazonaws.services.kinesis.model.Shard; -import com.amazonaws.services.kinesis.model.StreamDescription; - -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; - -import static java.lang.Integer.parseInt; - -/** - * Mock kinesis client for testing that is primarily used for reading from the - * stream as we do here in Trino. - *

- * This is to help prove that the API is being used correctly and debug any - * issues that arise without incurring AWS load and charges. It is far from a complete - * implementation of Kinesis. - *

- */ -public class MockKinesisClient - extends AmazonKinesisClient -{ - private final List streams = new ArrayList<>(); - - public static class InternalShard - extends Shard - { - private final List recs = new ArrayList<>(); - private final int index; - - public InternalShard(String streamName, int index) - { - super(); - this.index = index; - this.setShardId(streamName + "_" + this.index); - } - - public List getRecords() - { - return recs; - } - - public List getRecordsFrom(ShardIterator iterator) - { - List returnRecords = new ArrayList<>(); - - for (Record record : this.recs) { - if (parseInt(record.getSequenceNumber()) >= iterator.recordIndex) { - returnRecords.add(record); - } - } - - return returnRecords; - } - - public int getIndex() - { - return index; - } - - public void addRecord(Record rec) - { - recs.add(rec); - } - - public void clearRecords() - { - recs.clear(); - } - } - - public static class InternalStream - { - private final String streamName; - private final String streamAmazonResourceName; - private String streamStatus = "CREATING"; - private List shards = new ArrayList<>(); - private int sequenceNo = 100; - private int nextShard; - - public InternalStream(String streamName, int shardCount, boolean isActive) - { - this.streamName = streamName; - this.streamAmazonResourceName = "local:fake.stream:" + streamName; - if (isActive) { - this.streamStatus = "ACTIVE"; - } - - for (int i = 0; i < shardCount; i++) { - InternalShard newShard = new InternalShard(this.streamName, i); - newShard.setSequenceNumberRange(new SequenceNumberRange().withStartingSequenceNumber("100").withEndingSequenceNumber("999")); - this.shards.add(newShard); - } - } - - public String getStreamName() - { - return streamName; - } - - public String getStreamAmazonResourceName() - { - return streamAmazonResourceName; - } - - public String getStreamStatus() - { - return streamStatus; - } - - public List getShards() - { - return shards; - } - - public List getShardsFrom(String afterShardId) - { - String[] comps = afterShardId.split("_"); - if (comps.length == 2) { - List returnArray = new ArrayList<>(); - int afterIndex = parseInt(comps[1]); - if (shards.size() > afterIndex + 1) { - for (InternalShard shard : shards) { - if (shard.getIndex() > afterIndex) { - returnArray.add(shard); - } - } - } - - return returnArray; - } - return new ArrayList<>(); - } - - public PutRecordResult putRecord(ByteBuffer data, String partitionKey) - { - // Create record and insert into the shards. Initially just do it - // on a round robin basis. - long timestamp = System.currentTimeMillis() - 50000; - Record record = new Record(); - record = record.withData(data).withPartitionKey(partitionKey).withSequenceNumber(String.valueOf(sequenceNo)); - record.setApproximateArrivalTimestamp(new Date(timestamp)); - - if (nextShard == shards.size()) { - nextShard = 0; - } - InternalShard shard = shards.get(nextShard); - shard.addRecord(record); - - PutRecordResult result = new PutRecordResult(); - result.setSequenceNumber(String.valueOf(sequenceNo)); - result.setShardId(shard.getShardId()); - - nextShard++; - sequenceNo++; - - return result; - } - } - - public static class ShardIterator - { - public final String streamId; - public final int shardIndex; - public int recordIndex; - - public ShardIterator(String streamId, int shardIndex, int recordIndex) - { - this.streamId = streamId; - this.shardIndex = shardIndex; - this.recordIndex = recordIndex; - } - - public String makeString() - { - return this.streamId + "_" + this.shardIndex + "_" + this.recordIndex; - } - - public static ShardIterator fromStreamAndShard(String streamName, String shardId) - { - ShardIterator newInst = null; - String[] comps = shardId.split("_"); - if (streamName.equals(comps[0]) && comps[1].matches("[0-9]+")) { - newInst = new ShardIterator(comps[0], parseInt(comps[1]), 0); - } - - return newInst; - } - - public static ShardIterator fromString(String input) - { - ShardIterator newInst = null; - String[] comps = input.split("_"); - if (comps.length == 3) { - if (comps[1].matches("[0-9]+") && comps[2].matches("[0-9]+")) { - newInst = new ShardIterator(comps[0], parseInt(comps[1]), parseInt(comps[2])); - } - } - - return newInst; - } - } - - public MockKinesisClient() - { - super(); - } - - protected InternalStream getStream(String name) - { - InternalStream foundStream = null; - for (InternalStream stream : this.streams) { - if (stream.getStreamName().equals(name)) { - foundStream = stream; - break; - } - } - return foundStream; - } - - protected List getShards(InternalStream theStream) - { - List externalList = new ArrayList<>(); - for (InternalShard intshard : theStream.getShards()) { - externalList.add(intshard); - } - - return externalList; - } - - protected List getShards(InternalStream theStream, String fromShardId) - { - List externalList = new ArrayList<>(); - for (InternalShard intshard : theStream.getShardsFrom(fromShardId)) { - externalList.add(intshard); - } - - return externalList; - } - - @Override - public PutRecordResult putRecord(PutRecordRequest putRecordRequest) - throws AmazonClientException - { - // Setup method to add a new record: - InternalStream theStream = this.getStream(putRecordRequest.getStreamName()); - if (theStream != null) { - return theStream.putRecord(putRecordRequest.getData(), putRecordRequest.getPartitionKey()); - } - throw new AmazonClientException("This stream does not exist!"); - } - - @Override - public CreateStreamResult createStream(CreateStreamRequest createStreamRequest) - throws AmazonClientException - { - // Setup method to create a new stream: - InternalStream stream = new InternalStream(createStreamRequest.getStreamName(), createStreamRequest.getShardCount(), true); - this.streams.add(stream); - return new CreateStreamResult(); - } - - @Override - public CreateStreamResult createStream(String streamName, Integer integer) - throws AmazonClientException - { - return this.createStream(new CreateStreamRequest().withStreamName(streamName).withShardCount(integer)); - } - - @Override - public PutRecordsResult putRecords(PutRecordsRequest putRecordsRequest) - throws AmazonClientException - { - // Setup method to add a batch of new records: - InternalStream theStream = this.getStream(putRecordsRequest.getStreamName()); - if (theStream != null) { - PutRecordsResult result = new PutRecordsResult(); - List resultList = new ArrayList<>(); - for (PutRecordsRequestEntry entry : putRecordsRequest.getRecords()) { - PutRecordResult putResult = theStream.putRecord(entry.getData(), entry.getPartitionKey()); - resultList.add(new PutRecordsResultEntry().withShardId(putResult.getShardId()).withSequenceNumber(putResult.getSequenceNumber())); - } - - result.setRecords(resultList); - return result; - } - throw new AmazonClientException("This stream does not exist!"); - } - - @Override - public DescribeStreamResult describeStream(DescribeStreamRequest describeStreamRequest) - throws AmazonClientException - { - InternalStream theStream = this.getStream(describeStreamRequest.getStreamName()); - if (theStream == null) { - throw new AmazonClientException("This stream does not exist!"); - } - - StreamDescription desc = new StreamDescription(); - desc = desc.withStreamName(theStream.getStreamName()).withStreamStatus(theStream.getStreamStatus()).withStreamARN(theStream.getStreamAmazonResourceName()); - - if (describeStreamRequest.getExclusiveStartShardId() == null || describeStreamRequest.getExclusiveStartShardId().isEmpty()) { - desc.setShards(this.getShards(theStream)); - desc.setHasMoreShards(false); - } - else { - // Filter from given shard Id, or may not have any more - String startId = describeStreamRequest.getExclusiveStartShardId(); - desc.setShards(this.getShards(theStream, startId)); - desc.setHasMoreShards(false); - } - - DescribeStreamResult result = new DescribeStreamResult(); - result = result.withStreamDescription(desc); - return result; - } - - @Override - public GetShardIteratorResult getShardIterator(GetShardIteratorRequest getShardIteratorRequest) - throws AmazonClientException - { - ShardIterator iter = ShardIterator.fromStreamAndShard(getShardIteratorRequest.getStreamName(), getShardIteratorRequest.getShardId()); - if (iter == null) { - throw new AmazonClientException("Bad stream or shard iterator!"); - } - - InternalStream theStream = this.getStream(iter.streamId); - if (theStream == null) { - throw new AmazonClientException("Unknown stream or bad shard iterator!"); - } - - String seqAsString = getShardIteratorRequest.getStartingSequenceNumber(); - if (seqAsString != null && !seqAsString.isEmpty() && getShardIteratorRequest.getShardIteratorType().equals("AFTER_SEQUENCE_NUMBER")) { - int sequence = parseInt(seqAsString); - iter.recordIndex = sequence + 1; - } - else { - iter.recordIndex = 100; - } - - GetShardIteratorResult result = new GetShardIteratorResult(); - return result.withShardIterator(iter.makeString()); - } - - @Override - public GetRecordsResult getRecords(GetRecordsRequest getRecordsRequest) - throws AmazonClientException - { - ShardIterator iterator = ShardIterator.fromString(getRecordsRequest.getShardIterator()); - if (iterator == null) { - throw new AmazonClientException("Bad shard iterator."); - } - - // TODO: incorporate maximum batch size (getRecordsRequest.getLimit) - InternalStream stream = this.getStream(iterator.streamId); - if (stream == null) { - throw new AmazonClientException("Unknown stream or bad shard iterator."); - } - - InternalShard shard = stream.getShards().get(iterator.shardIndex); - - GetRecordsResult result; - if (iterator.recordIndex == 100) { - result = new GetRecordsResult(); - List recs = shard.getRecords(); - result.setRecords(recs); // NOTE: getting all for now - result.setNextShardIterator(getNextShardIterator(iterator, recs).makeString()); - result.setMillisBehindLatest(100L); - } - else { - result = new GetRecordsResult(); - List recs = shard.getRecordsFrom(iterator); - result.setRecords(recs); // may be empty - result.setNextShardIterator(getNextShardIterator(iterator, recs).makeString()); - result.setMillisBehindLatest(100L); - } - - return result; - } - - protected ShardIterator getNextShardIterator(ShardIterator previousIter, List records) - { - if (records.isEmpty()) { - return previousIter; - } - - Record rec = records.getLast(); - int lastSeq = Integer.valueOf(rec.getSequenceNumber()); - return new ShardIterator(previousIter.streamId, previousIter.shardIndex, lastSeq + 1); - } - - //// Unsupported methods - - @Override - public ListTagsForStreamResult listTagsForStream(ListTagsForStreamRequest listTagsForStreamRequest) - throws AmazonClientException - { - return null; - } - - @Override - public ListStreamsResult listStreams(ListStreamsRequest listStreamsRequest) - throws AmazonClientException - { - return null; - } - - @Override - public ListStreamsResult listStreams() - throws AmazonServiceException, AmazonClientException - { - return null; - } - - @Override - public PutRecordResult putRecord(String s, ByteBuffer byteBuffer, String s1) - throws AmazonServiceException, AmazonClientException - { - throw new UnsupportedOperationException("MockKinesisClient doesn't support this."); - } - - @Override - public PutRecordResult putRecord(String s, ByteBuffer byteBuffer, String s1, String s2) - throws AmazonServiceException, AmazonClientException - { - throw new UnsupportedOperationException("MockKinesisClient doesn't support this."); - } - - @Override - public DescribeStreamResult describeStream(String streamName) - throws AmazonServiceException, AmazonClientException - { - return null; - } - - @Override - public DescribeStreamResult describeStream(String streamName, String exclusiveStartShardId) - throws AmazonServiceException, AmazonClientException - { - return null; - } - - @Override - public DescribeStreamResult describeStream(String streamName, Integer integer, String exclusiveStartShardId) - throws AmazonServiceException, AmazonClientException - { - return null; - } - - @Override - public GetShardIteratorResult getShardIterator(String streamName, String shardId, String shardIteratorType) - throws AmazonServiceException, AmazonClientException - { - return null; - } - - @Override - public GetShardIteratorResult getShardIterator(String streamName, String shardId, String shardIteratorType, String startingSequenceNumber) - throws AmazonServiceException, AmazonClientException - { - return null; - } - - @Override - public ListStreamsResult listStreams(String exclusiveStartStreamName) - throws AmazonServiceException, AmazonClientException - { - return null; - } - - @Override - public ListStreamsResult listStreams(Integer limit, String exclusiveStartStreamName) - throws AmazonServiceException, AmazonClientException - { - return null; - } - - @Override - public void shutdown() - { - return; // Nothing to shutdown here - } - - @Override - public ResponseMetadata getCachedResponseMetadata(AmazonWebServiceRequest amazonWebServiceRequest) - { - return null; - } -} diff --git a/plugin/trino-kinesis/src/test/java/io/trino/plugin/kinesis/util/TestUtils.java b/plugin/trino-kinesis/src/test/java/io/trino/plugin/kinesis/util/TestUtils.java deleted file mode 100644 index 11398190803d..000000000000 --- a/plugin/trino-kinesis/src/test/java/io/trino/plugin/kinesis/util/TestUtils.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.kinesis.util; - -import com.google.common.collect.ImmutableMap; -import io.trino.plugin.kinesis.KinesisConnector; -import io.trino.plugin.kinesis.KinesisConnectorFactory; -import io.trino.plugin.kinesis.KinesisPlugin; -import io.trino.plugin.kinesis.TestingKinesisConnectorFactory; -import io.trino.spi.connector.Connector; -import io.trino.spi.connector.ConnectorFactory; -import io.trino.testing.QueryRunner; -import io.trino.testing.TestingConnectorContext; - -import java.util.Map; - -import static java.util.Objects.requireNonNull; -import static org.assertj.core.api.Assertions.assertThat; - -public final class TestUtils -{ - private static final String NONE_KEY = "NONE"; - - private TestUtils() {} - - public static KinesisConnector createConnector(KinesisPlugin plugin, Map properties, boolean withMockClient) - { - requireNonNull(plugin, "Plugin instance should not be null"); - requireNonNull(properties, "Properties map should not be null (can be empty)"); - ConnectorFactory factory = plugin.getConnectorFactories().iterator().next(); - assertThat(factory).isNotNull(); - - Connector connector = factory.create("kinesis", properties, new TestingConnectorContext()); - return (KinesisConnector) connector; - } - - /** - * Install the plugin into the given query runner, using the mock client and the given table descriptions. - */ - public static MockKinesisClient installKinesisPlugin(QueryRunner queryRunner) - { - KinesisTestClientManager kinesisTestClientManager = new KinesisTestClientManager(); - MockKinesisClient mockClient = (MockKinesisClient) kinesisTestClientManager.getClient(); - mockClient.createStream("test123", 2); - mockClient.createStream("sampleTable", 2); - mockClient.createStream("sampleGzipCompressTable", 2); - mockClient.createStream("sampleAutomaticCompressTable", 2); - KinesisConnectorFactory kinesisConnectorFactory = new TestingKinesisConnectorFactory(kinesisTestClientManager); - - KinesisPlugin kinesisPlugin = new KinesisPlugin(kinesisConnectorFactory); - queryRunner.installPlugin(kinesisPlugin); - - Map kinesisConfig = ImmutableMap.of( - "kinesis.default-schema", "default", - "kinesis.access-key", "", - "kinesis.secret-key", "", - "kinesis.table-description-location", "src/test/resources/tableDescriptions"); - queryRunner.createCatalog("kinesis", "kinesis", kinesisConfig); - - return mockClient; - } - - /** - * Install the plug in into the given query runner, using normal setup but with the given table descriptions. - *

- * Note that this uses the actual client and will incur charges from AWS when run. Mainly for full - * integration tests. - */ - public static void installKinesisPlugin(QueryRunner queryRunner, String tableDescriptionLocation, String accessKey, String secretKey) - { - KinesisPlugin kinesisPlugin = new KinesisPlugin(); - queryRunner.installPlugin(kinesisPlugin); - - Map kinesisConfig = ImmutableMap.of( - "kinesis.default-schema", "default", - "kinesis.access-key", accessKey, - "kinesis.secret-key", secretKey, - "kinesis.table-description-location", tableDescriptionLocation); - - queryRunner.createCatalog("kinesis", "kinesis", kinesisConfig); - } - - public static String noneToBlank(String awsValue) - { - if (awsValue.equals(NONE_KEY)) { - return ""; - } - return awsValue; - } -} diff --git a/plugin/trino-kinesis/src/test/resources/decoder/json/event.json b/plugin/trino-kinesis/src/test/resources/decoder/json/event.json deleted file mode 100644 index b5bfa9ab5275..000000000000 --- a/plugin/trino-kinesis/src/test/resources/decoder/json/event.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "created_at": "Mon Jul 28 20:38:07 +0000 2016", - "id": 493857959588286460, - "timestamp": 1450214872847, - "source": "otherworld", - "environment": { - "windowDims": { - "w": 661, - "h": 904 - }, - "screenDims": { - "w": 1680, - "h": 1050 - }, - "browser": { - "ua": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.80 Safari/537.36", - "name": "Chrome", - "version": "47.0.2526.80", - "os": "OS X", - "osVersion": "10.11.2" - } - }, - "user": { - "email": "joeblow@wherever.com", - "handle": "joeblow" - }, - "tags": [ - "tag1", - "tag2", - "tag3" - ] -} \ No newline at end of file diff --git a/plugin/trino-kinesis/src/test/resources/decoder/json/message.json b/plugin/trino-kinesis/src/test/resources/decoder/json/message.json deleted file mode 100644 index bbab321b1bfd..000000000000 --- a/plugin/trino-kinesis/src/test/resources/decoder/json/message.json +++ /dev/null @@ -1,80 +0,0 @@ -{ - "created_at": "Mon Jul 28 20:38:07 +0000 2014", - "id": 493857959588286460, - "id_str": "493857959588286465", - "text": "Lots of Important Preseason Football Dates on the Horizon - EKU Sports: Lots of Important Preseason Football D... http://t.co/F7iz6APFTW", - "source": "twitterfeed", - "truncated": false, - "in_reply_to_status_id": null, - "in_reply_to_status_id_str": null, - "in_reply_to_user_id": null, - "in_reply_to_user_id_str": null, - "in_reply_to_screen_name": null, - "user": { - "id": 98247748, - "id_str": "98247748", - "name": "Eastern KY News", - "screen_name": "EKentuckyNews", - "location": "Eastern Kentucky", - "url": null, - "description": "Your Eastern Kentucky News Source.", - "protected": false, - "verified": false, - "followers_count": 305, - "friends_count": 186, - "listed_count": 8, - "favourites_count": 0, - "statuses_count": 7630, - "created_at": "Mon Dec 21 01:17:22 +0000 2009", - "utc_offset": -14400, - "time_zone": "Eastern Time (US & Canada)", - "geo_enabled": true, - "lang": "en", - "contributors_enabled": false, - "is_translator": false, - "profile_background_color": "C6E2EE", - "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", - "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", - "profile_background_tile": false, - "profile_link_color": "1F98C7", - "profile_sidebar_border_color": "C6E2EE", - "profile_sidebar_fill_color": "DAECF4", - "profile_text_color": "8F433C", - "profile_use_background_image": true, - "profile_image_url": "http://pbs.twimg.com/profile_images/1297233295/Kentucky_at_Work_logo_normal.jpeg", - "profile_image_url_https": "https://pbs.twimg.com/profile_images/1297233295/Kentucky_at_Work_logo_normal.jpeg", - "default_profile": false, - "default_profile_image": false, - "following": null, - "follow_request_sent": null, - "notifications": null - }, - "geo": null, - "coordinates": null, - "place": null, - "contributors": null, - "retweet_count": 0, - "favorite_count": 0, - "entities": { - "hashtags": [], - "trends": [], - "urls": [ - { - "url": "http://t.co/F7iz6APFTW", - "expanded_url": "http://bit.ly/1rTEYRM", - "display_url": "bit.ly/1rTEYRM", - "indices": [ - 114, - 136 - ] - } - ], - "user_mentions": [], - "symbols": [] - }, - "favorited": false, - "retweeted": false, - "possibly_sensitive": false, - "filter_level": "medium", - "lang": "en" -} diff --git a/plugin/trino-kinesis/src/test/resources/etc/catalog/kinesis.properties b/plugin/trino-kinesis/src/test/resources/etc/catalog/kinesis.properties deleted file mode 100644 index 9889d4f208ce..000000000000 --- a/plugin/trino-kinesis/src/test/resources/etc/catalog/kinesis.properties +++ /dev/null @@ -1,33 +0,0 @@ -# Configuration for the kinesis connector -# -# This is a sample configuration file for the Kinesis connector. -# The values given here are the defaults. - -# Connector name, usually keep this as kinesis -connector.name=kinesis - -kinesis.default-schema=default -kinesis.aws-region=us-east-1 -kinesis.hide-internal-columns=false -kinesis.batch-size=10000 -kinesis.max-batches=100 -kinesis.fetch-attempts=2 -kinesis.sleep-time=1000ms - -# Use an initial shard iterator type of AT_TIMESTAMP starting -# iteratorOffsetSeconds before the current time -kinesis.iterator-from-timestamp=true -kinesis.iterator-offset-seconds=86400 -# info log one line each time a Kinesis batch is read with some useful info -#kinesis.log-batches=true - -# enable query checkpointing via Dynamo DB (configure other -# properties below if needed) -kinesis.checkpoint-enabled=false - -# properties related to checkpointing with Dynamo DB -#kinesis.dynamo-read-capacity=50 -#kinesis.dynamo-write-capacity=10 -#kinesis.checkpoint-interval-ms=60000ms -#kinesis.checkpoint-logical-name=process1 -#kinesis.iterator-number=0 diff --git a/plugin/trino-kinesis/src/test/resources/etc/kinesis/testtable.json b/plugin/trino-kinesis/src/test/resources/etc/kinesis/testtable.json deleted file mode 100644 index bcd9cd18a5b6..000000000000 --- a/plugin/trino-kinesis/src/test/resources/etc/kinesis/testtable.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "tableName": "test_table", - "schemaName": "prod", - "streamName": "test_kinesis_stream", - "message": { - "dataFormat": "json", - "fields": [ - { - "name": "client_id", - "type": "BIGINT", - "mapping": "client_id", - "comment": "The client ID field" - }, - { - "name": "acct_balance", - "type": "DOUBLE", - "mapping": "acct_balance", - "comment": "Current account balance" - }, - { - "name": "service_type", - "mapping": "service_type", - "type": "VARCHAR(20)" - }, - { - "name": "signup_date", - "mapping": "signup_date", - "type": "DATE", - "dataFormat": "iso8601" - } - ] - }, - "comment" : "This test adds some extra fields to make sure they are ignored and don't cause issues.", - "client_metadata" : { - "name" : "Sample Query", - "query" : "select client_id, service_type, signup_date, _shard_id, _message_length from prod.test_table" - } -} \ No newline at end of file diff --git a/plugin/trino-kinesis/src/test/resources/tableDescriptions/EmptyTable.json b/plugin/trino-kinesis/src/test/resources/tableDescriptions/EmptyTable.json deleted file mode 100644 index e1d61dd1bc8e..000000000000 --- a/plugin/trino-kinesis/src/test/resources/tableDescriptions/EmptyTable.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "tableName": "TABLE_NAME", - "schemaName": "default", - "streamName": "STREAM_NAME" -} \ No newline at end of file diff --git a/plugin/trino-kinesis/src/test/resources/tableDescriptions/MinimalTable.json b/plugin/trino-kinesis/src/test/resources/tableDescriptions/MinimalTable.json deleted file mode 100644 index 700a0ddc60b5..000000000000 --- a/plugin/trino-kinesis/src/test/resources/tableDescriptions/MinimalTable.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "tableName": "test123", - "schemaName": "default", - "streamName": "test123", - "message": { - "dataFormat": "json", - "fields": [{ - "name": "id", - "type": "BIGINT", - "mapping": "id", - "comment": "comment", - "hidden": "false" - }, - { - "name": "name", - "type": "VARCHAR", - "mapping": "name", - "comment": "comment", - "hidden": "false" - } - ] - } -} \ No newline at end of file diff --git a/plugin/trino-kinesis/src/test/resources/tableDescriptions/SampleAutomaticCompressTable.json b/plugin/trino-kinesis/src/test/resources/tableDescriptions/SampleAutomaticCompressTable.json deleted file mode 100644 index cdd94d08cf7a..000000000000 --- a/plugin/trino-kinesis/src/test/resources/tableDescriptions/SampleAutomaticCompressTable.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "tableName": "sampleAutomaticCompressTable", - "schemaName": "default", - "streamName": "sampleAutomaticCompressTable", - "message": { - "dataFormat": "json", - "compressionCodec": "AUTOMATIC", - "fields": [{ - "name": "id", - "type": "BIGINT", - "mapping": "id", - "comment": "comment", - "hidden": "false" - }, - { - "name": "name", - "type": "VARCHAR", - "mapping": "name", - "comment": "comment", - "hidden": "false" - } - ] - } -} diff --git a/plugin/trino-kinesis/src/test/resources/tableDescriptions/SampleGzipCompressTable.json b/plugin/trino-kinesis/src/test/resources/tableDescriptions/SampleGzipCompressTable.json deleted file mode 100644 index 7a864f4ce13f..000000000000 --- a/plugin/trino-kinesis/src/test/resources/tableDescriptions/SampleGzipCompressTable.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "tableName": "sampleGzipCompressTable", - "schemaName": "default", - "streamName": "sampleGzipCompressTable", - "message": { - "dataFormat": "json", - "compressionCodec": "GZIP", - "fields": [{ - "name": "id", - "type": "BIGINT", - "mapping": "id", - "comment": "comment", - "hidden": "false" - }, - { - "name": "name", - "type": "VARCHAR", - "mapping": "name", - "comment": "comment", - "hidden": "false" - } - ] - } -} diff --git a/plugin/trino-kinesis/src/test/resources/tableDescriptions/SampleTable.json b/plugin/trino-kinesis/src/test/resources/tableDescriptions/SampleTable.json deleted file mode 100644 index 92d16ee77144..000000000000 --- a/plugin/trino-kinesis/src/test/resources/tableDescriptions/SampleTable.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "tableName": "sampleTable", - "schemaName": "default", - "streamName": "sampleTable", - "message": { - "dataFormat": "json", - "fields": [{ - "name": "id", - "type": "BIGINT", - "mapping": "id", - "comment": "comment", - "hidden": "false" - }, - { - "name": "name", - "type": "VARCHAR", - "mapping": "name", - "comment": "comment", - "hidden": "false" - } - ] - } -} diff --git a/plugin/trino-kinesis/src/test/resources/tableDescriptions/TableWithMessage.json b/plugin/trino-kinesis/src/test/resources/tableDescriptions/TableWithMessage.json deleted file mode 100644 index 6d7b6ab7eb42..000000000000 --- a/plugin/trino-kinesis/src/test/resources/tableDescriptions/TableWithMessage.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "tableName": "test_table", - "schemaName": "prod", - "streamName": "test_kinesis_stream", - "message": { - "dataFormat": "json", - "fields": [{ - "name": "id", - "type": "BIGINT", - "mapping": "id", - "comment": "comment", - "hidden": "false" - }, - { - "name": "name", - "type": "VARCHAR", - "mapping": "name", - "comment": "comment", - "hidden": "false" - } - ] - } -} \ No newline at end of file diff --git a/pom.xml b/pom.xml index 4bdb5eb4fa50..534696330a17 100644 --- a/pom.xml +++ b/pom.xml @@ -84,7 +84,6 @@ plugin/trino-jmx plugin/trino-kafka plugin/trino-kafka-event-listener - plugin/trino-kinesis plugin/trino-kudu plugin/trino-mariadb plugin/trino-memory @@ -319,38 +318,6 @@ 3.11.0 - - com.amazonaws - amazon-kinesis-client - 1.15.2 - - - com.amazonaws - aws-java-sdk - - - com.amazonaws - aws-java-sdk-core - - - com.google.protobuf - protobuf-java - - - commons-lang - commons-lang - - - commons-logging - commons-logging - - - joda-time - joda-time - - - - com.amazonaws aws-java-sdk-core @@ -389,22 +356,6 @@ - - com.amazonaws - aws-java-sdk-kinesis - ${dep.aws-sdk.version} - - - commons-logging - commons-logging - - - joda-time - joda-time - - - - com.amazonaws aws-java-sdk-s3 diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeAllConnectors.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeAllConnectors.java index 069c79301e54..8502f3e85f02 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeAllConnectors.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeAllConnectors.java @@ -59,7 +59,6 @@ public void extendEnvironment(Environment.Builder builder) "iceberg", "ignite", "kafka", - "kinesis", "kudu", "mariadb", "memory", diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/multinode-all/kinesis.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/multinode-all/kinesis.properties deleted file mode 100644 index 1d218cadd126..000000000000 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/multinode-all/kinesis.properties +++ /dev/null @@ -1,3 +0,0 @@ -connector.name=kinesis -kinesis.access-key=XXXXXX -kinesis.secret-key=XXXXXX