Skip to content

Commit

Permalink
feat(client-glue): This release includes(1)Zero-ETL integration to in…
Browse files Browse the repository at this point in the history
…gest data from 3P SaaS and DynamoDB to Redshift/Redlake (2)new properties on Connections to enable reuse; new connection APIs for retrieve/preview metadata (3)support of CRUD operations for Multi-catalog (4)support of automatic statistics collections
  • Loading branch information
awstools committed Dec 3, 2024
1 parent e8b4cd8 commit 9c425d5
Show file tree
Hide file tree
Showing 106 changed files with 20,786 additions and 7,637 deletions.
176 changes: 176 additions & 0 deletions clients/client-glue/README.md

Large diffs are not rendered by default.

464 changes: 464 additions & 0 deletions clients/client-glue/src/Glue.ts

Large diffs are not rendered by default.

99 changes: 99 additions & 0 deletions clients/client-glue/src/GlueClient.ts

Large diffs are not rendered by default.

3 changes: 2 additions & 1 deletion clients/client-glue/src/commands/BatchStopJobRunCommand.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,8 @@ import { MetadataBearer as __MetadataBearer } from "@smithy/types";

import { commonParams } from "../endpoint/EndpointParameters";
import { GlueClientResolvedConfig, ServiceInputTypes, ServiceOutputTypes } from "../GlueClient";
import { BatchStopJobRunRequest, BatchStopJobRunResponse } from "../models/models_0";
import { BatchStopJobRunRequest } from "../models/models_0";
import { BatchStopJobRunResponse } from "../models/models_1";
import { de_BatchStopJobRunCommand, se_BatchStopJobRunCommand } from "../protocols/Aws_json1_1";

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import { MetadataBearer as __MetadataBearer } from "@smithy/types";

import { commonParams } from "../endpoint/EndpointParameters";
import { GlueClientResolvedConfig, ServiceInputTypes, ServiceOutputTypes } from "../GlueClient";
import { BatchUpdatePartitionRequest, BatchUpdatePartitionResponse } from "../models/models_0";
import { BatchUpdatePartitionRequest, BatchUpdatePartitionResponse } from "../models/models_1";
import { de_BatchUpdatePartitionCommand, se_BatchUpdatePartitionCommand } from "../protocols/Aws_json1_1";

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,10 @@ import { MetadataBearer as __MetadataBearer } from "@smithy/types";

import { commonParams } from "../endpoint/EndpointParameters";
import { GlueClientResolvedConfig, ServiceInputTypes, ServiceOutputTypes } from "../GlueClient";
import { CancelDataQualityRuleRecommendationRunRequest } from "../models/models_0";
import { CancelDataQualityRuleRecommendationRunResponse } from "../models/models_1";
import {
CancelDataQualityRuleRecommendationRunRequest,
CancelDataQualityRuleRecommendationRunResponse,
} from "../models/models_1";
import {
de_CancelDataQualityRuleRecommendationRunCommand,
se_CancelDataQualityRuleRecommendationRunCommand,
Expand Down
170 changes: 170 additions & 0 deletions clients/client-glue/src/commands/CreateCatalogCommand.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,170 @@
// smithy-typescript generated code
import { getEndpointPlugin } from "@smithy/middleware-endpoint";
import { getSerdePlugin } from "@smithy/middleware-serde";
import { Command as $Command } from "@smithy/smithy-client";
import { MetadataBearer as __MetadataBearer } from "@smithy/types";

import { commonParams } from "../endpoint/EndpointParameters";
import { GlueClientResolvedConfig, ServiceInputTypes, ServiceOutputTypes } from "../GlueClient";
import { CreateCatalogRequest, CreateCatalogResponse } from "../models/models_1";
import { de_CreateCatalogCommand, se_CreateCatalogCommand } from "../protocols/Aws_json1_1";

/**
* @public
*/
export type { __MetadataBearer };
export { $Command };
/**
* @public
*
* The input for {@link CreateCatalogCommand}.
*/
export interface CreateCatalogCommandInput extends CreateCatalogRequest {}
/**
* @public
*
* The output of {@link CreateCatalogCommand}.
*/
export interface CreateCatalogCommandOutput extends CreateCatalogResponse, __MetadataBearer {}

/**
* <p>Creates a new catalog in the Glue Data Catalog.</p>
* @example
* Use a bare-bones client and the command you need to make an API call.
* ```javascript
* import { GlueClient, CreateCatalogCommand } from "@aws-sdk/client-glue"; // ES Modules import
* // const { GlueClient, CreateCatalogCommand } = require("@aws-sdk/client-glue"); // CommonJS import
* const client = new GlueClient(config);
* const input = { // CreateCatalogRequest
* Name: "STRING_VALUE", // required
* CatalogInput: { // CatalogInput
* Description: "STRING_VALUE",
* FederatedCatalog: { // FederatedCatalog
* Identifier: "STRING_VALUE",
* ConnectionName: "STRING_VALUE",
* },
* Parameters: { // ParametersMap
* "<keys>": "STRING_VALUE",
* },
* TargetRedshiftCatalog: { // TargetRedshiftCatalog
* CatalogArn: "STRING_VALUE", // required
* },
* CatalogProperties: { // CatalogProperties
* DataLakeAccessProperties: { // DataLakeAccessProperties
* DataLakeAccess: true || false,
* DataTransferRole: "STRING_VALUE",
* KmsKey: "STRING_VALUE",
* CatalogType: "STRING_VALUE",
* },
* CustomProperties: {
* "<keys>": "STRING_VALUE",
* },
* },
* CreateTableDefaultPermissions: [ // PrincipalPermissionsList
* { // PrincipalPermissions
* Principal: { // DataLakePrincipal
* DataLakePrincipalIdentifier: "STRING_VALUE",
* },
* Permissions: [ // PermissionList
* "ALL" || "SELECT" || "ALTER" || "DROP" || "DELETE" || "INSERT" || "CREATE_DATABASE" || "CREATE_TABLE" || "DATA_LOCATION_ACCESS",
* ],
* },
* ],
* CreateDatabaseDefaultPermissions: [
* {
* Principal: {
* DataLakePrincipalIdentifier: "STRING_VALUE",
* },
* Permissions: [
* "ALL" || "SELECT" || "ALTER" || "DROP" || "DELETE" || "INSERT" || "CREATE_DATABASE" || "CREATE_TABLE" || "DATA_LOCATION_ACCESS",
* ],
* },
* ],
* },
* Tags: { // TagsMap
* "<keys>": "STRING_VALUE",
* },
* };
* const command = new CreateCatalogCommand(input);
* const response = await client.send(command);
* // {};
*
* ```
*
* @param CreateCatalogCommandInput - {@link CreateCatalogCommandInput}
* @returns {@link CreateCatalogCommandOutput}
* @see {@link CreateCatalogCommandInput} for command's `input` shape.
* @see {@link CreateCatalogCommandOutput} for command's `response` shape.
* @see {@link GlueClientResolvedConfig | config} for GlueClient's `config` shape.
*
* @throws {@link AccessDeniedException} (client fault)
* <p>Access to a resource was denied.</p>
*
* @throws {@link AlreadyExistsException} (client fault)
* <p>A resource to be created or added already exists.</p>
*
* @throws {@link ConcurrentModificationException} (client fault)
* <p>Two processes are trying to modify a resource simultaneously.</p>
*
* @throws {@link EntityNotFoundException} (client fault)
* <p>A specified entity does not exist</p>
*
* @throws {@link FederatedResourceAlreadyExistsException} (client fault)
* <p>A federated resource already exists.</p>
*
* @throws {@link FederationSourceException} (client fault)
* <p>A federation source failed.</p>
*
* @throws {@link GlueEncryptionException} (client fault)
* <p>An encryption operation failed.</p>
*
* @throws {@link InternalServiceException} (server fault)
* <p>An internal service error occurred.</p>
*
* @throws {@link InvalidInputException} (client fault)
* <p>The input provided was not valid.</p>
*
* @throws {@link OperationTimeoutException} (client fault)
* <p>The operation timed out.</p>
*
* @throws {@link ResourceNumberLimitExceededException} (client fault)
* <p>A resource numerical limit was exceeded.</p>
*
* @throws {@link GlueServiceException}
* <p>Base exception class for all service exceptions from Glue service.</p>
*
* @public
*/
export class CreateCatalogCommand extends $Command
.classBuilder<
CreateCatalogCommandInput,
CreateCatalogCommandOutput,
GlueClientResolvedConfig,
ServiceInputTypes,
ServiceOutputTypes
>()
.ep(commonParams)
.m(function (this: any, Command: any, cs: any, config: GlueClientResolvedConfig, o: any) {
return [
getSerdePlugin(config, this.serialize, this.deserialize),
getEndpointPlugin(config, Command.getEndpointParameterInstructions()),
];
})
.s("AWSGlue", "CreateCatalog", {})
.n("GlueClient", "CreateCatalogCommand")
.f(void 0, void 0)
.ser(se_CreateCatalogCommand)
.de(de_CreateCatalogCommand)
.build() {
/** @internal type navigation helper, not in runtime. */
protected declare static __types: {
api: {
input: CreateCatalogRequest;
output: {};
};
sdk: {
input: CreateCatalogCommandInput;
output: CreateCatalogCommandOutput;
};
};
}
29 changes: 26 additions & 3 deletions clients/client-glue/src/commands/CreateConnectionCommand.ts
Original file line number Diff line number Diff line change
Expand Up @@ -45,14 +45,20 @@ export interface CreateConnectionCommandOutput extends CreateConnectionResponse,
* ConnectionInput: { // ConnectionInput
* Name: "STRING_VALUE", // required
* Description: "STRING_VALUE",
* ConnectionType: "JDBC" || "SFTP" || "MONGODB" || "KAFKA" || "NETWORK" || "MARKETPLACE" || "CUSTOM" || "SALESFORCE" || "VIEW_VALIDATION_REDSHIFT" || "VIEW_VALIDATION_ATHENA", // required
* ConnectionType: "JDBC" || "SFTP" || "MONGODB" || "KAFKA" || "NETWORK" || "MARKETPLACE" || "CUSTOM" || "SALESFORCE" || "VIEW_VALIDATION_REDSHIFT" || "VIEW_VALIDATION_ATHENA" || "GOOGLEADS" || "GOOGLESHEETS" || "GOOGLEANALYTICS4" || "SERVICENOW" || "MARKETO" || "SAPODATA" || "ZENDESK" || "JIRACLOUD" || "NETSUITEERP" || "HUBSPOT" || "FACEBOOKADS" || "INSTAGRAMADS" || "ZOHOCRM" || "SALESFORCEPARDOT" || "SALESFORCEMARKETINGCLOUD" || "SLACK" || "STRIPE" || "INTERCOM" || "SNAPCHATADS", // required
* MatchCriteria: [ // MatchCriteria
* "STRING_VALUE",
* ],
* ConnectionProperties: { // ConnectionProperties // required
* "<keys>": "STRING_VALUE",
* },
* AthenaProperties: { // PropertyMap
* SparkProperties: { // PropertyMap
* "<keys>": "STRING_VALUE",
* },
* AthenaProperties: {
* "<keys>": "STRING_VALUE",
* },
* PythonProperties: {
* "<keys>": "STRING_VALUE",
* },
* PhysicalConnectionRequirements: { // PhysicalConnectionRequirements
Expand All @@ -63,7 +69,7 @@ export interface CreateConnectionCommandOutput extends CreateConnectionResponse,
* AvailabilityZone: "STRING_VALUE",
* },
* AuthenticationConfiguration: { // AuthenticationConfigurationInput
* AuthenticationType: "BASIC" || "OAUTH2" || "CUSTOM",
* AuthenticationType: "BASIC" || "OAUTH2" || "CUSTOM" || "IAM",
* OAuth2Properties: { // OAuth2PropertiesInput
* OAuth2GrantType: "AUTHORIZATION_CODE" || "CLIENT_CREDENTIALS" || "JWT_BEARER",
* OAuth2ClientApplication: { // OAuth2ClientApplication
Expand All @@ -78,10 +84,27 @@ export interface CreateConnectionCommandOutput extends CreateConnectionResponse,
* AuthorizationCode: "STRING_VALUE",
* RedirectUri: "STRING_VALUE",
* },
* OAuth2Credentials: { // OAuth2Credentials
* UserManagedClientApplicationClientSecret: "STRING_VALUE",
* AccessToken: "STRING_VALUE",
* RefreshToken: "STRING_VALUE",
* JwtToken: "STRING_VALUE",
* },
* },
* SecretArn: "STRING_VALUE",
* KmsKeyArn: "STRING_VALUE",
* BasicAuthenticationCredentials: { // BasicAuthenticationCredentials
* Username: "STRING_VALUE",
* Password: "STRING_VALUE",
* },
* CustomAuthenticationCredentials: { // CredentialMap
* "<keys>": "STRING_VALUE",
* },
* },
* ValidateCredentials: true || false,
* ValidateForComputeEnvironments: [ // ComputeEnvironmentList
* "SPARK" || "ATHENA" || "PYTHON",
* ],
* },
* Tags: { // TagsMap
* "<keys>": "STRING_VALUE",
Expand Down
6 changes: 6 additions & 0 deletions clients/client-glue/src/commands/CreateDatabaseCommand.ts
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,12 @@ export interface CreateDatabaseCommandOutput extends CreateDatabaseResponse, __M
* @throws {@link FederatedResourceAlreadyExistsException} (client fault)
* <p>A federated resource already exists.</p>
*
* @throws {@link FederationSourceException} (client fault)
* <p>A federation source failed.</p>
*
* @throws {@link FederationSourceRetryableException} (client fault)
* <p>A federation source failed, but the operation may be retried.</p>
*
* @throws {@link GlueEncryptionException} (client fault)
* <p>An encryption operation failed.</p>
*
Expand Down
Loading

0 comments on commit 9c425d5

Please sign in to comment.