feat(client-glue): AWS Glue Crawlers can now accept SerDe overrides f…

…rom a custom csv classifier. The two SerDe options are LazySimpleSerDe and OpenCSVSerDe. In case, the user wants crawler to do the selection, "None" can be selected for this purpose.
aws · Aug 15, 2023 · b0474f6 · b0474f6
1 parent 964f281
commit b0474f6
Show file tree

Hide file tree

Showing 10 changed files with 151 additions and 72 deletions.
diff --git a/clients/client-glue/src/commands/CreateClassifierCommand.ts b/clients/client-glue/src/commands/CreateClassifierCommand.ts
@@ -75,6 +75,7 @@ export interface CreateClassifierCommandOutput extends CreateClassifierResponse,
  *     CustomDatatypes: [ // CustomDatatypes
  *       "STRING_VALUE",
  *     ],
+ *     Serde: "OpenCSVSerDe" || "LazySimpleSerDe" || "None",
  *   },
  * };
  * const command = new CreateClassifierCommand(input);

diff --git a/clients/client-glue/src/commands/GetClassifierCommand.ts b/clients/client-glue/src/commands/GetClassifierCommand.ts
@@ -91,6 +91,7 @@ export interface GetClassifierCommandOutput extends GetClassifierResponse, __Met
  * //       CustomDatatypes: [ // CustomDatatypes
  * //         "STRING_VALUE",
  * //       ],
+ * //       Serde: "OpenCSVSerDe" || "LazySimpleSerDe" || "None",
  * //     },
  * //   },
  * // };

diff --git a/clients/client-glue/src/commands/GetClassifiersCommand.ts b/clients/client-glue/src/commands/GetClassifiersCommand.ts
@@ -93,6 +93,7 @@ export interface GetClassifiersCommandOutput extends GetClassifiersResponse, __M
  * //         CustomDatatypes: [ // CustomDatatypes
  * //           "STRING_VALUE",
  * //         ],
+ * //         Serde: "OpenCSVSerDe" || "LazySimpleSerDe" || "None",
  * //       },
  * //     },
  * //   ],

diff --git a/clients/client-glue/src/commands/GetUnfilteredPartitionMetadataCommand.ts b/clients/client-glue/src/commands/GetUnfilteredPartitionMetadataCommand.ts
@@ -14,8 +14,7 @@ import {
 } from "@smithy/types";
 
 import { GlueClientResolvedConfig, ServiceInputTypes, ServiceOutputTypes } from "../GlueClient";
-import { GetUnfilteredPartitionMetadataRequest } from "../models/models_1";
-import { GetUnfilteredPartitionMetadataResponse } from "../models/models_2";
+import { GetUnfilteredPartitionMetadataRequest, GetUnfilteredPartitionMetadataResponse } from "../models/models_2";
 import {
   de_GetUnfilteredPartitionMetadataCommand,
   se_GetUnfilteredPartitionMetadataCommand,

diff --git a/clients/client-glue/src/commands/UpdateClassifierCommand.ts b/clients/client-glue/src/commands/UpdateClassifierCommand.ts
@@ -75,6 +75,7 @@ export interface UpdateClassifierCommandOutput extends UpdateClassifierResponse,
  *     CustomDatatypes: [ // CustomDatatypes
  *       "STRING_VALUE",
  *     ],
+ *     Serde: "OpenCSVSerDe" || "LazySimpleSerDe" || "None",
  *   },
  * };
  * const command = new UpdateClassifierCommand(input);

diff --git a/clients/client-glue/src/models/models_0.ts b/clients/client-glue/src/models/models_0.ts
@@ -8206,6 +8206,21 @@ export const CsvHeaderOption = {
  */
 export type CsvHeaderOption = (typeof CsvHeaderOption)[keyof typeof CsvHeaderOption];
 
+/**
+ * @public
+ * @enum
+ */
+export const CsvSerdeOption = {
+  LazySimpleSerDe: "LazySimpleSerDe",
+  None: "None",
+  OpenCSVSerDe: "OpenCSVSerDe",
+} as const;
+
+/**
+ * @public
+ */
+export type CsvSerdeOption = (typeof CsvSerdeOption)[keyof typeof CsvSerdeOption];
+
 /**
  * @public
  * <p>Specifies a custom CSV classifier for <code>CreateClassifier</code> to create.</p>
@@ -8264,6 +8279,12 @@ export interface CreateCsvClassifierRequest {
    * <p>Creates a list of supported custom datatypes.</p>
    */
   CustomDatatypes?: string[];
+
+  /**
+   * @public
+   * <p>Sets the SerDe for processing CSV in the classifier, which will be applied in the Data Catalog. Valid values are <code>OpenCSVSerDe</code>, <code>LazySimpleSerDe</code>, and <code>None</code>. You can specify the <code>None</code> value when you want the crawler to do the detection.</p>
+   */
+  Serde?: CsvSerdeOption | string;
 }
 
 /**
@@ -9391,29 +9412,3 @@ export interface CreateDevEndpointResponse {
    */
   Arguments?: Record<string, string>;
 }
-
-/**
- * @public
- * <p>A value could not be validated.</p>
- */
-export class ValidationException extends __BaseException {
-  readonly name: "ValidationException" = "ValidationException";
-  readonly $fault: "client" = "client";
-  /**
-   * @public
-   * <p>A message describing the problem.</p>
-   */
-  Message?: string;
-  /**
-   * @internal
-   */
-  constructor(opts: __ExceptionOptionType<ValidationException, __BaseException>) {
-    super({
-      name: "ValidationException",
-      $fault: "client",
-      ...opts,
-    });
-    Object.setPrototypeOf(this, ValidationException.prototype);
-    this.Message = opts.Message;
-  }
-}
diff --git a/clients/client-glue/src/models/models_1.ts b/clients/client-glue/src/models/models_1.ts
@@ -4,13 +4,13 @@ import { ExceptionOptionType as __ExceptionOptionType } from "@smithy/smithy-cli
 import { GlueServiceException as __BaseException } from "./GlueServiceException";
 import {
   Action,
-  AuditContext,
   Blueprint,
   Column,
   ConnectionsList,
   ConnectionType,
   Crawler,
   CsvHeaderOption,
+  CsvSerdeOption,
   DatabaseIdentifier,
   DataFormat,
   DataQualityRuleResult,
@@ -36,6 +36,32 @@ import {
   WorkerType,
 } from "./models_0";
 
+/**
+ * @public
+ * <p>A value could not be validated.</p>
+ */
+export class ValidationException extends __BaseException {
+  readonly name: "ValidationException" = "ValidationException";
+  readonly $fault: "client" = "client";
+  /**
+   * @public
+   * <p>A message describing the problem.</p>
+   */
+  Message?: string;
+  /**
+   * @internal
+   */
+  constructor(opts: __ExceptionOptionType<ValidationException, __BaseException>) {
+    super({
+      name: "ValidationException",
+      $fault: "client",
+      ...opts,
+    });
+    Object.setPrototypeOf(this, ValidationException.prototype);
+    this.Message = opts.Message;
+  }
+}
+
 /**
  * @public
  */
@@ -2887,6 +2913,12 @@ export interface CsvClassifier {
    * <p>A list of custom datatypes including "BINARY", "BOOLEAN", "DATE", "DECIMAL", "DOUBLE", "FLOAT", "INT", "LONG", "SHORT", "STRING", "TIMESTAMP".</p>
    */
   CustomDatatypes?: string[];
+
+  /**
+   * @public
+   * <p>Sets the SerDe for processing CSV in the classifier, which will be applied in the Data Catalog. Valid values are <code>OpenCSVSerDe</code>, <code>LazySimpleSerDe</code>, and <code>None</code>. You can specify the <code>None</code> value when you want the crawler to do the detection.</p>
+   */
+  Serde?: CsvSerdeOption | string;
 }
 
 /**
@@ -7892,44 +7924,3 @@ export const PermissionType = {
  * @public
  */
 export type PermissionType = (typeof PermissionType)[keyof typeof PermissionType];
-
-/**
- * @public
- */
-export interface GetUnfilteredPartitionMetadataRequest {
-  /**
-   * @public
-   * <p>The catalog ID where the partition resides.</p>
-   */
-  CatalogId: string | undefined;
-
-  /**
-   * @public
-   * <p>(Required) Specifies the name of a database that contains the partition.</p>
-   */
-  DatabaseName: string | undefined;
-
-  /**
-   * @public
-   * <p>(Required) Specifies the name of a table that contains the partition.</p>
-   */
-  TableName: string | undefined;
-
-  /**
-   * @public
-   * <p>(Required) A list of partition key values.</p>
-   */
-  PartitionValues: string[] | undefined;
-
-  /**
-   * @public
-   * <p>A structure containing Lake Formation audit context information.</p>
-   */
-  AuditContext?: AuditContext;
-
-  /**
-   * @public
-   * <p>(Required) A list of supported permission types. </p>
-   */
-  SupportedPermissionTypes: (PermissionType | string)[] | undefined;
-}
diff --git a/clients/client-glue/src/models/models_2.ts b/clients/client-glue/src/models/models_2.ts
@@ -19,6 +19,7 @@ import {
   ConnectionsList,
   CrawlerTargets,
   CsvHeaderOption,
+  CsvSerdeOption,
   CustomCode,
   CustomEntityType,
   DatabaseInput,
@@ -130,6 +131,47 @@ import {
   UserDefinedFunctionInput,
 } from "./models_1";
 
+/**
+ * @public
+ */
+export interface GetUnfilteredPartitionMetadataRequest {
+  /**
+   * @public
+   * <p>The catalog ID where the partition resides.</p>
+   */
+  CatalogId: string | undefined;
+
+  /**
+   * @public
+   * <p>(Required) Specifies the name of a database that contains the partition.</p>
+   */
+  DatabaseName: string | undefined;
+
+  /**
+   * @public
+   * <p>(Required) Specifies the name of a table that contains the partition.</p>
+   */
+  TableName: string | undefined;
+
+  /**
+   * @public
+   * <p>(Required) A list of partition key values.</p>
+   */
+  PartitionValues: string[] | undefined;
+
+  /**
+   * @public
+   * <p>A structure containing Lake Formation audit context information.</p>
+   */
+  AuditContext?: AuditContext;
+
+  /**
+   * @public
+   * <p>(Required) A list of supported permission types. </p>
+   */
+  SupportedPermissionTypes: (PermissionType | string)[] | undefined;
+}
+
 /**
  * @public
  */
@@ -3818,6 +3860,12 @@ export interface UpdateCsvClassifierRequest {
    * <p>Specifies a list of supported custom datatypes.</p>
    */
   CustomDatatypes?: string[];
+
+  /**
+   * @public
+   * <p>Sets the SerDe for processing CSV in the classifier, which will be applied in the Data Catalog. Valid values are <code>OpenCSVSerDe</code>, <code>LazySimpleSerDe</code>, and <code>None</code>. You can specify the <code>None</code> value when you want the crawler to do the detection.</p>
+   */
+  Serde?: CsvSerdeOption | string;
 }
 
 /**

diff --git a/clients/client-glue/src/protocols/Aws_json1_1.ts b/clients/client-glue/src/protocols/Aws_json1_1.ts
@@ -657,7 +657,6 @@ import {
   TransformConfigParameter,
   Union,
   UpsertRedshiftTargetOptions,
-  ValidationException,
   Workflow,
   WorkflowGraph,
   WorkflowRun,
@@ -830,7 +829,6 @@ import {
   GetTagsRequest,
   GetTriggerRequest,
   GetTriggersRequest,
-  GetUnfilteredPartitionMetadataRequest,
   GluePolicy,
   GrokClassifier,
   IcebergInput,
@@ -868,6 +866,7 @@ import {
   TransformParameters,
   TransformSortCriteria,
   UserDefinedFunctionInput,
+  ValidationException,
   XMLClassifier,
 } from "../models/models_1";
 import {
@@ -892,6 +891,7 @@ import {
   DevEndpointCustomLibraries,
   GetJobResponse,
   GetJobsResponse,
+  GetUnfilteredPartitionMetadataRequest,
   GetUnfilteredPartitionMetadataResponse,
   GetUnfilteredPartitionsMetadataRequest,
   GetUnfilteredPartitionsMetadataResponse,
@@ -18360,6 +18360,7 @@ const de_CsvClassifier = (output: any, context: __SerdeContext): CsvClassifier =
     LastUpdated: (_: any) => __expectNonNull(__parseEpochTimestamp(__expectNumber(_))),
     Name: __expectString,
     QuoteSymbol: __expectString,
+    Serde: __expectString,
     Version: __expectLong,
   }) as any;
 };

diff --git a/codegen/sdk-codegen/aws-models/glue.json b/codegen/sdk-codegen/aws-models/glue.json
@@ -7459,6 +7459,12 @@
           "traits": {
             "smithy.api#documentation": "<p>Creates a list of supported custom datatypes.</p>"
           }
+        },
+        "Serde": {
+          "target": "com.amazonaws.glue#CsvSerdeOption",
+          "traits": {
+            "smithy.api#documentation": "<p>Sets the SerDe for processing CSV in the classifier, which will be applied in the Data Catalog. Valid values are <code>OpenCSVSerDe</code>, <code>LazySimpleSerDe</code>, and <code>None</code>. You can specify the <code>None</code> value when you want the crawler to do the detection.</p>"
+          }
         }
       },
       "traits": {
@@ -9570,6 +9576,12 @@
           "traits": {
             "smithy.api#documentation": "<p>A list of custom datatypes including \"BINARY\", \"BOOLEAN\", \"DATE\", \"DECIMAL\", \"DOUBLE\", \"FLOAT\", \"INT\", \"LONG\", \"SHORT\", \"STRING\", \"TIMESTAMP\".</p>"
           }
+        },
+        "Serde": {
+          "target": "com.amazonaws.glue#CsvSerdeOption",
+          "traits": {
+            "smithy.api#documentation": "<p>Sets the SerDe for processing CSV in the classifier, which will be applied in the Data Catalog. Valid values are <code>OpenCSVSerDe</code>, <code>LazySimpleSerDe</code>, and <code>None</code>. You can specify the <code>None</code> value when you want the crawler to do the detection.</p>"
+          }
         }
       },
       "traits": {
@@ -9625,6 +9637,29 @@
         "smithy.api#pattern": "^[^\\r\\n]$"
       }
     },
+    "com.amazonaws.glue#CsvSerdeOption": {
+      "type": "enum",
+      "members": {
+        "OpenCSVSerDe": {
+          "target": "smithy.api#Unit",
+          "traits": {
+            "smithy.api#enumValue": "OpenCSVSerDe"
+          }
+        },
+        "LazySimpleSerDe": {
+          "target": "smithy.api#Unit",
+          "traits": {
+            "smithy.api#enumValue": "LazySimpleSerDe"
+          }
+        },
+        "None": {
+          "target": "smithy.api#Unit",
+          "traits": {
+            "smithy.api#enumValue": "None"
+          }
+        }
+      }
+    },
     "com.amazonaws.glue#CustomCode": {
       "type": "structure",
       "members": {
@@ -33170,6 +33205,12 @@
           "traits": {
             "smithy.api#documentation": "<p>Specifies a list of supported custom datatypes.</p>"
           }
+        },
+        "Serde": {
+          "target": "com.amazonaws.glue#CsvSerdeOption",
+          "traits": {
+            "smithy.api#documentation": "<p>Sets the SerDe for processing CSV in the classifier, which will be applied in the Data Catalog. Valid values are <code>OpenCSVSerDe</code>, <code>LazySimpleSerDe</code>, and <code>None</code>. You can specify the <code>None</code> value when you want the crawler to do the detection.</p>"
+          }
         }
       },
       "traits": {
-Original file line number
+Diff line change
@@ Expand Up @@
      * //       CustomDatatypes: [ // CustomDatatypes
      * //         "STRING_VALUE",
      * //       ],
+     * //       Serde: "OpenCSVSerDe" || "LazySimpleSerDe" || "None",
      * //     },
      * //   },
      * // };
@@ Expand Down @@