diff --git a/packages/@aws-cdk/aws-glue/.gitignore b/packages/@aws-cdk/aws-glue/.gitignore index 018c65919d67c..266c0684c6844 100644 --- a/packages/@aws-cdk/aws-glue/.gitignore +++ b/packages/@aws-cdk/aws-glue/.gitignore @@ -14,5 +14,6 @@ nyc.config.js .LAST_PACKAGE *.snk !.eslintrc.js +!jest.config.js -junit.xml \ No newline at end of file +junit.xml diff --git a/packages/@aws-cdk/aws-glue/.npmignore b/packages/@aws-cdk/aws-glue/.npmignore index 95a6e5fe5bb87..bca0ae2513f1e 100644 --- a/packages/@aws-cdk/aws-glue/.npmignore +++ b/packages/@aws-cdk/aws-glue/.npmignore @@ -19,6 +19,7 @@ dist tsconfig.json .eslintrc.js +jest.config.js # exclude cdk artifacts **/cdk.out diff --git a/packages/@aws-cdk/aws-glue/jest.config.js b/packages/@aws-cdk/aws-glue/jest.config.js new file mode 100644 index 0000000000000..cd664e1d069e5 --- /dev/null +++ b/packages/@aws-cdk/aws-glue/jest.config.js @@ -0,0 +1,2 @@ +const baseConfig = require('../../../tools/cdk-build-tools/config/jest.config'); +module.exports = baseConfig; diff --git a/packages/@aws-cdk/aws-glue/lib/table.ts b/packages/@aws-cdk/aws-glue/lib/table.ts index f451e853dcb3a..6e65278fbb7d0 100644 --- a/packages/@aws-cdk/aws-glue/lib/table.ts +++ b/packages/@aws-cdk/aws-glue/lib/table.ts @@ -340,7 +340,7 @@ function validateSchema(columns: Column[], partitionKeys?: Column[]): void { const names = new Set(); (columns.concat(partitionKeys || [])).forEach(column => { if (names.has(column.name)) { - throw new Error('column names and partition keys must be unique, but \'p1\' is duplicated'); + throw new Error(`column names and partition keys must be unique, but \'${column.name}\' is duplicated`); } names.add(column.name); }); diff --git a/packages/@aws-cdk/aws-glue/package.json b/packages/@aws-cdk/aws-glue/package.json index d60997aff5011..0227fcaab0492 100644 --- a/packages/@aws-cdk/aws-glue/package.json +++ b/packages/@aws-cdk/aws-glue/package.json @@ -47,7 +47,8 @@ "compat": "cdk-compat" }, "cdk-build": { - "cloudformation": "AWS::Glue" + "cloudformation": "AWS::Glue", + "jest": true }, "keywords": [ "aws", @@ -67,7 +68,7 @@ "cdk-build-tools": "0.0.0", "cdk-integ-tools": "0.0.0", "cfn2ts": "0.0.0", - "nodeunit": "^0.11.3", + "jest": "^25.5.4", "pkglint": "0.0.0" }, "dependencies": { diff --git a/packages/@aws-cdk/aws-glue/test/database.test.ts b/packages/@aws-cdk/aws-glue/test/database.test.ts new file mode 100644 index 0000000000000..7ca9409525345 --- /dev/null +++ b/packages/@aws-cdk/aws-glue/test/database.test.ts @@ -0,0 +1,92 @@ +import { expect } from '@aws-cdk/assert'; +import '@aws-cdk/assert/jest'; +import { Stack } from '@aws-cdk/core'; +import { deepEqual, throws } from 'assert'; +import * as glue from '../lib'; + +test('default database does not create a bucket', () => { + const stack = new Stack(); + + new glue.Database(stack, 'Database', { + databaseName: 'test_database', + }); + + expect(stack).toMatch({ + Resources: { + DatabaseB269D8BB: { + Type: 'AWS::Glue::Database', + Properties: { + CatalogId: { + Ref: 'AWS::AccountId', + }, + DatabaseInput: { + Name: 'test_database', + }, + }, + }, + }, + }); + +}); + +test('explicit locationURI', () => { + const stack = new Stack(); + + new glue.Database(stack, 'Database', { + databaseName: 'test_database', + locationUri: 's3://my-uri/', + }); + + expect(stack).toMatch({ + Resources: { + DatabaseB269D8BB: { + Type: 'AWS::Glue::Database', + Properties: { + CatalogId: { + Ref: 'AWS::AccountId', + }, + DatabaseInput: { + LocationUri: 's3://my-uri/', + Name: 'test_database', + }, + }, + }, + }, + }); + +}); + +test('fromDatabase', () => { + // GIVEN + const stack = new Stack(); + + // WHEN + const database = glue.Database.fromDatabaseArn(stack, 'import', 'arn:aws:glue:us-east-1:123456789012:database/db1'); + + // THEN + deepEqual(database.databaseArn, 'arn:aws:glue:us-east-1:123456789012:database/db1'); + deepEqual(database.databaseName, 'db1'); + deepEqual(stack.resolve(database.catalogArn), { 'Fn::Join': [ '', + [ 'arn:', { Ref: 'AWS::Partition' }, ':glue:', { Ref: 'AWS::Region' }, ':', { Ref: 'AWS::AccountId' }, ':catalog' ] ] }); + deepEqual(stack.resolve(database.catalogId), { Ref: 'AWS::AccountId' }); +}); + +test('locationUri length must be >= 1', () => { + const stack = new Stack(); + throws(() => + new glue.Database(stack, 'Database', { + databaseName: 'test_database', + locationUri: '', + }), + ); +}); + +test('locationUri length must be <= 1024', () => { + const stack = new Stack(); + throws(() => + new glue.Database(stack, 'Database', { + databaseName: 'test_database', + locationUri: 'a'.repeat(1025), + }), + ); +}); diff --git a/packages/@aws-cdk/aws-glue/test/schema.test.ts b/packages/@aws-cdk/aws-glue/test/schema.test.ts new file mode 100644 index 0000000000000..f7910dc0bd852 --- /dev/null +++ b/packages/@aws-cdk/aws-glue/test/schema.test.ts @@ -0,0 +1,241 @@ +import '@aws-cdk/assert/jest'; +import { doesNotThrow, equal, throws } from 'assert'; +import { Schema } from '../lib'; + +test('boolean type', () => { + equal(Schema.BOOLEAN.inputString, 'boolean'); + equal(Schema.BOOLEAN.isPrimitive, true); +}); + +test('binary type', () => { + equal(Schema.BINARY.inputString, 'binary'); + equal(Schema.BINARY.isPrimitive, true); +}); + +test('bigint type', () => { + equal(Schema.BIG_INT.inputString, 'bigint'); + equal(Schema.BIG_INT.isPrimitive, true); +}); + +test('double type', () => { + equal(Schema.DOUBLE.inputString, 'double'); + equal(Schema.DOUBLE.isPrimitive, true); +}); + +test('float type', () => { + equal(Schema.FLOAT.inputString, 'float'); + equal(Schema.FLOAT.isPrimitive, true); +}); + +test('integer type', () => { + equal(Schema.INTEGER.inputString, 'int'); + equal(Schema.INTEGER.isPrimitive, true); +}); + +test('smallint type', () => { + equal(Schema.SMALL_INT.inputString, 'smallint'); + equal(Schema.SMALL_INT.isPrimitive, true); +}); + +test('tinyint type', () => { + equal(Schema.TINY_INT.inputString, 'tinyint'); + equal(Schema.TINY_INT.isPrimitive, true); +}); + +test('decimal type', () => { + equal(Schema.decimal(16).inputString, 'decimal(16)'); + equal(Schema.decimal(16, 1).inputString, 'decimal(16,1)'); + equal(Schema.decimal(16).isPrimitive, true); + equal(Schema.decimal(16, 1).isPrimitive, true); +}); +// TODO: decimal bounds + +test('date type', () => { + equal(Schema.DATE.inputString, 'date'); + equal(Schema.DATE.isPrimitive, true); +}); + +test('timestamp type', () => { + equal(Schema.TIMESTAMP.inputString, 'timestamp'); + equal(Schema.TIMESTAMP.isPrimitive, true); +}); + +test('string type', () => { + equal(Schema.STRING.inputString, 'string'); + equal(Schema.STRING.isPrimitive, true); +}); + +test('char type', () => { + equal(Schema.char(1).inputString, 'char(1)'); + equal(Schema.char(1).isPrimitive, true); +}); + +test('char length must be test(at least 1', () => { + doesNotThrow(() => Schema.char(1)); + throws(() => Schema.char(0)); + throws(() => Schema.char(-1)); +}); + +test('char length must test(be <= 255', () => { + doesNotThrow(() => Schema.char(255)); + throws(() => Schema.char(256)); +}); + +test('varchar type', () => { + equal(Schema.varchar(1).inputString, 'varchar(1)'); + equal(Schema.varchar(1).isPrimitive, true); +}); + +test('varchar length must be test(at least 1', () => { + doesNotThrow(() => Schema.varchar(1)); + throws(() => Schema.varchar(0)); + throws(() => Schema.varchar(-1)); +}); + +test('varchar length must test(be <= 65535', () => { + doesNotThrow(() => Schema.varchar(65535)); + throws(() => Schema.varchar(65536)); +}); + +test('test(array', () => { + const type = Schema.array(Schema.STRING); + equal(type.inputString, 'array'); + equal(type.isPrimitive, false); +}); + +test('array', () => { + const type = Schema.array(Schema.char(1)); + equal(type.inputString, 'array'); + equal(type.isPrimitive, false); +}); + +test('test(array', () => { + const type = Schema.array( + Schema.array(Schema.STRING)); + equal(type.inputString, 'array>'); + equal(type.isPrimitive, false); +}); + +test('test(array', () => { + const type = Schema.array( + Schema.map(Schema.STRING, Schema.STRING)); + equal(type.inputString, 'array>'); + equal(type.isPrimitive, false); +}); + +test('test(array', () => { + const type = Schema.array( + Schema.struct([{ + name: 'key', + type: Schema.STRING, + }])); + equal(type.inputString, 'array>'); + equal(type.isPrimitive, false); +}); + +test('map', () => { + const type = Schema.map( + Schema.STRING, + Schema.STRING, + ); + equal(type.inputString, 'map'); + equal(type.isPrimitive, false); +}); + +test('map', () => { + const type = Schema.map( + Schema.INTEGER, + Schema.STRING, + ); + equal(type.inputString, 'map'); + equal(type.isPrimitive, false); +}); + +test('map', () => { + const type = Schema.map( + Schema.char(1), + Schema.char(1), + ); + equal(type.inputString, 'map'); + equal(type.isPrimitive, false); +}); + +test('map', () => { + const type = Schema.map( + Schema.char(1), + Schema.array(Schema.STRING), + ); + equal(type.inputString, 'map>'); + equal(type.isPrimitive, false); +}); + +test('map', () => { + const type = Schema.map( + Schema.char(1), + Schema.map( + Schema.STRING, + Schema.STRING), + ); + equal(type.inputString, 'map>'); + equal(type.isPrimitive, false); +}); + +test('map', () => { + const type = Schema.map( + Schema.char(1), + Schema.struct([{ + name: 'key', + type: Schema.STRING, + }]), + ); + equal(type.inputString, 'map>'); + equal(type.isPrimitive, false); +}); + +test('map throws if keyType is test(non-primitive', () => { + throws(() => Schema.map( + Schema.array(Schema.STRING), + Schema.STRING, + )); + throws(() => Schema.map( + Schema.map(Schema.STRING, Schema.STRING), + Schema.STRING, + )); + throws(() => Schema.map( + Schema.struct([{ + name: 'key', + type: Schema.STRING, + }]), + Schema.STRING, + )); +}); + +test('struct type', () => { + const type = Schema.struct([{ + name: 'primitive', + type: Schema.STRING, + }, { + name: 'with_comment', + type: Schema.STRING, + comment: 'this has a comment', + }, { + name: 'array', + type: Schema.array(Schema.STRING), + }, { + name: 'map', + type: Schema.map(Schema.STRING, Schema.STRING), + }, { + name: 'nested_struct', + type: Schema.struct([{ + name: 'nested', + type: Schema.STRING, + comment: 'nested comment', + }]), + }]); + + equal(type.isPrimitive, false); + equal( + type.inputString, + // eslint-disable-next-line max-len + 'struct,map:map,nested_struct:struct>'); +}); \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue/test/table.test.ts b/packages/@aws-cdk/aws-glue/test/table.test.ts new file mode 100644 index 0000000000000..70d4caf3d5242 --- /dev/null +++ b/packages/@aws-cdk/aws-glue/test/table.test.ts @@ -0,0 +1,1599 @@ +import { expect as cdkExpect, haveResource, ResourcePart } from '@aws-cdk/assert'; +import '@aws-cdk/assert/jest'; +import * as iam from '@aws-cdk/aws-iam'; +import * as kms from '@aws-cdk/aws-kms'; +import * as s3 from '@aws-cdk/aws-s3'; +import * as cdk from '@aws-cdk/core'; +import { deepEqual, doesNotThrow, equal, notEqual, ok } from 'assert'; +import * as glue from '../lib'; + +test('unpartitioned JSON table', () => { + const app = new cdk.App(); + const dbStack = new cdk.Stack(app, 'db'); + const database = new glue.Database(dbStack, 'Database', { + databaseName: 'database', + }); + + const tableStack = new cdk.Stack(app, 'table'); + const table = new glue.Table(tableStack, 'Table', { + database, + tableName: 'table', + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + dataFormat: glue.DataFormat.JSON, + }); + equal(table.encryption, glue.TableEncryption.UNENCRYPTED); + + cdkExpect(tableStack).to(haveResource('AWS::S3::Bucket', { + Type: 'AWS::S3::Bucket', + DeletionPolicy: 'Retain', + UpdateReplacePolicy: 'Retain', + }, ResourcePart.CompleteDefinition)); + + cdkExpect(tableStack).to(haveResource('AWS::Glue::Table', { + CatalogId: { + Ref: 'AWS::AccountId', + }, + DatabaseName: { + 'Fn::ImportValue': 'db:ExportsOutputRefDatabaseB269D8BB88F4B1C4', + }, + TableInput: { + Name: 'table', + Description: 'table generated by CDK', + Parameters: { + has_encrypted_data: false, + }, + StorageDescriptor: { + Columns: [ + { + Name: 'col', + Type: 'string', + }, + ], + Compressed: false, + InputFormat: 'org.apache.hadoop.mapred.TextInputFormat', + Location: { + 'Fn::Join': [ + '', + [ + 's3://', + { + Ref: 'TableBucketDA42407C', + }, + '/data/', + ], + ], + }, + OutputFormat: 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat', + SerdeInfo: { + SerializationLibrary: 'org.openx.data.jsonserde.JsonSerDe', + }, + StoredAsSubDirectories: false, + }, + TableType: 'EXTERNAL_TABLE', + }, + })); + +}); + +test('partitioned JSON table', () => { + const app = new cdk.App(); + const dbStack = new cdk.Stack(app, 'db'); + const database = new glue.Database(dbStack, 'Database', { + databaseName: 'database', + }); + + const tableStack = new cdk.Stack(app, 'table'); + const table = new glue.Table(tableStack, 'Table', { + database, + tableName: 'table', + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + partitionKeys: [{ + name: 'year', + type: glue.Schema.SMALL_INT, + }], + dataFormat: glue.DataFormat.JSON, + }); + equal(table.encryption, glue.TableEncryption.UNENCRYPTED); + equal(table.encryptionKey, undefined); + equal(table.bucket.encryptionKey, undefined); + + cdkExpect(tableStack).to(haveResource('AWS::Glue::Table', { + CatalogId: { + Ref: 'AWS::AccountId', + }, + DatabaseName: { + 'Fn::ImportValue': 'db:ExportsOutputRefDatabaseB269D8BB88F4B1C4', + }, + TableInput: { + Name: 'table', + Description: 'table generated by CDK', + Parameters: { + has_encrypted_data: false, + }, + PartitionKeys: [ + { + Name: 'year', + Type: 'smallint', + }, + ], + StorageDescriptor: { + Columns: [ + { + Name: 'col', + Type: 'string', + }, + ], + Compressed: false, + InputFormat: 'org.apache.hadoop.mapred.TextInputFormat', + Location: { + 'Fn::Join': [ + '', + [ + 's3://', + { + Ref: 'TableBucketDA42407C', + }, + '/data/', + ], + ], + }, + OutputFormat: 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat', + SerdeInfo: { + SerializationLibrary: 'org.openx.data.jsonserde.JsonSerDe', + }, + StoredAsSubDirectories: false, + }, + TableType: 'EXTERNAL_TABLE', + }, + })); + +}); + +test('compressed table', () => { + const stack = new cdk.Stack(); + const database = new glue.Database(stack, 'Database', { + databaseName: 'database', + }); + + const table = new glue.Table(stack, 'Table', { + database, + tableName: 'table', + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + compressed: true, + dataFormat: glue.DataFormat.JSON, + }); + equal(table.encryptionKey, undefined); + equal(table.bucket.encryptionKey, undefined); + + cdkExpect(stack).to(haveResource('AWS::Glue::Table', { + CatalogId: { + Ref: 'AWS::AccountId', + }, + DatabaseName: { + Ref: 'DatabaseB269D8BB', + }, + TableInput: { + Name: 'table', + Description: 'table generated by CDK', + Parameters: { + has_encrypted_data: false, + }, + StorageDescriptor: { + Columns: [ + { + Name: 'col', + Type: 'string', + }, + ], + Compressed: true, + InputFormat: 'org.apache.hadoop.mapred.TextInputFormat', + Location: { + 'Fn::Join': [ + '', + [ + 's3://', + { + Ref: 'TableBucketDA42407C', + }, + '/data/', + ], + ], + }, + OutputFormat: 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat', + SerdeInfo: { + SerializationLibrary: 'org.openx.data.jsonserde.JsonSerDe', + }, + StoredAsSubDirectories: false, + }, + TableType: 'EXTERNAL_TABLE', + }, + })); + +}); + +test('table.node.defaultChild', () => { + // GIVEN + const stack = new cdk.Stack(); + const database = new glue.Database(stack, 'Database', { + databaseName: 'database', + }); + + // WHEN + const table = new glue.Table(stack, 'Table', { + database, + tableName: 'table', + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + compressed: true, + dataFormat: glue.DataFormat.JSON, + }); + + // THEN + ok(table.node.defaultChild instanceof glue.CfnTable); +}); + +test('encrypted table: SSE-S3', () => { + const stack = new cdk.Stack(); + const database = new glue.Database(stack, 'Database', { + databaseName: 'database', + }); + + const table = new glue.Table(stack, 'Table', { + database, + tableName: 'table', + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + encryption: glue.TableEncryption.S3_MANAGED, + dataFormat: glue.DataFormat.JSON, + }); + equal(table.encryption, glue.TableEncryption.S3_MANAGED); + equal(table.encryptionKey, undefined); + equal(table.bucket.encryptionKey, undefined); + + cdkExpect(stack).to(haveResource('AWS::Glue::Table', { + CatalogId: { + Ref: 'AWS::AccountId', + }, + DatabaseName: { + Ref: 'DatabaseB269D8BB', + }, + TableInput: { + Name: 'table', + Description: 'table generated by CDK', + Parameters: { + has_encrypted_data: true, + }, + StorageDescriptor: { + Columns: [ + { + Name: 'col', + Type: 'string', + }, + ], + Compressed: false, + InputFormat: 'org.apache.hadoop.mapred.TextInputFormat', + Location: { + 'Fn::Join': [ + '', + [ + 's3://', + { + Ref: 'TableBucketDA42407C', + }, + '/data/', + ], + ], + }, + OutputFormat: 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat', + SerdeInfo: { + SerializationLibrary: 'org.openx.data.jsonserde.JsonSerDe', + }, + StoredAsSubDirectories: false, + }, + TableType: 'EXTERNAL_TABLE', + }, + })); + + cdkExpect(stack).to(haveResource('AWS::S3::Bucket', { + BucketEncryption: { + ServerSideEncryptionConfiguration: [ + { + ServerSideEncryptionByDefault: { + SSEAlgorithm: 'AES256', + }, + }, + ], + }, + })); + +}); + +test('encrypted table: SSE-KMS (implicitly created key)', () => { + const stack = new cdk.Stack(); + const database = new glue.Database(stack, 'Database', { + databaseName: 'database', + }); + + const table = new glue.Table(stack, 'Table', { + database, + tableName: 'table', + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + encryption: glue.TableEncryption.KMS, + dataFormat: glue.DataFormat.JSON, + }); + equal(table.encryption, glue.TableEncryption.KMS); + equal(table.encryptionKey, table.bucket.encryptionKey); + + cdkExpect(stack).to(haveResource('AWS::KMS::Key', { + KeyPolicy: { + Statement: [ + { + Action: [ + 'kms:Create*', + 'kms:Describe*', + 'kms:Enable*', + 'kms:List*', + 'kms:Put*', + 'kms:Update*', + 'kms:Revoke*', + 'kms:Disable*', + 'kms:Get*', + 'kms:Delete*', + 'kms:ScheduleKeyDeletion', + 'kms:CancelKeyDeletion', + 'kms:GenerateDataKey', + 'kms:TagResource', + 'kms:UntagResource', + ], + Effect: 'Allow', + Principal: { + AWS: { + 'Fn::Join': [ + '', + [ + 'arn:', + { + Ref: 'AWS::Partition', + }, + ':iam::', + { + Ref: 'AWS::AccountId', + }, + ':root', + ], + ], + }, + }, + Resource: '*', + }, + ], + Version: '2012-10-17', + }, + Description: 'Created by Table/Bucket', + })); + + cdkExpect(stack).to(haveResource('AWS::S3::Bucket', { + BucketEncryption: { + ServerSideEncryptionConfiguration: [ + { + ServerSideEncryptionByDefault: { + KMSMasterKeyID: { + 'Fn::GetAtt': [ + 'TableBucketKey3E9F984A', + 'Arn', + ], + }, + SSEAlgorithm: 'aws:kms', + }, + }, + ], + }, + })); + + cdkExpect(stack).to(haveResource('AWS::Glue::Table', { + CatalogId: { + Ref: 'AWS::AccountId', + }, + DatabaseName: { + Ref: 'DatabaseB269D8BB', + }, + TableInput: { + Name: 'table', + Description: 'table generated by CDK', + Parameters: { + has_encrypted_data: true, + }, + StorageDescriptor: { + Columns: [ + { + Name: 'col', + Type: 'string', + }, + ], + Compressed: false, + InputFormat: 'org.apache.hadoop.mapred.TextInputFormat', + Location: { + 'Fn::Join': [ + '', + [ + 's3://', + { + Ref: 'TableBucketDA42407C', + }, + '/data/', + ], + ], + }, + OutputFormat: 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat', + SerdeInfo: { + SerializationLibrary: 'org.openx.data.jsonserde.JsonSerDe', + }, + StoredAsSubDirectories: false, + }, + TableType: 'EXTERNAL_TABLE', + }, + })); + +}); + +test('encrypted table: SSE-KMS (explicitly created key)', () => { + const stack = new cdk.Stack(); + const database = new glue.Database(stack, 'Database', { + databaseName: 'database', + }); + const encryptionKey = new kms.Key(stack, 'MyKey'); + + const table = new glue.Table(stack, 'Table', { + database, + tableName: 'table', + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + encryption: glue.TableEncryption.KMS, + encryptionKey, + dataFormat: glue.DataFormat.JSON, + }); + equal(table.encryption, glue.TableEncryption.KMS); + equal(table.encryptionKey, table.bucket.encryptionKey); + notEqual(table.encryptionKey, undefined); + + cdkExpect(stack).to(haveResource('AWS::KMS::Key', { + KeyPolicy: { + Statement: [ + { + Action: [ + 'kms:Create*', + 'kms:Describe*', + 'kms:Enable*', + 'kms:List*', + 'kms:Put*', + 'kms:Update*', + 'kms:Revoke*', + 'kms:Disable*', + 'kms:Get*', + 'kms:Delete*', + 'kms:ScheduleKeyDeletion', + 'kms:CancelKeyDeletion', + 'kms:GenerateDataKey', + 'kms:TagResource', + 'kms:UntagResource', + ], + Effect: 'Allow', + Principal: { + AWS: { + 'Fn::Join': [ + '', + [ + 'arn:', + { + Ref: 'AWS::Partition', + }, + ':iam::', + { + Ref: 'AWS::AccountId', + }, + ':root', + ], + ], + }, + }, + Resource: '*', + }, + ], + Version: '2012-10-17', + }, + })); + + cdkExpect(stack).to(haveResource('AWS::S3::Bucket', { + BucketEncryption: { + ServerSideEncryptionConfiguration: [ + { + ServerSideEncryptionByDefault: { + KMSMasterKeyID: { + 'Fn::GetAtt': [ + 'MyKey6AB29FA6', + 'Arn', + ], + }, + SSEAlgorithm: 'aws:kms', + }, + }, + ], + }, + })); + + cdkExpect(stack).to(haveResource('AWS::Glue::Table', { + CatalogId: { + Ref: 'AWS::AccountId', + }, + DatabaseName: { + Ref: 'DatabaseB269D8BB', + }, + TableInput: { + Description: 'table generated by CDK', + Name: 'table', + Parameters: { + has_encrypted_data: true, + }, + StorageDescriptor: { + Columns: [ + { + Name: 'col', + Type: 'string', + }, + ], + Compressed: false, + InputFormat: 'org.apache.hadoop.mapred.TextInputFormat', + Location: { + 'Fn::Join': [ + '', + [ + 's3://', + { + Ref: 'TableBucketDA42407C', + }, + '/data/', + ], + ], + }, + OutputFormat: 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat', + SerdeInfo: { + SerializationLibrary: 'org.openx.data.jsonserde.JsonSerDe', + }, + StoredAsSubDirectories: false, + }, + TableType: 'EXTERNAL_TABLE', + }, + })); + +}); + +test('encrypted table: SSE-KMS_MANAGED', () => { + const stack = new cdk.Stack(); + const database = new glue.Database(stack, 'Database', { + databaseName: 'database', + }); + + const table = new glue.Table(stack, 'Table', { + database, + tableName: 'table', + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + encryption: glue.TableEncryption.KMS_MANAGED, + dataFormat: glue.DataFormat.JSON, + }); + equal(table.encryption, glue.TableEncryption.KMS_MANAGED); + equal(table.encryptionKey, undefined); + equal(table.bucket.encryptionKey, undefined); + + cdkExpect(stack).to(haveResource('AWS::S3::Bucket', { + BucketEncryption: { + ServerSideEncryptionConfiguration: [ + { + ServerSideEncryptionByDefault: { + SSEAlgorithm: 'aws:kms', + }, + }, + ], + }, + })); + + cdkExpect(stack).to(haveResource('AWS::Glue::Table', { + CatalogId: { + Ref: 'AWS::AccountId', + }, + DatabaseName: { + Ref: 'DatabaseB269D8BB', + }, + TableInput: { + Name: 'table', + Description: 'table generated by CDK', + Parameters: { + has_encrypted_data: true, + }, + StorageDescriptor: { + Columns: [ + { + Name: 'col', + Type: 'string', + }, + ], + Compressed: false, + InputFormat: 'org.apache.hadoop.mapred.TextInputFormat', + Location: { + 'Fn::Join': [ + '', + [ + 's3://', + { + Ref: 'TableBucketDA42407C', + }, + '/data/', + ], + ], + }, + OutputFormat: 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat', + SerdeInfo: { + SerializationLibrary: 'org.openx.data.jsonserde.JsonSerDe', + }, + StoredAsSubDirectories: false, + }, + TableType: 'EXTERNAL_TABLE', + }, + })); + +}); + +test('encrypted table: CSE-KMS (implicitly created key)', () => { + const stack = new cdk.Stack(); + const database = new glue.Database(stack, 'Database', { + databaseName: 'database', + }); + + const table = new glue.Table(stack, 'Table', { + database, + tableName: 'table', + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + encryption: glue.TableEncryption.CLIENT_SIDE_KMS, + dataFormat: glue.DataFormat.JSON, + }); + equal(table.encryption, glue.TableEncryption.CLIENT_SIDE_KMS); + notEqual(table.encryptionKey, undefined); + equal(table.bucket.encryptionKey, undefined); + + cdkExpect(stack).to(haveResource('AWS::KMS::Key', { + KeyPolicy: { + Statement: [ + { + Action: [ + 'kms:Create*', + 'kms:Describe*', + 'kms:Enable*', + 'kms:List*', + 'kms:Put*', + 'kms:Update*', + 'kms:Revoke*', + 'kms:Disable*', + 'kms:Get*', + 'kms:Delete*', + 'kms:ScheduleKeyDeletion', + 'kms:CancelKeyDeletion', + 'kms:GenerateDataKey', + 'kms:TagResource', + 'kms:UntagResource', + ], + Effect: 'Allow', + Principal: { + AWS: { + 'Fn::Join': [ + '', + [ + 'arn:', + { + Ref: 'AWS::Partition', + }, + ':iam::', + { + Ref: 'AWS::AccountId', + }, + ':root', + ], + ], + }, + }, + Resource: '*', + }, + ], + Version: '2012-10-17', + }, + })); + + cdkExpect(stack).to(haveResource('AWS::Glue::Table', { + CatalogId: { + Ref: 'AWS::AccountId', + }, + DatabaseName: { + Ref: 'DatabaseB269D8BB', + }, + TableInput: { + Description: 'table generated by CDK', + Name: 'table', + Parameters: { + has_encrypted_data: true, + }, + StorageDescriptor: { + Columns: [ + { + Name: 'col', + Type: 'string', + }, + ], + Compressed: false, + InputFormat: 'org.apache.hadoop.mapred.TextInputFormat', + Location: { + 'Fn::Join': [ + '', + [ + 's3://', + { + Ref: 'TableBucketDA42407C', + }, + '/data/', + ], + ], + }, + OutputFormat: 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat', + SerdeInfo: { + SerializationLibrary: 'org.openx.data.jsonserde.JsonSerDe', + }, + StoredAsSubDirectories: false, + }, + TableType: 'EXTERNAL_TABLE', + }, + })); + +}); + +test('encrypted table: CSE-KMS (explicitly created key)', () => { + const stack = new cdk.Stack(); + const database = new glue.Database(stack, 'Database', { + databaseName: 'database', + }); + const encryptionKey = new kms.Key(stack, 'MyKey'); + + const table = new glue.Table(stack, 'Table', { + database, + tableName: 'table', + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + encryption: glue.TableEncryption.CLIENT_SIDE_KMS, + encryptionKey, + dataFormat: glue.DataFormat.JSON, + }); + equal(table.encryption, glue.TableEncryption.CLIENT_SIDE_KMS); + notEqual(table.encryptionKey, undefined); + equal(table.bucket.encryptionKey, undefined); + + cdkExpect(stack).to(haveResource('AWS::KMS::Key', { + KeyPolicy: { + Statement: [ + { + Action: [ + 'kms:Create*', + 'kms:Describe*', + 'kms:Enable*', + 'kms:List*', + 'kms:Put*', + 'kms:Update*', + 'kms:Revoke*', + 'kms:Disable*', + 'kms:Get*', + 'kms:Delete*', + 'kms:ScheduleKeyDeletion', + 'kms:CancelKeyDeletion', + 'kms:GenerateDataKey', + 'kms:TagResource', + 'kms:UntagResource', + ], + Effect: 'Allow', + Principal: { + AWS: { + 'Fn::Join': [ + '', + [ + 'arn:', + { + Ref: 'AWS::Partition', + }, + ':iam::', + { + Ref: 'AWS::AccountId', + }, + ':root', + ], + ], + }, + }, + Resource: '*', + }, + ], + Version: '2012-10-17', + }, + })); + + cdkExpect(stack).to(haveResource('AWS::Glue::Table', { + CatalogId: { + Ref: 'AWS::AccountId', + }, + DatabaseName: { + Ref: 'DatabaseB269D8BB', + }, + TableInput: { + Description: 'table generated by CDK', + Name: 'table', + Parameters: { + has_encrypted_data: true, + }, + StorageDescriptor: { + Columns: [ + { + Name: 'col', + Type: 'string', + }, + ], + Compressed: false, + InputFormat: 'org.apache.hadoop.mapred.TextInputFormat', + Location: { + 'Fn::Join': [ + '', + [ + 's3://', + { + Ref: 'TableBucketDA42407C', + }, + '/data/', + ], + ], + }, + OutputFormat: 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat', + SerdeInfo: { + SerializationLibrary: 'org.openx.data.jsonserde.JsonSerDe', + }, + StoredAsSubDirectories: false, + }, + TableType: 'EXTERNAL_TABLE', + }, + })); + +}); + +test('encrypted table: CSE-KMS (explicitly passed bucket and key)', () => { + const stack = new cdk.Stack(); + const database = new glue.Database(stack, 'Database', { + databaseName: 'database', + }); + const bucket = new s3.Bucket(stack, 'Bucket'); + const encryptionKey = new kms.Key(stack, 'MyKey'); + + const table = new glue.Table(stack, 'Table', { + database, + tableName: 'table', + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + bucket, + encryption: glue.TableEncryption.CLIENT_SIDE_KMS, + encryptionKey, + dataFormat: glue.DataFormat.JSON, + }); + equal(table.encryption, glue.TableEncryption.CLIENT_SIDE_KMS); + notEqual(table.encryptionKey, undefined); + equal(table.bucket.encryptionKey, undefined); + + cdkExpect(stack).to(haveResource('AWS::KMS::Key', { + KeyPolicy: { + Statement: [ + { + Action: [ + 'kms:Create*', + 'kms:Describe*', + 'kms:Enable*', + 'kms:List*', + 'kms:Put*', + 'kms:Update*', + 'kms:Revoke*', + 'kms:Disable*', + 'kms:Get*', + 'kms:Delete*', + 'kms:ScheduleKeyDeletion', + 'kms:CancelKeyDeletion', + 'kms:GenerateDataKey', + 'kms:TagResource', + 'kms:UntagResource', + ], + Effect: 'Allow', + Principal: { + AWS: { + 'Fn::Join': [ + '', + [ + 'arn:', + { + Ref: 'AWS::Partition', + }, + ':iam::', + { + Ref: 'AWS::AccountId', + }, + ':root', + ], + ], + }, + }, + Resource: '*', + }, + ], + Version: '2012-10-17', + }, + })); + + cdkExpect(stack).to(haveResource('AWS::Glue::Table', { + CatalogId: { + Ref: 'AWS::AccountId', + }, + DatabaseName: { + Ref: 'DatabaseB269D8BB', + }, + TableInput: { + Description: 'table generated by CDK', + Name: 'table', + Parameters: { + has_encrypted_data: true, + }, + StorageDescriptor: { + Columns: [ + { + Name: 'col', + Type: 'string', + }, + ], + Compressed: false, + InputFormat: 'org.apache.hadoop.mapred.TextInputFormat', + Location: { + 'Fn::Join': [ + '', + [ + 's3://', + { + Ref: 'Bucket83908E77', + }, + '/data/', + ], + ], + }, + OutputFormat: 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat', + SerdeInfo: { + SerializationLibrary: 'org.openx.data.jsonserde.JsonSerDe', + }, + StoredAsSubDirectories: false, + }, + TableType: 'EXTERNAL_TABLE', + }, + })); + +}); + +test('explicit s3 bucket and prefix', () => { + const app = new cdk.App(); + const dbStack = new cdk.Stack(app, 'db'); + const stack = new cdk.Stack(app, 'app'); + const bucket = new s3.Bucket(stack, 'ExplicitBucket'); + const database = new glue.Database(dbStack, 'Database', { + databaseName: 'database', + }); + + new glue.Table(stack, 'Table', { + database, + bucket, + s3Prefix: 'prefix/', + tableName: 'table', + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + dataFormat: glue.DataFormat.JSON, + }); + + cdkExpect(stack).to(haveResource('AWS::Glue::Table', { + CatalogId: { + Ref: 'AWS::AccountId', + }, + DatabaseName: { + 'Fn::ImportValue': 'db:ExportsOutputRefDatabaseB269D8BB88F4B1C4', + }, + TableInput: { + Description: 'table generated by CDK', + Name: 'table', + Parameters: { + has_encrypted_data: false, + }, + StorageDescriptor: { + Columns: [ + { + Name: 'col', + Type: 'string', + }, + ], + Compressed: false, + InputFormat: 'org.apache.hadoop.mapred.TextInputFormat', + Location: { + 'Fn::Join': [ + '', + [ + 's3://', + { + Ref: 'ExplicitBucket0AA51A3F', + }, + '/prefix/', + ], + ], + }, + OutputFormat: 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat', + SerdeInfo: { + SerializationLibrary: 'org.openx.data.jsonserde.JsonSerDe', + }, + StoredAsSubDirectories: false, + }, + TableType: 'EXTERNAL_TABLE', + }, + })); + +}); + +test('explicit s3 bucket and with empty prefix', () => { + const app = new cdk.App(); + const dbStack = new cdk.Stack(app, 'db'); + const stack = new cdk.Stack(app, 'app'); + const bucket = new s3.Bucket(stack, 'ExplicitBucket'); + const database = new glue.Database(dbStack, 'Database', { + databaseName: 'database', + }); + + new glue.Table(stack, 'Table', { + database, + bucket, + s3Prefix: '', + tableName: 'table', + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + dataFormat: glue.DataFormat.JSON, + }); + + cdkExpect(stack).to(haveResource('AWS::Glue::Table', { + CatalogId: { + Ref: 'AWS::AccountId', + }, + DatabaseName: { + 'Fn::ImportValue': 'db:ExportsOutputRefDatabaseB269D8BB88F4B1C4', + }, + TableInput: { + Description: 'table generated by CDK', + Name: 'table', + Parameters: { + has_encrypted_data: false, + }, + StorageDescriptor: { + Columns: [ + { + Name: 'col', + Type: 'string', + }, + ], + Compressed: false, + InputFormat: 'org.apache.hadoop.mapred.TextInputFormat', + Location: { + 'Fn::Join': [ + '', + [ + 's3://', + { + Ref: 'ExplicitBucket0AA51A3F', + }, + '/', + ], + ], + }, + OutputFormat: 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat', + SerdeInfo: { + SerializationLibrary: 'org.openx.data.jsonserde.JsonSerDe', + }, + StoredAsSubDirectories: false, + }, + TableType: 'EXTERNAL_TABLE', + }, + })); + +}); + +test('grants: read only', () => { + const stack = new cdk.Stack(); + const user = new iam.User(stack, 'User'); + const database = new glue.Database(stack, 'Database', { + databaseName: 'database', + }); + + const table = new glue.Table(stack, 'Table', { + database, + tableName: 'table', + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + compressed: true, + dataFormat: glue.DataFormat.JSON, + }); + + table.grantRead(user); + + cdkExpect(stack).to(haveResource('AWS::IAM::Policy', { + PolicyDocument: { + Statement: [ + { + Action: [ + 'glue:BatchDeletePartition', + 'glue:BatchGetPartition', + 'glue:GetPartition', + 'glue:GetPartitions', + 'glue:GetTable', + 'glue:GetTables', + 'glue:GetTableVersions', + ], + Effect: 'Allow', + Resource: { + 'Fn::Join': [ + '', + [ + 'arn:', + { + Ref: 'AWS::Partition', + }, + ':glue:', + { + Ref: 'AWS::Region', + }, + ':', + { + Ref: 'AWS::AccountId', + }, + ':table/', + { + Ref: 'DatabaseB269D8BB', + }, + '/', + { + Ref: 'Table4C2D914F', + }, + ], + ], + }, + }, + { + Action: [ + 's3:GetObject*', + 's3:GetBucket*', + 's3:List*', + ], + Effect: 'Allow', + Resource: [ + { + 'Fn::GetAtt': [ + 'TableBucketDA42407C', + 'Arn', + ], + }, + { + 'Fn::Join': [ + '', + [ + { + 'Fn::GetAtt': [ + 'TableBucketDA42407C', + 'Arn', + ], + }, + '/data/', + ], + ], + }, + ], + }, + ], + Version: '2012-10-17', + }, + PolicyName: 'UserDefaultPolicy1F97781E', + Users: [ + { + Ref: 'User00B015A1', + }, + ], + })); + +}); + +test('grants: write only', () => { + const stack = new cdk.Stack(); + const user = new iam.User(stack, 'User'); + const database = new glue.Database(stack, 'Database', { + databaseName: 'database', + }); + + const table = new glue.Table(stack, 'Table', { + database, + tableName: 'table', + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + compressed: true, + dataFormat: glue.DataFormat.JSON, + }); + + table.grantWrite(user); + + cdkExpect(stack).to(haveResource('AWS::IAM::Policy', { + PolicyDocument: { + Statement: [ + { + Action: [ + 'glue:BatchCreatePartition', + 'glue:BatchDeletePartition', + 'glue:CreatePartition', + 'glue:DeletePartition', + 'glue:UpdatePartition', + ], + Effect: 'Allow', + Resource: { + 'Fn::Join': [ + '', + [ + 'arn:', + { + Ref: 'AWS::Partition', + }, + ':glue:', + { + Ref: 'AWS::Region', + }, + ':', + { + Ref: 'AWS::AccountId', + }, + ':table/', + { + Ref: 'DatabaseB269D8BB', + }, + '/', + { + Ref: 'Table4C2D914F', + }, + ], + ], + }, + }, + { + Action: [ + 's3:DeleteObject*', + 's3:PutObject*', + 's3:Abort*', + ], + Effect: 'Allow', + Resource: [ + { + 'Fn::GetAtt': [ + 'TableBucketDA42407C', + 'Arn', + ], + }, + { + 'Fn::Join': [ + '', + [ + { + 'Fn::GetAtt': [ + 'TableBucketDA42407C', + 'Arn', + ], + }, + '/data/', + ], + ], + }, + ], + }, + ], + Version: '2012-10-17', + }, + PolicyName: 'UserDefaultPolicy1F97781E', + Users: [ + { + Ref: 'User00B015A1', + }, + ], + })); + +}); + +test('grants: read and write', () => { + const stack = new cdk.Stack(); + const user = new iam.User(stack, 'User'); + const database = new glue.Database(stack, 'Database', { + databaseName: 'database', + }); + + const table = new glue.Table(stack, 'Table', { + database, + tableName: 'table', + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + compressed: true, + dataFormat: glue.DataFormat.JSON, + }); + + table.grantReadWrite(user); + + cdkExpect(stack).to(haveResource('AWS::IAM::Policy', { + PolicyDocument: { + Statement: [ + { + Action: [ + 'glue:BatchDeletePartition', + 'glue:BatchGetPartition', + 'glue:GetPartition', + 'glue:GetPartitions', + 'glue:GetTable', + 'glue:GetTables', + 'glue:GetTableVersions', + 'glue:BatchCreatePartition', + 'glue:BatchDeletePartition', + 'glue:CreatePartition', + 'glue:DeletePartition', + 'glue:UpdatePartition', + ], + Effect: 'Allow', + Resource: { + 'Fn::Join': [ + '', + [ + 'arn:', + { + Ref: 'AWS::Partition', + }, + ':glue:', + { + Ref: 'AWS::Region', + }, + ':', + { + Ref: 'AWS::AccountId', + }, + ':table/', + { + Ref: 'DatabaseB269D8BB', + }, + '/', + { + Ref: 'Table4C2D914F', + }, + ], + ], + }, + }, + { + Action: [ + 's3:GetObject*', + 's3:GetBucket*', + 's3:List*', + 's3:DeleteObject*', + 's3:PutObject*', + 's3:Abort*', + ], + Effect: 'Allow', + Resource: [ + { + 'Fn::GetAtt': [ + 'TableBucketDA42407C', + 'Arn', + ], + }, + { + 'Fn::Join': [ + '', + [ + { + 'Fn::GetAtt': [ + 'TableBucketDA42407C', + 'Arn', + ], + }, + '/data/', + ], + ], + }, + ], + }, + ], + Version: '2012-10-17', + }, + PolicyName: 'UserDefaultPolicy1F97781E', + Users: [ + { + Ref: 'User00B015A1', + }, + ], + })); + +}); + +test('validate: at least one column', () => { + expect(() => { + createTable({ + columns: [], + tableName: 'name', + }); + }).toThrowError('you must specify at least one column for the table'); + +}); + +test('validate: unique column names', () => { + expect(() => { + createTable({ + tableName: 'name', + columns: [{ + name: 'col1', + type: glue.Schema.STRING, + }, { + name: 'col1', + type: glue.Schema.STRING, + }], + }); + }).toThrowError("column names and partition keys must be unique, but 'col1' is duplicated"); + +}); + +test('validate: unique partition keys', () => { + expect(() => { + createTable({ + tableName: 'name', + columns: [{ + name: 'col1', + type: glue.Schema.STRING, + }], + partitionKeys: [{ + name: 'p1', + type: glue.Schema.STRING, + }, { + name: 'p1', + type: glue.Schema.STRING, + }], + }); + }).toThrowError("column names and partition keys must be unique, but 'p1' is duplicated"); + +}); + +test('validate: column names and partition keys are all unique', () => { + expect(() => { createTable({ + tableName: 'name', + columns: [{ + name: 'col1', + type: glue.Schema.STRING, + }], + partitionKeys: [{ + name: 'col1', + type: glue.Schema.STRING, + }], + }); + }).toThrowError("column names and partition keys must be unique, but 'col1' is duplicated"); + +}); + +test('validate: can not specify an explicit bucket and encryption', () => { + expect(() => { + createTable({ + tableName: 'name', + columns: [{ + name: 'col1', + type: glue.Schema.STRING, + }], + bucket: new s3.Bucket(new cdk.Stack(), 'Bucket'), + encryption: glue.TableEncryption.KMS, + }); + }).toThrowError('you can not specify encryption settings if you also provide a bucket'); +}); + +test('validate: can explicitly pass bucket if Encryption undefined', () => { + doesNotThrow(() => createTable({ + tableName: 'name', + columns: [{ + name: 'col1', + type: glue.Schema.STRING, + }], + bucket: new s3.Bucket(new cdk.Stack(), 'Bucket'), + encryption: undefined, + })); +}); + +test('validate: can explicitly pass bucket if Unencrypted', () => { + doesNotThrow(() => createTable({ + tableName: 'name', + columns: [{ + name: 'col1', + type: glue.Schema.STRING, + }], + bucket: new s3.Bucket(new cdk.Stack(), 'Bucket'), + encryption: undefined, + })); +}); + +test('validate: can explicitly pass bucket if ClientSideKms', () => { + doesNotThrow(() => createTable({ + tableName: 'name', + columns: [{ + name: 'col1', + type: glue.Schema.STRING, + }], + bucket: new s3.Bucket(new cdk.Stack(), 'Bucket'), + encryption: glue.TableEncryption.CLIENT_SIDE_KMS, + })); +}); + +test('Table.fromTableArn', () => { + // GIVEN + const stack = new cdk.Stack(); + + // WHEN + const table = glue.Table.fromTableArn(stack, 'boom', 'arn:aws:glue:us-east-1:123456789012:table/db1/tbl1'); + + // THEN + deepEqual(table.tableArn, 'arn:aws:glue:us-east-1:123456789012:table/db1/tbl1'); + deepEqual(table.tableName, 'tbl1'); +}); + +function createTable(props: Pick>): void { + const stack = new cdk.Stack(); + new glue.Table(stack, 'table', { + ...props, + database: new glue.Database(stack, 'db', { + databaseName: 'database_name', + }), + dataFormat: glue.DataFormat.JSON, + }); +} diff --git a/packages/@aws-cdk/aws-glue/test/test.database.ts b/packages/@aws-cdk/aws-glue/test/test.database.ts deleted file mode 100644 index 121dd7a8ef594..0000000000000 --- a/packages/@aws-cdk/aws-glue/test/test.database.ts +++ /dev/null @@ -1,98 +0,0 @@ -import { expect } from '@aws-cdk/assert'; -import { Stack } from '@aws-cdk/core'; -import { Test } from 'nodeunit'; -import * as glue from '../lib'; - -export = { - 'default database does not create a bucket'(test: Test) { - const stack = new Stack(); - - new glue.Database(stack, 'Database', { - databaseName: 'test_database', - }); - - expect(stack).toMatch({ - Resources: { - DatabaseB269D8BB: { - Type: 'AWS::Glue::Database', - Properties: { - CatalogId: { - Ref: 'AWS::AccountId', - }, - DatabaseInput: { - Name: 'test_database', - }, - }, - }, - }, - }); - - test.done(); - }, - - 'explicit locationURI'(test: Test) { - const stack = new Stack(); - - new glue.Database(stack, 'Database', { - databaseName: 'test_database', - locationUri: 's3://my-uri/', - }); - - expect(stack).toMatch({ - Resources: { - DatabaseB269D8BB: { - Type: 'AWS::Glue::Database', - Properties: { - CatalogId: { - Ref: 'AWS::AccountId', - }, - DatabaseInput: { - LocationUri: 's3://my-uri/', - Name: 'test_database', - }, - }, - }, - }, - }); - - test.done(); - }, - - 'fromDatabase'(test: Test) { - // GIVEN - const stack = new Stack(); - - // WHEN - const database = glue.Database.fromDatabaseArn(stack, 'import', 'arn:aws:glue:us-east-1:123456789012:database/db1'); - - // THEN - test.deepEqual(database.databaseArn, 'arn:aws:glue:us-east-1:123456789012:database/db1'); - test.deepEqual(database.databaseName, 'db1'); - test.deepEqual(stack.resolve(database.catalogArn), { 'Fn::Join': [ '', - [ 'arn:', { Ref: 'AWS::Partition' }, ':glue:', { Ref: 'AWS::Region' }, ':', { Ref: 'AWS::AccountId' }, ':catalog' ] ] }); - test.deepEqual(stack.resolve(database.catalogId), { Ref: 'AWS::AccountId' }); - test.done(); - }, - - 'locationUri length must be >= 1'(test: Test) { - const stack = new Stack(); - test.throws(() => - new glue.Database(stack, 'Database', { - databaseName: 'test_database', - locationUri: '', - }), - ); - test.done(); - }, - - 'locationUri length must be <= 1024'(test: Test) { - const stack = new Stack(); - test.throws(() => - new glue.Database(stack, 'Database', { - databaseName: 'test_database', - locationUri: 'a'.repeat(1025), - }), - ); - test.done(); - }, -}; diff --git a/packages/@aws-cdk/aws-glue/test/test.schema.ts b/packages/@aws-cdk/aws-glue/test/test.schema.ts deleted file mode 100644 index 22246b5015723..0000000000000 --- a/packages/@aws-cdk/aws-glue/test/test.schema.ts +++ /dev/null @@ -1,274 +0,0 @@ -import { Test } from 'nodeunit'; - -import { Schema } from '../lib'; - -export = { - 'boolean type'(test: Test) { - test.equals(Schema.BOOLEAN.inputString, 'boolean'); - test.equals(Schema.BOOLEAN.isPrimitive, true); - test.done(); - }, - - 'binary type'(test: Test) { - test.equals(Schema.BINARY.inputString, 'binary'); - test.equals(Schema.BINARY.isPrimitive, true); - test.done(); - }, - - 'bigint type'(test: Test) { - test.equals(Schema.BIG_INT.inputString, 'bigint'); - test.equals(Schema.BIG_INT.isPrimitive, true); - test.done(); - }, - - 'double type'(test: Test) { - test.equals(Schema.DOUBLE.inputString, 'double'); - test.equals(Schema.DOUBLE.isPrimitive, true); - test.done(); - }, - - 'float type'(test: Test) { - test.equals(Schema.FLOAT.inputString, 'float'); - test.equals(Schema.FLOAT.isPrimitive, true); - test.done(); - }, - - 'integer type'(test: Test) { - test.equals(Schema.INTEGER.inputString, 'int'); - test.equals(Schema.INTEGER.isPrimitive, true); - test.done(); - }, - - 'smallint type'(test: Test) { - test.equals(Schema.SMALL_INT.inputString, 'smallint'); - test.equals(Schema.SMALL_INT.isPrimitive, true); - test.done(); - }, - - 'tinyint type'(test: Test) { - test.equals(Schema.TINY_INT.inputString, 'tinyint'); - test.equals(Schema.TINY_INT.isPrimitive, true); - test.done(); - }, - - 'decimal type'(test: Test) { - test.equals(Schema.decimal(16).inputString, 'decimal(16)'); - test.equals(Schema.decimal(16, 1).inputString, 'decimal(16,1)'); - test.equals(Schema.decimal(16).isPrimitive, true); - test.equals(Schema.decimal(16, 1).isPrimitive, true); - test.done(); - }, - // TODO: decimal bounds - - 'date type'(test: Test) { - test.equals(Schema.DATE.inputString, 'date'); - test.equals(Schema.DATE.isPrimitive, true); - test.done(); - }, - - 'timestamp type'(test: Test) { - test.equals(Schema.TIMESTAMP.inputString, 'timestamp'); - test.equals(Schema.TIMESTAMP.isPrimitive, true); - test.done(); - }, - - 'string type'(test: Test) { - test.equals(Schema.STRING.inputString, 'string'); - test.equals(Schema.STRING.isPrimitive, true); - test.done(); - }, - - 'char type'(test: Test) { - test.equals(Schema.char(1).inputString, 'char(1)'); - test.equals(Schema.char(1).isPrimitive, true); - test.done(); - }, - - 'char length must be at least 1'(test: Test) { - test.doesNotThrow(() => Schema.char(1)); - test.throws(() => Schema.char(0)); - test.throws(() => Schema.char(-1)); - test.done(); - }, - - 'char length must be <= 255'(test: Test) { - test.doesNotThrow(() => Schema.char(255)); - test.throws(() => Schema.char(256)); - test.done(); - }, - - 'varchar type'(test: Test) { - test.equals(Schema.varchar(1).inputString, 'varchar(1)'); - test.equals(Schema.varchar(1).isPrimitive, true); - test.done(); - }, - - 'varchar length must be at least 1'(test: Test) { - test.doesNotThrow(() => Schema.varchar(1)); - test.throws(() => Schema.varchar(0)); - test.throws(() => Schema.varchar(-1)); - test.done(); - }, - - 'varchar length must be <= 65535'(test: Test) { - test.doesNotThrow(() => Schema.varchar(65535)); - test.throws(() => Schema.varchar(65536)); - test.done(); - }, - - 'array'(test: Test) { - const type = Schema.array(Schema.STRING); - test.equals(type.inputString, 'array'); - test.equals(type.isPrimitive, false); - test.done(); - }, - - 'array'(test: Test) { - const type = Schema.array(Schema.char(1)); - test.equals(type.inputString, 'array'); - test.equals(type.isPrimitive, false); - test.done(); - }, - - 'array'(test: Test) { - const type = Schema.array( - Schema.array(Schema.STRING)); - test.equals(type.inputString, 'array>'); - test.equals(type.isPrimitive, false); - test.done(); - }, - - 'array'(test: Test) { - const type = Schema.array( - Schema.map(Schema.STRING, Schema.STRING)); - test.equals(type.inputString, 'array>'); - test.equals(type.isPrimitive, false); - test.done(); - }, - - 'array'(test: Test) { - const type = Schema.array( - Schema.struct([{ - name: 'key', - type: Schema.STRING, - }])); - test.equals(type.inputString, 'array>'); - test.equals(type.isPrimitive, false); - test.done(); - }, - - 'map'(test: Test) { - const type = Schema.map( - Schema.STRING, - Schema.STRING, - ); - test.equals(type.inputString, 'map'); - test.equals(type.isPrimitive, false); - test.done(); - }, - - 'map'(test: Test) { - const type = Schema.map( - Schema.INTEGER, - Schema.STRING, - ); - test.equals(type.inputString, 'map'); - test.equals(type.isPrimitive, false); - test.done(); - }, - - 'map'(test: Test) { - const type = Schema.map( - Schema.char(1), - Schema.char(1), - ); - test.equals(type.inputString, 'map'); - test.equals(type.isPrimitive, false); - test.done(); - }, - - 'map'(test: Test) { - const type = Schema.map( - Schema.char(1), - Schema.array(Schema.STRING), - ); - test.equals(type.inputString, 'map>'); - test.equals(type.isPrimitive, false); - test.done(); - }, - - 'map'(test: Test) { - const type = Schema.map( - Schema.char(1), - Schema.map( - Schema.STRING, - Schema.STRING), - ); - test.equals(type.inputString, 'map>'); - test.equals(type.isPrimitive, false); - test.done(); - }, - - 'map'(test: Test) { - const type = Schema.map( - Schema.char(1), - Schema.struct([{ - name: 'key', - type: Schema.STRING, - }]), - ); - test.equals(type.inputString, 'map>'); - test.equals(type.isPrimitive, false); - test.done(); - }, - - 'map throws if keyType is non-primitive'(test: Test) { - test.throws(() => Schema.map( - Schema.array(Schema.STRING), - Schema.STRING, - )); - test.throws(() => Schema.map( - Schema.map(Schema.STRING, Schema.STRING), - Schema.STRING, - )); - test.throws(() => Schema.map( - Schema.struct([{ - name: 'key', - type: Schema.STRING, - }]), - Schema.STRING, - )); - test.done(); - }, - - 'struct type'(test: Test) { - const type = Schema.struct([{ - name: 'primitive', - type: Schema.STRING, - }, { - name: 'with_comment', - type: Schema.STRING, - comment: 'this has a comment', - }, { - name: 'array', - type: Schema.array(Schema.STRING), - }, { - name: 'map', - type: Schema.map(Schema.STRING, Schema.STRING), - }, { - name: 'nested_struct', - type: Schema.struct([{ - name: 'nested', - type: Schema.STRING, - comment: 'nested comment', - }]), - }]); - - test.equals(type.isPrimitive, false); - test.equals( - type.inputString, - // eslint-disable-next-line max-len - 'struct,map:map,nested_struct:struct>'); - test.done(); - }, -}; \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue/test/test.table.ts b/packages/@aws-cdk/aws-glue/test/test.table.ts deleted file mode 100644 index b00466ebdb21d..0000000000000 --- a/packages/@aws-cdk/aws-glue/test/test.table.ts +++ /dev/null @@ -1,1626 +0,0 @@ -import { expect, haveResource, ResourcePart } from '@aws-cdk/assert'; -import * as iam from '@aws-cdk/aws-iam'; -import * as kms from '@aws-cdk/aws-kms'; -import * as s3 from '@aws-cdk/aws-s3'; -import * as cdk from '@aws-cdk/core'; -import { Test } from 'nodeunit'; -import * as glue from '../lib'; - -export = { - 'unpartitioned JSON table'(test: Test) { - const app = new cdk.App(); - const dbStack = new cdk.Stack(app, 'db'); - const database = new glue.Database(dbStack, 'Database', { - databaseName: 'database', - }); - - const tableStack = new cdk.Stack(app, 'table'); - const table = new glue.Table(tableStack, 'Table', { - database, - tableName: 'table', - columns: [{ - name: 'col', - type: glue.Schema.STRING, - }], - dataFormat: glue.DataFormat.JSON, - }); - test.equals(table.encryption, glue.TableEncryption.UNENCRYPTED); - - expect(tableStack).to(haveResource('AWS::S3::Bucket', { - Type: 'AWS::S3::Bucket', - DeletionPolicy: 'Retain', - UpdateReplacePolicy: 'Retain', - }, ResourcePart.CompleteDefinition)); - - expect(tableStack).to(haveResource('AWS::Glue::Table', { - CatalogId: { - Ref: 'AWS::AccountId', - }, - DatabaseName: { - 'Fn::ImportValue': 'db:ExportsOutputRefDatabaseB269D8BB88F4B1C4', - }, - TableInput: { - Name: 'table', - Description: 'table generated by CDK', - Parameters: { - has_encrypted_data: false, - }, - StorageDescriptor: { - Columns: [ - { - Name: 'col', - Type: 'string', - }, - ], - Compressed: false, - InputFormat: 'org.apache.hadoop.mapred.TextInputFormat', - Location: { - 'Fn::Join': [ - '', - [ - 's3://', - { - Ref: 'TableBucketDA42407C', - }, - '/data/', - ], - ], - }, - OutputFormat: 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat', - SerdeInfo: { - SerializationLibrary: 'org.openx.data.jsonserde.JsonSerDe', - }, - StoredAsSubDirectories: false, - }, - TableType: 'EXTERNAL_TABLE', - }, - })); - - test.done(); - }, - - 'partitioned JSON table'(test: Test) { - const app = new cdk.App(); - const dbStack = new cdk.Stack(app, 'db'); - const database = new glue.Database(dbStack, 'Database', { - databaseName: 'database', - }); - - const tableStack = new cdk.Stack(app, 'table'); - const table = new glue.Table(tableStack, 'Table', { - database, - tableName: 'table', - columns: [{ - name: 'col', - type: glue.Schema.STRING, - }], - partitionKeys: [{ - name: 'year', - type: glue.Schema.SMALL_INT, - }], - dataFormat: glue.DataFormat.JSON, - }); - test.equals(table.encryption, glue.TableEncryption.UNENCRYPTED); - test.equals(table.encryptionKey, undefined); - test.equals(table.bucket.encryptionKey, undefined); - - expect(tableStack).to(haveResource('AWS::Glue::Table', { - CatalogId: { - Ref: 'AWS::AccountId', - }, - DatabaseName: { - 'Fn::ImportValue': 'db:ExportsOutputRefDatabaseB269D8BB88F4B1C4', - }, - TableInput: { - Name: 'table', - Description: 'table generated by CDK', - Parameters: { - has_encrypted_data: false, - }, - PartitionKeys: [ - { - Name: 'year', - Type: 'smallint', - }, - ], - StorageDescriptor: { - Columns: [ - { - Name: 'col', - Type: 'string', - }, - ], - Compressed: false, - InputFormat: 'org.apache.hadoop.mapred.TextInputFormat', - Location: { - 'Fn::Join': [ - '', - [ - 's3://', - { - Ref: 'TableBucketDA42407C', - }, - '/data/', - ], - ], - }, - OutputFormat: 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat', - SerdeInfo: { - SerializationLibrary: 'org.openx.data.jsonserde.JsonSerDe', - }, - StoredAsSubDirectories: false, - }, - TableType: 'EXTERNAL_TABLE', - }, - })); - - test.done(); - }, - - 'compressed table'(test: Test) { - const stack = new cdk.Stack(); - const database = new glue.Database(stack, 'Database', { - databaseName: 'database', - }); - - const table = new glue.Table(stack, 'Table', { - database, - tableName: 'table', - columns: [{ - name: 'col', - type: glue.Schema.STRING, - }], - compressed: true, - dataFormat: glue.DataFormat.JSON, - }); - test.equals(table.encryptionKey, undefined); - test.equals(table.bucket.encryptionKey, undefined); - - expect(stack).to(haveResource('AWS::Glue::Table', { - CatalogId: { - Ref: 'AWS::AccountId', - }, - DatabaseName: { - Ref: 'DatabaseB269D8BB', - }, - TableInput: { - Name: 'table', - Description: 'table generated by CDK', - Parameters: { - has_encrypted_data: false, - }, - StorageDescriptor: { - Columns: [ - { - Name: 'col', - Type: 'string', - }, - ], - Compressed: true, - InputFormat: 'org.apache.hadoop.mapred.TextInputFormat', - Location: { - 'Fn::Join': [ - '', - [ - 's3://', - { - Ref: 'TableBucketDA42407C', - }, - '/data/', - ], - ], - }, - OutputFormat: 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat', - SerdeInfo: { - SerializationLibrary: 'org.openx.data.jsonserde.JsonSerDe', - }, - StoredAsSubDirectories: false, - }, - TableType: 'EXTERNAL_TABLE', - }, - })); - - test.done(); - }, - - 'table.node.defaultChild'(test: Test) { - // GIVEN - const stack = new cdk.Stack(); - const database = new glue.Database(stack, 'Database', { - databaseName: 'database', - }); - - // WHEN - const table = new glue.Table(stack, 'Table', { - database, - tableName: 'table', - columns: [{ - name: 'col', - type: glue.Schema.STRING, - }], - compressed: true, - dataFormat: glue.DataFormat.JSON, - }); - - // THEN - test.ok(table.node.defaultChild instanceof glue.CfnTable); - test.done(); - }, - - 'encrypted table': { - 'SSE-S3'(test: Test) { - const stack = new cdk.Stack(); - const database = new glue.Database(stack, 'Database', { - databaseName: 'database', - }); - - const table = new glue.Table(stack, 'Table', { - database, - tableName: 'table', - columns: [{ - name: 'col', - type: glue.Schema.STRING, - }], - encryption: glue.TableEncryption.S3_MANAGED, - dataFormat: glue.DataFormat.JSON, - }); - test.equals(table.encryption, glue.TableEncryption.S3_MANAGED); - test.equals(table.encryptionKey, undefined); - test.equals(table.bucket.encryptionKey, undefined); - - expect(stack).to(haveResource('AWS::Glue::Table', { - CatalogId: { - Ref: 'AWS::AccountId', - }, - DatabaseName: { - Ref: 'DatabaseB269D8BB', - }, - TableInput: { - Name: 'table', - Description: 'table generated by CDK', - Parameters: { - has_encrypted_data: true, - }, - StorageDescriptor: { - Columns: [ - { - Name: 'col', - Type: 'string', - }, - ], - Compressed: false, - InputFormat: 'org.apache.hadoop.mapred.TextInputFormat', - Location: { - 'Fn::Join': [ - '', - [ - 's3://', - { - Ref: 'TableBucketDA42407C', - }, - '/data/', - ], - ], - }, - OutputFormat: 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat', - SerdeInfo: { - SerializationLibrary: 'org.openx.data.jsonserde.JsonSerDe', - }, - StoredAsSubDirectories: false, - }, - TableType: 'EXTERNAL_TABLE', - }, - })); - - expect(stack).to(haveResource('AWS::S3::Bucket', { - BucketEncryption: { - ServerSideEncryptionConfiguration: [ - { - ServerSideEncryptionByDefault: { - SSEAlgorithm: 'AES256', - }, - }, - ], - }, - })); - - test.done(); - }, - - 'SSE-KMS (implicitly created key)'(test: Test) { - const stack = new cdk.Stack(); - const database = new glue.Database(stack, 'Database', { - databaseName: 'database', - }); - - const table = new glue.Table(stack, 'Table', { - database, - tableName: 'table', - columns: [{ - name: 'col', - type: glue.Schema.STRING, - }], - encryption: glue.TableEncryption.KMS, - dataFormat: glue.DataFormat.JSON, - }); - test.equals(table.encryption, glue.TableEncryption.KMS); - test.equals(table.encryptionKey, table.bucket.encryptionKey); - - expect(stack).to(haveResource('AWS::KMS::Key', { - KeyPolicy: { - Statement: [ - { - Action: [ - 'kms:Create*', - 'kms:Describe*', - 'kms:Enable*', - 'kms:List*', - 'kms:Put*', - 'kms:Update*', - 'kms:Revoke*', - 'kms:Disable*', - 'kms:Get*', - 'kms:Delete*', - 'kms:ScheduleKeyDeletion', - 'kms:CancelKeyDeletion', - 'kms:GenerateDataKey', - 'kms:TagResource', - 'kms:UntagResource', - ], - Effect: 'Allow', - Principal: { - AWS: { - 'Fn::Join': [ - '', - [ - 'arn:', - { - Ref: 'AWS::Partition', - }, - ':iam::', - { - Ref: 'AWS::AccountId', - }, - ':root', - ], - ], - }, - }, - Resource: '*', - }, - ], - Version: '2012-10-17', - }, - Description: 'Created by Table/Bucket', - })); - - expect(stack).to(haveResource('AWS::S3::Bucket', { - BucketEncryption: { - ServerSideEncryptionConfiguration: [ - { - ServerSideEncryptionByDefault: { - KMSMasterKeyID: { - 'Fn::GetAtt': [ - 'TableBucketKey3E9F984A', - 'Arn', - ], - }, - SSEAlgorithm: 'aws:kms', - }, - }, - ], - }, - })); - - expect(stack).to(haveResource('AWS::Glue::Table', { - CatalogId: { - Ref: 'AWS::AccountId', - }, - DatabaseName: { - Ref: 'DatabaseB269D8BB', - }, - TableInput: { - Name: 'table', - Description: 'table generated by CDK', - Parameters: { - has_encrypted_data: true, - }, - StorageDescriptor: { - Columns: [ - { - Name: 'col', - Type: 'string', - }, - ], - Compressed: false, - InputFormat: 'org.apache.hadoop.mapred.TextInputFormat', - Location: { - 'Fn::Join': [ - '', - [ - 's3://', - { - Ref: 'TableBucketDA42407C', - }, - '/data/', - ], - ], - }, - OutputFormat: 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat', - SerdeInfo: { - SerializationLibrary: 'org.openx.data.jsonserde.JsonSerDe', - }, - StoredAsSubDirectories: false, - }, - TableType: 'EXTERNAL_TABLE', - }, - })); - - test.done(); - }, - - 'SSE-KMS (explicitly created key)'(test: Test) { - const stack = new cdk.Stack(); - const database = new glue.Database(stack, 'Database', { - databaseName: 'database', - }); - const encryptionKey = new kms.Key(stack, 'MyKey'); - - const table = new glue.Table(stack, 'Table', { - database, - tableName: 'table', - columns: [{ - name: 'col', - type: glue.Schema.STRING, - }], - encryption: glue.TableEncryption.KMS, - encryptionKey, - dataFormat: glue.DataFormat.JSON, - }); - test.equals(table.encryption, glue.TableEncryption.KMS); - test.equals(table.encryptionKey, table.bucket.encryptionKey); - test.notEqual(table.encryptionKey, undefined); - - expect(stack).to(haveResource('AWS::KMS::Key', { - KeyPolicy: { - Statement: [ - { - Action: [ - 'kms:Create*', - 'kms:Describe*', - 'kms:Enable*', - 'kms:List*', - 'kms:Put*', - 'kms:Update*', - 'kms:Revoke*', - 'kms:Disable*', - 'kms:Get*', - 'kms:Delete*', - 'kms:ScheduleKeyDeletion', - 'kms:CancelKeyDeletion', - 'kms:GenerateDataKey', - 'kms:TagResource', - 'kms:UntagResource', - ], - Effect: 'Allow', - Principal: { - AWS: { - 'Fn::Join': [ - '', - [ - 'arn:', - { - Ref: 'AWS::Partition', - }, - ':iam::', - { - Ref: 'AWS::AccountId', - }, - ':root', - ], - ], - }, - }, - Resource: '*', - }, - ], - Version: '2012-10-17', - }, - })); - - expect(stack).to(haveResource('AWS::S3::Bucket', { - BucketEncryption: { - ServerSideEncryptionConfiguration: [ - { - ServerSideEncryptionByDefault: { - KMSMasterKeyID: { - 'Fn::GetAtt': [ - 'MyKey6AB29FA6', - 'Arn', - ], - }, - SSEAlgorithm: 'aws:kms', - }, - }, - ], - }, - })); - - expect(stack).to(haveResource('AWS::Glue::Table', { - CatalogId: { - Ref: 'AWS::AccountId', - }, - DatabaseName: { - Ref: 'DatabaseB269D8BB', - }, - TableInput: { - Description: 'table generated by CDK', - Name: 'table', - Parameters: { - has_encrypted_data: true, - }, - StorageDescriptor: { - Columns: [ - { - Name: 'col', - Type: 'string', - }, - ], - Compressed: false, - InputFormat: 'org.apache.hadoop.mapred.TextInputFormat', - Location: { - 'Fn::Join': [ - '', - [ - 's3://', - { - Ref: 'TableBucketDA42407C', - }, - '/data/', - ], - ], - }, - OutputFormat: 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat', - SerdeInfo: { - SerializationLibrary: 'org.openx.data.jsonserde.JsonSerDe', - }, - StoredAsSubDirectories: false, - }, - TableType: 'EXTERNAL_TABLE', - }, - })); - - test.done(); - }, - - 'SSE-KMS_MANAGED'(test: Test) { - const stack = new cdk.Stack(); - const database = new glue.Database(stack, 'Database', { - databaseName: 'database', - }); - - const table = new glue.Table(stack, 'Table', { - database, - tableName: 'table', - columns: [{ - name: 'col', - type: glue.Schema.STRING, - }], - encryption: glue.TableEncryption.KMS_MANAGED, - dataFormat: glue.DataFormat.JSON, - }); - test.equals(table.encryption, glue.TableEncryption.KMS_MANAGED); - test.equals(table.encryptionKey, undefined); - test.equals(table.bucket.encryptionKey, undefined); - - expect(stack).to(haveResource('AWS::S3::Bucket', { - BucketEncryption: { - ServerSideEncryptionConfiguration: [ - { - ServerSideEncryptionByDefault: { - SSEAlgorithm: 'aws:kms', - }, - }, - ], - }, - })); - - expect(stack).to(haveResource('AWS::Glue::Table', { - CatalogId: { - Ref: 'AWS::AccountId', - }, - DatabaseName: { - Ref: 'DatabaseB269D8BB', - }, - TableInput: { - Name: 'table', - Description: 'table generated by CDK', - Parameters: { - has_encrypted_data: true, - }, - StorageDescriptor: { - Columns: [ - { - Name: 'col', - Type: 'string', - }, - ], - Compressed: false, - InputFormat: 'org.apache.hadoop.mapred.TextInputFormat', - Location: { - 'Fn::Join': [ - '', - [ - 's3://', - { - Ref: 'TableBucketDA42407C', - }, - '/data/', - ], - ], - }, - OutputFormat: 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat', - SerdeInfo: { - SerializationLibrary: 'org.openx.data.jsonserde.JsonSerDe', - }, - StoredAsSubDirectories: false, - }, - TableType: 'EXTERNAL_TABLE', - }, - })); - - test.done(); - }, - - 'CSE-KMS (implicitly created key)'(test: Test) { - const stack = new cdk.Stack(); - const database = new glue.Database(stack, 'Database', { - databaseName: 'database', - }); - - const table = new glue.Table(stack, 'Table', { - database, - tableName: 'table', - columns: [{ - name: 'col', - type: glue.Schema.STRING, - }], - encryption: glue.TableEncryption.CLIENT_SIDE_KMS, - dataFormat: glue.DataFormat.JSON, - }); - test.equals(table.encryption, glue.TableEncryption.CLIENT_SIDE_KMS); - test.notEqual(table.encryptionKey, undefined); - test.equals(table.bucket.encryptionKey, undefined); - - expect(stack).to(haveResource('AWS::KMS::Key', { - KeyPolicy: { - Statement: [ - { - Action: [ - 'kms:Create*', - 'kms:Describe*', - 'kms:Enable*', - 'kms:List*', - 'kms:Put*', - 'kms:Update*', - 'kms:Revoke*', - 'kms:Disable*', - 'kms:Get*', - 'kms:Delete*', - 'kms:ScheduleKeyDeletion', - 'kms:CancelKeyDeletion', - 'kms:GenerateDataKey', - 'kms:TagResource', - 'kms:UntagResource', - ], - Effect: 'Allow', - Principal: { - AWS: { - 'Fn::Join': [ - '', - [ - 'arn:', - { - Ref: 'AWS::Partition', - }, - ':iam::', - { - Ref: 'AWS::AccountId', - }, - ':root', - ], - ], - }, - }, - Resource: '*', - }, - ], - Version: '2012-10-17', - }, - })); - - expect(stack).to(haveResource('AWS::Glue::Table', { - CatalogId: { - Ref: 'AWS::AccountId', - }, - DatabaseName: { - Ref: 'DatabaseB269D8BB', - }, - TableInput: { - Description: 'table generated by CDK', - Name: 'table', - Parameters: { - has_encrypted_data: true, - }, - StorageDescriptor: { - Columns: [ - { - Name: 'col', - Type: 'string', - }, - ], - Compressed: false, - InputFormat: 'org.apache.hadoop.mapred.TextInputFormat', - Location: { - 'Fn::Join': [ - '', - [ - 's3://', - { - Ref: 'TableBucketDA42407C', - }, - '/data/', - ], - ], - }, - OutputFormat: 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat', - SerdeInfo: { - SerializationLibrary: 'org.openx.data.jsonserde.JsonSerDe', - }, - StoredAsSubDirectories: false, - }, - TableType: 'EXTERNAL_TABLE', - }, - })); - - test.done(); - }, - - 'CSE-KMS (explicitly created key)'(test: Test) { - const stack = new cdk.Stack(); - const database = new glue.Database(stack, 'Database', { - databaseName: 'database', - }); - const encryptionKey = new kms.Key(stack, 'MyKey'); - - const table = new glue.Table(stack, 'Table', { - database, - tableName: 'table', - columns: [{ - name: 'col', - type: glue.Schema.STRING, - }], - encryption: glue.TableEncryption.CLIENT_SIDE_KMS, - encryptionKey, - dataFormat: glue.DataFormat.JSON, - }); - test.equals(table.encryption, glue.TableEncryption.CLIENT_SIDE_KMS); - test.notEqual(table.encryptionKey, undefined); - test.equals(table.bucket.encryptionKey, undefined); - - expect(stack).to(haveResource('AWS::KMS::Key', { - KeyPolicy: { - Statement: [ - { - Action: [ - 'kms:Create*', - 'kms:Describe*', - 'kms:Enable*', - 'kms:List*', - 'kms:Put*', - 'kms:Update*', - 'kms:Revoke*', - 'kms:Disable*', - 'kms:Get*', - 'kms:Delete*', - 'kms:ScheduleKeyDeletion', - 'kms:CancelKeyDeletion', - 'kms:GenerateDataKey', - 'kms:TagResource', - 'kms:UntagResource', - ], - Effect: 'Allow', - Principal: { - AWS: { - 'Fn::Join': [ - '', - [ - 'arn:', - { - Ref: 'AWS::Partition', - }, - ':iam::', - { - Ref: 'AWS::AccountId', - }, - ':root', - ], - ], - }, - }, - Resource: '*', - }, - ], - Version: '2012-10-17', - }, - })); - - expect(stack).to(haveResource('AWS::Glue::Table', { - CatalogId: { - Ref: 'AWS::AccountId', - }, - DatabaseName: { - Ref: 'DatabaseB269D8BB', - }, - TableInput: { - Description: 'table generated by CDK', - Name: 'table', - Parameters: { - has_encrypted_data: true, - }, - StorageDescriptor: { - Columns: [ - { - Name: 'col', - Type: 'string', - }, - ], - Compressed: false, - InputFormat: 'org.apache.hadoop.mapred.TextInputFormat', - Location: { - 'Fn::Join': [ - '', - [ - 's3://', - { - Ref: 'TableBucketDA42407C', - }, - '/data/', - ], - ], - }, - OutputFormat: 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat', - SerdeInfo: { - SerializationLibrary: 'org.openx.data.jsonserde.JsonSerDe', - }, - StoredAsSubDirectories: false, - }, - TableType: 'EXTERNAL_TABLE', - }, - })); - - test.done(); - }, - - 'CSE-KMS (explicitly passed bucket and key)'(test: Test) { - const stack = new cdk.Stack(); - const database = new glue.Database(stack, 'Database', { - databaseName: 'database', - }); - const bucket = new s3.Bucket(stack, 'Bucket'); - const encryptionKey = new kms.Key(stack, 'MyKey'); - - const table = new glue.Table(stack, 'Table', { - database, - tableName: 'table', - columns: [{ - name: 'col', - type: glue.Schema.STRING, - }], - bucket, - encryption: glue.TableEncryption.CLIENT_SIDE_KMS, - encryptionKey, - dataFormat: glue.DataFormat.JSON, - }); - test.equals(table.encryption, glue.TableEncryption.CLIENT_SIDE_KMS); - test.notEqual(table.encryptionKey, undefined); - test.equals(table.bucket.encryptionKey, undefined); - - expect(stack).to(haveResource('AWS::KMS::Key', { - KeyPolicy: { - Statement: [ - { - Action: [ - 'kms:Create*', - 'kms:Describe*', - 'kms:Enable*', - 'kms:List*', - 'kms:Put*', - 'kms:Update*', - 'kms:Revoke*', - 'kms:Disable*', - 'kms:Get*', - 'kms:Delete*', - 'kms:ScheduleKeyDeletion', - 'kms:CancelKeyDeletion', - 'kms:GenerateDataKey', - 'kms:TagResource', - 'kms:UntagResource', - ], - Effect: 'Allow', - Principal: { - AWS: { - 'Fn::Join': [ - '', - [ - 'arn:', - { - Ref: 'AWS::Partition', - }, - ':iam::', - { - Ref: 'AWS::AccountId', - }, - ':root', - ], - ], - }, - }, - Resource: '*', - }, - ], - Version: '2012-10-17', - }, - })); - - expect(stack).to(haveResource('AWS::Glue::Table', { - CatalogId: { - Ref: 'AWS::AccountId', - }, - DatabaseName: { - Ref: 'DatabaseB269D8BB', - }, - TableInput: { - Description: 'table generated by CDK', - Name: 'table', - Parameters: { - has_encrypted_data: true, - }, - StorageDescriptor: { - Columns: [ - { - Name: 'col', - Type: 'string', - }, - ], - Compressed: false, - InputFormat: 'org.apache.hadoop.mapred.TextInputFormat', - Location: { - 'Fn::Join': [ - '', - [ - 's3://', - { - Ref: 'Bucket83908E77', - }, - '/data/', - ], - ], - }, - OutputFormat: 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat', - SerdeInfo: { - SerializationLibrary: 'org.openx.data.jsonserde.JsonSerDe', - }, - StoredAsSubDirectories: false, - }, - TableType: 'EXTERNAL_TABLE', - }, - })); - - test.done(); - }, - }, - - 'explicit s3 bucket and prefix'(test: Test) { - const app = new cdk.App(); - const dbStack = new cdk.Stack(app, 'db'); - const stack = new cdk.Stack(app, 'app'); - const bucket = new s3.Bucket(stack, 'ExplicitBucket'); - const database = new glue.Database(dbStack, 'Database', { - databaseName: 'database', - }); - - new glue.Table(stack, 'Table', { - database, - bucket, - s3Prefix: 'prefix/', - tableName: 'table', - columns: [{ - name: 'col', - type: glue.Schema.STRING, - }], - dataFormat: glue.DataFormat.JSON, - }); - - expect(stack).to(haveResource('AWS::Glue::Table', { - CatalogId: { - Ref: 'AWS::AccountId', - }, - DatabaseName: { - 'Fn::ImportValue': 'db:ExportsOutputRefDatabaseB269D8BB88F4B1C4', - }, - TableInput: { - Description: 'table generated by CDK', - Name: 'table', - Parameters: { - has_encrypted_data: false, - }, - StorageDescriptor: { - Columns: [ - { - Name: 'col', - Type: 'string', - }, - ], - Compressed: false, - InputFormat: 'org.apache.hadoop.mapred.TextInputFormat', - Location: { - 'Fn::Join': [ - '', - [ - 's3://', - { - Ref: 'ExplicitBucket0AA51A3F', - }, - '/prefix/', - ], - ], - }, - OutputFormat: 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat', - SerdeInfo: { - SerializationLibrary: 'org.openx.data.jsonserde.JsonSerDe', - }, - StoredAsSubDirectories: false, - }, - TableType: 'EXTERNAL_TABLE', - }, - })); - - test.done(); - }, - - 'explicit s3 bucket and with empty prefix'(test: Test) { - const app = new cdk.App(); - const dbStack = new cdk.Stack(app, 'db'); - const stack = new cdk.Stack(app, 'app'); - const bucket = new s3.Bucket(stack, 'ExplicitBucket'); - const database = new glue.Database(dbStack, 'Database', { - databaseName: 'database', - }); - - new glue.Table(stack, 'Table', { - database, - bucket, - s3Prefix: '', - tableName: 'table', - columns: [{ - name: 'col', - type: glue.Schema.STRING, - }], - dataFormat: glue.DataFormat.JSON, - }); - - expect(stack).to(haveResource('AWS::Glue::Table', { - CatalogId: { - Ref: 'AWS::AccountId', - }, - DatabaseName: { - 'Fn::ImportValue': 'db:ExportsOutputRefDatabaseB269D8BB88F4B1C4', - }, - TableInput: { - Description: 'table generated by CDK', - Name: 'table', - Parameters: { - has_encrypted_data: false, - }, - StorageDescriptor: { - Columns: [ - { - Name: 'col', - Type: 'string', - }, - ], - Compressed: false, - InputFormat: 'org.apache.hadoop.mapred.TextInputFormat', - Location: { - 'Fn::Join': [ - '', - [ - 's3://', - { - Ref: 'ExplicitBucket0AA51A3F', - }, - '/', - ], - ], - }, - OutputFormat: 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat', - SerdeInfo: { - SerializationLibrary: 'org.openx.data.jsonserde.JsonSerDe', - }, - StoredAsSubDirectories: false, - }, - TableType: 'EXTERNAL_TABLE', - }, - })); - - test.done(); - }, - - 'grants': { - 'read only'(test: Test) { - const stack = new cdk.Stack(); - const user = new iam.User(stack, 'User'); - const database = new glue.Database(stack, 'Database', { - databaseName: 'database', - }); - - const table = new glue.Table(stack, 'Table', { - database, - tableName: 'table', - columns: [{ - name: 'col', - type: glue.Schema.STRING, - }], - compressed: true, - dataFormat: glue.DataFormat.JSON, - }); - - table.grantRead(user); - - expect(stack).to(haveResource('AWS::IAM::Policy', { - PolicyDocument: { - Statement: [ - { - Action: [ - 'glue:BatchDeletePartition', - 'glue:BatchGetPartition', - 'glue:GetPartition', - 'glue:GetPartitions', - 'glue:GetTable', - 'glue:GetTables', - 'glue:GetTableVersions', - ], - Effect: 'Allow', - Resource: { - 'Fn::Join': [ - '', - [ - 'arn:', - { - Ref: 'AWS::Partition', - }, - ':glue:', - { - Ref: 'AWS::Region', - }, - ':', - { - Ref: 'AWS::AccountId', - }, - ':table/', - { - Ref: 'DatabaseB269D8BB', - }, - '/', - { - Ref: 'Table4C2D914F', - }, - ], - ], - }, - }, - { - Action: [ - 's3:GetObject*', - 's3:GetBucket*', - 's3:List*', - ], - Effect: 'Allow', - Resource: [ - { - 'Fn::GetAtt': [ - 'TableBucketDA42407C', - 'Arn', - ], - }, - { - 'Fn::Join': [ - '', - [ - { - 'Fn::GetAtt': [ - 'TableBucketDA42407C', - 'Arn', - ], - }, - '/data/', - ], - ], - }, - ], - }, - ], - Version: '2012-10-17', - }, - PolicyName: 'UserDefaultPolicy1F97781E', - Users: [ - { - Ref: 'User00B015A1', - }, - ], - })); - - test.done(); - }, - - 'write only'(test: Test) { - const stack = new cdk.Stack(); - const user = new iam.User(stack, 'User'); - const database = new glue.Database(stack, 'Database', { - databaseName: 'database', - }); - - const table = new glue.Table(stack, 'Table', { - database, - tableName: 'table', - columns: [{ - name: 'col', - type: glue.Schema.STRING, - }], - compressed: true, - dataFormat: glue.DataFormat.JSON, - }); - - table.grantWrite(user); - - expect(stack).to(haveResource('AWS::IAM::Policy', { - PolicyDocument: { - Statement: [ - { - Action: [ - 'glue:BatchCreatePartition', - 'glue:BatchDeletePartition', - 'glue:CreatePartition', - 'glue:DeletePartition', - 'glue:UpdatePartition', - ], - Effect: 'Allow', - Resource: { - 'Fn::Join': [ - '', - [ - 'arn:', - { - Ref: 'AWS::Partition', - }, - ':glue:', - { - Ref: 'AWS::Region', - }, - ':', - { - Ref: 'AWS::AccountId', - }, - ':table/', - { - Ref: 'DatabaseB269D8BB', - }, - '/', - { - Ref: 'Table4C2D914F', - }, - ], - ], - }, - }, - { - Action: [ - 's3:DeleteObject*', - 's3:PutObject*', - 's3:Abort*', - ], - Effect: 'Allow', - Resource: [ - { - 'Fn::GetAtt': [ - 'TableBucketDA42407C', - 'Arn', - ], - }, - { - 'Fn::Join': [ - '', - [ - { - 'Fn::GetAtt': [ - 'TableBucketDA42407C', - 'Arn', - ], - }, - '/data/', - ], - ], - }, - ], - }, - ], - Version: '2012-10-17', - }, - PolicyName: 'UserDefaultPolicy1F97781E', - Users: [ - { - Ref: 'User00B015A1', - }, - ], - })); - - test.done(); - }, - - 'read and write'(test: Test) { - const stack = new cdk.Stack(); - const user = new iam.User(stack, 'User'); - const database = new glue.Database(stack, 'Database', { - databaseName: 'database', - }); - - const table = new glue.Table(stack, 'Table', { - database, - tableName: 'table', - columns: [{ - name: 'col', - type: glue.Schema.STRING, - }], - compressed: true, - dataFormat: glue.DataFormat.JSON, - }); - - table.grantReadWrite(user); - - expect(stack).to(haveResource('AWS::IAM::Policy', { - PolicyDocument: { - Statement: [ - { - Action: [ - 'glue:BatchDeletePartition', - 'glue:BatchGetPartition', - 'glue:GetPartition', - 'glue:GetPartitions', - 'glue:GetTable', - 'glue:GetTables', - 'glue:GetTableVersions', - 'glue:BatchCreatePartition', - 'glue:BatchDeletePartition', - 'glue:CreatePartition', - 'glue:DeletePartition', - 'glue:UpdatePartition', - ], - Effect: 'Allow', - Resource: { - 'Fn::Join': [ - '', - [ - 'arn:', - { - Ref: 'AWS::Partition', - }, - ':glue:', - { - Ref: 'AWS::Region', - }, - ':', - { - Ref: 'AWS::AccountId', - }, - ':table/', - { - Ref: 'DatabaseB269D8BB', - }, - '/', - { - Ref: 'Table4C2D914F', - }, - ], - ], - }, - }, - { - Action: [ - 's3:GetObject*', - 's3:GetBucket*', - 's3:List*', - 's3:DeleteObject*', - 's3:PutObject*', - 's3:Abort*', - ], - Effect: 'Allow', - Resource: [ - { - 'Fn::GetAtt': [ - 'TableBucketDA42407C', - 'Arn', - ], - }, - { - 'Fn::Join': [ - '', - [ - { - 'Fn::GetAtt': [ - 'TableBucketDA42407C', - 'Arn', - ], - }, - '/data/', - ], - ], - }, - ], - }, - ], - Version: '2012-10-17', - }, - PolicyName: 'UserDefaultPolicy1F97781E', - Users: [ - { - Ref: 'User00B015A1', - }, - ], - })); - - test.done(); - }, - }, - - 'validate': { - 'at least one column'(test: Test) { - test.throws(() => { - createTable({ - columns: [], - tableName: 'name', - }); - }, undefined, 'you must specify at least one column for the table'); - - test.done(); - }, - - 'unique column names'(test: Test) { - test.throws(() => { - createTable({ - tableName: 'name', - columns: [{ - name: 'col1', - type: glue.Schema.STRING, - }, { - name: 'col1', - type: glue.Schema.STRING, - }], - }); - }, undefined, "column names and partition keys must be unique, but 'col1' is duplicated."); - - test.done(); - }, - - 'unique partition keys'(test: Test) { - test.throws(() => createTable({ - tableName: 'name', - columns: [{ - name: 'col1', - type: glue.Schema.STRING, - }], - partitionKeys: [{ - name: 'p1', - type: glue.Schema.STRING, - }, { - name: 'p1', - type: glue.Schema.STRING, - }], - }), undefined, "column names and partition keys must be unique, but 'p1' is duplicated"); - - test.done(); - }, - - 'column names and partition keys are all unique'(test: Test) { - test.throws(() => createTable({ - tableName: 'name', - columns: [{ - name: 'col1', - type: glue.Schema.STRING, - }], - partitionKeys: [{ - name: 'col1', - type: glue.Schema.STRING, - }], - }), "column names and partition keys must be unique, but 'col1' is duplicated"); - - test.done(); - }, - - 'can not specify an explicit bucket and encryption'(test: Test) { - test.throws(() => createTable({ - tableName: 'name', - columns: [{ - name: 'col1', - type: glue.Schema.STRING, - }], - bucket: new s3.Bucket(new cdk.Stack(), 'Bucket'), - encryption: glue.TableEncryption.KMS, - }), undefined, 'you can not specify encryption settings if you also provide a bucket'); - test.done(); - }, - - 'can explicitly pass bucket if Encryption undefined'(test: Test) { - test.doesNotThrow(() => createTable({ - tableName: 'name', - columns: [{ - name: 'col1', - type: glue.Schema.STRING, - }], - bucket: new s3.Bucket(new cdk.Stack(), 'Bucket'), - encryption: undefined, - })); - test.done(); - }, - - 'can explicitly pass bucket if Unencrypted'(test: Test) { - test.doesNotThrow(() => createTable({ - tableName: 'name', - columns: [{ - name: 'col1', - type: glue.Schema.STRING, - }], - bucket: new s3.Bucket(new cdk.Stack(), 'Bucket'), - encryption: undefined, - })); - test.done(); - }, - - 'can explicitly pass bucket if ClientSideKms'(test: Test) { - test.doesNotThrow(() => createTable({ - tableName: 'name', - columns: [{ - name: 'col1', - type: glue.Schema.STRING, - }], - bucket: new s3.Bucket(new cdk.Stack(), 'Bucket'), - encryption: glue.TableEncryption.CLIENT_SIDE_KMS, - })); - test.done(); - }, - }, - - 'Table.fromTableArn'(test: Test) { - // GIVEN - const stack = new cdk.Stack(); - - // WHEN - const table = glue.Table.fromTableArn(stack, 'boom', 'arn:aws:glue:us-east-1:123456789012:table/db1/tbl1'); - - // THEN - test.deepEqual(table.tableArn, 'arn:aws:glue:us-east-1:123456789012:table/db1/tbl1'); - test.deepEqual(table.tableName, 'tbl1'); - test.done(); - }, -}; - -function createTable(props: Pick>): void { - const stack = new cdk.Stack(); - new glue.Table(stack, 'table', { - ...props, - database: new glue.Database(stack, 'db', { - databaseName: 'database_name', - }), - dataFormat: glue.DataFormat.JSON, - }); -}