feat(aws-s3): create default bucket policy when required (under feature flag)

packages/@aws-cdk/aws-ec2/README.md

@@ -1356,6 +1356,20 @@ new ec2.FlowLog(this, 'FlowLogWithKeyPrefix', {
 });
 ```
 
+When the S3 destination is configured, AWS will automatically create an S3 bucket policy
+that allows the service to write logs to the bucket. This makes it impossible to later update
+that bucket policy. To have CDK create the bucket policy so that future updates can be made,
+the `@aws-cdk/aws-s3:createDefaultLoggingPolicy` [feature flag](https://docs.aws.amazon.com/cdk/v2/guide/featureflags.html) can be used. This can be set
+in the `cdk.json` file.
+
+```json
+{
+  "context": {
+    "@aws-cdk/aws-s3:createDefaultLoggingPolicy": true
+  }
+}
+```
+
 ## User Data
 
 User data enables you to run a script when your instances start up.  In order to configure these scripts you can add commands directly to the script

packages/@aws-cdk/aws-ec2/lib/vpc-flow-logs.ts

@@ -1,15 +1,14 @@
 import * as iam from '@aws-cdk/aws-iam';
 import * as logs from '@aws-cdk/aws-logs';
 import * as s3 from '@aws-cdk/aws-s3';
-import { IResource, PhysicalName, RemovalPolicy, Resource } from '@aws-cdk/core';
+import { IResource, PhysicalName, RemovalPolicy, Resource, FeatureFlags, Stack } from '@aws-cdk/core';
+import { S3_CREATE_DEFAULT_LOGGING_POLICY } from '@aws-cdk/cx-api';
 import { Construct } from 'constructs';
 import { CfnFlowLog } from './ec2.generated';
 import { ISubnet, IVpc } from './vpc';
 
 /**
  * A FlowLog
- *
- *
  */
 export interface IFlowLog extends IResource {
   /**
@@ -22,8 +21,6 @@ export interface IFlowLog extends IResource {
 
 /**
  * The type of VPC traffic to log
- *
- *
  */
 export enum FlowLogTrafficType {
   /**
@@ -44,7 +41,6 @@ export enum FlowLogTrafficType {
 
 /**
  * The available destination types for Flow Logs
- *
  */
 export enum FlowLogDestinationType {
   /**
@@ -60,8 +56,6 @@ export enum FlowLogDestinationType {
 
 /**
  * The type of resource to create the flow log for
- *
- *
  */
 export abstract class FlowLogResourceType {
   /**
@@ -106,9 +100,29 @@ export abstract class FlowLogResourceType {
 }
 
 /**
- * The destination type for the flow log
- *
+ * Options for writing logs to a S3 destination
+ */
+export interface S3DestinationOptions {
+  /**
+   * Use Hive-compatible prefixes for flow logs
+   * stored in Amazon S3
+   *
+   * @default false
+   */
+  readonly hiveCompatiblePartitions?: boolean;
+}
+
+/**
+ * Options for writing logs to a destination
  *
+ * TODO: there are other destination options, currently they are
+ * only for s3 destinations (not sure if that will change)
+ */
+export interface DestinationOptions extends S3DestinationOptions { }
+
+
+/**
+ * The destination type for the flow log
  */
 export abstract class FlowLogDestination {
   /**
@@ -124,12 +138,18 @@ export abstract class FlowLogDestination {
 
   /**
    * Use S3 as the destination
+   *
+   * @param bucket optional s3 bucket to publish logs to. If one is not provided
+   * a default bucket will be created
+   * @param keyPrefix optional prefix within the bucket to write logs to
+   * @param options additional s3 destination options
    */
-  public static toS3(bucket?: s3.IBucket, keyPrefix?: string): FlowLogDestination {
+  public static toS3(bucket?: s3.IBucket, keyPrefix?: string, options?: S3DestinationOptions): FlowLogDestination {
     return new S3Destination({
       logDestinationType: FlowLogDestinationType.S3,
       s3Bucket: bucket,
       keyPrefix,
+      destinationOptions: options,
     });
   }
 
@@ -141,8 +161,6 @@ export abstract class FlowLogDestination {
 
 /**
  * Flow Log Destination configuration
- *
- *
  */
 export interface FlowLogDestinationConfig {
   /**
@@ -179,6 +197,13 @@ export interface FlowLogDestinationConfig {
    * @default - undefined
    */
   readonly keyPrefix?: string;
+
+  /**
+   * Options for writing flow logs to a supported destination
+   *
+   * @default - undefined
+   */
+  readonly destinationOptions?: DestinationOptions;
 }
 
 /**
@@ -196,13 +221,73 @@ class S3Destination extends FlowLogDestination {
         encryption: s3.BucketEncryption.UNENCRYPTED,
         removalPolicy: RemovalPolicy.RETAIN,
       });
+
     } else {
       s3Bucket = this.props.s3Bucket;
     }
+
+    // https://docs.aws.amazon.com/vpc/latest/userguide/flow-logs-s3.html#flow-logs-s3-permissions
+    if (FeatureFlags.of(scope).isEnabled(S3_CREATE_DEFAULT_LOGGING_POLICY)) {
+      const stack = Stack.of(scope);
+      let keyPrefix = this.props.keyPrefix ?? '';
+      if (keyPrefix && !keyPrefix.endsWith('/')) {
+        keyPrefix = keyPrefix + '/';
+      }
+      const prefix = this.props.destinationOptions?.hiveCompatiblePartitions
+        ? s3Bucket.arnForObjects(`${keyPrefix}AWSLogs/aws-account-id=${stack.account}/*`)
+        : s3Bucket.arnForObjects(`${keyPrefix}AWSLogs/${stack.account}/*`);
+
+      s3Bucket.addToResourcePolicy(new iam.PolicyStatement({
+        effect: iam.Effect.ALLOW,
+        principals: [
+          new iam.ServicePrincipal('delivery.logs.amazonaws.com'),
+        ],
+        resources: [
+          prefix,
+        ],
+        actions: ['s3:PutObject'],
+        conditions: {
+          StringEquals: {
+            's3:x-amz-acl': 'bucket-owner-full-control',
+            'aws:SourceAccount': stack.account,
+          },
+          ArnLike: {
+            'aws:SourceArn': stack.formatArn({
+              service: 'logs',
+              resource: '*',
+            }),
+          },
+        },
+      }));
+
+      s3Bucket.addToResourcePolicy(new iam.PolicyStatement({
+        effect: iam.Effect.ALLOW,
+        principals: [
+          new iam.ServicePrincipal('delivery.logs.amazonaws.com'),
+        ],
+        resources: [s3Bucket.bucketArn],
+        actions: [
+          's3:GetBucketAcl',
+          's3:ListBucket',
+        ],
+        conditions: {
+          StringEquals: {
+            'aws:SourceAccount': stack.account,
+          },
+          ArnLike: {
+            'aws:SourceArn': stack.formatArn({
+              service: 'logs',
+              resource: '*',
+            }),
+          },
+        },
+      }));
+    }
     return {
       logDestinationType: FlowLogDestinationType.S3,
       s3Bucket,
       keyPrefix: this.props.keyPrefix,
+      destinationOptions: this.props.destinationOptions,
     };
   }
 }
@@ -263,8 +348,6 @@ class CloudWatchLogsDestination extends FlowLogDestination {
 
 /**
  * Options to add a flow log to a VPC
- *
- *
  */
 export interface FlowLogOptions {
   /**
@@ -285,8 +368,6 @@ export interface FlowLogOptions {
 
 /**
  * Properties of a VPC Flow Log
- *
- *
  */
 export interface FlowLogProps extends FlowLogOptions {
   /**
@@ -307,8 +388,6 @@ export interface FlowLogProps extends FlowLogOptions {
 
 /**
  * The base class for a Flow Log
- *
- *
  */
 abstract class FlowLogBase extends Resource implements IFlowLog {
   /**
@@ -322,8 +401,6 @@ abstract class FlowLogBase extends Resource implements IFlowLog {
 /**
  * A VPC flow log.
  * @resource AWS::EC2::FlowLog
- *
- *
  */
 export class FlowLog extends FlowLogBase {
   /**
@@ -383,6 +460,7 @@ export class FlowLog extends FlowLogBase {
     }
 
     const flowLog = new CfnFlowLog(this, 'FlowLog', {
+      destinationOptions: destinationConfig.destinationOptions,
       deliverLogsPermissionArn: this.iamRole ? this.iamRole.roleArn : undefined,
       logDestinationType: destinationConfig.logDestinationType,
       logGroupName: this.logGroup ? this.logGroup.logGroupName : undefined,

packages/@aws-cdk/aws-ec2/package.json

@@ -83,6 +83,7 @@
     "@aws-cdk/assertions": "0.0.0",
     "@aws-cdk/cdk-build-tools": "0.0.0",
     "@aws-cdk/integ-runner": "0.0.0",
+    "@aws-cdk/integ-tests": "0.0.0",
     "@aws-cdk/cfn2ts": "0.0.0",
     "@aws-cdk/cloud-assembly-schema": "0.0.0",
     "@aws-cdk/cx-api": "0.0.0",

packages/@aws-cdk/aws-ec2/test/integ.vpc-flow-logs.ts

@@ -1,11 +1,35 @@
-/// !cdk-integ *
 import { PolicyStatement, Effect, ServicePrincipal } from '@aws-cdk/aws-iam';
 import * as s3 from '@aws-cdk/aws-s3';
 import { App, RemovalPolicy, Stack, StackProps } from '@aws-cdk/core';
-import { FlowLog, FlowLogDestination, FlowLogResourceType, Vpc } from '../lib';
+import { IntegTest, ExpectedResult, AssertionsProvider } from '@aws-cdk/integ-tests';
+import { FlowLog, FlowLogDestination, FlowLogResourceType, Vpc, Instance, InstanceType, InstanceClass, InstanceSize, MachineImage, AmazonLinuxGeneration } from '../lib';
 
 const app = new App();
 
+class FeatureFlagStack extends Stack {
+  public readonly bucketArn: string;
+  public readonly bucket: s3.IBucket;
+  constructor(scope: App, id: string, props?: StackProps) {
+    super(scope, id, props);
+
+    const vpc = new Vpc(this, 'VPC');
+
+    const flowLog = vpc.addFlowLog('FlowLogsS3', {
+      destination: FlowLogDestination.toS3(),
+    });
+    this.bucket = flowLog.bucket!;
+    this.bucketArn = this.exportValue(flowLog.bucket!.bucketArn);
+
+    new Instance(this, 'FlowLogsInstance', {
+      vpc,
+      instanceType: InstanceType.of(InstanceClass.T3, InstanceSize.SMALL),
+      machineImage: MachineImage.latestAmazonLinux({
+        generation: AmazonLinuxGeneration.AMAZON_LINUX_2,
+      }),
+    });
+  }
+}
+
 class TestStack extends Stack {
   constructor(scope: App, id: string, props?: StackProps) {
     super(scope, id, props);
@@ -66,6 +90,27 @@ class TestStack extends Stack {
   }
 }
 
-new TestStack(app, 'FlowLogsTestStack');
+const featureFlagTest = new FeatureFlagStack(app, 'FlowLogsFeatureFlag');
+
+const integ = new IntegTest(app, 'FlowLogs', {
+  testCases: [
+    new TestStack(app, 'FlowLogsTestStack'),
+    featureFlagTest,
+  ],
+});
+
+
+const objects = integ.assertions.awsApiCall('S3', 'listObjectsV2', {
+  Bucket: featureFlagTest.bucket.bucketName,
+  MaxKeys: 1,
+  Prefix: `AWSLogs/${featureFlagTest.account}/vpcflowlogs`,
+});
+const assertionProvider = objects.node.tryFindChild('SdkProvider') as AssertionsProvider;
+assertionProvider.addPolicyStatementFromSdkCall('s3', 'ListBucket', [featureFlagTest.bucketArn]);
+assertionProvider.addPolicyStatementFromSdkCall('s3', 'GetObject', [`${featureFlagTest.bucketArn}/*`]);
+
+objects.expect(ExpectedResult.objectLike({
+  KeyCount: 1,
+}));
 
 app.synth();