diff --git a/allowed-breaking-changes.txt b/allowed-breaking-changes.txt index 427980fb5be9c..a84098fb3ab06 100644 --- a/allowed-breaking-changes.txt +++ b/allowed-breaking-changes.txt @@ -40,4 +40,4 @@ change-return-type:@aws-cdk/aws-lambda-destinations.EventBridgeDestination.bind change-return-type:@aws-cdk/aws-lambda-destinations.LambdaDestination.bind change-return-type:@aws-cdk/aws-lambda-destinations.SnsDestination.bind change-return-type:@aws-cdk/aws-lambda-destinations.SqsDestination.bind - +removed:@aws-cdk/cdk-assets-schema.DockerImageDestination.imageUri diff --git a/packages/@aws-cdk/cdk-assets-schema/README.md b/packages/@aws-cdk/cdk-assets-schema/README.md index d9cce2f0049b9..80db1a478bfc7 100644 --- a/packages/@aws-cdk/cdk-assets-schema/README.md +++ b/packages/@aws-cdk/cdk-assets-schema/README.md @@ -1,5 +1,4 @@ # cdk-assets-schema - --- diff --git a/packages/@aws-cdk/cdk-assets-schema/lib/aws-destination.ts b/packages/@aws-cdk/cdk-assets-schema/lib/aws-destination.ts index 1f3688f5dec3a..e4b00ed4d308d 100644 --- a/packages/@aws-cdk/cdk-assets-schema/lib/aws-destination.ts +++ b/packages/@aws-cdk/cdk-assets-schema/lib/aws-destination.ts @@ -22,5 +22,24 @@ export interface AwsDestination { * @default - No ExternalId will be supplied */ readonly assumeRoleExternalId?: string; +} +/** + * Placeholders which can be used in the destinations + */ +export class Placeholders { + /** + * Insert this into the destination fields to be replaced with the current region + */ + public static readonly CURRENT_REGION = '${AWS::Region}'; + + /** + * Insert this into the destination fields to be replaced with the current account + */ + public static readonly CURRENT_ACCOUNT = '${AWS::AccountId}'; + + /** + * Insert this into the destination fields to be replaced with the current partition + */ + public static readonly CURRENT_PARTITION = '${AWS::Partition}'; } \ No newline at end of file diff --git a/packages/@aws-cdk/cdk-assets-schema/lib/docker-image-asset.ts b/packages/@aws-cdk/cdk-assets-schema/lib/docker-image-asset.ts index 24bffa056d2dd..dd56653288dc0 100644 --- a/packages/@aws-cdk/cdk-assets-schema/lib/docker-image-asset.ts +++ b/packages/@aws-cdk/cdk-assets-schema/lib/docker-image-asset.ts @@ -61,15 +61,4 @@ export interface DockerImageDestination extends AwsDestination { * Tag of the image to publish */ readonly imageTag: string; - - /** - * Full Docker tag coordinates (registry and repository and tag) - * - * Example: - * - * ``` - * 1234.dkr.ecr.REGION.amazonaws.com/REPO:TAG - * ``` - */ - readonly imageUri: string; } diff --git a/packages/@aws-cdk/cdk-assets-schema/lib/validate.ts b/packages/@aws-cdk/cdk-assets-schema/lib/validate.ts index 221ffc8524216..2e8279534a833 100644 --- a/packages/@aws-cdk/cdk-assets-schema/lib/validate.ts +++ b/packages/@aws-cdk/cdk-assets-schema/lib/validate.ts @@ -97,9 +97,8 @@ function isDockerImageAsset(entry: object): DockerImageAsset { expectKey(destination, 'assumeRoleExternalId', isString, true); expectKey(destination, 'repositoryName', isString); expectKey(destination, 'imageTag', isString); - expectKey(destination, 'imageUri', isString); return destination; })); return entry; -} \ No newline at end of file +} diff --git a/packages/@aws-cdk/cdk-assets-schema/test/validate.test.ts b/packages/@aws-cdk/cdk-assets-schema/test/validate.test.ts index 5beae92ce626b..88df3f9dea82f 100644 --- a/packages/@aws-cdk/cdk-assets-schema/test/validate.test.ts +++ b/packages/@aws-cdk/cdk-assets-schema/test/validate.test.ts @@ -14,7 +14,6 @@ test('Correctly validate Docker image asset', () => { region: 'us-north-20', repositoryName: 'REPO', imageTag: 'TAG', - imageUri: 'URI', }, }, }, @@ -79,4 +78,4 @@ test('Throw on invalid file asset', () => { }, }); }).toThrow(/Expected a string, got '3'/); -}); \ No newline at end of file +}); diff --git a/packages/aws-cdk/bin/cdk.ts b/packages/aws-cdk/bin/cdk.ts index 075a3e60c97ef..672d676b0d455 100644 --- a/packages/aws-cdk/bin/cdk.ts +++ b/packages/aws-cdk/bin/cdk.ts @@ -6,7 +6,8 @@ import * as colors from 'colors/safe'; import * as path from 'path'; import * as yargs from 'yargs'; -import { bootstrapEnvironment, BootstrapEnvironmentProps, SDK } from '../lib'; +import { bootstrapEnvironment, BootstrapEnvironmentProps } from '../lib'; +import { SdkProvider } from '../lib/api/aws-auth'; import { bootstrapEnvironment2 } from '../lib/api/bootstrap/bootstrap-environment2'; import { environmentsFromDescriptors, globEnvironmentsFromStacks } from '../lib/api/cxapp/environments'; import { execProgram } from '../lib/api/cxapp/exec'; @@ -111,11 +112,13 @@ async function initCommandLine() { debug('CDK toolkit version:', version.DISPLAY_VERSION); debug('Command line arguments:', argv); - const aws = new SDK({ + const aws = await SdkProvider.withAwsCliCompatibleDefaults({ profile: argv.profile, - proxyAddress: argv.proxy, - caBundlePath: argv['ca-bundle-path'], ec2creds: argv.ec2creds, + httpOptions: { + proxyAddress: argv.proxy, + caBundlePath: argv['ca-bundle-path'], + } }); const configuration = new Configuration(argv); diff --git a/packages/aws-cdk/lib/api/util/account-cache.ts b/packages/aws-cdk/lib/api/aws-auth/account-cache.ts similarity index 77% rename from packages/aws-cdk/lib/api/util/account-cache.ts rename to packages/aws-cdk/lib/api/aws-auth/account-cache.ts index 9dbc80ec8970f..4f3e7f2c1c414 100644 --- a/packages/aws-cdk/lib/api/util/account-cache.ts +++ b/packages/aws-cdk/lib/api/aws-auth/account-cache.ts @@ -2,6 +2,7 @@ import * as fs from 'fs-extra'; import * as os from 'os'; import * as path from 'path'; import { debug } from '../../logging'; +import { Account } from './sdk-provider'; /** * Disk cache which maps access key IDs to account IDs. @@ -21,7 +22,7 @@ export class AccountAccessKeyCache { * @param filePath Path to the cache file */ constructor(filePath?: string) { - this.cacheFile = filePath || path.join(os.homedir(), '.cdk', 'cache', 'accounts.json'); + this.cacheFile = filePath || path.join(os.homedir(), '.cdk', 'cache', 'accounts_partitions.json'); } /** @@ -38,31 +39,32 @@ export class AccountAccessKeyCache { * @param accessKeyId * @param resolver */ - public async fetch(accessKeyId: string, resolver: () => Promise) { + public async fetch(accessKeyId: string, resolver: () => Promise) { // try to get account ID based on this access key ID from disk. const cached = await this.get(accessKeyId); if (cached) { - debug(`Retrieved account ID ${cached} from disk cache`); + + debug(`Retrieved account ID ${cached.accountId} from disk cache`); return cached; } // if it's not in the cache, resolve and put in cache. - const accountId = await resolver(); - if (accountId) { - await this.put(accessKeyId, accountId); + const account = await resolver(); + if (account) { + await this.put(accessKeyId, account); } - return accountId; + return account; } /** Get the account ID from an access key or undefined if not in cache */ - public async get(accessKeyId: string): Promise { + public async get(accessKeyId: string): Promise { const map = await this.loadMap(); return map[accessKeyId]; } /** Put a mapping betweenn access key and account ID */ - public async put(accessKeyId: string, accountId: string) { + public async put(accessKeyId: string, account: Account) { let map = await this.loadMap(); // nuke cache if it's too big. @@ -70,11 +72,11 @@ export class AccountAccessKeyCache { map = { }; } - map[accessKeyId] = accountId; + map[accessKeyId] = account; await this.saveMap(map); } - private async loadMap(): Promise<{ [accessKeyId: string]: string }> { + private async loadMap(): Promise<{ [accessKeyId: string]: Account }> { if (!(await fs.pathExists(this.cacheFile))) { return { }; } @@ -82,7 +84,7 @@ export class AccountAccessKeyCache { return await fs.readJson(this.cacheFile); } - private async saveMap(map: { [accessKeyId: string]: string }) { + private async saveMap(map: { [accessKeyId: string]: Account }) { if (!(await fs.pathExists(this.cacheFile))) { await fs.mkdirs(path.dirname(this.cacheFile)); } diff --git a/packages/aws-cdk/lib/api/aws-auth/awscli-compatible.ts b/packages/aws-cdk/lib/api/aws-auth/awscli-compatible.ts new file mode 100644 index 0000000000000..2ad921ae5ed5c --- /dev/null +++ b/packages/aws-cdk/lib/api/aws-auth/awscli-compatible.ts @@ -0,0 +1,199 @@ +import * as AWS from 'aws-sdk'; +import * as child_process from 'child_process'; +import * as fs from 'fs-extra'; +import * as os from 'os'; +import * as path from 'path'; +import * as util from 'util'; +import { debug } from '../../logging'; +import { SharedIniFile } from "./sdk_ini_file"; + +/** + * Behaviors to match AWS CLI + * + * See these links: + * + * https://docs.aws.amazon.com/cli/latest/topic/config-vars.html + * https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html + */ +export class AwsCliCompatible { + /** + * Build an AWS CLI-compatible credential chain provider + * + * This is similar to the default credential provider chain created by the SDK + * except: + * + * 1. Accepts profile argument in the constructor (the SDK must have it prepopulated + * in the environment). + * 2. Conditionally checks EC2 credentials, because checking for EC2 + * credentials on a non-EC2 machine may lead to long delays (in the best case) + * or an exception (in the worst case). + * 3. Respects $AWS_SHARED_CREDENTIALS_FILE. + * 4. Respects $AWS_DEFAULT_PROFILE in addition to $AWS_PROFILE. + */ + public static async credentialChain(profile: string | undefined, ec2creds: boolean | undefined, containerCreds: boolean | undefined) { + await forceSdkToReadConfigIfPresent(); + + profile = profile || process.env.AWS_PROFILE || process.env.AWS_DEFAULT_PROFILE || 'default'; + + const sources = [ + () => new AWS.EnvironmentCredentials('AWS'), + () => new AWS.EnvironmentCredentials('AMAZON'), + ]; + + if (await fs.pathExists(credentialsFileName())) { + sources.push(() => new AWS.SharedIniFileCredentials({ profile, filename: credentialsFileName() })); + } + + if (await fs.pathExists(configFileName())) { + sources.push(() => new AWS.SharedIniFileCredentials({ profile, filename: credentialsFileName() })); + } + + if (containerCreds ?? hasEcsCredentials()) { + sources.push(() => new AWS.ECSCredentials()); + } else if (ec2creds ?? await hasEc2Credentials()) { + // else if: don't get EC2 creds if we should have gotten ECS creds--ECS instances also + // run on EC2 boxes but the creds represent something different. Same behavior as + // upstream code. + sources.push(() => new AWS.EC2MetadataCredentials()); + } + + return new AWS.CredentialProviderChain(sources); + } + + /** + * Return the default region in a CLI-compatible way + * + * Mostly copied from node_loader.js, but with the following differences to make it + * AWS CLI compatible: + * + * 1. Takes a profile name as an argument (instead of forcing it to be taken from $AWS_PROFILE). + * This requires having made a copy of the SDK's `SharedIniFile` (the original + * does not take an argument). + * 2. $AWS_DEFAULT_PROFILE and $AWS_DEFAULT_REGION are also respected. + * + * Lambda and CodeBuild set the $AWS_REGION variable. + * + * FIXME: EC2 instances require querying the metadata service to determine the current region. + */ + public static async region(profile: string | undefined): Promise { + profile = profile || process.env.AWS_PROFILE || process.env.AWS_DEFAULT_PROFILE || 'default'; + + // Defaults inside constructor + const toCheck = [ + { filename: credentialsFileName(), profile }, + { isConfig: true, filename: configFileName(), profile }, + { isConfig: true, filename: configFileName(), profile: 'default' }, + ]; + + let region = process.env.AWS_REGION || process.env.AMAZON_REGION || + process.env.AWS_DEFAULT_REGION || process.env.AMAZON_DEFAULT_REGION; + + while (!region && toCheck.length > 0) { + const options = toCheck.shift()!; + if (await fs.pathExists(options.filename)) { + const configFile = new SharedIniFile(options); + const section = await configFile.getProfile(options.profile); + region = section?.region; + } + } + + if (!region) { + const usedProfile = !profile ? '' : ` (profile: "${profile}")`; + region = 'us-east-1'; // This is what the AWS CLI does + debug(`Unable to determine AWS region from environment or AWS configuration${usedProfile}, defaulting to '${region}'`); + } + + return region; + } +} + +/** + * Return whether it looks like we'll have ECS credentials available + */ +function hasEcsCredentials(): boolean { + return (AWS.ECSCredentials.prototype as any).isConfiguredForEcsCredentials(); +} + +/** + * Return whether we're on an EC2 instance + */ +async function hasEc2Credentials() { + debug("Determining whether we're on an EC2 instance."); + + let instance = false; + if (process.platform === 'win32') { + // https://docs.aws.amazon.com/AWSEC2/latest/WindowsGuide/identify_ec2_instances.html + const result = await util.promisify(child_process.exec)('wmic path win32_computersystemproduct get uuid', { encoding: 'utf-8' }); + // output looks like + // UUID + // EC2AE145-D1DC-13B2-94ED-01234ABCDEF + const lines = result.stdout.toString().split('\n'); + instance = lines.some(x => matchesRegex(/^ec2/i, x)); + } else { + // https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/identify_ec2_instances.html + const files: Array<[string, RegExp]> = [ + // This recognizes the Xen hypervisor based instances (pre-5th gen) + ['/sys/hypervisor/uuid', /^ec2/i], + + // This recognizes the new Hypervisor (5th-gen instances and higher) + // Can't use the advertised file '/sys/devices/virtual/dmi/id/product_uuid' because it requires root to read. + // Instead, sys_vendor contains something like 'Amazon EC2'. + ['/sys/devices/virtual/dmi/id/sys_vendor', /ec2/i], + ]; + for (const [file, re] of files) { + if (matchesRegex(re, readIfPossible(file))) { + instance = true; + break; + } + } + } + + debug(instance ? 'Looks like EC2 instance.' : 'Does not look like EC2 instance.'); + return instance; +} + +function homeDir() { + return process.env.HOME || process.env.USERPROFILE + || (process.env.HOMEPATH ? ((process.env.HOMEDRIVE || 'C:/') + process.env.HOMEPATH) : null) || os.homedir(); +} + +function credentialsFileName() { + return process.env.AWS_SHARED_CREDENTIALS_FILE || path.join(homeDir(), '.aws', 'credentials'); +} + +function configFileName() { + return process.env.AWS_CONFIG_FILE || path.join(homeDir(), '.aws', 'config'); +} + +/** + * Force the JS SDK to honor the ~/.aws/config file (and various settings therein) + * + * For example, ther is just *NO* way to do AssumeRole credentials as long as AWS_SDK_LOAD_CONFIG is not set, + * or read credentials from that file. + * + * The SDK crashes if the variable is set but the file does not exist, so conditionally set it. + */ +async function forceSdkToReadConfigIfPresent() { + if (await fs.pathExists(configFileName())) { + process.env.AWS_SDK_LOAD_CONFIG = '1'; + } +} + +function matchesRegex(re: RegExp, s: string | undefined) { + return s !== undefined && re.exec(s) !== null; +} + +/** + * Read a file if it exists, or return undefined + * + * Not async because it is used in the constructor + */ +function readIfPossible(filename: string): string | undefined { + try { + if (!fs.pathExistsSync(filename)) { return undefined; } + return fs.readFileSync(filename, { encoding: 'utf-8' }); + } catch (e) { + debug(e); + return undefined; + } +} \ No newline at end of file diff --git a/packages/aws-cdk/lib/api/aws-auth/credential-plugins.ts b/packages/aws-cdk/lib/api/aws-auth/credential-plugins.ts new file mode 100644 index 0000000000000..8c5fa66f02285 --- /dev/null +++ b/packages/aws-cdk/lib/api/aws-auth/credential-plugins.ts @@ -0,0 +1,54 @@ +import { debug } from "../../logging"; +import { PluginHost } from "../../plugin"; +import { CredentialProviderSource, Mode } from "./credentials"; + +/** + * Cache for credential providers. + * + * Given an account and an operating mode (read or write) will return an + * appropriate credential provider for credentials for the given account. The + * credential provider will be cached so that multiple AWS clients for the same + * environment will not make multiple network calls to obtain credentials. + * + * Will use default credentials if they are for the right account; otherwise, + * all loaded credential provider plugins will be tried to obtain credentials + * for the given account. + */ +export class CredentialPlugins { + private readonly cache: {[key: string]: AWS.Credentials | undefined} = {}; + + public async fetchCredentialsFor(awsAccountId: string, mode: Mode): Promise { + const key = `${awsAccountId}-${mode}`; + if (!(key in this.cache)) { + this.cache[key] = await this.lookupCredentials(awsAccountId, mode); + } + return this.cache[key]; + } + + public get availablePluginNames(): string[] { + return PluginHost.instance.credentialProviderSources.map(s => s.name); + } + + private async lookupCredentials(awsAccountId: string, mode: Mode): Promise { + const triedSources: CredentialProviderSource[] = []; + // Otherwise, inspect the various credential sources we have + for (const source of PluginHost.instance.credentialProviderSources) { + if (!(await source.isAvailable())) { + debug('Credentials source %s is not available, ignoring it.', source.name); + continue; + } + triedSources.push(source); + if (!(await source.canProvideCredentials(awsAccountId))) { continue; } + debug(`Using ${source.name} credentials for account ${awsAccountId}`); + const providerOrCreds = await source.getProvider(awsAccountId, mode); + + // Backwards compatibility: if the plugin returns a ProviderChain, resolve that chain. + // Otherwise it must have returned credentials. + if ((providerOrCreds as any).resolvePromise) { + return await (providerOrCreds as any).resolvePromise(); + } + return providerOrCreds; + } + return undefined; + } +} diff --git a/packages/aws-cdk/lib/api/aws-auth/index.ts b/packages/aws-cdk/lib/api/aws-auth/index.ts new file mode 100644 index 0000000000000..cade9b2eada26 --- /dev/null +++ b/packages/aws-cdk/lib/api/aws-auth/index.ts @@ -0,0 +1,3 @@ +export * from './sdk'; +export * from './sdk-provider'; +export * from './credentials'; \ No newline at end of file diff --git a/packages/aws-cdk/lib/api/aws-auth/sdk-provider.ts b/packages/aws-cdk/lib/api/aws-auth/sdk-provider.ts new file mode 100644 index 0000000000000..b8e92a5f4d48f --- /dev/null +++ b/packages/aws-cdk/lib/api/aws-auth/sdk-provider.ts @@ -0,0 +1,356 @@ +import * as cxapi from '@aws-cdk/cx-api'; +import * as AWS from 'aws-sdk'; +import { ConfigurationOptions } from 'aws-sdk/lib/config'; +import * as fs from 'fs-extra'; +import * as https from 'https'; +import * as os from 'os'; +import * as path from 'path'; +import { debug } from '../../logging'; +import { cached } from '../../util/functions'; +import { CredentialPlugins } from '../aws-auth/credential-plugins'; +import { Mode } from "../aws-auth/credentials"; +import { AccountAccessKeyCache } from './account-cache'; +import { AwsCliCompatible } from './awscli-compatible'; +import { ISDK, SDK } from './sdk'; + +/** + * Options for the default SDK provider + */ +export interface SdkProviderOptions { + /** + * Profile to read from ~/.aws + * + * @default - No profile + */ + readonly profile?: string; + + /** + * Whether we should check for EC2 credentials + * + * @default - Autodetect + */ + readonly ec2creds?: boolean; + + /** + * Whether we should check for container credentials + * + * @default - Autodetect + */ + readonly containerCreds?: boolean; + + /** + * HTTP options for SDK + */ + readonly httpOptions?: SdkHttpOptions; +} + +/** + * Options for individual SDKs + */ +export interface SdkHttpOptions { + /** + * Proxy address to use + * + * @default No proxy + */ + readonly proxyAddress?: string; + + /** + * A path to a certificate bundle that contains a cert to be trusted. + * + * @default No certificate bundle + */ + readonly caBundlePath?: string; + + /** + * The custom user agent to use. + * + * @default - / + */ + readonly userAgent?: string; +} + +const CACHED_ACCOUNT = Symbol(); +const CACHED_DEFAULT_CREDENTIALS = Symbol(); + +/** + * Creates instances of the AWS SDK appropriate for a given account/region + * + * If an environment is given and the current credentials are NOT for the indicated + * account, will also search the set of credential plugin providers. + * + * If no environment is given, the default credentials will always be used. + */ +export class SdkProvider { + /** + * Create a new SdkProvider which gets its defaults in a way that haves like the AWS CLI does + * + * The AWS SDK for JS behaves slightly differently from the AWS CLI in a number of ways; see the + * class `AwsCliCompatible` for the details. + */ + public static async withAwsCliCompatibleDefaults(options: SdkProviderOptions = {}) { + const chain = await AwsCliCompatible.credentialChain(options.profile, options.ec2creds, options.containerCreds); + const region = await AwsCliCompatible.region(options.profile); + + return new SdkProvider(chain, region, options.httpOptions); + } + + private readonly accountCache = new AccountAccessKeyCache(); + private readonly plugins = new CredentialPlugins(); + private readonly httpOptions: ConfigurationOptions; + + public constructor( + private readonly defaultChain: AWS.CredentialProviderChain, + /** + * Default region + */ + public readonly defaultRegion: string, + httpOptions: SdkHttpOptions = {}) { + this.httpOptions = defaultHttpOptions(httpOptions); + } + + /** + * Return an SDK which can do operations in the given environment + * + * The `region` and `accountId` parameters are interpreted as in `resolveEnvironment()` (which is to + * say, `undefined` doesn't do what you expect). + */ + public async forEnvironment(accountId: string | undefined, region: string | undefined, mode: Mode): Promise { + const env = await this.resolveEnvironment(accountId, region); + const creds = await this.obtainCredentials(env.account, mode); + return new SDK(creds, env.region, this.httpOptions); + } + + /** + * Return an SDK which uses assumed role credentials + * + * The base credentials used to retrieve the assumed role credentials will be the + * current credentials (no plugin lookup will be done!). + * + * If `region` is undefined, the default value will be used. + */ + public async withAssumedRole(roleArn: string, externalId: string | undefined, region: string | undefined) { + debug(`Assuming role '${roleArn}'`); + region = region ?? this.defaultRegion; + + const creds = new AWS.ChainableTemporaryCredentials({ + params: { + RoleArn: roleArn, + ...externalId ? { ExternalId: externalId } : {}, + RoleSessionName: `aws-cdk-${os.userInfo().username}`, + }, + stsConfig: { + region, + ...this.httpOptions, + }, + masterCredentials: await this.defaultCredentials(), + }); + + return new SDK(creds, region, this.httpOptions); + } + + /** + * Resolve the environment for a stack + * + * `undefined` actually means `undefined`, and is NOT changed to default values! Only the magic values UNKNOWN_REGION + * and UNKNOWN_ACCOUNT will be replaced with looked-up values! + */ + public async resolveEnvironment(accountId: string | undefined, region: string | undefined) { + region = region !== cxapi.UNKNOWN_REGION ? region : this.defaultRegion; + accountId = accountId !== cxapi.UNKNOWN_ACCOUNT ? accountId : (await this.defaultAccount())?.accountId; + + if (!region) { + throw new Error(`AWS region must be configured either when you configure your CDK stack or through the environment`); + } + + if (!accountId) { + throw new Error(`Unable to resolve AWS account to use. It must be either configured when you define your CDK or through the environment`); + } + + const environment: cxapi.Environment = { + region, account: accountId, name: cxapi.EnvironmentUtils.format(accountId, region) + }; + + return environment; + } + + /** + * Use the default credentials to lookup our account number using STS. + * + * Uses a cache to avoid STS calls if we don't need 'em. + */ + public defaultAccount(): Promise { + return cached(this, CACHED_ACCOUNT, async () => { + try { + const creds = await this.defaultCredentials(); + + const accessKeyId = creds.accessKeyId; + if (!accessKeyId) { + throw new Error('Unable to resolve AWS credentials (setup with "aws configure")'); + } + + const account = await this.accountCache.fetch(creds.accessKeyId, async () => { + // if we don't have one, resolve from STS and store in cache. + debug('Looking up default account ID from STS'); + const result = await new AWS.STS({ ...this.httpOptions, credentials: creds, region: this.defaultRegion }).getCallerIdentity().promise(); + const accountId = result.Account; + const partition = result.Arn!.split(':')[1]; + if (!accountId) { + debug('STS didn\'t return an account ID'); + return undefined; + } + debug('Default account ID:', accountId); + return { accountId, partition }; + }); + + return account; + } catch (e) { + debug('Unable to determine the default AWS account (did you configure "aws configure"?):', e); + return undefined; + } + }); + } + + /** + * Get credentials for the given account ID in the given mode + * + * Use the current credentials if the destination account matches the current credentials' account. + * Otherwise try all credential plugins. + */ + protected async obtainCredentials(accountId: string, mode: Mode): Promise { + // First try 'current' credentials + const defaultAccountId = (await this.defaultAccount())?.accountId; + if (defaultAccountId === accountId) { + return this.defaultCredentials(); + } + + // Then try the plugins + const pluginCreds = await this.plugins.fetchCredentialsFor(accountId, mode); + if (pluginCreds) { + return pluginCreds; + } + + // No luck, format a useful error message + const error = [`Need to perform AWS calls for account ${accountId}`]; + error.push(defaultAccountId ? `but the current credentials are for ${defaultAccountId}` : `but no credentials have been configured`); + if (this.plugins.availablePluginNames.length > 0) { + error.push(`and none of these plugins found any: ${this.plugins.availablePluginNames.join(', ')}`); + } + + throw new Error(`${error.join(', ')}.`); + } + + /** + * Resolve the default chain to the first set of credentials that is available + */ + private defaultCredentials(): Promise { + return cached(this, CACHED_DEFAULT_CREDENTIALS, () => { + debug('Resolving default credentials'); + return this.defaultChain.resolvePromise(); + }); + } +} + +/** + * An AWS account + * + * An AWS account always exists in only one partition. Usually we don't care about + * the partition, but when we need to form ARNs we do. + */ +export interface Account { + /** + * The account number + */ + readonly accountId: string; + + /** + * The partition ('aws' or 'aws-cn' or otherwise) + */ + readonly partition: string; +} + +/** + * Get HTTP options for the SDK + * + * Read from user input or environment variables. + */ +function defaultHttpOptions(options: SdkHttpOptions) { + const config: ConfigurationOptions = {}; + config.httpOptions = {}; + + let userAgent = options.userAgent; + if (userAgent == null) { + // Find the package.json from the main toolkit + const pkg = JSON.parse(readIfPossible(path.join(__dirname, '..', '..', '..', 'package.json')) ?? '{}'); + userAgent = `${pkg.name}/${pkg.version}`; + } + config.customUserAgent = userAgent; + + const proxyAddress = options.proxyAddress || httpsProxyFromEnvironment(); + const caBundlePath = options.caBundlePath || caBundlePathFromEnvironment(); + + if (proxyAddress && caBundlePath) { + throw new Error(`At the moment, cannot specify Proxy (${proxyAddress}) and CA Bundle (${caBundlePath}) at the same time. See https://github.com/aws/aws-cdk/issues/5804`); + // Maybe it's possible after all, but I've been staring at + // https://github.com/TooTallNate/node-proxy-agent/blob/master/index.js#L79 + // a while now trying to figure out what to pass in so that the underlying Agent + // object will get the 'ca' argument. It's not trivial and I don't want to risk it. + } + + if (proxyAddress) { // Ignore empty string on purpose + // https://aws.amazon.com/blogs/developer/using-the-aws-sdk-for-javascript-from-behind-a-proxy/ + debug('Using proxy server: %s', proxyAddress); + // eslint-disable-next-line @typescript-eslint/no-require-imports + const ProxyAgent: any = require('proxy-agent'); + config.httpOptions.agent = new ProxyAgent(proxyAddress); + } + if (caBundlePath) { + debug('Using CA bundle path: %s', caBundlePath); + config.httpOptions.agent = new https.Agent({ + ca: readIfPossible(caBundlePath) + }); + } + + return config; +} + +/** + * Find and return the configured HTTPS proxy address + */ +function httpsProxyFromEnvironment(): string | undefined { + if (process.env.https_proxy) { + return process.env.https_proxy; + } + if (process.env.HTTPS_PROXY) { + return process.env.HTTPS_PROXY; + } + return undefined; +} + +/** + * Find and return a CA certificate bundle path to be passed into the SDK. + */ +function caBundlePathFromEnvironment(): string | undefined { + if (process.env.aws_ca_bundle) { + return process.env.aws_ca_bundle; + } + if (process.env.AWS_CA_BUNDLE) { + return process.env.AWS_CA_BUNDLE; + } + return undefined; +} + +/** + * Read a file if it exists, or return undefined + * + * Not async because it is used in the constructor + */ +function readIfPossible(filename: string): string | undefined { + try { + if (!fs.pathExistsSync(filename)) { return undefined; } + return fs.readFileSync(filename, { encoding: 'utf-8' }); + } catch (e) { + debug(e); + return undefined; + } +} \ No newline at end of file diff --git a/packages/aws-cdk/lib/api/aws-auth/sdk.ts b/packages/aws-cdk/lib/api/aws-auth/sdk.ts new file mode 100644 index 0000000000000..cd178d735acec --- /dev/null +++ b/packages/aws-cdk/lib/api/aws-auth/sdk.ts @@ -0,0 +1,69 @@ +import * as AWS from 'aws-sdk'; +import { ConfigurationOptions } from 'aws-sdk/lib/config'; + +/** @experimental */ +export interface ISDK { + cloudFormation(): AWS.CloudFormation; + + ec2(): AWS.EC2; + + ssm(): AWS.SSM; + + s3(): AWS.S3; + + route53(): AWS.Route53; + + ecr(): AWS.ECR; +} + +/** + * Base functionality of SDK without credential fetching + */ +export class SDK implements ISDK { + private readonly config: ConfigurationOptions; + + /** + * Default retry options for SDK clients + * + * Biggest bottleneck is CloudFormation, with a 1tps call rate. We want to be + * a little more tenacious than the defaults, and with a little more breathing + * room between calls (defaults are {retries=3, base=100}). + * + * I've left this running in a tight loop for an hour and the throttle errors + * haven't escaped the retry mechanism. + */ + private readonly retryOptions = { maxRetries: 6, retryDelayOptions: { base: 300 }}; + + constructor(credentials: AWS.Credentials, region: string, httpOptions: ConfigurationOptions = {}) { + this.config = { + ...httpOptions, + ...this.retryOptions, + credentials, + region, + }; + } + + public cloudFormation(): AWS.CloudFormation { + return new AWS.CloudFormation(this.config); + } + + public ec2(): AWS.EC2 { + return new AWS.EC2(this.config); + } + + public ssm(): AWS.SSM { + return new AWS.SSM(this.config); + } + + public s3(): AWS.S3 { + return new AWS.S3(this.config); + } + + public route53(): AWS.Route53 { + return new AWS.Route53(this.config); + } + + public ecr(): AWS.ECR { + return new AWS.ECR(this.config); + } +} \ No newline at end of file diff --git a/packages/aws-cdk/lib/api/aws-auth/sdk_ini_file.ts b/packages/aws-cdk/lib/api/aws-auth/sdk_ini_file.ts new file mode 100644 index 0000000000000..40845d00b8a15 --- /dev/null +++ b/packages/aws-cdk/lib/api/aws-auth/sdk_ini_file.ts @@ -0,0 +1,59 @@ +/** + * A reimplementation of JS AWS SDK's SharedIniFile class + * + * We need that class to parse the ~/.aws/config file to determine the correct + * region at runtime, but unfortunately it is private upstream. + */ + +import * as AWS from 'aws-sdk'; +import * as fs from 'fs-extra'; +import * as os from 'os'; +import * as path from 'path'; + +export interface SharedIniFileOptions { + isConfig?: boolean; + filename?: string; +} + +export class SharedIniFile { + private readonly isConfig: boolean; + private readonly filename: string; + private parsedContents?: { [key: string]: { [key: string]: string } }; + + constructor(options?: SharedIniFileOptions) { + options = options || {}; + this.isConfig = options.isConfig === true; + this.filename = options.filename || this.getDefaultFilepath(); + } + + public async getProfile(profile: string) { + await this.ensureFileLoaded(); + + const profileIndex = profile !== (AWS as any).util.defaultProfile && this.isConfig ? + 'profile ' + profile : profile; + + return this.parsedContents![profileIndex]; + } + + private getDefaultFilepath(): string { + return path.join( + os.homedir(), + '.aws', + this.isConfig ? 'config' : 'credentials' + ); + } + + private async ensureFileLoaded() { + if (this.parsedContents) { + return; + } + + if (!await fs.pathExists(this.filename)) { + this.parsedContents = {}; + return; + } + + const contents: string = (await fs.readFile(this.filename)).toString(); + this.parsedContents = (AWS as any).util.ini.parse(contents); + } +} diff --git a/packages/aws-cdk/lib/api/bootstrap-environment.ts b/packages/aws-cdk/lib/api/bootstrap-environment.ts index e1272e8e6ddc7..72c25eb140a50 100644 --- a/packages/aws-cdk/lib/api/bootstrap-environment.ts +++ b/packages/aws-cdk/lib/api/bootstrap-environment.ts @@ -2,15 +2,17 @@ import * as cxapi from '@aws-cdk/cx-api'; import * as fs from 'fs-extra'; import * as os from 'os'; import * as path from 'path'; +import { SdkProvider } from './aws-auth'; import {Tag} from "./cxapp/stacks"; import { deployStack, DeployStackResult } from './deploy-stack'; -import { ISDK } from './util/sdk'; // tslint:disable:max-line-length /** @experimental */ export const BUCKET_NAME_OUTPUT = 'BucketName'; /** @experimental */ +export const REPOSITORY_NAME_OUTPUT = 'RepositoryName'; +/** @experimental */ export const BUCKET_DOMAIN_NAME_OUTPUT = 'BucketDomainName'; export interface BootstrapEnvironmentProps { @@ -57,7 +59,7 @@ export interface BootstrapEnvironmentProps { } /** @experimental */ -export async function bootstrapEnvironment(environment: cxapi.Environment, aws: ISDK, toolkitStackName: string, roleArn: string | undefined, props: BootstrapEnvironmentProps = {}): Promise { +export async function bootstrapEnvironment(environment: cxapi.Environment, aws: SdkProvider, toolkitStackName: string, roleArn: string | undefined, props: BootstrapEnvironmentProps = {}): Promise { if (props.trustedAccounts?.length) { throw new Error('--trust can only be passed for the new bootstrap experience!'); } diff --git a/packages/aws-cdk/lib/api/bootstrap/bootstrap-environment2.ts b/packages/aws-cdk/lib/api/bootstrap/bootstrap-environment2.ts index 596450cd16939..8d4b9b5159716 100644 --- a/packages/aws-cdk/lib/api/bootstrap/bootstrap-environment2.ts +++ b/packages/aws-cdk/lib/api/bootstrap/bootstrap-environment2.ts @@ -2,9 +2,10 @@ import * as cxapi from '@aws-cdk/cx-api'; import * as fs from 'fs-extra'; import * as os from 'os'; import * as path from 'path'; -import { BootstrapEnvironmentProps, deployStack, DeployStackResult, ISDK } from '..'; +import { BootstrapEnvironmentProps, deployStack, DeployStackResult } from '..'; +import { SdkProvider } from '../aws-auth'; -export async function bootstrapEnvironment2(environment: cxapi.Environment, sdk: ISDK, +export async function bootstrapEnvironment2(environment: cxapi.Environment, sdk: SdkProvider, toolkitStackName: string, roleArn: string | undefined, props: BootstrapEnvironmentProps = {}): Promise { if (props.trustedAccounts?.length && !props.cloudFormationExecutionPolicies?.length) { diff --git a/packages/aws-cdk/lib/api/cxapp/environments.ts b/packages/aws-cdk/lib/api/cxapp/environments.ts index a4c2aae0376b2..c34c066e010ce 100644 --- a/packages/aws-cdk/lib/api/cxapp/environments.ts +++ b/packages/aws-cdk/lib/api/cxapp/environments.ts @@ -1,9 +1,9 @@ import * as cxapi from '@aws-cdk/cx-api'; import * as minimatch from 'minimatch'; -import { ISDK } from '../util/sdk'; +import { SdkProvider } from '../aws-auth'; import { AppStacks } from './stacks'; -export async function globEnvironmentsFromStacks(appStacks: AppStacks, environmentGlobs: string[], sdk: ISDK): Promise { +export async function globEnvironmentsFromStacks(appStacks: AppStacks, environmentGlobs: string[], sdk: SdkProvider): Promise { if (environmentGlobs.length === 0) { environmentGlobs = [ '**' ]; // default to ALL } @@ -12,7 +12,7 @@ export async function globEnvironmentsFromStacks(appStacks: AppStacks, environme const availableEnvironments = new Array(); for (const stack of stacks) { - const actual = await parseEnvironment(sdk, stack.environment); + const actual = await sdk.resolveEnvironment(stack.environment.account, stack.environment.region); availableEnvironments.push(actual); } @@ -26,20 +26,6 @@ export async function globEnvironmentsFromStacks(appStacks: AppStacks, environme return environments; } -async function parseEnvironment(sdk: ISDK, env: cxapi.Environment): Promise { - const account = env.account === cxapi.UNKNOWN_ACCOUNT ? await sdk.defaultAccount() : env.account; - const region = env.region === cxapi.UNKNOWN_REGION ? await sdk.defaultRegion() : env.region; - - if (!account || !region) { - throw new Error(`Unable to determine default account and/or region`); - } - - return { - account, region, - name: cxapi.EnvironmentUtils.format(account, region) - }; -} - /** * Given a set of "/" strings, construct environments for them */ diff --git a/packages/aws-cdk/lib/api/cxapp/exec.ts b/packages/aws-cdk/lib/api/cxapp/exec.ts index 8b9432140aba9..4de8c73509c01 100644 --- a/packages/aws-cdk/lib/api/cxapp/exec.ts +++ b/packages/aws-cdk/lib/api/cxapp/exec.ts @@ -5,10 +5,10 @@ import * as path from 'path'; import { debug } from '../../logging'; import { Configuration, PROJECT_CONFIG, USER_DEFAULTS } from '../../settings'; import { versionNumber } from '../../version'; -import { ISDK } from '../util/sdk'; +import { SdkProvider } from '../aws-auth'; /** Invokes the cloud executable and returns JSON output */ -export async function execProgram(aws: ISDK, config: Configuration): Promise { +export async function execProgram(aws: SdkProvider, config: Configuration): Promise { const env: { [key: string]: string } = { }; const context = config.context.all; @@ -131,12 +131,15 @@ export async function execProgram(aws: ISDK, config: Configuration): Promise Promise; +type Synthesizer = (aws: SdkProvider, config: Configuration) => Promise; export interface AppStacksProps { /** @@ -43,7 +43,7 @@ export interface AppStacksProps { /** * AWS object (used by synthesizer and contextprovider) */ - aws: ISDK; + aws: SdkProvider; /** * Callback invoked to synthesize the actual stacks diff --git a/packages/aws-cdk/lib/api/deploy-stack.ts b/packages/aws-cdk/lib/api/deploy-stack.ts index 0a51851080483..51af071a5948a 100644 --- a/packages/aws-cdk/lib/api/deploy-stack.ts +++ b/packages/aws-cdk/lib/api/deploy-stack.ts @@ -3,15 +3,18 @@ import * as aws from 'aws-sdk'; import * as colors from 'colors/safe'; import * as uuid from 'uuid'; import { Tag } from "../api/cxapp/stacks"; -import { prepareAssets } from '../assets'; +import { addMetadataAssetsToManifest } from '../assets'; import { debug, error, print } from '../logging'; import { deserializeStructure, toYAML } from '../serialize'; +import { AssetManifestBuilder } from '../util/asset-manifest-builder'; +import { publishAssets } from '../util/asset-publishing'; +import { contentHash } from '../util/content-hash'; +import { SdkProvider } from './aws-auth'; import { Mode } from './aws-auth/credentials'; import { ToolkitInfo } from './toolkit-info'; import { changeSetHasNoChanges, describeStack, stackExists, stackFailedCreating, waitForChangeSet, waitForStack } from './util/cloudformation'; import { StackActivityMonitor } from './util/cloudformation/stack-activity-monitor'; import { StackStatus } from './util/cloudformation/stack-status'; -import { ISDK } from './util/sdk'; type TemplateBodyParameter = { TemplateBody?: string @@ -29,7 +32,7 @@ export interface DeployStackResult { /** @experimental */ export interface DeployStackOptions { stack: cxapi.CloudFormationStackArtifact; - sdk: ISDK; + sdk: SdkProvider; toolkitInfo?: ToolkitInfo; roleArn?: string; notificationArns?: string[]; @@ -66,34 +69,40 @@ const LARGE_TEMPLATE_SIZE_KB = 50; /** @experimental */ export async function deployStack(options: DeployStackOptions): Promise { - if (!options.stack.environment) { - throw new Error(`The stack ${options.stack.displayName} does not have an environment`); + const stack = options.stack; + + if (!stack.environment) { + throw new Error(`The stack ${stack.displayName} does not have an environment`); } - const cfn = await options.sdk.cloudFormation(options.stack.environment.account, options.stack.environment.region, Mode.ForWriting); - const deployName = options.deployName || options.stack.stackName; + // Translate symbolic/unknown environment references to concrete environment references + const stackEnv = await options.sdk.resolveEnvironment(stack.environment.account, stack.environment.region); + + const cfn = (await options.sdk.forEnvironment(stackEnv.account, stackEnv.region, Mode.ForWriting)).cloudFormation(); + const deployName = options.deployName || stack.stackName; if (!options.force) { + // bail out if the current template is exactly the same as the one we are about to deploy + // in cdk-land, this means nothing changed because assets (and therefore nested stacks) are immutable. debug(`checking if we can skip this stack based on the currently deployed template and tags (use --force to override)`); const deployed = await getDeployedStack(cfn, deployName); const tagsIdentical = compareTags(deployed?.tags ?? [], options.tags ?? []); - if (deployed && JSON.stringify(options.stack.template) === JSON.stringify(deployed.template) && tagsIdentical) { + if (deployed && JSON.stringify(stack.template) === JSON.stringify(deployed.template) && tagsIdentical) { debug(`${deployName}: no change in template and tags, skipping (use --force to override)`); return { noOp: true, outputs: await getStackOutputs(cfn, deployName), stackArn: deployed.stackId, - stackArtifact: options.stack + stackArtifact: stack }; } else { debug(`${deployName}: template changed, deploying...`); } } - // bail out if the current template is exactly the same as the one we are about to deploy - // in cdk-land, this means nothing changed because assets (and therefore nested stacks) are immutable. + const assets = new AssetManifestBuilder(); - const params = await prepareAssets(options.stack, options.toolkitInfo, options.reuseAssets); + const params = await addMetadataAssetsToManifest(stack, assets, options.toolkitInfo, options.reuseAssets); // add passed CloudFormation parameters for (const [paramName, paramValue] of Object.entries((options.parameters || {}))) { @@ -107,7 +116,7 @@ export async function deployStack(options: DeployStackOptions): Promise { +async function makeBodyParameter( + stack: cxapi.CloudFormationStackArtifact, + assetManifest: AssetManifestBuilder, + toolkitInfo?: ToolkitInfo): Promise { const templateJson = toYAML(stack.template); - if (toolkitInfo) { - const s3KeyPrefix = `cdk/${stack.id}/`; - const s3KeySuffix = '.yml'; - const { key } = await toolkitInfo.uploadIfChanged(templateJson, { - s3KeyPrefix, s3KeySuffix, contentType: 'application/x-yaml' - }); - const templateURL = `${toolkitInfo.bucketUrl}/${key}`; - debug('Stored template in S3 at:', templateURL); - return { TemplateURL: templateURL }; - } else if (templateJson.length > LARGE_TEMPLATE_SIZE_KB * 1024) { + + if (templateJson.length <= LARGE_TEMPLATE_SIZE_KB * 1024) { + return { TemplateBody: templateJson }; + } + + if (!toolkitInfo) { error( `The template for stack "${stack.displayName}" is ${Math.round(templateJson.length / 1024)}KiB. ` + `Templates larger than ${LARGE_TEMPLATE_SIZE_KB}KiB must be uploaded to S3.\n` + @@ -204,15 +218,27 @@ async function makeBodyParameter(stack: cxapi.CloudFormationStackArtifact, toolk colors.blue(`\t$ cdk bootstrap ${stack.environment!.name}\n`)); throw new Error(`Template too large to deploy ("cdk bootstrap" is required)`); - } else { - return { TemplateBody: templateJson }; } + + const templateHash = contentHash(templateJson); + const key = `cdk/${stack.id}/${templateHash}.yml`; + const templateURL = `${toolkitInfo.bucketUrl}/${key}`; + + assetManifest.addFileAsset(templateHash, { + path: stack.templateFile, + }, { + bucketName: toolkitInfo.bucketName, + objectKey: key, + }); + + debug('Storing template in S3 at:', templateURL); + return { TemplateURL: templateURL }; } /** @experimental */ export interface DestroyStackOptions { stack: cxapi.CloudFormationStackArtifact; - sdk: ISDK; + sdk: SdkProvider; roleArn?: string; deployName?: string; quiet?: boolean; @@ -225,7 +251,8 @@ export async function destroyStack(options: DestroyStackOptions) { } const deployName = options.deployName || options.stack.stackName; - const cfn = await options.sdk.cloudFormation(options.stack.environment.account, options.stack.environment.region, Mode.ForWriting); + const { account, region } = options.stack.environment; + const cfn = (await options.sdk.forEnvironment(account, region, Mode.ForWriting)).cloudFormation(); if (!await stackExists(cfn, deployName)) { return; } @@ -305,4 +332,4 @@ function compareTags(a: Tag[], b: Tag[]): boolean { } return true; -} +} \ No newline at end of file diff --git a/packages/aws-cdk/lib/api/deployment-target.ts b/packages/aws-cdk/lib/api/deployment-target.ts index bf72c7f40b285..46c74ab61c3bc 100644 --- a/packages/aws-cdk/lib/api/deployment-target.ts +++ b/packages/aws-cdk/lib/api/deployment-target.ts @@ -1,10 +1,9 @@ import { CloudFormationStackArtifact } from '@aws-cdk/cx-api'; import { Tag } from "../api/cxapp/stacks"; import { debug } from '../logging'; -import { Mode } from './aws-auth/credentials'; +import { Mode, SdkProvider } from './aws-auth'; import { deployStack, DeployStackResult, readCurrentTemplate } from './deploy-stack'; import { loadToolkitInfo } from './toolkit-info'; -import { ISDK } from './util/sdk'; export const DEFAULT_TOOLKIT_STACK_NAME = 'CDKToolkit'; @@ -45,14 +44,14 @@ export interface DeployStackOptions { } export interface ProvisionerProps { - aws: ISDK; + aws: SdkProvider; } /** * Default provisioner (applies to CloudFormation). */ export class CloudFormationDeploymentTarget implements IDeploymentTarget { - private readonly aws: ISDK; + private readonly aws: SdkProvider; constructor(props: ProvisionerProps) { this.aws = props.aws; @@ -60,7 +59,7 @@ export class CloudFormationDeploymentTarget implements IDeploymentTarget { public async readCurrentTemplate(stack: CloudFormationStackArtifact): Promise