chore(core): update CDK Metadata to report construct-level details (#…

…13423)

See CDK RFC 253 (aws/aws-cdk-rfcs#254) for background and details.

Currently -- if a user has not opted out -- an AWS::CDK::Metadata resource
is added to each generated stack template with details about each loaded module
and version that matches an Amazon-specific allow list.

This modules list is used to:

- Track what library versions customers are using so they can be contacted in
  the event of a severe (security) issue with a library.
- Get business metrics on the adoption of CDK and its libraries.

This modules list is sometimes inaccurate (a module may be loaded into memory
without actually being used) and too braod to support CDK v2.

This feature (mostly) implements the specification proposed in RFC 253 to
include metadata about what constructs are present in each stack, rather than
modules loaded into memory. The allow-list is still used to ensure only CDK/AWS
constructs are reported on.

Implementation notes:
- The format of the Analytics property has changed slightly since the RFC. See
  the service-side code for justification and latest spec.
- How to handle the jsii runtime information was left un-spec'd. I've chosen to
  create a psuedo-Construct to add to the list as the simplest solution.
- `runtime-info.test.ts` leaps through some serious hoops to work equally well
  for both v1 and v2, and to fail somewhat gracefully locally if `tsc` was used
  to compile the module instead of `jsii`. Critques of this approach welcome!
- I removed an annoyance from `resolve-version-lib.js` that produced error
  messages when running unit tests.


----

*By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*

packages/@aws-cdk/core/lib/private/metadata-resource.ts

@@ -1,4 +1,4 @@
-import * as cxapi from '@aws-cdk/cx-api';
+import * as zlib from 'zlib';
 import { RegionInfo } from '@aws-cdk/region-info';
 import { CfnCondition } from '../cfn-condition';
 import { Fn } from '../cfn-fn';
@@ -8,41 +8,12 @@ import { Construct } from '../construct-compat';
 import { Lazy } from '../lazy';
 import { Stack } from '../stack';
 import { Token } from '../token';
-import { collectRuntimeInformation } from './runtime-info';
+import { ConstructInfo, constructInfoFromStack } from './runtime-info';
 
 /**
  * Construct that will render the metadata resource
  */
 export class MetadataResource extends Construct {
-  /**
-   * Clear the modules cache
-   *
-   * The next time the MetadataResource is rendered, it will do a lookup of the
-   * modules from the NodeJS module cache again.
-   *
-   * Used only for unit tests.
-   */
-  public static clearModulesCache() {
-    this._modulesPropertyCache = undefined;
-  }
-
-  /**
-   * Cached version of the _modulesProperty() accessor
-   *
-   * No point in calculating this fairly expensive list more than once.
-   */
-  private static _modulesPropertyCache?: string;
-
-  /**
-   * Calculate the modules property
-   */
-  private static modulesProperty(): string {
-    if (this._modulesPropertyCache === undefined) {
-      this._modulesPropertyCache = formatModules(collectRuntimeInformation());
-    }
-    return this._modulesPropertyCache;
-  }
-
   constructor(scope: Stack, id: string) {
     super(scope, id);
 
@@ -51,7 +22,7 @@ export class MetadataResource extends Construct {
       const resource = new CfnResource(this, 'Default', {
         type: 'AWS::CDK::Metadata',
         properties: {
-          Modules: Lazy.string({ produce: () => MetadataResource.modulesProperty() }),
+          Analytics: Lazy.string({ produce: () => formatAnalytics(constructInfoFromStack(scope)) }),
         },
       });
 
@@ -76,17 +47,81 @@ function makeCdkMetadataAvailableCondition() {
     .map(ri => Fn.conditionEquals(Aws.REGION, ri.name)));
 }
 
-function formatModules(runtime: cxapi.RuntimeInfo): string {
-  const modules = new Array<string>();
+/** Convenience type for arbitrarily-nested map */
+class Trie extends Map<string, Trie> { }
 
-  // inject toolkit version to list of modules
-  const cliVersion = process.env[cxapi.CLI_VERSION_ENV];
-  if (cliVersion) {
-    modules.push(`aws-cdk=${cliVersion}`);
-  }
+/**
+ * Formats a list of construct fully-qualified names (FQNs) and versions into a (possibly compressed) prefix-encoded string.
+ *
+ * The list of ConstructInfos is logically formatted into:
+ * ${version}!${fqn} (e.g., "1.90.0!aws-cdk-lib.Stack")
+ * and then all of the construct-versions are grouped with common prefixes together, grouping common parts in '{}' and separating items with ','.
+ *
+ * Example:
+ * [1.90.0!aws-cdk-lib.Stack, 1.90.0!aws-cdk-lib.Construct, 1.90.0!aws-cdk-lib.service.Resource, 0.42.1!aws-cdk-lib-experiments.NewStuff]
+ * Becomes:
+ * 1.90.0!aws-cdk-lib.{Stack,Construct,service.Resource},0.42.1!aws-cdk-lib-experiments.NewStuff
+ *
+ * The whole thing is then either included directly as plaintext as:
+ * v2:plaintext:{prefixEncodedList}
+ * Or is compressed and base64-encoded, and then formatted as:
+ * v2:deflate64:{prefixEncodedListCompressedAndEncoded}
+ *
+ * Exported/visible for ease of testing.
+ */
+export function formatAnalytics(infos: ConstructInfo[]) {
+  const trie = new Trie();
+  infos.forEach(info => insertFqnInTrie(`${info.version}!${info.fqn}`, trie));
 
-  for (const key of Object.keys(runtime.libraries).sort()) {
-    modules.push(`${key}=${runtime.libraries[key]}`);
+  const plaintextEncodedConstructs = prefixEncodeTrie(trie);
+  const compressedConstructs = zlib.gzipSync(Buffer.from(plaintextEncodedConstructs)).toString('base64');
+
+  return `v2:deflate64:${compressedConstructs}`;
+}
+
+/**
+ * Splits after non-alphanumeric characters (e.g., '.', '/') in the FQN
+ * and insert each piece of the FQN in nested map (i.e., simple trie).
+ */
+function insertFqnInTrie(fqn: string, trie: Trie) {
+  for (const fqnPart of fqn.replace(/[^a-z0-9]/gi, '$& ').split(' ')) {
+    const nextLevelTreeRef = trie.get(fqnPart) ?? new Trie();
+    trie.set(fqnPart, nextLevelTreeRef);
+    trie = nextLevelTreeRef;
   }
-  return modules.join(',');
-}
+  return trie;
+}
+
+/**
+ * Prefix-encodes a "trie-ish" structure, using '{}' to group and ',' to separate siblings.
+ *
+ * Example input:
+ * ABC,ABD,AEF
+ *
+ * Example trie:
+ * A --> B --> C
+ *  |     \--> D
+ *  \--> E --> F
+ *
+ * Becomes:
+ * A{B{C,D},EF}
+ */
+function prefixEncodeTrie(trie: Trie) {
+  let prefixEncoded = '';
+  let isFirstEntryAtLevel = true;
+  [...trie.entries()].forEach(([key, value]) => {
+    if (!isFirstEntryAtLevel) {
+      prefixEncoded += ',';
+    }
+    isFirstEntryAtLevel = false;
+    prefixEncoded += key;
+    if (value.size > 1) {
+      prefixEncoded += '{';
+      prefixEncoded += prefixEncodeTrie(value);
+      prefixEncoded += '}';
+    } else {
+      prefixEncoded += prefixEncodeTrie(value);
+    }
+  });
+  return prefixEncoded;
+}

packages/@aws-cdk/core/lib/private/runtime-info.ts

@@ -1,95 +1,82 @@
-import { basename, dirname } from 'path';
-import * as cxschema from '@aws-cdk/cloud-assembly-schema';
-import { major as nodeMajorVersion } from './node-version';
+import { IConstruct } from '../construct-compat';
+import { Stack } from '../stack';
+import { Stage } from '../stage';
 
-// list of NPM scopes included in version reporting e.g. @aws-cdk and @aws-solutions-konstruk
-const ALLOWED_SCOPES = ['@aws-cdk', '@aws-cdk-containers', '@aws-solutions-konstruk', '@aws-solutions-constructs', '@amzn'];
-// list of NPM packages included in version reporting
-const ALLOWED_PACKAGES = ['aws-rfdk', 'aws-cdk-lib', 'monocdk'];
+const ALLOWED_FQN_PREFIXES = [
+  // SCOPES
+  '@aws-cdk/', '@aws-cdk-containers/', '@aws-solutions-konstruk/', '@aws-solutions-constructs/', '@amzn/',
+  // PACKAGES
+  'aws-rfdk.', 'aws-cdk-lib.', 'monocdk.',
+];
 
 /**
- * Returns a list of loaded modules and their versions.
+ * Symbol for accessing jsii runtime information
+ *
+ * Introduced in jsii 1.19.0, cdk 1.90.0.
  */
-export function collectRuntimeInformation(): cxschema.RuntimeInfo {
-  const libraries: { [name: string]: string } = {};
-
-  for (const fileName of Object.keys(require.cache)) {
-    const pkg = findNpmPackage(fileName);
-    if (pkg && !pkg.private) {
-      libraries[pkg.name] = pkg.version;
-    }
-  }
+const JSII_RUNTIME_SYMBOL = Symbol.for('jsii.rtti');
 
-  // include only libraries that are in the allowlistLibraries list
-  for (const name of Object.keys(libraries)) {
-    let foundMatch = false;
-    for (const scope of ALLOWED_SCOPES) {
-      if (name.startsWith(`${scope}/`)) {
-        foundMatch = true;
-      }
-    }
-    foundMatch = foundMatch || ALLOWED_PACKAGES.includes(name);
+/**
+ * Source information on a construct (class fqn and version)
+ */
+export interface ConstructInfo {
+  readonly fqn: string;
+  readonly version: string;
+}
 
-    if (!foundMatch) {
-      delete libraries[name];
-    }
+export function constructInfoFromConstruct(construct: IConstruct): ConstructInfo | undefined {
+  const jsiiRuntimeInfo = Object.getPrototypeOf(construct).constructor[JSII_RUNTIME_SYMBOL];
+  if (typeof jsiiRuntimeInfo === 'object'
+    && jsiiRuntimeInfo !== null
+    && typeof jsiiRuntimeInfo.fqn === 'string'
+    && typeof jsiiRuntimeInfo.version === 'string') {
+    return { fqn: jsiiRuntimeInfo.fqn, version: jsiiRuntimeInfo.version };
+  } else if (jsiiRuntimeInfo) {
+    // There is something defined, but doesn't match our expectations. Fail fast and hard.
+    throw new Error(`malformed jsii runtime info for construct: '${construct.node.path}'`);
   }
-
-  // add jsii runtime version
-  libraries['jsii-runtime'] = getJsiiAgentVersion();
-
-  return { libraries };
+  return undefined;
 }
 
 /**
- * Determines which NPM module a given loaded javascript file is from.
- *
- * The only infromation that is available locally is a list of Javascript files,
- * and every source file is associated with a search path to resolve the further
- * ``require`` calls made from there, which includes its own directory on disk,
- * and parent directories - for example:
- *
- * [ '...repo/packages/aws-cdk-resources/lib/cfn/node_modules',
- *   '...repo/packages/aws-cdk-resources/lib/node_modules',
- *   '...repo/packages/aws-cdk-resources/node_modules',
- *   '...repo/packages/node_modules',
- *   // etc...
- * ]
- *
- * We are looking for ``package.json`` that is anywhere in the tree, except it's
- * in the parent directory, not in the ``node_modules`` directory. For this
- * reason, we strip the ``/node_modules`` suffix off each path and use regular
- * module resolution to obtain a reference to ``package.json``.
- *
- * @param fileName a javascript file name.
- * @returns the NPM module infos (aka ``package.json`` contents), or
- *      ``undefined`` if the lookup was unsuccessful.
+ * For a given stack, walks the tree and finds the runtime info for all constructs within the tree.
+ * Returns the unique list of construct info present in the stack,
+ * as long as the construct fully-qualified names match the defined allow list.
  */
-function findNpmPackage(fileName: string): { name: string, version: string, private?: boolean } | undefined {
-  const mod = require.cache[fileName];
+export function constructInfoFromStack(stack: Stack): ConstructInfo[] {
+  const isDefined = (value: ConstructInfo | undefined): value is ConstructInfo => value !== undefined;
 
-  if (!mod?.paths) {
-    // sometimes this can be undefined. for example when querying for .json modules
-    // inside a jest runtime environment.
-    // see https://github.com/aws/aws-cdk/issues/7657
-    // potentially we can remove this if it turns out to be a bug in how jest implemented the 'require' module.
-    return undefined;
-  }
+  const allConstructInfos = constructsInStack(stack)
+    .map(construct => constructInfoFromConstruct(construct))
+    .filter(isDefined)
+    .filter(info => ALLOWED_FQN_PREFIXES.find(prefix => info.fqn.startsWith(prefix)));
 
-  // For any path in ``mod.paths`` that is a node_modules folder, use its parent directory instead.
-  const paths = mod?.paths.map((path: string) => basename(path) === 'node_modules' ? dirname(path) : path);
+  // Adds the jsii runtime as a psuedo construct for reporting purposes.
+  allConstructInfos.push({
+    fqn: 'jsii-runtime.Runtime',
+    version: getJsiiAgentVersion(),
+  });
 
-  try {
-    const packagePath = require.resolve(
-      // Resolution behavior changed in node 12.0.0 - https://github.com/nodejs/node/issues/27583
-      nodeMajorVersion >= 12 ? './package.json' : 'package.json',
-      { paths },
-    );
-    // eslint-disable-next-line @typescript-eslint/no-require-imports
-    return require(packagePath);
-  } catch (e) {
-    return undefined;
-  }
+  // Filter out duplicate values
+  const uniqKeys = new Set();
+  return allConstructInfos.filter(construct => {
+    const constructKey = `${construct.fqn}@${construct.version}`;
+    const isDuplicate = uniqKeys.has(constructKey);
+    uniqKeys.add(constructKey);
+    return !isDuplicate;
+  });
+}
+
+/**
+ * Returns all constructs under the parent construct (including the parent),
+ * stopping when it reaches a boundary of another stack (e.g., Stack, Stage, NestedStack).
+ */
+function constructsInStack(construct: IConstruct): IConstruct[] {
+  const constructs = [construct];
+  construct.node.children
+    .filter(child => !Stage.isStage(child) && !Stack.isStack(child))
+    .forEach(child => constructs.push(...constructsInStack(child)));
+  return constructs;
 }
 
 function getJsiiAgentVersion() {

packages/@aws-cdk/core/lib/private/tree-metadata.ts

@@ -6,16 +6,10 @@ import { Annotations } from '../annotations';
 import { Construct, IConstruct, ISynthesisSession } from '../construct-compat';
 import { Stack } from '../stack';
 import { IInspectable, TreeInspector } from '../tree';
+import { ConstructInfo, constructInfoFromConstruct } from './runtime-info';
 
 const FILE_PATH = 'tree.json';
 
-/**
- * Symbol for accessing jsii runtime information
- *
- * Introduced in jsii 1.19.0, cdk 1.90.0.
- */
-const JSII_RUNTIME_SYMBOL = Symbol.for('jsii.rtti');
-
 /**
  * Construct that is automatically attached to the top-level `App`.
  * This generates, as part of synthesis, a file containing the construct tree and the metadata for each node in the tree.
@@ -48,14 +42,12 @@ export class TreeMetadata extends Construct {
         .filter((child) => child !== undefined)
         .reduce((map, child) => Object.assign(map, { [child!.id]: child }), {});
 
-      const jsiiRuntimeInfo = Object.getPrototypeOf(construct).constructor[JSII_RUNTIME_SYMBOL];
-
       const node: Node = {
         id: construct.node.id || 'App',
         path: construct.node.path,
         children: Object.keys(childrenMap).length === 0 ? undefined : childrenMap,
         attributes: this.synthAttributes(construct),
-        constructInfo: constructInfoFromRuntimeInfo(jsiiRuntimeInfo),
+        constructInfo: constructInfoFromConstruct(construct),
       };
 
       lookup[node.path] = node;
@@ -96,16 +88,6 @@ export class TreeMetadata extends Construct {
   }
 }
 
-function constructInfoFromRuntimeInfo(jsiiRuntimeInfo: any): ConstructInfo | undefined {
-  if (typeof jsiiRuntimeInfo === 'object'
-    && jsiiRuntimeInfo !== null
-    && typeof jsiiRuntimeInfo.fqn === 'string'
-    && typeof jsiiRuntimeInfo.version === 'string') {
-    return { fqn: jsiiRuntimeInfo.fqn, version: jsiiRuntimeInfo.version };
-  }
-  return undefined;
-}
-
 interface Node {
   readonly id: string;
   readonly path: string;
@@ -117,11 +99,3 @@ interface Node {
    */
   readonly constructInfo?: ConstructInfo;
 }
-
-/**
- * Source information on a construct (class fqn and version)
- */
-interface ConstructInfo {
-  readonly fqn: string;
-  readonly version: string;
-}