Merge pull request #1654 from snyk/feat/iac-experimental-terraform-su…

…pport feat: iac experimental terraform support
snyk · Feb 28, 2021 · a988600 · a988600
2 parents 7dfd3ea + b455497
commit a988600
Show file tree

Hide file tree

Showing 14 changed files with 242 additions and 73 deletions.
diff --git a/package.json b/package.json
@@ -77,6 +77,7 @@
     "configstore": "^5.0.1",
     "debug": "^4.1.1",
     "diff": "^4.0.1",
+    "hcl-to-json": "^0.1.1",
     "lodash.assign": "^4.2.0",
     "lodash.camelcase": "^4.3.0",
     "lodash.clonedeep": "^4.5.0",

diff --git a/src/cli/commands/test/iac-local-execution/index.ts b/src/cli/commands/test/iac-local-execution/index.ts
@@ -1,21 +1,27 @@
 import * as fs from 'fs';
-import * as YAML from 'js-yaml';
 import { isLocalFolder } from '../../../../lib/detect';
 import { getFileType } from '../../../../lib/iac/iac-parser';
 import * as util from 'util';
 import { IacFileTypes } from '../../../../lib/iac/constants';
 import { IacFileScanResult, IacFileMetadata, IacFileData } from './types';
-import { buildPolicyEngine } from './policy-engine';
+import { getPolicyEngine } from './policy-engine';
 import { formatResults } from './results-formatter';
+import { tryParseIacFile } from './parsers';
+import { isLocalCacheExists, REQUIRED_LOCAL_CACHE_FILES } from './local-cache';
 
 const readFileContentsAsync = util.promisify(fs.readFile);
-const REQUIRED_K8S_FIELDS = ['apiVersion', 'kind', 'metadata'];
 
 // this method executes the local processing engine and then formats the results to adapt with the CLI output.
 // the current version is dependent on files to be present locally which are not part of the source code.
 // without these files this method would fail.
 // if you're interested in trying out the experimental local execution model for IaC scanning, please reach-out.
 export async function test(pathToScan: string, options) {
+  if (!isLocalCacheExists())
+    throw Error(
+      `Missing IaC local cache data, please validate you have: \n${REQUIRED_LOCAL_CACHE_FILES.join(
+        '\n',
+      )}`,
+    );
   // TODO: add support for proper typing of old TestResult interface.
   const results = await localProcessing(pathToScan);
   const formattedResults = formatResults(results, options);
@@ -27,12 +33,9 @@ export async function test(pathToScan: string, options) {
 async function localProcessing(
   pathToScan: string,
 ): Promise<IacFileScanResult[]> {
-  const policyEngine = await buildPolicyEngine();
   const filePathsToScan = await getFilePathsToScan(pathToScan);
-  const fileDataToScan = await parseFileContentsForPolicyEngine(
-    filePathsToScan,
-  );
-  const scanResults = await policyEngine.scanFiles(fileDataToScan);
+  const fileDataToScan = await parseFilesForScan(filePathsToScan);
+  const scanResults = await scanFilesForIssues(fileDataToScan);
   return scanResults;
 }
 
@@ -42,18 +45,13 @@ async function getFilePathsToScan(pathToScan): Promise<IacFileMetadata[]> {
       'IaC Experimental version does not support directory scan yet.',
     );
   }
-  if (getFileType(pathToScan) === 'tf') {
-    throw new Error(
-      'IaC Experimental version does not support Terraform scan yet.',
-    );
-  }
 
   return [
     { filePath: pathToScan, fileType: getFileType(pathToScan) as IacFileTypes },
   ];
 }
 
-async function parseFileContentsForPolicyEngine(
+async function parseFilesForScan(
   filesMetadata: IacFileMetadata[],
 ): Promise<IacFileData[]> {
   const parsedFileData: Array<IacFileData> = [];
@@ -62,25 +60,23 @@ async function parseFileContentsForPolicyEngine(
       fileMetadata.filePath,
       'utf-8',
     );
-    const yamlDocuments = YAML.safeLoadAll(fileContent);
-
-    yamlDocuments.forEach((parsedYamlDocument, docId) => {
-      if (
-        REQUIRED_K8S_FIELDS.every((requiredField) =>
-          parsedYamlDocument.hasOwnProperty(requiredField),
-        )
-      ) {
-        parsedFileData.push({
-          ...fileMetadata,
-          fileContent: fileContent,
-          jsonContent: parsedYamlDocument,
-          docId,
-        });
-      } else {
-        throw new Error('Invalid K8s File!');
-      }
-    });
+    const parsedFiles = tryParseIacFile(fileMetadata, fileContent);
+    parsedFileData.push(...parsedFiles);
   }
 
   return parsedFileData;
 }
+
+async function scanFilesForIssues(
+  parsedFiles: Array<IacFileData>,
+): Promise<IacFileScanResult[]> {
+  // TODO: when adding dir support move implementation to queue.
+  // TODO: when adding dir support gracefully handle failed scans
+  return Promise.all(
+    parsedFiles.map(async (file) => {
+      const policyEngine = await getPolicyEngine(file.engineType);
+      const scanResults = policyEngine.scanFile(file);
+      return scanResults;
+    }),
+  );
+}
diff --git a/src/cli/commands/test/iac-local-execution/local-cache.ts b/src/cli/commands/test/iac-local-execution/local-cache.ts
@@ -0,0 +1,48 @@
+import * as path from 'path';
+import * as fs from 'fs';
+import { EngineType } from './types';
+
+export const LOCAL_POLICY_ENGINE_DIR = `.iac-data`;
+
+const KUBERNETES_POLICY_ENGINE_WASM_PATH = path.join(
+  LOCAL_POLICY_ENGINE_DIR,
+  'k8s_policy.wasm',
+);
+const KUBERNETES_POLICY_ENGINE_DATA_PATH = path.join(
+  LOCAL_POLICY_ENGINE_DIR,
+  'k8s_data.json',
+);
+const TERRAFORM_POLICY_ENGINE_WASM_PATH = path.join(
+  LOCAL_POLICY_ENGINE_DIR,
+  'tf_policy.wasm',
+);
+const TERRAFORM_POLICY_ENGINE_DATA_PATH = path.join(
+  LOCAL_POLICY_ENGINE_DIR,
+  'tf_data.json',
+);
+
+export const REQUIRED_LOCAL_CACHE_FILES = [
+  KUBERNETES_POLICY_ENGINE_WASM_PATH,
+  KUBERNETES_POLICY_ENGINE_DATA_PATH,
+  TERRAFORM_POLICY_ENGINE_WASM_PATH,
+  TERRAFORM_POLICY_ENGINE_DATA_PATH,
+];
+
+export function isLocalCacheExists(): boolean {
+  return REQUIRED_LOCAL_CACHE_FILES.every(fs.existsSync);
+}
+
+export function getLocalCachePath(engineType: EngineType) {
+  switch (engineType) {
+    case EngineType.Kubernetes:
+      return [
+        `${process.cwd()}/${KUBERNETES_POLICY_ENGINE_WASM_PATH}`,
+        `${process.cwd()}/${KUBERNETES_POLICY_ENGINE_DATA_PATH}`,
+      ];
+    case EngineType.Terraform:
+      return [
+        `${process.cwd()}/${TERRAFORM_POLICY_ENGINE_WASM_PATH}`,
+        `${process.cwd()}/${TERRAFORM_POLICY_ENGINE_DATA_PATH}`,
+      ];
+  }
+}
diff --git a/src/cli/commands/test/iac-local-execution/parsers.ts b/src/cli/commands/test/iac-local-execution/parsers.ts
@@ -0,0 +1,65 @@
+import * as hclToJson from 'hcl-to-json';
+import * as YAML from 'js-yaml';
+import { EngineType, IacFileData, IacFileMetadata } from './types';
+
+const REQUIRED_K8S_FIELDS = ['apiVersion', 'kind', 'metadata'];
+
+export function tryParseIacFile(
+  fileMetadata: IacFileMetadata,
+  fileContent: string,
+): Array<IacFileData> {
+  switch (fileMetadata.fileType) {
+    case 'yaml':
+    case 'yml':
+    case 'json':
+      return tryParsingKubernetesFile(fileContent, fileMetadata);
+    case 'tf':
+      return [tryParsingTerraformFile(fileContent, fileMetadata)];
+    default:
+      throw new Error('Invalid IaC file');
+  }
+}
+
+function tryParsingKubernetesFile(
+  fileContent: string,
+  fileMetadata: IacFileMetadata,
+): IacFileData[] {
+  const yamlDocuments = YAML.safeLoadAll(fileContent);
+
+  return yamlDocuments.map((parsedYamlDocument, docId) => {
+    if (
+      REQUIRED_K8S_FIELDS.every((requiredField) =>
+        parsedYamlDocument.hasOwnProperty(requiredField),
+      )
+    ) {
+      return {
+        ...fileMetadata,
+        fileContent: fileContent,
+        jsonContent: parsedYamlDocument,
+        engineType: EngineType.Kubernetes,
+        docId,
+      };
+    } else {
+      throw new Error('Invalid K8s File!');
+    }
+  });
+}
+
+function tryParsingTerraformFile(
+  fileContent: string,
+  fileMetadata: IacFileMetadata,
+): IacFileData {
+  try {
+    // TODO: This parser does not fail on inavlid Terraform files! it is here temporarily.
+    // cloud-config team will replace it to a valid parser for the beta release.
+    const parsedData = hclToJson(fileContent);
+    return {
+      ...fileMetadata,
+      fileContent: fileContent,
+      jsonContent: parsedData,
+      engineType: EngineType.Terraform,
+    };
+  } catch (err) {
+    throw new Error('Invalid Terraform File!');
+  }
+}
diff --git a/src/cli/commands/test/iac-local-execution/policy-engine.ts b/src/cli/commands/test/iac-local-execution/policy-engine.ts
@@ -3,18 +3,36 @@ import {
   IacFileData,
   IacFileScanResult,
   PolicyMetadata,
+  EngineType,
 } from './types';
 import { loadPolicy } from '@open-policy-agent/opa-wasm';
 import * as fs from 'fs';
-import * as path from 'path';
+import { getLocalCachePath, LOCAL_POLICY_ENGINE_DIR } from './local-cache';
 
-const LOCAL_POLICY_ENGINE_DIR = `.iac-data`;
-const LOCAL_POLICY_ENGINE_WASM_PATH = `${LOCAL_POLICY_ENGINE_DIR}${path.sep}policy.wasm`;
-const LOCAL_POLICY_ENGINE_DATA_PATH = `${LOCAL_POLICY_ENGINE_DIR}${path.sep}data.json`;
+export async function getPolicyEngine(
+  engineType: EngineType,
+): Promise<PolicyEngine> {
+  if (policyEngineCache[engineType]) {
+    return policyEngineCache[engineType]!;
+  }
+
+  policyEngineCache[engineType] = await buildPolicyEngine(engineType);
+  return policyEngineCache[engineType]!;
+}
+
+const policyEngineCache: { [key in EngineType]: PolicyEngine | null } = {
+  [EngineType.Kubernetes]: null,
+  [EngineType.Terraform]: null,
+};
+
+async function buildPolicyEngine(
+  engineType: EngineType,
+): Promise<PolicyEngine> {
+  const [
+    policyEngineCoreDataPath,
+    policyEngineMetaDataPath,
+  ] = getLocalCachePath(engineType);
 
-export async function buildPolicyEngine(): Promise<PolicyEngine> {
-  const policyEngineCoreDataPath = `${process.cwd()}/${LOCAL_POLICY_ENGINE_WASM_PATH}`;
-  const policyEngineMetaDataPath = `${process.cwd()}/${LOCAL_POLICY_ENGINE_DATA_PATH}`;
   try {
     const wasmFile = fs.readFileSync(policyEngineCoreDataPath);
     const policyMetaData = fs.readFileSync(policyEngineMetaDataPath);
@@ -44,17 +62,13 @@ class PolicyEngine {
     return this.opaWasmInstance.evaluate(data)[0].result;
   }
 
-  public async scanFiles(
-    filesToScan: IacFileData[],
-  ): Promise<IacFileScanResult[]> {
+  public scanFile(iacFile: IacFileData): IacFileScanResult {
     try {
-      return filesToScan.map((iacFile: IacFileData) => {
-        const violatedPolicies = this.evaluate(iacFile.jsonContent);
-        return {
-          ...iacFile,
-          violatedPolicies,
-        };
-      });
+      const violatedPolicies = this.evaluate(iacFile.jsonContent);
+      return {
+        ...iacFile,
+        violatedPolicies,
+      };
     } catch (err) {
       // TODO: to distinguish between different failure reasons
       throw new Error(`Failed to run policy engine: ${err}`);

diff --git a/src/cli/commands/test/iac-local-execution/results-formatter.ts b/src/cli/commands/test/iac-local-execution/results-formatter.ts
@@ -1,5 +1,6 @@
-import { IacFileScanResult, PolicyMetadata } from './types';
+import { EngineType, IacFileScanResult, PolicyMetadata } from './types';
 import { SEVERITY } from '../../../../lib/snyk-test/common';
+import { IacProjectType } from '../../../../lib/iac/constants';
 // import {
 //   issuesToLineNumbers,
 //   CloudConfigFileTypes,
@@ -34,16 +35,22 @@ export function formatResults(
 //   }
 // }
 
+const engineTypeToProjectType = {
+  [EngineType.Kubernetes]: IacProjectType.K8S,
+  [EngineType.Terraform]: IacProjectType.TERRAFORM,
+};
+
 function iacLocalFileScanToFormattedResult(
   iacFileScanResult: IacFileScanResult,
   severityThreshold?: SEVERITY,
 ) {
   const formattedIssues = iacFileScanResult.violatedPolicies.map((policy) => {
     // TODO: make sure we handle this issue with annotations:
     // https://github.com/snyk/registry/pull/17277
-    const cloudConfigPath = [`[DocId:${iacFileScanResult.docId}]`].concat(
-      policy.msg.split('.'),
-    );
+    const cloudConfigPath =
+      iacFileScanResult.docId !== undefined
+        ? [`[DocId:${iacFileScanResult.docId}]`].concat(policy.msg.split('.'))
+        : policy.msg.split('.');
     const lineNumber = -1;
     // TODO: once package becomes public, restore the commented out code for having the issue-to-line-number functionality
     // try {
@@ -80,7 +87,7 @@ function iacLocalFileScanToFormattedResult(
       ),
     },
     isPrivate: true,
-    packageManager: 'k8sconfig',
+    packageManager: engineTypeToProjectType[iacFileScanResult.engineType],
     targetFile: iacFileScanResult.filePath,
   };
 }

diff --git a/src/cli/commands/test/iac-local-execution/types.ts b/src/cli/commands/test/iac-local-execution/types.ts
@@ -5,6 +5,7 @@ export type IacFileMetadata = IacFileInDirectory;
 export interface IacFileData extends IacFileMetadata {
   jsonContent: Record<string, any>;
   fileContent: string;
+  engineType: EngineType;
   docId?: number;
 }
 export interface IacFileScanResult extends IacFileData {
@@ -16,6 +17,10 @@ export interface OpaWasmInstance {
   setData: (data: Record<string, any>) => void;
 }
 
+export enum EngineType {
+  Kubernetes,
+  Terraform,
+}
 export interface PolicyMetadata {
   id: string;
   publicId: string;

diff --git a/src/cli/commands/test/iac-output.ts b/src/cli/commands/test/iac-output.ts
@@ -30,8 +30,6 @@ function formatIacIssue(
     introducedBy = `\n    introduced by ${pathStr}`;
   }
 
-  const description = extractOverview(issue.description).trim();
-  const descriptionLine = `\n    ${description}\n`;
   const severityColor = getSeveritiesColour(issue.severity);
 
   return (
@@ -43,20 +41,10 @@ function formatIacIssue(
     ` [${issue.id}]` +
     name +
     introducedBy +
-    descriptionLine
+    '\n'
   );
 }
 
-function extractOverview(description: string): string {
-  if (!description) {
-    return '';
-  }
-
-  const overviewRegExp = /## Overview([\s\S]*?)(?=##|(# Details))/m;
-  const overviewMatches = overviewRegExp.exec(description);
-  return (overviewMatches && overviewMatches[1]) || '';
-}
-
 export function getIacDisplayedOutput(
   iacTest: IacTestResponse,
   testedInfoText: string,