Merge pull request #1831 from Azure/antmarti/fix_generator_pipeline

Fix generator pipeline
AzureSDKAutomation · Aug 11, 2021 · 94c6d5e · 94c6d5e
2 parents fd45b5c + 0143ae3
commit 94c6d5e
Show file tree

Hide file tree

Showing 6 changed files with 49 additions and 55 deletions.
diff --git a/generator/autogenlist.ts b/generator/autogenlist.ts
@@ -1047,10 +1047,22 @@ const autoGenList: AutoGenConfig[] = [
     },
 ];
 
-export function getAutoGenList(): AutoGenConfig[] {
-    return autoGenList;
-}
-
 export function findAutogenEntries(basePath: string): AutoGenConfig[] {
     return autoGenList.filter(w => lowerCaseEquals(w.basePath, basePath));
 }
+
+export function findOrGenerateAutogenEntries(basePath: string, namespaces: string[]): AutoGenConfig[] {
+    const entries = findAutogenEntries(basePath).filter(e => namespaces.some(ns => lowerCaseEquals(e.namespace, ns)));
+
+    for (const namespace of namespaces) {
+        if (!entries.some(e => lowerCaseEquals(e.namespace, namespace))) {
+            // Generate configuration for any RPs not explicitly declared in the autogen list
+            entries.push({
+                basePath,
+                namespace,
+            });
+        }
+    }
+
+    return entries;
+}
diff --git a/generator/cmd/generateall.ts b/generator/cmd/generateall.ts
@@ -1,7 +1,7 @@
 import * as constants from '../constants';
 import { cloneAndGenerateBasePaths, generateBasePaths, getPackageString, resolveAbsolutePath, validateAndReturnReadmePath } from '../specs';
 import { SchemaConfiguration, generateSchemas, clearAutoGeneratedSchemaRefs, saveAutoGeneratedSchemaRefs, getApiVersionsByNamespace } from '../generate';
-import { findAutogenEntries } from '../autogenlist';
+import { findOrGenerateAutogenEntries } from '../autogenlist';
 import chalk from 'chalk';
 import { flatten, keys, partition } from 'lodash';
 import { executeSynchronous, chunker, writeJsonFile, lowerCaseEquals } from '../utils';
@@ -47,22 +47,7 @@ executeSynchronous(async () => {
     for (const basePath of basePaths) {
         const readme = await validateAndReturnReadmePath(localPath, basePath);
         const namespaces = keys(await getApiVersionsByNamespace(readme));
-        const autogenlistEntries = findAutogenEntries(basePath);
-
-        const [unautogened, autogened] = partition(
-            namespaces,
-            n => autogenlistEntries.filter(w => lowerCaseEquals(w.namespace, n))[0]?.disabledForAutogen === true);
-
-        // Generate configuration for any RPs not explicitly declared in the autogen list
-        const generatedConfig = unautogened.map(namespace => ({
-            basePath,
-            namespace,
-        } as AutoGenConfig));
-
-        let filteredAutoGenList = [
-            ...autogenlistEntries,
-            ...generatedConfig,
-        ];
+        let filteredAutoGenList = findOrGenerateAutogenEntries(basePath, namespaces);
 
         if (!!params.readmeFiles) {
             filteredAutoGenList = filteredAutoGenList.filter(c => {
@@ -81,7 +66,7 @@ executeSynchronous(async () => {
             if (autoGenConfig.disabledForAutogen === true) {
                 continue;
             }
-            
+
             let pkg = {
                 path: ['schemas']
             } as Package;

diff --git a/generator/cmd/generateonboardedreport.ts b/generator/cmd/generateonboardedreport.ts
@@ -1,7 +1,7 @@
 import * as constants from '../constants';
 import { cloneAndGenerateBasePaths, validateAndReturnReadmePath } from '../specs';
 import chalk from 'chalk';
-import { findAutogenEntries } from '../autogenlist';
+import { findOrGenerateAutogenEntries } from '../autogenlist';
 import { executeSynchronous, lowerCaseEquals, writeJsonFile, safeMkdir } from '../utils';
 import { getApiVersionsByNamespace } from '../generate';
 import { keys, partition } from 'lodash';
@@ -15,11 +15,11 @@ executeSynchronous(async () => {
     for (const basePath of basePaths) {
         const readme = await validateAndReturnReadmePath(constants.specsRepoPath, basePath);
         const namespaces = keys(await getApiVersionsByNamespace(readme));
-        const autogenlistEntries = findAutogenEntries(basePath);
+        const autogenlistEntries = findOrGenerateAutogenEntries(basePath, namespaces);
 
         const [unautogened, autogened] = partition(
-            namespaces,
-            n => autogenlistEntries.filter(w => lowerCaseEquals(w.namespace, n))[0]?.disabledForAutogen === true);
+            autogenlistEntries,
+            e => e.disabledForAutogen === true);
 
         if (unautogened.length > 0 && autogened.length > 0) {
             // For partial autogeneration only, add two items
@@ -28,23 +28,23 @@ executeSynchronous(async () => {
             allBasePaths.push({
                 'basePath': basePath,
                 'onboardedToAutogen': 'no',
-                'missing': unautogened,
+                'missing': unautogened.map(x => x.namespace),
                 'onboarded': []
             });
 
             allBasePaths.push({
                 'basePath': basePath,
                 'onboardedToAutogen': 'yes',
                 'missing': [],
-                'onboarded': autogened
+                'onboarded': autogened.map(x => x.namespace)
             });
         }
         else {
             // unautogened.length === 0 means all resource types are onboarded for autogeneration
             allBasePaths.push({
                 'basePath': basePath,
                 'onboardedToAutogen': unautogened.length === 0 ? 'yes' : 'no',
-                'missing': unautogened,
+                'missing': unautogened.map(x => x.namespace),
                 'onboarded': []
             });
         }

diff --git a/generator/cmd/generatesingle.ts b/generator/cmd/generatesingle.ts
@@ -1,9 +1,10 @@
 import * as constants from '../constants';
-import { cloneAndGenerateBasePaths, resolveAbsolutePath, validateAndReturnReadmePath, getPackageString } from '../specs';
-import { generateSchemas, saveAutoGeneratedSchemaRefs } from '../generate';
+import { cloneAndGenerateBasePaths, resolveAbsolutePath, validateAndReturnReadmePath } from '../specs';
+import { generateSchemas, saveAutoGeneratedSchemaRefs, getApiVersionsByNamespace } from '../generate';
 import process from 'process';
-import { findAutogenEntries } from '../autogenlist';
+import { findOrGenerateAutogenEntries } from '../autogenlist';
 import chalk from 'chalk';
+import { keys } from 'lodash';
 import { executeSynchronous } from '../utils';
 
 executeSynchronous(async () => {
@@ -24,25 +25,21 @@ executeSynchronous(async () => {
     }
 
     const schemaConfigs = [];
-    const autoGenEntries = findAutogenEntries(basePath);
+    const namespaces = keys(await getApiVersionsByNamespace(readme));
+    const autoGenEntries = findOrGenerateAutogenEntries(basePath, namespaces);
 
-    if (autoGenEntries.length === 0) {
-        const localSchemaConfigs = await generateSchemas(readme);
-        schemaConfigs.push(...localSchemaConfigs);
-    } else {
-        for (const autoGenConfig of autoGenEntries) {
-            if (autoGenConfig.disabledForAutogen === true) {
-                console.log(`Path ${autoGenConfig.basePath} has been disabled for generation:`)
-                console.log(chalk.red(JSON.stringify(autoGenConfig, null, 2)));
-                continue;
-            }
+    for (const autoGenConfig of autoGenEntries) {
+        if (autoGenConfig.disabledForAutogen === true) {
+            console.log(`Path ${autoGenConfig.basePath} has been disabled for generation:`)
+            console.log(chalk.red(JSON.stringify(autoGenConfig, null, 2)));
+            continue;
+        }
 
-            console.log(`Using autogenlist config:`)
-            console.log(chalk.green(JSON.stringify(autoGenConfig, null, 2)));
+        console.log(`Using autogenlist config:`)
+        console.log(chalk.green(JSON.stringify(autoGenConfig, null, 2)));
 
-            const localSchemaConfigs = await generateSchemas(readme, autoGenConfig);
-            schemaConfigs.push(...localSchemaConfigs);
-        }
+        const localSchemaConfigs = await generateSchemas(readme, autoGenConfig);
+        schemaConfigs.push(...localSchemaConfigs);
     }
 
     await saveAutoGeneratedSchemaRefs(schemaConfigs);

diff --git a/generator/cmd/listbasepaths.ts b/generator/cmd/listbasepaths.ts
@@ -1,7 +1,7 @@
 import * as constants from '../constants';
 import { cloneAndGenerateBasePaths, validateAndReturnReadmePath } from '../specs';
 import chalk from 'chalk';
-import { findAutogenEntries } from '../autogenlist';
+import { findOrGenerateAutogenEntries } from '../autogenlist';
 import { executeSynchronous, lowerCaseEquals } from '../utils';
 import { getApiVersionsByNamespace } from '../generate';
 import { keys, partition } from 'lodash';
@@ -12,18 +12,18 @@ executeSynchronous(async () => {
     for (const basePath of basePaths) {
         const readme = await validateAndReturnReadmePath(constants.specsRepoPath, basePath);
         const namespaces = keys(await getApiVersionsByNamespace(readme));
-        const autogenlistEntries = findAutogenEntries(basePath);
+        const autogenlistEntries = findOrGenerateAutogenEntries(basePath, namespaces);
 
         const [unautogened, autogened] = partition(
-            namespaces,
-            n => autogenlistEntries.filter(w => lowerCaseEquals(w.namespace, n))[0]?.disabledForAutogen === true);
+            autogenlistEntries,
+            e => e.disabledForAutogen === true);
 
         if (unautogened.length === 0) {
             console.log(`Discovered '${chalk.green(basePath)}'. enabled for auto-generation: ${chalk.green('yes')}.`);
         } else if (autogened.length > 0) {
-            console.log(`Discovered '${chalk.green(basePath)}'. enabled for auto-generation: ${chalk.yellow('partial')}. Missing: ${unautogened.map(p => chalk.yellow(p)).join(', ')}.`);
+            console.log(`Discovered '${chalk.green(basePath)}'. enabled for auto-generation: ${chalk.yellow('partial')}. Missing: ${unautogened.map(p => chalk.yellow(p.namespace)).join(', ')}.`);
         } else {
-            console.log(`Discovered '${chalk.green(basePath)}'. enabled for auto-generation: ${chalk.red('no')}. Missing: ${unautogened.map(p => chalk.yellow(p)).join(', ')}.`);
+            console.log(`Discovered '${chalk.green(basePath)}'. enabled for auto-generation: ${chalk.red('no')}. Missing: ${unautogened.map(p => chalk.yellow(p.namespace)).join(', ')}.`);
         }
     }
 });
diff --git a/generator/processors/Microsoft.Insights.Application.ts b/generator/processors/Microsoft.Insights.Application.ts
@@ -3,7 +3,7 @@ import { SchemaPostProcessor } from '../models';
 export const postProcessor: SchemaPostProcessor = (namespace: string, apiVersion: string, schema: any) => {
   // this shouldn't be a resource definition, and it causes Export failures as it contains duplicate properties "Type" and "type"
   const resources = Object.values<any>(schema.resourceDefinitions || {});
-  console.log(resources);
+
   for (const resource of resources) {
     if (resource?.properties['Type'] && resource?.properties['type']) {
       delete resource.properties['Type'];