From 0143ae32c5a2f5d9381fabfde5a801b75be91df6 Mon Sep 17 00:00:00 2001 From: Anthony Martin Date: Wed, 11 Aug 2021 15:33:05 +0000 Subject: [PATCH] Fix generator pipeline --- generator/autogenlist.ts | 20 ++++++++--- generator/cmd/generateall.ts | 21 ++--------- generator/cmd/generateonboardedreport.ts | 14 ++++---- generator/cmd/generatesingle.ts | 35 +++++++++---------- generator/cmd/listbasepaths.ts | 12 +++---- .../Microsoft.Insights.Application.ts | 2 +- 6 files changed, 49 insertions(+), 55 deletions(-) diff --git a/generator/autogenlist.ts b/generator/autogenlist.ts index 9a1d58815e..2b1f7dd7bc 100644 --- a/generator/autogenlist.ts +++ b/generator/autogenlist.ts @@ -1047,10 +1047,22 @@ const autoGenList: AutoGenConfig[] = [ }, ]; -export function getAutoGenList(): AutoGenConfig[] { - return autoGenList; -} - export function findAutogenEntries(basePath: string): AutoGenConfig[] { return autoGenList.filter(w => lowerCaseEquals(w.basePath, basePath)); } + +export function findOrGenerateAutogenEntries(basePath: string, namespaces: string[]): AutoGenConfig[] { + const entries = findAutogenEntries(basePath).filter(e => namespaces.some(ns => lowerCaseEquals(e.namespace, ns))); + + for (const namespace of namespaces) { + if (!entries.some(e => lowerCaseEquals(e.namespace, namespace))) { + // Generate configuration for any RPs not explicitly declared in the autogen list + entries.push({ + basePath, + namespace, + }); + } + } + + return entries; +} \ No newline at end of file diff --git a/generator/cmd/generateall.ts b/generator/cmd/generateall.ts index b2dc23470a..7e199361f2 100644 --- a/generator/cmd/generateall.ts +++ b/generator/cmd/generateall.ts @@ -1,7 +1,7 @@ import * as constants from '../constants'; import { cloneAndGenerateBasePaths, generateBasePaths, getPackageString, resolveAbsolutePath, validateAndReturnReadmePath } from '../specs'; import { SchemaConfiguration, generateSchemas, clearAutoGeneratedSchemaRefs, saveAutoGeneratedSchemaRefs, getApiVersionsByNamespace } from '../generate'; -import { findAutogenEntries } from '../autogenlist'; +import { findOrGenerateAutogenEntries } from '../autogenlist'; import chalk from 'chalk'; import { flatten, keys, partition } from 'lodash'; import { executeSynchronous, chunker, writeJsonFile, lowerCaseEquals } from '../utils'; @@ -47,22 +47,7 @@ executeSynchronous(async () => { for (const basePath of basePaths) { const readme = await validateAndReturnReadmePath(constants.specsRepoPath, basePath); const namespaces = keys(await getApiVersionsByNamespace(readme)); - const autogenlistEntries = findAutogenEntries(basePath); - - const [unautogened, autogened] = partition( - namespaces, - n => autogenlistEntries.filter(w => lowerCaseEquals(w.namespace, n))[0]?.disabledForAutogen === true); - - // Generate configuration for any RPs not explicitly declared in the autogen list - const generatedConfig = unautogened.map(namespace => ({ - basePath, - namespace, - } as AutoGenConfig)); - - let filteredAutoGenList = [ - ...autogenlistEntries, - ...generatedConfig, - ]; + let filteredAutoGenList = findOrGenerateAutogenEntries(basePath, namespaces); if (!!params.readmeFiles) { filteredAutoGenList = filteredAutoGenList.filter(c => { @@ -81,7 +66,7 @@ executeSynchronous(async () => { if (autoGenConfig.disabledForAutogen === true) { continue; } - + let pkg = { path: ['schemas'] } as Package; diff --git a/generator/cmd/generateonboardedreport.ts b/generator/cmd/generateonboardedreport.ts index 9735a03e16..f5b99ae948 100644 --- a/generator/cmd/generateonboardedreport.ts +++ b/generator/cmd/generateonboardedreport.ts @@ -1,7 +1,7 @@ import * as constants from '../constants'; import { cloneAndGenerateBasePaths, validateAndReturnReadmePath } from '../specs'; import chalk from 'chalk'; -import { findAutogenEntries } from '../autogenlist'; +import { findOrGenerateAutogenEntries } from '../autogenlist'; import { executeSynchronous, lowerCaseEquals, writeJsonFile, safeMkdir } from '../utils'; import { getApiVersionsByNamespace } from '../generate'; import { keys, partition } from 'lodash'; @@ -15,11 +15,11 @@ executeSynchronous(async () => { for (const basePath of basePaths) { const readme = await validateAndReturnReadmePath(constants.specsRepoPath, basePath); const namespaces = keys(await getApiVersionsByNamespace(readme)); - const autogenlistEntries = findAutogenEntries(basePath); + const autogenlistEntries = findOrGenerateAutogenEntries(basePath, namespaces); const [unautogened, autogened] = partition( - namespaces, - n => autogenlistEntries.filter(w => lowerCaseEquals(w.namespace, n))[0]?.disabledForAutogen === true); + autogenlistEntries, + e => e.disabledForAutogen === true); if (unautogened.length > 0 && autogened.length > 0) { // For partial autogeneration only, add two items @@ -28,7 +28,7 @@ executeSynchronous(async () => { allBasePaths.push({ 'basePath': basePath, 'onboardedToAutogen': 'no', - 'missing': unautogened, + 'missing': unautogened.map(x => x.namespace), 'onboarded': [] }); @@ -36,7 +36,7 @@ executeSynchronous(async () => { 'basePath': basePath, 'onboardedToAutogen': 'yes', 'missing': [], - 'onboarded': autogened + 'onboarded': autogened.map(x => x.namespace) }); } else { @@ -44,7 +44,7 @@ executeSynchronous(async () => { allBasePaths.push({ 'basePath': basePath, 'onboardedToAutogen': unautogened.length === 0 ? 'yes' : 'no', - 'missing': unautogened, + 'missing': unautogened.map(x => x.namespace), 'onboarded': [] }); } diff --git a/generator/cmd/generatesingle.ts b/generator/cmd/generatesingle.ts index 243c1d811b..6e63604dc7 100644 --- a/generator/cmd/generatesingle.ts +++ b/generator/cmd/generatesingle.ts @@ -1,9 +1,10 @@ import * as constants from '../constants'; -import { cloneAndGenerateBasePaths, resolveAbsolutePath, validateAndReturnReadmePath, getPackageString } from '../specs'; -import { generateSchemas, saveAutoGeneratedSchemaRefs } from '../generate'; +import { cloneAndGenerateBasePaths, resolveAbsolutePath, validateAndReturnReadmePath } from '../specs'; +import { generateSchemas, saveAutoGeneratedSchemaRefs, getApiVersionsByNamespace } from '../generate'; import process from 'process'; -import { findAutogenEntries } from '../autogenlist'; +import { findOrGenerateAutogenEntries } from '../autogenlist'; import chalk from 'chalk'; +import { keys } from 'lodash'; import { executeSynchronous } from '../utils'; executeSynchronous(async () => { @@ -24,25 +25,21 @@ executeSynchronous(async () => { } const schemaConfigs = []; - const autoGenEntries = findAutogenEntries(basePath); + const namespaces = keys(await getApiVersionsByNamespace(readme)); + const autoGenEntries = findOrGenerateAutogenEntries(basePath, namespaces); - if (autoGenEntries.length === 0) { - const localSchemaConfigs = await generateSchemas(readme); - schemaConfigs.push(...localSchemaConfigs); - } else { - for (const autoGenConfig of autoGenEntries) { - if (autoGenConfig.disabledForAutogen === true) { - console.log(`Path ${autoGenConfig.basePath} has been disabled for generation:`) - console.log(chalk.red(JSON.stringify(autoGenConfig, null, 2))); - continue; - } + for (const autoGenConfig of autoGenEntries) { + if (autoGenConfig.disabledForAutogen === true) { + console.log(`Path ${autoGenConfig.basePath} has been disabled for generation:`) + console.log(chalk.red(JSON.stringify(autoGenConfig, null, 2))); + continue; + } - console.log(`Using autogenlist config:`) - console.log(chalk.green(JSON.stringify(autoGenConfig, null, 2))); + console.log(`Using autogenlist config:`) + console.log(chalk.green(JSON.stringify(autoGenConfig, null, 2))); - const localSchemaConfigs = await generateSchemas(readme, autoGenConfig); - schemaConfigs.push(...localSchemaConfigs); - } + const localSchemaConfigs = await generateSchemas(readme, autoGenConfig); + schemaConfigs.push(...localSchemaConfigs); } await saveAutoGeneratedSchemaRefs(schemaConfigs); diff --git a/generator/cmd/listbasepaths.ts b/generator/cmd/listbasepaths.ts index 6f249f3dc9..18e5e93a4e 100644 --- a/generator/cmd/listbasepaths.ts +++ b/generator/cmd/listbasepaths.ts @@ -1,7 +1,7 @@ import * as constants from '../constants'; import { cloneAndGenerateBasePaths, validateAndReturnReadmePath } from '../specs'; import chalk from 'chalk'; -import { findAutogenEntries } from '../autogenlist'; +import { findOrGenerateAutogenEntries } from '../autogenlist'; import { executeSynchronous, lowerCaseEquals } from '../utils'; import { getApiVersionsByNamespace } from '../generate'; import { keys, partition } from 'lodash'; @@ -12,18 +12,18 @@ executeSynchronous(async () => { for (const basePath of basePaths) { const readme = await validateAndReturnReadmePath(constants.specsRepoPath, basePath); const namespaces = keys(await getApiVersionsByNamespace(readme)); - const autogenlistEntries = findAutogenEntries(basePath); + const autogenlistEntries = findOrGenerateAutogenEntries(basePath, namespaces); const [unautogened, autogened] = partition( - namespaces, - n => autogenlistEntries.filter(w => lowerCaseEquals(w.namespace, n))[0]?.disabledForAutogen === true); + autogenlistEntries, + e => e.disabledForAutogen === true); if (unautogened.length === 0) { console.log(`Discovered '${chalk.green(basePath)}'. enabled for auto-generation: ${chalk.green('yes')}.`); } else if (autogened.length > 0) { - console.log(`Discovered '${chalk.green(basePath)}'. enabled for auto-generation: ${chalk.yellow('partial')}. Missing: ${unautogened.map(p => chalk.yellow(p)).join(', ')}.`); + console.log(`Discovered '${chalk.green(basePath)}'. enabled for auto-generation: ${chalk.yellow('partial')}. Missing: ${unautogened.map(p => chalk.yellow(p.namespace)).join(', ')}.`); } else { - console.log(`Discovered '${chalk.green(basePath)}'. enabled for auto-generation: ${chalk.red('no')}. Missing: ${unautogened.map(p => chalk.yellow(p)).join(', ')}.`); + console.log(`Discovered '${chalk.green(basePath)}'. enabled for auto-generation: ${chalk.red('no')}. Missing: ${unautogened.map(p => chalk.yellow(p.namespace)).join(', ')}.`); } } }); \ No newline at end of file diff --git a/generator/processors/Microsoft.Insights.Application.ts b/generator/processors/Microsoft.Insights.Application.ts index e9111d5a0e..8bd94a65c4 100644 --- a/generator/processors/Microsoft.Insights.Application.ts +++ b/generator/processors/Microsoft.Insights.Application.ts @@ -3,7 +3,7 @@ import { SchemaPostProcessor } from '../models'; export const postProcessor: SchemaPostProcessor = (namespace: string, apiVersion: string, schema: any) => { // this shouldn't be a resource definition, and it causes Export failures as it contains duplicate properties "Type" and "type" const resources = Object.values(schema.resourceDefinitions || {}); - console.log(resources); + for (const resource of resources) { if (resource?.properties['Type'] && resource?.properties['type']) { delete resource.properties['Type'];