From 4b8b0cfb165ad106e4ebd87ce4488b9fa26c20af Mon Sep 17 00:00:00 2001 From: Jorge Cotillo Date: Mon, 10 May 2021 14:53:34 -0700 Subject: [PATCH 1/2] generate onboarded report --- generator/cmd/generateonboardedreport.ts | 56 ++++++++++++++++++++++++ generator/constants.ts | 1 + generator/package.json | 1 + 3 files changed, 58 insertions(+) create mode 100644 generator/cmd/generateonboardedreport.ts diff --git a/generator/cmd/generateonboardedreport.ts b/generator/cmd/generateonboardedreport.ts new file mode 100644 index 0000000000..9aa0975a26 --- /dev/null +++ b/generator/cmd/generateonboardedreport.ts @@ -0,0 +1,56 @@ +import * as constants from '../constants'; +import { cloneAndGenerateBasePaths, validateAndReturnReadmePath } from '../specs'; +import chalk from 'chalk'; +import { findAutogenEntries } from '../autogenlist'; +import { executeSynchronous, lowerCaseEquals, writeJsonFile, safeMkdir } from '../utils'; +import { getApiVersionsByNamespace } from '../generate'; +import { keys, partition } from 'lodash'; +import path from 'path'; + +executeSynchronous(async () => { + const basePaths = await cloneAndGenerateBasePaths(constants.specsRepoPath, constants.specsRepoUri, constants.specsRepoCommitHash); + + let allBasePaths = []; + + for (const basePath of basePaths) { + const readme = await validateAndReturnReadmePath(constants.specsRepoPath, basePath); + const namespaces = keys(await getApiVersionsByNamespace(readme)); + const autogenlistEntries = findAutogenEntries(basePath); + + const [autogened, unautogened] = partition( + namespaces, + n => autogenlistEntries.findIndex(w => lowerCaseEquals(w.namespace, n)) > -1); + + if (unautogened.length > 0 && autogened.length > 0) { + // For partial autogeneration only, add two items + // one item containing resource types not onboarded for autogeneration + // and the other item containing resource types onboarded for autogeneration + allBasePaths.push({ + 'basePath': basePath, + 'onboardedToAutogen': 'no', + 'missing': unautogened, + 'included': [] + }); + + allBasePaths.push({ + 'basePath': basePath, + 'onboardedToAutogen': 'yes', + 'missing': [], + 'included': autogened + }); + } + else { + // unautogened.length === 0 means all resource types are onboarded for autogeneration + allBasePaths.push({ + 'basePath': basePath, + 'onboardedToAutogen': unautogened.length === 0 ? 'yes' : 'no', + 'missing': unautogened, + 'onboarded': [] + }); + } + } + + const autogenResultPath = path.join(constants.autogenResultPath, 'result.json'); + await safeMkdir(constants.autogenResultPath); + await writeJsonFile(autogenResultPath, allBasePaths); +}); \ No newline at end of file diff --git a/generator/constants.ts b/generator/constants.ts index e724775681..6da2952640 100644 --- a/generator/constants.ts +++ b/generator/constants.ts @@ -12,6 +12,7 @@ export const autoRestVerboseOutput = false; export const schemasBaseUri = 'https://schema.management.azure.com/schemas'; export const schemasBasePath = path.join(generatorRoot, 'schemas'); +export const autogenResultPath = path.join(generatorRoot, 'onboarded-report'); export const resourceGroupRootSchema = { file: path.join(schemasBasePath, 'common/autogeneratedResources.json'), jsonPath: 'allOf[1].oneOf' diff --git a/generator/package.json b/generator/package.json index 5bf0cf5c6c..37f1708e80 100644 --- a/generator/package.json +++ b/generator/package.json @@ -8,6 +8,7 @@ "npm": ">=6.0.0" }, "scripts": { + "generate-onboarded-report": "ts-node cmd/getrpsonboarded", "list-basepaths": "ts-node cmd/listbasepaths", "list-resources": "ts-node cmd/listresources", "generate-all": "ts-node cmd/generateall", From 02f2726e57ff06d1a37fc59891eaed779838eb46 Mon Sep 17 00:00:00 2001 From: Jorge Cotillo Date: Mon, 10 May 2021 14:55:26 -0700 Subject: [PATCH 2/2] new pipeline --- azure-pipelines-generate-onboarded-report.yml | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 azure-pipelines-generate-onboarded-report.yml diff --git a/azure-pipelines-generate-onboarded-report.yml b/azure-pipelines-generate-onboarded-report.yml new file mode 100644 index 0000000000..47ee40743a --- /dev/null +++ b/azure-pipelines-generate-onboarded-report.yml @@ -0,0 +1,69 @@ +pr: none +trigger: none + +schedules: +- cron: "0 6 * * 5" + branches: + include: + - master + displayName: Weekly report of onboarded RPs + +jobs: +- job: Generate + timeoutInMinutes: 180 + + pool: + vmImage: 'ubuntu-latest' + + steps: + - checkout: self + clean: true + + - task: DownloadSecureFile@1 + name: schemasDeployKey + inputs: + secureFile: schemas_rsa + displayName: "Download GitHub Deploy Key" + + - task: NodeTool@0 + inputs: + versionSpec: '14.x' + + - script: | + set -Eeuxo pipefail + mkdir ~/.ssh && mv $(schemasDeployKey.secureFilePath) ~/.ssh/schemas_rsa + chmod 700 ~/.ssh + chmod 600 ~/.ssh/schemas_rsa + git config user.name "Autogenerator Pipeline" + git config user.email "azure-resource-manager-schemas@noreply.github.com" + git config core.sshCommand "ssh -i ~/.ssh/schemas_rsa -F /dev/null" + git remote set-url origin git@github.com:Azure/azure-resource-manager-schemas.git + ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts + git fetch + displayName: Configure git + + - script: | + set -Eeuxo pipefail + git checkout $(Build.SourceBranchName) + displayName: Checkout repo + + - script: | + set -Eeuxo pipefail + cd generator + npm install + displayName: 'Install packages' + + - script: | + set -Eeuxo pipefail + cd generator + npm run generate-onboarded-report + displayName: 'Generate report' + + - script: | + set -Eeuxo pipefail + git add --all onboarded-report + if ! git diff-index --quiet HEAD --; then + git commit -m "Generated report" + git push origin $(Build.SourceBranchName) + fi + displayName: Commit report \ No newline at end of file