Fix #8305: failed to generate the changelog between HLC and Modular (#…

…8332)

* fix #8305

* update version

* changelog-temp should keeps for gen changelog.md

* Revert "changelog-temp should keeps for gen changelog.md"

This reverts commit 2f37ada.

* cleanup

* cleanup

* WIP on wanl/version-extraction

* part 1

* part 1

* fix no feature bug: specify wrong api.md path for MLC

* cleanup

* support mix api versions in operations

* get api.md in a better way

* rename  sdktype

---------

Co-authored-by: albertxavier100 <[email protected]>

tools/js-sdk-release-tools/package.json

@@ -1,8 +1,9 @@
 {
   "name": "@azure-tools/js-sdk-release-tools",
-  "version": "2.7.9",
+  "version": "2.7.10",
   "description": "",
   "scripts": {
+    "dev": "nodemon src/changelogToolCli.ts",
     "start": "node dist/changelogToolCli.js",
     "debug": "node --inspect-brk dist/changelogToolCli.js",
     "build": "rimraf dist && tsc -p .",
@@ -33,7 +34,11 @@
     "yaml": "^1.10.2"
   },
   "devDependencies": {
+    "@types/node": "^20.12.12",
+    "@types/shelljs": "^0.8.15",
+    "nodemon": "^3.1.0",
     "rimraf": "^3.0.2",
+    "ts-node": "^10.9.2",
     "typescript": "^3.9.7"
   }
 }

tools/js-sdk-release-tools/src/common/interfaces.ts

@@ -0,0 +1,5 @@
+import { ApiVersionType } from "./types";
+
+export interface IApiVersionTypeExtractor {
+    (packageRoot: string): ApiVersionType;
+}

tools/js-sdk-release-tools/src/common/types.ts

@@ -0,0 +1,11 @@
+export enum SDKType {
+    HighLevelClient = 'HighLevelClient',
+    RestLevelClient = 'RestLevelClient',
+    ModularClient = 'ModularClient',
+};
+
+export enum ApiVersionType {
+    None = 'None',
+    Stable = 'Stable',
+    Preview = 'Preview',
+}

tools/js-sdk-release-tools/src/common/utils.ts

@@ -0,0 +1,51 @@
+import shell from 'shelljs';
+import path from 'path';
+import fs from 'fs';
+
+import { SDKType } from './types'
+import { logger } from "../utils/logger";
+import { Project, ScriptTarget, SourceFile } from 'ts-morph';
+
+export function getClassicClientParametersPath(packageRoot: string): string {
+    return path.join(packageRoot, 'src', 'models', 'parameters.ts');
+}
+
+export function getSDKType(packageRoot: string): SDKType {
+    const paraPath = getClassicClientParametersPath(packageRoot);
+    const exist = shell.test('-e', paraPath);
+    const type = exist ? SDKType.HighLevelClient : SDKType.ModularClient;
+    logger.logInfo(`SDK type: ${type} detected in ${packageRoot}`);
+    return type;
+}
+
+export function getNpmPackageName(packageRoot: string): string {
+    const packageJsonPath = path.join(packageRoot, 'package.json');
+    const packageJson = fs.readFileSync(packageJsonPath, { encoding: 'utf-8' });
+    const packageName = JSON.parse(packageJson).name;
+    return packageName;
+}
+
+export function getApiReviewPath(packageRoot: string): string {
+    const sdkType = getSDKType(packageRoot);
+    const reviewDir = path.join(packageRoot, 'review');
+    switch (sdkType) {
+        case SDKType.ModularClient:
+            const npmPackageName = getNpmPackageName(packageRoot);
+            const packageName = npmPackageName.substring("@azure/".length);
+            const apiViewFileName = `${packageName}.api.md`;
+            return path.join(packageRoot, 'review', apiViewFileName);
+        case SDKType.HighLevelClient:
+        case SDKType.RestLevelClient:
+        default:
+            // only one xxx.api.md
+            return path.join(packageRoot, 'review', fs.readdirSync(reviewDir)[0]);
+    }
+}
+
+export function getTsSourceFile(filePath: string): SourceFile | undefined {
+    const target = ScriptTarget.ES2015;
+    const compilerOptions = { target };
+    const project = new Project({ compilerOptions });
+    project.addSourceFileAtPath(filePath);
+    return project.getSourceFile(filePath);
+}

tools/js-sdk-release-tools/src/hlc/apiVersion/apiVersionTypeExtractor.ts

@@ -0,0 +1,24 @@
+import { ApiVersionType } from "../../common/types"
+import { IApiVersionTypeExtractor } from "../../common/interfaces";
+import { getClassicClientParametersPath, getTsSourceFile } from "../../common/utils";
+
+// TODO: add unit test
+export const getApiVersionType: IApiVersionTypeExtractor = (packageRoot: string): ApiVersionType => {
+    const paraPath = getClassicClientParametersPath(packageRoot);
+    const source = getTsSourceFile(paraPath);
+    const variableDeclarations = source?.getVariableDeclarations();
+    if (!variableDeclarations) return ApiVersionType.Stable;
+    for (const variableDeclaration of variableDeclarations) {
+        const fullText = variableDeclaration.getFullText();
+        if (fullText.toLowerCase().includes('apiversion')) {
+            const match = fullText.match(/defaultValue: "([0-9a-z-]+)"/);
+            if (!match || match.length !== 2) {
+                continue;
+            }
+            if (match[1].includes('preview')) {
+                return ApiVersionType.Preview;
+            }
+        }
+    }
+    return ApiVersionType.Stable;
+}

tools/js-sdk-release-tools/src/hlc/utils/automaticGenerateChangeLogAndBumpVersion.ts

@@ -1,3 +1,7 @@
+import fs from 'fs';
+import path from 'path';
+import shell from 'shelljs';
+
 import {extractExportAndGenerateChangelog, readSourceAndExtractMetaData} from "../../changelog/extractMetaData";
 import {Changelog, changelogGenerator} from "../../changelog/changelogGenerator";
 import {NPMScope, NPMViewResult} from "@ts-common/azure-js-dev-tools";
@@ -14,19 +18,18 @@ import {
     getVersion,
     isBetaVersion
 } from "../../utils/version";
-import {isGeneratedCodeStable} from "./isGeneratedCodeStable";
 import {execSync} from "child_process";
 import { getversionDate } from "../../utils/version";
-
-const fs = require('fs');
-const path = require('path');
+import { ApiVersionType } from "../../common/types"
+import { getApiVersionType } from '../../xlc/apiVersion/apiVersionTypeExtractor'
+import { getApiReviewPath, getNpmPackageName } from '../../common/utils';
 
 export async function generateChangelogAndBumpVersion(packageFolderPath: string) {
-    const shell = require('shelljs');
     const jsSdkRepoPath = String(shell.pwd());
     packageFolderPath = path.join(jsSdkRepoPath, packageFolderPath);
-    const isStableRelease = isGeneratedCodeStable(path.join(packageFolderPath, 'src', 'models', 'parameters.ts'));
-    const packageName = JSON.parse(fs.readFileSync(path.join(packageFolderPath, 'package.json'), {encoding: 'utf-8'})).name;
+    const ApiType = getApiVersionType(packageFolderPath);
+    const isStableRelease = ApiType != ApiVersionType.Preview;
+    const packageName = getNpmPackageName(packageFolderPath);
     const npm = new NPMScope({ executionFolderPath: packageFolderPath });
     const npmViewResult: NPMViewResult = await npm.view({ packageName });
     const stableVersion = getVersion(npmViewResult,"latest");
@@ -45,29 +48,31 @@ export async function generateChangelogAndBumpVersion(packageFolderPath: string)
         const usedVersions = npmViewResult['versions'];
         // in our rule, we always compare to stableVersion. But here wo should pay attention to the some stableVersion which contains beta, which means the package has not been GA.
         try {
-            await shell.mkdir(path.join(packageFolderPath, 'changelog-temp'));
-            await shell.cd(path.join(packageFolderPath, 'changelog-temp'));
-            await shell.exec(`npm pack ${packageName}@${stableVersion}`);
-            await shell.exec('tar -xzf *.tgz');
-            await shell.cd(packageFolderPath);
+            shell.mkdir(path.join(packageFolderPath, 'changelog-temp'));
+            shell.cd(path.join(packageFolderPath, 'changelog-temp'));
+            shell.exec(`npm pack ${packageName}@${stableVersion}`);
+            const files = shell.ls('*.tgz');
+            shell.exec(`tar -xzf ${files[0]}`);
+            shell.cd(packageFolderPath);
 
             // only track2 sdk includes sdk-type with value mgmt
             const sdkType = JSON.parse(fs.readFileSync(path.join(packageFolderPath, 'changelog-temp', 'package', 'package.json'), {encoding: 'utf-8'}))['sdk-type'];
             if (sdkType && sdkType === 'mgmt') {
                 logger.log(`Package ${packageName} released before is track2 sdk`);
                 logger.log('Generating changelog by comparing api.md...');
-                const reviewFolder = path.join(packageFolderPath, 'changelog-temp', 'package', 'review');
-                let apiMdFileNPM: string = path.join(reviewFolder, fs.readdirSync(reviewFolder)[0]);
-                let apiMdFileLocal: string = path.join(packageFolderPath, 'review', fs.readdirSync(path.join(packageFolderPath, 'review'))[0]);
+                const npmPackageRoot = path.join(packageFolderPath, 'changelog-temp', 'package');
+                const apiMdFileNPM = getApiReviewPath(npmPackageRoot);
+                const apiMdFileLocal = getApiReviewPath(packageFolderPath);
                 const changelog: Changelog = await extractExportAndGenerateChangelog(apiMdFileNPM, apiMdFileLocal);
                 let originalChangeLogContent = fs.readFileSync(path.join(packageFolderPath, 'changelog-temp', 'package', 'CHANGELOG.md'), {encoding: 'utf-8'});
                 if(nextVersion){
-                    await shell.cd(path.join(packageFolderPath, 'changelog-temp'));
-                    await shell.mkdir(path.join(packageFolderPath, 'changelog-temp', 'next'));
-                    await shell.cd(path.join(packageFolderPath,'changelog-temp', 'next'));
-                    await shell.exec(`npm pack ${packageName}@${nextVersion}`);
-                    await shell.exec('tar -xzf *.tgz');
-                    await shell.cd(packageFolderPath);
+                    shell.cd(path.join(packageFolderPath, 'changelog-temp'));
+                    shell.mkdir(path.join(packageFolderPath, 'changelog-temp', 'next'));
+                    shell.cd(path.join(packageFolderPath,'changelog-temp', 'next'));
+                    shell.exec(`npm pack ${packageName}@${nextVersion}`);
+                    const files = shell.ls('*.tgz');
+                    shell.exec(`tar -xzf ${files[0]}`);
+                    shell.cd(packageFolderPath);
                     logger.log("Create next folder successfully")
 
                     const latestDate = getversionDate(npmViewResult, stableVersion);
@@ -106,8 +111,8 @@ export async function generateChangelogAndBumpVersion(packageFolderPath: string)
                 logger.log('Generate changelogs and setting version for migrating track1 to track2 successfully');
             }
         } finally {
-            await shell.exec(`rm -r ${path.join(packageFolderPath, 'changelog-temp')}`);
-            await shell.cd(jsSdkRepoPath);
+            shell.rm('-r', `${path.join(packageFolderPath, 'changelog-temp')}`);
+            shell.cd(jsSdkRepoPath);
         }
     }
 }

tools/js-sdk-release-tools/src/llc/utils/generateChangelog.ts

@@ -4,6 +4,7 @@ import {NPMScope} from "@ts-common/azure-js-dev-tools";
 import {logger} from "../../utils/logger";
 import {getLatestStableVersion} from "../../utils/version";
 import {extractExportAndGenerateChangelog} from "../../changelog/extractMetaData";
+import { getApiReviewPath } from "../../common/utils";
 
 const shell = require('shelljs');
 const todayDate = new Date();
@@ -60,8 +61,10 @@ export async function generateChangelog(packagePath) {
                 logger.logWarn("The latest package released in NPM doesn't contain review folder, so generate changelog same as first release");
                 generateChangelogForFirstRelease(packagePath, version);
             } else {
-                let apiMdFileNPM = path.join(tempReviewFolder, fs.readdirSync(tempReviewFolder)[0]);
-                let apiMdFileLocal = path.join(packagePath, 'review', fs.readdirSync(path.join(packagePath, 'review'))[0]);
+                const npmPackageRoot = path.join(packagePath, 'changelog-temp', 'package');
+                // TODO: error out if it's comparing between RLC and HLC or Modular api layer and HLC
+                const apiMdFileNPM = getApiReviewPath(npmPackageRoot);
+                const apiMdFileLocal = getApiReviewPath(packagePath);
                 const changelog = await extractExportAndGenerateChangelog(apiMdFileNPM, apiMdFileLocal);
                 if (!changelog.hasBreakingChange && !changelog.hasFeature) {
                     logger.logError('Cannot generate changelog because the codes of local and npm may be the same.');

tools/js-sdk-release-tools/src/mlc/apiVersion/apiVersionTypeExtractor.ts

@@ -0,0 +1,81 @@
+import { SourceFile, SyntaxKind } from "ts-morph";
+import shell from 'shelljs';
+import path from 'path';
+
+import { ApiVersionType } from "../../common/types"
+import { IApiVersionTypeExtractor } from "../../common/interfaces";
+import { getTsSourceFile } from "../../common/utils";
+
+const findRestClientPath = (packageRoot: string): string => {
+    const restPath = path.join(packageRoot, 'src/rest/');
+    const fileNames = shell.ls(restPath);
+    const clientFiles = fileNames.filter(f => f.endsWith("Client.ts"));
+    if (clientFiles.length !== 1) throw new Error(`Single client is supported, but found ${clientFiles}`);
+
+    const clientPath = path.join(restPath, clientFiles[0]);
+    return clientPath;
+};
+
+const matchPattern = (text: string, pattern: RegExp): string | undefined => {
+    const match = text.match(pattern);
+    const found = match != null && match.length === 2;
+    return found ? match?.at(1) : undefined;
+}
+
+const findApiVersionInRestClient = (clientPath: string): string | undefined => {
+    const sourceFile = getTsSourceFile(clientPath);
+    const createClientFunction = sourceFile?.getFunction("createClient");
+    if (!createClientFunction) throw new Error("Function 'createClient' not found.");
+
+    const apiVersionStatements = createClientFunction.getStatements()
+        .filter(s =>
+            s.getKind() === SyntaxKind.ExpressionStatement &&
+            s.getText().indexOf("options.apiVersion") > -1);
+    if (apiVersionStatements.length === 0) return undefined;
+
+    const text = apiVersionStatements[apiVersionStatements.length - 1].getText();
+    const pattern = /(\d{4}-\d{2}-\d{2}(?:-preview)?)/;
+    const apiVersion = matchPattern(text, pattern);
+    return apiVersion;
+};
+
+const getApiVersionTypeFromRestClient: IApiVersionTypeExtractor = (packageRoot: string): ApiVersionType => {
+    const clientPath = findRestClientPath(packageRoot);
+    const apiVersion = findApiVersionInRestClient(clientPath);
+    if (apiVersion && apiVersion.indexOf("-preview") >= 0) return ApiVersionType.Preview;
+    if (apiVersion && apiVersion.indexOf("-preview") < 0) return ApiVersionType.Stable;
+    return ApiVersionType.None;
+};
+
+const findApiVersionsInOperations = (sourceFile: SourceFile | undefined): Array<string> | undefined => {
+    const interfaces = sourceFile?.getInterfaces();
+    const interfacesWithApiVersion = interfaces?.filter(itf => itf.getProperty('"api-version"'));
+    const apiVersions = interfacesWithApiVersion?.map(itf => {
+        const property = itf.getMembers()
+            .filter(m => {
+                const defaultValue = m.getChildrenOfKind(SyntaxKind.StringLiteral)[0];
+                return defaultValue && defaultValue.getText() === '"api-version"';
+            })[0];
+        const apiVersion = property.getChildrenOfKind(SyntaxKind.LiteralType)[0].getText();
+        return apiVersion;
+    });
+    return apiVersions;
+}
+
+const getApiVersionTypeFromOperations: IApiVersionTypeExtractor = (packageRoot: string): ApiVersionType => {
+    const paraPath = path.join(packageRoot, 'src/rest/parameters.ts');
+    const sourceFile = getTsSourceFile(paraPath);
+    const apiVersions = findApiVersionsInOperations(sourceFile);
+    if (!apiVersions) return ApiVersionType.None;
+    const previewVersions = apiVersions.filter(v => v.indexOf("-preview") >= 0);
+    return previewVersions.length > 0 ? ApiVersionType.Preview : ApiVersionType.Stable;
+};
+
+// TODO: add unit test
+export const getApiVersionType: IApiVersionTypeExtractor = (packageRoot: string): ApiVersionType => {
+    const typeFromClient = getApiVersionTypeFromRestClient(packageRoot);
+    if (typeFromClient !== ApiVersionType.None) return typeFromClient;
+    const typeFromOperations = getApiVersionTypeFromOperations(packageRoot);
+    if (typeFromOperations !== ApiVersionType.None) return typeFromOperations;
+    return ApiVersionType.Stable;
+}

tools/js-sdk-release-tools/src/xlc/apiVersion/apiVersionTypeExtractor.ts

@@ -0,0 +1,19 @@
+import { getSDKType } from "../../common/utils";
+import { ApiVersionType, SDKType } from "../../common/types";
+import { IApiVersionTypeExtractor } from "../../common/interfaces";
+import * as mlcApi from '../../mlc/apiVersion/apiVersionTypeExtractor'
+import * as hlcApi from '../../hlc/apiVersion/apiVersionTypeExtractor'
+
+// TODO: move to x-level-client folder
+export const getApiVersionType: IApiVersionTypeExtractor = (packageRoot: string): ApiVersionType => {
+    const sdkType = getSDKType(packageRoot);
+    switch (sdkType) {
+        case SDKType.ModularClient:
+            return mlcApi.getApiVersionType(packageRoot);
+        case SDKType.HighLevelClient:
+            return hlcApi.getApiVersionType(packageRoot);
+        default:
+            console.warn(`Unsupported SDK type ${sdkType} to get detact api version`);
+            return ApiVersionType.None;
+    }
+}