From 8783442a9bf9e7133f4ec78d4b1ee7a78897c4bd Mon Sep 17 00:00:00 2001 From: Catalina Peralta Date: Thu, 26 Oct 2023 14:59:24 -0700 Subject: [PATCH 01/11] fix cloneRepo --- tools/tsp-client/src/git.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/tsp-client/src/git.ts b/tools/tsp-client/src/git.ts index 4376dc7184c..1735d987399 100644 --- a/tools/tsp-client/src/git.ts +++ b/tools/tsp-client/src/git.ts @@ -6,7 +6,7 @@ export async function getRepoRoot(repoPath: string): Promise { } export async function cloneRepo(rootUrl: string, cloneDir: string, repo: string): Promise { - await simpleGit(rootUrl).clone(repo, cloneDir, ["--no-checkout", "--filter=tree:0"], (err) => { + simpleGit(rootUrl).clone(repo, cloneDir, ["--no-checkout", "--filter=tree:0"], (err) => { if (err) { throw err; } From a60e394ffc18dd3860f30ed6a07fb19783f9072a Mon Sep 17 00:00:00 2001 From: Catalina Peralta Date: Thu, 26 Oct 2023 15:00:41 -0700 Subject: [PATCH 02/11] fix git functions --- tools/tsp-client/src/git.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/tsp-client/src/git.ts b/tools/tsp-client/src/git.ts index 1735d987399..6feed847bae 100644 --- a/tools/tsp-client/src/git.ts +++ b/tools/tsp-client/src/git.ts @@ -2,7 +2,7 @@ import { spawn } from "child_process"; import { simpleGit } from "simple-git"; export async function getRepoRoot(repoPath: string): Promise { - return await simpleGit(repoPath).revparse(["--show-toplevel"]); + return simpleGit(repoPath).revparse(["--show-toplevel"]); } export async function cloneRepo(rootUrl: string, cloneDir: string, repo: string): Promise { @@ -52,7 +52,7 @@ export async function cloneRepo(rootUrl: string, cloneDir: string, repo: string) } export async function checkoutCommit(cloneDir: string, commit: string): Promise { - await simpleGit(cloneDir).checkout(commit, undefined, (err) => { + simpleGit(cloneDir).checkout(commit, undefined, (err) => { if (err) { throw err; } From d454589450a3b587477f97ac092d6b06a3c3fc67 Mon Sep 17 00:00:00 2001 From: Catalina Peralta Date: Mon, 30 Oct 2023 17:49:46 -0700 Subject: [PATCH 03/11] update undefined checks --- tools/tsp-client/src/index.ts | 12 ++++++------ tools/tsp-client/src/options.ts | 2 +- tools/tsp-client/src/typespec.ts | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tools/tsp-client/src/index.ts b/tools/tsp-client/src/index.ts index 1084da21230..be53b9d23aa 100644 --- a/tools/tsp-client/src/index.ts +++ b/tools/tsp-client/src/index.ts @@ -35,7 +35,7 @@ async function sdkInit( const configYaml = parseYaml(tspConfig); if (configYaml["parameters"] && configYaml["parameters"]["service-dir"]){ const serviceDir = configYaml["parameters"]["service-dir"]["default"]; - if (serviceDir === undefined) { + if (!serviceDir) { Logger.error(`Parameter service-dir is not defined correctly in tspconfig.yaml. Please refer to https://github.com/Azure/azure-rest-api-specs/blob/main/specification/contosowidgetmanager/Contoso.WidgetManager/tspconfig.yaml for the right schema.`) } Logger.debug(`Service directory: ${serviceDir}`) @@ -69,7 +69,7 @@ async function sdkInit( if (configYaml["options"][emitter] && configYaml["options"][emitter]["package-dir"]) { packageDir = configYaml["options"][emitter]["package-dir"]; } - if (packageDir === undefined) { + if (!packageDir) { throw new Error(`Missing package-dir in ${emitter} options of tspconfig.yaml. Please refer to https://github.com/Azure/azure-rest-api-specs/blob/main/specification/contosowidgetmanager/Contoso.WidgetManager/tspconfig.yaml for the right schema.`); } const newPackageDir = path.join(outputDir, serviceDir, packageDir) @@ -96,14 +96,14 @@ async function syncTspFiles(outputDir: string, localSpecRepo?: string) { const repoRoot = await getRepoRoot(outputDir); Logger.debug(`Repo root is ${repoRoot}`); - if (repoRoot === undefined) { + if (!repoRoot) { throw new Error("Could not find repo root"); } const [ directory, commit, repo, additionalDirectories ] = await readTspLocation(outputDir); const dirSplit = directory.split("/"); let projectName = dirSplit[dirSplit.length - 1]; Logger.debug(`Using project name: ${projectName}`) - if (projectName === undefined) { + if (!projectName) { projectName = "src"; } const srcDir = path.join(tempRoot, projectName); @@ -123,7 +123,7 @@ async function syncTspFiles(outputDir: string, localSpecRepo?: string) { await cp(localSpecRepo, srcDir, { recursive: true, filter: filter }); const localSpecRepoRoot = await getRepoRoot(localSpecRepo); Logger.info(`Local spec repo root is ${localSpecRepoRoot}`) - if (localSpecRepoRoot === undefined) { + if (!localSpecRepoRoot) { throw new Error("Could not find local spec repo root, please make sure the path is correct"); } for (const dir of additionalDirectories) { @@ -171,7 +171,7 @@ async function generate({ const tspLocation = await readTspLocation(rootUrl); const dirSplit = tspLocation[0].split("/"); let projectName = dirSplit[dirSplit.length - 1]; - if (projectName === undefined) { + if (!projectName) { throw new Error("cannot find project name"); } const srcDir = path.join(tempRoot, projectName); diff --git a/tools/tsp-client/src/options.ts b/tools/tsp-client/src/options.ts index d756054dfd7..94bd92d3601 100644 --- a/tools/tsp-client/src/options.ts +++ b/tools/tsp-client/src/options.ts @@ -95,7 +95,7 @@ export async function getOptions(): Promise { isUrl = true; } if (!isUrl) { - if (values.commit === undefined || values.repo === undefined) { + if (!values.commit || !values.repo) { Logger.error("The commit and repo options are required when tspConfig is a local directory"); printUsage(); process.exit(1); diff --git a/tools/tsp-client/src/typespec.ts b/tools/tsp-client/src/typespec.ts index 3c00fc69d0d..4260724f8ff 100644 --- a/tools/tsp-client/src/typespec.ts +++ b/tools/tsp-client/src/typespec.ts @@ -28,7 +28,7 @@ export async function getEmitterOptions(rootUrl: string, tempRoot: string, emitt emitterOptions[emitter]!["emitter-output-dir"] = rootUrl; } if (saveInputs) { - if (emitterOptions[emitter] === undefined) { + if (!emitterOptions[emitter]) { emitterOptions[emitter] = {}; } emitterOptions[emitter]!["save-inputs"] = true; From 4f1d39af1010058ed89aebfe314287692ecd9b5c Mon Sep 17 00:00:00 2001 From: Catalina Peralta Date: Mon, 30 Oct 2023 18:29:45 -0700 Subject: [PATCH 04/11] fix chaining --- tools/tsp-client/src/index.ts | 49 ++++++++++++++++------------------- 1 file changed, 22 insertions(+), 27 deletions(-) diff --git a/tools/tsp-client/src/index.ts b/tools/tsp-client/src/index.ts index be53b9d23aa..b1b4c67dfb1 100644 --- a/tools/tsp-client/src/index.ts +++ b/tools/tsp-client/src/index.ts @@ -27,14 +27,14 @@ async function sdkInit( repo: string | undefined; isUrl: boolean; }): Promise { - if (isUrl) { - // URL scenario - const resolvedConfigUrl = resolveTspConfigUrl(config); - Logger.debug(`Resolved config url: ${resolvedConfigUrl.resolvedUrl}`) - const tspConfig = await fetch(resolvedConfigUrl.resolvedUrl); - const configYaml = parseYaml(tspConfig); - if (configYaml["parameters"] && configYaml["parameters"]["service-dir"]){ - const serviceDir = configYaml["parameters"]["service-dir"]["default"]; + try { + if (isUrl) { + // URL scenario + const resolvedConfigUrl = resolveTspConfigUrl(config); + Logger.debug(`Resolved config url: ${resolvedConfigUrl.resolvedUrl}`) + const tspConfig = await fetch(resolvedConfigUrl.resolvedUrl); + const configYaml = parseYaml(tspConfig); + const serviceDir = configYaml?.parameters?.["service-dir"]?.default; if (!serviceDir) { Logger.error(`Parameter service-dir is not defined correctly in tspconfig.yaml. Please refer to https://github.com/Azure/azure-rest-api-specs/blob/main/specification/contosowidgetmanager/Contoso.WidgetManager/tspconfig.yaml for the right schema.`) } @@ -51,24 +51,18 @@ async function sdkInit( `directory: ${resolvedConfigUrl.path}\ncommit: ${resolvedConfigUrl.commit}\nrepo: ${resolvedConfigUrl.repo}\nadditionalDirectories: ${additionalDirs}`); return newPackageDir; } else { - Logger.error("Missing service-dir in parameters section of tspconfig.yaml. Please refer to https://github.com/Azure/azure-rest-api-specs/blob/main/specification/contosowidgetmanager/Contoso.WidgetManager/tspconfig.yaml for the right schema.") - } - } else { - // Local directory scenario - let configFile = path.join(config, "tspconfig.yaml") - const data = await readFile(configFile, "utf8"); - const configYaml = parseYaml(data); - if (configYaml["parameters"] && configYaml["parameters"]["service-dir"]) { - const serviceDir = configYaml["parameters"]["service-dir"]["default"]; - var additionalDirs: string[] = []; - if (configYaml["parameters"]["dependencies"] && configYaml["parameters"]["dependencies"]["additionalDirectories"]) { - additionalDirs = configYaml["parameters"]["dependencies"]["additionalDirectories"]; + // Local directory scenario + let configFile = path.join(config, "tspconfig.yaml") + const data = await readFile(configFile, "utf8"); + const configYaml = parseYaml(data); + const serviceDir = configYaml?.parameters?.["service-dir"]?.default; + if (!serviceDir) { + Logger.error(`Parameter service-dir is not defined correctly in tspconfig.yaml. Please refer to https://github.com/Azure/azure-rest-api-specs/blob/main/specification/contosowidgetmanager/Contoso.WidgetManager/tspconfig.yaml for the right schema.`) } + Logger.debug(`Service directory: ${serviceDir}`) + const additionalDirs: string[] = configYaml?.parameters?.dependencies?.additionalDirectories ?? []; Logger.info(`Additional directories: ${additionalDirs}`) - let packageDir: string | undefined = undefined; - if (configYaml["options"][emitter] && configYaml["options"][emitter]["package-dir"]) { - packageDir = configYaml["options"][emitter]["package-dir"]; - } + const packageDir = configYaml?.options?.[emitter]?.["package-dir"]; if (!packageDir) { throw new Error(`Missing package-dir in ${emitter} options of tspconfig.yaml. Please refer to https://github.com/Azure/azure-rest-api-specs/blob/main/specification/contosowidgetmanager/Contoso.WidgetManager/tspconfig.yaml for the right schema.`); } @@ -86,9 +80,10 @@ async function sdkInit( `directory: ${directory}\ncommit: ${commit}\nrepo: ${repo}\nadditionalDirectories: ${additionalDirs}`); return newPackageDir; } - throw new Error("Missing service-dir in parameters section of tspconfig.yaml. Please refer to https://github.com/Azure/azure-rest-api-specs/blob/main/specification/contosowidgetmanager/Contoso.WidgetManager/tspconfig.yaml for the right schema.") - } - throw new Error("Invalid tspconfig.yaml. Please refer to https://github.com/Azure/azure-rest-api-specs/blob/main/specification/contosowidgetmanager/Contoso.WidgetManager/tspconfig.yaml for the right schema."); + } catch (err) { + Logger.error("Invalid tspconfig.yaml. Please refer to https://github.com/Azure/azure-rest-api-specs/blob/main/specification/contosowidgetmanager/Contoso.WidgetManager/tspconfig.yaml for the right schema."); + throw err; + } } async function syncTspFiles(outputDir: string, localSpecRepo?: string) { From 24520798b13b6cae893be3b7f9c65f5a67541fb9 Mon Sep 17 00:00:00 2001 From: Catalina Peralta Date: Mon, 30 Oct 2023 18:46:30 -0700 Subject: [PATCH 05/11] remove fileTree.ts --- tools/tsp-client/src/fileTree.ts | 79 -------------------------------- 1 file changed, 79 deletions(-) delete mode 100644 tools/tsp-client/src/fileTree.ts diff --git a/tools/tsp-client/src/fileTree.ts b/tools/tsp-client/src/fileTree.ts deleted file mode 100644 index cd10076a254..00000000000 --- a/tools/tsp-client/src/fileTree.ts +++ /dev/null @@ -1,79 +0,0 @@ -export function createFileTree(url: string): FileTree { - const rootUrl = url; - const fileMap = new Map(); - - function longestCommonPrefix(a: string, b: string): string { - if (a === b) { - return a; - } - let lastCommonSlash = -1; - for (let i = 0; i < Math.min(a.length, b.length); i++) { - if (a[i] === b[i]) { - if (a[i] === "/") { - lastCommonSlash = i; - } - } else { - break; - } - } - if (lastCommonSlash === -1) { - throw new Error("no common prefix found"); - } - return a.slice(0, lastCommonSlash + 1); - } - - function findCommonRoot(): string { - let candidate = ""; - for (const fileUrl of fileMap.keys()) { - const lastSlashIndex = fileUrl.lastIndexOf("/"); - const dirUrl = fileUrl.slice(0, lastSlashIndex + 1); - if (!candidate) { - candidate = dirUrl; - } else { - candidate = longestCommonPrefix(candidate, dirUrl); - } - } - return candidate; - } - - return { - addFile(url: string, contents: string): void { - if (fileMap.has(url)) { - throw new Error(`file already parsed: ${url}`); - } - fileMap.set(url, contents); - }, - async createTree(): Promise { - const outputFiles = new Map(); - // calculate the highest common root - const root = findCommonRoot(); - let mainFilePath = ""; - for (const [url, contents] of fileMap.entries()) { - const relativePath = url.slice(root.length); - outputFiles.set(relativePath, contents); - if (url === rootUrl) { - mainFilePath = relativePath; - } - } - if (!mainFilePath) { - throw new RangeError( - `Main file ${rootUrl} not added to FileTree. Did you forget to add it?`, - ); - } - return { - mainFilePath, - files: outputFiles, - }; - }, - }; -} - -export interface FileTreeResult { - mainFilePath: string; - files: Map; -} - -export interface FileTree { - addFile(url: string, contents: string): void; - createTree(): Promise; -} From a1647df4e98e84f3cd5d8773f14dbb01fb784ca4 Mon Sep 17 00:00:00 2001 From: Catalina Peralta Date: Mon, 30 Oct 2023 18:52:39 -0700 Subject: [PATCH 06/11] add TspLocation interface --- tools/tsp-client/src/fs.ts | 47 ++++++-------------------------- tools/tsp-client/src/index.ts | 42 ++++++++++++++-------------- tools/tsp-client/src/typespec.ts | 7 +++++ 3 files changed, 36 insertions(+), 60 deletions(-) diff --git a/tools/tsp-client/src/fs.ts b/tools/tsp-client/src/fs.ts index d874232762c..30b8c9ab005 100644 --- a/tools/tsp-client/src/fs.ts +++ b/tools/tsp-client/src/fs.ts @@ -1,8 +1,8 @@ -import { mkdir, rm, writeFile, stat, readFile, access } from "node:fs/promises"; -import { FileTreeResult } from "./fileTree.js"; +import { mkdir, rm, stat, readFile, access } from "node:fs/promises"; import * as path from "node:path"; import { Logger } from "./log.js"; import { parse as parseYaml } from "yaml"; +import { TspLocation } from "./typespec.js"; export async function ensureDirectory(path: string) { await mkdir(path, { recursive: true }); @@ -19,51 +19,20 @@ export async function createTempDirectory(outputDir: string): Promise { return tempRoot; } -export async function writeFileTree(rootDir: string, files: FileTreeResult["files"]) { - for (const [relativeFilePath, contents] of files) { - const filePath = path.join(rootDir, relativeFilePath); - await ensureDirectory(path.dirname(filePath)); - Logger.debug(`writing ${filePath}`); - await writeFile(filePath, contents); - } -} - -export async function tryReadTspLocation(rootDir: string): Promise { - try { - const yamlPath = path.resolve(rootDir, "tsp-location.yaml"); - const fileStat = await stat(yamlPath); - if (fileStat.isFile()) { - const fileContents = await readFile(yamlPath, "utf8"); - const locationYaml = parseYaml(fileContents); - const { directory, commit, repo } = locationYaml; - if (!directory || !commit || !repo) { - throw new Error("Invalid tsp-location.yaml"); - } - // make GitHub URL - return `https://raw.githubusercontent.com/${repo}/${commit}/${directory}/`; - } - } catch (e) { - Logger.error(`Error reading tsp-location.yaml: ${e}`); - } - return undefined; -} - -export async function readTspLocation(rootDir: string): Promise<[string, string, string, string[]]> { +export async function readTspLocation(rootDir: string): Promise { try { const yamlPath = path.resolve(rootDir, "tsp-location.yaml"); const fileStat = await stat(yamlPath); if (fileStat.isFile()) { const fileContents = await readFile(yamlPath, "utf8"); - const locationYaml = parseYaml(fileContents); - let { directory, commit, repo, additionalDirectories } = locationYaml; - if (!directory || !commit || !repo) { + const tspLocation: TspLocation = parseYaml(fileContents); + if (!tspLocation.directory || !tspLocation.commit || !tspLocation.repo) { throw new Error("Invalid tsp-location.yaml"); } - Logger.info(`Additional directories: ${additionalDirectories}`) - if (!additionalDirectories) { - additionalDirectories = []; + if (!tspLocation.additionalDirectories) { + tspLocation.additionalDirectories = []; } - return [ directory, commit, repo, additionalDirectories ]; + return tspLocation; } throw new Error("Could not find tsp-location.yaml"); } catch (e) { diff --git a/tools/tsp-client/src/index.ts b/tools/tsp-client/src/index.ts index b1b4c67dfb1..4a9156b48ba 100644 --- a/tools/tsp-client/src/index.ts +++ b/tools/tsp-client/src/index.ts @@ -1,9 +1,9 @@ import * as path from "node:path"; import { installDependencies } from "./npm.js"; -import { createTempDirectory, removeDirectory,readTspLocation, getEmitterFromRepoConfig } from "./fs.js"; +import { createTempDirectory, removeDirectory, readTspLocation, getEmitterFromRepoConfig } from "./fs.js"; import { Logger, printBanner, enableDebug, printVersion } from "./log.js"; -import { compileTsp, discoverMainFile, getEmitterOptions, resolveTspConfigUrl } from "./typespec.js"; +import { TspLocation, compileTsp, discoverMainFile, getEmitterOptions, resolveTspConfigUrl } from "./typespec.js"; import { getOptions } from "./options.js"; import { mkdir, writeFile, cp, readFile } from "node:fs/promises"; import { addSpecFiles, checkoutCommit, cloneRepo, getRepoRoot, sparseCheckout } from "./git.js"; @@ -94,8 +94,8 @@ async function syncTspFiles(outputDir: string, localSpecRepo?: string) { if (!repoRoot) { throw new Error("Could not find repo root"); } - const [ directory, commit, repo, additionalDirectories ] = await readTspLocation(outputDir); - const dirSplit = directory.split("/"); + const tspLocation: TspLocation = await readTspLocation(outputDir); + const dirSplit = tspLocation.directory.split("/"); let projectName = dirSplit[dirSplit.length - 1]; Logger.debug(`Using project name: ${projectName}`) if (!projectName) { @@ -121,7 +121,7 @@ async function syncTspFiles(outputDir: string, localSpecRepo?: string) { if (!localSpecRepoRoot) { throw new Error("Could not find local spec repo root, please make sure the path is correct"); } - for (const dir of additionalDirectories) { + for (const dir of tspLocation.additionalDirectories!) { await cp(path.join(localSpecRepoRoot, dir), srcDir, { recursive: true, filter: filter }); } } else { @@ -129,16 +129,16 @@ async function syncTspFiles(outputDir: string, localSpecRepo?: string) { await mkdir(cloneDir, { recursive: true }); Logger.debug(`Created temporary sparse-checkout directory ${cloneDir}`); Logger.debug(`Cloning repo to ${cloneDir}`); - await cloneRepo(tempRoot, cloneDir, `https://github.com/${repo}.git`); + await cloneRepo(tempRoot, cloneDir, `https://github.com/${tspLocation.repo}.git`); await sparseCheckout(cloneDir); - await addSpecFiles(cloneDir, directory) - Logger.info(`Processing additional directories: ${additionalDirectories}`) - for (const dir of additionalDirectories) { + await addSpecFiles(cloneDir, tspLocation.directory) + Logger.info(`Processing additional directories: ${tspLocation.additionalDirectories}`) + for (const dir of tspLocation.additionalDirectories!) { await addSpecFiles(cloneDir, dir); } - await checkoutCommit(cloneDir, commit); - await cp(path.join(cloneDir, directory), srcDir, { recursive: true }); - for (const dir of additionalDirectories) { + await checkoutCommit(cloneDir, tspLocation.commit); + await cp(path.join(cloneDir, tspLocation.directory), srcDir, { recursive: true }); + for (const dir of tspLocation.additionalDirectories!) { const dirSplit = dir.split("/"); let projectName = dirSplit[dirSplit.length - 1]; const dirName = path.join(tempRoot, projectName!); @@ -164,7 +164,7 @@ async function generate({ }) { const tempRoot = path.join(rootUrl, "TempTypeSpecFiles"); const tspLocation = await readTspLocation(rootUrl); - const dirSplit = tspLocation[0].split("/"); + const dirSplit = tspLocation.directory.split("/"); let projectName = dirSplit[dirSplit.length - 1]; if (!projectName) { throw new Error("cannot find project name"); @@ -229,17 +229,17 @@ async function main() { throw new Error("Commit SHA is required when specifying `--repo`, please specify a commit using `--commit`"); } if (options.commit) { - let [ directory, commit, repo, additionalDirectories ] = await readTspLocation(rootUrl); - commit = options.commit ?? commit; - repo = options.repo ?? repo; - await writeFile(path.join(rootUrl, "tsp-location.yaml"), `directory: ${directory}\ncommit: ${commit}\nrepo: ${repo}\nadditionalDirectories: ${additionalDirectories}`); + let tspLocation: TspLocation = await readTspLocation(rootUrl); + tspLocation.commit = options.commit ?? tspLocation.commit; + tspLocation.repo = options.repo ?? tspLocation.repo; + await writeFile(path.join(rootUrl, "tsp-location.yaml"), `directory: ${tspLocation.directory}\ncommit: ${tspLocation.commit}\nrepo: ${tspLocation.repo}\nadditionalDirectories: ${tspLocation.additionalDirectories}`); } if (options.tspConfig) { - let [ directory, commit, repo, additionalDirectories ] = await readTspLocation(rootUrl); + let tspLocation: TspLocation = await readTspLocation(rootUrl); let tspConfig = resolveTspConfigUrl(options.tspConfig); - commit = tspConfig.commit ?? commit; - repo = tspConfig.repo ?? repo; - await writeFile(path.join(rootUrl, "tsp-location.yaml"), `directory: ${directory}\ncommit: ${commit}\nrepo: ${repo}\nadditionalDirectories: ${additionalDirectories}`); + tspLocation.commit = tspConfig.commit ?? tspLocation.commit; + tspLocation.repo = tspConfig.repo ?? tspLocation.repo; + await writeFile(path.join(rootUrl, "tsp-location.yaml"), `directory: ${tspLocation.directory}\ncommit: ${tspLocation.commit}\nrepo: ${tspLocation.repo}\nadditionalDirectories: ${tspLocation.additionalDirectories}`); } await syncTspFiles(rootUrl); await generate({ rootUrl, noCleanup: options.noCleanup, additionalEmitterOptions: options.emitterOptions}); diff --git a/tools/tsp-client/src/typespec.ts b/tools/tsp-client/src/typespec.ts index 4260724f8ff..bd962d27d62 100644 --- a/tools/tsp-client/src/typespec.ts +++ b/tools/tsp-client/src/typespec.ts @@ -6,6 +6,13 @@ import * as path from "node:path"; import { parse as parseYaml } from "yaml"; +export interface TspLocation { + directory: string; + commit: string; + repo: string; + additionalDirectories?: string[]; +} + export async function getEmitterOptions(rootUrl: string, tempRoot: string, emitter: string, saveInputs: boolean, additionalOptions?: string): Promise> { // TODO: Add a way to specify emitter options like Language-Settings.ps1, could be a languageSettings.ts file let emitterOptions: Record> = {}; From 5f4de6b78194116e7ebc880133b90d7c06f5af02 Mon Sep 17 00:00:00 2001 From: Catalina Peralta Date: Mon, 30 Oct 2023 19:35:06 -0700 Subject: [PATCH 07/11] clean up network.ts --- tools/tsp-client/src/network.ts | 66 --------------------------------- 1 file changed, 66 deletions(-) diff --git a/tools/tsp-client/src/network.ts b/tools/tsp-client/src/network.ts index dabac0e22a3..504f9c07110 100644 --- a/tools/tsp-client/src/network.ts +++ b/tools/tsp-client/src/network.ts @@ -1,7 +1,4 @@ import { createDefaultHttpClient, createPipelineRequest } from "@azure/core-rest-pipeline"; -import { createFileTree } from "./fileTree.js"; -import { resolveImports } from "./typespec.js"; -import { Logger } from "./log.js"; const httpClient = createDefaultHttpClient(); @@ -27,66 +24,3 @@ export function doesFileExist(url: string): Promise { .then(() => true) .catch(() => false); } - -export function rewriteGitHubUrl(url: string): string { - if (url.includes("github.com")) { - const result = url.replace("github.com", "raw.githubusercontent.com"); - Logger.debug(`rewriting github url to direct link: ${result}`); - return result; - } - return url; -} - -export async function downloadTsp(rootUrl: string) { - const seenFiles = new Set(); - const moduleImports = new Set(); - // fetch the root file - const filesToProcess = [rootUrl]; - seenFiles.add(rootUrl); - const fileTree = createFileTree(rootUrl); - - while (filesToProcess.length > 0) { - const url = filesToProcess.shift()!; - const sourceFile = await fetch(url); - fileTree.addFile(url, sourceFile); - // process imports, fetching any relatively referenced files - const imports = await resolveImports(sourceFile); - for (const fileImport of imports) { - // Check if the module name is referencing a path(./foo, /foo, file:/foo) - if (/^(?:\.\.?(?:\/|$)|\/|([A-Za-z]:)?[/\\])/.test(fileImport)) { - if (fileImport.startsWith("file:")) { - throw new Error(`file protocol imports are not supported: ${fileImport}`); - } - let resolvedImport: string; - if (fileImport.startsWith("http:")) { - throw new Error(`absolute url imports are not supported: ${fileImport}`); - } else { - resolvedImport = new URL(fileImport, url).toString(); - } - if (!seenFiles.has(resolvedImport)) { - Logger.debug(`discovered import ${resolvedImport}`); - filesToProcess.push(resolvedImport); - seenFiles.add(resolvedImport); - } - } else { - Logger.debug(`discovered module import ${fileImport}`); - moduleImports.add(fileImport); - } - } - } - - // look for a tspconfig.yaml next to the root - try { - const tspConfigUrl = new URL("tspconfig.yaml", rootUrl).toString(); - const tspConfig = await fetch(tspConfigUrl); - Logger.debug("found tspconfig.yaml"); - fileTree.addFile(tspConfigUrl, tspConfig); - } catch (e) { - Logger.debug("no tspconfig.yaml found"); - } - - return { - moduleImports, - fileTree, - }; -} From 1bfdcda67f75edad7ad0f98f648535b2c0396238 Mon Sep 17 00:00:00 2001 From: Catalina Peralta Date: Thu, 9 Nov 2023 17:24:41 -0800 Subject: [PATCH 08/11] fix async funcs --- tools/tsp-client/src/git.ts | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/tools/tsp-client/src/git.ts b/tools/tsp-client/src/git.ts index 6feed847bae..04f0253c413 100644 --- a/tools/tsp-client/src/git.ts +++ b/tools/tsp-client/src/git.ts @@ -6,10 +6,13 @@ export async function getRepoRoot(repoPath: string): Promise { } export async function cloneRepo(rootUrl: string, cloneDir: string, repo: string): Promise { - simpleGit(rootUrl).clone(repo, cloneDir, ["--no-checkout", "--filter=tree:0"], (err) => { - if (err) { - throw err; - } + return new Promise((resolve, reject) => { + simpleGit(rootUrl).clone(repo, cloneDir, ["--no-checkout", "--filter=tree:0"], (err) => { + if (err) { + reject(err); + } + }); + resolve(); }); } @@ -52,9 +55,12 @@ export async function cloneRepo(rootUrl: string, cloneDir: string, repo: string) } export async function checkoutCommit(cloneDir: string, commit: string): Promise { - simpleGit(cloneDir).checkout(commit, undefined, (err) => { - if (err) { - throw err; - } + return new Promise((resolve, reject) => { + simpleGit(cloneDir).checkout(commit, undefined, (err) => { + if (err) { + reject(err); + } + }); + resolve(); }); } From 1aecd12c615d6cdcd4d0fa1f3466baeaa29cffc4 Mon Sep 17 00:00:00 2001 From: Catalina Peralta Date: Thu, 9 Nov 2023 17:56:49 -0800 Subject: [PATCH 09/11] revert init change --- tools/tsp-client/src/index.ts | 105 ++++++++++++++++------------------ 1 file changed, 50 insertions(+), 55 deletions(-) diff --git a/tools/tsp-client/src/index.ts b/tools/tsp-client/src/index.ts index 4a9156b48ba..12407f992f3 100644 --- a/tools/tsp-client/src/index.ts +++ b/tools/tsp-client/src/index.ts @@ -27,63 +27,58 @@ async function sdkInit( repo: string | undefined; isUrl: boolean; }): Promise { - try { - if (isUrl) { - // URL scenario - const resolvedConfigUrl = resolveTspConfigUrl(config); - Logger.debug(`Resolved config url: ${resolvedConfigUrl.resolvedUrl}`) - const tspConfig = await fetch(resolvedConfigUrl.resolvedUrl); - const configYaml = parseYaml(tspConfig); - const serviceDir = configYaml?.parameters?.["service-dir"]?.default; - if (!serviceDir) { - Logger.error(`Parameter service-dir is not defined correctly in tspconfig.yaml. Please refer to https://github.com/Azure/azure-rest-api-specs/blob/main/specification/contosowidgetmanager/Contoso.WidgetManager/tspconfig.yaml for the right schema.`) - } - Logger.debug(`Service directory: ${serviceDir}`) - const additionalDirs: string[] = configYaml?.parameters?.dependencies?.additionalDirectories ?? []; - const packageDir: string | undefined = configYaml?.options?.[emitter]?.["package-dir"]; - if (!packageDir) { - Logger.error(`Missing package-dir in ${emitter} options of tspconfig.yaml. Please refer to https://github.com/Azure/azure-rest-api-specs/blob/main/specification/contosowidgetmanager/Contoso.WidgetManager/tspconfig.yaml for the right schema.`); - } - const newPackageDir = path.join(outputDir, serviceDir, packageDir!) - await mkdir(newPackageDir, { recursive: true }); - await writeFile( - path.join(newPackageDir, "tsp-location.yaml"), - `directory: ${resolvedConfigUrl.path}\ncommit: ${resolvedConfigUrl.commit}\nrepo: ${resolvedConfigUrl.repo}\nadditionalDirectories: ${additionalDirs}`); - return newPackageDir; - } else { - // Local directory scenario - let configFile = path.join(config, "tspconfig.yaml") - const data = await readFile(configFile, "utf8"); - const configYaml = parseYaml(data); - const serviceDir = configYaml?.parameters?.["service-dir"]?.default; - if (!serviceDir) { - Logger.error(`Parameter service-dir is not defined correctly in tspconfig.yaml. Please refer to https://github.com/Azure/azure-rest-api-specs/blob/main/specification/contosowidgetmanager/Contoso.WidgetManager/tspconfig.yaml for the right schema.`) - } - Logger.debug(`Service directory: ${serviceDir}`) - const additionalDirs: string[] = configYaml?.parameters?.dependencies?.additionalDirectories ?? []; - Logger.info(`Additional directories: ${additionalDirs}`) - const packageDir = configYaml?.options?.[emitter]?.["package-dir"]; - if (!packageDir) { - throw new Error(`Missing package-dir in ${emitter} options of tspconfig.yaml. Please refer to https://github.com/Azure/azure-rest-api-specs/blob/main/specification/contosowidgetmanager/Contoso.WidgetManager/tspconfig.yaml for the right schema.`); - } - const newPackageDir = path.join(outputDir, serviceDir, packageDir) - await mkdir(newPackageDir, { recursive: true }); - configFile = configFile.replaceAll("\\", "/"); - const matchRes = configFile.match('.*/(?specification/.*)/tspconfig.yaml$') - var directory = ""; - if (matchRes) { - if (matchRes.groups) { - directory = matchRes.groups!["path"]!; - } + if (isUrl) { + // URL scenario + const resolvedConfigUrl = resolveTspConfigUrl(config); + Logger.debug(`Resolved config url: ${resolvedConfigUrl.resolvedUrl}`) + const tspConfig = await fetch(resolvedConfigUrl.resolvedUrl); + const configYaml = parseYaml(tspConfig); + const serviceDir = configYaml?.parameters?.["service-dir"]?.default; + if (!serviceDir) { + Logger.error(`Parameter service-dir is not defined correctly in tspconfig.yaml. Please refer to https://github.com/Azure/azure-rest-api-specs/blob/main/specification/contosowidgetmanager/Contoso.WidgetManager/tspconfig.yaml for the right schema.`) + } + Logger.debug(`Service directory: ${serviceDir}`) + const additionalDirs: string[] = configYaml?.parameters?.dependencies?.additionalDirectories ?? []; + const packageDir: string | undefined = configYaml?.options?.[emitter]?.["package-dir"]; + if (!packageDir) { + Logger.error(`Missing package-dir in ${emitter} options of tspconfig.yaml. Please refer to https://github.com/Azure/azure-rest-api-specs/blob/main/specification/contosowidgetmanager/Contoso.WidgetManager/tspconfig.yaml for the right schema.`); + } + const newPackageDir = path.join(outputDir, serviceDir, packageDir!) + await mkdir(newPackageDir, { recursive: true }); + await writeFile( + path.join(newPackageDir, "tsp-location.yaml"), + `directory: ${resolvedConfigUrl.path}\ncommit: ${resolvedConfigUrl.commit}\nrepo: ${resolvedConfigUrl.repo}\nadditionalDirectories: ${additionalDirs}`); + return newPackageDir; + } else { + // Local directory scenario + let configFile = path.join(config, "tspconfig.yaml") + const data = await readFile(configFile, "utf8"); + const configYaml = parseYaml(data); + const serviceDir = configYaml?.parameters?.["service-dir"]?.default; + if (!serviceDir) { + Logger.error(`Parameter service-dir is not defined correctly in tspconfig.yaml. Please refer to https://github.com/Azure/azure-rest-api-specs/blob/main/specification/contosowidgetmanager/Contoso.WidgetManager/tspconfig.yaml for the right schema.`) + } + Logger.debug(`Service directory: ${serviceDir}`) + const additionalDirs: string[] = configYaml?.parameters?.dependencies?.additionalDirectories ?? []; + Logger.info(`Additional directories: ${additionalDirs}`) + const packageDir = configYaml?.options?.[emitter]?.["package-dir"]; + if (!packageDir) { + throw new Error(`Missing package-dir in ${emitter} options of tspconfig.yaml. Please refer to https://github.com/Azure/azure-rest-api-specs/blob/main/specification/contosowidgetmanager/Contoso.WidgetManager/tspconfig.yaml for the right schema.`); + } + const newPackageDir = path.join(outputDir, serviceDir, packageDir) + await mkdir(newPackageDir, { recursive: true }); + configFile = configFile.replaceAll("\\", "/"); + const matchRes = configFile.match('.*/(?specification/.*)/tspconfig.yaml$') + var directory = ""; + if (matchRes) { + if (matchRes.groups) { + directory = matchRes.groups!["path"]!; } - writeFile(path.join(newPackageDir, "tsp-location.yaml"), - `directory: ${directory}\ncommit: ${commit}\nrepo: ${repo}\nadditionalDirectories: ${additionalDirs}`); - return newPackageDir; } - } catch (err) { - Logger.error("Invalid tspconfig.yaml. Please refer to https://github.com/Azure/azure-rest-api-specs/blob/main/specification/contosowidgetmanager/Contoso.WidgetManager/tspconfig.yaml for the right schema."); - throw err; - } + writeFile(path.join(newPackageDir, "tsp-location.yaml"), + `directory: ${directory}\ncommit: ${commit}\nrepo: ${repo}\nadditionalDirectories: ${additionalDirs}`); + return newPackageDir; + } } async function syncTspFiles(outputDir: string, localSpecRepo?: string) { From 7d4f5e9269486d3192efba907c458a67aced9188 Mon Sep 17 00:00:00 2001 From: Catalina Peralta Date: Wed, 15 Nov 2023 19:26:41 -0800 Subject: [PATCH 10/11] fix git functions --- tools/tsp-client/src/git.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/tsp-client/src/git.ts b/tools/tsp-client/src/git.ts index 04f0253c413..e14d3e75228 100644 --- a/tools/tsp-client/src/git.ts +++ b/tools/tsp-client/src/git.ts @@ -11,8 +11,8 @@ export async function cloneRepo(rootUrl: string, cloneDir: string, repo: string) if (err) { reject(err); } + resolve(); }); - resolve(); }); } @@ -60,7 +60,7 @@ export async function checkoutCommit(cloneDir: string, commit: string): Promise< if (err) { reject(err); } + resolve(); }); - resolve(); }); } From 011622971292fd5c739397d66e4bbaccd64c53fb Mon Sep 17 00:00:00 2001 From: Catalina Peralta Date: Wed, 15 Nov 2023 19:31:42 -0800 Subject: [PATCH 11/11] use const --- tools/tsp-client/src/index.ts | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tools/tsp-client/src/index.ts b/tools/tsp-client/src/index.ts index 12407f992f3..a6c23b49f23 100644 --- a/tools/tsp-client/src/index.ts +++ b/tools/tsp-client/src/index.ts @@ -224,14 +224,13 @@ async function main() { throw new Error("Commit SHA is required when specifying `--repo`, please specify a commit using `--commit`"); } if (options.commit) { - let tspLocation: TspLocation = await readTspLocation(rootUrl); + const tspLocation: TspLocation = await readTspLocation(rootUrl); tspLocation.commit = options.commit ?? tspLocation.commit; tspLocation.repo = options.repo ?? tspLocation.repo; await writeFile(path.join(rootUrl, "tsp-location.yaml"), `directory: ${tspLocation.directory}\ncommit: ${tspLocation.commit}\nrepo: ${tspLocation.repo}\nadditionalDirectories: ${tspLocation.additionalDirectories}`); - } - if (options.tspConfig) { - let tspLocation: TspLocation = await readTspLocation(rootUrl); - let tspConfig = resolveTspConfigUrl(options.tspConfig); + } else if (options.tspConfig) { + const tspLocation: TspLocation = await readTspLocation(rootUrl); + const tspConfig = resolveTspConfigUrl(options.tspConfig); tspLocation.commit = tspConfig.commit ?? tspLocation.commit; tspLocation.repo = tspConfig.repo ?? tspLocation.repo; await writeFile(path.join(rootUrl, "tsp-location.yaml"), `directory: ${tspLocation.directory}\ncommit: ${tspLocation.commit}\nrepo: ${tspLocation.repo}\nadditionalDirectories: ${tspLocation.additionalDirectories}`);