Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[tsp-client] Code fixes #7202

Merged
merged 11 commits into from
Nov 16, 2023
Merged

[tsp-client] Code fixes #7202

Show file tree
Hide file tree
Changes from 8 commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
8783442
fix cloneRepo
catalinaperalta Oct 26, 2023
a60e394
fix git functions
catalinaperalta Oct 26, 2023
d454589
update undefined checks
catalinaperalta Oct 31, 2023
4f1d39a
fix chaining
catalinaperalta Oct 31, 2023
2452079
remove fileTree.ts
catalinaperalta Oct 31, 2023
a1647df
add TspLocation interface
catalinaperalta Oct 31, 2023
5f4de6b
clean up network.ts
catalinaperalta Oct 31, 2023
1bfdcda
fix async funcs
catalinaperalta Nov 10, 2023
1aecd12
revert init change
catalinaperalta Nov 10, 2023
7d4f5e9
fix git functions
catalinaperalta Nov 16, 2023
0116229
use const
catalinaperalta Nov 16, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
79 changes: 0 additions & 79 deletions tools/tsp-client/src/fileTree.ts
View file
Open in desktop

This file was deleted.

47 changes: 8 additions & 39 deletions tools/tsp-client/src/fs.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
import { mkdir, rm, writeFile, stat, readFile, access } from "node:fs/promises";
import { FileTreeResult } from "./fileTree.js";
import { mkdir, rm, stat, readFile, access } from "node:fs/promises";
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

does it work if you import from "fs/promises"? without node: just curious, as I'm not used to this notation.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It does seem to work without it. I'm not sure why this was added for me. I could look into removing it in the future if it's better. Let me know! I'll merge this PR in without that change though

import * as path from "node:path";
import { Logger } from "./log.js";
import { parse as parseYaml } from "yaml";
import { TspLocation } from "./typespec.js";

export async function ensureDirectory(path: string) {
await mkdir(path, { recursive: true });
Expand All @@ -19,51 +19,20 @@ export async function createTempDirectory(outputDir: string): Promise<string> {
return tempRoot;
}

export async function writeFileTree(rootDir: string, files: FileTreeResult["files"]) {
for (const [relativeFilePath, contents] of files) {
const filePath = path.join(rootDir, relativeFilePath);
await ensureDirectory(path.dirname(filePath));
Logger.debug(`writing ${filePath}`);
await writeFile(filePath, contents);
}
}

export async function tryReadTspLocation(rootDir: string): Promise<string | undefined> {
try {
const yamlPath = path.resolve(rootDir, "tsp-location.yaml");
const fileStat = await stat(yamlPath);
if (fileStat.isFile()) {
const fileContents = await readFile(yamlPath, "utf8");
const locationYaml = parseYaml(fileContents);
const { directory, commit, repo } = locationYaml;
if (!directory || !commit || !repo) {
throw new Error("Invalid tsp-location.yaml");
}
// make GitHub URL
return `https://raw.githubusercontent.com/${repo}/${commit}/${directory}/`;
}
} catch (e) {
Logger.error(`Error reading tsp-location.yaml: ${e}`);
}
return undefined;
}

export async function readTspLocation(rootDir: string): Promise<[string, string, string, string[]]> {
export async function readTspLocation(rootDir: string): Promise<TspLocation> {
try {
const yamlPath = path.resolve(rootDir, "tsp-location.yaml");
const fileStat = await stat(yamlPath);
if (fileStat.isFile()) {
const fileContents = await readFile(yamlPath, "utf8");
const locationYaml = parseYaml(fileContents);
let { directory, commit, repo, additionalDirectories } = locationYaml;
if (!directory || !commit || !repo) {
const tspLocation: TspLocation = parseYaml(fileContents);
if (!tspLocation.directory || !tspLocation.commit || !tspLocation.repo) {
throw new Error("Invalid tsp-location.yaml");
}
Logger.info(`Additional directories: ${additionalDirectories}`)
if (!additionalDirectories) {
additionalDirectories = [];
if (!tspLocation.additionalDirectories) {
tspLocation.additionalDirectories = [];
}
return [ directory, commit, repo, additionalDirectories ];
return tspLocation;
}
throw new Error("Could not find tsp-location.yaml");
} catch (e) {
Expand Down
24 changes: 15 additions & 9 deletions tools/tsp-client/src/git.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,17 @@ import { spawn } from "child_process";
import { simpleGit } from "simple-git";

export async function getRepoRoot(repoPath: string): Promise<string> {
return await simpleGit(repoPath).revparse(["--show-toplevel"]);
return simpleGit(repoPath).revparse(["--show-toplevel"]);
catalinaperalta marked this conversation as resolved.
Show resolved Hide resolved
}

export async function cloneRepo(rootUrl: string, cloneDir: string, repo: string): Promise<void> {
await simpleGit(rootUrl).clone(repo, cloneDir, ["--no-checkout", "--filter=tree:0"], (err) => {
if (err) {
throw err;
}
return new Promise((resolve, reject) => {
simpleGit(rootUrl).clone(repo, cloneDir, ["--no-checkout", "--filter=tree:0"], (err) => {
if (err) {
reject(err);
}
});
resolve();
catalinaperalta marked this conversation as resolved.
Show resolved Hide resolved
});
}

Expand Down Expand Up @@ -52,9 +55,12 @@ export async function cloneRepo(rootUrl: string, cloneDir: string, repo: string)
}

export async function checkoutCommit(cloneDir: string, commit: string): Promise<void> {
await simpleGit(cloneDir).checkout(commit, undefined, (err) => {
if (err) {
throw err;
}
return new Promise((resolve, reject) => {
simpleGit(cloneDir).checkout(commit, undefined, (err) => {
if (err) {
reject(err);
}
});
resolve();
});
}
103 changes: 49 additions & 54 deletions tools/tsp-client/src/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
import * as path from "node:path";

import { installDependencies } from "./npm.js";
import { createTempDirectory, removeDirectory,readTspLocation, getEmitterFromRepoConfig } from "./fs.js";
import { createTempDirectory, removeDirectory, readTspLocation, getEmitterFromRepoConfig } from "./fs.js";
import { Logger, printBanner, enableDebug, printVersion } from "./log.js";
import { compileTsp, discoverMainFile, getEmitterOptions, resolveTspConfigUrl } from "./typespec.js";
import { TspLocation, compileTsp, discoverMainFile, getEmitterOptions, resolveTspConfigUrl } from "./typespec.js";
import { getOptions } from "./options.js";
import { mkdir, writeFile, cp, readFile } from "node:fs/promises";
import { addSpecFiles, checkoutCommit, cloneRepo, getRepoRoot, sparseCheckout } from "./git.js";
Expand All @@ -27,15 +27,15 @@ async function sdkInit(
repo: string | undefined;
isUrl: boolean;
}): Promise<string> {
if (isUrl) {
// URL scenario
const resolvedConfigUrl = resolveTspConfigUrl(config);
Logger.debug(`Resolved config url: ${resolvedConfigUrl.resolvedUrl}`)
const tspConfig = await fetch(resolvedConfigUrl.resolvedUrl);
const configYaml = parseYaml(tspConfig);
if (configYaml["parameters"] && configYaml["parameters"]["service-dir"]){
const serviceDir = configYaml["parameters"]["service-dir"]["default"];
if (serviceDir === undefined) {
try {
if (isUrl) {
// URL scenario
const resolvedConfigUrl = resolveTspConfigUrl(config);
Logger.debug(`Resolved config url: ${resolvedConfigUrl.resolvedUrl}`)
const tspConfig = await fetch(resolvedConfigUrl.resolvedUrl);
const configYaml = parseYaml(tspConfig);
const serviceDir = configYaml?.parameters?.["service-dir"]?.default;
if (!serviceDir) {
Logger.error(`Parameter service-dir is not defined correctly in tspconfig.yaml. Please refer to https://github.com/Azure/azure-rest-api-specs/blob/main/specification/contosowidgetmanager/Contoso.WidgetManager/tspconfig.yaml for the right schema.`)
}
Logger.debug(`Service directory: ${serviceDir}`)
Expand All @@ -51,25 +51,19 @@ async function sdkInit(
`directory: ${resolvedConfigUrl.path}\ncommit: ${resolvedConfigUrl.commit}\nrepo: ${resolvedConfigUrl.repo}\nadditionalDirectories: ${additionalDirs}`);
return newPackageDir;
} else {
Logger.error("Missing service-dir in parameters section of tspconfig.yaml. Please refer to https://github.com/Azure/azure-rest-api-specs/blob/main/specification/contosowidgetmanager/Contoso.WidgetManager/tspconfig.yaml for the right schema.")
}
} else {
// Local directory scenario
let configFile = path.join(config, "tspconfig.yaml")
const data = await readFile(configFile, "utf8");
const configYaml = parseYaml(data);
if (configYaml["parameters"] && configYaml["parameters"]["service-dir"]) {
const serviceDir = configYaml["parameters"]["service-dir"]["default"];
var additionalDirs: string[] = [];
if (configYaml["parameters"]["dependencies"] && configYaml["parameters"]["dependencies"]["additionalDirectories"]) {
additionalDirs = configYaml["parameters"]["dependencies"]["additionalDirectories"];
// Local directory scenario
let configFile = path.join(config, "tspconfig.yaml")
const data = await readFile(configFile, "utf8");
const configYaml = parseYaml(data);
const serviceDir = configYaml?.parameters?.["service-dir"]?.default;
if (!serviceDir) {
Logger.error(`Parameter service-dir is not defined correctly in tspconfig.yaml. Please refer to https://github.com/Azure/azure-rest-api-specs/blob/main/specification/contosowidgetmanager/Contoso.WidgetManager/tspconfig.yaml for the right schema.`)
}
Logger.debug(`Service directory: ${serviceDir}`)
const additionalDirs: string[] = configYaml?.parameters?.dependencies?.additionalDirectories ?? [];
Logger.info(`Additional directories: ${additionalDirs}`)
let packageDir: string | undefined = undefined;
if (configYaml["options"][emitter] && configYaml["options"][emitter]["package-dir"]) {
packageDir = configYaml["options"][emitter]["package-dir"];
}
if (packageDir === undefined) {
const packageDir = configYaml?.options?.[emitter]?.["package-dir"];
if (!packageDir) {
throw new Error(`Missing package-dir in ${emitter} options of tspconfig.yaml. Please refer to https://github.com/Azure/azure-rest-api-specs/blob/main/specification/contosowidgetmanager/Contoso.WidgetManager/tspconfig.yaml for the right schema.`);
}
const newPackageDir = path.join(outputDir, serviceDir, packageDir)
Expand All @@ -86,24 +80,25 @@ async function sdkInit(
`directory: ${directory}\ncommit: ${commit}\nrepo: ${repo}\nadditionalDirectories: ${additionalDirs}`);
return newPackageDir;
}
throw new Error("Missing service-dir in parameters section of tspconfig.yaml. Please refer to https://github.com/Azure/azure-rest-api-specs/blob/main/specification/contosowidgetmanager/Contoso.WidgetManager/tspconfig.yaml for the right schema.")
}
throw new Error("Invalid tspconfig.yaml. Please refer to https://github.com/Azure/azure-rest-api-specs/blob/main/specification/contosowidgetmanager/Contoso.WidgetManager/tspconfig.yaml for the right schema.");
} catch (err) {
Logger.error("Invalid tspconfig.yaml. Please refer to https://github.com/Azure/azure-rest-api-specs/blob/main/specification/contosowidgetmanager/Contoso.WidgetManager/tspconfig.yaml for the right schema.");
throw err;
}
}

async function syncTspFiles(outputDir: string, localSpecRepo?: string) {
const tempRoot = await createTempDirectory(outputDir);

const repoRoot = await getRepoRoot(outputDir);
Logger.debug(`Repo root is ${repoRoot}`);
if (repoRoot === undefined) {
if (!repoRoot) {
throw new Error("Could not find repo root");
}
const [ directory, commit, repo, additionalDirectories ] = await readTspLocation(outputDir);
const dirSplit = directory.split("/");
const tspLocation: TspLocation = await readTspLocation(outputDir);
const dirSplit = tspLocation.directory.split("/");
let projectName = dirSplit[dirSplit.length - 1];
Logger.debug(`Using project name: ${projectName}`)
if (projectName === undefined) {
if (!projectName) {
projectName = "src";
}
const srcDir = path.join(tempRoot, projectName);
Expand All @@ -123,27 +118,27 @@ async function syncTspFiles(outputDir: string, localSpecRepo?: string) {
await cp(localSpecRepo, srcDir, { recursive: true, filter: filter });
const localSpecRepoRoot = await getRepoRoot(localSpecRepo);
Logger.info(`Local spec repo root is ${localSpecRepoRoot}`)
if (localSpecRepoRoot === undefined) {
if (!localSpecRepoRoot) {
throw new Error("Could not find local spec repo root, please make sure the path is correct");
}
for (const dir of additionalDirectories) {
for (const dir of tspLocation.additionalDirectories!) {
await cp(path.join(localSpecRepoRoot, dir), srcDir, { recursive: true, filter: filter });
}
} else {
const cloneDir = path.join(repoRoot, "..", "sparse-spec");
await mkdir(cloneDir, { recursive: true });
Logger.debug(`Created temporary sparse-checkout directory ${cloneDir}`);
Logger.debug(`Cloning repo to ${cloneDir}`);
await cloneRepo(tempRoot, cloneDir, `https://github.com/${repo}.git`);
await cloneRepo(tempRoot, cloneDir, `https://github.com/${tspLocation.repo}.git`);
await sparseCheckout(cloneDir);
await addSpecFiles(cloneDir, directory)
Logger.info(`Processing additional directories: ${additionalDirectories}`)
for (const dir of additionalDirectories) {
await addSpecFiles(cloneDir, tspLocation.directory)
Logger.info(`Processing additional directories: ${tspLocation.additionalDirectories}`)
for (const dir of tspLocation.additionalDirectories!) {
await addSpecFiles(cloneDir, dir);
}
await checkoutCommit(cloneDir, commit);
await cp(path.join(cloneDir, directory), srcDir, { recursive: true });
for (const dir of additionalDirectories) {
await checkoutCommit(cloneDir, tspLocation.commit);
await cp(path.join(cloneDir, tspLocation.directory), srcDir, { recursive: true });
for (const dir of tspLocation.additionalDirectories!) {
const dirSplit = dir.split("/");
let projectName = dirSplit[dirSplit.length - 1];
const dirName = path.join(tempRoot, projectName!);
Expand All @@ -169,9 +164,9 @@ async function generate({
}) {
const tempRoot = path.join(rootUrl, "TempTypeSpecFiles");
const tspLocation = await readTspLocation(rootUrl);
const dirSplit = tspLocation[0].split("/");
const dirSplit = tspLocation.directory.split("/");
let projectName = dirSplit[dirSplit.length - 1];
if (projectName === undefined) {
if (!projectName) {
throw new Error("cannot find project name");
}
const srcDir = path.join(tempRoot, projectName);
Expand Down Expand Up @@ -234,17 +229,17 @@ async function main() {
throw new Error("Commit SHA is required when specifying `--repo`, please specify a commit using `--commit`");
}
if (options.commit) {
let [ directory, commit, repo, additionalDirectories ] = await readTspLocation(rootUrl);
commit = options.commit ?? commit;
repo = options.repo ?? repo;
await writeFile(path.join(rootUrl, "tsp-location.yaml"), `directory: ${directory}\ncommit: ${commit}\nrepo: ${repo}\nadditionalDirectories: ${additionalDirectories}`);
let tspLocation: TspLocation = await readTspLocation(rootUrl);
catalinaperalta marked this conversation as resolved.
Show resolved Hide resolved
tspLocation.commit = options.commit ?? tspLocation.commit;
tspLocation.repo = options.repo ?? tspLocation.repo;
await writeFile(path.join(rootUrl, "tsp-location.yaml"), `directory: ${tspLocation.directory}\ncommit: ${tspLocation.commit}\nrepo: ${tspLocation.repo}\nadditionalDirectories: ${tspLocation.additionalDirectories}`);
}
if (options.tspConfig) {
let [ directory, commit, repo, additionalDirectories ] = await readTspLocation(rootUrl);
let tspLocation: TspLocation = await readTspLocation(rootUrl);
let tspConfig = resolveTspConfigUrl(options.tspConfig);
commit = tspConfig.commit ?? commit;
repo = tspConfig.repo ?? repo;
await writeFile(path.join(rootUrl, "tsp-location.yaml"), `directory: ${directory}\ncommit: ${commit}\nrepo: ${repo}\nadditionalDirectories: ${additionalDirectories}`);
tspLocation.commit = tspConfig.commit ?? tspLocation.commit;
tspLocation.repo = tspConfig.repo ?? tspLocation.repo;
await writeFile(path.join(rootUrl, "tsp-location.yaml"), `directory: ${tspLocation.directory}\ncommit: ${tspLocation.commit}\nrepo: ${tspLocation.repo}\nadditionalDirectories: ${tspLocation.additionalDirectories}`);
}
await syncTspFiles(rootUrl);
await generate({ rootUrl, noCleanup: options.noCleanup, additionalEmitterOptions: options.emitterOptions});
Expand Down
Loading