diff --git a/.github/actions/cache/action.yml b/.github/actions/cache/action.yml index fad3ac8125..2c21fbc547 100644 --- a/.github/actions/cache/action.yml +++ b/.github/actions/cache/action.yml @@ -39,11 +39,6 @@ runs: shell: bash run: curl -L "https://github.com/google/google-java-format/releases/download/v1.13.0/google-java-format-1.13.0-all-deps.jar" > /tmp/java-formatter.jar - - name: Download openapi generator jar for java (TODO REMOVE) - if: ${{ inputs.language == 'java' || inputs.job == 'cts' }} - shell: bash - run: curl -L "https://repo1.maven.org/maven2/org/openapitools/openapi-generator-cli/5.4.0/openapi-generator-cli-5.4.0.jar" > /tmp/openapi-generator-cli.jar - # Restore bundled specs: used during 'client' generation or pushing the 'codegen' - name: Restore built abtesting spec if: ${{ inputs.job == 'client' || inputs.job == 'codegen' }} diff --git a/.gitignore b/.gitignore index 0717881bb1..369c3c63af 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,5 @@ build dist .openapi-generator + +tests/output/*/.openapi-generator-ignore diff --git a/Dockerfile b/Dockerfile index de045f51b4..b567072464 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,9 +10,6 @@ ENV JAVA_HOME=/usr/lib/jvm/default-jvm # Java formatter ADD https://github.com/google/google-java-format/releases/download/v1.13.0/google-java-format-1.13.0-all-deps.jar /tmp/java-formatter.jar -# openapi generator jar (TODO: REMOVE) -ADD https://repo1.maven.org/maven2/org/openapitools/openapi-generator-cli/5.4.0/openapi-generator-cli-5.4.0.jar /tmp/openapi-generator-cli.jar - # PHP dependencies RUN apk add -U composer php8 php8-tokenizer php8-dom php8-xml php8-xmlwriter diff --git a/config/clients.config.json b/config/clients.config.json index 06e736304c..6fcbd37f77 100644 --- a/config/clients.config.json +++ b/config/clients.config.json @@ -1,6 +1,7 @@ { "java": { "folder": "clients/algoliasearch-client-java-2", + "customGenerator": "algolia-java", "tests": { "extension": ".test.java", "outputFolder": "src/test/java/com/algolia" diff --git a/config/openapitools-java-cts.json b/config/openapitools-java-cts.json deleted file mode 100644 index 8cb33abe13..0000000000 --- a/config/openapitools-java-cts.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "generatorName": "algolia-cts", - "templateDir": "tests/CTS/methods/requests/templates/java", - "outputDir": "tests/output/java", - "artifactId": "java-tests", - "groupId": "com.algolia", - "invokerPackage": "com.algolia", - "inputSpec": "specs/bundled/search.yml" -} diff --git a/config/openapitools-java.json b/config/openapitools-java.json deleted file mode 100644 index a891ec42e2..0000000000 --- a/config/openapitools-java.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "generatorName": "algolia-java", - "templateDir": "templates/java/", - "outputDir": "clients/algoliasearch-client-java-2", - "artifactId": "algoliasearch-client-java-2", - "groupId": "com.algolia", - "apiPackage": "com.algolia.search", - "invokerPackage": "com.algolia", - "modelPackage": "com.algolia.model.search", - "library": "okhttp-gson", - "inputSpec": "specs/bundled/search.yml", - "gitHost": "algolia", - "gitUserId": "algolia", - "gitRepoId": "algoliasearch-client-java-2", - "additionalProperties": { - "sourceFolder": "algoliasearch-core", - "java8": true, - "dateLibrary": "java8", - "packageName": "algoliasearch-client-java-2" - } -} diff --git a/generators/src/main/java/com/algolia/codegen/Utils.java b/generators/src/main/java/com/algolia/codegen/Utils.java new file mode 100644 index 0000000000..b8cbd3514f --- /dev/null +++ b/generators/src/main/java/com/algolia/codegen/Utils.java @@ -0,0 +1,7 @@ +package com.algolia.codegen; + +public class Utils { + public static String capitalize(String str) { + return str.substring(0, 1).toUpperCase() + str.substring(1); + } +} diff --git a/generators/src/main/java/com/algolia/codegen/cts/AlgoliaCtsGenerator.java b/generators/src/main/java/com/algolia/codegen/cts/AlgoliaCtsGenerator.java index fc2ad5cfc1..f7ebeb486a 100644 --- a/generators/src/main/java/com/algolia/codegen/cts/AlgoliaCtsGenerator.java +++ b/generators/src/main/java/com/algolia/codegen/cts/AlgoliaCtsGenerator.java @@ -5,8 +5,10 @@ import java.util.*; import java.util.Map.Entry; +import com.algolia.codegen.Utils; import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.databind.JsonMappingException; +import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap.Builder; import com.samskivert.mustache.Mustache.Lambda; @@ -19,6 +21,10 @@ public class AlgoliaCtsGenerator extends DefaultCodegen { // cache the models private final Map models = new HashMap<>(); + private String language; + private String client; + private String packageName; + private boolean hasRegionalHost; /** * Configures the type of generator. @@ -49,10 +55,35 @@ public String getName() { * * @return A string value for the help message */ + @Override public String getHelp() { return "Generates the CTS"; } + @Override + public void processOpts() { + super.processOpts(); + + language = (String) additionalProperties.get("language"); + client = (String) additionalProperties.get("client"); + packageName = (String) additionalProperties.get("packageName"); + hasRegionalHost = additionalProperties.get("hasRegionalHost").equals("true"); + + try { + JsonNode config = Json.mapper().readTree(new File("config/clients.config.json")); + TestConfig testConfig = Json.mapper().treeToValue(config.get(language).get("tests"), TestConfig.class); + + setTemplateDir("tests/CTS/methods/requests/templates/" + language); + setOutputDir("tests/output/" + language); + supportingFiles + .add(new SupportingFile("requests.mustache", testConfig.outputFolder + "/methods/requests", + client + testConfig.extension)); + } catch (IOException e) { + e.printStackTrace(); + System.exit(1); + } + } + @Override public Map postProcessAllModels(Map objs) { Map mod = super.postProcessAllModels(objs); @@ -65,12 +96,6 @@ public Map postProcessAllModels(Map objs) { return mod; } - public AlgoliaCtsGenerator() { - super(); - supportingFiles - .add(new SupportingFile("requests.mustache", "src/test/java/com/algolia/methods/requests", "search.test.java")); - } - @Override protected Builder addMustacheLambdas() { Builder lambdas = super.addMustacheLambdas(); @@ -85,22 +110,30 @@ public Map postProcessSupportingFileData(Map obj try { cts = loadCTS(); - Map operations = buildOperations(objs).get("Search"); + Map operations = buildOperations(objs); - Map bundle = super.postProcessSupportingFileData(objs); + // The return value of this function is not used, we need to modify the param + // itself. + Object lambda = objs.get("lambda"); + Map bundle = objs; + bundle.clear(); // We can put whatever we want in the bundle, and it will be accessible in the // template - bundle.put("client", "SearchApi"); + bundle.put("client", createClientName()); + bundle.put("import", packageName); + bundle.put("hasRegionalHost", hasRegionalHost); + bundle.put("lambda", lambda); List blocks = new ArrayList<>(); ParametersWithDataType paramsType = new ParametersWithDataType(models); for (Entry entry : cts.entrySet()) { - if (!operations.containsKey(entry.getKey())) { - throw new CTSException("operationId " + entry.getKey() + " does not exist in the spec"); + String operationId = entry.getKey(); + if (!operations.containsKey(operationId)) { + throw new CTSException("operationId " + operationId + " does not exist in the spec"); } - CodegenOperation op = operations.get(entry.getKey()); + CodegenOperation op = operations.get(operationId); List tests = new ArrayList<>(); for (int i = 0; i < entry.getValue().length; i++) { @@ -109,11 +142,17 @@ public Map postProcessSupportingFileData(Map obj } Map testObj = new HashMap<>(); testObj.put("tests", tests); + testObj.put("operationId", operationId); blocks.add(testObj); } bundle.put("blocks", blocks); return bundle; + } catch (CTSException e) { + if (e.isSkipable()) { + System.out.println(e.getMessage()); + System.exit(0); + } } catch (Exception e) { e.printStackTrace(); System.exit(1); @@ -121,33 +160,54 @@ public Map postProcessSupportingFileData(Map obj return null; } - private Map loadCTS() throws JsonParseException, JsonMappingException, IOException { + private Map loadCTS() throws JsonParseException, JsonMappingException, IOException, CTSException { TreeMap cts = new TreeMap<>(); - File dir = new File("tests/CTS/methods/requests/search"); + File dir = new File("tests/CTS/methods/requests/" + client); + if (!dir.exists()) { + throw new CTSException("CTS not found at " + dir.getAbsolutePath(), true); + } for (File f : dir.listFiles()) { cts.put(f.getName().replace(".json", ""), Json.mapper().readValue(f, Request[].class)); } return cts; } - // Client -> operationId -> CodegenOperation - private HashMap> buildOperations(Map objs) { - HashMap> result = new HashMap<>(); + // operationId -> CodegenOperation + private HashMap buildOperations(Map objs) { + HashMap result = new HashMap<>(); List> apis = ((Map>>) objs.get("apiInfo")).get("apis"); for (Map api : apis) { - String apiName = (String) api.get("baseName"); + String apiName = ((String) api.get("baseName")).toLowerCase(); + if (!apiName.equals(client.replace("-", ""))) { + continue; + } List operations = ((Map>) api.get("operations")) .get("operation"); - - HashMap allOp = new HashMap<>(); for (CodegenOperation ope : operations) { - allOp.put(ope.operationId, ope); + result.put(ope.operationId, ope); } - result.put(apiName, allOp); } return result; } + private String createClientName() { + String[] clientParts = client.split("-"); + String clientName = ""; + if (language.equals("javascript")) { + // do not capitalize the first part + clientName = clientParts[0]; + for (int i = 1; i < clientParts.length; i++) { + clientName += Utils.capitalize(clientParts[i]); + } + } else { + for (int i = 0; i < clientParts.length; i++) { + clientName += Utils.capitalize(clientParts[i]); + } + } + + return clientName + "Api"; + } + /** * override with any special text escaping logic to handle unsafe * characters so as to avoid code injection diff --git a/generators/src/main/java/com/algolia/codegen/cts/CTSException.java b/generators/src/main/java/com/algolia/codegen/cts/CTSException.java index ba3765a6ff..8a965af88c 100644 --- a/generators/src/main/java/com/algolia/codegen/cts/CTSException.java +++ b/generators/src/main/java/com/algolia/codegen/cts/CTSException.java @@ -1,7 +1,18 @@ package com.algolia.codegen.cts; public class CTSException extends Exception { - public CTSException(String message) { - super(message); - } + private boolean skipable; + + public CTSException(String message) { + super(message); + } + + public CTSException(String message, boolean skipable) { + this(message); + this.skipable = skipable; + } + + public boolean isSkipable() { + return skipable; + } } diff --git a/generators/src/main/java/com/algolia/codegen/cts/EscapeQuotesLambda.java b/generators/src/main/java/com/algolia/codegen/cts/EscapeQuotesLambda.java index a66dfa10e8..57ab52c6a8 100644 --- a/generators/src/main/java/com/algolia/codegen/cts/EscapeQuotesLambda.java +++ b/generators/src/main/java/com/algolia/codegen/cts/EscapeQuotesLambda.java @@ -7,9 +7,9 @@ import com.samskivert.mustache.Template; public class EscapeQuotesLambda implements Mustache.Lambda { - @Override - public void execute(Template.Fragment fragment, Writer writer) throws IOException { - String text = fragment.execute(); - writer.write(text.replace("\"", "\\\"")); - } + @Override + public void execute(Template.Fragment fragment, Writer writer) throws IOException { + String text = fragment.execute(); + writer.write(text.replace("\"", "\\\"")); + } } diff --git a/generators/src/main/java/com/algolia/codegen/cts/ParametersWithDataType.java b/generators/src/main/java/com/algolia/codegen/cts/ParametersWithDataType.java index 7f5d717511..db00b3950f 100644 --- a/generators/src/main/java/com/algolia/codegen/cts/ParametersWithDataType.java +++ b/generators/src/main/java/com/algolia/codegen/cts/ParametersWithDataType.java @@ -3,6 +3,10 @@ import java.util.*; import java.util.Map.Entry; +import com.algolia.codegen.Utils; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonMappingException; + import org.openapitools.codegen.CodegenModel; import org.openapitools.codegen.CodegenOperation; import org.openapitools.codegen.CodegenParameter; @@ -10,7 +14,7 @@ import org.openapitools.codegen.CodegenResponse; import org.openapitools.codegen.IJsonSchemaValidationProperties; -import io.swagger.v3.core.util.Json; +import io.swagger.util.Json; @SuppressWarnings("unchecked") public class ParametersWithDataType { @@ -21,13 +25,21 @@ public ParametersWithDataType(Map models) { } public Map buildJSONForRequest(Request req, CodegenOperation ope, int testIndex) - throws CTSException { + throws CTSException, JsonMappingException, JsonProcessingException { Map test = new HashMap<>(); test.put("method", req.method); test.put("testName", req.testName == null ? req.method : req.testName); test.put("testIndex", testIndex); test.put("request", req.request); + test.put("hasParameters", req.parameters.size() != 0); + + if (req.parameters.size() == 0) { + return test; + } + // Give the stringified version to mustache + test.put("parameters", Json.mapper().writeValueAsString(req.parameters)); + List parametersWithDataType = new ArrayList<>(); // special case if there is only bodyParam which is not an array @@ -86,7 +98,7 @@ private Map traverseParams(String paramName, Object param, IJson testOutput.put("parentSuffix", suffix - 1); testOutput.put("suffix", suffix); testOutput.put("parent", parent); - testOutput.put("objectName", baseType.substring(0, 1).toUpperCase() + baseType.substring(1)); + testOutput.put("objectName", Utils.capitalize(baseType)); if (spec.getIsArray()) { handleArray(paramName, param, testOutput, spec, suffix); @@ -260,6 +272,12 @@ private String inferDataType(Object param, CodegenParameter spec, Map parameters; - public RequestProp request; + public String testName; + public String method; - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append("class Request {\n"); - sb.append(" testName: ").append(testName).append("\n"); - sb.append(" method: ").append(method).append("\n"); - sb.append(" parameters: ").append(parameters).append("\n"); - sb.append(" request: ").append(request).append("\n"); - sb.append("}"); - return sb.toString(); - } + public Map parameters; + + public RequestProp request; + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("class Request {\n"); + sb.append(" testName: ").append(testName).append("\n"); + sb.append(" method: ").append(method).append("\n"); + sb.append(" parameters: ").append(parameters).append("\n"); + sb.append(" request: ").append(request).append("\n"); + sb.append("}"); + return sb.toString(); + } } class RequestProp { - public String path; - public String method; + public String path; + public String method; - @JsonDeserialize(using = RawDeserializer.class) - public Object data; + @JsonDeserialize(using = RawDeserializer.class) + public String data; - @JsonDeserialize(using = RawDeserializer.class) - public Object searchParams; + @JsonDeserialize(using = RawDeserializer.class) + public String searchParams; - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append("class RequestProp {\n"); - sb.append(" path: ").append(path).append("\n"); - sb.append(" method: ").append(method).append("\n"); - sb.append(" data: ").append(data).append("\n"); - sb.append(" searchParams: ").append(searchParams).append("\n"); - sb.append("}"); - return sb.toString(); - } + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("class RequestProp {\n"); + sb.append(" path: ").append(path).append("\n"); + sb.append(" method: ").append(method).append("\n"); + sb.append(" data: ").append(data).append("\n"); + sb.append(" searchParams: ").append(searchParams).append("\n"); + sb.append("}"); + return sb.toString(); + } } // Output json to raw string with quotes class RawDeserializer extends JsonDeserializer { - @Override - public String deserialize(JsonParser jp, DeserializationContext ctxt) - throws IOException, JsonProcessingException { + @Override + public String deserialize(JsonParser jp, DeserializationContext ctxt) + throws IOException, JsonProcessingException { - TreeNode tree = jp.getCodec().readTree(jp); - return tree.toString(); - } + TreeNode tree = jp.getCodec().readTree(jp); + return tree.toString(); + } } diff --git a/generators/src/main/java/com/algolia/codegen/cts/TestConfig.java b/generators/src/main/java/com/algolia/codegen/cts/TestConfig.java new file mode 100644 index 0000000000..eb4589d565 --- /dev/null +++ b/generators/src/main/java/com/algolia/codegen/cts/TestConfig.java @@ -0,0 +1,6 @@ +package com.algolia.codegen.cts; + +public class TestConfig { + public String extension; + public String outputFolder; +} diff --git a/openapitools.json b/openapitools.json index b9c3d8ae97..a2425c7ae3 100644 --- a/openapitools.json +++ b/openapitools.json @@ -1,5 +1,5 @@ { - "$schema": "./node_modules/@openapitools/openapi-generator-cli/config.schema.json", + "$schema": "./node_modules/@experimental-api-clients-automation/openapi-generator-cli/config.schema.json", "generator-cli": { "version": "5.4.0", "generators": { @@ -223,7 +223,7 @@ } }, "java-search": { - "generatorName": "java", + "generatorName": "algolia-java", "templateDir": "#{cwd}/templates/java/", "config": "#{cwd}/openapitools.json", "output": "#{cwd}/clients/algoliasearch-client-java-2", @@ -403,4 +403,4 @@ } } } -} +} \ No newline at end of file diff --git a/package.json b/package.json index 1e5662dfab..0935bbb140 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,7 @@ "website": "yarn workspace website start --host 0.0.0.0" }, "devDependencies": { - "@openapitools/openapi-generator-cli": "2.4.26", + "@experimental-api-clients-automation/openapi-generator-cli": "0.0.1", "@redocly/openapi-cli": "1.0.0-beta.79", "@typescript-eslint/eslint-plugin": "5.5.0", "@typescript-eslint/parser": "5.5.0", diff --git a/scripts/common.ts b/scripts/common.ts index f2a0dda131..46477c5cf1 100644 --- a/scripts/common.ts +++ b/scripts/common.ts @@ -5,6 +5,7 @@ import execa from 'execa'; // https://github.com/sindresorhus/execa/tree/v5.1.1 import openapitools from '../openapitools.json'; +import { createSpinner } from './oraLog'; import type { Generator, RunOptions } from './types'; export const CI = Boolean(process.env.CI); @@ -61,7 +62,9 @@ export const CLIENTS = CLIENTS_JS.filter( /** * Takes a generator key in the form 'language-client' and returns the Generator object. */ -export function splitGeneratorKey(generatorKey: string): Generator { +export function splitGeneratorKey( + generatorKey: string +): Pick { const language = generatorKey.slice(0, generatorKey.indexOf('-')); const client = generatorKey.slice(generatorKey.indexOf('-') + 1); return { language, client, key: generatorKey }; @@ -151,3 +154,11 @@ export async function runIfExists( } return ''; } + +export async function buildCustomGenerators(verbose: boolean): Promise { + const spinner = createSpinner('building custom generators', verbose).start(); + await run('./gradle/gradlew --no-daemon -p generators assemble', { + verbose, + }); + spinner.succeed(); +} diff --git a/scripts/config.ts b/scripts/config.ts index f4558505a9..76fcee8c61 100644 --- a/scripts/config.ts +++ b/scripts/config.ts @@ -11,3 +11,7 @@ export function getTestExtension(language: string): string | undefined { export function getTestOutputFolder(language: string): string | undefined { return clientsConfig[language]?.tests?.outputFolder; } + +export function getCustomGenerator(language: string): string | undefined { + return clientsConfig[language]?.customGenerator; +} diff --git a/scripts/cts/generate.ts b/scripts/cts/generate.ts index f6200e7e15..0542cb1a50 100644 --- a/scripts/cts/generate.ts +++ b/scripts/cts/generate.ts @@ -1,31 +1,26 @@ -import { run, toAbsolutePath } from '../common'; +import { buildCustomGenerators, run, toAbsolutePath } from '../common'; import { getTestOutputFolder } from '../config'; import { formatter } from '../formatter'; import { createSpinner } from '../oraLog'; import type { Generator } from '../types'; import { generateClientTests } from './client/generate'; -import { generateRequestsTests } from './methods/requests/generate'; -async function ctsGenerate( - generator: Generator, - verbose: boolean -): Promise { - createSpinner(`generating CTS for ${generator.key}`, verbose).start().info(); - switch (generator.language) { +async function ctsGenerate(gen: Generator, verbose: boolean): Promise { + createSpinner(`generating CTS for ${gen.key}`, verbose).start().info(); + const spinner = createSpinner( + { text: 'generating requests tests', indent: 4 }, + verbose + ).start(); + await run( + `yarn openapi-generator-cli --custom-generator=generators/build/libs/algolia-java-openapi-generator-1.0.0.jar generate \ + -g algolia-cts -i specs/bundled/${gen.client}.yml --additional-properties="language=${gen.language},client=${gen.client},packageName=${gen.additionalProperties.packageName},hasRegionalHost=${gen.additionalProperties.hasRegionalHost}"`, + { verbose } + ); + spinner.succeed(); + switch (gen.language) { case 'javascript': - await generateRequestsTests(generator, verbose); - await generateClientTests(generator, verbose); - break; - case 'java': - // eslint-disable-next-line no-warning-comments - // TODO: We can remove this once https://github.com/OpenAPITools/openapi-generator-cli/issues/439 is fixed, - // and just call it with `yarn openapi-generator-cli --custom-generator=generators/build/libs/algolia-java-openapi-generator-1.0.0.jar` - await run( - `./gradle/gradlew --no-daemon -p generators assemble && \ - java -cp /tmp/openapi-generator-cli.jar:generators/build/libs/algolia-java-openapi-generator-1.0.0.jar -ea org.openapitools.codegen.OpenAPIGenerator generate -c config/openapitools-java-cts.json`, - { verbose } - ); + await generateClientTests(gen, verbose); break; default: } @@ -35,6 +30,8 @@ export async function ctsGenerateMany( generators: Generator[], verbose: boolean ): Promise { + await buildCustomGenerators(verbose); + for (const gen of generators) { if (!getTestOutputFolder(gen.language)) { continue; diff --git a/scripts/cts/methods/requests/cts.ts b/scripts/cts/methods/requests/cts.ts deleted file mode 100644 index 06e9e0599d..0000000000 --- a/scripts/cts/methods/requests/cts.ts +++ /dev/null @@ -1,225 +0,0 @@ -import fsp from 'fs/promises'; - -import SwaggerParser from '@apidevtools/swagger-parser'; -import type { OpenAPIV3 } from 'openapi-types'; - -import { exists, toAbsolutePath } from '../../../common'; -import { removeEnumType, removeObjectName, walk } from '../../utils'; - -import type { - CTSBlock, - ParametersWithDataType, - RequestCTS, - RequestCTSOutput, -} from './types'; - -/** - * Provide the `key` and `is*` params to apply custom logic in templates - * include the `-last` param to join with comma in mustache. - */ -function transformParam({ - key = '$root', - value, - last = true, - testName, - parent, - suffix = 0, -}: { - key?: string; - value: any; - last?: boolean; - testName: string; - parent?: string; - suffix?: number; -}): ParametersWithDataType | ParametersWithDataType[] { - const isDate = key === 'endAt'; - const isArray = Array.isArray(value); - let isObject = typeof value === 'object' && !isArray; - const isEnum = isObject && '$enumType' in value; - const isString = typeof value === 'string' && !isDate; - const isBoolean = typeof value === 'boolean'; - const isInteger = Number.isInteger(value); - const isDouble = typeof value === 'number' && !isInteger; - const objectName: string | undefined = (value as any).$objectName; - const isFreeFormObject = objectName === 'Object'; - - if (isEnum) { - isObject = false; - } - - const isTypes = { - isArray, - isObject: isObject && !isFreeFormObject, - isFreeFormObject, - isEnum, - isString, - isBoolean, - isInteger, - isDouble, - }; - - const isRoot = key === '$root'; - - let out = value; - if (isEnum) { - out = { enumType: value.$enumType, value: value.value }; - } else if (isObject) { - // recursive on every key:value - out = Object.entries(value) - .filter(([prop]) => prop !== '$objectName') - .map(([inKey, inValue], i, arr) => - transformParam({ - key: inKey, - value: inValue, - last: i === arr.length - 1, - testName, - parent: isRoot ? 'param' : key, - suffix: suffix + 1, - }) - ); - - // Special case for root - if (isRoot) { - if (objectName) { - return { - key: 'param', - value: out, - objectName, - suffix, - parentSuffix: suffix, - ...isTypes, - '-last': true, - }; - } - return out; - } - } else if (isArray) { - // recursive on all value - out = value.map((v, i) => - transformParam({ - key: `${key}Param${i}`, - value: v, - last: i === value.length - 1, - testName, - parent: key, - suffix: suffix + 1, - }) - ); - } - - return { - key, - value: out, - objectName, - parent, - suffix, - parentSuffix: suffix - 1, - ...isTypes, - '-last': last, - }; -} - -function createParamWithDataType({ - parameters, - testName, -}: { - parameters: Record; - testName: string; -}): ParametersWithDataType[] { - const transformed = transformParam({ value: parameters, testName }); - if (Array.isArray(transformed)) { - return transformed; - } - return [transformed]; -} - -export async function loadRequestsCTS(client: string): Promise { - // load the list of operations from the spec - const spec = await SwaggerParser.validate( - toAbsolutePath(`specs/${client}/spec.yml`) - ); - if (!spec.paths) { - throw new Error(`No paths found for spec ${client}/spec.yml`); - } - if (!(await exists(toAbsolutePath(`tests/CTS/methods/requests/${client}`)))) { - // skip it if no CTS for this client - return []; - } - - const operations = Object.values(spec.paths) - .flatMap((p) => Object.values(p)) - .map((obj) => obj.operationId); - - const ctsClient: CTSBlock[] = []; - - for await (const file of walk( - toAbsolutePath(`tests/CTS/methods/requests/${client}`) - )) { - if (!file.name.endsWith('json')) { - continue; - } - const fileName = file.name.replace('.json', ''); - const fileContent = (await fsp.readFile(file.path)).toString(); - - if (!fileContent) { - throw new Error(`cannot read empty file ${fileName} - ${client} client`); - } - - const tests: RequestCTS[] = JSON.parse(fileContent); - - // check test validity against spec - if (!operations.includes(fileName)) { - throw new Error(`cannot find ${fileName} for the ${client} client`); - } - - const testsOutput: RequestCTSOutput[] = []; - let testIndex = 0; - for (const test of tests) { - const testOutput = test as RequestCTSOutput; - testOutput.testName = test.testName || test.method; - testOutput.testIndex = testIndex++; - - // stringify request.data too - testOutput.request.data = JSON.stringify(test.request.data); - testOutput.request.searchParams = JSON.stringify( - test.request.searchParams - ); - - if ( - typeof test.parameters !== 'object' || - Array.isArray(test.parameters) - ) { - throw new Error( - `parameters of ${testOutput.testName} must be an object` - ); - } - - if (Object.keys(test.parameters).length === 0) { - testOutput.parameters = undefined; - testOutput.parametersWithDataType = undefined; - testOutput.hasParameters = false; - } else { - testOutput.parametersWithDataType = createParamWithDataType({ - parameters: test.parameters, - testName: testOutput.testName, - }); - - // we stringify the param for mustache to render them properly - testOutput.parameters = JSON.stringify( - removeEnumType(removeObjectName(test.parameters)) - ); - testOutput.hasParameters = true; - } - testsOutput.push(testOutput); - } - - ctsClient.push({ - operationId: fileName, - tests: testsOutput, - }); - } - - return ctsClient.sort((t1, t2) => - t1.operationId.localeCompare(t2.operationId) - ); -} diff --git a/scripts/cts/methods/requests/generate.ts b/scripts/cts/methods/requests/generate.ts deleted file mode 100644 index 9cb1591375..0000000000 --- a/scripts/cts/methods/requests/generate.ts +++ /dev/null @@ -1,73 +0,0 @@ -import fsp from 'fs/promises'; - -import Mustache from 'mustache'; - -import { createSpinner } from '../../../oraLog'; -import type { Generator } from '../../../types'; -import { - createClientName, - capitalize, - getOutputPath, - createOutputDir, - loadTemplates, -} from '../../utils'; - -import { loadRequestsCTS } from './cts'; - -const testPath = 'methods/requests'; - -export async function generateRequestsTests( - { - language, - client, - additionalProperties: { hasRegionalHost, packageName }, - }: Generator, - verbose: boolean -): Promise { - createSpinner({ text: 'generating requests tests', indent: 4 }, verbose) - .start() - .info(); - const spinner = createSpinner( - { text: 'loading templates', indent: 8 }, - verbose - ).start(); - const { requests: template, ...partialTemplates } = await loadTemplates({ - language, - testPath, - }); - - spinner.text = 'loading CTS'; - const cts = await loadRequestsCTS(client); - - if (cts.length === 0) { - spinner.warn("skipping because tests doesn't exist"); - return; - } - - await createOutputDir({ language, testPath }); - - spinner.text = 'rendering templates'; - const code = Mustache.render( - template, - { - import: packageName, - client: createClientName(client, language), - blocks: cts, - hasRegionalHost: hasRegionalHost ? true : undefined, - capitalize() { - return function (text: string, render: (t: string) => string): string { - return capitalize(render(text)); - }; - }, - escapeQuotes() { - return function (text: string, render: (t: string) => string): string { - return render(text).replace(/"/g, '\\"'); - }; - }, - }, - partialTemplates - ); - - await fsp.writeFile(getOutputPath({ language, client, testPath }), code); - spinner.succeed(); -} diff --git a/scripts/cts/methods/requests/types.ts b/scripts/cts/methods/requests/types.ts deleted file mode 100644 index 4f8e57d66e..0000000000 --- a/scripts/cts/methods/requests/types.ts +++ /dev/null @@ -1,52 +0,0 @@ -export type ParametersWithDataType = { - key: string; - value: Record | string; - parent?: string; - suffix: number; - parentSuffix: number; - isArray: boolean; - isObject: boolean; - isFreeFormObject: boolean; - isString: boolean; - isBoolean: boolean; - isInteger: boolean; - isDouble: boolean; - '-last': boolean; - objectName?: string; -}; - -export type RequestCTS = { - testName?: string; - method: string; - parameters: Record; - request: { - path: string; - method: string; - data?: Record; - searchParams?: Record; - }; -}; - -export type RequestCTSOutput = { - testName: string; - testIndex: number; - method: string; - parameters: any; - parametersWithDataType?: ParametersWithDataType[]; - hasParameters: boolean; - request: { - path: string; - method: string; - data?: string; - searchParams?: string; - }; -}; - -export type CTSBlock = { - operationId: string; - tests: RequestCTSOutput[]; -}; - -export type CTS = { - requests: CTSBlock[]; -}; diff --git a/scripts/cts/utils.test.ts b/scripts/cts/utils.test.ts index dc1401bb46..ff9302b001 100644 --- a/scripts/cts/utils.test.ts +++ b/scripts/cts/utils.test.ts @@ -1,9 +1,4 @@ -import { - capitalize, - createClientName, - removeEnumType, - removeObjectName, -} from './utils'; +import { capitalize, createClientName } from './utils'; describe('utils', () => { describe('capitalize', () => { @@ -44,75 +39,4 @@ describe('utils', () => { ); }); }); - - describe('removeObjectName', () => { - it('should remove simple $objectName from root', () => { - expect(removeObjectName({ $objectName: 'test', key: 'val' })).toEqual({ - key: 'val', - }); - }); - - it('should remove $objectName in nested objects', () => { - expect( - removeObjectName({ - $objectName: 'test', - key: { $objectName: 'other', otherKey: 'val2' }, - }) - ).toEqual({ - key: { otherKey: 'val2' }, - }); - }); - - it('should remove $objectName in arrays', () => { - expect( - removeObjectName({ - $objectName: 'test', - arr: [{ $objectName: 'other', otherKey: 'val2' }, '$objectName'], - }) - ).toEqual({ - arr: [{ otherKey: 'val2' }, '$objectName'], - }); - }); - }); - - describe('removeEnumType', () => { - it('should replace $enumType with the value', () => { - expect( - removeEnumType({ - val: 'test', - key: { $enumType: 'Action', value: 'addEntry' }, - }) - ).toEqual({ - val: 'test', - key: 'addEntry', - }); - }); - - it('should replace $enumType in nested objects', () => { - expect( - removeEnumType({ - val: 'test', - key: { - first: 'basic', - second: { $enumType: 'Action', value: 'addEntry' }, - }, - }) - ).toEqual({ - val: 'test', - key: { first: 'basic', second: 'addEntry' }, - }); - }); - - it('should replace $enumType in arrays', () => { - expect( - removeEnumType({ - val: 'test', - arr: [{ $enumType: 'Action', value: 'addEntry' }, '$enumType'], - }) - ).toEqual({ - val: 'test', - arr: ['addEntry', '$enumType'], - }); - }); - }); }); diff --git a/scripts/cts/utils.ts b/scripts/cts/utils.ts index e6ace617ab..a2abc733db 100644 --- a/scripts/cts/utils.ts +++ b/scripts/cts/utils.ts @@ -32,42 +32,6 @@ export function createClientName(client: string, language: string): string { return `${clientName}Api`; } - -export function removeObjectName(obj: any): any { - if (typeof obj === 'object') { - if (Array.isArray(obj)) { - return obj.map((k) => removeObjectName(k)); - } - const copy = {}; - for (const prop in obj) { - if (!Object.prototype.hasOwnProperty.call(obj, prop)) { - continue; - } - if (prop === '$objectName') { - continue; - } - copy[prop] = removeObjectName(obj[prop]); - } - return copy; - } - return obj; -} - -export function removeEnumType(obj: any): any { - if (typeof obj === 'object') { - if (Array.isArray(obj)) { - return obj.map((k) => removeEnumType(k)); - } - if ('$enumType' in obj) { - return obj.value; - } - return Object.fromEntries( - Object.entries(obj).map(([k, v]) => [k, removeEnumType(v)]) - ); - } - return obj; -} - export async function createOutputDir({ language, testPath, diff --git a/scripts/generate.ts b/scripts/generate.ts index c3b19c65f4..8473c9719a 100644 --- a/scripts/generate.ts +++ b/scripts/generate.ts @@ -1,7 +1,7 @@ import { buildJSClientUtils } from './buildClients'; import { buildSpecs } from './buildSpecs'; -import { CI, run, runIfExists } from './common'; -import { getLanguageFolder } from './config'; +import { buildCustomGenerators, CI, run, runIfExists } from './common'; +import { getCustomGenerator, getLanguageFolder } from './config'; import { formatter } from './formatter'; import { createSpinner } from './oraLog'; import { setHostsOptions } from './pre-gen/setHostsOptions'; @@ -22,19 +22,17 @@ async function generateClient( { language, key }: Generator, verbose?: boolean ): Promise { - if (language === 'java') { - // eslint-disable-next-line no-warning-comments - // TODO: We can remove this once https://github.com/OpenAPITools/openapi-generator-cli/issues/439 is fixed - await run( - `./gradle/gradlew --no-daemon -p generators assemble && \ - java -cp /tmp/openapi-generator-cli.jar:generators/build/libs/algolia-java-openapi-generator-1.0.0.jar -ea org.openapitools.codegen.OpenAPIGenerator generate -c config/openapitools-java.json`, - { verbose } - ); - return; - } - await run(`yarn openapi-generator-cli generate --generator-key ${key}`, { - verbose, - }); + const customGenerator = getCustomGenerator(language); + await run( + `yarn openapi-generator-cli ${ + customGenerator + ? '--custom-generator=generators/build/libs/algolia-java-openapi-generator-1.0.0.jar' + : '' + } generate --generator-key ${key}`, + { + verbose, + } + ); } async function postGen( @@ -55,6 +53,14 @@ export async function generate( await buildSpecs(clients, 'yml', verbose, true); } + const langs = [...new Set(generators.map((gen) => gen.language))]; + const useCustomGenerator = langs + .map((lang) => getCustomGenerator(lang)) + .some(Boolean); + if (useCustomGenerator) { + await buildCustomGenerators(verbose); + } + for (const gen of generators) { const spinner = createSpinner(`pre-gen ${gen.key}`, verbose).start(); await preGen(gen, verbose); @@ -73,7 +79,6 @@ export async function generate( spinner.succeed(); } - const langs = [...new Set(generators.map((gen) => gen.language))]; for (const lang of langs) { if (!(CI && lang === 'javascript')) { await formatter(lang, getLanguageFolder(lang), verbose); @@ -85,10 +90,4 @@ export async function generate( await buildJSClientUtils(verbose, 'all'); } } - - if (!CI) { - const spinner = createSpinner('formatting specs', verbose).start(); - await run(`yarn specs:fix`, { verbose }); - spinner.succeed(); - } } diff --git a/scripts/index.ts b/scripts/index.ts index 95c1a1faae..d63daa021f 100644 --- a/scripts/index.ts +++ b/scripts/index.ts @@ -286,7 +286,6 @@ program await playground({ language, client, - key: createGeneratorKey({ language, client }), }); } ); diff --git a/scripts/playground.ts b/scripts/playground.ts index da7fe28a6f..981b4709d5 100644 --- a/scripts/playground.ts +++ b/scripts/playground.ts @@ -4,7 +4,7 @@ import type { Generator } from './types'; export async function playground({ language, client, -}: Generator): Promise { +}: Pick): Promise { const verbose = true; switch (language) { case 'javascript': diff --git a/scripts/types.ts b/scripts/types.ts index c2dcba8f55..22a7720e73 100644 --- a/scripts/types.ts +++ b/scripts/types.ts @@ -2,6 +2,10 @@ export type Generator = Record & { language: string; client: string; key: string; + additionalProperties: Record & { + packageName: string; + hasRegionalHost?: boolean; + }; }; export type RunOptions = { diff --git a/tests/output/java/.openapi-generator-ignore b/tests/output/java/.openapi-generator-ignore deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/yarn.lock b/yarn.lock index 1b0471abef..d7da400358 100644 --- a/yarn.lock +++ b/yarn.lock @@ -9,7 +9,7 @@ __metadata: version: 0.0.0-use.local resolution: "@algolia/api-client-automation@workspace:." dependencies: - "@openapitools/openapi-generator-cli": 2.4.26 + "@experimental-api-clients-automation/openapi-generator-cli": 0.0.1 "@redocly/openapi-cli": 1.0.0-beta.79 "@typescript-eslint/eslint-plugin": 5.5.0 "@typescript-eslint/parser": 5.5.0 @@ -2486,6 +2486,30 @@ __metadata: languageName: unknown linkType: soft +"@experimental-api-clients-automation/openapi-generator-cli@npm:0.0.1": + version: 0.0.1 + resolution: "@experimental-api-clients-automation/openapi-generator-cli@npm:0.0.1" + dependencies: + "@nestjs/common": 8.2.6 + "@nestjs/core": 8.2.6 + chalk: 4.1.2 + commander: 8.3.0 + compare-versions: 3.6.0 + concurrently: 6.5.1 + console.table: 0.10.0 + fs-extra: 10.0.0 + glob: 7.1.6 + inquirer: 8.2.0 + lodash: 4.17.21 + reflect-metadata: 0.1.13 + rxjs: 7.5.2 + tslib: 2.0.3 + bin: + openapi-generator-cli: main.js + checksum: 4d0073d5520b0beec0dd08a20d11d0201727d28fd7d1a720add0ca8fae67557c2df61d7def1da4358856c7901d89fd701857c1354a1b6633a76b4885d54bf630 + languageName: node + linkType: hard + "@experimental-api-clients-automation/recommend@0.0.4, @experimental-api-clients-automation/recommend@workspace:clients/algoliasearch-client-javascript/packages/recommend": version: 0.0.0-use.local resolution: "@experimental-api-clients-automation/recommend@workspace:clients/algoliasearch-client-javascript/packages/recommend" @@ -3168,31 +3192,6 @@ __metadata: languageName: node linkType: hard -"@openapitools/openapi-generator-cli@npm:2.4.26": - version: 2.4.26 - resolution: "@openapitools/openapi-generator-cli@npm:2.4.26" - dependencies: - "@nestjs/common": 8.2.6 - "@nestjs/core": 8.2.6 - "@nuxtjs/opencollective": 0.3.2 - chalk: 4.1.2 - commander: 8.3.0 - compare-versions: 3.6.0 - concurrently: 6.5.1 - console.table: 0.10.0 - fs-extra: 10.0.0 - glob: 7.1.6 - inquirer: 8.2.0 - lodash: 4.17.21 - reflect-metadata: 0.1.13 - rxjs: 7.5.2 - tslib: 2.0.3 - bin: - openapi-generator-cli: main.js - checksum: ff6f75e194c0df492c4f6951a922a1da5472e7c3ca09b7b8e02f9b910e1fcc49832ca19e18ee7fdd87fcf43ef1ed6443ef5d1954932137c759d87f8a51e276eb - languageName: node - linkType: hard - "@parcel/bundler-default@npm:2.3.1": version: 2.3.1 resolution: "@parcel/bundler-default@npm:2.3.1"