From adde15a1605bbfd45f25cb9b535a62e701842dc9 Mon Sep 17 00:00:00 2001 From: mkralik3 Date: Fri, 4 Aug 2023 12:59:10 +0200 Subject: [PATCH] fix: Add in-memory parser option (gh-826) --- README.md | 2 +- .../camelroute/CamelRestDSLParseCatalog.java | 8 +- .../camelroute/CamelRouteParseCatalog.java | 6 + .../step/kamelet/KameletParseCatalog.java | 7 + .../api/metadata/catalog/StepCatalogTest.java | 162 ++++++++++++++++-- .../step/kamelet/KameletParseCatalogTest.java | 1 + .../kamelet/KameletStepParserServiceTest.java | 1 + .../api/metadata/catalog/StepCatalog.java | 21 ++- .../metadata/catalog/StepCatalogParser.java | 9 + update-resources.sh | 24 ++- 10 files changed, 209 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index 404a8712a..9676f2c07 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ For more information and all configuration properties, see [Quarkus HTTP Referen * Kaoto camel components: **3ee2af43623923a5c5e09df6f3f70657e1ccd09f** * Kaoto view definitions: **94aae37dee4356d51ac34bfb757eb43a85ad2c0a** * Camel-connectors: **3.21.0** -* Camel-connectors: **3.21.0** +* Camel-kamelets: **3.21.0** #### Updating step resources The repository contains steps repositories zip files which are bundled with Kaoto-backend during building. diff --git a/camel-support/src/main/java/io/kaoto/backend/camel/metadata/parser/step/camelroute/CamelRestDSLParseCatalog.java b/camel-support/src/main/java/io/kaoto/backend/camel/metadata/parser/step/camelroute/CamelRestDSLParseCatalog.java index 7f88a1616..d4d6839b9 100644 --- a/camel-support/src/main/java/io/kaoto/backend/camel/metadata/parser/step/camelroute/CamelRestDSLParseCatalog.java +++ b/camel-support/src/main/java/io/kaoto/backend/camel/metadata/parser/step/camelroute/CamelRestDSLParseCatalog.java @@ -180,10 +180,16 @@ private static Step getConsumesStep() { } @Override - public ParseCatalog getParser(String url) { + public ParseCatalog getParser() { return new CamelRestDSLParser(); } + @Override + public ParseCatalog getParser(String url) { + //We are not expecting to get anything from here + return new EmptyParseCatalog<>(); + } + @Override public ParseCatalog getParser(String url, String tag) { //We are not expecting to get anything from here diff --git a/camel-support/src/main/java/io/kaoto/backend/camel/metadata/parser/step/camelroute/CamelRouteParseCatalog.java b/camel-support/src/main/java/io/kaoto/backend/camel/metadata/parser/step/camelroute/CamelRouteParseCatalog.java index 45e6bbfac..f5a2847f9 100644 --- a/camel-support/src/main/java/io/kaoto/backend/camel/metadata/parser/step/camelroute/CamelRouteParseCatalog.java +++ b/camel-support/src/main/java/io/kaoto/backend/camel/metadata/parser/step/camelroute/CamelRouteParseCatalog.java @@ -22,6 +22,12 @@ public ParseCatalog getParser(final String url, final String tag) { } + @Override + public ParseCatalog getParser() { + //We are not expecting to get Camel Operators from memory + return new EmptyParseCatalog<>(); + } + @Override public ParseCatalog getParser(final String url) { ParseCatalog parseCatalog = new JarParseCatalog<>(url); diff --git a/camel-support/src/main/java/io/kaoto/backend/camel/metadata/parser/step/kamelet/KameletParseCatalog.java b/camel-support/src/main/java/io/kaoto/backend/camel/metadata/parser/step/kamelet/KameletParseCatalog.java index d7feaaa4b..1e3a0f975 100644 --- a/camel-support/src/main/java/io/kaoto/backend/camel/metadata/parser/step/kamelet/KameletParseCatalog.java +++ b/camel-support/src/main/java/io/kaoto/backend/camel/metadata/parser/step/kamelet/KameletParseCatalog.java @@ -4,6 +4,7 @@ import io.kaoto.backend.api.metadata.catalog.StepCatalogParser; import io.kaoto.backend.metadata.ParseCatalog; import io.kaoto.backend.metadata.parser.ClusterParseCatalog; +import io.kaoto.backend.metadata.parser.EmptyParseCatalog; import io.kaoto.backend.metadata.parser.GitParseCatalog; import io.kaoto.backend.metadata.parser.JarParseCatalog; import io.kaoto.backend.metadata.parser.LocalFolderParseCatalog; @@ -48,6 +49,12 @@ public ParseCatalog getParser(final String url, final String tag) { return parseCatalog; } + @Override + public ParseCatalog getParser() { + //We are not expecting to get Kamelets from memory + return new EmptyParseCatalog<>(); + } + @Override public ParseCatalog getParser(final String url) { ParseCatalog parseCatalog = new JarParseCatalog<>(url); diff --git a/camel-support/src/test/java/io/kaoto/backend/api/metadata/catalog/StepCatalogTest.java b/camel-support/src/test/java/io/kaoto/backend/api/metadata/catalog/StepCatalogTest.java index 4f61c4088..ff159a585 100644 --- a/camel-support/src/test/java/io/kaoto/backend/api/metadata/catalog/StepCatalogTest.java +++ b/camel-support/src/test/java/io/kaoto/backend/api/metadata/catalog/StepCatalogTest.java @@ -6,13 +6,16 @@ import io.quarkus.test.junit.QuarkusTest; import jakarta.enterprise.inject.Instance; import jakarta.inject.Inject; +import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import org.junit.jupiter.params.provider.ValueSource; import java.util.Collection; import java.util.List; import java.util.Optional; +import java.util.stream.Stream; import static org.assertj.core.api.Assertions.assertThat; @@ -25,37 +28,150 @@ public class StepCatalogTest { @Inject private StepCatalog mainStepCatalog; + //helping class with all information about version/repo/file for Kamelets + private record KameletInfo() { + static final String REPO = "https://github.com/apache/camel-kamelets"; + //don't rename, update-resources.sh script uses/updates this + static final String KAMELET_VERSION = "3.21.0"; + static final String TAG = "v" + KAMELET_VERSION; + + static final String FILE = "camel-kamelets-" + KAMELET_VERSION + ".jar"; + static final int EXPECT_NUMBER = 215; + + static String getFileFromRepo() { + return String.format("%s/archive/refs/tags/%s.zip", REPO, TAG); + } + + static String getFileFromCentral() { + return String.format( + "https://repo1.maven.org/maven2/org/apache/camel/kamelets/camel-kamelets/%s/camel-kamelets-%s.jar", + KAMELET_VERSION, KAMELET_VERSION); + } + + static String getFileFromResources() { + return "resource://" + FILE; + } + } + + //helping class with all information about version/repo/file for Kaoto component metadata + private record ComponentMetadataInfo() { + static final String REPO = "https://github.com/KaotoIO/camel-component-metadata"; + static final String TAG = "test-202308"; + static final String FILE = "camel-component-metadata.zip"; + static final int EXPECT_REPO_NUMBER = 54; + static final int EXPECT_FILE_NUMBER = 54; + + static String getFileFromRepo() { + return String.format("%s/archive/refs/tags/%s.zip", REPO, TAG); + } + + static String getFileFromResources() { + return "resource://" + FILE; + } + } + + //helping class with all information about version/repo/file for Camel connectors + private record CamelConnectorsInfo() { + //don't rename, update-resources.sh script uses/updates this + static final String CAMEL_CONNECTORS_VERSION = "3.21.0"; + static final String FILE = "camel-connectors-" + CAMEL_CONNECTORS_VERSION + ".zip"; + + static final int EXPECT_NUMBER = 847; + + static String getFileFromResources() { + return "resource://" + FILE; + } + } + + //helping class with all information about in-memory steps + private record InMemoryStepsInfo() { + //can be more steps (also with different types) in the future + static final int CAMEL_REST_DSL_IN_MEMORY_STEPS = 11; + + static int getAllInMemoryStepsNumber() { + return CAMEL_REST_DSL_IN_MEMORY_STEPS; + } + + } + + private static Stream provideTestParametersForGitRepoTest() { + return Stream.of( + Arguments.of(KameletInfo.REPO + ".git", KameletInfo.TAG, KameletInfo.EXPECT_NUMBER), + Arguments.of(ComponentMetadataInfo.REPO + ".git", + ComponentMetadataInfo.TAG, ComponentMetadataInfo.EXPECT_REPO_NUMBER) + ); + } + @ParameterizedTest - @CsvSource({"https://github.com/KaotoIO/camel-component-metadata.git,test-202308", - "https://github.com/apache/camel-kamelets.git,v3.21.0"}) - void testParseFromGit(String repo, String tag) { + @MethodSource("provideTestParametersForGitRepoTest") + void testParseFromGit(String repo, String tag, int expectNumbers) { StepRepository stepRepository = new StepRepository( Optional.empty(), Optional.of(List.of( new GitRepository(repo, tag, true, "all"))), Optional.empty()); - tryLoadSteps(stepRepository, repo + ":" + tag); + tryLoadSteps(stepRepository, repo + ":" + tag, expectNumbers); + } + + private static Stream provideTestParametersForJarRepoTest() { + return Stream.of( + //kamelets + Arguments.of(KameletInfo.getFileFromRepo(), KameletInfo.EXPECT_NUMBER), + Arguments.of(KameletInfo.getFileFromCentral(), KameletInfo.EXPECT_NUMBER), + Arguments.of(KameletInfo.getFileFromResources(), KameletInfo.EXPECT_NUMBER), + //component-metadata + Arguments.of(ComponentMetadataInfo.getFileFromRepo(), ComponentMetadataInfo.EXPECT_REPO_NUMBER), + Arguments.of(ComponentMetadataInfo.getFileFromResources(), ComponentMetadataInfo.EXPECT_FILE_NUMBER), + //connectors + Arguments.of(CamelConnectorsInfo.getFileFromResources(), CamelConnectorsInfo.EXPECT_NUMBER) + ); } @ParameterizedTest - @ValueSource(strings = { - "https://github.com/KaotoIO/camel-component-metadata/archive/refs/tags/test-202308.zip", - "https://repo1.maven.org/maven2/org/apache/camel/kamelets/camel-kamelets/3.21.0/camel-kamelets-3.21.0.jar", - "resource://camel-component-metadata.zip", - "resource://camel-kamelets-3.21.0.jar", - "resource://camel-connectors-3.21.0.zip" - }) - void testParseFromFile(String resource) { + @MethodSource("provideTestParametersForJarRepoTest") + void testParseFromJar(String resource, int expectNumbers) { StepRepository stepRepository = new StepRepository( Optional.of(List.of( new LocationRepository(resource, true, "all"))), Optional.empty(), Optional.empty()); - tryLoadSteps(stepRepository, resource); + tryLoadSteps(stepRepository, resource, expectNumbers); } - private void tryLoadSteps(StepRepository stepRepository, String resource) { + @Test + void testSameResourceFromMoreRepositories() { + StepRepository stepRepository = new StepRepository( + Optional.of(List.of( + new LocationRepository(KameletInfo.getFileFromCentral(), true, "all"), + new LocationRepository(KameletInfo.getFileFromRepo(), true, "all"))), + Optional.of(List.of( + new GitRepository(KameletInfo.REPO + ".git", KameletInfo.TAG, true, "all"))), + Optional.empty()); + tryLoadSteps(stepRepository, KameletInfo.getFileFromCentral() + " + " + KameletInfo.getFileFromRepo(), + KameletInfo.EXPECT_NUMBER); + } + + @Test + void testMultipleResource() { + StepRepository stepRepository = new StepRepository( + Optional.empty(), + Optional.of(List.of( + new GitRepository(KameletInfo.REPO + ".git", KameletInfo.TAG, true, "all"), + new GitRepository(ComponentMetadataInfo.REPO + ".git", ComponentMetadataInfo.TAG, + true, "all"))), + Optional.empty()); + tryLoadSteps(stepRepository, KameletInfo.REPO + " + " + ComponentMetadataInfo.REPO, + KameletInfo.EXPECT_NUMBER + ComponentMetadataInfo.EXPECT_REPO_NUMBER); + } + + private void tryLoadSteps(StepRepository stepRepository, String resource, int expectNumbers) { + tryLoadSteps(stepRepository, resource, expectNumbers, false); + } + + private void tryLoadSteps(StepRepository stepRepository, String resource, int expectNumbers, + boolean includeInMemorySteps) { StepCatalog anotherStepCatalog = new StepCatalog(); + anotherStepCatalog.includeInMemoryCatalogs = includeInMemorySteps; anotherStepCatalog.setStepCatalogParsers(stepCatalogParsers); anotherStepCatalog.setKclient(kclient); anotherStepCatalog.setRepository(stepRepository); @@ -63,14 +179,24 @@ private void tryLoadSteps(StepRepository stepRepository, String resource) { anotherStepCatalog.warmUpCatalog(); Collection steps = anotherStepCatalog.getReadOnlyCatalog().getAll(); assertThat(steps) - .isNotEmpty() - .as("Test that more than 50 steps was loaded from resource: " + resource) - .hasSizeGreaterThan(50) + .as("Test that all expected steps '" + expectNumbers + "' was loaded from resource: " + resource) + .hasSize(expectNumbers) .as("Test that catalog contains steps only from resource '" + resource + "' and not from different resources.") .hasSizeLessThan(mainStepCatalog.getReadOnlyCatalog().getAll().size()); } + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void testInMemorySteps(boolean shouldContains) { + StepRepository stepRepository = new StepRepository( + Optional.empty(), + Optional.empty(), + Optional.empty()); + int numberOfSteps = shouldContains ? InMemoryStepsInfo.getAllInMemoryStepsNumber() : 0; + tryLoadSteps(stepRepository, "In memory steps (loaded from no repo)", numberOfSteps, shouldContains); + } + // GitRepository implementation used for testing private record GitRepository(String url, String tag, boolean ifNoCluster, String kind) implements Repository.Git { diff --git a/camel-support/src/test/java/io/kaoto/backend/camel/metadata/parser/step/kamelet/KameletParseCatalogTest.java b/camel-support/src/test/java/io/kaoto/backend/camel/metadata/parser/step/kamelet/KameletParseCatalogTest.java index 06f9ac15a..8d469f6f8 100644 --- a/camel-support/src/test/java/io/kaoto/backend/camel/metadata/parser/step/kamelet/KameletParseCatalogTest.java +++ b/camel-support/src/test/java/io/kaoto/backend/camel/metadata/parser/step/kamelet/KameletParseCatalogTest.java @@ -41,6 +41,7 @@ class KameletParseCatalogTest { private static final String FILE_NAME = "camel-kamelets-%s.jar"; + //don't rename, update-resources.sh script uses/updates this private static final String VERSION = "3.21.0"; private static final Logger LOG = Logger.getLogger(KameletParseCatalogTest.class); diff --git a/camel-support/src/test/java/io/kaoto/backend/camel/service/step/parser/kamelet/KameletStepParserServiceTest.java b/camel-support/src/test/java/io/kaoto/backend/camel/service/step/parser/kamelet/KameletStepParserServiceTest.java index 3d88b46b3..a1f4cc24b 100644 --- a/camel-support/src/test/java/io/kaoto/backend/camel/service/step/parser/kamelet/KameletStepParserServiceTest.java +++ b/camel-support/src/test/java/io/kaoto/backend/camel/service/step/parser/kamelet/KameletStepParserServiceTest.java @@ -39,6 +39,7 @@ @QuarkusTest class KameletStepParserServiceTest { + //don't rename, update-resources.sh script uses/updates this private static final String VERSION = "3.21.0"; private static String kamelet; diff --git a/catalog/src/main/java/io/kaoto/backend/api/metadata/catalog/StepCatalog.java b/catalog/src/main/java/io/kaoto/backend/api/metadata/catalog/StepCatalog.java index a7e8cc960..395b16f58 100644 --- a/catalog/src/main/java/io/kaoto/backend/api/metadata/catalog/StepCatalog.java +++ b/catalog/src/main/java/io/kaoto/backend/api/metadata/catalog/StepCatalog.java @@ -17,6 +17,7 @@ import jakarta.enterprise.context.ApplicationScoped; import jakarta.enterprise.inject.Instance; import jakarta.inject.Inject; +import org.eclipse.microprofile.config.inject.ConfigProperty; /** * 🐱class StepCatalog @@ -29,6 +30,10 @@ @ApplicationScoped public class StepCatalog extends AbstractCatalog { + @ConfigProperty(name = "kaoto.step.catalog.include.inmemory.catalogs", + defaultValue = "true") + boolean includeInMemoryCatalogs; + public static final String ALL = "all"; private StepRepository repository; @@ -45,7 +50,7 @@ protected List> loadParsers() { addZipJar(catalogs, clusterAvailable); addLocalFolder(catalogs, clusterAvailable); addGit(catalogs, clusterAvailable); - + if (includeInMemoryCatalogs) addInMemoryParsers(catalogs); return catalogs; } @@ -95,10 +100,10 @@ private void addLocalFolder(final List> catalogs, final boole //And call only the parsers that apply .flatMap(folder -> stepCatalogParsers.stream().parallel() .filter(parser -> ALL.equalsIgnoreCase(folder.kind()) || parser.generatesKind(folder.kind())) - .map(parser -> { - File dir = new File(folder.url()); - return parser.getLocalFolder(dir.toPath()); - }) + .map(parser -> { + File dir = new File(folder.url()); + return parser.getLocalFolder(dir.toPath()); + }) ).toList(); catalogs.addAll(items); @@ -119,6 +124,12 @@ private void addZipJar(final List> catalogs, final boolean cl } + private void addInMemoryParsers(final List> catalogs) { + List> items = stepCatalogParsers.stream().parallel().map(StepCatalogParser::getParser) + .toList(); + catalogs.addAll(items); + } + @Inject public void setRepository(final StepRepository repo) { this.repository = repo; diff --git a/catalog/src/main/java/io/kaoto/backend/api/metadata/catalog/StepCatalogParser.java b/catalog/src/main/java/io/kaoto/backend/api/metadata/catalog/StepCatalogParser.java index 925ac9b66..36a934433 100644 --- a/catalog/src/main/java/io/kaoto/backend/api/metadata/catalog/StepCatalogParser.java +++ b/catalog/src/main/java/io/kaoto/backend/api/metadata/catalog/StepCatalogParser.java @@ -14,6 +14,15 @@ */ public interface StepCatalogParser { + /* + * 🐱method getParser : ParseCatalog + * + * Loads all the elements from memory(code). + * + */ + @WithSpan + ParseCatalog getParser(); + /* * 🐱method getParser : ParseCatalog * 🐱param url : String diff --git a/update-resources.sh b/update-resources.sh index a91798097..c39954958 100755 --- a/update-resources.sh +++ b/update-resources.sh @@ -58,8 +58,8 @@ then read -r versionCamel echo "Removing old zip files" - git rm api/src/main/resources/camel-connectors-*.jar - git rm camel-support/src/test/resources/camel-connectors-*.jar + git rm api/src/main/resources/camel-connectors-*.zip + git rm camel-support/src/test/resources/camel-connectors-*.zip echo "Preparing zip file with camel component metadata" cd /tmp || exit 1 @@ -86,12 +86,14 @@ then sed -i 's/camel-connectors-.*/camel-connectors-'"$versionCamel"'.zip"/g' camel-support/src/test/resources/application.yaml sed -i 's/camel-connectors-.*/camel-connectors-'"$versionCamel"'.zip";/g' camel-support/src/test/java/io/kaoto/backend/camel/metadata/parser/step/camelroute/CamelRouteParseCatalogTest.java sed -i 's/VERSION = ".*/VERSION = "'"$versionCamel"'";/g' camel-support/src/test/java/io/kaoto/backend/camel/service/step/parser/kamelet/KameletStepParserServiceTest.java + sed -i 's/CAMEL_CONNECTORS_VERSION = ".*/CAMEL_CONNECTORS_VERSION = "'"$versionCamel"'";/g' camel-support/src/test/java/io/kaoto/backend/api/metadata/catalog/StepCatalogTest.java git add api/src/main/resources/resources-config.json api/src/main/resources/application.yaml \ api/src/test/resources/application.yaml camel-support/src/test/resources/application.yaml \ camel-support/src/test/java/io/kaoto/backend/camel/metadata/parser/step/camelroute/CamelRouteParseCatalogTest.java \ camel-support/src/main/resources/io/kaoto/backend/camel/service/deployment/generator/camelroute/camel-yaml-dsl.json \ - camel-support/src/test/java/io/kaoto/backend/camel/service/step/parser/kamelet/KameletStepParserServiceTest.java + camel-support/src/test/java/io/kaoto/backend/camel/service/step/parser/kamelet/KameletStepParserServiceTest.java \ + camel-support/src/test/java/io/kaoto/backend/api/metadata/catalog/StepCatalogTest.java echo "Updating version info in Readme" sed -i 's/Camel-connectors: \*\*.*/Camel-connectors: \*\*'"$versionCamel"'\*\*/g' README.md @@ -105,7 +107,7 @@ read -r download if [ "$download" = "y" ]; then echo "Downloading kamelets." - echo "Enter kamelets version to use (tag in apache/camel-kamelets repo):" + echo "Enter kamelets version to use (tag without 'v' in apache/camel-kamelets repo):" echo "Example: 3.21.0" read -r versionKamelets @@ -136,19 +138,27 @@ then sed -i 's/camel-kamelets-.*/camel-kamelets-'"$versionKamelets"'.jar"/g' api/src/test/resources/application.yaml sed -i 's/camel-kamelets-.*/camel-kamelets-'"$versionKamelets"'.jar"/g' camel-support/src/test/resources/application.yaml sed -i 's/VERSION = ".*/VERSION = "'"$versionKamelets"'";/g' camel-support/src/test/java/io/kaoto/backend/camel/metadata/parser/step/kamelet/KameletParseCatalogTest.java + sed -i 's/KAMELET_VERSION = ".*/KAMELET_VERSION = "'"$versionKamelets"'";/g' camel-support/src/test/java/io/kaoto/backend/api/metadata/catalog/StepCatalogTest.java git add api/src/main/resources/resources-config.json api/src/main/resources/application.yaml \ api/src/test/resources/application.yaml camel-support/src/test/resources/application.yaml \ - camel-support/src/test/java/io/kaoto/backend/camel/metadata/parser/step/kamelet/KameletParseCatalogTest.java + camel-support/src/test/java/io/kaoto/backend/camel/metadata/parser/step/kamelet/KameletParseCatalogTest.java \ + camel-support/src/test/java/io/kaoto/backend/api/metadata/catalog/StepCatalogTest.java echo "Updating version info in Readme" - sed -i 's/Camel-kamelets: \*\*.*/Camel-connectors: \*\*'"$versionKamelets"'\*\*/g' README.md + sed -i 's/Camel-kamelets: \*\*.*/Camel-kamelets: \*\*'"$versionKamelets"'\*\*/g' README.md git add README.md rm -rf /tmp/kamelets fi -echo "All resources were updated successfully! Do you want to create commit? (y/n)" +echo "All resources were updated successfully!" + +Yellow='\033[1;33m' +NC='\033[0m' +echo -e "${Yellow}[WARNING] If updated resources are adding some new steps, you will need to update the number of steps manually in test classes (StepCatalogTest, StepResourceTestAbstract)${NC}" + +echo "Do you want to create commit automatically? (y/n)" read -r download if [ "$download" = "y" ]; then