From 7c6675af619ed7e76f11e94f95142588a8143165 Mon Sep 17 00:00:00 2001 From: Miles Mason Winther <42948872+mmwinther@users.noreply.github.com> Date: Tue, 26 Nov 2024 14:02:23 +0100 Subject: [PATCH] Filter catalog services based on keywords (#520) --- .../api/configuration/CatalogWrapper.java | 11 ++ .../api/dao/universe/CatalogLoader.java | 38 +++-- .../api/dao/universe/CatalogLoaderTest.java | 28 +++- .../catalogs.json5 | 36 +++++ .../index.yaml | 131 ++++++++++++++++++ 5 files changed, 232 insertions(+), 12 deletions(-) create mode 100644 onyxia-api/src/test/resources/catalog-loader-test-with-keywords/catalogs.json5 create mode 100644 onyxia-api/src/test/resources/catalog-loader-test-with-keywords/index.yaml diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/configuration/CatalogWrapper.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/configuration/CatalogWrapper.java index c663951c..06bbecc0 100644 --- a/onyxia-api/src/main/java/fr/insee/onyxia/api/configuration/CatalogWrapper.java +++ b/onyxia-api/src/main/java/fr/insee/onyxia/api/configuration/CatalogWrapper.java @@ -51,6 +51,9 @@ public class CatalogWrapper { @Schema(description = "names of declarative important charts of the catalog") private List highlightedCharts = new ArrayList<>(); + @Schema(description = "only include charts with one or more of the given keywords") + private List includeKeywords = new ArrayList<>(); + @Schema(description = "Skip tls certificate checks for the repository") private boolean skipTlsVerify; @@ -194,6 +197,14 @@ public void setHighlightedCharts(List highlightedCharts) { this.highlightedCharts = highlightedCharts; } + public List getIncludeKeywords() { + return includeKeywords; + } + + public void setIncludeKeywords(List includeKeywords) { + this.includeKeywords = includeKeywords; + } + public boolean getSkipTlsVerify() { return skipTlsVerify; } diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java index e0c7f537..2feab6b2 100644 --- a/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java +++ b/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java @@ -15,6 +15,7 @@ import fr.insee.onyxia.model.helm.Repository; import java.io.*; import java.util.*; +import java.util.stream.Collectors; import org.apache.commons.compress.archivers.tar.TarArchiveEntry; import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; @@ -63,17 +64,32 @@ private void updateHelmRepository(CatalogWrapper cw) { .getInputStream(), UTF_8); Repository repository = mapperHelm.readValue(reader, Repository.class); - // Remove excluded services from list - repository - .getEntries() - .entrySet() - .removeIf( - entry -> - cw.getExcludedCharts().stream() - .anyMatch( - excludedChart -> - excludedChart.equalsIgnoreCase( - entry.getKey()))); + + repository.setEntries( + repository.getEntries().entrySet().stream() + .filter( + // Remove explicitly excluded services + entry -> + cw.getExcludedCharts().stream() + .noneMatch( + excludedChart -> + excludedChart.equalsIgnoreCase( + entry.getKey()))) + .filter( + // If includeKeywords is defined, include only services where + // the latest version has the desired keyword. + entry -> + cw.getIncludeKeywords() == null + || cw.getIncludeKeywords().isEmpty() + || cw.getIncludeKeywords().stream() + .anyMatch( + include -> + entry.getValue() + .getFirst() + .getKeywords() + .contains( + include))) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))); // For each service, filter the multiple versions if needed then refresh remaining // versions repository.getEntries().values().parallelStream() diff --git a/onyxia-api/src/test/java/fr/insee/onyxia/api/dao/universe/CatalogLoaderTest.java b/onyxia-api/src/test/java/fr/insee/onyxia/api/dao/universe/CatalogLoaderTest.java index 851bc883..4f852b3f 100644 --- a/onyxia-api/src/test/java/fr/insee/onyxia/api/dao/universe/CatalogLoaderTest.java +++ b/onyxia-api/src/test/java/fr/insee/onyxia/api/dao/universe/CatalogLoaderTest.java @@ -3,7 +3,8 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.is; -import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.params.provider.Arguments.arguments; import fr.insee.onyxia.api.configuration.CatalogWrapper; import fr.insee.onyxia.api.configuration.CustomObjectMapper; @@ -12,10 +13,15 @@ import fr.insee.onyxia.api.util.TestUtils; import fr.insee.onyxia.model.helm.Chart; import java.util.List; +import java.util.Set; import java.util.stream.Collectors; +import java.util.stream.Stream; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.core.io.ResourceLoader; @@ -152,4 +158,24 @@ void packageOnClassPathNotFound() { + "Exception occurred during loading resource: class path resource " + "[catalog-loader-test/keepeme1.gz]")); } + + @ParameterizedTest + @MethodSource("includeKeywords") + void filterIncludeKeywordsTest(List includeKeywords, Set expectedServices) { + CatalogWrapper cw = new CatalogWrapper(); + cw.setType("helm"); + cw.setLocation("classpath:/catalog-loader-test-with-keywords"); + cw.setIncludeKeywords(includeKeywords); + catalogLoader.updateCatalog(cw); + assertEquals(expectedServices, cw.getCatalog().getEntries().keySet()); + } + + private static Stream includeKeywords() { + return Stream.of( + arguments(List.of("CD"), Set.of("keepme")), + arguments(List.of("CD", "Experimental"), Set.of("keepme", "excludeme")), + arguments(List.of(), Set.of("keepme", "excludeme")), + arguments(null, Set.of("keepme", "excludeme")), + arguments(List.of("no one knows"), Set.of())); + } } diff --git a/onyxia-api/src/test/resources/catalog-loader-test-with-keywords/catalogs.json5 b/onyxia-api/src/test/resources/catalog-loader-test-with-keywords/catalogs.json5 new file mode 100644 index 00000000..3ef7dc59 --- /dev/null +++ b/onyxia-api/src/test/resources/catalog-loader-test-with-keywords/catalogs.json5 @@ -0,0 +1,36 @@ +// This file is a json5-file, which onyxia-api should be able to parse +{ + "catalogs": [ + { + "id": "ide", + "name": "IDE", + "description": "Services for datascientists.", + "maintainer": "innovation@insee.fr", + "location": "https://inseefrlab.github.io/helm-charts-interactive-services", + "status": "PROD", + "highlightedCharts": ["jupyter-python", "rstudio", "vscode-python"], + // Single quote should be supported + "timeout": '10m', + "type": 'helm', + /* block comment + over several + lines are supported */ + "skipTlsVerify": false, + "caFile": null, + "allowSharing": false, + "visible": { + "user": true, + // Trailing comma is supported + "project": true, + }, + "restrictions": [ + { + "userAttribute": { + "key": "sub", + "match": "^onyxia.*" + } + } + ] + }, + ], +} diff --git a/onyxia-api/src/test/resources/catalog-loader-test-with-keywords/index.yaml b/onyxia-api/src/test/resources/catalog-loader-test-with-keywords/index.yaml new file mode 100644 index 00000000..0ec72541 --- /dev/null +++ b/onyxia-api/src/test/resources/catalog-loader-test-with-keywords/index.yaml @@ -0,0 +1,131 @@ +apiVersion: v1 +entries: + keepme: + - apiVersion: v2 + appVersion: "1" + created: "2022-10-03T11:53:20.589754116Z" + dependencies: + - name: argo-cd + repository: https://argoproj.github.io/argo-helm + version: 4.9.11 + - name: library-chart + repository: https://inseefrlab.github.io/helm-charts-datascience + version: 2.0.21 + description: Argo CD is a declarative, GitOps continuous delivery tool for Kubernetes. + digest: beedc71c7f5730cf68000b5665dd4da636e72a026edd9b12c1d048398d4166dd + home: https://argo-cd.readthedocs.io/en/stable/ + icon: https://minio.lab.sspcloud.fr/projet-onyxia/assets/servicesImg/argo.png + keywords: + - CD + name: keepme + sources: + - https://github.com/InseeFrLab/helm-charts-datascience/tree/master/charts/argo-cd + - https://github.com/argoproj/argo-helm/tree/master/charts/argo-cd + type: application + urls: + - keepeme1.gz + version: 2.5.1 + maintainers: + - email: test@example.com + name: test + - apiVersion: v2 + appVersion: "1" + created: "2022-10-03T11:53:20.589754116Z" + dependencies: + - name: argo-cd + repository: https://argoproj.github.io/argo-helm + version: 4.9.11 + - name: library-chart + repository: https://inseefrlab.github.io/helm-charts-datascience + version: 2.0.21 + description: Argo CD is a declarative, GitOps continuous delivery tool for Kubernetes. + digest: beedc71c7f5730cf68000b5665dd4da636e72a026edd9b12c1d048398d4166dd + home: https://argo-cd.readthedocs.io/en/stable/ + icon: https://minio.lab.sspcloud.fr/projet-onyxia/assets/servicesImg/argo.png + keywords: + - CD + name: keepme + sources: + - https://github.com/InseeFrLab/helm-charts-datascience/tree/master/charts/argo-cd + - https://github.com/argoproj/argo-helm/tree/master/charts/argo-cd + type: application + urls: + - keepeme1.gz + version: 2.4.1 + maintainers: + - email: test@example.com + name: test + - apiVersion: v2 + appVersion: "2" + created: "2022-10-03T11:53:20.589754116Z" + dependencies: + - name: argo-cd + repository: https://argoproj.github.io/argo-helm + version: 4.9.11 + - name: library-chart + repository: https://inseefrlab.github.io/helm-charts-datascience + version: 2.0.21 + description: Argo CD is a declarative, GitOps continuous delivery tool for Kubernetes. + digest: beedc71c7f5730cf68000b5665dd4da636e72a026edd9b12c1d048398d4166dd + home: https://argo-cd.readthedocs.io/en/stable/ + icon: https://minio.lab.sspcloud.fr/projet-onyxia/assets/servicesImg/argo.png + keywords: + - CD + name: keepme + sources: + - https://github.com/InseeFrLab/helm-charts-datascience/tree/master/charts/argo-cd + - https://github.com/argoproj/argo-helm/tree/master/charts/argo-cd + type: application + urls: + - keepeme2.gz + version: 2.4.0 + excludeme: + - apiVersion: v2 + appVersion: "1" + created: "2022-10-03T11:53:20.589754116Z" + dependencies: + - name: argo-cd + repository: https://argoproj.github.io/argo-helm + version: 4.9.11 + - name: library-chart + repository: https://inseefrlab.github.io/helm-charts-datascience + version: 2.0.21 + description: Argo CD is a declarative, GitOps continuous delivery tool for Kubernetes. + digest: beedc71c7f5730cf68000b5665dd4da636e72a026edd9b12c1d048398d4166dd + home: https://argo-cd.readthedocs.io/en/stable/ + icon: https://minio.lab.sspcloud.fr/projet-onyxia/assets/servicesImg/argo.png + name: excludeme + keywords: + - Experimental + sources: + - https://github.com/InseeFrLab/helm-charts-datascience/tree/master/charts/argo-cd + - https://github.com/argoproj/argo-helm/tree/master/charts/argo-cd + type: application + urls: + - excludeme1.gz + version: 2.4.1 + - apiVersion: v2 + appVersion: "2" + created: "2022-10-03T11:53:20.589754116Z" + dependencies: + - name: argo-cd + repository: https://argoproj.github.io/argo-helm + version: 4.9.11 + - name: library-chart + repository: https://inseefrlab.github.io/helm-charts-datascience + version: 2.0.21 + description: Argo CD is a declarative, GitOps continuous delivery tool for Kubernetes. + digest: beedc71c7f5730cf68000b5665dd4da636e72a026edd9b12c1d048398d4166dd + home: https://argo-cd.readthedocs.io/en/stable/ + icon: https://minio.lab.sspcloud.fr/projet-onyxia/assets/servicesImg/argo.png + name: excludeme + keywords: + - Experimental + - CD + sources: + - https://github.com/InseeFrLab/helm-charts-datascience/tree/master/charts/argo-cd + - https://github.com/argoproj/argo-helm/tree/master/charts/argo-cd + type: application + urls: + - excludeme2.gz + version: 2.4.0