Filter catalog services based on keywords (#520)

onyxia-api/src/main/java/fr/insee/onyxia/api/configuration/CatalogWrapper.java

@@ -51,6 +51,9 @@ public class CatalogWrapper {
     @Schema(description = "names of declarative important charts of the catalog")
     private List<String> highlightedCharts = new ArrayList<>();
 
+    @Schema(description = "only include charts with one or more of the given keywords")
+    private List<String> includeKeywords = new ArrayList<>();
+
     @Schema(description = "Skip tls certificate checks for the repository")
     private boolean skipTlsVerify;
 
@@ -194,6 +197,14 @@ public void setHighlightedCharts(List<String> highlightedCharts) {
         this.highlightedCharts = highlightedCharts;
     }
 
+    public List<String> getIncludeKeywords() {
+        return includeKeywords;
+    }
+
+    public void setIncludeKeywords(List<String> includeKeywords) {
+        this.includeKeywords = includeKeywords;
+    }
+
     public boolean getSkipTlsVerify() {
         return skipTlsVerify;
     }

onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java

@@ -15,6 +15,7 @@
 import fr.insee.onyxia.model.helm.Repository;
 import java.io.*;
 import java.util.*;
+import java.util.stream.Collectors;
 import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
 import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
 import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
@@ -63,17 +64,32 @@ private void updateHelmRepository(CatalogWrapper cw) {
                                     .getInputStream(),
                             UTF_8);
             Repository repository = mapperHelm.readValue(reader, Repository.class);
-            // Remove excluded services from list
-            repository
-                    .getEntries()
-                    .entrySet()
-                    .removeIf(
-                            entry ->
-                                    cw.getExcludedCharts().stream()
-                                            .anyMatch(
-                                                    excludedChart ->
-                                                            excludedChart.equalsIgnoreCase(
-                                                                    entry.getKey())));
+
+            repository.setEntries(
+                    repository.getEntries().entrySet().stream()
+                            .filter(
+                                    // Remove explicitly excluded services
+                                    entry ->
+                                            cw.getExcludedCharts().stream()
+                                                    .noneMatch(
+                                                            excludedChart ->
+                                                                    excludedChart.equalsIgnoreCase(
+                                                                            entry.getKey())))
+                            .filter(
+                                    // If includeKeywords is defined, include only services where
+                                    // the latest version has the desired keyword.
+                                    entry ->
+                                            cw.getIncludeKeywords() == null
+                                                    || cw.getIncludeKeywords().isEmpty()
+                                                    || cw.getIncludeKeywords().stream()
+                                                            .anyMatch(
+                                                                    include ->
+                                                                            entry.getValue()
+                                                                                    .getFirst()
+                                                                                    .getKeywords()
+                                                                                    .contains(
+                                                                                            include)))
+                            .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)));
             // For each service, filter the multiple versions if needed then refresh remaining
             // versions
             repository.getEntries().values().parallelStream()

onyxia-api/src/test/java/fr/insee/onyxia/api/dao/universe/CatalogLoaderTest.java

@@ -3,7 +3,8 @@
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.is;
-import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.*;
+import static org.junit.jupiter.params.provider.Arguments.arguments;
 
 import fr.insee.onyxia.api.configuration.CatalogWrapper;
 import fr.insee.onyxia.api.configuration.CustomObjectMapper;
@@ -12,10 +13,15 @@
 import fr.insee.onyxia.api.util.TestUtils;
 import fr.insee.onyxia.model.helm.Chart;
 import java.util.List;
+import java.util.Set;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 import org.junit.jupiter.api.DisplayName;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.core.io.ResourceLoader;
@@ -152,4 +158,24 @@ void packageOnClassPathNotFound() {
                                 + "Exception occurred during loading resource: class path resource "
                                 + "[catalog-loader-test/keepeme1.gz]"));
     }
+
+    @ParameterizedTest
+    @MethodSource("includeKeywords")
+    void filterIncludeKeywordsTest(List<String> includeKeywords, Set<String> expectedServices) {
+        CatalogWrapper cw = new CatalogWrapper();
+        cw.setType("helm");
+        cw.setLocation("classpath:/catalog-loader-test-with-keywords");
+        cw.setIncludeKeywords(includeKeywords);
+        catalogLoader.updateCatalog(cw);
+        assertEquals(expectedServices, cw.getCatalog().getEntries().keySet());
+    }
+
+    private static Stream<Arguments> includeKeywords() {
+        return Stream.of(
+                arguments(List.of("CD"), Set.of("keepme")),
+                arguments(List.of("CD", "Experimental"), Set.of("keepme", "excludeme")),
+                arguments(List.of(), Set.of("keepme", "excludeme")),
+                arguments(null, Set.of("keepme", "excludeme")),
+                arguments(List.of("no one knows"), Set.of()));
+    }
 }

onyxia-api/src/test/resources/catalog-loader-test-with-keywords/catalogs.json5

@@ -0,0 +1,36 @@
+// This file is a json5-file, which onyxia-api should be able to parse
+{
+  "catalogs": [
+    {
+      "id": "ide",
+      "name": "IDE",
+      "description": "Services for datascientists.",
+      "maintainer": "[email protected]",
+      "location": "https://inseefrlab.github.io/helm-charts-interactive-services",
+      "status": "PROD",
+      "highlightedCharts": ["jupyter-python", "rstudio", "vscode-python"],
+      // Single quote should be supported
+      "timeout": '10m',
+      "type": 'helm',
+      /* block comment
+      over several
+      lines are supported */
+      "skipTlsVerify": false,
+      "caFile": null,
+      "allowSharing": false,
+      "visible": {
+        "user": true,
+        // Trailing comma is supported
+        "project": true,
+      },
+      "restrictions": [
+        {
+          "userAttribute": {
+            "key": "sub",
+            "match": "^onyxia.*"
+          }
+        }
+      ]
+    },
+  ],
+}

onyxia-api/src/test/resources/catalog-loader-test-with-keywords/index.yaml

@@ -0,0 +1,131 @@
+apiVersion: v1
+entries:
+  keepme:
+    - apiVersion: v2
+      appVersion: "1"
+      created: "2022-10-03T11:53:20.589754116Z"
+      dependencies:
+        - name: argo-cd
+          repository: https://argoproj.github.io/argo-helm
+          version: 4.9.11
+        - name: library-chart
+          repository: https://inseefrlab.github.io/helm-charts-datascience
+          version: 2.0.21
+      description: Argo CD is a declarative, GitOps continuous delivery tool for Kubernetes.
+      digest: beedc71c7f5730cf68000b5665dd4da636e72a026edd9b12c1d048398d4166dd
+      home: https://argo-cd.readthedocs.io/en/stable/
+      icon: https://minio.lab.sspcloud.fr/projet-onyxia/assets/servicesImg/argo.png
+      keywords:
+        - CD
+      name: keepme
+      sources:
+        - https://github.com/InseeFrLab/helm-charts-datascience/tree/master/charts/argo-cd
+        - https://github.com/argoproj/argo-helm/tree/master/charts/argo-cd
+      type: application
+      urls:
+        - keepeme1.gz
+      version: 2.5.1
+      maintainers:
+        - email: [email protected]
+          name: test
+    - apiVersion: v2
+      appVersion: "1"
+      created: "2022-10-03T11:53:20.589754116Z"
+      dependencies:
+        - name: argo-cd
+          repository: https://argoproj.github.io/argo-helm
+          version: 4.9.11
+        - name: library-chart
+          repository: https://inseefrlab.github.io/helm-charts-datascience
+          version: 2.0.21
+      description: Argo CD is a declarative, GitOps continuous delivery tool for Kubernetes.
+      digest: beedc71c7f5730cf68000b5665dd4da636e72a026edd9b12c1d048398d4166dd
+      home: https://argo-cd.readthedocs.io/en/stable/
+      icon: https://minio.lab.sspcloud.fr/projet-onyxia/assets/servicesImg/argo.png
+      keywords:
+        - CD
+      name: keepme
+      sources:
+        - https://github.com/InseeFrLab/helm-charts-datascience/tree/master/charts/argo-cd
+        - https://github.com/argoproj/argo-helm/tree/master/charts/argo-cd
+      type: application
+      urls:
+        - keepeme1.gz
+      version: 2.4.1
+      maintainers:
+        - email: [email protected]
+          name: test
+    - apiVersion: v2
+      appVersion: "2"
+      created: "2022-10-03T11:53:20.589754116Z"
+      dependencies:
+        - name: argo-cd
+          repository: https://argoproj.github.io/argo-helm
+          version: 4.9.11
+        - name: library-chart
+          repository: https://inseefrlab.github.io/helm-charts-datascience
+          version: 2.0.21
+      description: Argo CD is a declarative, GitOps continuous delivery tool for Kubernetes.
+      digest: beedc71c7f5730cf68000b5665dd4da636e72a026edd9b12c1d048398d4166dd
+      home: https://argo-cd.readthedocs.io/en/stable/
+      icon: https://minio.lab.sspcloud.fr/projet-onyxia/assets/servicesImg/argo.png
+      keywords:
+        - CD
+      name: keepme
+      sources:
+        - https://github.com/InseeFrLab/helm-charts-datascience/tree/master/charts/argo-cd
+        - https://github.com/argoproj/argo-helm/tree/master/charts/argo-cd
+      type: application
+      urls:
+        - keepeme2.gz
+      version: 2.4.0
+  excludeme:
+    - apiVersion: v2
+      appVersion: "1"
+      created: "2022-10-03T11:53:20.589754116Z"
+      dependencies:
+        - name: argo-cd
+          repository: https://argoproj.github.io/argo-helm
+          version: 4.9.11
+        - name: library-chart
+          repository: https://inseefrlab.github.io/helm-charts-datascience
+          version: 2.0.21
+      description: Argo CD is a declarative, GitOps continuous delivery tool for Kubernetes.
+      digest: beedc71c7f5730cf68000b5665dd4da636e72a026edd9b12c1d048398d4166dd
+      home: https://argo-cd.readthedocs.io/en/stable/
+      icon: https://minio.lab.sspcloud.fr/projet-onyxia/assets/servicesImg/argo.png
+      name: excludeme
+      keywords:
+        - Experimental
+      sources:
+        - https://github.com/InseeFrLab/helm-charts-datascience/tree/master/charts/argo-cd
+        - https://github.com/argoproj/argo-helm/tree/master/charts/argo-cd
+      type: application
+      urls:
+        - excludeme1.gz
+      version: 2.4.1
+    - apiVersion: v2
+      appVersion: "2"
+      created: "2022-10-03T11:53:20.589754116Z"
+      dependencies:
+        - name: argo-cd
+          repository: https://argoproj.github.io/argo-helm
+          version: 4.9.11
+        - name: library-chart
+          repository: https://inseefrlab.github.io/helm-charts-datascience
+          version: 2.0.21
+      description: Argo CD is a declarative, GitOps continuous delivery tool for Kubernetes.
+      digest: beedc71c7f5730cf68000b5665dd4da636e72a026edd9b12c1d048398d4166dd
+      home: https://argo-cd.readthedocs.io/en/stable/
+      icon: https://minio.lab.sspcloud.fr/projet-onyxia/assets/servicesImg/argo.png
+      name: excludeme
+      keywords:
+        - Experimental
+        - CD
+      sources:
+        - https://github.com/InseeFrLab/helm-charts-datascience/tree/master/charts/argo-cd
+        - https://github.com/argoproj/argo-helm/tree/master/charts/argo-cd
+      type: application
+      urls:
+        - excludeme2.gz
+      version: 2.4.0