Fix excluded charts are downloaded for entries

Signed-off-by: CChemin <[email protected]>

onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java

@@ -8,7 +8,6 @@
 import fr.insee.onyxia.model.catalog.Pkg;
 import fr.insee.onyxia.model.helm.Chart;
 import fr.insee.onyxia.model.helm.Repository;
-import io.github.inseefrlab.helmwrapper.service.HelmRepoService;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
@@ -27,18 +26,14 @@
 @Service
 public class CatalogLoader {
 
-    private final Logger logger = LoggerFactory.getLogger(CatalogRefresher.class);
+    private final Logger logger = LoggerFactory.getLogger(CatalogLoader.class);
 
     @Autowired private ResourceLoader resourceLoader;
 
-    @Autowired private ObjectMapper mapper;
-
     @Autowired
     @Qualifier("helm")
     private ObjectMapper mapperHelm;
 
-    @Autowired private HelmRepoService helmRepoService;
-
     public void updateCatalog(CatalogWrapper cw) {
         logger.info("updating catalog with id :" + cw.getId() + " and type " + cw.getType());
         switch (cw.getType()) {
@@ -58,6 +53,16 @@ private void updateHelmRepository(CatalogWrapper cw) {
                                     .getInputStream(),
                             "UTF-8");
             Repository repository = mapperHelm.readValue(reader, Repository.class);
+            repository
+                    .getEntries()
+                    .entrySet()
+                    .removeIf(
+                            entry ->
+                                    cw.getExcludedCharts().stream()
+                                            .anyMatch(
+                                                    excludedChart ->
+                                                            excludedChart.equalsIgnoreCase(
+                                                                    entry.getKey())));
             repository.getEntries().values().parallelStream()
                     .forEach(
                             entry -> {
@@ -100,7 +105,7 @@ private void refreshPackage(CatalogWrapper cw, Pkg pkg) throws IOException {
         inputStream.close();
     }
 
-    public void extractDataFromTgz(InputStream in, Chart chart) throws IOException {
+    private void extractDataFromTgz(InputStream in, Chart chart) throws IOException {
         GzipCompressorInputStream gzipIn = new GzipCompressorInputStream(in);
         // HelmConfig config = null;
         try (TarArchiveInputStream tarIn = new TarArchiveInputStream(gzipIn)) {

onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogRefresher.java

@@ -6,15 +6,13 @@
 import java.util.Timer;
 import java.util.TimerTask;
 import java.util.concurrent.TimeoutException;
-import java.util.stream.Collectors;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.boot.ApplicationArguments;
 import org.springframework.boot.ApplicationRunner;
 import org.springframework.stereotype.Service;
-import org.springframework.util.CollectionUtils;
 
 @Service
 public class CatalogRefresher implements ApplicationRunner {
@@ -38,31 +36,6 @@ private void refresh() {
                                 logger.info(
                                         helmRepoService.addHelmRepo(c.getLocation(), c.getId()));
                                 catalogLoader.updateCatalog(c);
-                                if (c.getCatalog() != null
-                                        && !CollectionUtils.isEmpty(c.getCatalog().getPackages())) {
-                                    c.getCatalog()
-                                            .setPackages(
-                                                    c.getCatalog().getPackages().stream()
-                                                            .filter(
-                                                                    pkg -> {
-                                                                        if (c.getExcludedCharts()
-                                                                                .contains(
-                                                                                        pkg
-                                                                                                .getName())) {
-                                                                            logger.info(
-                                                                                    "Ignoring chart "
-                                                                                            + pkg
-                                                                                                    .getName()
-                                                                                            + " in catalog "
-                                                                                            + c
-                                                                                                    .getName());
-                                                                            return false;
-                                                                        }
-                                                                        return true;
-                                                                    })
-                                                            .collect(Collectors.toList()));
-                                }
-
                             } catch (Exception e) {
                                 e.printStackTrace();
                             }

onyxia-api/src/test/java/fr/insee/onyxia/api/dao/universe/CatalogLoaderTest.java

@@ -0,0 +1,49 @@
+package fr.insee.onyxia.api.dao.universe;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.is;
+
+import fr.insee.onyxia.api.configuration.CatalogWrapper;
+import fr.insee.onyxia.api.configuration.CustomObjectMapper;
+import java.util.List;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.test.context.junit.jupiter.SpringExtension;
+
+@ExtendWith(SpringExtension.class)
+@SpringBootTest(classes = {CatalogLoader.class, CustomObjectMapper.class})
+public class CatalogLoaderTest {
+
+    @Autowired CatalogLoader catalogLoader;
+
+    @DisplayName(
+            "Given a helm catalog wrapper with local charts and excluded charts, "
+                    + "when we update the catalog, "
+                    + "then the excluded chart is not set")
+    @Test
+    public void excludeChartTest() {
+        CatalogWrapper cw = new CatalogWrapper();
+        cw.setType("helm");
+        cw.setLocation("classpath:/catalog-loader-test");
+        cw.setExcludedCharts(List.of("excludemetoo", "excludeme"));
+        catalogLoader.updateCatalog(cw);
+        assertThat(
+                "cw has the not excluded entries",
+                cw.getCatalog().getEntries().get("keepme").size(),
+                is(2));
+        assertThat(
+                "cw has the not excluded package",
+                cw.getCatalog().getPackages().stream()
+                        .anyMatch(p -> p.getName().equalsIgnoreCase("keepme")));
+        assertThat(
+                "cw does not have the excluded entries",
+                !cw.getCatalog().getEntries().containsKey("excludeme"));
+        assertThat(
+                "cw does not have the excluded packages",
+                cw.getCatalog().getPackages().stream()
+                        .noneMatch(p -> p.getName().equalsIgnoreCase("excludeme")));
+    }
+}

onyxia-api/src/test/resources/catalog-loader-test/index.yaml

@@ -0,0 +1,93 @@
+apiVersion: v1
+entries:
+  keepme:
+  - apiVersion: v2
+    appVersion: "1"
+    created: "2022-10-03T11:53:20.589754116Z"
+    dependencies:
+    - name: argo-cd
+      repository: https://argoproj.github.io/argo-helm
+      version: 4.9.11
+    - name: library-chart
+      repository: https://inseefrlab.github.io/helm-charts-datascience
+      version: 2.0.21
+    description: Argo CD is a declarative, GitOps continuous delivery tool for Kubernetes.
+    digest: beedc71c7f5730cf68000b5665dd4da636e72a026edd9b12c1d048398d4166dd
+    home: https://argo-cd.readthedocs.io/en/stable/
+    icon: https://minio.lab.sspcloud.fr/projet-onyxia/assets/servicesImg/argo.png
+    name: keepme
+    sources:
+    - https://github.com/InseeFrLab/helm-charts-datascience/tree/master/charts/argo-cd
+    - https://github.com/argoproj/argo-helm/tree/master/charts/argo-cd
+    type: application
+    urls:
+    - keepeme1.gz
+    version: 2.4.0
+  - apiVersion: v2
+    appVersion: "2"
+    created: "2022-10-03T11:53:20.589754116Z"
+    dependencies:
+    - name: argo-cd
+      repository: https://argoproj.github.io/argo-helm
+      version: 4.9.11
+    - name: library-chart
+      repository: https://inseefrlab.github.io/helm-charts-datascience
+      version: 2.0.21
+    description: Argo CD is a declarative, GitOps continuous delivery tool for Kubernetes.
+    digest: beedc71c7f5730cf68000b5665dd4da636e72a026edd9b12c1d048398d4166dd
+    home: https://argo-cd.readthedocs.io/en/stable/
+    icon: https://minio.lab.sspcloud.fr/projet-onyxia/assets/servicesImg/argo.png
+    name: keepme
+    sources:
+    - https://github.com/InseeFrLab/helm-charts-datascience/tree/master/charts/argo-cd
+    - https://github.com/argoproj/argo-helm/tree/master/charts/argo-cd
+    type: application
+    urls:
+    - keepeme2.gz
+    version: 2.4.0
+  excludeme:
+  - apiVersion: v2
+    appVersion: "1"
+    created: "2022-10-03T11:53:20.589754116Z"
+    dependencies:
+    - name: argo-cd
+      repository: https://argoproj.github.io/argo-helm
+      version: 4.9.11
+    - name: library-chart
+      repository: https://inseefrlab.github.io/helm-charts-datascience
+      version: 2.0.21
+    description: Argo CD is a declarative, GitOps continuous delivery tool for Kubernetes.
+    digest: beedc71c7f5730cf68000b5665dd4da636e72a026edd9b12c1d048398d4166dd
+    home: https://argo-cd.readthedocs.io/en/stable/
+    icon: https://minio.lab.sspcloud.fr/projet-onyxia/assets/servicesImg/argo.png
+    name: excludeme
+    sources:
+    - https://github.com/InseeFrLab/helm-charts-datascience/tree/master/charts/argo-cd
+    - https://github.com/argoproj/argo-helm/tree/master/charts/argo-cd
+    type: application
+    urls:
+    - excludeme1.gz
+    version: 2.4.0
+  - apiVersion: v2
+    appVersion: "2"
+    created: "2022-10-03T11:53:20.589754116Z"
+    dependencies:
+    - name: argo-cd
+      repository: https://argoproj.github.io/argo-helm
+      version: 4.9.11
+    - name: library-chart
+      repository: https://inseefrlab.github.io/helm-charts-datascience
+      version: 2.0.21
+    description: Argo CD is a declarative, GitOps continuous delivery tool for Kubernetes.
+    digest: beedc71c7f5730cf68000b5665dd4da636e72a026edd9b12c1d048398d4166dd
+    home: https://argo-cd.readthedocs.io/en/stable/
+    icon: https://minio.lab.sspcloud.fr/projet-onyxia/assets/servicesImg/argo.png
+    name: excludeme
+    sources:
+    - https://github.com/InseeFrLab/helm-charts-datascience/tree/master/charts/argo-cd
+    - https://github.com/argoproj/argo-helm/tree/master/charts/argo-cd
+    type: application
+    urls:
+    - excludeme2.gz
+    version: 2.4.0
+