Merge remote-tracking branch 'origin/main' into get-error-from-helm

README.md

@@ -115,6 +115,8 @@ Configurable properties :
 | `logging.structured.format.console` | `` | Format for structured logging. Valid values : `ecs`, `gelf`, `logstash`. Leave empty for no structured logging (default). See https://docs.spring.io/spring-boot/reference/features/logging.html#features.logging.structured  |
 | `springdoc.swagger-ui.path` | `/` | Open API (swagger) UI path |
 | `springdoc.swagger-ui.oauth.clientId` | `` | clientid used by swagger to authenticate the user, in general the same which is used by onyxia-ui is ok. |
+| `DEBUG_JMX` | `` | Enable JMX monitoring. This is useful for profiling the app to improve performance but is not intended for production / daily use. Once enabled (`true`), use `kubectl port-forward` + a profiler (e.g VisualVM) to profile the app. |
+| `JMX_PORT` | `10000` | Port used by JMX if enabled. |
 
 ## Onyxia API dependency to Helm
 

helm-wrapper/pom.xml

@@ -61,8 +61,22 @@
         <dependency>
             <groupId>io.fabric8</groupId>
             <artifactId>kubernetes-client</artifactId>
+            <exclusions>
+                <exclusion>
+                    <groupId>io.fabric8</groupId>
+                    <artifactId>kubernetes-httpclient-vertx</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
 
+        <dependency>
+            <groupId>io.fabric8</groupId>
+            <artifactId>kubernetes-httpclient-okhttp</artifactId>
+            <version>7.0.0</version>
+            <scope>runtime</scope>
+        </dependency>
+
+
         <dependency>
             <groupId>org.apache.commons</groupId>
             <artifactId>commons-lang3</artifactId>

onyxia-api/entrypoint.sh

@@ -13,4 +13,9 @@ if [[ -n "$CACERTS_DIR" ]]; then
 fi
 
 # Run application
-java org.springframework.boot.loader.launch.JarLauncher
+if [ -n "$DEBUG_JMX" ]; then
+  JMX_PORT="${JMX_PORT:-10000}"
+  java -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=$JMX_PORT -Dcom.sun.management.jmxremote.rmi.port=$JMX_PORT -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.local.only=false -Djava.rmi.server.hostname=127.0.0.1 org.springframework.boot.loader.launch.JarLauncher
+else
+  java org.springframework.boot.loader.launch.JarLauncher
+fi

onyxia-api/src/main/java/fr/insee/onyxia/api/configuration/CatalogWrapper.java

@@ -8,7 +8,9 @@
 import fr.insee.onyxia.model.helm.Repository;
 import io.swagger.v3.oas.annotations.media.Schema;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.regex.Pattern;
 
 @JsonIgnoreProperties(ignoreUnknown = true)
@@ -54,6 +56,15 @@ public class CatalogWrapper {
     @Schema(description = "only include charts with one or more of the given keywords")
     private List<String> includeKeywords = new ArrayList<>();
 
+    @Schema(description = "exclude any charts which have one or more of the given keywords")
+    private List<String> excludeKeywords = new ArrayList<>();
+
+    @Schema(description = "only include charts with one or more of the given annotations")
+    private Map<String, String> includeAnnotations = new HashMap<>();
+
+    @Schema(description = "exclude any charts which have one or more of the given annotations")
+    private Map<String, String> excludeAnnotations = new HashMap<>();
+
     @Schema(description = "Skip tls certificate checks for the repository")
     private boolean skipTlsVerify;
 
@@ -205,6 +216,30 @@ public void setIncludeKeywords(List<String> includeKeywords) {
         this.includeKeywords = includeKeywords;
     }
 
+    public void setExcludeKeywords(List<String> excludeKeywords) {
+        this.excludeKeywords = excludeKeywords;
+    }
+
+    public List<String> getExcludeKeywords() {
+        return excludeKeywords;
+    }
+
+    public Map<String, String> getIncludeAnnotations() {
+        return includeAnnotations;
+    }
+
+    public void setIncludeAnnotations(Map<String, String> includeAnnotations) {
+        this.includeAnnotations = includeAnnotations;
+    }
+
+    public Map<String, String> getExcludeAnnotations() {
+        return excludeAnnotations;
+    }
+
+    public void setExcludeAnnotations(Map<String, String> excludeAnnotations) {
+        this.excludeAnnotations = excludeAnnotations;
+    }
+
     public boolean getSkipTlsVerify() {
         return skipTlsVerify;
     }

onyxia-api/src/main/java/fr/insee/onyxia/api/configuration/kubernetes/KubernetesClientProvider.java

@@ -6,6 +6,8 @@
 import io.fabric8.kubernetes.client.ConfigBuilder;
 import io.fabric8.kubernetes.client.KubernetesClient;
 import io.fabric8.kubernetes.client.KubernetesClientBuilder;
+import java.util.HashMap;
+import java.util.Map;
 import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -16,19 +18,39 @@ public class KubernetesClientProvider {
 
     private static final Logger LOGGER = LoggerFactory.getLogger(KubernetesClientProvider.class);
 
+    private static Map<String, KubernetesClient> rootKubernetesClientCache = new HashMap<>();
+
+    private static KubernetesClient userKubernetesClientCache = null;
+
     /**
      * This returns the root client which has extended permissions. Currently cluster-admin. User
      * calls should be done using the userClient which only has user permissions.
      *
      * @param region
      * @return
      */
-    public KubernetesClient getRootClient(Region region) {
-        final Config config = getDefaultConfiguration(region).build();
-        return new KubernetesClientBuilder().withConfig(config).build();
+    public synchronized KubernetesClient getRootClient(Region region) {
+        if (!rootKubernetesClientCache.containsKey(region.getId())) {
+            final Config config = getDefaultConfiguration(region).build();
+            rootKubernetesClientCache.put(
+                    region.getId(), new KubernetesClientBuilder().withConfig(config).build());
+        }
+        return rootKubernetesClientCache.get(region.getId());
     }
 
     public KubernetesClient getUserClient(Region region, User user) {
+        // In case of SERVICEACCOUNT authentication, we can safely mutualize and use a single
+        // KubernetesClient
+        if (region.getServices().getAuthenticationMode()
+                == Region.Services.AuthenticationMode.SERVICEACCOUNT) {
+            if (userKubernetesClientCache != null) {
+                return userKubernetesClientCache;
+            }
+            Config config = getDefaultConfiguration(region).build();
+            KubernetesClient client = new KubernetesClientBuilder().withConfig(config).build();
+            userKubernetesClientCache = client;
+            return client;
+        }
         final Config config = getDefaultConfiguration(region).build();
         String username = user.getIdep();
         if (region.getServices().getUsernamePrefix() != null) {

onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java

@@ -29,6 +29,7 @@
 import org.springframework.core.io.Resource;
 import org.springframework.core.io.ResourceLoader;
 import org.springframework.stereotype.Service;
+import org.springframework.util.CollectionUtils;
 import org.springframework.util.StringUtils;
 
 @Service
@@ -76,19 +77,33 @@ private void updateHelmRepository(CatalogWrapper cw) {
                                                                     excludedChart.equalsIgnoreCase(
                                                                             entry.getKey())))
                             .filter(
-                                    // If includeKeywords is defined, include only services where
-                                    // the latest version has the desired keyword.
                                     entry ->
-                                            cw.getIncludeKeywords() == null
-                                                    || cw.getIncludeKeywords().isEmpty()
-                                                    || cw.getIncludeKeywords().stream()
-                                                            .anyMatch(
-                                                                    include ->
-                                                                            entry.getValue()
-                                                                                    .getFirst()
-                                                                                    .getKeywords()
-                                                                                    .contains(
-                                                                                            include)))
+                                            (CollectionUtils.isEmpty(cw.getIncludeKeywords())
+                                                            || entry.getValue()
+                                                                    .getFirst()
+                                                                    .hasKeywords(
+                                                                            cw
+                                                                                    .getIncludeKeywords()))
+                                                    && (CollectionUtils.isEmpty(
+                                                                    cw.getIncludeAnnotations())
+                                                            || entry.getValue()
+                                                                    .getFirst()
+                                                                    .hasAnnotations(
+                                                                            cw
+                                                                                    .getIncludeAnnotations())))
+                            .filter(
+                                    entry ->
+                                            CollectionUtils.isEmpty(cw.getExcludeKeywords())
+                                                    || !entry.getValue()
+                                                            .getFirst()
+                                                            .hasKeywords(cw.getExcludeKeywords()))
+                            .filter(
+                                    entry ->
+                                            CollectionUtils.isEmpty(cw.getExcludeAnnotations())
+                                                    || !entry.getValue()
+                                                            .getFirst()
+                                                            .hasAnnotations(
+                                                                    cw.getExcludeAnnotations()))
                             .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)));
             // For each service, filter the multiple versions if needed then refresh remaining
             // versions

onyxia-api/src/main/java/fr/insee/onyxia/api/security/LogUserInfoFilter.java

@@ -0,0 +1,42 @@
+package fr.insee.onyxia.api.security;
+
+import fr.insee.onyxia.api.configuration.properties.RegionsConfiguration;
+import fr.insee.onyxia.api.services.UserProvider;
+import jakarta.servlet.FilterChain;
+import jakarta.servlet.ServletException;
+import jakarta.servlet.http.HttpServletRequest;
+import jakarta.servlet.http.HttpServletResponse;
+import java.io.IOException;
+import org.slf4j.MDC;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+import org.springframework.web.filter.OncePerRequestFilter;
+
+@Component
+public class LogUserInfoFilter extends OncePerRequestFilter {
+
+    private UserProvider userProvider;
+
+    private RegionsConfiguration regionsConfiguration;
+
+    @Autowired
+    public LogUserInfoFilter(UserProvider userProvider, RegionsConfiguration regionsConfiguration) {
+        this.userProvider = userProvider;
+        this.regionsConfiguration = regionsConfiguration;
+    }
+
+    @Override
+    protected void doFilterInternal(
+            HttpServletRequest request, HttpServletResponse response, FilterChain filterChain)
+            throws ServletException, IOException {
+        try {
+            MDC.put(
+                    "username",
+                    userProvider.getUser(regionsConfiguration.getDefaultRegion()).getIdep());
+        } catch (Exception ignored) {
+
+        }
+        filterChain.doFilter(request, response);
+        MDC.clear();
+    }
+}

onyxia-api/src/test/java/fr/insee/onyxia/api/UserControllerTest.java

@@ -12,6 +12,7 @@
 import fr.insee.onyxia.api.configuration.SecurityConfig;
 import fr.insee.onyxia.api.configuration.properties.RegionsConfiguration;
 import fr.insee.onyxia.api.controller.api.user.UserController;
+import fr.insee.onyxia.api.services.UserProvider;
 import fr.insee.onyxia.api.services.utils.HttpRequestUtils;
 import fr.insee.onyxia.api.user.OnyxiaUserProvider;
 import fr.insee.onyxia.model.OnyxiaUser;
@@ -41,6 +42,8 @@ public class UserControllerTest extends BaseTest {
 
     @MockBean private OnyxiaUserProvider onyxiaUserProvider;
 
+    @MockBean private UserProvider userProvider;
+
     @BeforeEach
     public void setUp() {
         User user = new User();