From f6ff68e82b90a495912cc355968458e7815c1083 Mon Sep 17 00:00:00 2001 From: Comte Date: Thu, 11 Jul 2024 12:42:46 +0200 Subject: [PATCH 01/53] init --- .../api/controller/pub/CatalogController.java | 52 -- .../api/dao/universe/CatalogLoader.java | 10 +- .../onyxia/model/catalog/Config/Category.java | 72 --- .../onyxia/model/catalog/Config/Config.java | 42 -- .../onyxia/model/catalog/Config/Property.java | 471 ------------------ .../fr/insee/onyxia/model/catalog/Pkg.java | 18 +- 6 files changed, 17 insertions(+), 648 deletions(-) delete mode 100644 onyxia-model/src/main/java/fr/insee/onyxia/model/catalog/Config/Category.java delete mode 100644 onyxia-model/src/main/java/fr/insee/onyxia/model/catalog/Config/Config.java delete mode 100644 onyxia-model/src/main/java/fr/insee/onyxia/model/catalog/Config/Property.java diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/pub/CatalogController.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/pub/CatalogController.java index f98c1ec3..abfd8ff7 100644 --- a/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/pub/CatalogController.java +++ b/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/pub/CatalogController.java @@ -82,7 +82,6 @@ public Pkg getPackage(@PathVariable String catalogId, @PathVariable String packa catalogService .getPackage(catalogId, packageName) .orElseThrow(NotFoundException::new); - addCustomOnyxiaProperties(pkg); return pkg; } @@ -116,7 +115,6 @@ public Chart getChartByVersion( catalogService .getChartByVersion(catalogId, chartName, version) .orElseThrow(NotFoundException::new); - addCustomOnyxiaProperties(chart); return chart; } @@ -140,57 +138,7 @@ public Chart getChartByVersion( public List getCharts(@PathVariable String catalogId, @PathVariable String chartName) { List charts = catalogService.getCharts(catalogId, chartName).orElseThrow(NotFoundException::new); - charts.forEach(this::addCustomOnyxiaProperties); return charts; } - private void addCustomOnyxiaProperties(Pkg pkg) { - Map properties = pkg.getConfig().getProperties().getProperties(); - Property property = new Property(); - property.setDescription("Onyxia specific configuration"); - property.setType("object"); - property.setProperties(new HashMap<>()); - Map onyxiaProperties = new HashMap<>(); - Property customName = new Property(); - String stringType = "string"; - customName.setType(stringType); - customName.setDescription("Service custom name"); - customName.setDefaut(pkg.getName()); - customName.setTitle("Custom name"); - onyxiaProperties.put("friendlyName", customName); - Property userDefinedValues = new Property(); - userDefinedValues.setType(stringType); - userDefinedValues.setDescription("Values defined by the end user"); - userDefinedValues.setDefaut(""); - userDefinedValues.setTitle("User defined values"); - XOnyxia xonyxiaUserDefinedValues = new XOnyxia(); - xonyxiaUserDefinedValues.setHidden(true); - userDefinedValues.setXonyxia(xonyxiaUserDefinedValues); - onyxiaProperties.put("userDefinedValues", userDefinedValues); - Property owner = new Property(); - owner.setType(stringType); - owner.setDescription("Owner of the chart"); - owner.setDefaut("owner"); - owner.setTitle("Owner"); - XForm xform = new XForm(); - xform.setValue("{{user.idep}}"); - xform.setHidden(true); - owner.setXform(xform); - XOnyxia xonyxia = new XOnyxia(); - xonyxia.setOverwriteDefaultWith("{{user.idep}}"); - xonyxia.setHidden(true); - owner.setXonyxia(xonyxia); - onyxiaProperties.put("owner", owner); - - Property share = new Property(); - share.setType("boolean"); - share.setDescription("Enable share for this service"); - share.setDefaut(false); - share.setTitle("Share"); - onyxiaProperties.put("share", share); - - property.setProperties(onyxiaProperties); - - properties.put("onyxia", property); - } } diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java index 706b0b06..24ab2449 100644 --- a/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java +++ b/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java @@ -15,6 +15,7 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.io.Reader; +import java.io.ByteArrayOutputStream; import java.util.Iterator; import java.util.List; import java.util.Optional; @@ -186,8 +187,13 @@ public void extractDataFromTgz(InputStream in, Chart chart) throws IOException { ObjectMapper mapper = new ObjectMapper(); mapper.configure(JsonParser.Feature.AUTO_CLOSE_SOURCE, false); mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - Config config = mapper.readValue(tarIn, Config.class); - chart.setConfig(config); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + byte[] buffer = new byte[1024]; + int len; + while ((len = tarIn.read(buffer)) != -1) { + baos.write(buffer, 0, len); + } + chart.setSchema(objectMapper.readTree(baos.toString("UTF-8"));); } } } diff --git a/onyxia-model/src/main/java/fr/insee/onyxia/model/catalog/Config/Category.java b/onyxia-model/src/main/java/fr/insee/onyxia/model/catalog/Config/Category.java deleted file mode 100644 index 505745c7..00000000 --- a/onyxia-model/src/main/java/fr/insee/onyxia/model/catalog/Config/Category.java +++ /dev/null @@ -1,72 +0,0 @@ -package fr.insee.onyxia.model.catalog.Config; - -import com.fasterxml.jackson.annotation.JsonAnyGetter; -import com.fasterxml.jackson.annotation.JsonAnySetter; -import io.swagger.v3.oas.annotations.media.Schema; -import java.util.LinkedHashMap; -import java.util.Map; - -/** Properties */ -@Schema(description = "") -public class Category { - - @Schema(description = "") - Map properties = new LinkedHashMap<>(); - - @Schema(description = "") - String description; - - @Schema(description = "") - String type; - - @Schema(description = "") - String[] required; - - public String getDescription() { - return description; - } - - public void setDescription(String description) { - this.description = description; - } - - public String getType() { - return type; - } - - public void setType(String type) { - this.type = type; - } - - @JsonAnyGetter - public Map getProperties() { - // Map map = new HashMap<>(); - // for (Entry entry : properties.entrySet()) { - // //if (entry.getValue().getApiDefined() == null) { - // map.put(entry.getKey(), entry.getValue()); - // } - // } - return properties; - } - - // @Transient - // public Map getAllProperties() { - // return properties; - // } - public String[] getRequired() { - return required; - } - - public void setRequired(String[] required) { - this.required = required; - } - - @JsonAnySetter - public void setUnrecognizedFields(String key, Property value) { - this.properties.put(key, value); - } - - public void setProperties(Map properties) { - this.properties = properties; - } -} diff --git a/onyxia-model/src/main/java/fr/insee/onyxia/model/catalog/Config/Config.java b/onyxia-model/src/main/java/fr/insee/onyxia/model/catalog/Config/Config.java deleted file mode 100644 index 1ffa4003..00000000 --- a/onyxia-model/src/main/java/fr/insee/onyxia/model/catalog/Config/Config.java +++ /dev/null @@ -1,42 +0,0 @@ -package fr.insee.onyxia.model.catalog.Config; - -import io.swagger.v3.oas.annotations.media.Schema; -import java.util.List; - -/** Config */ -@Schema(description = "") -public class Config { - - @Schema(description = "") - private String type; - - @Schema(description = "") - private Category properties; - - @Schema(description = "") - private List required = null; - - public String getType() { - return type; - } - - public void setType(String type) { - this.type = type; - } - - public Category getProperties() { - return properties; - } - - public void setProperties(Category properties) { - this.properties = properties; - } - - public List getRequired() { - return required; - } - - public void setRequired(List required) { - this.required = required; - } -} diff --git a/onyxia-model/src/main/java/fr/insee/onyxia/model/catalog/Config/Property.java b/onyxia-model/src/main/java/fr/insee/onyxia/model/catalog/Config/Property.java deleted file mode 100644 index a681c199..00000000 --- a/onyxia-model/src/main/java/fr/insee/onyxia/model/catalog/Config/Property.java +++ /dev/null @@ -1,471 +0,0 @@ -package fr.insee.onyxia.model.catalog.Config; - -import com.fasterxml.jackson.annotation.JsonAlias; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonProperty; -import io.swagger.v3.oas.annotations.media.Schema; -import java.util.Map; - -@JsonIgnoreProperties(ignoreUnknown = true) -@Schema(description = "") -public class Property { - @Schema(description = "") - String type; - - @Schema(description = "") - String description; - - @Schema(description = "") - String title; - - @Schema(description = "") - @JsonProperty("default") - Object defaut; - - @Schema(description = "") - String pattern; - - @Schema(description = "") - Media media; - - @Schema(description = "") - String minimum; - - @Schema(description = "") - String render; - - @Schema(description = "") - Integer sliderMin; - - @Schema(description = "") - Integer sliderMax; - - @Schema(description = "") - Integer sliderStep; - - @Schema(description = "") - String sliderUnit; - - @Schema(description = "") - String sliderExtremity; - - @Schema(description = "") - String sliderExtremitySemantic; - - @Schema(description = "") - String sliderRangeId; - - @Schema(description = "") - Hidden hidden; - - @JsonProperty("enum") - @Schema(description = "") - Object enumeration; - - @JsonProperty("listEnum") - @Schema(description = "") - Object listEnumeration; - - @Schema(description = "") - Map properties; - - @JsonProperty("x-form") - @Schema(description = "") - private XForm xform; - - @JsonProperty("x-security") - @Schema(description = "") - private XSecurity xsecurity; - - @JsonProperty("x-generated") - @Schema(description = "") - private XGenerated xGenerated; - - @JsonProperty("x-onyxia") - @Schema(description = "") - private XOnyxia xonyxia; - - public XForm getXform() { - return xform; - } - - public void setXform(XForm xform) { - this.xform = xform; - } - - public XOnyxia getXonyxia() { - return xonyxia; - } - - public void setXonyxia(XOnyxia xonyxia) { - this.xonyxia = xonyxia; - } - - public XSecurity getXsecurity() { - return xsecurity; - } - - public void setXsecurity(XSecurity xsecurity) { - this.xsecurity = xsecurity; - } - - public Map getProperties() { - return properties; - } - - public void setProperties(Map properties) { - this.properties = properties; - } - - public String getType() { - return type; - } - - public void setType(String type) { - this.type = type; - } - - public String getTitle() { - return title; - } - - public void setTitle(String title) { - this.title = title; - } - - public String getDescription() { - return description; - } - - public void setDescription(String description) { - this.description = description; - } - - public String getPattern() { - return pattern; - } - - public void setPattern(String pattern) { - this.pattern = pattern; - } - - public String getRender() { - return render; - } - - public void setRender(String render) { - this.render = render; - } - - public Integer getSliderMin() { - return sliderMin; - } - - public void setSliderMin(Integer sliderMin) { - this.sliderMin = sliderMin; - } - - public Integer getSliderMax() { - return sliderMax; - } - - public void setSliderMax(Integer sliderMax) { - this.sliderMax = sliderMax; - } - - public Integer getSliderStep() { - return sliderStep; - } - - public void setSliderStep(Integer sliderStep) { - this.sliderStep = sliderStep; - } - - public String getSliderUnit() { - return sliderUnit; - } - - public void setSliderUnit(String sliderUnit) { - this.sliderUnit = sliderUnit; - } - - public String getSliderExtremity() { - return sliderExtremity; - } - - public void setSliderExtremity(String sliderExtremity) { - this.sliderExtremity = sliderExtremity; - } - - public String getSliderExtremitySemantic() { - return sliderExtremitySemantic; - } - - public void setSliderExtremitySemantic(String sliderExtremitySemantic) { - this.sliderExtremitySemantic = sliderExtremitySemantic; - } - - public String getSliderRangeId() { - return sliderRangeId; - } - - public void setSliderRangeId(String sliderRangeId) { - this.sliderRangeId = sliderRangeId; - } - - public Hidden getHidden() { - return hidden; - } - - public void setHidden(Hidden hidden) { - this.hidden = hidden; - } - - public Media getMedia() { - return media; - } - - public void setMedia(Media media) { - this.media = media; - } - - public Object getDefaut() { - return defaut; - } - - public void setDefaut(Object defaut) { - this.defaut = defaut; - } - - public Object getEnumeration() { - return enumeration; - } - - public void setEnumeration(Object enumeration) { - this.enumeration = enumeration; - } - - public Object getListEnumeration() { - return listEnumeration; - } - - public void setListEnumeration(Object listEnumeration) { - this.listEnumeration = listEnumeration; - } - - public String getMinimum() { - return minimum; - } - - public void setMinimum(String minimum) { - this.minimum = minimum; - } - - public XGenerated getxGenerated() { - return xGenerated; - } - - public void setxGenerated(XGenerated xGenerated) { - this.xGenerated = xGenerated; - } - - @JsonIgnoreProperties(ignoreUnknown = true) - @Schema(description = "") - public static class Media { - String type; - - public String getType() { - return type; - } - - public void setType(String type) { - this.type = type; - } - } - - @JsonIgnoreProperties(ignoreUnknown = true) - @Schema(description = "") - public static class Hidden { - Object value; - String path; - - public Object getValue() { - return value; - } - - public void setValue(Object value) { - this.value = value; - } - - public String getPath() { - return path; - } - - public void setPath(String path) { - this.path = path; - } - } - - @JsonIgnoreProperties(ignoreUnknown = true) - public static class XSecurity { - - String pattern; - - public String getPattern() { - return pattern; - } - - public void setPattern(String pattern) { - this.pattern = pattern; - } - } - - @JsonIgnoreProperties(ignoreUnknown = true) - public static class XForm { - - boolean hidden = false; - boolean readonly = false; - String value; - - public boolean isHidden() { - return hidden; - } - - public void setHidden(boolean hidden) { - this.hidden = hidden; - } - - public boolean isReadonly() { - return readonly; - } - - public void setReadonly(boolean readonly) { - this.readonly = readonly; - } - - public String getValue() { - return value; - } - - public void setValue(String value) { - this.value = value; - } - } - - @JsonIgnoreProperties(ignoreUnknown = true) - public static class XOnyxia { - - boolean hidden = false; - boolean readonly = false; - - Object overwriteDefaultWith; - Object overwriteListEnumWith; - - String useRegionSliderConfig; - Object formFieldLabel; - Object formFieldHelperText; - - public boolean isHidden() { - return hidden; - } - - public void setHidden(boolean hidden) { - this.hidden = hidden; - } - - public boolean isReadonly() { - return readonly; - } - - public void setReadonly(boolean readonly) { - this.readonly = readonly; - } - - public Object getOverwriteDefaultWith() { - return overwriteDefaultWith; - } - - public void setOverwriteDefaultWith(Object overwriteDefaultWith) { - this.overwriteDefaultWith = overwriteDefaultWith; - } - - public Object getOverwriteListEnumWith() { - return overwriteListEnumWith; - } - - public void setOverwriteListEnumWith(Object overwriteListEnumWith) { - this.overwriteListEnumWith = overwriteListEnumWith; - } - - public String getUseRegionSliderConfig() { - return useRegionSliderConfig; - } - - public void setUseRegionSliderConfig(String useRegionSliderConfig) { - this.useRegionSliderConfig = useRegionSliderConfig; - } - - public Object getFormFieldLabel() { - return formFieldLabel; - } - - public void setFormFieldLabel(Object formFieldLabel) { - this.formFieldLabel = formFieldLabel; - } - - public Object getFormFieldHelperText() { - return formFieldHelperText; - } - - public void setFormFieldHelperText(Object formFieldHelperText) { - this.formFieldHelperText = formFieldHelperText; - } - } - - @JsonIgnoreProperties(ignoreUnknown = true) - public static class XGenerated { - - private XGeneratedType type; - private String scope; - private String name; - - public XGeneratedType getType() { - return type; - } - - public void setType(XGeneratedType type) { - this.type = type; - } - - public String getScope() { - return scope; - } - - public void setScope(String scope) { - this.scope = scope; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public static enum XGeneratedType { - GroupID, - AppID, - RandomID, - - @JsonProperty("internalDNS") - @JsonAlias("internal-DNS") - InternalDNS, - @JsonProperty("externalDNS") - @JsonAlias("external-DNS") - ExternalDNS, - @JsonProperty("initScript") - InitScript; - } - } -} diff --git a/onyxia-model/src/main/java/fr/insee/onyxia/model/catalog/Pkg.java b/onyxia-model/src/main/java/fr/insee/onyxia/model/catalog/Pkg.java index ba0b3743..7f324966 100644 --- a/onyxia-model/src/main/java/fr/insee/onyxia/model/catalog/Pkg.java +++ b/onyxia-model/src/main/java/fr/insee/onyxia/model/catalog/Pkg.java @@ -1,4 +1,5 @@ package fr.insee.onyxia.model.catalog; +import com.fasterxml.jackson.databind.JsonNode; import fr.insee.onyxia.model.catalog.Config.Config; import io.swagger.v3.oas.annotations.media.Schema; @@ -15,15 +16,7 @@ public class Pkg { private String version; @Schema(description = "") - private Config config; - - public void setConfig(Config config) { - this.config = config; - } - - public Config getConfig() { - return config; - } + private JsonNode schema; public String getName() { return this.name; @@ -59,6 +52,13 @@ public void setVersion(String version) { this.version = version; } + public JsonNode getSchema() { + return this.schema; + } + + public void setSchema(JsonNode schema) { + this.schema = schema; + } public Pkg version(String version) { this.version = version; return this; From 5aa57ae2d6a2f6b749486b886c335d1bcf38a4bb Mon Sep 17 00:00:00 2001 From: Comte Date: Thu, 11 Jul 2024 12:46:05 +0200 Subject: [PATCH 02/53] fix --- .../java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java | 1 - .../src/main/java/fr/insee/onyxia/model/catalog/Pkg.java | 1 - 2 files changed, 2 deletions(-) diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java index 24ab2449..44316657 100644 --- a/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java +++ b/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java @@ -7,7 +7,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.github.zafarkhaja.semver.Version; import fr.insee.onyxia.api.configuration.CatalogWrapper; -import fr.insee.onyxia.model.catalog.Config.Config; import fr.insee.onyxia.model.catalog.Pkg; import fr.insee.onyxia.model.helm.Chart; import fr.insee.onyxia.model.helm.Repository; diff --git a/onyxia-model/src/main/java/fr/insee/onyxia/model/catalog/Pkg.java b/onyxia-model/src/main/java/fr/insee/onyxia/model/catalog/Pkg.java index 7f324966..a2dba9cb 100644 --- a/onyxia-model/src/main/java/fr/insee/onyxia/model/catalog/Pkg.java +++ b/onyxia-model/src/main/java/fr/insee/onyxia/model/catalog/Pkg.java @@ -1,7 +1,6 @@ package fr.insee.onyxia.model.catalog; import com.fasterxml.jackson.databind.JsonNode; -import fr.insee.onyxia.model.catalog.Config.Config; import io.swagger.v3.oas.annotations.media.Schema; @Schema(description = "") From 175c6708da5dcd938b151e4573de82eb64ecc32d Mon Sep 17 00:00:00 2001 From: Comte Date: Thu, 11 Jul 2024 12:49:05 +0200 Subject: [PATCH 03/53] fix test --- .../onyxia/model/catalog/PropertyTest.java | 121 ------------------ 1 file changed, 121 deletions(-) delete mode 100644 onyxia-model/src/test/java/fr/insee/onyxia/model/catalog/PropertyTest.java diff --git a/onyxia-model/src/test/java/fr/insee/onyxia/model/catalog/PropertyTest.java b/onyxia-model/src/test/java/fr/insee/onyxia/model/catalog/PropertyTest.java deleted file mode 100644 index 5fc9979b..00000000 --- a/onyxia-model/src/test/java/fr/insee/onyxia/model/catalog/PropertyTest.java +++ /dev/null @@ -1,121 +0,0 @@ -package fr.insee.onyxia.model.catalog; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.module.SimpleModule; -import fr.insee.onyxia.model.catalog.Config.Property; -import java.io.IOException; -import java.util.Arrays; -import java.util.List; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -class PropertyTest { - - private ObjectMapper mapper; - private Property property; - - @BeforeEach - void setUp() { - mapper = new ObjectMapper(); - mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - - SimpleModule module = new SimpleModule(); - mapper.registerModule(module); - - property = new Property(); - property.setType("string"); - property.setDescription("A sample description"); - property.setTitle("A sample title"); - property.setDefaut("default value"); - property.setPattern(".*"); - property.setMinimum("1"); - property.setRender("render"); - property.setSliderMin(0); - property.setSliderMax(100); - property.setSliderStep(1); - property.setSliderUnit("units"); - property.setSliderExtremity("extremity"); - property.setSliderExtremitySemantic("semantic"); - property.setSliderRangeId("rangeId"); - - Property.XOnyxia xOnyxia = new Property.XOnyxia(); - xOnyxia.setOverwriteDefaultWith("user.email"); - xOnyxia.setOverwriteListEnumWith("user.decodedIdToken.groups"); - xOnyxia.setUseRegionSliderConfig("config"); - xOnyxia.setFormFieldLabel("label"); - xOnyxia.setFormFieldHelperText("helper text"); - property.setXonyxia(xOnyxia); - } - - @Test - void testSerialization() throws IOException { - String json = mapper.writeValueAsString(property); - System.out.println("Serialized JSON: \n" + json); - assertTrue(json.contains("\"type\":\"string\"")); - assertTrue(json.contains("\"description\":\"A sample description\"")); - assertTrue(json.contains("\"title\":\"A sample title\"")); - } - - @Test - void testDeserialization() throws IOException { - String json = mapper.writeValueAsString(property); - Property deserializedProperty = mapper.readValue(json, Property.class); - assertEquals(property.getType(), deserializedProperty.getType()); - assertEquals(property.getDescription(), deserializedProperty.getDescription()); - assertEquals(property.getTitle(), deserializedProperty.getTitle()); - } - - @Test - void testListEnumProperty() throws IOException { - Property pullPolicyProperty = new Property(); - pullPolicyProperty.setType("string"); - pullPolicyProperty.setDefaut("IfNotPresent"); - pullPolicyProperty.setListEnumeration(Arrays.asList("IfNotPresent", "Always", "Never")); - - String json = mapper.writeValueAsString(pullPolicyProperty); - Property deserializedProperty = mapper.readValue(json, Property.class); - - assertNotNull(deserializedProperty, "PullPolicy property should not be null"); - assertEquals("IfNotPresent", deserializedProperty.getDefaut()); - - List listEnum = (List) deserializedProperty.getListEnumeration(); - assertNotNull(listEnum, "listEnum should not be null"); - assertEquals(Arrays.asList("IfNotPresent", "Always", "Never"), listEnum); - } - - @Test - void testXOnyxiaProperty() throws IOException { - Property groupProperty = new Property(); - groupProperty.setType("string"); - groupProperty.setDefaut(""); - groupProperty.setListEnumeration(List.of("")); - - Property.XOnyxia xOnyxia = new Property.XOnyxia(); - xOnyxia.setOverwriteDefaultWith("user.decodedIdToken.groups[0]"); - xOnyxia.setOverwriteListEnumWith("user.decodedIdToken.groups"); - groupProperty.setXonyxia(xOnyxia); - - // Serialize and deserialize to test - String json = mapper.writeValueAsString(groupProperty); - Property deserializedGroupProperty = mapper.readValue(json, Property.class); - - assertNotNull(deserializedGroupProperty, "Group property should not be null"); - assertEquals("", deserializedGroupProperty.getDefaut()); - - List listEnum = (List) deserializedGroupProperty.getListEnumeration(); - assertNotNull(listEnum, "listEnum should not be null"); - assertEquals(List.of(""), listEnum); - - // Check x-onyxia values - Property.XOnyxia deserializedXOnyxia = deserializedGroupProperty.getXonyxia(); - assertNotNull(deserializedXOnyxia, "XOnyxia property should not be null"); - assertEquals( - "user.decodedIdToken.groups[0]", deserializedXOnyxia.getOverwriteDefaultWith()); - assertEquals("user.decodedIdToken.groups", deserializedXOnyxia.getOverwriteListEnumWith()); - } -} From 8e827c4ddecc5bc00c393dc58cedf142dd7c1c95 Mon Sep 17 00:00:00 2001 From: Comte Date: Thu, 11 Jul 2024 12:52:14 +0200 Subject: [PATCH 04/53] fix --- .../java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java index 44316657..1fde242e 100644 --- a/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java +++ b/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java @@ -192,7 +192,7 @@ public void extractDataFromTgz(InputStream in, Chart chart) throws IOException { while ((len = tarIn.read(buffer)) != -1) { baos.write(buffer, 0, len); } - chart.setSchema(objectMapper.readTree(baos.toString("UTF-8"));); + chart.setSchema(objectMapper.readTree(baos.toString("UTF-8")); } } } From 8b7c1b56f0d76a73570de7d2a291bddca69ae76c Mon Sep 17 00:00:00 2001 From: Comte Date: Thu, 11 Jul 2024 12:53:57 +0200 Subject: [PATCH 05/53] fix --- .../java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java index 1fde242e..d6520a1e 100644 --- a/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java +++ b/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java @@ -192,7 +192,7 @@ public void extractDataFromTgz(InputStream in, Chart chart) throws IOException { while ((len = tarIn.read(buffer)) != -1) { baos.write(buffer, 0, len); } - chart.setSchema(objectMapper.readTree(baos.toString("UTF-8")); + chart.setSchema(objectMapper.readTree(baos.toString("UTF-8"))); } } } From 6beb5b12bb73b3ed7246cf27ac71c2fdf3ac295e Mon Sep 17 00:00:00 2001 From: Comte Date: Thu, 11 Jul 2024 13:10:38 +0200 Subject: [PATCH 06/53] fix --- .../api/dao/universe/CatalogLoader.java | 2 +- .../control/AdmissionControllerHelm.java | 26 ------- .../control/commons/UrlGenerator.java | 22 ------ .../control/utils/PublishContext.java | 51 -------------- .../control/xgenerated/XGeneratedContext.java | 47 ------------- .../xgenerated/XGeneratedInjector.java | 22 ------ .../xgenerated/XGeneratedIterator.java | 63 ----------------- .../xgenerated/XGeneratedProcessor.java | 50 ------------- .../xgenerated/XGeneratedProvider.java | 20 ------ .../control/xgenerated/XGeneratedReader.java | 38 ---------- .../api/services/impl/HelmAppsService.java | 70 ------------------- 11 files changed, 1 insertion(+), 410 deletions(-) delete mode 100644 onyxia-api/src/main/java/fr/insee/onyxia/api/services/control/AdmissionControllerHelm.java delete mode 100644 onyxia-api/src/main/java/fr/insee/onyxia/api/services/control/commons/UrlGenerator.java delete mode 100644 onyxia-api/src/main/java/fr/insee/onyxia/api/services/control/utils/PublishContext.java delete mode 100644 onyxia-api/src/main/java/fr/insee/onyxia/api/services/control/xgenerated/XGeneratedContext.java delete mode 100644 onyxia-api/src/main/java/fr/insee/onyxia/api/services/control/xgenerated/XGeneratedInjector.java delete mode 100644 onyxia-api/src/main/java/fr/insee/onyxia/api/services/control/xgenerated/XGeneratedIterator.java delete mode 100644 onyxia-api/src/main/java/fr/insee/onyxia/api/services/control/xgenerated/XGeneratedProcessor.java delete mode 100644 onyxia-api/src/main/java/fr/insee/onyxia/api/services/control/xgenerated/XGeneratedProvider.java delete mode 100644 onyxia-api/src/main/java/fr/insee/onyxia/api/services/control/xgenerated/XGeneratedReader.java diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java index d6520a1e..b5fb22df 100644 --- a/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java +++ b/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java @@ -192,7 +192,7 @@ public void extractDataFromTgz(InputStream in, Chart chart) throws IOException { while ((len = tarIn.read(buffer)) != -1) { baos.write(buffer, 0, len); } - chart.setSchema(objectMapper.readTree(baos.toString("UTF-8"))); + chart.setSchema(mapper.readTree(baos.toString("UTF-8"))); } } } diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/services/control/AdmissionControllerHelm.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/services/control/AdmissionControllerHelm.java deleted file mode 100644 index bdec4793..00000000 --- a/onyxia-api/src/main/java/fr/insee/onyxia/api/services/control/AdmissionControllerHelm.java +++ /dev/null @@ -1,26 +0,0 @@ -package fr.insee.onyxia.api.services.control; - -import fr.insee.onyxia.api.services.control.utils.PublishContext; -import fr.insee.onyxia.model.User; -import fr.insee.onyxia.model.catalog.Pkg; -import fr.insee.onyxia.model.region.Region; -import java.util.Map; - -public interface AdmissionControllerHelm { - /** - * Validate that the contract is ok to deploy Implementations are free to modify the contract - * - * @return whether the contract is ok to deploy - */ - boolean validateContract( - Region region, Pkg pkg, Map values, User user, PublishContext context); - - /** - * Returns the priority. Higher priority will be applied first. Default is 0, can be negative. - * - * @return - */ - default Integer getPriority() { - return 0; - } -} diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/services/control/commons/UrlGenerator.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/services/control/commons/UrlGenerator.java deleted file mode 100644 index 0640577a..00000000 --- a/onyxia-api/src/main/java/fr/insee/onyxia/api/services/control/commons/UrlGenerator.java +++ /dev/null @@ -1,22 +0,0 @@ -package fr.insee.onyxia.api.services.control.commons; - -import org.apache.commons.lang3.StringUtils; -import org.springframework.stereotype.Service; - -@Service -public class UrlGenerator { - - public String generateUrl( - String userId, String id, String generatedId, String prefix, String baseDomain) { - String url = - userId - + "-" - + id - + "-" - + generatedId - + (StringUtils.isNotBlank(prefix) ? "-" + prefix : "") - + "." - + baseDomain; - return url; - } -} diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/services/control/utils/PublishContext.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/services/control/utils/PublishContext.java deleted file mode 100644 index a0756b02..00000000 --- a/onyxia-api/src/main/java/fr/insee/onyxia/api/services/control/utils/PublishContext.java +++ /dev/null @@ -1,51 +0,0 @@ -package fr.insee.onyxia.api.services.control.utils; - -import java.util.HashMap; -import java.util.Map; -import java.util.UUID; - -public class PublishContext { - - private Context globalContext = new Context(); - private Map localContexts = new HashMap<>(); - private String catalogId; - - public PublishContext() {} - - public PublishContext(String catalogId) {} - - private Context getLocalContext(String key) { - if (!localContexts.containsKey(key)) { - localContexts.put(key, new Context()); - } - - return localContexts.get(key); - } - - public Context getGlobalContext() { - return globalContext; - } - - public String getCatalogId() { - return catalogId; - } - - public static class Context { - private String randomizedId; - - private Map data = new HashMap<>(); - - public String getRandomizedId() { - if (randomizedId == null) { - UUID uuid = UUID.randomUUID(); - randomizedId = Long.toString(-uuid.getLeastSignificantBits()); - ; - } - return randomizedId; - } - - public Map getData() { - return data; - } - } -} diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/services/control/xgenerated/XGeneratedContext.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/services/control/xgenerated/XGeneratedContext.java deleted file mode 100644 index e7faa235..00000000 --- a/onyxia-api/src/main/java/fr/insee/onyxia/api/services/control/xgenerated/XGeneratedContext.java +++ /dev/null @@ -1,47 +0,0 @@ -package fr.insee.onyxia.api.services.control.xgenerated; - -import fr.insee.onyxia.model.catalog.Config.Property; -import java.util.HashMap; -import java.util.Map; - -public class XGeneratedContext { - private String groupIdKey; - private Map scopes = new HashMap<>(); - - public String getGroupIdKey() { - return groupIdKey; - } - - public void setGroupIdKey(String groupIdKey) { - this.groupIdKey = groupIdKey; - } - - public Map getScopes() { - return scopes; - } - - public void setScopes(Map scopes) { - this.scopes = scopes; - } - - public static class Scope { - private String scopeName; - private Map xGenerateds = new HashMap<>(); - - public String getScopeName() { - return scopeName; - } - - public void setScopeName(String scopeName) { - this.scopeName = scopeName; - } - - public Map getxGenerateds() { - return xGenerateds; - } - - public void setxGenerateds(Map xGenerateds) { - this.xGenerateds = xGenerateds; - } - } -} diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/services/control/xgenerated/XGeneratedInjector.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/services/control/xgenerated/XGeneratedInjector.java deleted file mode 100644 index b1a14249..00000000 --- a/onyxia-api/src/main/java/fr/insee/onyxia/api/services/control/xgenerated/XGeneratedInjector.java +++ /dev/null @@ -1,22 +0,0 @@ -package fr.insee.onyxia.api.services.control.xgenerated; - -import java.util.HashMap; -import java.util.Map; -import org.springframework.stereotype.Service; - -@Service -public class XGeneratedInjector { - - public void injectIntoContext(Map context, Map toInject) { - toInject.forEach( - (k, v) -> { - String[] splittedPath = k.split("\\."); - Map currentContext = context; - for (int i = 0; i < splittedPath.length - 1; i++) { - currentContext.putIfAbsent(splittedPath[i], new HashMap()); - currentContext = (Map) currentContext.get(splittedPath[i]); - } - currentContext.put(splittedPath[splittedPath.length - 1], v); - }); - } -} diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/services/control/xgenerated/XGeneratedIterator.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/services/control/xgenerated/XGeneratedIterator.java deleted file mode 100644 index 10cf31fa..00000000 --- a/onyxia-api/src/main/java/fr/insee/onyxia/api/services/control/xgenerated/XGeneratedIterator.java +++ /dev/null @@ -1,63 +0,0 @@ -package fr.insee.onyxia.api.services.control.xgenerated; - -import fr.insee.onyxia.model.catalog.Config.Property; -import java.util.Map; -import org.springframework.stereotype.Service; - -@Service -public class XGeneratedIterator { - - public void iterateOverContext( - XGeneratedContext context, - XGeneratedProvider xGeneratedProvider, - Map xGeneratedValues) { - - if (context.getGroupIdKey() != null) { - if (context.getGroupIdKey() != null) { - xGeneratedValues.put(context.getGroupIdKey(), xGeneratedProvider.getGroupId()); - } - } - - context.getScopes() - .forEach( - (scopeName, scope) -> { - scope.getxGenerateds() - .forEach( - (name, xGenerated) -> { - if (xGenerated.getType() - == Property.XGenerated.XGeneratedType - .AppID) { - xGeneratedValues.put( - name, - xGeneratedProvider.getAppId( - scopeName, scope, xGenerated)); - } - if (xGenerated.getType() - == Property.XGenerated.XGeneratedType - .ExternalDNS) { - xGeneratedValues.put( - name, - xGeneratedProvider.getExternalDns( - scopeName, scope, xGenerated)); - } - - if (xGenerated.getType() - == Property.XGenerated.XGeneratedType - .InternalDNS) { - xGeneratedValues.put( - name, - xGeneratedProvider.getInternalDns( - scopeName, scope, xGenerated)); - } - if (xGenerated.getType() - == Property.XGenerated.XGeneratedType - .InitScript) { - xGeneratedValues.put( - name, - xGeneratedProvider.getInitScript( - scopeName, scope, xGenerated)); - } - }); - }); - } -} diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/services/control/xgenerated/XGeneratedProcessor.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/services/control/xgenerated/XGeneratedProcessor.java deleted file mode 100644 index ddfd2bd9..00000000 --- a/onyxia-api/src/main/java/fr/insee/onyxia/api/services/control/xgenerated/XGeneratedProcessor.java +++ /dev/null @@ -1,50 +0,0 @@ -package fr.insee.onyxia.api.services.control.xgenerated; - -import fr.insee.onyxia.model.catalog.Pkg; -import java.util.*; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -@Service -public class XGeneratedProcessor { - - private final XGeneratedReader xGeneratedReader; - - private final XGeneratedIterator xGeneratedIterator; - - private final XGeneratedInjector xGeneratedInjector; - - @Autowired - public XGeneratedProcessor( - XGeneratedReader xGeneratedReader, - XGeneratedIterator xGeneratedIterator, - XGeneratedInjector xGeneratedInjector) { - this.xGeneratedReader = xGeneratedReader; - this.xGeneratedIterator = xGeneratedIterator; - this.xGeneratedInjector = xGeneratedInjector; - } - - public XGeneratedContext readContext(Pkg pkg) { - XGeneratedContext xGeneratedContext = new XGeneratedContext(); - pkg.getConfig() - .getProperties() - .getProperties() - .forEach( - (key, value) -> - xGeneratedReader.readXGenerated( - List.of(key), value, xGeneratedContext)); - return xGeneratedContext; - } - - public Map process( - XGeneratedContext xGeneratedContext, XGeneratedProvider provider) { - Map xGeneratedValues = new HashMap<>(); - xGeneratedIterator.iterateOverContext(xGeneratedContext, provider, xGeneratedValues); - return xGeneratedValues; - } - - public void injectIntoContext( - Map target, Map xGeneratedValues) { - xGeneratedInjector.injectIntoContext(target, xGeneratedValues); - } -} diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/services/control/xgenerated/XGeneratedProvider.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/services/control/xgenerated/XGeneratedProvider.java deleted file mode 100644 index bec8555c..00000000 --- a/onyxia-api/src/main/java/fr/insee/onyxia/api/services/control/xgenerated/XGeneratedProvider.java +++ /dev/null @@ -1,20 +0,0 @@ -package fr.insee.onyxia.api.services.control.xgenerated; - -import fr.insee.onyxia.model.catalog.Config.Property; - -public interface XGeneratedProvider { - - public String getGroupId(); - - public String getAppId( - String scopeName, XGeneratedContext.Scope scope, Property.XGenerated xGenerated); - - public String getExternalDns( - String scopeName, XGeneratedContext.Scope scope, Property.XGenerated xGenerated); - - public String getInternalDns( - String scopeName, XGeneratedContext.Scope scope, Property.XGenerated xGenerated); - - public String getInitScript( - String scopeName, XGeneratedContext.Scope scope, Property.XGenerated xGenerated); -} diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/services/control/xgenerated/XGeneratedReader.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/services/control/xgenerated/XGeneratedReader.java deleted file mode 100644 index 3f30ecac..00000000 --- a/onyxia-api/src/main/java/fr/insee/onyxia/api/services/control/xgenerated/XGeneratedReader.java +++ /dev/null @@ -1,38 +0,0 @@ -package fr.insee.onyxia.api.services.control.xgenerated; - -import fr.insee.onyxia.model.catalog.Config.Property; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import org.springframework.stereotype.Service; - -@Service -public class XGeneratedReader { - - public void readXGenerated(List path, Property property, XGeneratedContext context) { - String currentPath = path.stream().collect(Collectors.joining(".")); - if (property.getProperties() != null) { - for (Map.Entry prop : property.getProperties().entrySet()) { - List newPath = new ArrayList<>(); - newPath.addAll(path); - newPath.add(prop.getKey()); - readXGenerated(newPath, prop.getValue(), context); - } - } else if (property.getxGenerated() != null) { - Property.XGenerated xGenerated = property.getxGenerated(); - if (xGenerated.getType() == Property.XGenerated.XGeneratedType.GroupID) { - context.setGroupIdKey(path.stream().collect(Collectors.joining("."))); - return; - } - - String scopeName = xGenerated.getScope(); - if (!context.getScopes().containsKey(scopeName)) { - context.getScopes().put(scopeName, new XGeneratedContext.Scope()); - } - - XGeneratedContext.Scope scope = context.getScopes().get(scopeName); - scope.getxGenerateds().put(currentPath, property.getxGenerated()); - } - } -} diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/services/impl/HelmAppsService.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/services/impl/HelmAppsService.java index d3a014b5..c9eef45c 100644 --- a/onyxia-api/src/main/java/fr/insee/onyxia/api/services/impl/HelmAppsService.java +++ b/onyxia-api/src/main/java/fr/insee/onyxia/api/services/impl/HelmAppsService.java @@ -14,15 +14,10 @@ import fr.insee.onyxia.api.events.UninstallServiceEvent; import fr.insee.onyxia.api.services.AppsService; import fr.insee.onyxia.api.services.control.AdmissionControllerHelm; -import fr.insee.onyxia.api.services.control.commons.UrlGenerator; import fr.insee.onyxia.api.services.control.utils.PublishContext; -import fr.insee.onyxia.api.services.control.xgenerated.XGeneratedContext; -import fr.insee.onyxia.api.services.control.xgenerated.XGeneratedProcessor; -import fr.insee.onyxia.api.services.control.xgenerated.XGeneratedProvider; import fr.insee.onyxia.api.services.impl.kubernetes.KubernetesService; import fr.insee.onyxia.api.services.utils.Base64Utils; import fr.insee.onyxia.model.User; -import fr.insee.onyxia.model.catalog.Config.Property; import fr.insee.onyxia.model.catalog.Pkg; import fr.insee.onyxia.model.dto.CreateServiceDTO; import fr.insee.onyxia.model.dto.ServicesListing; @@ -122,71 +117,6 @@ public Collection installApp( final String caFile) throws IOException, TimeoutException, InterruptedException { - PublishContext context = new PublishContext(); - - XGeneratedContext xGeneratedContext = xGeneratedProcessor.readContext(pkg); - XGeneratedProvider xGeneratedProvider = - new XGeneratedProvider() { - @Override - public String getGroupId() { - return null; - } - - @Override - public String getAppId( - String scopeName, - XGeneratedContext.Scope scope, - Property.XGenerated xGenerated) { - return pkg.getName(); - } - - @Override - public String getExternalDns( - String scopeName, - XGeneratedContext.Scope scope, - Property.XGenerated xGenerated) { - return urlGenerator.generateUrl( - user.getIdep(), - pkg.getName(), - context.getGlobalContext().getRandomizedId(), - scopeName - + (StringUtils.isNotBlank(xGenerated.getName()) - ? "-" + xGenerated.getName() - : ""), - region.getServices().getExpose().getDomain()); - } - - @Override - public String getInternalDns( - String scopeName, - XGeneratedContext.Scope scope, - Property.XGenerated xGenerated) { - return ""; - } - - @Override - public String getInitScript( - String scopeName, - XGeneratedContext.Scope scope, - Property.XGenerated xGenerated) { - return region.getServices().getInitScript(); - } - }; - Map xGeneratedValues = - xGeneratedProcessor.process(xGeneratedContext, xGeneratedProvider); - xGeneratedProcessor.injectIntoContext(fusion, xGeneratedValues); - - long nbInvalidations = - admissionControllers.stream() - .map( - controller -> - controller.validateContract( - region, pkg, fusion, user, context)) - .filter(b -> !b) - .count(); - if (nbInvalidations > 0) { - throw new AccessDeniedException("Validation failed"); - } File values = File.createTempFile("values", ".yaml"); mapperHelm.writeValue(values, fusion); String namespaceId = From 9f391b61a0374eb381debcdcbbf744581c0aab2f Mon Sep 17 00:00:00 2001 From: Comte Date: Thu, 11 Jul 2024 13:17:42 +0200 Subject: [PATCH 07/53] fix --- .../api/controller/pub/CatalogController.java | 3 --- .../onyxia/api/services/impl/HelmAppsService.java | 13 ------------- 2 files changed, 16 deletions(-) diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/pub/CatalogController.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/pub/CatalogController.java index abfd8ff7..c8cd8a8b 100644 --- a/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/pub/CatalogController.java +++ b/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/pub/CatalogController.java @@ -4,9 +4,6 @@ import fr.insee.onyxia.api.configuration.Catalogs; import fr.insee.onyxia.api.configuration.NotFoundException; import fr.insee.onyxia.api.services.CatalogService; -import fr.insee.onyxia.model.catalog.Config.Property; -import fr.insee.onyxia.model.catalog.Config.Property.XForm; -import fr.insee.onyxia.model.catalog.Config.Property.XOnyxia; import fr.insee.onyxia.model.catalog.Pkg; import fr.insee.onyxia.model.helm.Chart; import fr.insee.onyxia.model.region.Region; diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/services/impl/HelmAppsService.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/services/impl/HelmAppsService.java index c9eef45c..78facd4b 100644 --- a/onyxia-api/src/main/java/fr/insee/onyxia/api/services/impl/HelmAppsService.java +++ b/onyxia-api/src/main/java/fr/insee/onyxia/api/services/impl/HelmAppsService.java @@ -13,8 +13,6 @@ import fr.insee.onyxia.api.events.SuspendResumeServiceEvent; import fr.insee.onyxia.api.events.UninstallServiceEvent; import fr.insee.onyxia.api.services.AppsService; -import fr.insee.onyxia.api.services.control.AdmissionControllerHelm; -import fr.insee.onyxia.api.services.control.utils.PublishContext; import fr.insee.onyxia.api.services.impl.kubernetes.KubernetesService; import fr.insee.onyxia.api.services.utils.Base64Utils; import fr.insee.onyxia.model.User; @@ -61,17 +59,12 @@ public class HelmAppsService implements AppsService { private final KubernetesService kubernetesService; - private final List admissionControllers; private final FastDateFormat helmDateFormat = FastDateFormat.getInstance("yyyy-MM-dd HH:mm:ss"); private final KubernetesClientProvider kubernetesClientProvider; private final HelmClientProvider helmClientProvider; - private final XGeneratedProcessor xGeneratedProcessor; - - private final UrlGenerator urlGenerator; - final OnyxiaEventPublisher onyxiaEventPublisher; public static final String ONYXIA_SECRET_PREFIX = "sh.onyxia.release.v1."; @@ -80,19 +73,13 @@ public class HelmAppsService implements AppsService { public HelmAppsService( @Qualifier("helm") ObjectMapper mapperHelm, KubernetesService kubernetesService, - List admissionControllers, KubernetesClientProvider kubernetesClientProvider, HelmClientProvider helmClientProvider, - XGeneratedProcessor xGeneratedProcessor, - UrlGenerator urlGenerator, OnyxiaEventPublisher onyxiaEventPublisher) { this.mapperHelm = mapperHelm; this.kubernetesService = kubernetesService; - this.admissionControllers = admissionControllers; this.kubernetesClientProvider = kubernetesClientProvider; this.helmClientProvider = helmClientProvider; - this.xGeneratedProcessor = xGeneratedProcessor; - this.urlGenerator = urlGenerator; this.onyxiaEventPublisher = onyxiaEventPublisher; } From c4c19f19f2c2b6bd2e2e87c2e984add3eb82d539 Mon Sep 17 00:00:00 2001 From: Comte Date: Thu, 11 Jul 2024 13:26:30 +0200 Subject: [PATCH 08/53] fix --- .../api/dao/universe/CatalogLoaderTest.java | 31 ------------------- 1 file changed, 31 deletions(-) diff --git a/onyxia-api/src/test/java/fr/insee/onyxia/api/dao/universe/CatalogLoaderTest.java b/onyxia-api/src/test/java/fr/insee/onyxia/api/dao/universe/CatalogLoaderTest.java index 3026d676..5fbd7537 100644 --- a/onyxia-api/src/test/java/fr/insee/onyxia/api/dao/universe/CatalogLoaderTest.java +++ b/onyxia-api/src/test/java/fr/insee/onyxia/api/dao/universe/CatalogLoaderTest.java @@ -147,35 +147,4 @@ void packageOnClassPathNotFound() { + "[catalog-loader-test/keepeme1.gz]")); } - @Test - /** - * This test is for a regression on commons compress 1.26 - * https://commons.apache.org/proper/commons-compress/changes-report.html#a1.26.0 that made it - * non parallelable - */ - void shouldExtractPackageInParallel() throws IOException { - List charts = new ArrayList<>(); - for (int i = 0; i < 20; i++) { - Chart chart = new Chart(); - chart.setName("vscode-python-darkmode"); - charts.add(chart); - } - charts.parallelStream() - .forEach( - chart -> { - try { - catalogLoader.extractDataFromTgz( - resourceLoader - .getResource( - "classpath:/catalog-loader-test/vscode-python-darkmode-1.11.11.tgz") - .getInputStream(), - chart); - assertEquals( - 19, - chart.getConfig().getProperties().getProperties().size()); - } catch (IOException e) { - throw new RuntimeException(e); - } - }); - } } From cf884b830d7eac8f8fd97d10a88f643dcc5b720d Mon Sep 17 00:00:00 2001 From: Comte Date: Thu, 11 Jul 2024 13:51:47 +0200 Subject: [PATCH 09/53] convert to config --- .../insee/onyxia/api/dao/universe/CatalogLoader.java | 2 +- .../main/java/fr/insee/onyxia/model/catalog/Pkg.java | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java index b5fb22df..61dd06c1 100644 --- a/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java +++ b/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java @@ -192,7 +192,7 @@ public void extractDataFromTgz(InputStream in, Chart chart) throws IOException { while ((len = tarIn.read(buffer)) != -1) { baos.write(buffer, 0, len); } - chart.setSchema(mapper.readTree(baos.toString("UTF-8"))); + chart.setConfig(mapper.readTree(baos.toString("UTF-8"))); } } } diff --git a/onyxia-model/src/main/java/fr/insee/onyxia/model/catalog/Pkg.java b/onyxia-model/src/main/java/fr/insee/onyxia/model/catalog/Pkg.java index a2dba9cb..c93bfaae 100644 --- a/onyxia-model/src/main/java/fr/insee/onyxia/model/catalog/Pkg.java +++ b/onyxia-model/src/main/java/fr/insee/onyxia/model/catalog/Pkg.java @@ -15,7 +15,7 @@ public class Pkg { private String version; @Schema(description = "") - private JsonNode schema; + private JsonNode config; public String getName() { return this.name; @@ -51,12 +51,12 @@ public void setVersion(String version) { this.version = version; } - public JsonNode getSchema() { - return this.schema; + public JsonNode getConfig() { + return this.config; } - public void setSchema(JsonNode schema) { - this.schema = schema; + public void setConfig(JsonNode config) { + this.config = config; } public Pkg version(String version) { this.version = version; From 05508e2e9af081e63273a0d9ab394d0a020fcdf5 Mon Sep 17 00:00:00 2001 From: Comte Date: Thu, 11 Jul 2024 18:19:46 +0200 Subject: [PATCH 10/53] add internal resolution --- .../api/dao/universe/CatalogLoader.java | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java index 61dd06c1..dd26564a 100644 --- a/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java +++ b/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java @@ -192,9 +192,27 @@ public void extractDataFromTgz(InputStream in, Chart chart) throws IOException { while ((len = tarIn.read(buffer)) != -1) { baos.write(buffer, 0, len); } - chart.setConfig(mapper.readTree(baos.toString("UTF-8"))); + chart.setConfig(resolveInternalReferences(mapper.readTree(baos.toString("UTF-8")))); } } } } + + public JsonNode resolveInternalReferences(JsonNode schemaNode) throws IOException { + // Convert the main schema JSON node to JSONObject + JSONObject schemaJson = new JSONObject(new JSONTokener(schemaNode.toString())); + + // Create a SchemaLoader + SchemaLoader loader = SchemaLoader.builder() + .schemaJson(schemaJson) + .resolutionScope("file:///") // Setting a base URI for relative references + .build(); + + // Load and expand the schema + Schema schema = loader.load().build(); + + // Convert the resolved schema back to JsonNode + JSONObject resolvedSchemaJson = new JSONObject(schema.toString()); + return objectMapper.readTree(resolvedSchemaJson.toString()); + } } From ba74ccad327a287a6cb312ffc5239b707be4469d Mon Sep 17 00:00:00 2001 From: Comte Date: Thu, 11 Jul 2024 18:21:41 +0200 Subject: [PATCH 11/53] fix --- .../fr/insee/onyxia/api/dao/universe/CatalogLoader.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java index dd26564a..b1ead618 100644 --- a/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java +++ b/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java @@ -5,6 +5,12 @@ import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.everit.json.schema.Schema; +import org.everit.json.schema.loader.SchemaLoader; +import org.json.JSONObject; +import org.json.JSONTokener; import com.github.zafarkhaja.semver.Version; import fr.insee.onyxia.api.configuration.CatalogWrapper; import fr.insee.onyxia.model.catalog.Pkg; From f082b3e800ea3043b1c895e0c49886c03d9222c7 Mon Sep 17 00:00:00 2001 From: Comte Date: Thu, 11 Jul 2024 18:52:23 +0200 Subject: [PATCH 12/53] fix pom --- onyxia-api/pom.xml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/onyxia-api/pom.xml b/onyxia-api/pom.xml index 2889bc64..aaf481dd 100644 --- a/onyxia-api/pom.xml +++ b/onyxia-api/pom.xml @@ -105,6 +105,12 @@ jackson-databind + + org.everit.json + org.everit.json.schema + 1.14.2 + + io.fabric8 kubernetes-server-mock From 823beca187a5663f9cf65f0f0f07d07eee65cc3c Mon Sep 17 00:00:00 2001 From: Comte Date: Thu, 11 Jul 2024 19:25:14 +0200 Subject: [PATCH 13/53] fix pom --- onyxia-api/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/onyxia-api/pom.xml b/onyxia-api/pom.xml index aaf481dd..c26b16f0 100644 --- a/onyxia-api/pom.xml +++ b/onyxia-api/pom.xml @@ -106,8 +106,8 @@ - org.everit.json - org.everit.json.schema + com.github.erosb + everit-json-schema 1.14.2 From dc704491b0bc0cf2654e633e6796c7239b3ed94a Mon Sep 17 00:00:00 2001 From: Comte Date: Thu, 11 Jul 2024 19:52:51 +0200 Subject: [PATCH 14/53] fix --- .../java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java index b1ead618..455344bb 100644 --- a/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java +++ b/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java @@ -198,13 +198,13 @@ public void extractDataFromTgz(InputStream in, Chart chart) throws IOException { while ((len = tarIn.read(buffer)) != -1) { baos.write(buffer, 0, len); } - chart.setConfig(resolveInternalReferences(mapper.readTree(baos.toString("UTF-8")))); + chart.setConfig(resolveInternalReferences(mapper.readTree(baos.toString("UTF-8")),mapper)); } } } } - public JsonNode resolveInternalReferences(JsonNode schemaNode) throws IOException { + public JsonNode resolveInternalReferences(JsonNode schemaNode, ObjectMapper objectMapper) throws IOException { // Convert the main schema JSON node to JSONObject JSONObject schemaJson = new JSONObject(new JSONTokener(schemaNode.toString())); From c761c437b13d56d404f63bfdd9cccf8757591b19 Mon Sep 17 00:00:00 2001 From: Comte Date: Thu, 11 Jul 2024 21:41:19 +0200 Subject: [PATCH 15/53] recursive resolution --- .../api/dao/universe/CatalogLoader.java | 35 ++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java index 455344bb..bb3e06be 100644 --- a/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java +++ b/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java @@ -198,7 +198,7 @@ public void extractDataFromTgz(InputStream in, Chart chart) throws IOException { while ((len = tarIn.read(buffer)) != -1) { baos.write(buffer, 0, len); } - chart.setConfig(resolveInternalReferences(mapper.readTree(baos.toString("UTF-8")),mapper)); + chart.setConfig(resolveInternalReferences(mapper.readTree(baos.toString("UTF-8")))); } } } @@ -221,4 +221,37 @@ public JsonNode resolveInternalReferences(JsonNode schemaNode, ObjectMapper obje JSONObject resolvedSchemaJson = new JSONObject(schema.toString()); return objectMapper.readTree(resolvedSchemaJson.toString()); } + + public JsonNode resolveInternalReferences(JsonNode schemaNode) { + return resolveInternalReferences(schemaNode, schemaNode); + } + + private JsonNode resolveInternalReferences(JsonNode schemaNode, JsonNode rootNode) { + if (schemaNode.isObject()) { + ObjectNode objectNode = (ObjectNode) schemaNode; + Iterator> fields = objectNode.fields(); + while (fields.hasNext()) { + Map.Entry field = fields.next(); + if (field.getKey().equals("$ref") && field.getValue().isTextual()) { + String ref = field.getValue().asText(); + if (ref.startsWith("#/definitions/")) { + String refName = ref.substring("#/definitions/".length()); + JsonNode refNode = rootNode.at("/definitions/" + refName); + if (!refNode.isMissingNode()) { + JsonNode resolvedNode = resolveInternalReferences(refNode.deepCopy(), rootNode); + objectNode.setAll((ObjectNode) resolvedNode); + objectNode.remove("$ref"); + } + } + } else { + objectNode.set(field.getKey(), resolveInternalReferences(field.getValue(), rootNode)); + } + } + } else if (schemaNode.isArray()) { + for (int i = 0; i < schemaNode.size(); i++) { + ((ObjectNode) schemaNode).set(i, resolveInternalReferences(schemaNode.get(i), rootNode)); + } + } + return schemaNode; + } } From 3df369a58ab43f546de0608ad04428bc6bf8eacf Mon Sep 17 00:00:00 2001 From: Comte Date: Thu, 11 Jul 2024 21:43:41 +0200 Subject: [PATCH 16/53] fix --- .../java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java index bb3e06be..cac07c79 100644 --- a/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java +++ b/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java @@ -6,7 +6,7 @@ import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; import org.everit.json.schema.Schema; import org.everit.json.schema.loader.SchemaLoader; import org.json.JSONObject; From 43709bca09e523c864b3eed58cb002e7e4cb31bf Mon Sep 17 00:00:00 2001 From: Comte Date: Thu, 11 Jul 2024 21:50:39 +0200 Subject: [PATCH 17/53] fix --- .../fr/insee/onyxia/api/dao/universe/CatalogLoader.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java index cac07c79..0ba2a16f 100644 --- a/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java +++ b/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java @@ -24,6 +24,7 @@ import java.util.Iterator; import java.util.List; import java.util.Optional; +import java.util.Map; import org.apache.commons.compress.archivers.tar.TarArchiveEntry; import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; @@ -248,8 +249,9 @@ private JsonNode resolveInternalReferences(JsonNode schemaNode, JsonNode rootNod } } } else if (schemaNode.isArray()) { - for (int i = 0; i < schemaNode.size(); i++) { - ((ObjectNode) schemaNode).set(i, resolveInternalReferences(schemaNode.get(i), rootNode)); + ArrayNode arrayNode = (ArrayNode) schemaNode; + for (int i = 0; i < arrayNode.size(); i++) { + arrayNode.set(i, resolveInternalReferences(arrayNode.get(i), rootNode)); } } return schemaNode; From a7e1cd282f9ec98cd8ad8b93ca8e9251c83eda65 Mon Sep 17 00:00:00 2001 From: Comte Date: Thu, 11 Jul 2024 21:52:04 +0200 Subject: [PATCH 18/53] fix --- .../java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java | 1 + 1 file changed, 1 insertion(+) diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java index 0ba2a16f..ac2d47b3 100644 --- a/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java +++ b/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java @@ -7,6 +7,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.node.ArrayNode; import org.everit.json.schema.Schema; import org.everit.json.schema.loader.SchemaLoader; import org.json.JSONObject; From bee9917874e5b02dc850c0b6406f8c9d9df351ac Mon Sep 17 00:00:00 2001 From: Comte Date: Thu, 11 Jul 2024 21:59:45 +0200 Subject: [PATCH 19/53] fix --- .../onyxia/api/dao/universe/CatalogLoader.java | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java index ac2d47b3..9e1ca58d 100644 --- a/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java +++ b/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java @@ -231,6 +231,8 @@ public JsonNode resolveInternalReferences(JsonNode schemaNode) { private JsonNode resolveInternalReferences(JsonNode schemaNode, JsonNode rootNode) { if (schemaNode.isObject()) { ObjectNode objectNode = (ObjectNode) schemaNode; + Map updates = new HashMap<>(); + Iterator> fields = objectNode.fields(); while (fields.hasNext()) { Map.Entry field = fields.next(); @@ -241,12 +243,20 @@ private JsonNode resolveInternalReferences(JsonNode schemaNode, JsonNode rootNod JsonNode refNode = rootNode.at("/definitions/" + refName); if (!refNode.isMissingNode()) { JsonNode resolvedNode = resolveInternalReferences(refNode.deepCopy(), rootNode); - objectNode.setAll((ObjectNode) resolvedNode); - objectNode.remove("$ref"); + updates.putAll((ObjectNode) resolvedNode); + updates.put("$ref", null); } } } else { - objectNode.set(field.getKey(), resolveInternalReferences(field.getValue(), rootNode)); + updates.put(field.getKey(), resolveInternalReferences(field.getValue(), rootNode)); + } + } + + for (Map.Entry update : updates.entrySet()) { + if (update.getValue() == null) { + objectNode.remove(update.getKey()); + } else { + objectNode.set(update.getKey(), update.getValue()); } } } else if (schemaNode.isArray()) { From d6bc50e5442a97600ff9e7542e25971e6f1dd3e3 Mon Sep 17 00:00:00 2001 From: Comte Date: Thu, 11 Jul 2024 22:02:14 +0200 Subject: [PATCH 20/53] fix --- .../java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java index 9e1ca58d..16d22712 100644 --- a/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java +++ b/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java @@ -26,6 +26,8 @@ import java.util.List; import java.util.Optional; import java.util.Map; +import java.util.HashMap; + import org.apache.commons.compress.archivers.tar.TarArchiveEntry; import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; From d58a73eaa2858546bc18d3528129f238af14b197 Mon Sep 17 00:00:00 2001 From: Comte Date: Thu, 11 Jul 2024 22:04:48 +0200 Subject: [PATCH 21/53] fix --- .../insee/onyxia/api/dao/universe/CatalogLoader.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java index 16d22712..76319f7d 100644 --- a/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java +++ b/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java @@ -245,7 +245,7 @@ private JsonNode resolveInternalReferences(JsonNode schemaNode, JsonNode rootNod JsonNode refNode = rootNode.at("/definitions/" + refName); if (!refNode.isMissingNode()) { JsonNode resolvedNode = resolveInternalReferences(refNode.deepCopy(), rootNode); - updates.putAll((ObjectNode) resolvedNode); + updates.putAll(convertToMap((ObjectNode) resolvedNode)); updates.put("$ref", null); } } @@ -269,4 +269,14 @@ private JsonNode resolveInternalReferences(JsonNode schemaNode, JsonNode rootNod } return schemaNode; } + + private Map convertToMap(ObjectNode objectNode) { + Map map = new HashMap<>(); + Iterator> fields = objectNode.fields(); + while (fields.hasNext()) { + Map.Entry field = fields.next(); + map.put(field.getKey(), field.getValue()); + } + return map; + } } From 419a1436882e8f7d6dc3fa4ac94d1717c586df4f Mon Sep 17 00:00:00 2001 From: Comte Date: Fri, 12 Jul 2024 11:59:12 +0200 Subject: [PATCH 22/53] add schema controller --- .../api/controller/RestExceptionHandler.java | 5 ++ .../exception/SchemaNotFoundException.java | 6 +++ .../controller/pub/JsonSchemaController.java | 31 ++++++++++++ .../services/JsonSchemaRegistryService.java | 47 +++++++++++++++++++ onyxia-api/src/main/resources/schemas/s3.json | 0 5 files changed, 89 insertions(+) create mode 100644 onyxia-api/src/main/java/fr/insee/onyxia/api/controller/exception/SchemaNotFoundException.java create mode 100644 onyxia-api/src/main/java/fr/insee/onyxia/api/controller/pub/JsonSchemaController.java create mode 100644 onyxia-api/src/main/java/fr/insee/onyxia/api/services/JsonSchemaRegistryService.java create mode 100644 onyxia-api/src/main/resources/schemas/s3.json diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/RestExceptionHandler.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/RestExceptionHandler.java index 8e684ea3..76bbd4de 100644 --- a/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/RestExceptionHandler.java +++ b/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/RestExceptionHandler.java @@ -12,4 +12,9 @@ public class RestExceptionHandler { @ResponseStatus(value = HttpStatus.FORBIDDEN) @ExceptionHandler(AccessDeniedException.class) public void handleAccessDeniedException(Exception ignored) {} + + @ExceptionHandler(SchemaNotFoundException.class) + public ResponseEntity handleSchemaNotFoundException(SchemaNotFoundException ex) { + return new ResponseEntity<>(ex.getMessage(), HttpStatus.NOT_FOUND); + } } diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/exception/SchemaNotFoundException.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/exception/SchemaNotFoundException.java new file mode 100644 index 00000000..341bc9a5 --- /dev/null +++ b/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/exception/SchemaNotFoundException.java @@ -0,0 +1,6 @@ +public class SchemaNotFoundException extends RuntimeException { + + public SchemaNotFoundException(String schemaName) { + super("Schema not found: " + schemaName); + } +} diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/pub/JsonSchemaController.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/pub/JsonSchemaController.java new file mode 100644 index 00000000..1c1951a4 --- /dev/null +++ b/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/pub/JsonSchemaController.java @@ -0,0 +1,31 @@ +import com.fasterxml.jackson.databind.JsonNode; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; +import fr.insee.onyxia.api.services.JsonSchemaRegistryService; + +import java.util.Map; + +@RestController +@RequestMapping("/api/schemas") +public class JsonSchemaController { + + @Autowired + private JsonSchemaRegistryService jsonSchemaRegistryService; + + @GetMapping + public Map listSchemas() { + return jsonSchemaRegistryService.listSchemas(); + } + + @GetMapping("/{schemaName}") + public JsonNode getSchema(@PathVariable String schemaName) { + JsonNode schema = jsonSchemaRegistryService.getSchema(schemaName); + if (schema == null) { + throw new SchemaNotFoundException(schemaName); + } + return schema; + } +} diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/services/JsonSchemaRegistryService.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/services/JsonSchemaRegistryService.java new file mode 100644 index 00000000..16635ca2 --- /dev/null +++ b/onyxia-api/src/main/java/fr/insee/onyxia/api/services/JsonSchemaRegistryService.java @@ -0,0 +1,47 @@ +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.springframework.stereotype.Service; + +import javax.annotation.PostConstruct; +import java.io.IOException; +import java.nio.file.*; +import java.util.HashMap; +import java.util.Map; + +@Service +public class JsonSchemaRegistryService { + + private static final String SCHEMA_DIRECTORY = "src/main/resources/schemas"; + private final ObjectMapper objectMapper; + private final Map schemaRegistry; + + public JsonSchemaRegistryService() { + this.objectMapper = new ObjectMapper(); + this.schemaRegistry = new HashMap<>(); + } + + @PostConstruct + private void loadSchemas() throws IOException { + Files.walk(Paths.get(SCHEMA_DIRECTORY)) + .filter(Files::isRegularFile) + .forEach(this::loadSchema); + } + + private void loadSchema(Path path) { + try { + JsonNode schema = objectMapper.readTree(path.toFile()); + String relativePath = Paths.get(SCHEMA_DIRECTORY).relativize(path).toString().replace("\\", "/"); + schemaRegistry.put(relativePath, schema); + } catch (IOException e) { + throw new RuntimeException("Failed to load schema: " + path.getFileName(), e); + } + } + + public Map listSchemas() { + return schemaRegistry; + } + + public JsonNode getSchema(String schemaName) { + return schemaRegistry.get(schemaName); + } +} diff --git a/onyxia-api/src/main/resources/schemas/s3.json b/onyxia-api/src/main/resources/schemas/s3.json new file mode 100644 index 00000000..e69de29b From deea725b615cabd66506b4a65eb0539eb3f581b8 Mon Sep 17 00:00:00 2001 From: Comte Date: Fri, 12 Jul 2024 13:30:23 +0200 Subject: [PATCH 23/53] Update RestExceptionHandler.java --- .../fr/insee/onyxia/api/controller/RestExceptionHandler.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/RestExceptionHandler.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/RestExceptionHandler.java index 76bbd4de..d6d38c18 100644 --- a/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/RestExceptionHandler.java +++ b/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/RestExceptionHandler.java @@ -1,5 +1,6 @@ package fr.insee.onyxia.api.controller; - +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; import org.springframework.http.HttpStatus; import org.springframework.security.access.AccessDeniedException; import org.springframework.web.bind.annotation.ExceptionHandler; From 016e448a134d5b1f093b79edfe937ce26342600d Mon Sep 17 00:00:00 2001 From: Comte Date: Fri, 12 Jul 2024 13:36:21 +0200 Subject: [PATCH 24/53] fix --- .../onyxia/api/controller/pub/JsonSchemaController.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/pub/JsonSchemaController.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/pub/JsonSchemaController.java index 1c1951a4..46ccaf49 100644 --- a/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/pub/JsonSchemaController.java +++ b/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/pub/JsonSchemaController.java @@ -12,8 +12,12 @@ @RequestMapping("/api/schemas") public class JsonSchemaController { - @Autowired private JsonSchemaRegistryService jsonSchemaRegistryService; + + @Autowired + public JsonSchemaController(JsonSchemaRegistryService jsonSchemaRegistryService) { + this.jsonSchemaRegistryService = jsonSchemaRegistryService; + } @GetMapping public Map listSchemas() { From e5498c246dfc3470a657d321ff85e64ab9dff3ec Mon Sep 17 00:00:00 2001 From: Comte Date: Fri, 12 Jul 2024 13:42:42 +0200 Subject: [PATCH 25/53] fix --- .../fr/insee/onyxia/api/services/JsonSchemaRegistryService.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/services/JsonSchemaRegistryService.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/services/JsonSchemaRegistryService.java index 16635ca2..fe73ced0 100644 --- a/onyxia-api/src/main/java/fr/insee/onyxia/api/services/JsonSchemaRegistryService.java +++ b/onyxia-api/src/main/java/fr/insee/onyxia/api/services/JsonSchemaRegistryService.java @@ -1,3 +1,5 @@ +package fr.insee.onyxia.api.services; + import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import org.springframework.stereotype.Service; From 3711858431cf75e10332fe9685fc2b62d0dd53cc Mon Sep 17 00:00:00 2001 From: Comte Date: Fri, 12 Jul 2024 13:51:29 +0200 Subject: [PATCH 26/53] add annotation postconstruct pom --- onyxia-api/pom.xml | 6 ++++++ .../api/controller/exception/SchemaNotFoundException.java | 2 ++ .../onyxia/api/controller/pub/JsonSchemaController.java | 2 +- 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/onyxia-api/pom.xml b/onyxia-api/pom.xml index c26b16f0..033725ea 100644 --- a/onyxia-api/pom.xml +++ b/onyxia-api/pom.xml @@ -111,6 +111,12 @@ 1.14.2 + + javax.annotation + javax.annotation-api + 1.3.2 + + io.fabric8 kubernetes-server-mock diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/exception/SchemaNotFoundException.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/exception/SchemaNotFoundException.java index 341bc9a5..126c71da 100644 --- a/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/exception/SchemaNotFoundException.java +++ b/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/exception/SchemaNotFoundException.java @@ -1,3 +1,5 @@ +package fr.insee.onyxia.api.controller.exception; + public class SchemaNotFoundException extends RuntimeException { public SchemaNotFoundException(String schemaName) { diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/pub/JsonSchemaController.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/pub/JsonSchemaController.java index 46ccaf49..79857275 100644 --- a/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/pub/JsonSchemaController.java +++ b/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/pub/JsonSchemaController.java @@ -5,7 +5,7 @@ import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; import fr.insee.onyxia.api.services.JsonSchemaRegistryService; - +import fr.insee.onyxia.api.controller.exception.SchemaNotFoundException; import java.util.Map; @RestController From 47f1f242bc80756cdd44a5e74497fa87bada6c3a Mon Sep 17 00:00:00 2001 From: Comte Date: Fri, 12 Jul 2024 13:53:11 +0200 Subject: [PATCH 27/53] fix --- .../fr/insee/onyxia/api/controller/RestExceptionHandler.java | 1 + 1 file changed, 1 insertion(+) diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/RestExceptionHandler.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/RestExceptionHandler.java index d6d38c18..38c7c81f 100644 --- a/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/RestExceptionHandler.java +++ b/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/RestExceptionHandler.java @@ -6,6 +6,7 @@ import org.springframework.web.bind.annotation.ExceptionHandler; import org.springframework.web.bind.annotation.ResponseStatus; import org.springframework.web.bind.annotation.RestControllerAdvice; +import fr.insee.onyxia.api.controller.exception.SchemaNotFoundException; @RestControllerAdvice public class RestExceptionHandler { From 1f0f0e620ca97ce424c9cc31b2773289aa596b6a Mon Sep 17 00:00:00 2001 From: Olivier Levitt Date: Fri, 12 Jul 2024 14:22:30 +0200 Subject: [PATCH 28/53] =?UTF-8?q?Fix=20fr=C3=A9do?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../api/controller/RestExceptionHandler.java | 4 +- .../exception/SchemaNotFoundException.java | 16 +-- .../api/controller/pub/CatalogController.java | 3 - .../controller/pub/JsonSchemaController.java | 72 +++++++------- .../api/dao/universe/CatalogLoader.java | 42 ++++---- .../services/JsonSchemaRegistryService.java | 99 ++++++++++--------- .../api/services/impl/HelmAppsService.java | 1 - onyxia-api/src/main/resources/schemas/s3.json | 72 ++++++++++++++ .../api/dao/universe/CatalogLoaderTest.java | 3 - .../fr/insee/onyxia/model/catalog/Pkg.java | 3 +- 10 files changed, 194 insertions(+), 121 deletions(-) diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/RestExceptionHandler.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/RestExceptionHandler.java index 38c7c81f..a3dd38af 100644 --- a/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/RestExceptionHandler.java +++ b/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/RestExceptionHandler.java @@ -1,12 +1,12 @@ package fr.insee.onyxia.api.controller; + +import fr.insee.onyxia.api.controller.exception.SchemaNotFoundException; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; -import org.springframework.http.HttpStatus; import org.springframework.security.access.AccessDeniedException; import org.springframework.web.bind.annotation.ExceptionHandler; import org.springframework.web.bind.annotation.ResponseStatus; import org.springframework.web.bind.annotation.RestControllerAdvice; -import fr.insee.onyxia.api.controller.exception.SchemaNotFoundException; @RestControllerAdvice public class RestExceptionHandler { diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/exception/SchemaNotFoundException.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/exception/SchemaNotFoundException.java index 126c71da..3387fe49 100644 --- a/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/exception/SchemaNotFoundException.java +++ b/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/exception/SchemaNotFoundException.java @@ -1,8 +1,8 @@ -package fr.insee.onyxia.api.controller.exception; - -public class SchemaNotFoundException extends RuntimeException { - - public SchemaNotFoundException(String schemaName) { - super("Schema not found: " + schemaName); - } -} +package fr.insee.onyxia.api.controller.exception; + +public class SchemaNotFoundException extends RuntimeException { + + public SchemaNotFoundException(String schemaName) { + super("Schema not found: " + schemaName); + } +} diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/pub/CatalogController.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/pub/CatalogController.java index c8cd8a8b..96d82dfb 100644 --- a/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/pub/CatalogController.java +++ b/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/pub/CatalogController.java @@ -11,9 +11,7 @@ import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.enums.ParameterIn; import io.swagger.v3.oas.annotations.tags.Tag; -import java.util.HashMap; import java.util.List; -import java.util.Map; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; @@ -137,5 +135,4 @@ public List getCharts(@PathVariable String catalogId, @PathVariable Strin catalogService.getCharts(catalogId, chartName).orElseThrow(NotFoundException::new); return charts; } - } diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/pub/JsonSchemaController.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/pub/JsonSchemaController.java index 79857275..00f4f695 100644 --- a/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/pub/JsonSchemaController.java +++ b/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/pub/JsonSchemaController.java @@ -1,35 +1,37 @@ -import com.fasterxml.jackson.databind.JsonNode; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; -import fr.insee.onyxia.api.services.JsonSchemaRegistryService; -import fr.insee.onyxia.api.controller.exception.SchemaNotFoundException; -import java.util.Map; - -@RestController -@RequestMapping("/api/schemas") -public class JsonSchemaController { - - private JsonSchemaRegistryService jsonSchemaRegistryService; - - @Autowired - public JsonSchemaController(JsonSchemaRegistryService jsonSchemaRegistryService) { - this.jsonSchemaRegistryService = jsonSchemaRegistryService; - } - - @GetMapping - public Map listSchemas() { - return jsonSchemaRegistryService.listSchemas(); - } - - @GetMapping("/{schemaName}") - public JsonNode getSchema(@PathVariable String schemaName) { - JsonNode schema = jsonSchemaRegistryService.getSchema(schemaName); - if (schema == null) { - throw new SchemaNotFoundException(schemaName); - } - return schema; - } -} +package fr.insee.onyxia.api.controller.pub; + +import com.fasterxml.jackson.databind.JsonNode; +import fr.insee.onyxia.api.controller.exception.SchemaNotFoundException; +import fr.insee.onyxia.api.services.JsonSchemaRegistryService; +import java.util.Map; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +@RestController +@RequestMapping("/public/schemas") +public class JsonSchemaController { + + private final JsonSchemaRegistryService jsonSchemaRegistryService; + + @Autowired + public JsonSchemaController(JsonSchemaRegistryService jsonSchemaRegistryService) { + this.jsonSchemaRegistryService = jsonSchemaRegistryService; + } + + @GetMapping + public Map listSchemas() { + return jsonSchemaRegistryService.listSchemas(); + } + + @GetMapping("/{schemaName}") + public JsonNode getSchema(@PathVariable String schemaName) { + JsonNode schema = jsonSchemaRegistryService.getSchema(schemaName); + if (schema == null) { + throw new SchemaNotFoundException(schemaName); + } + return schema; + } +} diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java index 76319f7d..dddea9a9 100644 --- a/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java +++ b/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java @@ -4,33 +4,32 @@ import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; -import org.everit.json.schema.Schema; -import org.everit.json.schema.loader.SchemaLoader; -import org.json.JSONObject; -import org.json.JSONTokener; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.github.zafarkhaja.semver.Version; import fr.insee.onyxia.api.configuration.CatalogWrapper; import fr.insee.onyxia.model.catalog.Pkg; import fr.insee.onyxia.model.helm.Chart; import fr.insee.onyxia.model.helm.Repository; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.Reader; -import java.io.ByteArrayOutputStream; +import java.util.HashMap; import java.util.Iterator; import java.util.List; -import java.util.Optional; import java.util.Map; -import java.util.HashMap; - +import java.util.Optional; import org.apache.commons.compress.archivers.tar.TarArchiveEntry; import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; +import org.everit.json.schema.Schema; +import org.everit.json.schema.loader.SchemaLoader; +import org.json.JSONObject; +import org.json.JSONTokener; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Qualifier; @@ -201,22 +200,25 @@ public void extractDataFromTgz(InputStream in, Chart chart) throws IOException { int len; while ((len = tarIn.read(buffer)) != -1) { baos.write(buffer, 0, len); - } - chart.setConfig(resolveInternalReferences(mapper.readTree(baos.toString("UTF-8")))); + } + chart.setConfig( + resolveInternalReferences(mapper.readTree(baos.toString("UTF-8")))); } } } } - public JsonNode resolveInternalReferences(JsonNode schemaNode, ObjectMapper objectMapper) throws IOException { + public JsonNode resolveInternalReferences(JsonNode schemaNode, ObjectMapper objectMapper) + throws IOException { // Convert the main schema JSON node to JSONObject JSONObject schemaJson = new JSONObject(new JSONTokener(schemaNode.toString())); // Create a SchemaLoader - SchemaLoader loader = SchemaLoader.builder() - .schemaJson(schemaJson) - .resolutionScope("file:///") // Setting a base URI for relative references - .build(); + SchemaLoader loader = + SchemaLoader.builder() + .schemaJson(schemaJson) + .resolutionScope("file:///") // Setting a base URI for relative references + .build(); // Load and expand the schema Schema schema = loader.load().build(); @@ -244,13 +246,15 @@ private JsonNode resolveInternalReferences(JsonNode schemaNode, JsonNode rootNod String refName = ref.substring("#/definitions/".length()); JsonNode refNode = rootNode.at("/definitions/" + refName); if (!refNode.isMissingNode()) { - JsonNode resolvedNode = resolveInternalReferences(refNode.deepCopy(), rootNode); + JsonNode resolvedNode = + resolveInternalReferences(refNode.deepCopy(), rootNode); updates.putAll(convertToMap((ObjectNode) resolvedNode)); updates.put("$ref", null); } } } else { - updates.put(field.getKey(), resolveInternalReferences(field.getValue(), rootNode)); + updates.put( + field.getKey(), resolveInternalReferences(field.getValue(), rootNode)); } } diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/services/JsonSchemaRegistryService.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/services/JsonSchemaRegistryService.java index fe73ced0..f4f83785 100644 --- a/onyxia-api/src/main/java/fr/insee/onyxia/api/services/JsonSchemaRegistryService.java +++ b/onyxia-api/src/main/java/fr/insee/onyxia/api/services/JsonSchemaRegistryService.java @@ -1,49 +1,50 @@ -package fr.insee.onyxia.api.services; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import org.springframework.stereotype.Service; - -import javax.annotation.PostConstruct; -import java.io.IOException; -import java.nio.file.*; -import java.util.HashMap; -import java.util.Map; - -@Service -public class JsonSchemaRegistryService { - - private static final String SCHEMA_DIRECTORY = "src/main/resources/schemas"; - private final ObjectMapper objectMapper; - private final Map schemaRegistry; - - public JsonSchemaRegistryService() { - this.objectMapper = new ObjectMapper(); - this.schemaRegistry = new HashMap<>(); - } - - @PostConstruct - private void loadSchemas() throws IOException { - Files.walk(Paths.get(SCHEMA_DIRECTORY)) - .filter(Files::isRegularFile) - .forEach(this::loadSchema); - } - - private void loadSchema(Path path) { - try { - JsonNode schema = objectMapper.readTree(path.toFile()); - String relativePath = Paths.get(SCHEMA_DIRECTORY).relativize(path).toString().replace("\\", "/"); - schemaRegistry.put(relativePath, schema); - } catch (IOException e) { - throw new RuntimeException("Failed to load schema: " + path.getFileName(), e); - } - } - - public Map listSchemas() { - return schemaRegistry; - } - - public JsonNode getSchema(String schemaName) { - return schemaRegistry.get(schemaName); - } -} +package fr.insee.onyxia.api.services; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.Map; +import javax.annotation.PostConstruct; +import org.springframework.stereotype.Service; + +@Service +public class JsonSchemaRegistryService { + + private static final String SCHEMA_DIRECTORY = "/schemas"; + private final ObjectMapper objectMapper; + private final Map schemaRegistry; + + public JsonSchemaRegistryService() { + this.objectMapper = new ObjectMapper(); + this.schemaRegistry = new HashMap<>(); + } + + @PostConstruct + private void loadSchemas() throws IOException, URISyntaxException { + Files.walk(Paths.get(JsonSchemaRegistryService.class.getResource(SCHEMA_DIRECTORY).toURI())) + .filter(Files::isRegularFile) + .forEach(this::loadSchema); + } + + private void loadSchema(Path path) { + try { + JsonNode schema = objectMapper.readTree(path.toFile()); + schemaRegistry.put(path.getFileName().toString(), schema); + } catch (IOException e) { + throw new RuntimeException("Failed to load schema: " + path.getFileName(), e); + } + } + + public Map listSchemas() { + return schemaRegistry; + } + + public JsonNode getSchema(String schemaName) { + return schemaRegistry.get(schemaName); + } +} diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/services/impl/HelmAppsService.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/services/impl/HelmAppsService.java index 78facd4b..93ba7702 100644 --- a/onyxia-api/src/main/java/fr/insee/onyxia/api/services/impl/HelmAppsService.java +++ b/onyxia-api/src/main/java/fr/insee/onyxia/api/services/impl/HelmAppsService.java @@ -59,7 +59,6 @@ public class HelmAppsService implements AppsService { private final KubernetesService kubernetesService; - private final FastDateFormat helmDateFormat = FastDateFormat.getInstance("yyyy-MM-dd HH:mm:ss"); private final KubernetesClientProvider kubernetesClientProvider; diff --git a/onyxia-api/src/main/resources/schemas/s3.json b/onyxia-api/src/main/resources/schemas/s3.json index e69de29b..e297bfd4 100644 --- a/onyxia-api/src/main/resources/schemas/s3.json +++ b/onyxia-api/src/main/resources/schemas/s3.json @@ -0,0 +1,72 @@ +{ + "$id": "https://example.com/schemas/s3.json", + "title": "S3 Configuration", + "description": "Configuration of temporary identity for AWS S3", + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "description": "Add S3 temporary identity inside your environment", + "default": true + }, + "accessKeyId": { + "description": "AWS Access Key", + "type": "string", + "x-onyxia": { + "overwriteDefaultWith": "{{s3.AWS_ACCESS_KEY_ID}}" + }, + "hidden": { + "value": false, + "path": "enabled" + } + }, + "endpoint": { + "description": "AWS S3 Endpoint", + "type": "string", + "x-onyxia": { + "overwriteDefaultWith": "{{s3.AWS_S3_ENDPOINT}}" + }, + "hidden": { + "value": false, + "path": "enabled" + } + }, + "defaultRegion": { + "description": "AWS S3 default region", + "type": "string", + "x-onyxia": { + "overwriteDefaultWith": "{{s3.AWS_DEFAULT_REGION}}" + }, + "hidden": { + "value": false, + "path": "enabled" + } + }, + "secretAccessKey": { + "description": "AWS S3 secret access key", + "type": "string", + "render": "password", + "x-onyxia": { + "overwriteDefaultWith": "{{s3.AWS_SECRET_ACCESS_KEY}}" + }, + "hidden": { + "value": false, + "path": "enabled" + } + }, + "sessionToken": { + "description": "AWS S3 session Token", + "type": "string", + "render": "password", + "x-onyxia": { + "overwriteDefaultWith": "{{s3.AWS_SESSION_TOKEN}}" + }, + "hidden": { + "value": false, + "path": "enabled" + } + } + }, + "required": ["enabled"] +} + diff --git a/onyxia-api/src/test/java/fr/insee/onyxia/api/dao/universe/CatalogLoaderTest.java b/onyxia-api/src/test/java/fr/insee/onyxia/api/dao/universe/CatalogLoaderTest.java index 5fbd7537..4739faed 100644 --- a/onyxia-api/src/test/java/fr/insee/onyxia/api/dao/universe/CatalogLoaderTest.java +++ b/onyxia-api/src/test/java/fr/insee/onyxia/api/dao/universe/CatalogLoaderTest.java @@ -9,8 +9,6 @@ import fr.insee.onyxia.api.configuration.CustomObjectMapper; import fr.insee.onyxia.api.util.TestUtils; import fr.insee.onyxia.model.helm.Chart; -import java.io.IOException; -import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; import org.junit.jupiter.api.DisplayName; @@ -146,5 +144,4 @@ void packageOnClassPathNotFound() { + "Exception occurred during loading resource: class path resource " + "[catalog-loader-test/keepeme1.gz]")); } - } diff --git a/onyxia-model/src/main/java/fr/insee/onyxia/model/catalog/Pkg.java b/onyxia-model/src/main/java/fr/insee/onyxia/model/catalog/Pkg.java index c93bfaae..d1a9d7f6 100644 --- a/onyxia-model/src/main/java/fr/insee/onyxia/model/catalog/Pkg.java +++ b/onyxia-model/src/main/java/fr/insee/onyxia/model/catalog/Pkg.java @@ -1,6 +1,6 @@ package fr.insee.onyxia.model.catalog; -import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.JsonNode; import io.swagger.v3.oas.annotations.media.Schema; @Schema(description = "") @@ -58,6 +58,7 @@ public JsonNode getConfig() { public void setConfig(JsonNode config) { this.config = config; } + public Pkg version(String version) { this.version = version; return this; From cf528da287fce12e55a850b94978d19ad4eda4f1 Mon Sep 17 00:00:00 2001 From: fcomte Date: Fri, 12 Jul 2024 14:23:45 +0000 Subject: [PATCH 29/53] via vscode ajout resolution service --- .../api/dao/universe/CatalogLoader.java | 8 ++- .../services/JsonSchemaResolutionService.java | 60 +++++++++++++++++++ 2 files changed, 65 insertions(+), 3 deletions(-) create mode 100644 onyxia-api/src/main/java/fr/insee/onyxia/api/services/JsonSchemaResolutionService.java diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java index dddea9a9..d9f854cb 100644 --- a/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java +++ b/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java @@ -13,6 +13,7 @@ import fr.insee.onyxia.model.catalog.Pkg; import fr.insee.onyxia.model.helm.Chart; import fr.insee.onyxia.model.helm.Repository; +import fr.insee.onyxia.api.services.JsonSchemaResolutionService; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; @@ -44,13 +45,14 @@ public class CatalogLoader { private static final Logger LOGGER = LoggerFactory.getLogger(CatalogLoader.class); private final ResourceLoader resourceLoader; - + private final JsonSchemaResolutionService jsonSchemaResolutionService; private final ObjectMapper mapperHelm; public CatalogLoader( - ResourceLoader resourceLoader, @Qualifier("helm") ObjectMapper mapperHelm) { + ResourceLoader resourceLoader, @Qualifier("helm") ObjectMapper mapperHelm, JsonSchemaResolutionService jsonSchemaResolutionService) { this.resourceLoader = resourceLoader; this.mapperHelm = mapperHelm; + this.jsonSchemaResolutionService = jsonSchemaResolutionService; } public void updateCatalog(CatalogWrapper cw) { @@ -202,7 +204,7 @@ public void extractDataFromTgz(InputStream in, Chart chart) throws IOException { baos.write(buffer, 0, len); } chart.setConfig( - resolveInternalReferences(mapper.readTree(baos.toString("UTF-8")))); + jsonSchemaResolutionService.resolveReferences(mapper.readTree(baos.toString("UTF-8")))); } } } diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/services/JsonSchemaResolutionService.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/services/JsonSchemaResolutionService.java new file mode 100644 index 00000000..a0880856 --- /dev/null +++ b/onyxia-api/src/main/java/fr/insee/onyxia/api/services/JsonSchemaResolutionService.java @@ -0,0 +1,60 @@ +package fr.insee.onyxia.api.services; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.Iterator; +import java.util.Map; + +@Service +public class JsonSchemaResolutionService { + + private final ObjectMapper objectMapper; + private final JsonSchemaRegistryService registryService; + + @Autowired + public JsonSchemaResolutionService(JsonSchemaRegistryService registryService) { + this.objectMapper = new ObjectMapper(); + this.registryService = registryService; + } + + public JsonNode resolveReferences(JsonNode schemaNode) { + return resolveReferences(schemaNode, schemaNode); + } + + private JsonNode resolveReferences(JsonNode schemaNode, JsonNode rootNode) { + if (schemaNode.isObject()) { + ObjectNode objectNode = (ObjectNode) schemaNode; + Iterator> fields = objectNode.fields(); + while (fields.hasNext()) { + Map.Entry field = fields.next(); + if (field.getKey().equals("$ref") && field.getValue().isTextual()) { + String ref = field.getValue().asText(); + JsonNode refNode = null; + if (ref.startsWith("#/definitions/")) { + refNode = rootNode.at(ref.substring(1)); + } else { + refNode = registryService.getSchema(ref); + } + if (refNode != null && !refNode.isMissingNode()) { + JsonNode resolvedNode = resolveReferences(refNode.deepCopy(), rootNode); + fields.remove(); // Remove the $ref field + objectNode.setAll((ObjectNode) resolvedNode); + } + } else { + objectNode.set(field.getKey(), resolveReferences(field.getValue(), rootNode)); + } + } + } else if (schemaNode.isArray()) { + ArrayNode arrayNode = (ArrayNode) schemaNode; + for (int i = 0; i < arrayNode.size(); i++) { + arrayNode.set(i, resolveReferences(arrayNode.get(i), rootNode)); + } + } + return schemaNode; + } +} From 6829323f6d78c82a1ec8c8ccecc0a87965676bfe Mon Sep 17 00:00:00 2001 From: fcomte Date: Fri, 12 Jul 2024 14:53:51 +0000 Subject: [PATCH 30/53] fredo fix tout seul spotless --- .../fr/insee/onyxia/api/dao/universe/CatalogLoader.java | 9 ++++++--- .../onyxia/api/services/JsonSchemaResolutionService.java | 9 ++++----- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java index d9f854cb..9ef7ebcc 100644 --- a/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java +++ b/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java @@ -10,10 +10,10 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.github.zafarkhaja.semver.Version; import fr.insee.onyxia.api.configuration.CatalogWrapper; +import fr.insee.onyxia.api.services.JsonSchemaResolutionService; import fr.insee.onyxia.model.catalog.Pkg; import fr.insee.onyxia.model.helm.Chart; import fr.insee.onyxia.model.helm.Repository; -import fr.insee.onyxia.api.services.JsonSchemaResolutionService; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; @@ -49,7 +49,9 @@ public class CatalogLoader { private final ObjectMapper mapperHelm; public CatalogLoader( - ResourceLoader resourceLoader, @Qualifier("helm") ObjectMapper mapperHelm, JsonSchemaResolutionService jsonSchemaResolutionService) { + ResourceLoader resourceLoader, + @Qualifier("helm") ObjectMapper mapperHelm, + JsonSchemaResolutionService jsonSchemaResolutionService) { this.resourceLoader = resourceLoader; this.mapperHelm = mapperHelm; this.jsonSchemaResolutionService = jsonSchemaResolutionService; @@ -204,7 +206,8 @@ public void extractDataFromTgz(InputStream in, Chart chart) throws IOException { baos.write(buffer, 0, len); } chart.setConfig( - jsonSchemaResolutionService.resolveReferences(mapper.readTree(baos.toString("UTF-8")))); + jsonSchemaResolutionService.resolveReferences( + mapper.readTree(baos.toString("UTF-8")))); } } } diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/services/JsonSchemaResolutionService.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/services/JsonSchemaResolutionService.java index a0880856..dcf309bc 100644 --- a/onyxia-api/src/main/java/fr/insee/onyxia/api/services/JsonSchemaResolutionService.java +++ b/onyxia-api/src/main/java/fr/insee/onyxia/api/services/JsonSchemaResolutionService.java @@ -2,13 +2,12 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.databind.node.ArrayNode; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - +import com.fasterxml.jackson.databind.node.ObjectNode; import java.util.Iterator; import java.util.Map; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; @Service public class JsonSchemaResolutionService { @@ -42,7 +41,7 @@ private JsonNode resolveReferences(JsonNode schemaNode, JsonNode rootNode) { } if (refNode != null && !refNode.isMissingNode()) { JsonNode resolvedNode = resolveReferences(refNode.deepCopy(), rootNode); - fields.remove(); // Remove the $ref field + fields.remove(); // Remove the $ref field objectNode.setAll((ObjectNode) resolvedNode); } } else { From f1b593513b4a92bf46f2d012b74fd36202f5e61d Mon Sep 17 00:00:00 2001 From: Olivier Levitt Date: Fri, 12 Jul 2024 18:43:14 +0200 Subject: [PATCH 31/53] Fix tests --- .../onyxia/api/dao/universe/CatalogLoaderTest.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/onyxia-api/src/test/java/fr/insee/onyxia/api/dao/universe/CatalogLoaderTest.java b/onyxia-api/src/test/java/fr/insee/onyxia/api/dao/universe/CatalogLoaderTest.java index 4739faed..851bc883 100644 --- a/onyxia-api/src/test/java/fr/insee/onyxia/api/dao/universe/CatalogLoaderTest.java +++ b/onyxia-api/src/test/java/fr/insee/onyxia/api/dao/universe/CatalogLoaderTest.java @@ -7,6 +7,8 @@ import fr.insee.onyxia.api.configuration.CatalogWrapper; import fr.insee.onyxia.api.configuration.CustomObjectMapper; +import fr.insee.onyxia.api.services.JsonSchemaRegistryService; +import fr.insee.onyxia.api.services.JsonSchemaResolutionService; import fr.insee.onyxia.api.util.TestUtils; import fr.insee.onyxia.model.helm.Chart; import java.util.List; @@ -21,7 +23,13 @@ import org.springframework.util.CollectionUtils; @ExtendWith(SpringExtension.class) -@SpringBootTest(classes = {CatalogLoader.class, CustomObjectMapper.class}) +@SpringBootTest( + classes = { + CatalogLoader.class, + CustomObjectMapper.class, + JsonSchemaResolutionService.class, + JsonSchemaRegistryService.class + }) public class CatalogLoaderTest { @Autowired CatalogLoader catalogLoader; From d2d68b091eb57a7e137c366404c7b85011035ac2 Mon Sep 17 00:00:00 2001 From: fcomte Date: Sat, 13 Jul 2024 21:09:04 +0000 Subject: [PATCH 32/53] fix concurrent modif --- .../services/JsonSchemaResolutionService.java | 27 ++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/services/JsonSchemaResolutionService.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/services/JsonSchemaResolutionService.java index dcf309bc..e595e7fa 100644 --- a/onyxia-api/src/main/java/fr/insee/onyxia/api/services/JsonSchemaResolutionService.java +++ b/onyxia-api/src/main/java/fr/insee/onyxia/api/services/JsonSchemaResolutionService.java @@ -4,6 +4,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import java.util.HashMap; import java.util.Iterator; import java.util.Map; import org.springframework.beans.factory.annotation.Autowired; @@ -28,6 +29,8 @@ public JsonNode resolveReferences(JsonNode schemaNode) { private JsonNode resolveReferences(JsonNode schemaNode, JsonNode rootNode) { if (schemaNode.isObject()) { ObjectNode objectNode = (ObjectNode) schemaNode; + Map updates = new HashMap<>(); + Iterator> fields = objectNode.fields(); while (fields.hasNext()) { Map.Entry field = fields.next(); @@ -41,11 +44,19 @@ private JsonNode resolveReferences(JsonNode schemaNode, JsonNode rootNode) { } if (refNode != null && !refNode.isMissingNode()) { JsonNode resolvedNode = resolveReferences(refNode.deepCopy(), rootNode); - fields.remove(); // Remove the $ref field - objectNode.setAll((ObjectNode) resolvedNode); + updates.putAll(convertToMap((ObjectNode) resolvedNode)); + updates.put("$ref", null); } } else { - objectNode.set(field.getKey(), resolveReferences(field.getValue(), rootNode)); + updates.put(field.getKey(), resolveReferences(field.getValue(), rootNode)); + } + } + + for (Map.Entry update : updates.entrySet()) { + if (update.getValue() == null) { + objectNode.remove(update.getKey()); + } else { + objectNode.set(update.getKey(), update.getValue()); } } } else if (schemaNode.isArray()) { @@ -56,4 +67,14 @@ private JsonNode resolveReferences(JsonNode schemaNode, JsonNode rootNode) { } return schemaNode; } + + private Map convertToMap(ObjectNode objectNode) { + Map map = new HashMap<>(); + Iterator> fields = objectNode.fields(); + while (fields.hasNext()) { + Map.Entry field = fields.next(); + map.put(field.getKey(), field.getValue()); + } + return map; + } } From a30612269e32de4a0b2569327a9563c10a7de098 Mon Sep 17 00:00:00 2001 From: fcomte Date: Sun, 14 Jul 2024 11:39:13 +0000 Subject: [PATCH 33/53] relative path s3 --- onyxia-api/src/main/resources/schemas/s3.json | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/onyxia-api/src/main/resources/schemas/s3.json b/onyxia-api/src/main/resources/schemas/s3.json index e297bfd4..f0d9d55e 100644 --- a/onyxia-api/src/main/resources/schemas/s3.json +++ b/onyxia-api/src/main/resources/schemas/s3.json @@ -17,7 +17,8 @@ }, "hidden": { "value": false, - "path": "enabled" + "path": "enabled", + "isPathRelative": true } }, "endpoint": { @@ -28,7 +29,8 @@ }, "hidden": { "value": false, - "path": "enabled" + "path": "enabled", + "isPathRelative": true } }, "defaultRegion": { @@ -39,7 +41,8 @@ }, "hidden": { "value": false, - "path": "enabled" + "path": "enabled", + "isPathRelative": true } }, "secretAccessKey": { @@ -51,7 +54,8 @@ }, "hidden": { "value": false, - "path": "enabled" + "path": "enabled", + "isPathRelative": true } }, "sessionToken": { @@ -63,7 +67,8 @@ }, "hidden": { "value": false, - "path": "enabled" + "path": "enabled", + "isPathRelative": true } } }, From 81debe0fdd60975d1ac3f86d2ff3f5e6ed8e54ce Mon Sep 17 00:00:00 2001 From: fcomte Date: Thu, 18 Jul 2024 13:09:40 +0000 Subject: [PATCH 34/53] controle schema when publishing --- .../onyxia/api/services/impl/HelmAppsService.java | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/services/impl/HelmAppsService.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/services/impl/HelmAppsService.java index 93ba7702..a3730ee8 100644 --- a/onyxia-api/src/main/java/fr/insee/onyxia/api/services/impl/HelmAppsService.java +++ b/onyxia-api/src/main/java/fr/insee/onyxia/api/services/impl/HelmAppsService.java @@ -41,6 +41,11 @@ import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.time.FastDateFormat; +import org.everit.json.schema.ValidationException; +import org.everit.json.schema.Schema; +import org.everit.json.schema.loader.SchemaLoader; +import org.json.JSONObject; +import org.json.JSONTokener; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -101,7 +106,15 @@ public Collection installApp( Map fusion, final boolean skipTlsVerify, final String caFile) - throws IOException, TimeoutException, InterruptedException { + throws IOException, TimeoutException, InterruptedException, ValidationException { + + JSONObject jsonSchema = new JSONObject(new JSONTokener(pkg.getConfig().toString())); + // Load the schema + Schema schema = SchemaLoader.load(jsonSchema); + // Convert the options map to a JSONObject + JSONObject jsonObject = new JSONObject(fusion); + // Validate the options object against the schema + schema.validate(jsonObject); File values = File.createTempFile("values", ".yaml"); mapperHelm.writeValue(values, fusion); From 3c9b50195ddc72a4b232c3bc7860d0b1b44947cf Mon Sep 17 00:00:00 2001 From: fcomte Date: Thu, 18 Jul 2024 13:15:28 +0000 Subject: [PATCH 35/53] spot --- .../java/fr/insee/onyxia/api/services/impl/HelmAppsService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/services/impl/HelmAppsService.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/services/impl/HelmAppsService.java index a3730ee8..a780cb20 100644 --- a/onyxia-api/src/main/java/fr/insee/onyxia/api/services/impl/HelmAppsService.java +++ b/onyxia-api/src/main/java/fr/insee/onyxia/api/services/impl/HelmAppsService.java @@ -41,8 +41,8 @@ import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.time.FastDateFormat; -import org.everit.json.schema.ValidationException; import org.everit.json.schema.Schema; +import org.everit.json.schema.ValidationException; import org.everit.json.schema.loader.SchemaLoader; import org.json.JSONObject; import org.json.JSONTokener; From 30ed8f34baa268e2d1125f4deb077e3166901a60 Mon Sep 17 00:00:00 2001 From: fcomte Date: Fri, 19 Jul 2024 15:24:46 +0000 Subject: [PATCH 36/53] add overwriteSchemaWith --- .../services/JsonSchemaResolutionService.java | 33 ++++++++++++++++--- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/services/JsonSchemaResolutionService.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/services/JsonSchemaResolutionService.java index e595e7fa..31cc6df6 100644 --- a/onyxia-api/src/main/java/fr/insee/onyxia/api/services/JsonSchemaResolutionService.java +++ b/onyxia-api/src/main/java/fr/insee/onyxia/api/services/JsonSchemaResolutionService.java @@ -34,8 +34,11 @@ private JsonNode resolveReferences(JsonNode schemaNode, JsonNode rootNode) { Iterator> fields = objectNode.fields(); while (fields.hasNext()) { Map.Entry field = fields.next(); - if (field.getKey().equals("$ref") && field.getValue().isTextual()) { - String ref = field.getValue().asText(); + JsonNode fieldValue = field.getValue(); + + // Handle $ref attribute + if (field.getKey().equals("$ref") && fieldValue.isTextual()) { + String ref = fieldValue.asText(); JsonNode refNode = null; if (ref.startsWith("#/definitions/")) { refNode = rootNode.at(ref.substring(1)); @@ -47,8 +50,20 @@ private JsonNode resolveReferences(JsonNode schemaNode, JsonNode rootNode) { updates.putAll(convertToMap((ObjectNode) resolvedNode)); updates.put("$ref", null); } - } else { - updates.put(field.getKey(), resolveReferences(field.getValue(), rootNode)); + } + // Handle x-onyxia.overwriteSchemaWith attribute + else if (field.getKey().equals("x-onyxia") + && fieldValue.has("overwriteSchemaWith")) { + String overrideSchemaName = fieldValue.get("overwriteSchemaWith").asText(); + JsonNode overrideSchemaNode = registryService.getSchema(overrideSchemaName); + if (overrideSchemaNode != null && !overrideSchemaNode.isMissingNode()) { + JsonNode mergedNode = mergeSchemas(schemaNode, overrideSchemaNode); + updates.putAll(convertToMap((ObjectNode) mergedNode)); + } + } + // Recursively resolve for nested objects + else if (fieldValue.isObject() || fieldValue.isArray()) { + updates.put(field.getKey(), resolveReferences(fieldValue, rootNode)); } } @@ -68,6 +83,16 @@ private JsonNode resolveReferences(JsonNode schemaNode, JsonNode rootNode) { return schemaNode; } + private ObjectNode mergeSchemas(JsonNode originalSchema, JsonNode overrideSchema) { + ObjectNode mergedSchema = (ObjectNode) originalSchema.deepCopy(); + Iterator> fields = overrideSchema.fields(); + while (fields.hasNext()) { + Map.Entry field = fields.next(); + mergedSchema.set(field.getKey(), field.getValue()); + } + return mergedSchema; + } + private Map convertToMap(ObjectNode objectNode) { Map map = new HashMap<>(); Iterator> fields = objectNode.fields(); From 16cd389f3f65a052cbc50eafd1f3839f4fbc65bd Mon Sep 17 00:00:00 2001 From: fcomte Date: Mon, 22 Jul 2024 14:30:20 +0000 Subject: [PATCH 37/53] add external schema --- .../services/JsonSchemaRegistryService.java | 51 ++++++++++++++++--- .../main/resources/schemas/nodeSelector.json | 14 +++++ onyxia-api/src/main/resources/schemas/s3.json | 1 - 3 files changed, 59 insertions(+), 7 deletions(-) create mode 100644 onyxia-api/src/main/resources/schemas/nodeSelector.json diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/services/JsonSchemaRegistryService.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/services/JsonSchemaRegistryService.java index f4f83785..315ea49d 100644 --- a/onyxia-api/src/main/java/fr/insee/onyxia/api/services/JsonSchemaRegistryService.java +++ b/onyxia-api/src/main/java/fr/insee/onyxia/api/services/JsonSchemaRegistryService.java @@ -2,6 +2,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.File; import java.io.IOException; import java.net.URISyntaxException; import java.nio.file.Files; @@ -10,15 +11,19 @@ import java.util.HashMap; import java.util.Map; import javax.annotation.PostConstruct; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; @Service public class JsonSchemaRegistryService { - private static final String SCHEMA_DIRECTORY = "/schemas"; + private static final String SCHEMA_DIRECTORY = "/schemas"; // Resource directory private final ObjectMapper objectMapper; private final Map schemaRegistry; + @Value("${external.schema.directory:/external-schemas}") // External directory path + private String externalSchemaDirectory; + public JsonSchemaRegistryService() { this.objectMapper = new ObjectMapper(); this.schemaRegistry = new HashMap<>(); @@ -26,25 +31,59 @@ public JsonSchemaRegistryService() { @PostConstruct private void loadSchemas() throws IOException, URISyntaxException { - Files.walk(Paths.get(JsonSchemaRegistryService.class.getResource(SCHEMA_DIRECTORY).toURI())) + // Load initial schemas from resources + loadResourceSchemas(); + + // Load schemas from the external directory if it exists + loadExternalSchemas(); + } + + private void loadResourceSchemas() throws IOException, URISyntaxException { + Path resourcePath = + Paths.get(JsonSchemaRegistryService.class.getResource(SCHEMA_DIRECTORY).toURI()); + Files.walk(resourcePath) .filter(Files::isRegularFile) - .forEach(this::loadSchema); + .forEach(path -> loadSchema(resourcePath, path)); + } + + private void loadExternalSchemas() throws IOException { + Path externalPath = Paths.get(externalSchemaDirectory); + if (Files.exists(externalPath)) { + Files.walk(externalPath) + .filter(Files::isRegularFile) + .forEach(path -> loadSchema(externalPath, path)); + } } - private void loadSchema(Path path) { + private void loadSchema(Path basePath, Path path) { try { JsonNode schema = objectMapper.readTree(path.toFile()); - schemaRegistry.put(path.getFileName().toString(), schema); + String key = generateKey(basePath, path); + schemaRegistry.put(key, schema); } catch (IOException e) { throw new RuntimeException("Failed to load schema: " + path.getFileName(), e); } } + private String generateKey(Path basePath, Path path) { + // Remove the base directory part from the path and replace the file separators with dots + Path relativePath = basePath.relativize(path); + return relativePath.toString().replace(File.separatorChar, '/'); + } + + public void refreshExternalSchemas() throws IOException { + loadExternalSchemas(); + } + public Map listSchemas() { - return schemaRegistry; + return new HashMap<>(schemaRegistry); } public JsonNode getSchema(String schemaName) { return schemaRegistry.get(schemaName); } + + public void overwriteSchema(String schemaName, JsonNode newSchema) { + schemaRegistry.put(schemaName, newSchema); + } } diff --git a/onyxia-api/src/main/resources/schemas/nodeSelector.json b/onyxia-api/src/main/resources/schemas/nodeSelector.json new file mode 100644 index 00000000..79849804 --- /dev/null +++ b/onyxia-api/src/main/resources/schemas/nodeSelector.json @@ -0,0 +1,14 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Node Selector", + "type": "object", + "description": "Node selector constraints for the pod", + "additionalProperties": { + "type": "string", + "description": "Key-value pairs to select nodes" + }, + "x-onyxia": { + "hidden": false, + "overwriteDefaultWith": "region.nodeSelector" + } +} \ No newline at end of file diff --git a/onyxia-api/src/main/resources/schemas/s3.json b/onyxia-api/src/main/resources/schemas/s3.json index f0d9d55e..d519eff5 100644 --- a/onyxia-api/src/main/resources/schemas/s3.json +++ b/onyxia-api/src/main/resources/schemas/s3.json @@ -1,5 +1,4 @@ { - "$id": "https://example.com/schemas/s3.json", "title": "S3 Configuration", "description": "Configuration of temporary identity for AWS S3", "type": "object", From 8bf5e0a329c781c61ad2b5cea89b510b14828621 Mon Sep 17 00:00:00 2001 From: fcomte Date: Wed, 24 Jul 2024 12:25:24 +0000 Subject: [PATCH 38/53] fix resolution, no merge --- .../services/JsonSchemaResolutionService.java | 7 +- .../main/resources/schemas/resources-ide.json | 76 +++++++++++++++++++ 2 files changed, 77 insertions(+), 6 deletions(-) create mode 100644 onyxia-api/src/main/resources/schemas/resources-ide.json diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/services/JsonSchemaResolutionService.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/services/JsonSchemaResolutionService.java index 31cc6df6..4aaca163 100644 --- a/onyxia-api/src/main/java/fr/insee/onyxia/api/services/JsonSchemaResolutionService.java +++ b/onyxia-api/src/main/java/fr/insee/onyxia/api/services/JsonSchemaResolutionService.java @@ -84,12 +84,7 @@ else if (fieldValue.isObject() || fieldValue.isArray()) { } private ObjectNode mergeSchemas(JsonNode originalSchema, JsonNode overrideSchema) { - ObjectNode mergedSchema = (ObjectNode) originalSchema.deepCopy(); - Iterator> fields = overrideSchema.fields(); - while (fields.hasNext()) { - Map.Entry field = fields.next(); - mergedSchema.set(field.getKey(), field.getValue()); - } + ObjectNode mergedSchema = (ObjectNode) overrideSchema.deepCopy(); return mergedSchema; } diff --git a/onyxia-api/src/main/resources/schemas/resources-ide.json b/onyxia-api/src/main/resources/schemas/resources-ide.json new file mode 100644 index 00000000..a1805592 --- /dev/null +++ b/onyxia-api/src/main/resources/schemas/resources-ide.json @@ -0,0 +1,76 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Resources", + "description": "Your service will have at least the requested resources and never more than its limits. No limit for a resource and you can consume everything left on the host machine.", + "type": "object", + "properties": { + "requests": { + "description": "Guaranteed resources", + "type": "object", + "properties": { + "cpu": { + "description": "The amount of cpu guaranteed", + "title": "CPU", + "type": "string", + "default": "100m", + "render": "slider", + "sliderMin": 50, + "sliderMax": 40000, + "sliderStep": 50, + "sliderUnit": "m", + "sliderExtremity": "down", + "sliderExtremitySemantic": "guaranteed", + "sliderRangeId": "cpu" + }, + "memory": { + "description": "The amount of memory guaranteed", + "title": "memory", + "type": "string", + "default": "2Gi", + "render": "slider", + "sliderMin": 1, + "sliderMax": 200, + "sliderStep": 1, + "sliderUnit": "Gi", + "sliderExtremity": "down", + "sliderExtremitySemantic": "guaranteed", + "sliderRangeId": "memory" + } + } + }, + "limits": { + "description": "max resources", + "type": "object", + "properties": { + "cpu": { + "description": "The maximum amount of cpu", + "title": "CPU", + "type": "string", + "default": "30000m", + "render": "slider", + "sliderMin": 50, + "sliderMax": 40000, + "sliderStep": 50, + "sliderUnit": "m", + "sliderExtremity": "up", + "sliderExtremitySemantic": "Maximum", + "sliderRangeId": "cpu" + }, + "memory": { + "description": "The maximum amount of memory", + "title": "Memory", + "type": "string", + "default": "50Gi", + "render": "slider", + "sliderMin": 1, + "sliderMax": 200, + "sliderStep": 1, + "sliderUnit": "Gi", + "sliderExtremity": "up", + "sliderExtremitySemantic": "Maximum", + "sliderRangeId": "memory" + } + } + } + } +} From c7570eef0e3fb3a3cb44946a92965aa97d9f2b9e Mon Sep 17 00:00:00 2001 From: fcomte Date: Wed, 24 Jul 2024 13:18:56 +0000 Subject: [PATCH 39/53] fix resolution --- .../services/JsonSchemaResolutionService.java | 29 ++++++++----------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/services/JsonSchemaResolutionService.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/services/JsonSchemaResolutionService.java index 4aaca163..9c43b77b 100644 --- a/onyxia-api/src/main/java/fr/insee/onyxia/api/services/JsonSchemaResolutionService.java +++ b/onyxia-api/src/main/java/fr/insee/onyxia/api/services/JsonSchemaResolutionService.java @@ -29,14 +29,13 @@ public JsonNode resolveReferences(JsonNode schemaNode) { private JsonNode resolveReferences(JsonNode schemaNode, JsonNode rootNode) { if (schemaNode.isObject()) { ObjectNode objectNode = (ObjectNode) schemaNode; + Iterator> fields = objectNode.fields(); Map updates = new HashMap<>(); - Iterator> fields = objectNode.fields(); while (fields.hasNext()) { Map.Entry field = fields.next(); JsonNode fieldValue = field.getValue(); - // Handle $ref attribute if (field.getKey().equals("$ref") && fieldValue.isTextual()) { String ref = fieldValue.asText(); JsonNode refNode = null; @@ -45,24 +44,25 @@ private JsonNode resolveReferences(JsonNode schemaNode, JsonNode rootNode) { } else { refNode = registryService.getSchema(ref); } + if (refNode != null && !refNode.isMissingNode()) { JsonNode resolvedNode = resolveReferences(refNode.deepCopy(), rootNode); updates.putAll(convertToMap((ObjectNode) resolvedNode)); updates.put("$ref", null); } - } - // Handle x-onyxia.overwriteSchemaWith attribute - else if (field.getKey().equals("x-onyxia") - && fieldValue.has("overwriteSchemaWith")) { - String overrideSchemaName = fieldValue.get("overwriteSchemaWith").asText(); + } else if (fieldValue.isObject() + && fieldValue.has("x-onyxia") + && fieldValue.get("x-onyxia").has("overwriteSchemaWith")) { + String overrideSchemaName = + fieldValue.get("x-onyxia").get("overwriteSchemaWith").asText(); JsonNode overrideSchemaNode = registryService.getSchema(overrideSchemaName); + if (overrideSchemaNode != null && !overrideSchemaNode.isMissingNode()) { - JsonNode mergedNode = mergeSchemas(schemaNode, overrideSchemaNode); - updates.putAll(convertToMap((ObjectNode) mergedNode)); + JsonNode resolvedNode = + resolveReferences(overrideSchemaNode.deepCopy(), rootNode); + updates.put(field.getKey(), resolvedNode); } - } - // Recursively resolve for nested objects - else if (fieldValue.isObject() || fieldValue.isArray()) { + } else if (fieldValue.isObject() || fieldValue.isArray()) { updates.put(field.getKey(), resolveReferences(fieldValue, rootNode)); } } @@ -83,11 +83,6 @@ else if (fieldValue.isObject() || fieldValue.isArray()) { return schemaNode; } - private ObjectNode mergeSchemas(JsonNode originalSchema, JsonNode overrideSchema) { - ObjectNode mergedSchema = (ObjectNode) overrideSchema.deepCopy(); - return mergedSchema; - } - private Map convertToMap(ObjectNode objectNode) { Map map = new HashMap<>(); Iterator> fields = objectNode.fields(); From 7a0f4ac25b264f19cd111b9517d33ad17b4e6a47 Mon Sep 17 00:00:00 2001 From: fcomte Date: Thu, 25 Jul 2024 08:26:52 +0000 Subject: [PATCH 40/53] handle validation exception --- .../api/controller/RestExceptionHandler.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/RestExceptionHandler.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/RestExceptionHandler.java index a3dd38af..eb6a7c89 100644 --- a/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/RestExceptionHandler.java +++ b/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/RestExceptionHandler.java @@ -1,6 +1,9 @@ package fr.insee.onyxia.api.controller; import fr.insee.onyxia.api.controller.exception.SchemaNotFoundException; +import java.util.List; +import java.util.stream.Collectors; +import org.everit.json.schema.ValidationException; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.security.access.AccessDeniedException; @@ -19,4 +22,17 @@ public void handleAccessDeniedException(Exception ignored) {} public ResponseEntity handleSchemaNotFoundException(SchemaNotFoundException ex) { return new ResponseEntity<>(ex.getMessage(), HttpStatus.NOT_FOUND); } + + @ExceptionHandler(ValidationException.class) + public ResponseEntity handleValidationException(ValidationException ex) { + List errors = + ex.getCausingExceptions().stream() + .map(ValidationException::getMessage) + .collect(Collectors.toList()); + + ErrorResponse errorResponse = + new ErrorResponse(HttpStatus.BAD_REQUEST.value(), "Validation failed", errors); + + return new ResponseEntity<>(errorResponse, HttpStatus.BAD_REQUEST); + } } From 590fd3ad453b56b040d27ad092176e409f894b7a Mon Sep 17 00:00:00 2001 From: fcomte Date: Thu, 25 Jul 2024 08:40:46 +0000 Subject: [PATCH 41/53] fix exception --- .../api/controller/RestExceptionHandler.java | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/RestExceptionHandler.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/RestExceptionHandler.java index eb6a7c89..e23c703b 100644 --- a/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/RestExceptionHandler.java +++ b/onyxia-api/src/main/java/fr/insee/onyxia/api/controller/RestExceptionHandler.java @@ -35,4 +35,41 @@ public ResponseEntity handleValidationException(ValidationExcepti return new ResponseEntity<>(errorResponse, HttpStatus.BAD_REQUEST); } + + // Define the ErrorResponse class within the GlobalExceptionHandler + public static class ErrorResponse { + private int status; + private String message; + private List errors; + + public ErrorResponse(int status, String message, List errors) { + this.status = status; + this.message = message; + this.errors = errors; + } + + public int getStatus() { + return status; + } + + public void setStatus(int status) { + this.status = status; + } + + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } + + public List getErrors() { + return errors; + } + + public void setErrors(List errors) { + this.errors = errors; + } + } } From 600eab8fc574747ba9ea32c6742b42c4e608cc9d Mon Sep 17 00:00:00 2001 From: fcomte Date: Thu, 25 Jul 2024 09:42:34 +0000 Subject: [PATCH 42/53] add schemas --- .../resources/schemas/resources-ide-gpu.json | 87 +++++++++++++++++++ .../src/main/resources/schemas/role.json | 26 ++++++ .../main/resources/schemas/tolerations.json | 35 ++++++++ .../src/main/resources/schemas/vault.json | 72 +++++++++++++++ 4 files changed, 220 insertions(+) create mode 100644 onyxia-api/src/main/resources/schemas/resources-ide-gpu.json create mode 100644 onyxia-api/src/main/resources/schemas/role.json create mode 100644 onyxia-api/src/main/resources/schemas/tolerations.json create mode 100644 onyxia-api/src/main/resources/schemas/vault.json diff --git a/onyxia-api/src/main/resources/schemas/resources-ide-gpu.json b/onyxia-api/src/main/resources/schemas/resources-ide-gpu.json new file mode 100644 index 00000000..e60858f3 --- /dev/null +++ b/onyxia-api/src/main/resources/schemas/resources-ide-gpu.json @@ -0,0 +1,87 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Resources", + "description": "Your service will have at least the requested resources and never more than its limits. No limit for a resource and you can consume everything left on the host machine.", + "type": "object", + "properties": { + "requests": { + "description": "Guaranteed resources", + "type": "object", + "properties": { + "cpu": { + "description": "The amount of cpu guaranteed", + "title": "CPU", + "type": "string", + "default": "100m", + "render": "slider", + "sliderMin": 50, + "sliderMax": 40000, + "sliderStep": 50, + "sliderUnit": "m", + "sliderExtremity": "down", + "sliderExtremitySemantic": "guaranteed", + "sliderRangeId": "cpu" + }, + "memory": { + "description": "The amount of memory guaranteed", + "title": "memory", + "type": "string", + "default": "2Gi", + "render": "slider", + "sliderMin": 1, + "sliderMax": 200, + "sliderStep": 1, + "sliderUnit": "Gi", + "sliderExtremity": "down", + "sliderExtremitySemantic": "guaranteed", + "sliderRangeId": "memory" + } + } + }, + "limits": { + "description": "max resources", + "type": "object", + "properties": { + "cpu": { + "description": "The maximum amount of cpu", + "title": "CPU", + "type": "string", + "default": "30000m", + "render": "slider", + "sliderMin": 50, + "sliderMax": 40000, + "sliderStep": 50, + "sliderUnit": "m", + "sliderExtremity": "up", + "sliderExtremitySemantic": "Maximum", + "sliderRangeId": "cpu" + }, + "memory": { + "description": "The maximum amount of memory", + "title": "Memory", + "type": "string", + "default": "50Gi", + "render": "slider", + "sliderMin": 1, + "sliderMax": 200, + "sliderStep": 1, + "sliderUnit": "Gi", + "sliderExtremity": "up", + "sliderExtremitySemantic": "Maximum", + "sliderRangeId": "memory" + }, + "nvidia.com/gpu": { + "description": "GPU to allocate to this instance. This is also requested", + "type": "string", + "default": "1", + "render": "slider", + "sliderMin": 1, + "sliderMax": 3, + "sliderStep": 1, + "sliderUnit": "" + } + } + } + } + +} diff --git a/onyxia-api/src/main/resources/schemas/role.json b/onyxia-api/src/main/resources/schemas/role.json new file mode 100644 index 00000000..e5832fb6 --- /dev/null +++ b/onyxia-api/src/main/resources/schemas/role.json @@ -0,0 +1,26 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Role", + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "description": "allow your service to access your namespace ressources", + "default": true + }, + "role": { + "type": "string", + "description": "bind your service account to this kubernetes default role", + "default": "view", + "hidden": { + "value": false, + "path": "kubernetes/enabled" + }, + "enum": [ + "view", + "edit", + "admin" + ] + } + } +} \ No newline at end of file diff --git a/onyxia-api/src/main/resources/schemas/tolerations.json b/onyxia-api/src/main/resources/schemas/tolerations.json new file mode 100644 index 00000000..8b85cafd --- /dev/null +++ b/onyxia-api/src/main/resources/schemas/tolerations.json @@ -0,0 +1,35 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Kubernetes Tolerations", + "type": "array", + "items": { + "type": "object", + "properties": { + "key": { + "type": "string", + "description": "The taint key that the toleration applies to." + }, + "operator": { + "type": "string", + "enum": ["Exists", "Equal"], + "description": "The operator indicates the relationship between the key and value." + }, + "value": { + "type": "string", + "description": "The taint value that the toleration matches to." + }, + "effect": { + "type": "string", + "enum": ["NoSchedule", "PreferNoSchedule", "NoExecute"], + "description": "The effect indicates what action should be taken when the toleration is matched." + }, + "tolerationSeconds": { + "type": "integer", + "description": "The period of time the toleration (which must be of effect NoExecute, otherwise this field is ignored) tolerates the taint. Represented in seconds." + } + }, + "required": ["key", "operator", "effect"], + "additionalProperties": false + } + } + \ No newline at end of file diff --git a/onyxia-api/src/main/resources/schemas/vault.json b/onyxia-api/src/main/resources/schemas/vault.json new file mode 100644 index 00000000..150e4a0e --- /dev/null +++ b/onyxia-api/src/main/resources/schemas/vault.json @@ -0,0 +1,72 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Vault", + "description": "Configuration of vault client", + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "description": "Add vault temporary identity inside your environment", + "default": true + }, + "token": { + "description": "token vault", + "type": "string", + "render": "password", + "x-onyxia": { + "overwriteDefaultWith": "{{vault.VAULT_TOKEN}}" + }, + "hidden": { + "value": false, + "path": "enabled", + "isPathRelative": true + } + }, + "url": { + "description": "url of vault server", + "type": "string", + "x-onyxia": { + "overwriteDefaultWith": "{{vault.VAULT_ADDR}}" + }, + "hidden": { + "value": false, + "path": "enabled", + "isPathRelative": true + } + }, + "mount": { + "description": "mount of the v2 secret engine", + "type": "string", + "x-onyxia": { + "overwriteDefaultWith": "{{vault.VAULT_MOUNT}}" + }, + "hidden": { + "value": false, + "path": "enabled", + "isPathRelative": true + } + }, + "directory": { + "description": "top level directory", + "type": "string", + "x-onyxia": { + "overwriteDefaultWith": "{{vault.VAULT_TOP_DIR}}" + }, + "hidden": { + "value": false, + "path": "enabled", + "isPathRelative": true + } + }, + "secret": { + "description": "the path of the secret to convert into a list of environment variables", + "type": "string", + "default": "", + "hidden": { + "value": false, + "path": "enabled", + "isPathRelative": true + } + } + } +} \ No newline at end of file From 4a6b5f909beba18738f2b454fffa2e007c74293a Mon Sep 17 00:00:00 2001 From: fcomte Date: Thu, 25 Jul 2024 15:01:14 +0000 Subject: [PATCH 43/53] add schemas --- .../main/resources/schemas/customImage.json | 7 ++ .../src/main/resources/schemas/git.json | 89 +++++++++++++++++++ .../src/main/resources/schemas/init-ide.json | 27 ++++++ .../resources/schemas/persistance-ide.json | 29 ++++++ 4 files changed, 152 insertions(+) create mode 100644 onyxia-api/src/main/resources/schemas/customImage.json create mode 100644 onyxia-api/src/main/resources/schemas/git.json create mode 100644 onyxia-api/src/main/resources/schemas/init-ide.json create mode 100644 onyxia-api/src/main/resources/schemas/persistance-ide.json diff --git a/onyxia-api/src/main/resources/schemas/customImage.json b/onyxia-api/src/main/resources/schemas/customImage.json new file mode 100644 index 00000000..d9dd8f6d --- /dev/null +++ b/onyxia-api/src/main/resources/schemas/customImage.json @@ -0,0 +1,7 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "custom image", + "type": "boolean", + "description": "use a custom jupyter docker image", + "default": false +} \ No newline at end of file diff --git a/onyxia-api/src/main/resources/schemas/git.json b/onyxia-api/src/main/resources/schemas/git.json new file mode 100644 index 00000000..419dfeb6 --- /dev/null +++ b/onyxia-api/src/main/resources/schemas/git.json @@ -0,0 +1,89 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "GIT", + "description": "Git user configuration", + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "description": "Add git config inside your environment", + "default": true + }, + "name": { + "type": "string", + "description": "user name for git", + "default": "", + "x-onyxia": { + "overwriteDefaultWith": "{{git.name}}" + }, + "hidden": { + "value": false, + "path": "enabled", + "isPathRelative": true + } + }, + "email": { + "type": "string", + "description": "user email for git", + "default": "", + "x-onyxia": { + "overwriteDefaultWith": "{{git.email}}" + }, + "hidden": { + "value": false, + "path": "enabled", + "isPathRelative": true + } + }, + "cache": { + "type": "string", + "description": "duration in seconds of the credentials cache duration", + "default": "", + "x-onyxia": { + "overwriteDefaultWith": "{{git.credentials_cache_duration}}" + }, + "hidden": { + "value": false, + "path": "enabled", + "isPathRelative": true + } + }, + "token": { + "type": "string", + "description": "personal access token", + "default": "", + "render": "password", + "x-onyxia": { + "overwriteDefaultWith": "{{git.token}}" + }, + "hidden": { + "value": false, + "path": "enabled", + "isPathRelative": true + } + }, + "repository": { + "type": "string", + "description": "Repository url", + "default": "", + "x-onyxia": { + "overwriteDefaultWith": "{{git.project}}" + }, + "hidden": { + "value": false, + "path": "enabled", + "isPathRelative": true + } + }, + "branch": { + "type": "string", + "description": "Branch automatically checked out", + "default": "", + "hidden": { + "value": false, + "path": "enabled", + "isPathRelative": true + } + } + } +} \ No newline at end of file diff --git a/onyxia-api/src/main/resources/schemas/init-ide.json b/onyxia-api/src/main/resources/schemas/init-ide.json new file mode 100644 index 00000000..686cd406 --- /dev/null +++ b/onyxia-api/src/main/resources/schemas/init-ide.json @@ -0,0 +1,27 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Init", + "description": "Init parameters", + "type": "object", + "properties": { + "regionInit": { + "type": "string", + "description": "region initialization script", + "default": "", + "x-onyxia": { + "hidden": true, + "overwriteDefaultWith": "{{k8s.initScriptUrl}}" + } + }, + "personalInit": { + "type": "string", + "description": "user initialization script", + "default": "" + }, + "personalInitArgs": { + "type": "string", + "description": "args for user initialization script", + "default": "" + } + } +} \ No newline at end of file diff --git a/onyxia-api/src/main/resources/schemas/persistance-ide.json b/onyxia-api/src/main/resources/schemas/persistance-ide.json new file mode 100644 index 00000000..70e925e2 --- /dev/null +++ b/onyxia-api/src/main/resources/schemas/persistance-ide.json @@ -0,0 +1,29 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Configuration for persistence", + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "description": "Create a persistent volume", + "default": true + }, + "size": { + "type": "string", + "title": "Persistent volume size", + "description": "Size of the persistent volume", + "default": "10Gi", + "form": true, + "render": "slider", + "sliderMin": 1, + "sliderMax": 100, + "sliderStep": 1, + "sliderUnit": "Gi", + "hidden": { + "value": false, + "path": "enabled", + "isPathRelative": true + } + } + } +} \ No newline at end of file From 32d9556d6fe1abbd138c15ecbb1be6ae0eaafe3b Mon Sep 17 00:00:00 2001 From: ihiverlet <94982379+ihiverlet@users.noreply.github.com> Date: Fri, 26 Jul 2024 14:42:26 +0200 Subject: [PATCH 44/53] add message.json in schemas --- onyxia-api/src/main/resources/schemas/message.json | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 onyxia-api/src/main/resources/schemas/message.json diff --git a/onyxia-api/src/main/resources/schemas/message.json b/onyxia-api/src/main/resources/schemas/message.json new file mode 100644 index 00000000..5cb02c54 --- /dev/null +++ b/onyxia-api/src/main/resources/schemas/message.json @@ -0,0 +1,7 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "message", + "type": "string", + "description": "add a message in the notes", + "default": "" +} \ No newline at end of file From 4543b1ceba4f5db8d236e99433317b6c1b218569 Mon Sep 17 00:00:00 2001 From: fcomte Date: Mon, 29 Jul 2024 13:04:42 +0000 Subject: [PATCH 45/53] add schemas --- .../custom-image.json} | 0 .../main/resources/schemas/{ => ide}/git.json | 0 .../main/resources/schemas/ide/ingress.json | 64 +++++++++++++++++++ .../schemas/{init-ide.json => ide/init.json} | 0 .../resources/schemas/{ => ide}/message.json | 0 .../resources/schemas/ide/network-policy.json | 19 ++++++ .../resources/schemas/ide/openshiftSCC.json | 18 ++++++ .../main/resources/schemas/ide/password.json | 10 +++ .../persistance.json} | 0 .../resources-gpu.json} | 0 .../resources.json} | 0 .../resources/schemas/{ => ide}/role.json | 0 .../src/main/resources/schemas/ide/route.json | 35 ++++++++++ .../main/resources/schemas/{ => ide}/s3.json | 0 .../resources/schemas/{ => ide}/vault.json | 0 15 files changed, 146 insertions(+) rename onyxia-api/src/main/resources/schemas/{customImage.json => ide/custom-image.json} (100%) rename onyxia-api/src/main/resources/schemas/{ => ide}/git.json (100%) create mode 100644 onyxia-api/src/main/resources/schemas/ide/ingress.json rename onyxia-api/src/main/resources/schemas/{init-ide.json => ide/init.json} (100%) rename onyxia-api/src/main/resources/schemas/{ => ide}/message.json (100%) create mode 100644 onyxia-api/src/main/resources/schemas/ide/network-policy.json create mode 100644 onyxia-api/src/main/resources/schemas/ide/openshiftSCC.json create mode 100644 onyxia-api/src/main/resources/schemas/ide/password.json rename onyxia-api/src/main/resources/schemas/{persistance-ide.json => ide/persistance.json} (100%) rename onyxia-api/src/main/resources/schemas/{resources-ide-gpu.json => ide/resources-gpu.json} (100%) rename onyxia-api/src/main/resources/schemas/{resources-ide.json => ide/resources.json} (100%) rename onyxia-api/src/main/resources/schemas/{ => ide}/role.json (100%) create mode 100644 onyxia-api/src/main/resources/schemas/ide/route.json rename onyxia-api/src/main/resources/schemas/{ => ide}/s3.json (100%) rename onyxia-api/src/main/resources/schemas/{ => ide}/vault.json (100%) diff --git a/onyxia-api/src/main/resources/schemas/customImage.json b/onyxia-api/src/main/resources/schemas/ide/custom-image.json similarity index 100% rename from onyxia-api/src/main/resources/schemas/customImage.json rename to onyxia-api/src/main/resources/schemas/ide/custom-image.json diff --git a/onyxia-api/src/main/resources/schemas/git.json b/onyxia-api/src/main/resources/schemas/ide/git.json similarity index 100% rename from onyxia-api/src/main/resources/schemas/git.json rename to onyxia-api/src/main/resources/schemas/ide/git.json diff --git a/onyxia-api/src/main/resources/schemas/ide/ingress.json b/onyxia-api/src/main/resources/schemas/ide/ingress.json new file mode 100644 index 00000000..83af0879 --- /dev/null +++ b/onyxia-api/src/main/resources/schemas/ide/ingress.json @@ -0,0 +1,64 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Ingress", + "description": "Ingress parameters", + "type": "object", + "properties": { + "enabled": { + "description": "Enable Ingress", + "type": "boolean", + "default": true, + "x-onyxia": { + "hidden": true, + "overwriteDefaultWith": "k8s.ingress" + } + }, + "hostname": { + "type": "string", + "form": true, + "title": "Hostname", + "x-onyxia": { + "hidden": true, + "overwriteDefaultWith": "{{project.id}}-{{k8s.randomSubdomain}}-0.{{k8s.domain}}" + } + }, + "userHostname": { + "type": "string", + "form": true, + "title": "Hostname", + "x-onyxia": { + "hidden": true, + "overwriteDefaultWith": "{{project.id}}-{{k8s.randomSubdomain}}-user.{{k8s.domain}}" + } + }, + "ingressClassName": { + "type": "string", + "form": true, + "title": "ingressClassName", + "default": "", + "x-onyxia": { + "hidden": true, + "overwriteDefaultWith": "{{k8s.ingressClassName}}" + } + }, + "useCertManager": { + "type": "boolean", + "description": "Whether CertManager should be used to generate a certificate", + "default": false, + "x-onyxia": { + "hidden": true, + "overwriteDefaultWith": "k8s.useCertManager" + } + }, + "certManagerClusterIssuer":{ + "type": "string", + "description": "certManager cluster issuer", + "title": "CertManager Cluster Issuer", + "default": "", + "x-onyxia": { + "hidden": true, + "overwriteDefaultWith": "k8s.certManagerClusterIssuer" + } + } + } +} \ No newline at end of file diff --git a/onyxia-api/src/main/resources/schemas/init-ide.json b/onyxia-api/src/main/resources/schemas/ide/init.json similarity index 100% rename from onyxia-api/src/main/resources/schemas/init-ide.json rename to onyxia-api/src/main/resources/schemas/ide/init.json diff --git a/onyxia-api/src/main/resources/schemas/message.json b/onyxia-api/src/main/resources/schemas/ide/message.json similarity index 100% rename from onyxia-api/src/main/resources/schemas/message.json rename to onyxia-api/src/main/resources/schemas/ide/message.json diff --git a/onyxia-api/src/main/resources/schemas/ide/network-policy.json b/onyxia-api/src/main/resources/schemas/ide/network-policy.json new file mode 100644 index 00000000..bf4665bf --- /dev/null +++ b/onyxia-api/src/main/resources/schemas/ide/network-policy.json @@ -0,0 +1,19 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "title": "Network Policy", + "description": "Define access policy to the service", + "properties": { + "enabled": { + "type": "boolean", + "title": "Enable network policy", + "description": "Only pod from the same namespace will be allowed", + "default": false + }, + "from": { + "type": "array", + "description": "Array of source allowed to have network access to your service", + "default": [] + } + } +} \ No newline at end of file diff --git a/onyxia-api/src/main/resources/schemas/ide/openshiftSCC.json b/onyxia-api/src/main/resources/schemas/ide/openshiftSCC.json new file mode 100644 index 00000000..c2898a8a --- /dev/null +++ b/onyxia-api/src/main/resources/schemas/ide/openshiftSCC.json @@ -0,0 +1,18 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Openshift SCC", + "description": "configuration for openshift compatibility", + "type": "object", + "properties": { + "enabled": { + "description": "enable rolebinding with openshift scc", + "type": "boolean", + "default": false + }, + "scc": { + "type": "string", + "description": "name of scc for rolebinding", + "default": "anyuid" + } + } +} diff --git a/onyxia-api/src/main/resources/schemas/ide/password.json b/onyxia-api/src/main/resources/schemas/ide/password.json new file mode 100644 index 00000000..613d0187 --- /dev/null +++ b/onyxia-api/src/main/resources/schemas/ide/password.json @@ -0,0 +1,10 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "string", + "description": "Password", + "default": "changeme", + "render": "password", + "x-onyxia": { + "overwriteDefaultWith": "{{service.oneTimePassword}}" + } +} \ No newline at end of file diff --git a/onyxia-api/src/main/resources/schemas/persistance-ide.json b/onyxia-api/src/main/resources/schemas/ide/persistance.json similarity index 100% rename from onyxia-api/src/main/resources/schemas/persistance-ide.json rename to onyxia-api/src/main/resources/schemas/ide/persistance.json diff --git a/onyxia-api/src/main/resources/schemas/resources-ide-gpu.json b/onyxia-api/src/main/resources/schemas/ide/resources-gpu.json similarity index 100% rename from onyxia-api/src/main/resources/schemas/resources-ide-gpu.json rename to onyxia-api/src/main/resources/schemas/ide/resources-gpu.json diff --git a/onyxia-api/src/main/resources/schemas/resources-ide.json b/onyxia-api/src/main/resources/schemas/ide/resources.json similarity index 100% rename from onyxia-api/src/main/resources/schemas/resources-ide.json rename to onyxia-api/src/main/resources/schemas/ide/resources.json diff --git a/onyxia-api/src/main/resources/schemas/role.json b/onyxia-api/src/main/resources/schemas/ide/role.json similarity index 100% rename from onyxia-api/src/main/resources/schemas/role.json rename to onyxia-api/src/main/resources/schemas/ide/role.json diff --git a/onyxia-api/src/main/resources/schemas/ide/route.json b/onyxia-api/src/main/resources/schemas/ide/route.json new file mode 100644 index 00000000..c003c0a4 --- /dev/null +++ b/onyxia-api/src/main/resources/schemas/ide/route.json @@ -0,0 +1,35 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Route", + "description": "Route parameters", + "type": "object", + "properties": { + "enabled": { + "description": "Enable route", + "type": "boolean", + "default": false, + "x-onyxia": { + "hidden": true, + "overwriteDefaultWith": "k8s.route" + } + }, + "hostname": { + "type": "string", + "form": true, + "title": "Hostname", + "x-onyxia": { + "hidden": true, + "overwriteDefaultWith": "{{project.id}}-{{k8s.randomSubdomain}}-0.{{k8s.domain}}" + } + }, + "userHostname": { + "type": "string", + "form": true, + "title": "Hostname", + "x-onyxia": { + "hidden": true, + "overwriteDefaultWith": "{{project.id}}-{{k8s.randomSubdomain}}-user.{{k8s.domain}}" + } + } + } +} \ No newline at end of file diff --git a/onyxia-api/src/main/resources/schemas/s3.json b/onyxia-api/src/main/resources/schemas/ide/s3.json similarity index 100% rename from onyxia-api/src/main/resources/schemas/s3.json rename to onyxia-api/src/main/resources/schemas/ide/s3.json diff --git a/onyxia-api/src/main/resources/schemas/vault.json b/onyxia-api/src/main/resources/schemas/ide/vault.json similarity index 100% rename from onyxia-api/src/main/resources/schemas/vault.json rename to onyxia-api/src/main/resources/schemas/ide/vault.json From c7ace556989821098a0b97b39b9e55dd28573aac Mon Sep 17 00:00:00 2001 From: Olivier Levitt Date: Thu, 1 Aug 2024 17:57:21 +0200 Subject: [PATCH 46/53] Fix spotless --- .../onyxia/api/dao/universe/CatalogLoader.java | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java b/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java index 29dbb89f..93173209 100644 --- a/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java +++ b/onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java @@ -1,5 +1,7 @@ package fr.insee.onyxia.api.dao.universe; +import static java.nio.charset.StandardCharsets.UTF_8; + import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.JsonNode; @@ -12,6 +14,8 @@ import fr.insee.onyxia.model.catalog.Pkg; import fr.insee.onyxia.model.helm.Chart; import fr.insee.onyxia.model.helm.Repository; +import java.io.*; +import java.util.*; import org.apache.commons.compress.archivers.tar.TarArchiveEntry; import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; @@ -27,11 +31,6 @@ import org.springframework.stereotype.Service; import org.springframework.util.StringUtils; -import java.io.*; -import java.util.*; - -import static java.nio.charset.StandardCharsets.UTF_8; - @Service public class CatalogLoader { @@ -196,9 +195,11 @@ public void extractDataFromTgz(InputStream in, Chart chart) throws IOException { mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) { tarIn.transferTo(baos); - chart.setConfig(jsonSchemaResolutionService.resolveReferences(mapper.readTree(baos.toString(UTF_8)))); + chart.setConfig( + jsonSchemaResolutionService.resolveReferences( + mapper.readTree(baos.toString(UTF_8)))); } - } else if (entryName.endsWith(chartName + "/values.yaml") + } else if (entryName.endsWith(chartName + "/values.yaml") && !entryName.endsWith("charts/" + chartName + "/values.yaml")) { try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) { tarIn.transferTo(baos); From 842eb8aaec489939fba6096ac6148c0823e1d9bd Mon Sep 17 00:00:00 2001 From: fcomte Date: Sat, 3 Aug 2024 13:28:21 +0000 Subject: [PATCH 47/53] remove many configuration --- .../fr/insee/onyxia/model/region/Region.java | 368 ------------------ 1 file changed, 368 deletions(-) diff --git a/onyxia-model/src/main/java/fr/insee/onyxia/model/region/Region.java b/onyxia-model/src/main/java/fr/insee/onyxia/model/region/Region.java index b13af9d4..e720e450 100644 --- a/onyxia-model/src/main/java/fr/insee/onyxia/model/region/Region.java +++ b/onyxia-model/src/main/java/fr/insee/onyxia/model/region/Region.java @@ -199,50 +199,13 @@ public static class Services { private Expose expose; private Server server; private Monitoring monitoring; - private String initScript; private String allowedURIPattern = "^https://"; private Quotas quotas = new Quotas(); - private DefaultConfiguration defaultConfiguration = new DefaultConfiguration(); private K8sPublicEndpoint k8sPublicEndpoint = new K8sPublicEndpoint(); - private CustomInitScript customInitScript = new CustomInitScript(); - private OpenshiftSCC openshiftSCC = new OpenshiftSCC(); - private Map customValues = new HashMap<>(); private NamespaceAnnotationsDynamic namespaceAnnotationsDynamic = new NamespaceAnnotationsDynamic(); - public DefaultConfiguration getDefaultConfiguration() { - return defaultConfiguration; - } - - public void setDefaultConfiguration(DefaultConfiguration defaultConfiguration) { - this.defaultConfiguration = defaultConfiguration; - } - - public Map getCustomValues() { - return customValues; - } - - public void setCustomValues(Map customValues) { - this.customValues = customValues; - } - - public CustomInitScript getCustomInitScript() { - return customInitScript; - } - - public void setCustomInitScript(CustomInitScript customInitScript) { - this.customInitScript = customInitScript; - } - - public OpenshiftSCC getOpenshiftSCC() { - return openshiftSCC; - } - - public void setOpenshiftSCC(OpenshiftSCC openshiftSCC) { - this.openshiftSCC = openshiftSCC; - } - public boolean isSingleNamespace() { return singleNamespace; } @@ -340,14 +303,6 @@ public void setMonitoring(Monitoring monitoring) { this.monitoring = monitoring; } - public String getInitScript() { - return initScript; - } - - public void setInitScript(String initScript) { - this.initScript = initScript; - } - public String getAllowedURIPattern() { return allowedURIPattern; } @@ -429,250 +384,6 @@ public void setUserAttributes(List userAttributes) { } } - public static class DefaultConfiguration { - private boolean IPProtection = false; - private boolean networkPolicy = false; - private List from = new ArrayList<>(); - private List tolerations = new ArrayList<>(); - private Object nodeSelector; - private Object startupProbe; - private Kafka kafka = new Kafka(); - private Sliders sliders = new Sliders(); - private Resources resources = new Resources(); - - public boolean isIPProtection() { - return IPProtection; - } - - public void setIPProtection(boolean IPProtection) { - this.IPProtection = IPProtection; - } - - public boolean isNetworkPolicy() { - return networkPolicy; - } - - public void setNetworkPolicy(boolean networkPolicy) { - this.networkPolicy = networkPolicy; - } - - public List getFrom() { - return from; - } - - public void setFrom(List from) { - this.from = from; - } - - public List getTolerations() { - return tolerations; - } - - public void setTolerations(List tolerations) { - this.tolerations = tolerations; - } - - public Object getNodeSelector() { - return nodeSelector; - } - - public void setNodeSelector(Object nodeSelector) { - this.nodeSelector = nodeSelector; - } - - public Object getStartupProbe() { - return startupProbe; - } - - public void setStartupProbe(Object startupProbe) { - this.startupProbe = startupProbe; - } - - public Kafka getKafka() { - return kafka; - } - - public void setKafka(Kafka kafka) { - this.kafka = kafka; - } - - public Sliders getSliders() { - return sliders; - } - - public void setSliders(Sliders sliders) { - this.sliders = sliders; - } - - public Resources getResources() { - return resources; - } - - public void setResources(Resources resources) { - this.resources = resources; - } - - public static class Kafka { - @JsonProperty("URL") - private String url; - - private String topicName; - - public String getUrl() { - return url; - } - - public void setUrl(String url) { - this.url = url; - } - - public String getTopicName() { - return topicName; - } - - public void setTopicName(String topicName) { - this.topicName = topicName; - } - } - - public static class Sliders { - - Slider cpu; - Slider memory; - Slider gpu; - Slider disk; - - public Slider getCpu() { - return cpu; - } - - public void setCpu(Slider cpu) { - this.cpu = cpu; - } - - public Slider getMemory() { - return memory; - } - - public void setMemory(Slider memory) { - this.memory = memory; - } - - public Slider getGpu() { - return gpu; - } - - public void setGpu(Slider gpu) { - this.gpu = gpu; - } - - public Slider getDisk() { - return disk; - } - - public void setDisk(Slider disk) { - this.disk = disk; - } - - public static class Slider { - - Integer sliderMin; - Integer sliderMax; - Integer sliderStep; - String sliderUnit; - - public Integer getSliderMin() { - return sliderMin; - } - - public void setSliderMin(Integer sliderMin) { - this.sliderMin = sliderMin; - } - - public Integer getSliderMax() { - return sliderMax; - } - - public void setSliderMax(Integer sliderMax) { - this.sliderMax = sliderMax; - } - - public Integer getSliderStep() { - return sliderStep; - } - - public void setSliderStep(Integer sliderStep) { - this.sliderStep = sliderStep; - } - - public String getSliderUnit() { - return sliderUnit; - } - - public void setSliderUnit(String sliderUnit) { - this.sliderUnit = sliderUnit; - } - } - } - - public static class Resources { - private String cpuRequest; - private String cpuLimit; - private String memoryRequest; - private String memoryLimit; - private String disk; - private String gpu; - - public String getCpuRequest() { - return cpuRequest; - } - - public void setCpuRequest(String cpuRequest) { - this.cpuRequest = cpuRequest; - } - - public String getCpuLimit() { - return cpuLimit; - } - - public void setCpuLimit(String cpuLimit) { - this.cpuLimit = cpuLimit; - } - - public String getMemoryRequest() { - return memoryRequest; - } - - public void setMemoryRequest(String memoryRequest) { - this.memoryRequest = memoryRequest; - } - - public String getMemoryLimit() { - return memoryLimit; - } - - public void setMemoryLimit(String memoryLimit) { - this.memoryLimit = memoryLimit; - } - - public String getDisk() { - return disk; - } - - public void setDisk(String disk) { - this.disk = disk; - } - - public String getGpu() { - return gpu; - } - - public void setGpu(String gpu) { - this.gpu = gpu; - } - } - } - public static class Quotas { // could be deprecated as userQuota/groupQuota is enough private boolean enabled = false; @@ -764,19 +475,9 @@ public void setUrlPattern(String urlPattern) { @Schema(description = "") public static class Data { - private Atlas atlas; - @JsonProperty("S3") private S3 s3; - public Atlas getAtlas() { - return atlas; - } - - public void setAtlas(Atlas atlas) { - this.atlas = atlas; - } - public S3 getS3() { return s3; } @@ -786,30 +487,6 @@ public void setS3(S3 s3) { } } - public static class Atlas { - - @JsonProperty("URL") - private String url; - - private OIDCConfiguration oidcConfiguration = null; - - public String getUrl() { - return url; - } - - public void setUrl(String url) { - this.url = url; - } - - public OIDCConfiguration getOidcConfiguration() { - return oidcConfiguration; - } - - public void setOidcConfiguration(OIDCConfiguration oidcConfiguration) { - this.oidcConfiguration = oidcConfiguration; - } - } - @Schema(description = "Vault Configuration") public static class Vault { @@ -942,51 +619,6 @@ public void setOidcConfiguration(OIDCConfiguration oidcConfiguration) { } } - public static class CustomInitScript { - - @JsonProperty("URL") - private String url; - - private String checksum; - - public String getUrl() { - return url; - } - - public void setUrl(String url) { - this.url = url; - } - - public String getChecksum() { - return checksum; - } - - public void setChecksum(String checksum) { - this.checksum = checksum; - } - } - - public static class OpenshiftSCC { - private boolean enabled; - private String scc; - - public boolean isEnabled() { - return enabled; - } - - public void setEnabled(boolean enabled) { - this.enabled = enabled; - } - - public String getScc() { - return scc; - } - - public void setScc(String scc) { - this.scc = scc; - } - } - @Schema(description = "Configuration to be used by the S3 client associated to Onyxia") public static class S3 { From c5af6714740ca6f06d4ebae337b7aa329d253b26 Mon Sep 17 00:00:00 2001 From: fcomte Date: Sat, 3 Aug 2024 13:31:28 +0000 Subject: [PATCH 48/53] docs --- docs/region-configuration.md | 96 +----------------------------------- 1 file changed, 2 insertions(+), 94 deletions(-) diff --git a/docs/region-configuration.md b/docs/region-configuration.md index d6234116..4cd4f549 100644 --- a/docs/region-configuration.md +++ b/docs/region-configuration.md @@ -11,20 +11,14 @@ See [regions.json](/onyxia-api/src/main/resources/regions.json) for a complete e - [Region configuration](#region-configuration) - [Main region properties](#main-region-properties) - [Services properties](#services-properties) - - [CustomInitScript properties](#custominitscript-properties) - [Server properties](#server-properties) - [K8sPublicEndpoint properties](#k8spublicendpoint-properties) - [Quotas properties](#quotas-properties) - [Expose properties](#expose-properties) - [istio](#istio) - [CertManager](#certManager) - - [Default configuration properties](#default-configuration-properties) - - [Kafka](#kafka) - - [Sliders](#sliders) - - [Resources](#resources) - [Data properties](#data-properties) - [S3](#s3) - - [Atlas](#atlas) - [Vault properties](#vault-properties) - [Git properties](#git-properties) - [ProxyConfiguration properties](#proxyconfiguration-properties) @@ -70,34 +64,14 @@ Users can work on Onyxia as a User or as a Group to which they belong. Each user | `groupPrefix` | | not used | | | `authenticationMode` | serviceAccount | serviceAccount, impersonate or tokenPassthrough : on serviceAccount mode Onyxia API uses its own serviceAccount (by default admin or cluster-admin), with impersonate mode Onyxia requests the API with user's permissions (helm option `--kube-as-user`). With tokenPassthrough, the authentication token is passed to the API server. | | | `expose` | | When users request to expose their service, only subdomain of this object domain are allowed | See [Expose properties](#expose-properties) | -| `monitoring` | | Define the URL pattern of the monitoring service that is to be launched with each service. Only for client purposes. | {URLPattern: "https://$NAMESPACE-$INSTANCE.mymonitoring.sspcloud.fr"} | -| `initScript` | | Define where to fetch a script that will be launched on some service on startup. | "https://inseefrlab.github.io/onyxia/onyxia-init.sh" | +| `monitoring` | | Define the URL pattern of the monitoring service that is to be launched with each service. Only for client purposes. | {URLPattern: "https://$NAMESPACE-$INSTANCE.mymonitoring.sspcloud.fr"} | | | `allowedURIPattern` | "^https://" | Init scripts set by the user have to respect this pattern. | | | `server` | | Define the configuration of the services provider API server, this value is not served on the API as it contains credentials for the API. | See [Server properties](#server-properties) | | `k8sPublicEndpoint` | | Define external access to Kubernetes API if available. It helps Onyxia users to directly connect to Kubernetes outside the datalab | See [K8sPublicEndpoint properties](#k8sPublicEndpoint-properties) | | `quotas` | | Properties setting quotas on how many resources a user can get on the services provider. | See [Quotas properties](#quotas-properties) | -| `defaultConfiguration` | | Default configuration on services that a user can override. For client purposes only. | See [Default Configuration](#default-configuration-properties) | -| `customInitScript` | | This can be used to customize user environments using a regional script executed by some users' pods. | See [CustomInitScript properties](#custominitscript-properties) -| `openshiftSCC` | | This can be used to inject SCC (Security Context Constraints) in openshift clusters | See [OpenshiftSCC properties](#openshiftSCC-properties) | -| `customValues` | | This can be used to specify custom values that will be available for helm chart injection in the web app. Nested values are supported. | ` "customValues": {"myCustomKey": "myValue", "myNestedCustomKey": {"nestedKey": "nestedValue"} }` | -### CustomInitScript properties -These properties define how to reach the **service provider API**. - -| Key | Description | Example | -| --------------------- | ------------------------------------------------------------------ | ---- | -| `URL` | URL of the init script | "api.kub.sspcloud.fr" | -| `checksum` | checksum of the init script | | - -### OpenshiftSCC properties -These properties define if SCC should be injected in services for openshift clusters - -| Key | Description | Example | -| --------------------- | ------------------------------------------------------------------ | ---- | -| `enabled` | defaults to `false` | `true` | -| `scc` | name of the SCC | `anyuid` | ### Server properties @@ -168,62 +142,6 @@ A quota follows the Kubernetes model which is composed of: -### Default configuration properties - -| Key | Default | Description | -| --------------------- | ------- | ------------------------------------------------------------------ | -| `IPProtection` | false | Whether or not the default behavior of the reverse proxy serving the service is to block a request from an IP other than the one from which it has been created. For client purposes only. | -| `networkPolicy` | false | Whether or not services can be reached by pods outside of the current namespace. For client purposes only. | -| `from` | NA | List of allowed sources (Kubernetes network policies format for from) to reach user HTTP services. Used to allow ingress access to users' services | -| `nodeSelector` | NA | This node selector can be injected in a service to restrain on which node it can be launched | -| `tolerations` | NA | This node selector can be injected in a service to force it to run on nodes with this taint | -| `startupProbe` | NA | This startup probe can be injected into a service. It can help you in an environment with a slow network to specify a long duration before killing a container | -| `kafka` | | See [Kafka](#kafka) | -| `sliders` | | See [Sliders](#sliders) | -| `Resources` | | See [Resources](#resources) | - -#### Kafka - -Kafka can be used to get some events in the user chart like Hive metastore. - -| Key | Default | Description | -| --------------------- | ------- | ------------------------------------------------------------------ | -| `URL` | N.A | brokerURL | -| `topicName` | N.A | topic name for those events | - -#### Sliders - -Sliders specify some slider parameters that may overwrite some defaults. - -| Key | Default | Description | -| --------------------- | ------- | ------------------------------------------------------------------ | -| `cpu` | N.A | cpu slider parameters | -| `memory` | N.A | memory slider parameters | -| `gpu` | N.A | gpu slider parameters | -| `disk` | N.A | disk slider parameters | - - -| Key | Default | Description | -| --------------------- | ------- | ------------------------------------------------------------------ | -| `sliderMin` | N.A | sliderMin | -| `sliderMax` | N.A | sliderMax | -| `sliderStep` | N.A | sliderStep | -| `sliderUnit` | N.A | sliderUnit | - -#### Resources - -Resources specify some values that may overwrite some defaults. - -| Key | Default | Description | -| --------------------- | ------- | ------------------------------------------------------------------ | -| `cpuRequest` | N.A | overwrite default CPU request if asked by helm-charts | -| `cpuLimit` | N.A | overwrite default CPU limit if asked by helm-charts | -| `memoryRequest` | N.A | overwrite default memory request if asked by helm-charts | -| `memoryLimit` | N.A | overwrite default memory limit if asked by helm-charts | -| `disk` | N.A | overwrite default disk size if asked by helm-charts | -| `gpu` | N.A | overwrite default GPU if asked by helm-charts | - - ## Data properties ### S3 @@ -367,17 +285,7 @@ type Region = { }; }; ``` - -### Atlas - -Atlas is a data management tool. - -It can be used to add additional features to the file explorer to transform it into a data explorer - -| Key | Default | Description | Example | -| --------------------- | ------- | ------------------------------------------------------------------ | ---- | -| `URL` | | URL of the atlas service for the region. | "https://atlas.change.me" | -| `oidcConfiguration` | | Allow override of openidconnect authentication for this specific service. If not defined then global Onyxia authentication will be used. | {clientID: "onyxia", issuerURI: "https://auth.lab.sspcloud.fr/auth"} | +| ## Vault properties From 81bc2090901d1555db789867ca6e5d498e4b9f97 Mon Sep 17 00:00:00 2001 From: ihiverlet <94982379+ihiverlet@users.noreply.github.com> Date: Tue, 20 Aug 2024 15:05:27 +0200 Subject: [PATCH 49/53] Update message.json --- .../main/resources/schemas/ide/message.json | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/onyxia-api/src/main/resources/schemas/ide/message.json b/onyxia-api/src/main/resources/schemas/ide/message.json index 5cb02c54..63583c08 100644 --- a/onyxia-api/src/main/resources/schemas/ide/message.json +++ b/onyxia-api/src/main/resources/schemas/ide/message.json @@ -1,7 +1,21 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "title": "message", - "type": "string", + "type": "object", "description": "add a message in the notes", - "default": "" -} \ No newline at end of file + "x-onyxia":{ + "hidden": true + }, + "properties": { + "fr":{ + "type": string, + "description":"message à ajouter dans les notes, + "default": "NB: ce service pourrait être supprimé après 7 jours d'utilisation en raison de nos règles de gestion" + }, + "en":{ + "type": string, + "description: "message to add in notes", + "default": "Warning: this service may be deleted after 7 days due to our management policies", + } + } +} From a2fb7d7e4bdcf8aa638cad4de207a3c068c02d1d Mon Sep 17 00:00:00 2001 From: ihiverlet <94982379+ihiverlet@users.noreply.github.com> Date: Tue, 20 Aug 2024 15:06:06 +0200 Subject: [PATCH 50/53] Update message.json --- onyxia-api/src/main/resources/schemas/ide/message.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/onyxia-api/src/main/resources/schemas/ide/message.json b/onyxia-api/src/main/resources/schemas/ide/message.json index 63583c08..20e9c5e4 100644 --- a/onyxia-api/src/main/resources/schemas/ide/message.json +++ b/onyxia-api/src/main/resources/schemas/ide/message.json @@ -9,12 +9,12 @@ "properties": { "fr":{ "type": string, - "description":"message à ajouter dans les notes, + "description":"message à ajouter dans les notes", "default": "NB: ce service pourrait être supprimé après 7 jours d'utilisation en raison de nos règles de gestion" }, "en":{ "type": string, - "description: "message to add in notes", + "description": "message to add in notes", "default": "Warning: this service may be deleted after 7 days due to our management policies", } } From 6f2ec22f796563510429546d2a50c289f9af3e70 Mon Sep 17 00:00:00 2001 From: ihiverlet <94982379+ihiverlet@users.noreply.github.com> Date: Tue, 20 Aug 2024 15:06:41 +0200 Subject: [PATCH 51/53] Update message.json --- onyxia-api/src/main/resources/schemas/ide/message.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/onyxia-api/src/main/resources/schemas/ide/message.json b/onyxia-api/src/main/resources/schemas/ide/message.json index 20e9c5e4..c32fa9b3 100644 --- a/onyxia-api/src/main/resources/schemas/ide/message.json +++ b/onyxia-api/src/main/resources/schemas/ide/message.json @@ -8,12 +8,12 @@ }, "properties": { "fr":{ - "type": string, + "type": "string", "description":"message à ajouter dans les notes", "default": "NB: ce service pourrait être supprimé après 7 jours d'utilisation en raison de nos règles de gestion" }, "en":{ - "type": string, + "type": "string", "description": "message to add in notes", "default": "Warning: this service may be deleted after 7 days due to our management policies", } From d6544d96bd3bdfbff7c15f6d3ebc68bcda6f4f87 Mon Sep 17 00:00:00 2001 From: ihiverlet <94982379+ihiverlet@users.noreply.github.com> Date: Tue, 20 Aug 2024 15:13:15 +0200 Subject: [PATCH 52/53] Update message.json --- onyxia-api/src/main/resources/schemas/ide/message.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/onyxia-api/src/main/resources/schemas/ide/message.json b/onyxia-api/src/main/resources/schemas/ide/message.json index c32fa9b3..075342ae 100644 --- a/onyxia-api/src/main/resources/schemas/ide/message.json +++ b/onyxia-api/src/main/resources/schemas/ide/message.json @@ -10,12 +10,12 @@ "fr":{ "type": "string", "description":"message à ajouter dans les notes", - "default": "NB: ce service pourrait être supprimé après 7 jours d'utilisation en raison de nos règles de gestion" + "default": "" }, "en":{ "type": "string", "description": "message to add in notes", - "default": "Warning: this service may be deleted after 7 days due to our management policies", + "default": "", } } } From 4dd5f6105298413886d6a8397ad5a9f38a92399c Mon Sep 17 00:00:00 2001 From: ihiverlet <94982379+ihiverlet@users.noreply.github.com> Date: Tue, 20 Aug 2024 15:28:44 +0200 Subject: [PATCH 53/53] Update message.json --- onyxia-api/src/main/resources/schemas/ide/message.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onyxia-api/src/main/resources/schemas/ide/message.json b/onyxia-api/src/main/resources/schemas/ide/message.json index 075342ae..e7c91d13 100644 --- a/onyxia-api/src/main/resources/schemas/ide/message.json +++ b/onyxia-api/src/main/resources/schemas/ide/message.json @@ -15,7 +15,7 @@ "en":{ "type": "string", "description": "message to add in notes", - "default": "", + "default": "" } } }