Skip to content

Commit

Permalink
feat: add ephemeral volume size parameter
Browse files Browse the repository at this point in the history
  • Loading branch information
@lmanelphe
lmanelphe committed Sep 24, 2024
1 parent 661d2ea commit 405e873
Show file tree
Hide file tree
Showing 14 changed files with 110 additions and 89 deletions.
1 change: 1 addition & 0 deletions arc-batch/src/main/resources/applicationContext.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@
<property name="kubernetesExecutorCpu" value="#{systemEnvironment['KUBERNETES_EXECUTOR_CPU'] ?: '${fr.insee.arc.kubernetes.executor.cpu}' }"></property>
<property name="kubernetesExecutorRam" value="#{systemEnvironment['KUBERNETES_EXECUTOR_RAM'] ?: '${fr.insee.arc.kubernetes.executor.ram}' }"></property>
<property name="kubernetesExecutorEphemeral" value="#{systemEnvironment['KUBERNETES_EXECUTOR_EPHEMERAL'] ?: '${fr.insee.arc.kubernetes.executor.ephemeral}' }"></property>
<property name="kubernetesExecutorEphemeralVolumeSize" value="#{systemEnvironment['KUBERNETES_EXECUTOR_EPHEMERAL_VOLUME_SIZE'] ?: '${fr.insee.arc.kubernetes.executor.ephemeral.volumeSize}' }"></property>
<property name="kubernetesExecutorVolatile" value="#{systemEnvironment['KUBERNETES_EXECUTOR_VOLATILE'] ?: '${fr.insee.arc.kubernetes.executor.volatile}' }"></property>

<property name="processExport" value="#{systemEnvironment['PROCESS_EXPORT'] ?: '${fr.insee.arc.process.export}' }"></property>
Expand Down
1 change: 1 addition & 0 deletions arc-batch/src/main/resources/fr/insee/config/arc.properties
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ fr.insee.arc.kubernetes.executor.port=${env.kubernetesExecutorPort:}
fr.insee.arc.kubernetes.executor.cpu=${env.kubernetesExecutorCpu:}
fr.insee.arc.kubernetes.executor.ram=${env.kubernetesExecutorRam:}
fr.insee.arc.kubernetes.executor.ephemeral=${env.kubernetesExecutorEphemeral:}
fr.insee.arc.kubernetes.executor.ephemeral.volumeSize=${env.kubernetesExecutorEphemeralVolumeSize:}
fr.insee.arc.kubernetes.executor.volatile=${env.kubernetesExecutorVolatile:}

# Properties to activate export phase
Expand Down
1 change: 1 addition & 0 deletions arc-core/src/main/java/fr/insee/arc/core/service/kubernetes/bo/JsonFileParameter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ private JsonFileParameter() {
public static final String CPU = "{cpu}";
public static final String RAM = "{ram}";
public static final String EPHEMERAL = "{ephemeral}";
public static final String EPHEMERAL_VOLUME_SIZE = "{generic_ephemeral_volume_size}";


}
1 change: 1 addition & 0 deletions .../main/java/fr/insee/arc/core/service/kubernetes/configuration/BuildJsonConfiguration.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,7 @@ private static String[] kubernetesParameters(int executorReplicaIndex)
, JsonFileParameter.CPU, properties.getKubernetesExecutorCpu()
, JsonFileParameter.RAM, properties.getKubernetesExecutorRam()
, JsonFileParameter.EPHEMERAL, properties.getKubernetesExecutorEphemeral()
, JsonFileParameter.EPHEMERAL_VOLUME_SIZE, properties.getKubernetesExecutorEphemeralVolumeSize()
};
}

Expand Down
9 changes: 9 additions & 0 deletions arc-utils/src/main/java/fr/insee/arc/utils/ressourceUtils/PropertiesHandler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ public class PropertiesHandler {
private String kubernetesExecutorCpu;
private String kubernetesExecutorRam;
private String kubernetesExecutorEphemeral;
private String kubernetesExecutorEphemeralVolumeSize;
private String kubernetesExecutorVolatile;

/* export data end of batch ? */
Expand Down Expand Up @@ -419,6 +420,14 @@ public void setKubernetesExecutorEphemeral(String kubernetesExecutorEphemeral) {
this.kubernetesExecutorEphemeral = kubernetesExecutorEphemeral;
}

public String getKubernetesExecutorEphemeralVolumeSize() {
return kubernetesExecutorEphemeralVolumeSize;
}

public void setKubernetesExecutorEphemeralVolumeSize(String kubernetesExecutorEphemeralVolumeSize) {
this.kubernetesExecutorEphemeralVolumeSize = kubernetesExecutorEphemeralVolumeSize;
}

public String getKubernetesExecutorVolatile() {
return kubernetesExecutorVolatile;
}
Expand Down
1 change: 1 addition & 0 deletions arc-web/src/main/resources/applicationContext.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@
<property name="kubernetesExecutorCpu" value="#{systemEnvironment['KUBERNETES_EXECUTOR_CPU'] ?: '${fr.insee.arc.kubernetes.executor.cpu}' }"></property>
<property name="kubernetesExecutorRam" value="#{systemEnvironment['KUBERNETES_EXECUTOR_RAM'] ?: '${fr.insee.arc.kubernetes.executor.ram}' }"></property>
<property name="kubernetesExecutorEphemeral" value="#{systemEnvironment['KUBERNETES_EXECUTOR_EPHEMERAL'] ?: '${fr.insee.arc.kubernetes.executor.ephemeral}' }"></property>
<property name="kubernetesExecutorEphemeralVolumeSize" value="#{systemEnvironment['KUBERNETES_EXECUTOR_EPHEMERAL_VOLUME_SIZE'] ?: '${fr.insee.arc.kubernetes.executor.ephemeral.volumeSize}' }"></property>
<property name="kubernetesExecutorVolatile" value="#{systemEnvironment['KUBERNETES_EXECUTOR_VOLATILE'] ?: '${fr.insee.arc.kubernetes.executor.volatile}' }"></property>
<property name="processExport" value="#{systemEnvironment['PROCESS_EXPORT'] ?: '${fr.insee.arc.process.export}' }"></property>

Expand Down
1 change: 1 addition & 0 deletions arc-web/src/main/resources/fr/insee/config/arc.properties
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ fr.insee.arc.kubernetes.executor.port=${env.kubernetesExecutorPort:}
fr.insee.arc.kubernetes.executor.cpu=${env.kubernetesExecutorCpu:}
fr.insee.arc.kubernetes.executor.ram=${env.kubernetesExecutorRam:}
fr.insee.arc.kubernetes.executor.ephemeral=${env.kubernetesExecutorEphemeral:}
fr.insee.arc.kubernetes.executor.ephemeral.volumeSize=${env.kubernetesExecutorEphemeralVolumeSize:}
fr.insee.arc.kubernetes.executor.volatile=${env.kubernetesExecutorVolatile:}

# Properties to activate export phase
Expand Down
1 change: 1 addition & 0 deletions arc-ws/src/main/resources/fr/insee/config/arc.properties
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ fr.insee.arc.kubernetes.executor.port=${env.kubernetesExecutorPort:}
fr.insee.arc.kubernetes.executor.cpu=${env.kubernetesExecutorCpu:}
fr.insee.arc.kubernetes.executor.ram=${env.kubernetesExecutorRam:}
fr.insee.arc.kubernetes.executor.ephemeral=${env.kubernetesExecutorEphemeral:}
fr.insee.arc.kubernetes.executor.ephemeral.volumeSize=${env.kubernetesExecutorEphemeralVolumeSize:}
fr.insee.arc.kubernetes.executor.volatile=${env.kubernetesExecutorVolatile:}

# Properties to activate export phase
Expand Down
1 change: 1 addition & 0 deletions arc-ws/src/main/webapp/WEB-INF/applicationContext.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@
<property name="kubernetesExecutorCpu" value="#{systemEnvironment['KUBERNETES_EXECUTOR_CPU'] ?: '${fr.insee.arc.kubernetes.executor.cpu}' }"></property>
<property name="kubernetesExecutorRam" value="#{systemEnvironment['KUBERNETES_EXECUTOR_RAM'] ?: '${fr.insee.arc.kubernetes.executor.ram}' }"></property>
<property name="kubernetesExecutorEphemeral" value="#{systemEnvironment['KUBERNETES_EXECUTOR_EPHEMERAL'] ?: '${fr.insee.arc.kubernetes.executor.ephemeral}' }"></property>
<property name="kubernetesExecutorEphemeralVolumeSize" value="#{systemEnvironment['KUBERNETES_EXECUTOR_EPHEMERAL_VOLUME_SIZE'] ?: '${fr.insee.arc.kubernetes.executor.ephemeral.volumeSize}' }"></property>
<property name="kubernetesExecutorVolatile" value="#{systemEnvironment['KUBERNETES_EXECUTOR_VOLATILE'] ?: '${fr.insee.arc.kubernetes.executor.volatile}' }"></property>

<property name="processExport" value="#{systemEnvironment['PROCESS_EXPORT'] ?: '${fr.insee.arc.process.export}' }"></property>
Expand Down
1 change: 1 addition & 0 deletions docker/app-batch.Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ ARG KUBERNETES_EXECUTOR_PORT
ARG KUBERNETES_EXECUTOR_CPU
ARG KUBERNETES_EXECUTOR_RAM
ARG KUBERNETES_EXECUTOR_EPHEMERAL
ARG KUBERNETES_EXECUTOR_EPHEMERAL_VOLUME_SIZE
ARG KUBERNETES_EXECUTOR_VOLATILE
ARG S3_INPUT_API_URI
ARG S3_INPUT_BUCKET
Expand Down
1 change: 1 addition & 0 deletions docker/app-web.Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ ARG KUBERNETES_EXECUTOR_PORT
ARG KUBERNETES_EXECUTOR_CPU
ARG KUBERNETES_EXECUTOR_RAM
ARG KUBERNETES_EXECUTOR_EPHEMERAL
ARG KUBERNETES_EXECUTOR_EPHEMERAL_VOLUME_SIZE
ARG KUBERNETES_EXECUTOR_VOLATILE
ARG S3_INPUT_API_URI
ARG S3_INPUT_BUCKET
Expand Down
1 change: 1 addition & 0 deletions docker/app-ws.Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ ARG KUBERNETES_EXECUTOR_PORT
ARG KUBERNETES_EXECUTOR_CPU
ARG KUBERNETES_EXECUTOR_RAM
ARG KUBERNETES_EXECUTOR_EPHEMERAL
ARG KUBERNETES_EXECUTOR_EPHEMERAL_VOLUME_SIZE
ARG KUBERNETES_EXECUTOR_VOLATILE
ARG PROCESS_EXPORT
ARG S3_INPUT_API_URI
Expand Down
2 changes: 1 addition & 1 deletion docker/script.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,6 @@ echo "maven settings.xml set to $MAVEN_SETTINGS";
MAVEN_CONF="-s usr/src/app/$MAVEN_SETTINGS";
fi

mvn -f /usr/src/app/pom.xml clean package -DskipTests $MAVEN_CONF -Pdocker -Denv.logSettings=$LOG_SETTINGS -Denv.urlDatabase=$DATABASE_URL -Denv.usernameDatabase=$DATABASE_USER -Denv.passwordDatabase=$DATABASE_PASSWORD -Denv.restrictedUserDatabase=$DATABASE_RESTRICTED_USER -Denv.applicationDirectory=$APPLICATION_DIRECTORY -Denv.disableDebugGui=$DISABLE_DEBUG_GUI -Denv.kubernetesApiUri=$KUBERNETES_API_URI -Denv.kubernetesApiNamespace=KUBERNETES_API_NAMESPACE -Denv.kubernetesApiTokenPath=$KUBERNETES_API_TOKEN_PATH -Denv.kubernetesApiTokenValue=$KUBERNETES_API_TOKEN_VALUE -Denv.kubernetesExecutorImage=$KUBERNETES_EXECUTOR_IMAGE -Denv.kubernetesExecutorNumber=$KUBERNETES_EXECUTOR_NUMBER -Denv.kubernetesExecutorLabel=$KUBERNETES_EXECUTOR_LABEL -Denv.kubernetesExecutorUser=$KUBERNETES_EXECUTOR_USER -Denv.kubernetesExecutorDatabase=$KUBERNETES_EXECUTOR_DATABASE -Denv.kubernetesExecutorPort=$KUBERNETES_EXECUTOR_PORT -Denv.kubernetesExecutorCpu=$KUBERNETES_EXECUTOR_CPU -Denv.kubernetesExecutorRam=$KUBERNETES_EXECUTOR_RAM -Denv.kubernetesExecutorEphemeral=$KUBERNETES_EXECUTOR_EPHEMERAL -Denv.kubernetesExecutorVolatile=$KUBERNETES_EXECUTOR_VOLATILE -Denv.processExport=$PROCESS_EXPORT -Denv.s3InputApiUri=$S3_INPUT_API_URI -Denv.s3InputBucket=$S3_INPUT_BUCKET -Denv.s3InputAccess=$S3_INPUT_ACCESS -Denv.s3InputSecret=$S3_INPUT_SECRET -Denv.s3OutputApiUri=$S3_OUTPUT_API_URI -Denv.s3OutputBucket=$S3_OUTPUT_BUCKET -Denv.s3OutputAccess=$S3_OUTPUT_ACCESS -Denv.s3OutputSecret=$S3_OUTPUT_SECRET -Denv.s3OutputParquetKey=$S3_OUTPUT_PARQUET_KEY -Denv.envExecution=$ENV_EXECUTION -Dfr.insee.arc.roles.admin=$KEYCLOAK_AUTHORIZED_ROLES -Dfr.insee.keycloak.realm=$KEYCLOAK_REALM -Dfr.insee.keycloak.server=$KEYCLOAK_SERVER -Dfr.insee.keycloak.resource=$KEYCLOAK_RESOURCE -Dfr.insee.keycloak.credentials.secret=$KEYCLOAK_CREDENTIALS -Dfr.insee.arc.files.retention.days=FILES_RETENTION_DAYS;
mvn -f /usr/src/app/pom.xml clean package -DskipTests $MAVEN_CONF -Pdocker -Denv.logSettings=$LOG_SETTINGS -Denv.urlDatabase=$DATABASE_URL -Denv.usernameDatabase=$DATABASE_USER -Denv.passwordDatabase=$DATABASE_PASSWORD -Denv.restrictedUserDatabase=$DATABASE_RESTRICTED_USER -Denv.applicationDirectory=$APPLICATION_DIRECTORY -Denv.disableDebugGui=$DISABLE_DEBUG_GUI -Denv.kubernetesApiUri=$KUBERNETES_API_URI -Denv.kubernetesApiNamespace=KUBERNETES_API_NAMESPACE -Denv.kubernetesApiTokenPath=$KUBERNETES_API_TOKEN_PATH -Denv.kubernetesApiTokenValue=$KUBERNETES_API_TOKEN_VALUE -Denv.kubernetesExecutorImage=$KUBERNETES_EXECUTOR_IMAGE -Denv.kubernetesExecutorNumber=$KUBERNETES_EXECUTOR_NUMBER -Denv.kubernetesExecutorLabel=$KUBERNETES_EXECUTOR_LABEL -Denv.kubernetesExecutorUser=$KUBERNETES_EXECUTOR_USER -Denv.kubernetesExecutorDatabase=$KUBERNETES_EXECUTOR_DATABASE -Denv.kubernetesExecutorPort=$KUBERNETES_EXECUTOR_PORT -Denv.kubernetesExecutorCpu=$KUBERNETES_EXECUTOR_CPU -Denv.kubernetesExecutorRam=$KUBERNETES_EXECUTOR_RAM -Denv.kubernetesExecutorEphemeral=$KUBERNETES_EXECUTOR_EPHEMERAL -Denv.kubernetesExecutorEphemeralVolumeSize=$KUBERNETES_EXECUTOR_EPHEMERAL_VOLUME_SIZE -Denv.kubernetesExecutorVolatile=$KUBERNETES_EXECUTOR_VOLATILE -Denv.processExport=$PROCESS_EXPORT -Denv.s3InputApiUri=$S3_INPUT_API_URI -Denv.s3InputBucket=$S3_INPUT_BUCKET -Denv.s3InputAccess=$S3_INPUT_ACCESS -Denv.s3InputSecret=$S3_INPUT_SECRET -Denv.s3OutputApiUri=$S3_OUTPUT_API_URI -Denv.s3OutputBucket=$S3_OUTPUT_BUCKET -Denv.s3OutputAccess=$S3_OUTPUT_ACCESS -Denv.s3OutputSecret=$S3_OUTPUT_SECRET -Denv.s3OutputParquetKey=$S3_OUTPUT_PARQUET_KEY -Denv.envExecution=$ENV_EXECUTION -Dfr.insee.arc.roles.admin=$KEYCLOAK_AUTHORIZED_ROLES -Dfr.insee.keycloak.realm=$KEYCLOAK_REALM -Dfr.insee.keycloak.server=$KEYCLOAK_SERVER -Dfr.insee.keycloak.resource=$KEYCLOAK_RESOURCE -Dfr.insee.keycloak.credentials.secret=$KEYCLOAK_CREDENTIALS -Dfr.insee.arc.files.retention.days=FILES_RETENTION_DAYS;


Loading

0 comments on commit 405e873

Please sign in to comment.