From 326ddb6c920ede3be775b86b2dc040d3ed885879 Mon Sep 17 00:00:00 2001 From: Karan Date: Mon, 16 Apr 2018 21:41:26 -0700 Subject: [PATCH] [INFINITY - 2985] backport isolation to sdk 0.40 (#2478) * isolation backport * using relative hostpath for isolation (#2476) * fixing compatibility with 1.9 * latest changes * setting executor info * fixing isolation compatibility with 1.9 * [INFINITY-2985] Isolation support for elastic search (#2481) [INFINITY-2985] Isolation support for elastic search * fixing style warnings --- frameworks/elastic/src/main/dist/svc.yml | 4 ++ .../helloworld/src/main/dist/isolation.yml | 16 +++++++ frameworks/helloworld/tests/test_isolation.py | 46 +++++++++++++++++++ frameworks/helloworld/universe/config.json | 9 +++- .../universe/marathon.json.mustache | 1 + .../sdk/offer/evaluate/EvaluationOutcome.java | 1 - .../sdk/offer/evaluate/PodInfoBuilder.java | 20 +++++++- .../sdk/specification/DefaultPodSpec.java | 27 ++++++++++- .../mesosphere/sdk/specification/PodSpec.java | 3 ++ .../sdk/specification/yaml/RawPod.java | 8 +++- .../yaml/YAMLToInternalMappers.java | 3 +- .../sdk/specification/DefaultPodSpecTest.java | 1 + 12 files changed, 132 insertions(+), 7 deletions(-) create mode 100644 frameworks/helloworld/src/main/dist/isolation.yml create mode 100644 frameworks/helloworld/tests/test_isolation.py diff --git a/frameworks/elastic/src/main/dist/svc.yml b/frameworks/elastic/src/main/dist/svc.yml index 8afc793219f..cc5590cc901 100644 --- a/frameworks/elastic/src/main/dist/svc.yml +++ b/frameworks/elastic/src/main/dist/svc.yml @@ -4,6 +4,7 @@ scheduler: user: {{FRAMEWORK_USER}} pods: master: + isolate-tmp: true count: 3 {{#ENABLE_VIRTUAL_NETWORK}} networks: @@ -73,6 +74,7 @@ pods: type: TLS {{/TASKCFG_ALL_SECURITY_TRANSPORT_ENCRYPTION_ENABLED}} data: + isolate-tmp: true count: {{DATA_NODE_COUNT}} {{#ENABLE_VIRTUAL_NETWORK}} networks: @@ -140,6 +142,7 @@ pods: type: TLS {{/TASKCFG_ALL_SECURITY_TRANSPORT_ENCRYPTION_ENABLED}} ingest: + isolate-tmp: true count: {{INGEST_NODE_COUNT}} {{#ENABLE_VIRTUAL_NETWORK}} networks: @@ -207,6 +210,7 @@ pods: type: TLS {{/TASKCFG_ALL_SECURITY_TRANSPORT_ENCRYPTION_ENABLED}} coordinator: + isolate-tmp: true count: {{COORDINATOR_NODE_COUNT}} {{#ENABLE_VIRTUAL_NETWORK}} networks: diff --git a/frameworks/helloworld/src/main/dist/isolation.yml b/frameworks/helloworld/src/main/dist/isolation.yml new file mode 100644 index 00000000000..9d77d50234d --- /dev/null +++ b/frameworks/helloworld/src/main/dist/isolation.yml @@ -0,0 +1,16 @@ +name: {{FRAMEWORK_NAME}} +scheduler: + principal: {{SERVICE_PRINCIPAL}} + user: {{SERVICE_USER}} +pods: + hello: + count: {{HELLO_COUNT}} + isolate-tmp: {{HELLO_ISOLATION}} + tasks: + server: + goal: FINISHED + cmd: echo foo > tmp/foo && echo bar > /tmp/bar && cat tmp/bar | grep bar + cpus: {{HELLO_CPUS}} + memory: {{HELLO_MEM}} + + diff --git a/frameworks/helloworld/tests/test_isolation.py b/frameworks/helloworld/tests/test_isolation.py new file mode 100644 index 00000000000..357ffcf3fc1 --- /dev/null +++ b/frameworks/helloworld/tests/test_isolation.py @@ -0,0 +1,46 @@ +import logging +import re + +import dcos.marathon +import pytest +import sdk_cmd +import sdk_install +import sdk_marathon +import sdk_plan +import sdk_tasks +import sdk_utils +import shakedown +from tests import config + + +log = logging.getLogger(__name__) + + +@pytest.fixture(scope='module', autouse=True) +def configure_package(configure_security): + try: + sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME) + yield + finally: + sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME) + + +@pytest.mark.sanity +def test_tmp_directory_created(): + + sdk_install.install( + config.PACKAGE_NAME, + config.SERVICE_NAME, + 0, + additional_options={"service": {"name": config.SERVICE_NAME, "yaml": "isolation"}}, + wait_for_deployment=False) + + pl = sdk_plan.get_deployment_plan(config.SERVICE_NAME) + + assert pl['status'] != 'COMPLETE' + + marathon_config = sdk_marathon.get_config(config.SERVICE_NAME) + marathon_config['env']['HELLO_ISOLATION'] = 'true' + sdk_marathon.update_app(config.SERVICE_NAME, marathon_config) + + sdk_plan.wait_for_completed_deployment(config.SERVICE_NAME) diff --git a/frameworks/helloworld/universe/config.json b/frameworks/helloworld/universe/config.json index 64858fe2faa..f9d8fe41547 100644 --- a/frameworks/helloworld/universe/config.json +++ b/frameworks/helloworld/universe/config.json @@ -108,7 +108,9 @@ "svc", "tls", "uri", - "web-url" + "web-url", + "isolation", + "" ], "default": "svc" }, @@ -175,6 +177,11 @@ "description": "The number of seconds of grace to await a clean shutdown following SIGTERM before sending SIGKILL, default: `0`", "type": "integer", "default": 0 + }, + "isolation": { + "description": "boolean flag to control isolation", + "type": "boolean", + "default": false } }, "required": [ diff --git a/frameworks/helloworld/universe/marathon.json.mustache b/frameworks/helloworld/universe/marathon.json.mustache index 9018e2ca0eb..31e71c061fb 100644 --- a/frameworks/helloworld/universe/marathon.json.mustache +++ b/frameworks/helloworld/universe/marathon.json.mustache @@ -77,6 +77,7 @@ {{#world.secret3}} "WORLD_SECRET3" : "{{world.secret3}}", {{/world.secret3}} + "HELLO_ISOLATION": "{{hello.isolation}}", {{#tls.discovery_task_prefix}} "DISCOVERY_TASK_PREFIX": "{{tls.discovery_task_prefix}}", diff --git a/sdk/scheduler/src/main/java/com/mesosphere/sdk/offer/evaluate/EvaluationOutcome.java b/sdk/scheduler/src/main/java/com/mesosphere/sdk/offer/evaluate/EvaluationOutcome.java index 5a76f9c28d2..23d55c93bf9 100644 --- a/sdk/scheduler/src/main/java/com/mesosphere/sdk/offer/evaluate/EvaluationOutcome.java +++ b/sdk/scheduler/src/main/java/com/mesosphere/sdk/offer/evaluate/EvaluationOutcome.java @@ -26,7 +26,6 @@ private enum Type { private final Collection offerRecommendations; private final Collection children; private final String reason; - /** * Returns a new passing outcome object with the provided descriptive reason. * diff --git a/sdk/scheduler/src/main/java/com/mesosphere/sdk/offer/evaluate/PodInfoBuilder.java b/sdk/scheduler/src/main/java/com/mesosphere/sdk/offer/evaluate/PodInfoBuilder.java index 773949ec69e..282346bc336 100644 --- a/sdk/scheduler/src/main/java/com/mesosphere/sdk/offer/evaluate/PodInfoBuilder.java +++ b/sdk/scheduler/src/main/java/com/mesosphere/sdk/offer/evaluate/PodInfoBuilder.java @@ -294,6 +294,15 @@ private Protos.TaskInfo.Builder createTaskInfo( taskInfoBuilder.setContainer(Protos.ContainerInfo.newBuilder().setType(Protos.ContainerInfo.Type.MESOS)); } + if (podSpec.getIsolateTmp() && useDefaultExecutor) { + // Isolate the tmp directory of tasks + //switch to SANDBOX SELF after dc/os 1.13 + taskInfoBuilder.setContainer(taskInfoBuilder.getContainerBuilder().addVolumes(Protos.Volume.newBuilder() + .setContainerPath("/tmp") + .setHostPath("tmp") + .setMode(Protos.Volume.Mode.RW))); + } + setHealthCheck(taskInfoBuilder, serviceName, podInstance, taskSpec, override, schedulerConfig); setReadinessCheck(taskInfoBuilder, serviceName, podInstance, taskSpec, override, schedulerConfig); setTaskKillGracePeriod(taskInfoBuilder, taskSpec); @@ -370,6 +379,14 @@ private Protos.ExecutorInfo.Builder getExecutorInfoBuilder( // This includes networks, rlimits, secret volumes... executorInfoBuilder.setContainer(getContainerInfo(podSpec, true, false)); + if (podSpec.getIsolateTmp() && !useDefaultExecutor) { + executorInfoBuilder.setContainer(executorInfoBuilder.getContainerBuilder().addVolumes( + Protos.Volume.newBuilder() + .setContainerPath("/tmp") + .setHostPath("tmp") + .setMode(Protos.Volume.Mode.RW))); + } + return executorInfoBuilder; } @@ -606,7 +623,8 @@ private Protos.ContainerInfo getContainerInfo( if (!podSpec.getImage().isPresent() && podSpec.getNetworks().isEmpty() && podSpec.getRLimits().isEmpty() - && secretVolumes.isEmpty()) { + && secretVolumes.isEmpty() + && podSpec.getIsolateTmp() == false) { // Nothing left to do. return containerInfo.build(); } diff --git a/sdk/scheduler/src/main/java/com/mesosphere/sdk/specification/DefaultPodSpec.java b/sdk/scheduler/src/main/java/com/mesosphere/sdk/specification/DefaultPodSpec.java index 252dfb94ada..a0da6e11b32 100644 --- a/sdk/scheduler/src/main/java/com/mesosphere/sdk/specification/DefaultPodSpec.java +++ b/sdk/scheduler/src/main/java/com/mesosphere/sdk/specification/DefaultPodSpec.java @@ -52,6 +52,8 @@ public class DefaultPodSpec implements PodSpec { private String preReservedRole; @NotNull private Boolean sharePidNamespace; + @NotNull + private final Boolean isolateTmp; @JsonCreator public DefaultPodSpec( @@ -68,7 +70,8 @@ public DefaultPodSpec( @JsonProperty("pre-reserved-role") String preReservedRole, @JsonProperty("secrets") Collection secrets, @JsonProperty("share-pid-namespace") Boolean sharePidNamespace, - @JsonProperty("allow-decommission") Boolean allowDecommission) { + @JsonProperty("allow-decommission") Boolean allowDecommission, + @JsonProperty("isolate-tmp") Boolean isolateTmp) { this( new Builder(Optional.empty()) // Assume that Executor URI is already present .type(type) @@ -84,7 +87,8 @@ public DefaultPodSpec( .preReservedRole(preReservedRole) .secrets(secrets) .sharePidNamespace(sharePidNamespace) - .allowDecommission(allowDecommission)); + .allowDecommission(allowDecommission) + .isolateTmp(isolateTmp)); } private DefaultPodSpec(Builder builder) { @@ -102,6 +106,7 @@ private DefaultPodSpec(Builder builder) { this.user = builder.user; this.volumes = builder.volumes; this.sharePidNamespace = builder.sharePidNamespace; + this.isolateTmp = builder.isolateTmp; ValidationUtils.validate(this); } @@ -125,6 +130,7 @@ public static Builder newBuilder(PodSpec copy) { builder.user = copy.getUser().isPresent() ? copy.getUser().get() : null; builder.volumes = copy.getVolumes(); builder.sharePidNamespace = copy.getSharePidNamespace(); + builder.isolateTmp = copy.getIsolateTmp(); return builder; } @@ -198,6 +204,11 @@ public Boolean getSharePidNamespace() { return sharePidNamespace; } + @Override + public Boolean getIsolateTmp() { + return isolateTmp; + } + @Override public boolean equals(Object o) { return EqualsBuilder.reflectionEquals(this, o); @@ -233,6 +244,7 @@ public static final class Builder { private Collection volumes = new ArrayList<>(); private Collection secrets = new ArrayList<>(); private Boolean sharePidNamespace = false; + private Boolean isolateTmp = false; private Builder(Optional executorUri) { this.executorUri = executorUri; @@ -457,6 +469,17 @@ public Builder sharePidNamespace(Boolean sharePidNamespace) { return this; } + /** + * Sets whether tasks in this pod will have tmp directories isolated from the host. + * + * @param isolateTmp Whether the pod should isolate the tmp directories of tasks. + * @return a reference to this Builder + */ + public Builder isolateTmp(Boolean isolateTmp) { + this.isolateTmp = isolateTmp != null && isolateTmp; + return this; + } + /** * Returns a {@code DefaultPodSpec} built from the parameters previously set. * diff --git a/sdk/scheduler/src/main/java/com/mesosphere/sdk/specification/PodSpec.java b/sdk/scheduler/src/main/java/com/mesosphere/sdk/specification/PodSpec.java index 578592795cd..ba1f9747f80 100644 --- a/sdk/scheduler/src/main/java/com/mesosphere/sdk/specification/PodSpec.java +++ b/sdk/scheduler/src/main/java/com/mesosphere/sdk/specification/PodSpec.java @@ -57,6 +57,9 @@ public interface PodSpec { @JsonProperty("share-pid-namespace") Boolean getSharePidNamespace(); + @JsonProperty("isolate-tmp") + Boolean getIsolateTmp(); + @JsonIgnore static String getName(PodSpec podSpec, int index) { return podSpec.getType() + "-" + index; diff --git a/sdk/scheduler/src/main/java/com/mesosphere/sdk/specification/yaml/RawPod.java b/sdk/scheduler/src/main/java/com/mesosphere/sdk/specification/yaml/RawPod.java index 33b3f41833a..1f5cc6ca018 100644 --- a/sdk/scheduler/src/main/java/com/mesosphere/sdk/specification/yaml/RawPod.java +++ b/sdk/scheduler/src/main/java/com/mesosphere/sdk/specification/yaml/RawPod.java @@ -29,6 +29,7 @@ public class RawPod { private final WriteOnceLinkedHashMap secrets; private final Boolean sharePidNamespace; private final Boolean allowDecommission; + private final Boolean isolateTmp; private RawPod( @JsonProperty("resource-sets") WriteOnceLinkedHashMap resourceSets, @@ -44,7 +45,8 @@ private RawPod( @JsonProperty("pre-reserved-role") String preReservedRole, @JsonProperty("secrets") WriteOnceLinkedHashMap secrets, @JsonProperty("share-pid-namespace") Boolean sharePidNamespace, - @JsonProperty("allow-decommission") Boolean allowDecommission) { + @JsonProperty("allow-decommission") Boolean allowDecommission, + @JsonProperty("isolate-tmp") Boolean isolateTmp) { this.placement = placement; this.count = count; this.image = image; @@ -59,6 +61,7 @@ private RawPod( this.secrets = secrets == null ? new WriteOnceLinkedHashMap<>() : secrets; this.sharePidNamespace = sharePidNamespace != null && sharePidNamespace; this.allowDecommission = allowDecommission != null && allowDecommission; + this.isolateTmp = isolateTmp != null && isolateTmp; } public String getPlacement() { @@ -116,4 +119,7 @@ public Boolean getSharePidNamespace() { public Boolean getAllowDecommission() { return allowDecommission; } + + public Boolean getIsolateTmp() { return isolateTmp; } + } diff --git a/sdk/scheduler/src/main/java/com/mesosphere/sdk/specification/yaml/YAMLToInternalMappers.java b/sdk/scheduler/src/main/java/com/mesosphere/sdk/specification/yaml/YAMLToInternalMappers.java index 1dddd97eb6e..9c90fa781d4 100644 --- a/sdk/scheduler/src/main/java/com/mesosphere/sdk/specification/yaml/YAMLToInternalMappers.java +++ b/sdk/scheduler/src/main/java/com/mesosphere/sdk/specification/yaml/YAMLToInternalMappers.java @@ -195,7 +195,8 @@ private static PodSpec convertPod( .user(user) .preReservedRole(rawPod.getPreReservedRole()) .sharePidNamespace(rawPod.getSharePidNamespace()) - .allowDecommission(rawPod.getAllowDecommission()); + .allowDecommission(rawPod.getAllowDecommission()) + .isolateTmp(rawPod.getIsolateTmp()); List networkNames = new ArrayList<>(); List rlimits = new ArrayList<>(); diff --git a/sdk/scheduler/src/test/java/com/mesosphere/sdk/specification/DefaultPodSpecTest.java b/sdk/scheduler/src/test/java/com/mesosphere/sdk/specification/DefaultPodSpecTest.java index 8fd9b76ab8f..ed7723c4233 100644 --- a/sdk/scheduler/src/test/java/com/mesosphere/sdk/specification/DefaultPodSpecTest.java +++ b/sdk/scheduler/src/test/java/com/mesosphere/sdk/specification/DefaultPodSpecTest.java @@ -64,6 +64,7 @@ private static PodSpec getPodSpec(List taskSpecs) throws InvalidRLimit "slave_public", Arrays.asList(new DefaultSecretSpec("secretPath", "envKey", "filePath")), true, + true, true); } }