Skip to content

Commit

Permalink
Fix/blue green deploy (#4414)
Browse files Browse the repository at this point in the history 
* fix(blue-green-deploy): removing old failed manifests

* fix(blue-green-deploy): refactoring

* fix(blue-green-deploy): refactoring

* fix(blue-green-deploy): added docs

* fix(blue-green-deploy): added test to validate new behaviour

(cherry picked from commit 5156eec)

# Conflicts:
#	orca-clouddriver/src/main/java/com/netflix/spinnaker/orca/clouddriver/pipeline/manifest/DeployManifestStage.java
#	orca-clouddriver/src/test/java/com/netflix/spinnaker/orca/clouddriver/pipeline/manifest/DeployManifestStageTest.java
  • Loading branch information
@kkotula @mergify
kkotula authored and mergify[bot] committed Mar 27, 2023
1 parent c990e35 commit f065d3e
Show file tree
Hide file tree
Showing 2 changed files with 425 additions and 30 deletions.
256 changes: 250 additions & 6 deletions ...in/java/com/netflix/spinnaker/orca/clouddriver/pipeline/manifest/DeployManifestStage.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,27 +30,31 @@
import com.netflix.spinnaker.orca.clouddriver.tasks.artifacts.CleanupArtifactsTask;
import com.netflix.spinnaker.orca.clouddriver.tasks.manifest.*;
import com.netflix.spinnaker.orca.clouddriver.tasks.manifest.DeployManifestContext.TrafficManagement;
import com.netflix.spinnaker.orca.clouddriver.tasks.manifest.ResolveDeploySourceManifestTask;
import com.netflix.spinnaker.orca.pipeline.ExpressionAwareStageDefinitionBuilder;
import com.netflix.spinnaker.orca.pipeline.tasks.artifacts.BindProducedArtifactsTask;
import com.netflix.spinnaker.orca.pipeline.util.ContextParameterProcessor;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.function.BiConsumer;
import java.util.stream.Collectors;
import javax.annotation.Nonnull;
import org.springframework.beans.factory.annotation.Autowired;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import org.jetbrains.annotations.NotNull;
import org.springframework.stereotype.Component;

@Component
@RequiredArgsConstructor
public class DeployManifestStage extends ExpressionAwareStageDefinitionBuilder {

public static final String PIPELINE_CONFIG_TYPE = "deployManifest";

private final OortService oortService;
private final OldManifestActionAppender oldManifestActionAppender;

@Autowired
public DeployManifestStage(OortService oortService) {
this.oortService = oortService;
private static boolean shouldRemoveStageOutputs(@NotNull StageExecution stage) {
return stage.getContext().getOrDefault("noOutput", "false").toString().equals("true");
}

@Override
Expand All @@ -67,6 +71,7 @@ public void taskGraph(@Nonnull StageExecution stage, @Nonnull TaskNode.Builder b
.withTask(BindProducedArtifactsTask.TASK_NAME, BindProducedArtifactsTask.class);
}

<<<<<<< HEAD
public void afterStages(@Nonnull StageExecution stage, @Nonnull StageGraphBuilder graph) {
TrafficManagement trafficManagement =
stage.mapTo(DeployManifestContext.class).getTrafficManagement();
Expand Down Expand Up @@ -149,6 +154,8 @@ private ImmutableList<String> getOldManifestNames(
.collect(toImmutableList());
}

=======
>>>>>>> 5156eecb3 (Fix/blue green deploy (#4414))
@Override
public boolean processExpressions(
@Nonnull StageExecution stage,
Expand All @@ -162,4 +169,241 @@ public boolean processExpressions(
}
return true;
}

@Override
public void afterStages(@Nonnull StageExecution stage, @Nonnull StageGraphBuilder graph) {
TrafficManagement trafficManagement =
stage.mapTo(DeployManifestContext.class).getTrafficManagement();
if (trafficManagement.isEnabled()) {
switch (trafficManagement.getOptions().getStrategy()) {
case RED_BLACK:
case BLUE_GREEN:
oldManifestActionAppender.deleteOrDisableOldManifest(stage.getContext(), graph);
break;
case HIGHLANDER:
oldManifestActionAppender.disableOldManifest(stage.getContext(), graph);
oldManifestActionAppender.deleteOldManifest(stage.getContext(), graph);
break;
case NONE:
// do nothing
}
}
if (shouldRemoveStageOutputs(stage)) {
stage.setOutputs(emptyMap());
}
}

/** {@code OldManifestActionAppender} appends new stages to old manifests */
@Component
@RequiredArgsConstructor
static class OldManifestActionAppender {

private final GetDeployedManifests deployedManifests;
private final ManifestOperationsHelper manifestOperationsHelper;

/**
* Appends delete stages to already deployed manifests that preceded the current stage manifest
*
* @param parentContext of currently executed stage
* @param graph stage execution graph
*/
private void deleteOldManifest(Map<String, Object> parentContext, StageGraphBuilder graph) {
applyAction(
parentContext,
(name, manifest) ->
appendStage(graph, manifest, name, DeleteManifestStage.PIPELINE_CONFIG_TYPE));
}

/**
* Appends disable stages to already deployed manifests that preceded the current stage manifest
*
* @param parentContext of currently executed stage
* @param graph stage execution graph
*/
private void disableOldManifest(Map<String, Object> parentContext, StageGraphBuilder graph) {
applyAction(
parentContext,
(name, manifest) ->
appendStage(graph, manifest, name, DisableManifestStage.PIPELINE_CONFIG_TYPE));
}

/**
* Appends disable or delete stages to already deployed manifests that preceded the current
* stage manifest. The specific stage that will be appended depends on the status of the
* previous deployment.
*
* @param parentContext of currently executed stage
* @param graph stage execution graph
*/
private void deleteOrDisableOldManifest(
Map<String, Object> parentContext, StageGraphBuilder graph) {
applyAction(
parentContext,
(name, manifest) -> {
var oldManifestIsUnstable =
this.manifestOperationsHelper.previousDeploymentNeitherStableNorFailed(
manifest.getAccount(), name);
var nextStageType =
oldManifestIsUnstable
? DeleteManifestStage.PIPELINE_CONFIG_TYPE
: DisableManifestStage.PIPELINE_CONFIG_TYPE;
appendStage(graph, manifest, name, nextStageType);
});
}

private void applyAction(
Map<String, Object> parentContext, BiConsumer<String, DeployedManifest> action) {
this.deployedManifests
.get(parentContext)
.forEach(
currentlyDeployedManifest ->
manifestOperationsHelper
.getOldManifestNames(currentlyDeployedManifest)
.forEach(
oldManifestName ->
action.accept(oldManifestName, currentlyDeployedManifest)));
}

private void appendStage(
StageGraphBuilder graph, DeployedManifest manifest, String name, String stageType) {
graph.append(
(stage) -> {
stage.setType(stageType);
Map<String, Object> context = stage.getContext();
context.put("account", manifest.getAccount());
context.put("app", manifest.getApplication());
context.put("cloudProvider", manifest.getCloudProvider());
context.put("manifestName", name);
context.put("location", manifest.getNamespace());
});
}
}

/**
* Delegate class to handle all manifest-related actions in this file such as fetching manifest
* from external service or extracting manifest params from parentContext
*/
@Component
@RequiredArgsConstructor
static class ManifestOperationsHelper {

private static final String REPLICA_SET = "replicaSet";
private static final String KIND = "kind";
private static final String OUTPUTS_MANIFEST = "outputs.manifests";

private final OortService oortService;

/**
* This returns all replicaSet manifests from the cluster. The search is performed in an
* external service, and search parameters match manifests deployed in the current stage.
*
* @param dm - deployment manifest of current stage
* @return list of all manifest already deployed to the cluster
*/
ImmutableList<String> getOldManifestNames(DeployedManifest dm) {
return oortService
.getClusterManifests(
dm.getAccount(),
dm.getNamespace(),
REPLICA_SET,
dm.getApplication(),
dm.getClusterName())
.stream()
.filter(m -> !m.getFullResourceName().equals(dm.getManifestName()))
.map(ManifestCoordinates::getFullResourceName)
.collect(toImmutableList());
}

/**
* Returns replicaSet manifests from the {@code parentContext}
*
* @param parentContext of currently processed stage
* @return list of replicaSet manifests deployed in the cluster - obtained directly from the
* {@code parentContext}
*/
List<Map<String, ?>> getNewManifests(Map<String, Object> parentContext) {
var manifestsFromParentContext = (List<Map<String, ?>>) parentContext.get(OUTPUTS_MANIFEST);
return manifestsFromParentContext.stream()
.filter(manifest -> REPLICA_SET.equalsIgnoreCase((String) manifest.get(KIND)))
.collect(Collectors.toList());
}

/**
* During a B/G deployment, if the blue deployment fails to create pods (due to issues such as
* an incorrect image name), the deployment will not fail, but will wait indefinitely to achieve
* stability. This is indicated by status.failed=false and status.stable=false. This method
* checks for such a situation.
*
* @param account used to run deployment
* @param name of the manifest
* @return true, if manifest was not deployed correctly and waits to get stable, false otherwise
*/
boolean previousDeploymentNeitherStableNorFailed(String account, String name) {
var oldManifest = this.oortService.getManifest(account, name, false);

var status = oldManifest.getStatus();
var notStable = !status.getStable().isState();
var notFailed = !status.getFailed().isState();

return notFailed && notStable;
}
}

/** Delegate class for fetching and mapping manifests deployed in the cluster */
@Component
@RequiredArgsConstructor
static class GetDeployedManifests {

private final ManifestOperationsHelper manifestOperationsHelper;

/**
* This method encapsulates fetching deployed manifests and mapping them to a new designated
* type, {@code DeployedManifest}
*
* @param parentContext is the context of currently processed stage
* @return list of replicaSet manifests deployed in currently processed stage
*/
List<DeployedManifest> get(Map<String, Object> parentContext) {

var deployedManifests = new ArrayList<DeployedManifest>();
var account = (String) parentContext.get("account");
var manifestMoniker = (Map) parentContext.get("moniker");
var application = (String) manifestMoniker.get("app");

this.manifestOperationsHelper
.getNewManifests(parentContext)
.forEach(
manifest -> {
var manifestMetadata = (Map) manifest.get("metadata");
var annotations = (Map) manifestMetadata.get("annotations");

deployedManifests.add(
new DeployedManifest(
account,
manifestMoniker,
application,
(Map) manifest.get("metadata"),
String.format("replicaSet %s", manifestMetadata.get("name")),
(String) manifestMetadata.get("namespace"),
(Map) manifestMetadata.get("annotations"),
(String) annotations.get("moniker.spinnaker.io/cluster"),
"kubernetes"));
});
return deployedManifests;
}
}

@Getter
@RequiredArgsConstructor
static class DeployedManifest {
private final String account;
private final Map manifestMoniker;
private final String application;
private final Map manifestMetadata;
private final String manifestName;
private final String namespace;
private final Map annotations;
private final String clusterName;
private final String cloudProvider;
}
}
Loading

0 comments on commit f065d3e

Please sign in to comment.