Cleaner will remove artifacts without reference in suites e.g after rerun one test

api/datastorage-api/src/main/java/com/cognifide/aet/vs/ArtifactsDAO.java

@@ -77,6 +77,12 @@ public interface ArtifactsDAO extends Serializable {
    */
   String getArtifactAsString(DBKey dbKey, String objectID) throws IOException;
 
+  /**
+   * @param dbKey - key with project and company name
+   * @return Set of all artifacts id contained in database as String set or empty set
+   */
+  Set<String> getArtifactsId(DBKey dbKey);
+
   /**
    * @param dbKey - key with project and company name
    * @param objectID - suite run identificator

core/cleaner/src/main/java/com/cognifide/aet/cleaner/context/SuiteAggregationCounter.java

@@ -21,11 +21,15 @@ public class SuiteAggregationCounter {
 
   private final int suitesToAggregate;
 
-  public SuiteAggregationCounter(int suitesToAggregate) {
-    this.suitesToAggregate = suitesToAggregate;
+  public SuiteAggregationCounter() {
+    this.suitesToAggregate = 0;
   }
 
   public int getSuitesToAggregate() {
     return suitesToAggregate;
   }
+
+  public int addSuitesToAggregate(int suitesToAggregate) {
+    return this.suitesToAggregate + suitesToAggregate;
+  }
 }

core/cleaner/src/main/java/com/cognifide/aet/cleaner/processors/FetchAllProjectSuitesProcessor.java

@@ -16,6 +16,7 @@
 package com.cognifide.aet.cleaner.processors;
 
 import com.cognifide.aet.cleaner.context.CleanerContext;
+import com.cognifide.aet.cleaner.context.SuiteAggregationCounter;
 import com.cognifide.aet.cleaner.processors.exchange.AllSuiteVersionsMessageBody;
 import com.cognifide.aet.communication.api.metadata.Suite;
 import com.cognifide.aet.vs.DBKey;
@@ -73,6 +74,7 @@ public AllSuiteVersionsMessageBody apply(Map.Entry<String, Collection<Suite>> in
             }).toList();
 
     exchange.getOut().setBody(body);
+    exchange.getOut().setHeader(SuiteAggregationCounter.NAME_KEY, new SuiteAggregationCounter());
     exchange.getOut().setHeader(CleanerContext.KEY_NAME, cleanerContext);
   }
 

core/cleaner/src/main/java/com/cognifide/aet/cleaner/processors/GetMetadataArtifactsProcessor.java

@@ -77,9 +77,7 @@ public void process(Exchange exchange) throws Exception {
     ReferencedArtifactsMessageBody body = new ReferencedArtifactsMessageBody(
         messageBody.getData().getName(),
         messageBody.getDbKey());
-    if (messageBody.shouldBeRemoved()) {
-      body.setArtifactsToRemove(metatadaArtifacts);
-    } else {
+    if (messageBody.shouldBeKeeped()) {
       body.setArtifactsToKeep(metatadaArtifacts);
     }
 

core/cleaner/src/main/java/com/cognifide/aet/cleaner/processors/RemoveArtifactsProcessor.java

@@ -19,6 +19,11 @@
 import com.cognifide.aet.cleaner.processors.exchange.ReferencedArtifactsMessageBody;
 import com.cognifide.aet.vs.ArtifactsDAO;
 import com.google.common.collect.Sets;
+import com.google.common.collect.Sets.SetView;
+import java.util.ArrayList;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 import org.apache.camel.Exchange;
 import org.apache.camel.Processor;
 import org.osgi.service.component.annotations.Component;
@@ -34,6 +39,15 @@ public class RemoveArtifactsProcessor implements Processor {
   @Reference
   private ArtifactsDAO artifactsDAO;
 
+  public RemoveArtifactsProcessor() {
+    //default constructor
+  }
+
+  // for unit tests
+  public RemoveArtifactsProcessor(ArtifactsDAO artifactsDAO) {
+    this.artifactsDAO = artifactsDAO;
+  }
+
   @Override
   @SuppressWarnings("unchecked")
   public void process(Exchange exchange) throws Exception {
@@ -42,8 +56,7 @@ public void process(Exchange exchange) throws Exception {
     final ReferencedArtifactsMessageBody messageBody = exchange.getIn()
         .getBody(ReferencedArtifactsMessageBody.class);
 
-    final Sets.SetView<String> artifactsToRemove =
-        Sets.difference(messageBody.getArtifactsToRemove(), messageBody.getArtifactsToKeep());
+    Set<String> artifactsToRemove = getArtifactsIdsToRemove(artifactsDAO, messageBody);
 
     LOGGER.debug("Artifacts that will be removed: {}", artifactsToRemove);
     if (!cleanerContext.isDryRun()) {
@@ -57,4 +70,11 @@ public void process(Exchange exchange) throws Exception {
           artifactsToRemove.size(), messageBody.getDbKey(), messageBody.getData());
     }
   }
+
+  public static Set<String> getArtifactsIdsToRemove(ArtifactsDAO artifactsDAO,
+      ReferencedArtifactsMessageBody messageBody) {
+    Set<String> artifactsToRemove = artifactsDAO.getArtifactsId(messageBody.getDbKey());
+    artifactsToRemove.removeAll(messageBody.getArtifactsToKeep());
+    return artifactsToRemove;
+  }
 }

core/cleaner/src/main/java/com/cognifide/aet/cleaner/processors/SuitesRemovePredicateProcessor.java

@@ -49,6 +49,10 @@ public void process(Exchange exchange) throws Exception {
     final Collection<Suite> suiteVersions = allSuiteVersions.getAllVersions();
     final DBKey dbKey = allSuiteVersions.getDbKey();
 
+    SuiteAggregationCounter suiteAggregationCounter = exchange.getIn()
+        .getHeader(SuiteAggregationCounter.NAME_KEY, SuiteAggregationCounter.class);
+    suiteAggregationCounter.addSuitesToAggregate(suiteVersions.size());
+
     LOGGER.info("Processing suite `{}` with {} version(s) in {}", allSuiteVersions.getData(),
         suiteVersions.size(), dbKey);
 
@@ -65,7 +69,7 @@ public SuiteMessageBody apply(Suite suite) {
 
     exchange.getOut().setBody(body);
     exchange.getOut().setHeader(SuiteAggregationCounter.NAME_KEY,
-        new SuiteAggregationCounter(suiteVersions.size()));
+        suiteAggregationCounter);
     exchange.getOut().setHeader(CleanerContext.KEY_NAME, cleanerContext);
   }
 }

core/cleaner/src/main/java/com/cognifide/aet/cleaner/processors/exchange/ReferencedArtifactsMessageBody.java

@@ -22,9 +22,7 @@
 
 public class ReferencedArtifactsMessageBody extends MessageBody<String> {
 
-  private static final long serialVersionUID = 3811933888080393100L;
-
-  private Set<String> artifactsToRemove;
+  private static final long serialVersionUID = 3748474548512567069L;
 
   private Set<String> artifactsToKeep;
 
@@ -36,15 +34,6 @@ public String getId() {
     return getDbKey().getCompany() + "_" + getDbKey().getProject() + "|" + getData();
   }
 
-  public Set<String> getArtifactsToRemove() {
-    return artifactsToRemove != null ? Collections.unmodifiableSet(artifactsToRemove)
-        : Collections.<String>emptySet();
-  }
-
-  public void setArtifactsToRemove(Set<String> artifactsToRemove) {
-    this.artifactsToRemove = artifactsToRemove;
-  }
-
   public Set<String> getArtifactsToKeep() {
     return artifactsToKeep != null ? Collections.unmodifiableSet(artifactsToKeep)
         : Collections.<String>emptySet();
@@ -55,11 +44,6 @@ public void setArtifactsToKeep(Set<String> artifactsToKeep) {
   }
 
   public void update(ReferencedArtifactsMessageBody body) {
-    if (artifactsToRemove == null) {
-      artifactsToRemove = new HashSet<>();
-    }
-    artifactsToRemove.addAll(body.getArtifactsToRemove());
-
     if (artifactsToKeep == null) {
       artifactsToKeep = new HashSet<>();
     }

core/cleaner/src/main/java/com/cognifide/aet/cleaner/processors/exchange/SuiteMessageBody.java

@@ -32,8 +32,8 @@ public SuiteMessageBody(Suite data, DBKey dbKey, boolean toRemove) {
     this.toRemove = toRemove;
   }
 
-  public boolean shouldBeRemoved() {
-    return toRemove;
+  public boolean shouldBeKeeped() {
+    return !toRemove;
   }
 
   public Set<String> getSuiteArtifacts() {

core/cleaner/src/main/java/com/cognifide/aet/cleaner/route/MetadataCleanerRouteBuilder.java

@@ -72,7 +72,7 @@ public void configure() throws Exception {
         .process(suitesRemovePredicateProcessor)
         .split(body())
         .choice()
-        .when(body().method("shouldBeRemoved").isEqualTo(true)).to(direct("removeMetadata"))
+        .when(body().method("shouldBeKeeped").isEqualTo(false)).to(direct("removeMetadata"))
         .otherwise().to(direct("getMetadataArtifacts"))
         .endChoice();
 
@@ -86,7 +86,7 @@ public void configure() throws Exception {
         .to(direct(AGGREGATE_SUITES_STEP));
 
     from(direct(AGGREGATE_SUITES_STEP))
-        .aggregate(body().method("getId"), new SuitesAggregationStrategy())
+        .aggregate(body().method("getDbKey"), new SuitesAggregationStrategy())
         .completionSize(header(SuiteAggregationCounter.NAME_KEY).method("getSuitesToAggregate"))
         .completionTimeout(60000L).forceCompletionOnStop()
         .to(direct("removeArtifacts"));

core/cleaner/src/main/java/com/cognifide/aet/cleaner/route/SuitesAggregationStrategy.java

@@ -57,6 +57,6 @@ private boolean isFirstAggregation(Exchange oldExchange) {
   public void onCompletion(Exchange exchange) {
     ReferencedArtifactsMessageBody body = exchange.getIn()
         .getBody(ReferencedArtifactsMessageBody.class);
-    LOGGER.debug("Finished aggregating {}", body.getId());
+    LOGGER.debug("Finished aggregating {}", body.getData());
   }
 }

core/cleaner/src/test/java/com/cognifide/aet/cleaner/processors/RemoveArtifactsProcessorTest.java

@@ -0,0 +1,160 @@
+/**
+ * AET
+ *
+ * Copyright (C) 2013 Cognifide Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package com.cognifide.aet.cleaner.processors;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import com.cognifide.aet.cleaner.context.CleanerContext;
+import com.cognifide.aet.cleaner.processors.exchange.ReferencedArtifactsMessageBody;
+import com.cognifide.aet.vs.ArtifactsDAO;
+import com.cognifide.aet.vs.DBKey;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+import org.apache.camel.CamelContext;
+import org.apache.camel.Exchange;
+import org.apache.camel.Message;
+import org.apache.camel.impl.DefaultCamelContext;
+import org.apache.camel.impl.DefaultExchange;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.runners.MockitoJUnitRunner;
+
+@RunWith(MockitoJUnitRunner.class)
+public class RemoveArtifactsProcessorTest {
+
+  private static final Set<String> ONE_TO_SEVEN_ARTIFACTS_ID_SET = new HashSet<>(Arrays
+      .asList("1", "2", "3", "4", "5", "6", "7"));
+
+  private static final Set<String> ONE_TO_FIVE_ARTIFACTS_ID_SET = new HashSet<>(Arrays
+      .asList("1", "2", "3", "4", "5"));
+
+  private static final Set<String> SIX_TO_SEVEN_ARTIFACTS_ID_SET = new HashSet<>(Arrays
+      .asList("6", "7"));
+
+  private static final Set<String> EMPTY_ARTIFACTS_ID_SET = new HashSet<>();
+
+  private Exchange exchange;
+
+  @Mock
+  private ArtifactsDAO artifactDAO;
+
+  @Mock
+  private CleanerContext cleanerContext;
+
+  @Mock
+  private DBKey dbKey;
+
+  private RemoveArtifactsProcessor removeArtifactsProcessor;
+
+  @Before
+  public void setUp() {
+    when(cleanerContext.isDryRun()).thenReturn(false);
+
+    CamelContext ctx = new DefaultCamelContext();
+    exchange = new DefaultExchange(ctx);
+
+    Message in = exchange.getIn();
+    in.setBody(new ReferencedArtifactsMessageBody("", dbKey));
+    in.setHeader(CleanerContext.KEY_NAME, cleanerContext);
+  }
+
+  @Test
+  public void check_ifRemoveArtifactsWasCalled_expectTrue() throws Exception {
+    when(artifactDAO.getArtifactsId(any(DBKey.class)))
+        .thenReturn(new HashSet<>(ONE_TO_SEVEN_ARTIFACTS_ID_SET));
+    setArtifactsIdToKeep(ONE_TO_FIVE_ARTIFACTS_ID_SET);
+
+    removeArtifactsProcessor = new RemoveArtifactsProcessor(artifactDAO);
+    removeArtifactsProcessor.process(exchange);
+
+    verify(artifactDAO, times(1)).removeArtifacts(any(DBKey.class), any(Set.class));
+    verify(artifactDAO, times(1)).getArtifactsId(any(DBKey.class));
+  }
+
+  @Test
+  public void check_ifRemoveArtifactsWasCalled_expectFalse() throws Exception {
+    when(cleanerContext.isDryRun()).thenReturn(true);
+    when(artifactDAO.getArtifactsId(any(DBKey.class)))
+        .thenReturn(new HashSet<>(ONE_TO_SEVEN_ARTIFACTS_ID_SET));
+    setArtifactsIdToKeep(ONE_TO_FIVE_ARTIFACTS_ID_SET);
+
+    removeArtifactsProcessor = new RemoveArtifactsProcessor(artifactDAO);
+    removeArtifactsProcessor.process(exchange);
+
+    verify(artifactDAO, times(0)).removeArtifacts(any(DBKey.class), any(Set.class));
+    verify(artifactDAO, times(1)).getArtifactsId(any(DBKey.class));
+  }
+
+  @Test
+  public void check_substractArtifactsSets_expectSetOfTwoVariables() throws Exception {
+    when(artifactDAO.getArtifactsId(any(DBKey.class)))
+        .thenReturn(new HashSet<>(ONE_TO_SEVEN_ARTIFACTS_ID_SET));
+    setArtifactsIdToKeep(ONE_TO_FIVE_ARTIFACTS_ID_SET);
+    ReferencedArtifactsMessageBody messageBody = (ReferencedArtifactsMessageBody) exchange.getIn()
+        .getBody();
+    assertEquals(SIX_TO_SEVEN_ARTIFACTS_ID_SET,
+        removeArtifactsProcessor.getArtifactsIdsToRemove(artifactDAO, messageBody));
+  }
+
+  @Test
+  public void check_substractArtifactsSets_expectEmptySet() throws Exception {
+    when(artifactDAO.getArtifactsId(any(DBKey.class)))
+        .thenReturn(new HashSet<>(ONE_TO_FIVE_ARTIFACTS_ID_SET));
+    setArtifactsIdToKeep(ONE_TO_SEVEN_ARTIFACTS_ID_SET);
+    ReferencedArtifactsMessageBody messageBody = (ReferencedArtifactsMessageBody) exchange.getIn()
+        .getBody();
+    assertEquals(EMPTY_ARTIFACTS_ID_SET,
+        removeArtifactsProcessor.getArtifactsIdsToRemove(artifactDAO, messageBody));
+  }
+
+  @Test
+  public void check_substractArtifactsSets_expectSetOfFiveVariables() throws Exception {
+    when(artifactDAO.getArtifactsId(any(DBKey.class)))
+        .thenReturn(new HashSet<>(ONE_TO_SEVEN_ARTIFACTS_ID_SET));
+    setArtifactsIdToKeep(SIX_TO_SEVEN_ARTIFACTS_ID_SET);
+
+    ReferencedArtifactsMessageBody messageBody = (ReferencedArtifactsMessageBody) exchange.getIn()
+        .getBody();
+
+    assertEquals(ONE_TO_FIVE_ARTIFACTS_ID_SET,
+        removeArtifactsProcessor.getArtifactsIdsToRemove(artifactDAO, messageBody));
+  }
+
+  @Test
+  public void check_substractArtifactsSetsWhenDbIsEmpty_expectEmptySet() throws Exception {
+    when(artifactDAO.getArtifactsId(any(DBKey.class)))
+        .thenReturn(new HashSet<>(EMPTY_ARTIFACTS_ID_SET));
+    setArtifactsIdToKeep(SIX_TO_SEVEN_ARTIFACTS_ID_SET);
+    ReferencedArtifactsMessageBody messageBody = (ReferencedArtifactsMessageBody) exchange.getIn()
+        .getBody();
+    assertEquals(EMPTY_ARTIFACTS_ID_SET,
+        removeArtifactsProcessor.getArtifactsIdsToRemove(artifactDAO, messageBody));
+  }
+
+  private void setArtifactsIdToKeep(Set<String> artifactsIdToRemove) {
+    ReferencedArtifactsMessageBody body = (ReferencedArtifactsMessageBody) exchange.getIn()
+        .getBody();
+    body.setArtifactsToKeep(artifactsIdToRemove);
+  }
+
+}