Skip to content

Commit

Permalink
Merge pull request #336 from Cognifide/feature/cleaner-removes-artifa…
Browse files Browse the repository at this point in the history
…cts-after-rerun

Cleaner will remove artifacts without reference in suites e.g after rerun one test
  • Loading branch information
plutasnyy authored Oct 2, 2018
2 parents dedc37c + 1878579 commit 3143b41
Show file tree
Hide file tree
Showing 15 changed files with 220 additions and 30 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ All notable changes to AET will be documented in this file.
## Unreleased
**List of changes that are finished but not yet released in any final version.**

- [PR-336](https://github.com/Cognifide/aet/pull/336) Cleaner will remove artifacts without reference in suites e.g after rerun one test
- [PR-337](https://github.com/Cognifide/aet/pull/337) Added rerun one test or whole suite

## Version 3.0.0
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,12 @@ public interface ArtifactsDAO extends Serializable {
*/
String getArtifactAsString(DBKey dbKey, String objectID) throws IOException;

/**
* @param dbKey - key with project and company name
* @return Set of all artifacts id contained in database as String set or empty set
*/
Set<String> getArtifactsIds(DBKey dbKey);

/**
* @param dbKey - key with project and company name
* @param objectID - suite run identificator
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,4 +28,5 @@ public SuiteAggregationCounter(int suitesToAggregate) {
public int getSuitesToAggregate() {
return suitesToAggregate;
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
package com.cognifide.aet.cleaner.processors;

import com.cognifide.aet.cleaner.context.CleanerContext;
import com.cognifide.aet.cleaner.context.SuiteAggregationCounter;
import com.cognifide.aet.cleaner.processors.exchange.AllSuiteVersionsMessageBody;
import com.cognifide.aet.communication.api.metadata.Suite;
import com.cognifide.aet.vs.DBKey;
Expand Down Expand Up @@ -73,6 +74,8 @@ public AllSuiteVersionsMessageBody apply(Map.Entry<String, Collection<Suite>> in
}).toList();

exchange.getOut().setBody(body);
exchange.getOut().setHeader(SuiteAggregationCounter.NAME_KEY,
new SuiteAggregationCounter(allProjectSuites.size()));
exchange.getOut().setHeader(CleanerContext.KEY_NAME, cleanerContext);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -77,9 +77,7 @@ public void process(Exchange exchange) throws Exception {
ReferencedArtifactsMessageBody body = new ReferencedArtifactsMessageBody(
messageBody.getData().getName(),
messageBody.getDbKey());
if (messageBody.shouldBeRemoved()) {
body.setArtifactsToRemove(metatadaArtifacts);
} else {
if (messageBody.shouldBeKept()) {
body.setArtifactsToKeep(metatadaArtifacts);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
import com.cognifide.aet.cleaner.context.CleanerContext;
import com.cognifide.aet.cleaner.processors.exchange.ReferencedArtifactsMessageBody;
import com.cognifide.aet.vs.ArtifactsDAO;
import com.google.common.collect.Sets;
import java.util.Set;
import org.apache.camel.Exchange;
import org.apache.camel.Processor;
import org.osgi.service.component.annotations.Component;
Expand All @@ -34,6 +34,15 @@ public class RemoveArtifactsProcessor implements Processor {
@Reference
private ArtifactsDAO artifactsDAO;

public RemoveArtifactsProcessor() {
//default constructor
}

// for unit tests
public RemoveArtifactsProcessor(ArtifactsDAO artifactsDAO) {
this.artifactsDAO = artifactsDAO;
}

@Override
@SuppressWarnings("unchecked")
public void process(Exchange exchange) throws Exception {
Expand All @@ -42,8 +51,7 @@ public void process(Exchange exchange) throws Exception {
final ReferencedArtifactsMessageBody messageBody = exchange.getIn()
.getBody(ReferencedArtifactsMessageBody.class);

final Sets.SetView<String> artifactsToRemove =
Sets.difference(messageBody.getArtifactsToRemove(), messageBody.getArtifactsToKeep());
Set<String> artifactsToRemove = getArtifactsIdsToRemove(artifactsDAO, messageBody);

LOGGER.debug("Artifacts that will be removed: {}", artifactsToRemove);
if (!cleanerContext.isDryRun()) {
Expand All @@ -57,4 +65,11 @@ public void process(Exchange exchange) throws Exception {
artifactsToRemove.size(), messageBody.getDbKey(), messageBody.getData());
}
}

static Set<String> getArtifactsIdsToRemove(ArtifactsDAO artifactsDAO,
ReferencedArtifactsMessageBody messageBody) {
Set<String> artifactsToRemove = artifactsDAO.getArtifactsIds(messageBody.getDbKey());
artifactsToRemove.removeAll(messageBody.getArtifactsToKeep());
return artifactsToRemove;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -64,8 +64,8 @@ public SuiteMessageBody apply(Suite suite) {
}).toList();

exchange.getOut().setBody(body);
exchange.getOut().setHeader(SuiteAggregationCounter.NAME_KEY,
new SuiteAggregationCounter(suiteVersions.size()));
exchange.getOut().setHeader(SuiteAggregationCounter.NAME_KEY, exchange.getIn()
.getHeader(SuiteAggregationCounter.NAME_KEY, SuiteAggregationCounter.class));
exchange.getOut().setHeader(CleanerContext.KEY_NAME, cleanerContext);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,7 @@

public class ReferencedArtifactsMessageBody extends MessageBody<String> {

private static final long serialVersionUID = 3811933888080393100L;

private Set<String> artifactsToRemove;
private static final long serialVersionUID = 3748474548512567069L;

private Set<String> artifactsToKeep;

Expand All @@ -36,15 +34,6 @@ public String getId() {
return getDbKey().getCompany() + "_" + getDbKey().getProject() + "|" + getData();
}

public Set<String> getArtifactsToRemove() {
return artifactsToRemove != null ? Collections.unmodifiableSet(artifactsToRemove)
: Collections.<String>emptySet();
}

public void setArtifactsToRemove(Set<String> artifactsToRemove) {
this.artifactsToRemove = artifactsToRemove;
}

public Set<String> getArtifactsToKeep() {
return artifactsToKeep != null ? Collections.unmodifiableSet(artifactsToKeep)
: Collections.<String>emptySet();
Expand All @@ -55,11 +44,6 @@ public void setArtifactsToKeep(Set<String> artifactsToKeep) {
}

public void update(ReferencedArtifactsMessageBody body) {
if (artifactsToRemove == null) {
artifactsToRemove = new HashSet<>();
}
artifactsToRemove.addAll(body.getArtifactsToRemove());

if (artifactsToKeep == null) {
artifactsToKeep = new HashSet<>();
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,8 @@ public SuiteMessageBody(Suite data, DBKey dbKey, boolean toRemove) {
this.toRemove = toRemove;
}

public boolean shouldBeRemoved() {
return toRemove;
public boolean shouldBeKept() {
return !toRemove;
}

public Set<String> getSuiteArtifacts() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ public void configure() throws Exception {
.process(suitesRemovePredicateProcessor)
.split(body())
.choice()
.when(body().method("shouldBeRemoved").isEqualTo(true)).to(direct("removeMetadata"))
.when(body().method("shouldBeKept").isEqualTo(false)).to(direct("removeMetadata"))
.otherwise().to(direct("getMetadataArtifacts"))
.endChoice();

Expand All @@ -86,7 +86,7 @@ public void configure() throws Exception {
.to(direct(AGGREGATE_SUITES_STEP));

from(direct(AGGREGATE_SUITES_STEP))
.aggregate(body().method("getId"), new SuitesAggregationStrategy())
.aggregate(body().method("getDbKey"), new SuitesAggregationStrategy())
.completionSize(header(SuiteAggregationCounter.NAME_KEY).method("getSuitesToAggregate"))
.completionTimeout(60000L).forceCompletionOnStop()
.to(direct("removeArtifacts"));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,6 @@ private boolean isFirstAggregation(Exchange oldExchange) {
public void onCompletion(Exchange exchange) {
ReferencedArtifactsMessageBody body = exchange.getIn()
.getBody(ReferencedArtifactsMessageBody.class);
LOGGER.debug("Finished aggregating {}", body.getId());
LOGGER.debug("Finished aggregating {}", body.getDbKey());
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,157 @@
/**
* AET
*
* Copyright (C) 2013 Cognifide Limited
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied. See the License for the specific language governing permissions and limitations under
* the License.
*/
package com.cognifide.aet.cleaner.processors;

import static junit.framework.TestCase.assertTrue;
import static org.junit.Assert.assertEquals;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;

import com.cognifide.aet.cleaner.context.CleanerContext;
import com.cognifide.aet.cleaner.processors.exchange.ReferencedArtifactsMessageBody;
import com.cognifide.aet.vs.ArtifactsDAO;
import com.cognifide.aet.vs.DBKey;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Set;
import org.apache.camel.CamelContext;
import org.apache.camel.Exchange;
import org.apache.camel.Message;
import org.apache.camel.impl.DefaultCamelContext;
import org.apache.camel.impl.DefaultExchange;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.runners.MockitoJUnitRunner;

@RunWith(MockitoJUnitRunner.class)
public class RemoveArtifactsProcessorTest {

private static final Set<String> ONE_TO_SEVEN_ARTIFACTS_ID_SET = new HashSet<>(Arrays
.asList("1", "2", "3", "4", "5", "6", "7"));

private static final Set<String> ONE_TO_FIVE_ARTIFACTS_ID_SET = new HashSet<>(Arrays
.asList("1", "2", "3", "4", "5"));

private static final Set<String> SIX_TO_SEVEN_ARTIFACTS_ID_SET = new HashSet<>(Arrays
.asList("6", "7"));

private Exchange exchange;

@Mock
private ArtifactsDAO artifactDAO;

@Mock
private CleanerContext cleanerContext;

@Mock
private DBKey dbKey;

private RemoveArtifactsProcessor removeArtifactsProcessor;

@Before
public void setUp() {
when(cleanerContext.isDryRun()).thenReturn(false);

CamelContext ctx = new DefaultCamelContext();
exchange = new DefaultExchange(ctx);

Message in = exchange.getIn();
in.setBody(new ReferencedArtifactsMessageBody("", dbKey));
in.setHeader(CleanerContext.KEY_NAME, cleanerContext);
}

@Test
public void checkIfRemoveArtifactsWasCalled_whenDryRunIsFalse_expectTrue() throws Exception {
when(artifactDAO.getArtifactsIds(any(DBKey.class)))
.thenReturn(new HashSet<>(ONE_TO_SEVEN_ARTIFACTS_ID_SET));
setArtifactsIdToKeep(ONE_TO_FIVE_ARTIFACTS_ID_SET);

removeArtifactsProcessor = new RemoveArtifactsProcessor(artifactDAO);
removeArtifactsProcessor.process(exchange);

verify(artifactDAO, times(1)).removeArtifacts(any(DBKey.class), any(Set.class));
verify(artifactDAO, times(1)).getArtifactsIds(any(DBKey.class));
}

@Test
public void checkIfRemoveArtifactsWasCalled_whenDryRunIsTrue_expectFalse() throws Exception {
when(cleanerContext.isDryRun()).thenReturn(true);
when(artifactDAO.getArtifactsIds(any(DBKey.class)))
.thenReturn(new HashSet<>(ONE_TO_SEVEN_ARTIFACTS_ID_SET));
setArtifactsIdToKeep(ONE_TO_FIVE_ARTIFACTS_ID_SET);

removeArtifactsProcessor = new RemoveArtifactsProcessor(artifactDAO);
removeArtifactsProcessor.process(exchange);

verify(artifactDAO, times(0)).removeArtifacts(any(DBKey.class), any(Set.class));
verify(artifactDAO, times(1)).getArtifactsIds(any(DBKey.class));
}

@Test
public void check_substractArtifactsSets_expectSetOfTwoVariables() throws Exception {
when(artifactDAO.getArtifactsIds(any(DBKey.class)))
.thenReturn(new HashSet<>(ONE_TO_SEVEN_ARTIFACTS_ID_SET));
setArtifactsIdToKeep(ONE_TO_FIVE_ARTIFACTS_ID_SET);
ReferencedArtifactsMessageBody messageBody = (ReferencedArtifactsMessageBody) exchange.getIn()
.getBody();
assertEquals(SIX_TO_SEVEN_ARTIFACTS_ID_SET,
removeArtifactsProcessor.getArtifactsIdsToRemove(artifactDAO, messageBody));
}

@Test
public void check_substractArtifactsSets_expectEmptySet() throws Exception {
when(artifactDAO.getArtifactsIds(any(DBKey.class)))
.thenReturn(new HashSet<>(ONE_TO_FIVE_ARTIFACTS_ID_SET));
setArtifactsIdToKeep(ONE_TO_SEVEN_ARTIFACTS_ID_SET);
ReferencedArtifactsMessageBody messageBody = (ReferencedArtifactsMessageBody) exchange.getIn()
.getBody();
assertTrue(removeArtifactsProcessor.getArtifactsIdsToRemove(artifactDAO, messageBody).isEmpty());
}

@Test
public void check_substractArtifactsSets_expectSetOfFiveVariables() throws Exception {
when(artifactDAO.getArtifactsIds(any(DBKey.class)))
.thenReturn(new HashSet<>(ONE_TO_SEVEN_ARTIFACTS_ID_SET));
setArtifactsIdToKeep(SIX_TO_SEVEN_ARTIFACTS_ID_SET);

ReferencedArtifactsMessageBody messageBody = (ReferencedArtifactsMessageBody) exchange.getIn()
.getBody();

assertEquals(ONE_TO_FIVE_ARTIFACTS_ID_SET,
removeArtifactsProcessor.getArtifactsIdsToRemove(artifactDAO, messageBody));
}

@Test
public void check_substractArtifactsSetsWhenDbIsEmpty_expectEmptySet() throws Exception {
when(artifactDAO.getArtifactsIds(any(DBKey.class)))
.thenReturn(new HashSet<>());
setArtifactsIdToKeep(SIX_TO_SEVEN_ARTIFACTS_ID_SET);
ReferencedArtifactsMessageBody messageBody = (ReferencedArtifactsMessageBody) exchange.getIn()
.getBody();
assertTrue(removeArtifactsProcessor.getArtifactsIdsToRemove(artifactDAO, messageBody).isEmpty());
}

private void setArtifactsIdToKeep(Set<String> artifactsIdToKeep) {
ReferencedArtifactsMessageBody body = (ReferencedArtifactsMessageBody) exchange.getIn()
.getBody();
body.setArtifactsToKeep(artifactsIdToKeep);
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@
*/
package com.cognifide.aet.vs.artifacts;

import static com.mongodb.client.model.Projections.fields;
import static com.mongodb.client.model.Projections.include;
import com.cognifide.aet.vs.Artifact;
import com.cognifide.aet.vs.ArtifactsDAO;
import com.cognifide.aet.vs.DBKey;
Expand All @@ -30,6 +32,7 @@
import java.lang.reflect.Type;
import java.nio.charset.StandardCharsets;
import java.util.Date;
import java.util.HashSet;
import java.util.Set;
import org.apache.commons.io.IOUtils;
import org.bson.Document;
Expand Down Expand Up @@ -103,6 +106,22 @@ public Artifact getArtifact(DBKey dbKey, String objectID) {
return artifact;
}

@Override
public Set<String> getArtifactsIds(DBKey dbKey) {
final String dbName = MongoDBClient.getDbName(dbKey.getCompany(), dbKey.getProject());
FindIterable<Document> findIterable =
client.getDatabase(dbName)
.getCollection(ARTIFACTS_COLLECTION_NAME + FILES_COLLECTION_SUFFIX)
.find().projection(fields(include(ID_FIELD_NAME)));

Set<String> artifactsId = new HashSet<>();
for (Document document : findIterable) {
artifactsId.add(document.get(ID_FIELD_NAME).toString());
}
return artifactsId;
}


@Override
public String getArtifactAsString(DBKey dbKey, String objectID) throws IOException {
String result = null;
Expand Down
Loading

0 comments on commit 3143b41

Please sign in to comment.