Skip to content
This repository has been archived by the owner on Sep 21, 2021. It is now read-only.

Commit

Permalink
Saving test information directly to a file instead of keeping it in m…
Browse files Browse the repository at this point in the history 
…emory (#782)

* Dumping test information to a file every 50 tests.

* Reverting change in the dockerfile.

* Saving test information directly to a file instead of keeping it in memory.
  • Loading branch information
@diemol
diemol authored Nov 27, 2018
1 parent 1b06541 commit f19f4c1
Show file tree
Hide file tree
Showing 6 changed files with 56 additions and 128 deletions.
6 changes: 3 additions & 3 deletions docker/Dockerfile
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
#== Ubuntu xenial is 16.04, i.e. FROM ubuntu:16.04
# Find latest images at https://hub.docker.com/r/library/ubuntu/
# Layer size: big: ~130 MB
FROM ubuntu:xenial-20181005
ENV UBUNTU_FLAVOR="xenial" \
UBUNTU_DATE="20181005"

FROM ubuntu:xenial-20181113
ENV UBUNTU_FLAVOR="xenial" \
UBUNTU_DATE="20181113"
ARG UBUNTU_MIRROR=http://archive.ubuntu.com/ubuntu

#== Ubuntu flavors - common
Expand Down
81 changes: 39 additions & 42 deletions src/main/java/de/zalando/ep/zalenium/dashboard/Dashboard.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,20 +5,17 @@
import com.google.gson.GsonBuilder;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import com.google.gson.reflect.TypeToken;
import de.zalando.ep.zalenium.util.CommonProxyUtilities;
import de.zalando.ep.zalenium.util.Environment;

import org.apache.commons.io.FileUtils;

import java.io.File;
import java.io.IOException;
import java.lang.reflect.Type;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.stream.Collectors;

import org.slf4j.Logger;
Expand Down Expand Up @@ -48,24 +45,18 @@ public class Dashboard implements DashboardInterface {
private static final String ZALENIUM_RETENTION_PERIOD = "ZALENIUM_RETENTION_PERIOD";
private static final int DEFAULT_RETENTION_PERIOD = 3;
private static final Logger LOGGER = LoggerFactory.getLogger(Dashboard.class.getName());
private static List<TestInformation> executedTestsInformation = new ArrayList<>();
private static CommonProxyUtilities commonProxyUtilities = new CommonProxyUtilities();
private static final Environment defaultEnvironment = new Environment();
private static Environment env = defaultEnvironment;
private static int executedTests = 0;
private static int executedTestsWithVideo = 0;
private static int retentionPeriod;
private static AtomicBoolean shutdownHookAdded = new AtomicBoolean(false);


public Dashboard() {
retentionPeriod = env.getIntEnvVariable(ZALENIUM_RETENTION_PERIOD,
DEFAULT_RETENTION_PERIOD);
}

public static List<TestInformation> getExecutedTestsInformation() {
return executedTestsInformation;
}

public static String getCurrentLocalPath() {
return commonProxyUtilities.currentLocalPath();
}
Expand Down Expand Up @@ -155,32 +146,31 @@ public synchronized void updateDashboard(TestInformation testInformation) {
if (!jsFolder.exists()) {
FileUtils.copyDirectory(new File(getCurrentLocalPath() + JS_FOLDER), jsFolder);
}
executedTestsInformation.add(testInformation);
saveTestInformation(testInformation);
} catch (IOException e) {
LOGGER.warn("Error while updating the dashboard.", e);
}
}

public synchronized void cleanupDashboard() throws IOException {
Map<Boolean, List<TestInformation>> partitioned = executedTestsInformation.stream()
public synchronized void cleanupDashboard() throws IOException {
List<TestInformation> informationList = loadTestInformationFromFile();
Map<Boolean, List<TestInformation>> partitioned = informationList.stream()
.collect(Collectors.partitioningBy(testInformation -> testInformation.getRetentionDate().getTime() > new Date().getTime()));

List<TestInformation> validTestsInformation = partitioned.get(true);
List<TestInformation> invalidTestsInformation = partitioned.get(false);

if(invalidTestsInformation.size() > 0) {
LOGGER.info("Cleaning up " + invalidTestsInformation.size() + " test from Dashboard");
LOGGER.info("Cleaning up " + invalidTestsInformation.size() + " test(s) from Dashboard");
File testCountFile = new File(getLocalVideosPath(), TEST_COUNT_FILE);
File dashboardHtml = new File(getLocalVideosPath(), DASHBOARD_FILE);
File testList = new File(getLocalVideosPath(), TEST_LIST_FILE);

for(TestInformation testInformation : invalidTestsInformation) {
deleteIfExists(new File(getLocalVideosPath() + "/" + testInformation.getFileName()));
deleteIfExists(new File(testInformation.getLogsFolderPath()));

executedTestsInformation.remove(testInformation);
}

deleteIfExists(dashboardHtml);
deleteIfExists(testList);
deleteIfExists(testCountFile);
Expand All @@ -193,15 +183,17 @@ public synchronized void cleanupDashboard() throws IOException {
for(TestInformation testInformation : validTestsInformation) {
updateDashboard(testInformation);
}
dumpTestInformationToFile(validTestsInformation);
}
}

public synchronized void resetDashboard() throws IOException {
LOGGER.info("Reseting Dashboard");
LOGGER.info("Resetting Dashboard");
File testList = new File(getLocalVideosPath(), TEST_LIST_FILE);
File testCountFile = new File(getLocalVideosPath(), TEST_COUNT_FILE);
File dashboardHtml = new File(getLocalVideosPath(), DASHBOARD_FILE);
File logsFolder = new File(getLocalVideosPath(), LOGS_FOLDER_NAME);
File testInformationFile = new File(getLocalVideosPath(), TEST_INFORMATION_FILE);
File videosFolder = new File(getLocalVideosPath());
String[] extensions = new String[] { "mp4", "mkv" };
for (File file : FileUtils.listFiles(videosFolder, extensions, true)) {
Expand All @@ -211,12 +203,10 @@ public synchronized void resetDashboard() throws IOException {
deleteIfExists(testList);
deleteIfExists(testCountFile);
deleteIfExists(dashboardHtml);
deleteIfExists(testInformationFile);
String dashboard = FileUtils.readFileToString(new File(getCurrentLocalPath(), DASHBOARD_TEMPLATE_FILE), UTF_8);
dashboard = dashboard.replace("{testList}", "").
replace("{executedTests}", "0");
dashboard = dashboard.replace("{testList}", "").replace("{executedTests}", "0");
FileUtils.writeStringToFile(dashboardHtml, dashboard, UTF_8);

executedTestsInformation = new ArrayList<>();
}

public static void deleteIfExists(File file) {
Expand Down Expand Up @@ -250,42 +240,49 @@ public static void synchronizeExecutedTestsValues(File testCountFile) {
}

@VisibleForTesting
public static void loadTestInformationFromFile() {
public static List<TestInformation> loadTestInformationFromFile() {
try {
if (executedTestsInformation.size() == 0) {
File testInformationFile = new File(getLocalVideosPath(), TEST_INFORMATION_FILE);
if (testInformationFile.exists()) {
String testInformationContents = FileUtils.readFileToString(testInformationFile, UTF_8);
Type collectionType = new TypeToken<ArrayList<TestInformation>>(){}.getType();
executedTestsInformation = new Gson().fromJson(testInformationContents, collectionType);
List<TestInformation> testInformation = new ArrayList<>();
File testInformationFile = new File(getLocalVideosPath(), TEST_INFORMATION_FILE);
if (testInformationFile.exists()) {
List<String> lines = FileUtils.readLines(testInformationFile, UTF_8);
Gson gson = new Gson();
for (String line : lines) {
testInformation.add(gson.fromJson(line, TestInformation.class));
}
}
return testInformation;
} catch (Exception e) {
LOGGER.warn(e.toString(), e);
}
return new ArrayList<>();
}

@VisibleForTesting
public static void dumpTestInformationToFile() {
public static void dumpTestInformationToFile(List<TestInformation> testInformationList) {
try {
if (executedTestsInformation.size() > 0) {
File testInformationFile = new File(getLocalVideosPath(), TEST_INFORMATION_FILE);
Gson gson = new GsonBuilder().setPrettyPrinting().create();
FileUtils.writeStringToFile(testInformationFile, gson.toJson(executedTestsInformation), UTF_8);
File testInformationFile = new File(getLocalVideosPath(), TEST_INFORMATION_FILE);
// Emptying the file first and then replacing it with what comes from testInformationList
FileUtils.writeStringToFile(testInformationFile, "", UTF_8);
Gson gson = new GsonBuilder().create();
for (TestInformation information : testInformationList) {
FileUtils.writeStringToFile(testInformationFile, gson.toJson(information) + System.lineSeparator(),
UTF_8, true);
}
} catch (Exception e) {
LOGGER.warn(e.toString(), e);
}
}

public static void setShutDownHook() {
if (!shutdownHookAdded.getAndSet(true)) {
try {
Runtime.getRuntime().addShutdownHook(new Thread(Dashboard::dumpTestInformationToFile, "Dashboard dumpTestInformationToFile shutdown hook"));
} catch (Exception e) {
LOGGER.warn(e.toString(), e);
}
private void saveTestInformation(TestInformation testInformation) {
try {
File testInformationFile = new File(getLocalVideosPath(), TEST_INFORMATION_FILE);
Gson gson = new GsonBuilder().create();
FileUtils.writeStringToFile(testInformationFile, gson.toJson(testInformation) + System.lineSeparator(),
UTF_8, true);
} catch (Exception e) {
LOGGER.warn(e.toString(), e);
}

}

@VisibleForTesting
Expand Down
5 changes: 3 additions & 2 deletions src/main/java/de/zalando/ep/zalenium/dashboard/DashboardInformationServlet.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,19 +10,20 @@
import java.io.IOException;
import java.util.List;

import static com.google.common.net.MediaType.*;
import static com.google.common.net.MediaType.JSON_UTF_8;
import static java.net.HttpURLConnection.HTTP_OK;
import static java.nio.charset.StandardCharsets.UTF_8;

public class DashboardInformationServlet extends HttpServlet {

@Override
protected void doGet(HttpServletRequest request, HttpServletResponse response) throws IOException {
process(request, response);
}

@SuppressWarnings("unused")
protected void process(HttpServletRequest request, HttpServletResponse response) throws IOException {
List<TestInformation> executedTestsInformation = Dashboard.getExecutedTestsInformation();
List<TestInformation> executedTestsInformation = Dashboard.loadTestInformationFromFile();
Gson gson = new GsonBuilder().setPrettyPrinting().create();
byte[] testInformation = gson.toJson(executedTestsInformation).getBytes(UTF_8);
response.setStatus(HTTP_OK);
Expand Down
6 changes: 2 additions & 4 deletions src/main/java/de/zalando/ep/zalenium/registry/ZaleniumRegistry.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@
import java.util.Optional;
import java.util.Random;
import java.util.Set;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.ReentrantLock;
Expand All @@ -62,7 +62,7 @@ public class ZaleniumRegistry extends BaseGridRegistry implements GridRegistry {
private final ActiveTestSessions activeTestSessions = new ActiveTestSessions();
private final NewSessionRequestQueue newSessionQueue;
private final Matcher matcherThread = new Matcher();
private final List<RemoteProxy> registeringProxies = new CopyOnWriteArrayList<>();
private final Set<RemoteProxy> registeringProxies = ConcurrentHashMap.newKeySet();
private volatile boolean stop = false;

private static final Environment defaultEnvironment = new Environment();
Expand Down Expand Up @@ -96,8 +96,6 @@ public ZaleniumRegistry(Hub hub) {
boolean waitForAvailableNodes = true;

DockeredSeleniumStarter starter = new DockeredSeleniumStarter();
Dashboard.loadTestInformationFromFile();
Dashboard.setShutDownHook();

AutoStartProxySet autoStart = new AutoStartProxySet(false, minContainers, maxContainers, timeToWaitToStart,
waitForAvailableNodes, starter, Clock.systemDefaultZone());
Expand Down
75 changes: 0 additions & 75 deletions src/test/java/de/zalando/ep/zalenium/dashboard/DashboardInformationServletTest.java
View file

This file was deleted.

11 changes: 9 additions & 2 deletions src/test/java/de/zalando/ep/zalenium/registry/ZaleniumRunningLocallyTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,15 @@ public void runLocally() {
gridHubConfiguration.registry = ZaleniumRegistry.class.getCanonicalName();
gridHubConfiguration.port = 4445;
Hub hub = new Hub(gridHubConfiguration);
// hub.start();
// Thread.sleep(1000 * 60 * 60);
/*
hub.start();
try {
Thread.sleep(1000 * 60 * 60);
} catch (InterruptedException e) {
e.printStackTrace();
}
*/

}

@After
Expand Down

0 comments on commit f19f4c1

Please sign in to comment.