From f19f4c16d2ba986b4e519daae4752cf066836742 Mon Sep 17 00:00:00 2001 From: Diego Molina Date: Tue, 27 Nov 2018 12:27:26 +0100 Subject: [PATCH] Saving test information directly to a file instead of keeping it in memory (#782) * Dumping test information to a file every 50 tests. * Reverting change in the dockerfile. * Saving test information directly to a file instead of keeping it in memory. --- docker/Dockerfile | 6 +- .../ep/zalenium/dashboard/Dashboard.java | 81 +++++++++---------- .../DashboardInformationServlet.java | 5 +- .../zalenium/registry/ZaleniumRegistry.java | 6 +- .../DashboardInformationServletTest.java | 75 ----------------- .../registry/ZaleniumRunningLocallyTest.java | 11 ++- 6 files changed, 56 insertions(+), 128 deletions(-) delete mode 100644 src/test/java/de/zalando/ep/zalenium/dashboard/DashboardInformationServletTest.java diff --git a/docker/Dockerfile b/docker/Dockerfile index 0111f0fa0a..79806a3e8e 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,10 +1,10 @@ #== Ubuntu xenial is 16.04, i.e. FROM ubuntu:16.04 # Find latest images at https://hub.docker.com/r/library/ubuntu/ # Layer size: big: ~130 MB -FROM ubuntu:xenial-20181005 -ENV UBUNTU_FLAVOR="xenial" \ - UBUNTU_DATE="20181005" +FROM ubuntu:xenial-20181113 +ENV UBUNTU_FLAVOR="xenial" \ + UBUNTU_DATE="20181113" ARG UBUNTU_MIRROR=http://archive.ubuntu.com/ubuntu #== Ubuntu flavors - common diff --git a/src/main/java/de/zalando/ep/zalenium/dashboard/Dashboard.java b/src/main/java/de/zalando/ep/zalenium/dashboard/Dashboard.java index e1a9123508..65a726b90e 100644 --- a/src/main/java/de/zalando/ep/zalenium/dashboard/Dashboard.java +++ b/src/main/java/de/zalando/ep/zalenium/dashboard/Dashboard.java @@ -5,7 +5,6 @@ import com.google.gson.GsonBuilder; import com.google.gson.JsonObject; import com.google.gson.JsonParser; -import com.google.gson.reflect.TypeToken; import de.zalando.ep.zalenium.util.CommonProxyUtilities; import de.zalando.ep.zalenium.util.Environment; @@ -13,12 +12,10 @@ import java.io.File; import java.io.IOException; -import java.lang.reflect.Type; import java.util.ArrayList; import java.util.Date; import java.util.List; import java.util.Map; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; import org.slf4j.Logger; @@ -48,24 +45,18 @@ public class Dashboard implements DashboardInterface { private static final String ZALENIUM_RETENTION_PERIOD = "ZALENIUM_RETENTION_PERIOD"; private static final int DEFAULT_RETENTION_PERIOD = 3; private static final Logger LOGGER = LoggerFactory.getLogger(Dashboard.class.getName()); - private static List executedTestsInformation = new ArrayList<>(); private static CommonProxyUtilities commonProxyUtilities = new CommonProxyUtilities(); private static final Environment defaultEnvironment = new Environment(); private static Environment env = defaultEnvironment; private static int executedTests = 0; private static int executedTestsWithVideo = 0; private static int retentionPeriod; - private static AtomicBoolean shutdownHookAdded = new AtomicBoolean(false); - + public Dashboard() { retentionPeriod = env.getIntEnvVariable(ZALENIUM_RETENTION_PERIOD, DEFAULT_RETENTION_PERIOD); } - public static List getExecutedTestsInformation() { - return executedTestsInformation; - } - public static String getCurrentLocalPath() { return commonProxyUtilities.currentLocalPath(); } @@ -155,21 +146,22 @@ public synchronized void updateDashboard(TestInformation testInformation) { if (!jsFolder.exists()) { FileUtils.copyDirectory(new File(getCurrentLocalPath() + JS_FOLDER), jsFolder); } - executedTestsInformation.add(testInformation); + saveTestInformation(testInformation); } catch (IOException e) { LOGGER.warn("Error while updating the dashboard.", e); } } - public synchronized void cleanupDashboard() throws IOException { - Map> partitioned = executedTestsInformation.stream() + public synchronized void cleanupDashboard() throws IOException { + List informationList = loadTestInformationFromFile(); + Map> partitioned = informationList.stream() .collect(Collectors.partitioningBy(testInformation -> testInformation.getRetentionDate().getTime() > new Date().getTime())); List validTestsInformation = partitioned.get(true); List invalidTestsInformation = partitioned.get(false); if(invalidTestsInformation.size() > 0) { - LOGGER.info("Cleaning up " + invalidTestsInformation.size() + " test from Dashboard"); + LOGGER.info("Cleaning up " + invalidTestsInformation.size() + " test(s) from Dashboard"); File testCountFile = new File(getLocalVideosPath(), TEST_COUNT_FILE); File dashboardHtml = new File(getLocalVideosPath(), DASHBOARD_FILE); File testList = new File(getLocalVideosPath(), TEST_LIST_FILE); @@ -177,10 +169,8 @@ public synchronized void cleanupDashboard() throws IOException { for(TestInformation testInformation : invalidTestsInformation) { deleteIfExists(new File(getLocalVideosPath() + "/" + testInformation.getFileName())); deleteIfExists(new File(testInformation.getLogsFolderPath())); - - executedTestsInformation.remove(testInformation); } - + deleteIfExists(dashboardHtml); deleteIfExists(testList); deleteIfExists(testCountFile); @@ -193,15 +183,17 @@ public synchronized void cleanupDashboard() throws IOException { for(TestInformation testInformation : validTestsInformation) { updateDashboard(testInformation); } + dumpTestInformationToFile(validTestsInformation); } } public synchronized void resetDashboard() throws IOException { - LOGGER.info("Reseting Dashboard"); + LOGGER.info("Resetting Dashboard"); File testList = new File(getLocalVideosPath(), TEST_LIST_FILE); File testCountFile = new File(getLocalVideosPath(), TEST_COUNT_FILE); File dashboardHtml = new File(getLocalVideosPath(), DASHBOARD_FILE); File logsFolder = new File(getLocalVideosPath(), LOGS_FOLDER_NAME); + File testInformationFile = new File(getLocalVideosPath(), TEST_INFORMATION_FILE); File videosFolder = new File(getLocalVideosPath()); String[] extensions = new String[] { "mp4", "mkv" }; for (File file : FileUtils.listFiles(videosFolder, extensions, true)) { @@ -211,12 +203,10 @@ public synchronized void resetDashboard() throws IOException { deleteIfExists(testList); deleteIfExists(testCountFile); deleteIfExists(dashboardHtml); + deleteIfExists(testInformationFile); String dashboard = FileUtils.readFileToString(new File(getCurrentLocalPath(), DASHBOARD_TEMPLATE_FILE), UTF_8); - dashboard = dashboard.replace("{testList}", ""). - replace("{executedTests}", "0"); + dashboard = dashboard.replace("{testList}", "").replace("{executedTests}", "0"); FileUtils.writeStringToFile(dashboardHtml, dashboard, UTF_8); - - executedTestsInformation = new ArrayList<>(); } public static void deleteIfExists(File file) { @@ -250,42 +240,49 @@ public static void synchronizeExecutedTestsValues(File testCountFile) { } @VisibleForTesting - public static void loadTestInformationFromFile() { + public static List loadTestInformationFromFile() { try { - if (executedTestsInformation.size() == 0) { - File testInformationFile = new File(getLocalVideosPath(), TEST_INFORMATION_FILE); - if (testInformationFile.exists()) { - String testInformationContents = FileUtils.readFileToString(testInformationFile, UTF_8); - Type collectionType = new TypeToken>(){}.getType(); - executedTestsInformation = new Gson().fromJson(testInformationContents, collectionType); + List testInformation = new ArrayList<>(); + File testInformationFile = new File(getLocalVideosPath(), TEST_INFORMATION_FILE); + if (testInformationFile.exists()) { + List lines = FileUtils.readLines(testInformationFile, UTF_8); + Gson gson = new Gson(); + for (String line : lines) { + testInformation.add(gson.fromJson(line, TestInformation.class)); } } + return testInformation; } catch (Exception e) { LOGGER.warn(e.toString(), e); } + return new ArrayList<>(); } - @VisibleForTesting - public static void dumpTestInformationToFile() { + public static void dumpTestInformationToFile(List testInformationList) { try { - if (executedTestsInformation.size() > 0) { - File testInformationFile = new File(getLocalVideosPath(), TEST_INFORMATION_FILE); - Gson gson = new GsonBuilder().setPrettyPrinting().create(); - FileUtils.writeStringToFile(testInformationFile, gson.toJson(executedTestsInformation), UTF_8); + File testInformationFile = new File(getLocalVideosPath(), TEST_INFORMATION_FILE); + // Emptying the file first and then replacing it with what comes from testInformationList + FileUtils.writeStringToFile(testInformationFile, "", UTF_8); + Gson gson = new GsonBuilder().create(); + for (TestInformation information : testInformationList) { + FileUtils.writeStringToFile(testInformationFile, gson.toJson(information) + System.lineSeparator(), + UTF_8, true); } } catch (Exception e) { LOGGER.warn(e.toString(), e); } } - public static void setShutDownHook() { - if (!shutdownHookAdded.getAndSet(true)) { - try { - Runtime.getRuntime().addShutdownHook(new Thread(Dashboard::dumpTestInformationToFile, "Dashboard dumpTestInformationToFile shutdown hook")); - } catch (Exception e) { - LOGGER.warn(e.toString(), e); - } + private void saveTestInformation(TestInformation testInformation) { + try { + File testInformationFile = new File(getLocalVideosPath(), TEST_INFORMATION_FILE); + Gson gson = new GsonBuilder().create(); + FileUtils.writeStringToFile(testInformationFile, gson.toJson(testInformation) + System.lineSeparator(), + UTF_8, true); + } catch (Exception e) { + LOGGER.warn(e.toString(), e); } + } @VisibleForTesting diff --git a/src/main/java/de/zalando/ep/zalenium/dashboard/DashboardInformationServlet.java b/src/main/java/de/zalando/ep/zalenium/dashboard/DashboardInformationServlet.java index f150d42933..1476bb8acb 100644 --- a/src/main/java/de/zalando/ep/zalenium/dashboard/DashboardInformationServlet.java +++ b/src/main/java/de/zalando/ep/zalenium/dashboard/DashboardInformationServlet.java @@ -10,11 +10,12 @@ import java.io.IOException; import java.util.List; -import static com.google.common.net.MediaType.*; +import static com.google.common.net.MediaType.JSON_UTF_8; import static java.net.HttpURLConnection.HTTP_OK; import static java.nio.charset.StandardCharsets.UTF_8; public class DashboardInformationServlet extends HttpServlet { + @Override protected void doGet(HttpServletRequest request, HttpServletResponse response) throws IOException { process(request, response); @@ -22,7 +23,7 @@ protected void doGet(HttpServletRequest request, HttpServletResponse response) t @SuppressWarnings("unused") protected void process(HttpServletRequest request, HttpServletResponse response) throws IOException { - List executedTestsInformation = Dashboard.getExecutedTestsInformation(); + List executedTestsInformation = Dashboard.loadTestInformationFromFile(); Gson gson = new GsonBuilder().setPrettyPrinting().create(); byte[] testInformation = gson.toJson(executedTestsInformation).getBytes(UTF_8); response.setStatus(HTTP_OK); diff --git a/src/main/java/de/zalando/ep/zalenium/registry/ZaleniumRegistry.java b/src/main/java/de/zalando/ep/zalenium/registry/ZaleniumRegistry.java index bccacde824..b87a02c8da 100644 --- a/src/main/java/de/zalando/ep/zalenium/registry/ZaleniumRegistry.java +++ b/src/main/java/de/zalando/ep/zalenium/registry/ZaleniumRegistry.java @@ -39,7 +39,7 @@ import java.util.Optional; import java.util.Random; import java.util.Set; -import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.ReentrantLock; @@ -62,7 +62,7 @@ public class ZaleniumRegistry extends BaseGridRegistry implements GridRegistry { private final ActiveTestSessions activeTestSessions = new ActiveTestSessions(); private final NewSessionRequestQueue newSessionQueue; private final Matcher matcherThread = new Matcher(); - private final List registeringProxies = new CopyOnWriteArrayList<>(); + private final Set registeringProxies = ConcurrentHashMap.newKeySet(); private volatile boolean stop = false; private static final Environment defaultEnvironment = new Environment(); @@ -96,8 +96,6 @@ public ZaleniumRegistry(Hub hub) { boolean waitForAvailableNodes = true; DockeredSeleniumStarter starter = new DockeredSeleniumStarter(); - Dashboard.loadTestInformationFromFile(); - Dashboard.setShutDownHook(); AutoStartProxySet autoStart = new AutoStartProxySet(false, minContainers, maxContainers, timeToWaitToStart, waitForAvailableNodes, starter, Clock.systemDefaultZone()); diff --git a/src/test/java/de/zalando/ep/zalenium/dashboard/DashboardInformationServletTest.java b/src/test/java/de/zalando/ep/zalenium/dashboard/DashboardInformationServletTest.java deleted file mode 100644 index 9f160a81f2..0000000000 --- a/src/test/java/de/zalando/ep/zalenium/dashboard/DashboardInformationServletTest.java +++ /dev/null @@ -1,75 +0,0 @@ -package de.zalando.ep.zalenium.dashboard; - -import com.google.common.reflect.TypeToken; -import com.google.gson.Gson; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - -import javax.servlet.ServletOutputStream; -import javax.servlet.WriteListener; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import java.io.IOException; -import java.lang.reflect.Type; -import java.util.ArrayList; -import java.util.List; - -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -public class DashboardInformationServletTest { - @Rule - public TemporaryFolder temporaryFolder = new TemporaryFolder(); - - private HttpServletRequest request; - private HttpServletResponse response; - private DashboardInformationServlet dashboardInformationServlet; - - @Before - public void initMocksAndService() throws IOException { - request = mock(HttpServletRequest.class); - response = mock(HttpServletResponse.class); - when(response.getOutputStream()).thenReturn(new ServletOutputStream() { - private StringBuilder stringBuilder = new StringBuilder(); - - @Override - public boolean isReady() { - System.out.println("isReady"); - return false; - } - - @Override - public void setWriteListener(WriteListener writeListener) { - System.out.println("setWriteListener"); - } - - @Override - public void write(int b) { - this.stringBuilder.append((char) b); - } - - public String toString() { - return stringBuilder.toString(); - } - }); - dashboardInformationServlet = new DashboardInformationServlet(); - } - - @Test - public void getReturnsCurrentTestInformationCount() throws IOException { - try { - dashboardInformationServlet.doGet(request, response); - String responseContents = response.getOutputStream().toString(); - Type collectionType = new TypeToken>(){}.getType(); - List testInformationList = new Gson().fromJson(responseContents, collectionType); - Assert.assertEquals(testInformationList.size(), Dashboard.getExecutedTestsInformation().size()); - - } finally { - Dashboard.restoreCommonProxyUtilities(); - } - } - -} diff --git a/src/test/java/de/zalando/ep/zalenium/registry/ZaleniumRunningLocallyTest.java b/src/test/java/de/zalando/ep/zalenium/registry/ZaleniumRunningLocallyTest.java index 4b574550dc..cbf262c496 100644 --- a/src/test/java/de/zalando/ep/zalenium/registry/ZaleniumRunningLocallyTest.java +++ b/src/test/java/de/zalando/ep/zalenium/registry/ZaleniumRunningLocallyTest.java @@ -25,8 +25,15 @@ public void runLocally() { gridHubConfiguration.registry = ZaleniumRegistry.class.getCanonicalName(); gridHubConfiguration.port = 4445; Hub hub = new Hub(gridHubConfiguration); - // hub.start(); - // Thread.sleep(1000 * 60 * 60); + /* + hub.start(); + try { + Thread.sleep(1000 * 60 * 60); + } catch (InterruptedException e) { + e.printStackTrace(); + } + */ + } @After