Merge pull request #1054 from elsloo/master

Add cache type to Traffic Stats and Traffic Monitor; fixes a config file path issue and reduces logging.

traffic_monitor/README.md

@@ -3,4 +3,19 @@
 ### Why Tests are not in exactly matching packages
 
 The "com.comcast.cdn.traffic_control.traffic_monitor" portion of the package name was omitted from unit
-tests to prevent improper referencing of package private fields and methods of the code under test.
+tests to prevent improper referencing of package private fields and methods of the code under test.
+
+### Running Traffic Monitor locally
+
+The "com.comcast.cdn.traffic_control.traffic_monitor.Start" class allows one to run Traffic Monitor
+locally provided that necessary configuration is in place. By default, the files are specified
+with paths relevant to certain IDEs, but these paths can be changed by specifying different
+properties via System.properties. These properties are:
+
+* traffic_monitor.path.config
+* traffic_monitor.path.db
+
+The first property refers to the location of traffic_monitor_config.js. The second property
+refers to the directory that will be used for certain data files that are downloaded at runtime.
+If you need to specify a different path, use the -D option to the Java command, or modify the
+paths in the Start class directly.

traffic_monitor/src/main/java/com/comcast/cdn/traffic_control/traffic_monitor/MonitorApplication.java

@@ -62,7 +62,7 @@ public void init() {
 		super.init();
 
 		if (!ConfigHandler.getInstance().configFileExists()) {
-			LOGGER.fatal("Cannot find configuration file: " + ConfigHandler.CONFIG_FILEPATH);
+			LOGGER.fatal("Cannot find configuration file: " + ConfigHandler.getInstance().getConfigFile());
 			// This will only stop Tomcat if the security manager allows it
 			// https://tomcat.apache.org/tomcat-6.0-doc/security-manager-howto.html
 			System.exit(1);

traffic_monitor/src/main/java/com/comcast/cdn/traffic_control/traffic_monitor/config/ConfigHandler.java

@@ -26,14 +26,16 @@
 
 public class ConfigHandler {
 	private static final Logger LOGGER = Logger.getLogger(ConfigHandler.class);
-	public static final String CONFIG_FILEPATH =   "/opt/traffic_monitor/conf/traffic_monitor_config.js";
-	public static final String DB_FILEPATH = "/opt/traffic_monitor/db/";
+	private static final String CONFIG_FILEPATH = "/opt/traffic_monitor/conf/traffic_monitor_config.js";
+	private static final String DB_FILEPATH = "/opt/traffic_monitor/db";
+	public static final String CONFIG_PROPERTY = "traffic_monitor.path.config";
+	public static final String DB_PROPERTY = "traffic_monitor.path.db";
 
 	private final Object lok = new Object();
 	private MonitorConfig config = null;
 	private boolean shutdown;
-	private final File configFile = new File(CONFIG_FILEPATH);
-	private final File dbDirectory = new File(DB_FILEPATH);
+	private final File configFile = new File(getFilePath(CONFIG_PROPERTY, CONFIG_FILEPATH));
+	private final String dbPath = getFilePath(DB_PROPERTY, DB_FILEPATH);
 
 	// Recommended Singleton Pattern implementation
 	// https://community.oracle.com/docs/DOC-918906
@@ -71,7 +73,7 @@ public MonitorConfig getConfig() {
 				final JSONObject jsonConfig = new JSONObject(IOUtils.toString(new FileReader(configFile)));
 				config = new MonitorConfig(jsonConfig.getJSONObject("traffic_monitor_config"));
 			} catch (FileNotFoundException e) {
-				LOGGER.error("Failed to find traffic monitor configuration file " + CONFIG_FILEPATH);
+				LOGGER.error("Failed to find traffic monitor configuration file " + configFile.toString());
 			} catch (Exception e) {
 				LOGGER.warn(e, e);
 			}
@@ -84,13 +86,23 @@ public MonitorConfig getConfig() {
 		return config;
 	}
 
-	public String getDbDir() {
-		synchronized (lok) {
-			return dbDirectory.exists() ? DB_FILEPATH : null;
-		}
+	public File getDbFile(final String filename) {
+		return new File(dbPath, filename);
+	}
+
+	public File getConfigFile() {
+		return configFile;
 	}
 
 	public boolean configFileExists() {
 		return configFile.exists();
 	}
+
+	private String getFilePath(final String property, final String staticFilePath) {
+		if (property != null && System.getProperty(property) != null) {
+			return System.getProperty(property);
+		}
+
+		return staticFilePath;
+	}
 }

traffic_monitor/src/main/java/com/comcast/cdn/traffic_control/traffic_monitor/config/RouterConfig.java

@@ -63,17 +63,17 @@ public RouterConfig(final JSONObject crConfigJson, final HealthDeterminer health
 				final Peer peer = new Peer(id, peers.optJSONObject(id));
 
 				if (Network.isIpAddressLocal(peer.getIpAddress())) {
-					LOGGER.warn("Skipping monitor " + id + "; IP address " + peer.getIpAddress() + " is local");
+					LOGGER.debug("Skipping monitor " + id + "; IP address " + peer.getIpAddress() + " is local");
 					continue;
 				}
 
 				if (Network.isLocalName(peer.getFqdn())) {
-					LOGGER.warn("Skipping monitor " + id + "; fqdn " + peer.getFqdn() + " is the local fully qualified name");
+					LOGGER.debug("Skipping monitor " + id + "; fqdn " + peer.getFqdn() + " is the local fully qualified name");
 					continue;
 				}
 
 				if (Network.isLocalName(id)) {
-					LOGGER.warn("Skipping monitor " + id + "; short name " + id + " is the local hostname");
+					LOGGER.debug("Skipping monitor " + id + "; short name " + id + " is the local hostname");
 					continue;
 				}
 

traffic_monitor/src/main/java/com/comcast/cdn/traffic_control/traffic_monitor/health/AbstractState.java

@@ -16,16 +16,21 @@
 
 package com.comcast.cdn.traffic_control.traffic_monitor.health;
 
+import java.util.ArrayList;
 import java.util.Deque;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
 import com.comcast.cdn.traffic_control.traffic_monitor.data.StatisticsLog;
+import com.comcast.cdn.traffic_control.traffic_monitor.health.Event.EventType;
+
 import org.apache.log4j.Logger;
 import org.apache.wicket.ajax.json.JSONArray;
 import org.apache.wicket.ajax.json.JSONException;
 import org.apache.wicket.ajax.json.JSONObject;
+import org.apache.wicket.util.string.Strings;
 
 import com.comcast.cdn.traffic_control.traffic_monitor.data.DataPoint;
 import com.comcast.cdn.traffic_control.traffic_monitor.data.DataSummary;
@@ -49,6 +54,10 @@ public String getId() {
 	}
 
 	protected void putDataPoints(final Map<String, String> statistics) {
+		if (statistics == null) {
+			return;
+		}
+
 		synchronized(this) {
 			for(String key : statistics.keySet()) {
 				putDataPoint(key, statistics.get(key));
@@ -137,19 +146,35 @@ public void setHistoryTime(final long t) {
 	}
 
 	public String getStatusString() {
-		String error = getLastValue("error-string");
-		if(error == null) {
-			error = isAvailable()? "available" : "";
+		final List<String> errors = new ArrayList<String>();
+		final String error = getLastValue("error-string");
+		final StringBuilder status = new StringBuilder();
+
+		status.append(getLastValue("status"));
+
+		if (error != null) {
+			errors.add(error);
+		} else {
+			errors.add(isAvailable() ? "available" : "unavailable");
 		}
-		if(getBool("clearData")) { error = "No query"; }
-		return getLastValue("status") + " - " + error;
+
+		if (getBool("clearData")) {
+			errors.add("monitoring disabled");
+		}
+
+		if (!errors.isEmpty()) {
+			status.append(" - ");
+			status.append(Strings.join(", ", errors));
+		}
+
+		return status.toString();
 	}
 
 	public boolean isError() {
 		return getLastValue("error-string") != null;
 	}
 
-	public void setAvailable(final boolean isAvailable, final String error) {
+	public void setAvailable(final EventType type, final boolean isAvailable, final String error) {
 		final boolean isHealthy = (error == null);
 		boolean logChange = true;
 
@@ -165,7 +190,7 @@ public void setAvailable(final boolean isAvailable, final String error) {
 		putDataPoint(IS_HEALTHY_STR, String.valueOf(isHealthy));
 
 		if (logChange) {
-			lastEvent = Event.logStateChange(this.getId(), isAvailable, getStatusString());
+			lastEvent = Event.logStateChange(this.getId(), type, isAvailable, getStatusString());
 		}
 	}
 

traffic_monitor/src/main/java/com/comcast/cdn/traffic_control/traffic_monitor/health/CacheWatcher.java

@@ -169,7 +169,7 @@ public void run() {
 					queryIntervalActual.set(completedTime - time);
 					queryIntervalDelta.set((completedTime - time) - config.getHealthPollingInterval());
 
-					LOGGER.warn("Check time of " + states.size() + " caches elapsed: " + mytime + " msec, (Active time was " + (completedTime - time) + ") msec, " + cancelCount.get() + " checks were cancelled, " + failCount.get() + " failed");
+					LOGGER.debug("Check time of " + states.size() + " caches elapsed: " + mytime + " msec, (Active time was " + (completedTime - time) + ") msec, " + cancelCount.get() + " checks were cancelled, " + failCount.get() + " failed");
 				} catch (Exception e) {
 					LOGGER.warn(e, e);
 

traffic_monitor/src/main/java/com/comcast/cdn/traffic_control/traffic_monitor/health/DsState.java

@@ -16,22 +16,25 @@
 
 package com.comcast.cdn.traffic_control.traffic_monitor.health;
 
+import java.util.Collection;
 import java.util.HashMap;
 import java.util.Map;
 
 import org.apache.wicket.ajax.json.JSONArray;
 import org.apache.wicket.ajax.json.JSONException;
 import org.apache.wicket.ajax.json.JSONObject;
 
+import com.comcast.cdn.traffic_control.traffic_monitor.health.EmbeddedStati.StatType;
+
 public class DsState extends AbstractState {
 	private static final long serialVersionUID = 1L;
+	final private Map<StatType, Map<String, EmbeddedStati>> aggregateStats = new HashMap<StatType, Map<String, EmbeddedStati>>();
+	final public static String DISABLED_LOCATIONS = "disabledLocations";
 
 	private DsStati currentDsStati;
 	private int cachesConfigured = 0;
 	private int cachesAvailable = 0;
 	private int cachesReporting = 0;
-	final private Map<String, EmbeddedStati> locs = new HashMap<String, EmbeddedStati>();
-	final private Map<String, EmbeddedStati> cacheStatiMap = new HashMap<String, EmbeddedStati>();
 
 	public DsState(final String id) {
 		super(id);
@@ -48,23 +51,23 @@ public void accumulate(final DsStati stati, final String location, final CacheSt
 			currentDsStati.accumulate(stati);
 		}
 
-		EmbeddedStati loc = locs.get(location);
+		aggregateStats(StatType.LOCATION, location, stati);
+		aggregateStats(StatType.CACHE, state.id, stati);
+		aggregateStats(StatType.TYPE, state.getCache().getType(), stati);
+	}
 
-		if (loc == null) {
-			loc = new EmbeddedStati("location", location);
-			locs.put(location,loc);
+	private void aggregateStats(final StatType statType, final String statKey, final DsStati dsStat) {
+		if (!aggregateStats.containsKey(statType)) {
+			aggregateStats.put(statType, new HashMap<String, EmbeddedStati>());
 		}
 
-		loc.accumulate(stati);
+		final Map<String, EmbeddedStati> aggregate = aggregateStats.get(statType);
 
-		EmbeddedStati cacheStati = cacheStatiMap.get(state.id);
-
-		if (cacheStati == null) {
-			cacheStati = new EmbeddedStati("cache", state.id);
-			cacheStatiMap.put(state.id, cacheStati);
+		if (!aggregate.containsKey(statKey)) {
+			aggregate.put(statKey, new EmbeddedStati(statType, statKey));
 		}
 
-		cacheStati.accumulate(stati);
+		aggregate.get(statKey).accumulate(dsStat);
 	}
 
 	public boolean completeRound(final JSONObject dsControls) {
@@ -83,43 +86,52 @@ public boolean completeRound(final JSONObject dsControls) {
 
 		HealthDeterminer.setIsAvailable(this, dsControls);
 
-		final StringBuilder sb = new StringBuilder();
+		for (Map<String, EmbeddedStati> aggregate : aggregateStats.values()) {
+			processDataPoints(aggregate.values(), dsControls);
+		}
+
+		return true;
+	}
+
+	private void processDataPoints(final Collection<EmbeddedStati> stats, final JSONObject dsControls) {
+		final Map<StatType, StringBuilder> disabled = new HashMap<StatType, StringBuilder>();
 
-		for (String locId : locs.keySet()) {
-			final EmbeddedStati loc = locs.get(locId);
-			final Map<String, String> stati = loc.completeRound();
+		for (EmbeddedStati stat : stats) {
+			final Map<String, String> points = stat.completeRound();
 
-			if (stati == null) {
+			if (points == null) {
 				continue;
 			}
 
-			putDataPoints(stati);
+			putDataPoints(points);
 
-			if (!HealthDeterminer.setIsAvailable(this, loc, dsControls)) {
-				sb.append("\"").append(locId).append("\", ");
+			if (stat.isHidden()) {
+				addHiddenStats(points.keySet());
 			}
-		}
-
-		putDataPoint("disabledLocations", sb.toString());
 
-		for (String cacheId : cacheStatiMap.keySet()) {
-			final EmbeddedStati cacheStat = cacheStatiMap.get(cacheId);
-			final Map<String, String> stati = cacheStat.completeRound();
+			if (stat.getStatType() == StatType.LOCATION) {
+				if (!disabled.containsKey(stat.getStatType())) {
+					disabled.put(stat.getStatType(), new StringBuilder());
+				}
 
-			if (stati == null) {
-				continue;
+				if (!HealthDeterminer.setIsLocationAvailable(this, stat, dsControls)) {
+					disabled.get(stat.getStatType()).append("\"").append(stat.getId()).append("\", ");
+				}
 			}
+		}
 
-			addHiddenStats(stati.keySet());
+		for (StatType statType : disabled.keySet()) {
+			final StringBuilder sb = disabled.get(statType);
 
-			putDataPoints(stati);
+			if (statType == StatType.LOCATION && sb != null) {
+				final String s = sb.toString();
+				putDataPoint(DISABLED_LOCATIONS, s);
+			}
 		}
-
-		return true;
 	}
 
 	public JSONArray getDisabledLocations() throws JSONException {
-		return new JSONArray("["+this.getLastValue("disabledLocations")+"]");
+		return new JSONArray("["+this.getLastValue(DISABLED_LOCATIONS)+"]");
 	}
 
 	public void addCacheConfigured() {

traffic_monitor/src/main/java/com/comcast/cdn/traffic_control/traffic_monitor/health/EmbeddedStati.java

@@ -7,22 +7,27 @@ public class EmbeddedStati implements java.io.Serializable {
 	private static final long serialVersionUID = 1L;
 	private DsStati currentDtati;
 	private final String id;
+	private StatType statType;
 
-	public EmbeddedStati(final String base, final String id, final String delimiter) {
-		final StringBuilder statId = new StringBuilder();
+	public enum StatType {
+		LOCATION,
+		CACHE,
+		TYPE
+	};
 
-		if (base != null) {
-			statId.append(base);
-			statId.append(delimiter);
-		}
+	public EmbeddedStati(final StatType statType, final String id, final String delimiter) {
+		final StringBuilder statId = new StringBuilder();
 
+		statId.append(statType.toString().toLowerCase());
+		statId.append(delimiter);
 		statId.append(id);
 
 		this.id = statId.toString();
+		this.statType = statType;
 	}
 
-	public EmbeddedStati(final String base, final String id) {
-		this(base, id, ".");
+	public EmbeddedStati(final StatType statType, final String id) {
+		this(statType, id, ".");
 	}
 
 	public void accumulate(final DsStati stati) {
@@ -49,4 +54,12 @@ public Map<String, String> completeRound() {
 	public String getId() {
 		return id;
 	}
+
+	public boolean isHidden() {
+		return (statType == StatType.CACHE) ? true : false;
+	}
+
+	public StatType getStatType() {
+		return statType;
+	}
 }