From 1591df5d233832a7a89b86eb36817b817d38cc25 Mon Sep 17 00:00:00 2001 From: Olivier Vielpeau Date: Thu, 11 Jun 2015 14:33:26 -0400 Subject: [PATCH] Support regexes on domain and bean names --- .../java/org/datadog/jmxfetch/Filter.java | 47 +++++++++- .../org/datadog/jmxfetch/JMXAttribute.java | 90 ++++++++++++------- .../java/org/datadog/jmxfetch/TestApp.java | 86 +++++++++++++++++- src/test/resources/jmx_domain_regex.yaml | 14 +++ .../jmx_list_beans_regex_exclude.yaml | 18 ++++ .../jmx_list_beans_regex_include.yaml | 17 ++++ 6 files changed, 238 insertions(+), 34 deletions(-) create mode 100644 src/test/resources/jmx_domain_regex.yaml create mode 100644 src/test/resources/jmx_list_beans_regex_exclude.yaml create mode 100644 src/test/resources/jmx_list_beans_regex_include.yaml diff --git a/src/main/java/org/datadog/jmxfetch/Filter.java b/src/main/java/org/datadog/jmxfetch/Filter.java index f3524b05a..6ace5b43c 100644 --- a/src/main/java/org/datadog/jmxfetch/Filter.java +++ b/src/main/java/org/datadog/jmxfetch/Filter.java @@ -4,16 +4,19 @@ import java.util.ArrayList; import java.util.Set; import java.lang.ClassCastException; +import java.util.regex.Pattern; class Filter { LinkedHashMap filter; + Pattern domainRegex; + ArrayList beanRegexes = null; /** * A simple class to manipulate include/exclude filter elements more easily * A filter may contain: - * - A domain (key: 'domain') - * - Bean names (key: 'bean' or 'bean_name') + * - A domain (key: 'domain') or a domain regex (key: 'domain_regex') + * - Bean names (key: 'bean' or 'bean_name') or bean regexes (key: 'bean_regex') * - Attributes (key: 'attribute') * - Additional bean parameters (other keys) */ @@ -23,7 +26,7 @@ public Filter(Object filter) { LinkedHashMap castFilter; if (filter != null) { castFilter = (LinkedHashMap) filter; - } else{ + } else { castFilter = new LinkedHashMap(); } this.filter = castFilter; @@ -78,10 +81,48 @@ public ArrayList getBeanNames() { return toStringArrayList(beanNames); } + private static ArrayList toPatternArrayList(final Object toCast) { + ArrayList patternArrayList = new ArrayList(); + ArrayList stringArrayList = toStringArrayList(toCast); + for (String string : stringArrayList) { + patternArrayList.add(Pattern.compile(string)); + } + + return patternArrayList; + } + + public ArrayList getBeanRegexes() { + // Return bean regexes as an ArrayList of Pattern whether it's defined as + // a list or not + + if (this.beanRegexes == null) { + if (filter.get("bean_regex") == null){ + this.beanRegexes = new ArrayList(); + } else { + final Object beanRegexNames = filter.get("bean_regex"); + this.beanRegexes = toPatternArrayList(beanRegexNames); + } + } + + return this.beanRegexes; + } + public String getDomain() { return (String) filter.get("domain"); } + public Pattern getDomainRegex() { + if (this.filter.get("domain_regex") == null) { + return null; + } + + if (this.domainRegex == null) { + this.domainRegex = Pattern.compile((String) this.filter.get("domain_regex")); + } + + return this.domainRegex; + } + public Object getAttribute() { return filter.get("attribute"); } diff --git a/src/main/java/org/datadog/jmxfetch/JMXAttribute.java b/src/main/java/org/datadog/jmxfetch/JMXAttribute.java index 860dce57e..3374b5dad 100644 --- a/src/main/java/org/datadog/jmxfetch/JMXAttribute.java +++ b/src/main/java/org/datadog/jmxfetch/JMXAttribute.java @@ -10,6 +10,7 @@ import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; +import java.util.regex.Pattern; import javax.management.AttributeNotFoundException; import javax.management.InstanceNotFoundException; @@ -23,7 +24,7 @@ public abstract class JMXAttribute { private final static Logger LOGGER = Logger.getLogger(Instance.class.getName()); - private static final List EXCLUDED_BEAN_PARAMS = Arrays.asList("domain", "bean_name", "bean", "attribute"); + private static final List EXCLUDED_BEAN_PARAMS = Arrays.asList("domain", "domain_regex", "bean_name", "bean", "bean_regex", "attribute"); private static final String FIRST_CAP_PATTERN = "(.)([A-Z][a-z]+)"; private static final String ALL_CAP_PATTERN = "([a-z0-9])([A-Z])"; private static final String METRIC_REPLACEMENT = "([^a-zA-Z0-9_.]+)|(^[^a-zA-Z]+)"; @@ -149,39 +150,18 @@ Object getJmxValue() throws AttributeNotFoundException, InstanceNotFoundExceptio boolean matchDomain(Configuration conf) { String includeDomain = conf.getInclude().getDomain(); - return includeDomain == null || includeDomain.equals(domain); + Pattern includeDomainRegex = conf.getInclude().getDomainRegex(); + + return (includeDomain == null || includeDomain.equals(domain)) + && (includeDomainRegex == null || includeDomainRegex.matcher(domain).matches()); } boolean excludeMatchDomain(Configuration conf) { String excludeDomain = conf.getExclude().getDomain(); - return excludeDomain != null && excludeDomain.equals(domain); - } + Pattern excludeDomainRegex = conf.getExclude().getDomainRegex(); - boolean excludeMatchBean(Configuration conf) { - Filter exclude = conf.getExclude(); - ArrayList beanNames = exclude.getBeanNames(); - - if(beanNames.contains(beanName)){ - return true; - } - - for (String bean_attr : exclude.keySet()) { - if (EXCLUDED_BEAN_PARAMS.contains(bean_attr)) { - continue; - } - - if (beanParameters.get(bean_attr) == null) { - continue; - } - - ArrayList beanValues = exclude.getParameterValues(bean_attr); - for (String beanVal : beanValues) { - if (beanParameters.get(bean_attr).equals(beanVal)) { - return true; - } - } - } - return false; + return excludeDomain != null && excludeDomain.equals(domain) + || excludeDomainRegex != null && excludeDomainRegex.matcher(domain).matches(); } Object convertMetricValue(Object metricValue) { @@ -227,7 +207,22 @@ Object convertMetricValue(Object metricValue) { } } - boolean matchBean(Configuration configuration) { + private boolean matchBeanRegex(Filter filter, boolean matchIfNoRegex) { + ArrayList beanRegexes = filter.getBeanRegexes(); + if (beanRegexes.isEmpty()) { + return matchIfNoRegex; + } + + for (Pattern beanRegex : beanRegexes) { + if(beanRegex.matcher(beanName).matches()) { + return true; + } + } + + return false; + } + + private boolean matchBeanName(Configuration configuration) { boolean matchBeanAttr = true; Filter include = configuration.getInclude(); @@ -261,6 +256,41 @@ boolean matchBean(Configuration configuration) { return matchBeanAttr; } + private boolean excludeMatchBeanName(Configuration conf) { + Filter exclude = conf.getExclude(); + ArrayList beanNames = exclude.getBeanNames(); + + if(beanNames.contains(beanName)){ + return true; + } + + for (String bean_attr : exclude.keySet()) { + if (EXCLUDED_BEAN_PARAMS.contains(bean_attr)) { + continue; + } + + if (beanParameters.get(bean_attr) == null) { + continue; + } + + ArrayList beanValues = exclude.getParameterValues(bean_attr); + for (String beanVal : beanValues) { + if (beanParameters.get(bean_attr).equals(beanVal)) { + return true; + } + } + } + return false; + } + + boolean matchBean(Configuration configuration) { + return matchBeanName(configuration) && matchBeanRegex(configuration.getInclude(), true); + } + + boolean excludeMatchBean(Configuration configuration) { + return excludeMatchBeanName(configuration) || matchBeanRegex(configuration.getExclude(), false); + } + @SuppressWarnings("unchecked") HashMap getValueConversions() { if (valueConversions == null) { diff --git a/src/test/java/org/datadog/jmxfetch/TestApp.java b/src/test/java/org/datadog/jmxfetch/TestApp.java index 738bfe35a..1c2e85e9d 100644 --- a/src/test/java/org/datadog/jmxfetch/TestApp.java +++ b/src/test/java/org/datadog/jmxfetch/TestApp.java @@ -3,7 +3,6 @@ import com.beust.jcommander.JCommander; import org.apache.log4j.Level; -import org.datadog.jmxfetch.Status; import org.datadog.jmxfetch.reporter.Reporter; import org.datadog.jmxfetch.reporter.ConsoleReporter; import org.datadog.jmxfetch.util.CustomLogger; @@ -135,6 +134,35 @@ public void testDomainExclude() throws Exception { mbs.unregisterMBean(excludeMe); } + @Test + public void testDomainRegex() throws Exception { + // We expose a few metrics through JMX + MBeanServer mbs = ManagementFactory.getPlatformMBeanServer(); + + ObjectName includeObjectName1 = new ObjectName("org.datadog.jmxfetch.includeme:type=AType"); + ObjectName includeObjectName2 = new ObjectName("org.datadog.jmxfetch.includeme.too:type=AType"); + ObjectName excludeObjectName = new ObjectName("org.datadog.jmxfetch.includeme.not.me:type=AType"); + SimpleTestJavaApp testApp = new SimpleTestJavaApp(); + mbs.registerMBean(testApp, includeObjectName1); + mbs.registerMBean(testApp, includeObjectName2); + mbs.registerMBean(testApp, excludeObjectName); + + // Initializing application + AppConfig appConfig = new AppConfig(); + App app = initApp("jmx_domain_regex.yaml", appConfig); + + // Collecting metrics + app.doIteration(); + LinkedList> metrics = ((ConsoleReporter) appConfig.getReporter()).getMetrics(); + + // First filter 15 = 13 metrics from java.lang + 3 metrics explicitly defined - 1 implicitly defined in exclude section + assertEquals(15, metrics.size()); + + mbs.unregisterMBean(includeObjectName1); + mbs.unregisterMBean(includeObjectName2); + mbs.unregisterMBean(excludeObjectName); + } + @Test public void testParameterMatch() throws Exception { // Do not match beans which do not contain types specified in the conf @@ -224,6 +252,62 @@ public void testListBeansInclude() throws Exception { mbs.unregisterMBean(includeMe); } + @Test + public void testListBeansRegexInclude() throws Exception { + // We expose a few metrics through JMX + MBeanServer mbs = ManagementFactory.getPlatformMBeanServer(); + ObjectName includeMe = new ObjectName("org.datadog.jmxfetch.test:type=IncludeMe"); + ObjectName includeMeToo = new ObjectName("org.datadog.jmxfetch.test:type=IncludeMeToo"); + ObjectName notIncludeMe = new ObjectName("org.datadog.jmxfetch.test:type=RightType"); + SimpleTestJavaApp testApp = new SimpleTestJavaApp(); + mbs.registerMBean(testApp, includeMe); + mbs.registerMBean(testApp, includeMeToo); + mbs.registerMBean(testApp, notIncludeMe); + + // Initializing application + AppConfig appConfig = new AppConfig(); + App app = initApp("jmx_list_beans_regex_include.yaml", appConfig); + + // Collecting metrics + app.doIteration(); + LinkedList> metrics = ((ConsoleReporter) appConfig.getReporter()).getMetrics(); + + // First filter 15 = 13 metrics from java.lang + 2 metrics explicitly defined + assertEquals(15, metrics.size()); + + mbs.unregisterMBean(includeMe); + mbs.unregisterMBean(includeMeToo); + mbs.unregisterMBean(notIncludeMe); + } + + @Test + public void testListBeansRegexExclude() throws Exception { + // We expose a few metrics through JMX + MBeanServer mbs = ManagementFactory.getPlatformMBeanServer(); + ObjectName includeMe = new ObjectName("org.datadog.jmxfetch.test:type=IncludeMe"); + ObjectName excludeMe = new ObjectName("org.datadog.jmxfetch.test:type=ExcludeMe,scope=InScope"); + ObjectName excludeMeToo = new ObjectName("org.datadog.jmxfetch.test:scope=OutOfScope"); + SimpleTestJavaApp testApp = new SimpleTestJavaApp(); + mbs.registerMBean(testApp, includeMe); + mbs.registerMBean(testApp, excludeMe); + mbs.registerMBean(testApp, excludeMeToo); + + // Initializing application + AppConfig appConfig = new AppConfig(); + App app = initApp("jmx_list_beans_regex_exclude.yaml", appConfig); + + // Collecting metrics + app.doIteration(); + LinkedList> metrics = ((ConsoleReporter) appConfig.getReporter()).getMetrics(); + + // First filter 14 = 13 metrics from java.lang + 1 metrics explicitly defined + assertEquals(14, metrics.size()); + + mbs.unregisterMBean(includeMe); + mbs.unregisterMBean(excludeMe); + mbs.unregisterMBean(excludeMeToo); + } + @Test public void testListBeansExclude() throws Exception { // We expose a few metrics through JMX diff --git a/src/test/resources/jmx_domain_regex.yaml b/src/test/resources/jmx_domain_regex.yaml new file mode 100644 index 000000000..4bee8bc08 --- /dev/null +++ b/src/test/resources/jmx_domain_regex.yaml @@ -0,0 +1,14 @@ +init_config: + +instances: + - process_name_regex: .*surefire.* + name: jmx_test_instance + conf: + - include: + domain_regex: .*includeme.* + attribute: + ShouldBe100: + metric_type: gauge + alias: this.is.100 + exclude: + domain_regex: .*\.me$ diff --git a/src/test/resources/jmx_list_beans_regex_exclude.yaml b/src/test/resources/jmx_list_beans_regex_exclude.yaml new file mode 100644 index 000000000..2335b98f7 --- /dev/null +++ b/src/test/resources/jmx_list_beans_regex_exclude.yaml @@ -0,0 +1,18 @@ +init_config: + +instances: + - process_name_regex: .*surefire.* + name: jmx_test_instance + tags: + env: stage + newTag: test + conf: + - include: + attribute: + ShouldBe100: + metric_type: gauge + alias: this.is.100 + exclude: + bean_regex: + - .*[,:]type=ExcludeMe.* + - .*[,:]scope=Out.* diff --git a/src/test/resources/jmx_list_beans_regex_include.yaml b/src/test/resources/jmx_list_beans_regex_include.yaml new file mode 100644 index 000000000..8b1a0200b --- /dev/null +++ b/src/test/resources/jmx_list_beans_regex_include.yaml @@ -0,0 +1,17 @@ +init_config: + +instances: + - process_name_regex: .*surefire.* + name: jmx_test_instance + tags: + env: stage + newTag: test + conf: + - include: + bean_regex: + - .*type=\w*WrongType + - .*type=Include.* + attribute: + ShouldBe100: + metric_type: gauge + alias: this.is.100