-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathplugin.py
179 lines (145 loc) · 6.53 KB
/
plugin.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
from __future__ import (absolute_import, division, print_function, unicode_literals)
import logging
import datetime
import time
import sys
import urllib2
import urlparse
import json
import boundary_plugin
import boundary_accumulator
"""
If getting statistics fails, we will retry up to this number of times before
giving up and aborting the plugin. Use 0 for unlimited retries.
"""
PLUGIN_RETRY_COUNT = 0
"""
If getting statistics fails, we will wait this long (in seconds) before retrying.
"""
PLUGIN_RETRY_DELAY = 5
"""
We have multiple endpoints for retriving the different status
"""
SYSTEM_ENDPOINT='admin/info/system?wt=json'
THREAD_ENDPOINT='admin/info/threads?wt=json'
MBEANS_ENDPOINT='admin/mbeans?stats=true&wt=json&json.nl=map'
SYSTEM_KEY_MAPPING = (
(["mode"], "SOLR_RUN_MODE", False),
(["system", "committedVirtualMemorySize"], "SOLR_SYSTEM_COMMITED_VIRTUAL_MEMORY_SIZE", False),
(["system", "freePhysicalMemorySize"], "SOLR_SYSTEM_FREE_PHYSICAL_MEMORY_SIZE", False),
(["system", "processCpuTime"], "SOLR_SYSTEM_PROCESS_CPU_TIME", False),
(["system", "openFileDescriptorCount"], "SOLR_SYSTEM_OPEN_FILE_DESCRIPTOR_COUNT", False),
(["system", "maxFileDescriptorCount"], "SOLR_SYSTEM_MAX_FILE_DESCRIPTOR_COUNT", False),
(["jvm", "jmx", "upTimeMS"], "SOLR_JVM_UPTIME", False),
(["jvm", "processors"], "SOLR_JVM_PROCESSORS", False),
(["jvm", "memory", "raw", "free"], "SOLR_JVM_MEMORY_FREE", False),
(["jvm", "memory", "raw", "total"], "SOLR_JVM_MEMORY_TOTAL", False),
(["jvm", "memory", "raw", "max"], "SOLR_JVM_MEMORY_MAX", False),
(["jvm", "memory", "raw", "used"], "SOLR_JVM_MEMORY_USED", False)
)
THREAD_KEY_MAPPING = (
(["system", "threadCount", "current"], "SOLR_THREAD_CURRENT", False),
(["system", "threadCount", "peak"], "SOLR_THREAD_PEAK", False),
(["system", "threadCount", "daemon"], "SOLR_THREAD_DAEMON", False)
)
MBEANS_KEY_MAPPING = (
(["solr-mbeans", "CACHE", "documentCache", "stats", "lookups"], "SOLR_CACHE_DOCUMENT_LOOKUPS", False),
(["solr-mbeans", "CACHE", "documentCache", "stats", "hits"], "SOLR_CACHE_DOCUMENT_HITS", False),
(["solr-mbeans", "CACHE", "documentCache", "stats", "hitratio"], "SOLR_CACHE_DOCUMENT_HITRATIO", False),
(["solr-mbeans", "CACHE", "documentCache", "stats", "inserts"], "SOLR_CACHE_DOCUMENT_INSERTS", False),
(["solr-mbeans", "CACHE", "documentCache", "stats", "size"], "SOLR_CACHE_DOCUMENT_SIZE", False),
(["solr-mbeans", "CACHE", "documentCache", "stats", "evictions"], "SOLR_CACHE_DOCUMENT_EVICTIONS", False),
(["solr-mbeans", "CACHE", "documentCache", "stats", "warmupTime"], "SOLR_CACHE_DOCUMENT_WARMUPTIME", False),
)
class SolrPlugin(object):
def __init__(self, boundary_metric_prefix):
self.boundary_metric_prefix = boundary_metric_prefix
self.settings = boundary_plugin.parse_params()
self.accumulator = boundary_accumulator
self.base_url = self.settings.get("base_url", "http://localhost:8983/solr/")
def get_stats(self):
mbeans_endpoint_with_core = MBEANS_ENDPOINT
system = self.get_raw_data(SYSTEM_ENDPOINT)
threads = self.get_raw_data(THREAD_ENDPOINT)
if self.core:
mbeans_endpoint_with_core = self.core + '/' + MBEANS_ENDPOINT
mbeans = self.get_raw_data(mbeans_endpoint_with_core)
return {'system': system, 'threads': threads, 'mbeans': mbeans}
def get_raw_data(self, endpoint):
req = urllib2.urlopen(urlparse.urljoin(self.base_url, endpoint))
res = req.read()
req.close()
data = json.loads(res)
return data
def get_stats_with_retries(self, *args, **kwargs):
"""
Calls the get_stats function, taking into account retry configuration.
"""
retry_range = xrange(PLUGIN_RETRY_COUNT) if PLUGIN_RETRY_COUNT > 0 else iter(int, 1)
for _ in retry_range:
try:
return self.get_stats(*args, **kwargs)
except Exception as e:
logging.error("Error retrieving data: %s" % e)
time.sleep(PLUGIN_RETRY_DELAY)
logging.fatal("Max retries exceeded retrieving data")
raise Exception("Max retries exceeded retrieving data")
def handle_metric_for_system(self, data):
for metric_path, boundary_name, accumulate in SYSTEM_KEY_MAPPING:
value = data
try:
for p in metric_path:
value = value[p]
except KeyError:
value = None
if not value:
continue
if accumulate:
value = self.accumulator.accumulate(metric_path, value)
boundary_plugin.boundary_report_metric(self.boundary_metric_prefix + boundary_name, value)
def handle_metric_for_threads(self, data):
for metric_path, boundary_name, accumulate in THREAD_KEY_MAPPING:
value = data
try:
for p in metric_path:
value = value[p]
except KeyError:
value = None
if not value:
continue
if accumulate:
value = self.accumulator.accumulate(metric_path, value)
boundary_plugin.boundary_report_metric(self.boundary_metric_prefix + boundary_name, value)
def handle_metric_for_caches(self, data):
for metric_path, boundary_name, accumulate in MBEANS_KEY_MAPPING:
value = data
try:
for p in metric_path:
value = value[p]
except KeyError:
value = None
if not value:
continue
if accumulate:
value = self.accumulator.accumulate(metric_path, value)
boundary_plugin.boundary_report_metric(self.boundary_metric_prefix + boundary_name, value)
def handle_metrics(self, data):
self.handle_metric_for_system(data['system'])
self.handle_metric_for_threads(data['threads'])
self.handle_metric_for_caches(data['mbeans'])
def main(self):
logging.basicConfig(level=logging.ERROR, filename=self.settings.get('log_file', None))
reports_log = self.settings.get('report_log_file', None)
if reports_log:
boundary_plugin.log_metrics_to_file(reports_log)
self.core = self.settings.get('core_name', None)
boundary_plugin.start_keepalive_subprocess()
while True:
data = self.get_stats_with_retries()
self.handle_metrics(data)
boundary_plugin.sleep_interval()
if __name__ == '__main__':
if len(sys.argv) > 1 and sys.argv[1] == '-v':
logging.basicConfig(level=logging.INFO)
plugin = SolrPlugin('')
plugin.main()