From 2de8252921452853b01c6841ef636017c8ae0c67 Mon Sep 17 00:00:00 2001 From: YevhenLukomskyi Date: Thu, 3 Jan 2019 22:09:20 +0200 Subject: [PATCH 01/49] set package long description --- MANIFEST.in | 1 + setup.py | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/MANIFEST.in b/MANIFEST.in index bd94f4e0d..18ccc8e90 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,5 @@ include LICENSE +include README.md include MANIFEST.in include check-dependencies.py include examples/* diff --git a/setup.py b/setup.py index 8e4ee9805..bf9ac8c62 100644 --- a/setup.py +++ b/setup.py @@ -71,6 +71,10 @@ conf_files = [ ('conf', glob('conf/*.example')) ] examples = [ ('examples', glob('examples/example-*')) ] +def read(fname): + with open(os.path.join(os.path.dirname(__file__), fname)) as f: + return f.read() + try: setup( name='graphite-web', @@ -80,6 +84,8 @@ author_email='chrismd@gmail.com', license='Apache Software License 2.0', description='Enterprise scalable realtime graphing', + long_description=read('README.md'), + long_description_content_type='text/markdown', package_dir={'' : 'webapp'}, packages=[ 'graphite', From c06c324d98ef854815ceffb4491ece7af0337aea Mon Sep 17 00:00:00 2001 From: Pierce Lopez Date: Tue, 5 Feb 2019 20:33:20 -0500 Subject: [PATCH 02/49] fix dashboard graph metric list icon paths with URL_PREFIX use global *_ICON vars like the other image assets --- webapp/content/js/dashboard.js | 8 ++++---- webapp/graphite/templates/dashboard.html | 10 ++++------ 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/webapp/content/js/dashboard.js b/webapp/content/js/dashboard.js index 6c981ddb7..66d8331dc 100644 --- a/webapp/content/js/dashboard.js +++ b/webapp/content/js/dashboard.js @@ -4,7 +4,7 @@ /*global AUTOCOMPLETE_DELAY CALENDAR_ICON CLOCK_ICON CONTEXT_FIELD_WIDTH FINDER_QUERY_DELAY*/ /*global HELP_ICON NEW_DASHBOARD_REMOVE_GRAPHS REFRESH_ICON REMOVE_ICON RESIZE_ICON*/ /*global SHARE_ICON UI_CONFIG initialState initialError permissions queryString userName*/ -/*global permissionsUnauthenticated schemes*/ +/*global UP_ICON DOWN_ICON TRASH_ICON permissionsUnauthenticated schemes*/ // Defined in composer_widgets.js /*global createFunctionsMenu createOptionsMenu updateCheckItems*/ @@ -1916,7 +1916,7 @@ function graphClicked(graphView, graphIndex, element, evt) { width: 30, sortable: false, items: [{ - icon: '/static/img/move_up.png', + icon: UP_ICON, tooltip: 'Move Up', handler: function(grid, rowIndex, colIndex) { var record = targetStore.getAt(rowIndex); @@ -1935,7 +1935,7 @@ function graphClicked(graphView, graphIndex, element, evt) { width: 30, sortable: false, items: [{ - icon: '/static/img/move_down.png', + icon: DOWN_ICON, tooltip: 'Move Down', handler: function(grid, rowIndex, colIndex) { var record = targetStore.getAt(rowIndex); @@ -1954,7 +1954,7 @@ function graphClicked(graphView, graphIndex, element, evt) { width: 30, sortable: false, items: [{ - icon: '/static/img/trash.png', + icon: TRASH_ICON, tooltip: 'Delete Row', handler: function(grid, rowIndex, colIndex) { var record = targetStore.getAt(rowIndex); diff --git a/webapp/graphite/templates/dashboard.html b/webapp/graphite/templates/dashboard.html index 688a3fde2..8a9a15b36 100644 --- a/webapp/graphite/templates/dashboard.html +++ b/webapp/graphite/templates/dashboard.html @@ -19,14 +19,12 @@ var REMOVE_ICON = '{% static "js/ext/resources/icons/fam/cross.gif" %}'; var REFRESH_ICON = '{% static "js/ext/resources/icons/fam/table_refresh.png" %}'; var SHARE_ICON = '{% static "js/ext/resources/icons/fam/application_go.png" %}'; + var HELP_ICON = '{% static "js/ext/resources/icons/fam/information.png" %}'; var CALENDAR_ICON = '{% static "js/ext/resources/images/default/shared/calendar.gif" %}'; var CLOCK_ICON = '{% static "img/clock_16.png" %}'; - var HELP_ICON = '{% static "js/ext/resources/icons/fam/information.png" %}'; - - // Prefetch images for use in Target grid - (new Image()).src = '{% static "img/move_up.png" %}'; - (new Image()).src = '{% static "img/move_down.png" %}'; - (new Image()).src = '{% static "img/trash.png" %}'; + var UP_ICON = '{% static "img/move_up.png" %}'; + var DOWN_ICON = '{% static "img/move_down.png" %}'; + var TRASH_ICON = '{% static "img/trash.png" %}'; {% if initialState %} var initialState = JSON.parse('{{ initialState|escapejs }}'); From 67d0d341cc87b875bd9032b2aeee94c22a55d39b Mon Sep 17 00:00:00 2001 From: Pierce Lopez Date: Thu, 7 Feb 2019 03:19:49 -0500 Subject: [PATCH 03/49] docs: for sql db migration to 1.1 recommend --fake-initial ... and --run-syncdb should no longer be needed. Initial migrations were created for all models just before 1.1.0 in d1912d4bc0f0. Prior to that, some models did not have any migrations (and no initial migration), so --run-syncdb was needed to create tables for them. But now that they have initial migrations, those migrations run because django knows they have never been applied, and they fail because the tables already exist. The --fake-initial option is designed for this situation: it considers initial migrations already done if the tables already exist. --- docs/config-database-setup.rst | 5 ++++- docs/config-local-settings.rst | 2 +- docs/releases/1_1_1.rst | 4 +++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/docs/config-database-setup.rst b/docs/config-database-setup.rst index 87b309fcb..18ec781ff 100644 --- a/docs/config-database-setup.rst +++ b/docs/config-database-setup.rst @@ -20,7 +20,10 @@ To set up a new database and create the initial schema, run: .. code-block:: none - PYTHONPATH=$GRAPHITE_ROOT/webapp django-admin.py migrate --settings=graphite.settings --run-syncdb + PYTHONPATH=$GRAPHITE_ROOT/webapp django-admin.py migrate --settings=graphite.settings + +.. note :: + Graphite-Web 1.0 and earlier had some models without migrations, and with Django 1.9 or later, the ``--run-syncdb`` option was needed for migrate to create tables for these models. (Django 1.8 and earlier did not have this option, but always exhibited this behavior.) In Graphite-Web 1.1 and later all models have migrations, so ``--run-syncdb`` is no longer needed. If upgrading a database created by Graphite-Web 1.0 or earlier, you need to use the ``--fake-initial`` option for migrate: it considers an initial migration to already be applied if the tables it creates already exist. If you are experiencing problems, uncomment the following line in /opt/graphite/webapp/graphite/local_settings.py: diff --git a/docs/config-local-settings.rst b/docs/config-local-settings.rst index 4af89463e..aa1b6928f 100644 --- a/docs/config-local-settings.rst +++ b/docs/config-local-settings.rst @@ -373,7 +373,7 @@ The following configures the Django database settings. Graphite uses the databas See the `Django documentation `_ for full documentation of the DATABASES setting. .. note :: - Remember, setting up a new database requires running ``PYTHONPATH=$GRAPHITE_ROOT/webapp django-admin.py migrate --settings=graphite.settings --run-syncdb`` to create the initial schema. + Remember, setting up a new database requires running ``PYTHONPATH=$GRAPHITE_ROOT/webapp django-admin.py migrate --settings=graphite.settings`` to create the initial schema. .. note :: If you are using a custom database backend (other than SQLite) you must first create a $GRAPHITE_ROOT/webapp/graphite/local_settings.py file that overrides the database related settings from settings.py. Use $GRAPHITE_ROOT/webapp/graphite/local_settings.py.example as a template. diff --git a/docs/releases/1_1_1.rst b/docs/releases/1_1_1.rst index 31eae4cc8..66a1393fc 100644 --- a/docs/releases/1_1_1.rst +++ b/docs/releases/1_1_1.rst @@ -176,7 +176,9 @@ If you're not already running from the *master* branch, Graphite-Web's applicati sudo cp /opt/graphite/storage/graphite.db \ /opt/graphite/storage/graphite.db.backup-`date +%Y%m%d_%H%M%S` sudo PYTHONPATH=/opt/graphite/webapp django-admin.py migrate \ - --noinput --settings=graphite.settings --run-syncdb + --noinput --settings=graphite.settings --fake-initial + +In this release of Graphite-Web, migrations have been added for any Django models that did not have them. Previously, if using Django 1.9 or later, one needed the ``--run-syncdb`` option for migrate to create tables for Graphite-Web models without migrations (and Django 1.8 or earlier did not have this option but always exhibited this behavior). Django keeps track of which migrations have been applied and attempts to run any that have not, and these new initial migrations try to create tables that already exist from previous versions of Graphite-Web, and fail. This common Django situation is resolved by the ``--fake-initial`` option for migrate: it considers an initial migration to already be applied if the tables it creates already exist. Other Changes From f242d5de56f6ca9857bf4574bbc85d309c2a480a Mon Sep 17 00:00:00 2001 From: Mauro Stettler Date: Tue, 12 Feb 2019 17:31:33 -0300 Subject: [PATCH 04/49] add tag formatting docs --- docs/tags.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/tags.rst b/docs/tags.rst index 647b2943f..bdbfa9f87 100644 --- a/docs/tags.rst +++ b/docs/tags.rst @@ -17,6 +17,8 @@ Carbon will automatically decode the tags, normalize the tag order, and register .. _querying-tagged-series: +Tag names must have a length of ``>=1`` and they may contain any Ascii characters, except ``;!^=``. Tag values must also have a length of ``>=1``, and they may contain any Ascii characters, except ``;~``. UTF-8 characters may work for names and values, but they are not well tested and it is not recommendable to use UTF-8 anywhere in metric names or tags. + Querying -------- @@ -88,6 +90,8 @@ Finally, the `aliasByTags # web01.disk.used # web02.disk.used +If a tag name or value contains quotes (``'"``), then they will need to be escaped properly. For example a series with a tag ``tagName='quotedValue'`` could be queried with ``seriesByTag('tagName=\'quotedValue\'')`` or alternatively ``seriesByTag("tagName='quotedValue'")``. + Database Storage ---------------- As Whisper and other storage backends are designed to hold simple time-series data (metric key, value, and timestamp), Graphite stores tag information in a separate tag database (TagDB). The TagDB is a pluggable store, by default it uses the Graphite SQLite, MySQL or PostgreSQL database, but it can also be configured to use an external Redis server or a custom plugin. From c24533022bd80a958dcb0497234372bf45dd02d0 Mon Sep 17 00:00:00 2001 From: Dan Cech Date: Tue, 12 Feb 2019 15:55:50 -0500 Subject: [PATCH 05/49] Update tags.rst --- docs/tags.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tags.rst b/docs/tags.rst index bdbfa9f87..f8ca9af7f 100644 --- a/docs/tags.rst +++ b/docs/tags.rst @@ -17,7 +17,7 @@ Carbon will automatically decode the tags, normalize the tag order, and register .. _querying-tagged-series: -Tag names must have a length of ``>=1`` and they may contain any Ascii characters, except ``;!^=``. Tag values must also have a length of ``>=1``, and they may contain any Ascii characters, except ``;~``. UTF-8 characters may work for names and values, but they are not well tested and it is not recommendable to use UTF-8 anywhere in metric names or tags. +Tag names must have a length >= 1 and may contain any ascii characters except ``;!^=``. Tag values must also have a length >= 1, and they may contain any ascii characters except ``;~``. UTF-8 characters may work for names and values, but they are not well tested and it is not recommended to use non-ascii characters in metric names or tags. Querying -------- From 0ed6a1f0edb8d88e66c9f772997bc64c206b645b Mon Sep 17 00:00:00 2001 From: Mauro Stettler Date: Tue, 12 Feb 2019 19:51:21 -0300 Subject: [PATCH 06/49] add tag character validator --- webapp/graphite/tags/utils.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/webapp/graphite/tags/utils.py b/webapp/graphite/tags/utils.py index 37854b8e7..632cd753f 100644 --- a/webapp/graphite/tags/utils.py +++ b/webapp/graphite/tags/utils.py @@ -5,6 +5,22 @@ class TaggedSeries(object): + prohibitedTagChars = ';!^=' + prohibitedValueChars = ';~' + + @classmethod + def validateCharacters(cls, tag, value): + """validate that there are no prohibited characters in given tag/value""" + for char in cls.prohibitedTagChars: + if char in tag: + return False + + for char in cls.prohibitedValueChars: + if char in value: + return False + + return True + @classmethod def parse(cls, path): # if path is in openmetrics format: metric{tag="value",...} @@ -31,7 +47,13 @@ def parse_openmetrics(cls, path): if not m: raise Exception('Cannot parse path %s, invalid segment %s' % (path, rawtags)) - tags[m.group(1)] = m.group(2).replace(r'\"', '"').replace(r'\\', '\\') + tag = m.group(1) + value = m.group(2).replace(r'\"', '"').replace(r'\\', '\\') + + if not cls.validateCharacters(tag, value): + raise Exception('Tag/Value contains invalid characters: %s/%s' % (tag, value)) + + tags[tag] = value rawtags = rawtags[len(m.group(0)):] tags['name'] = metric @@ -53,6 +75,9 @@ def parse_carbon(cls, path): if len(tag) != 2 or not tag[0]: raise Exception('Cannot parse path %s, invalid segment %s' % (path, segment)) + if not cls.validateCharacters(*tag): + raise Exception('Tag/Value contains invalid characters: %s/%s' % (tag[0], tag[1])) + tags[tag[0]] = tag[1] tags['name'] = metric From c263a8258650a128918acb6dc880b426c910d3d7 Mon Sep 17 00:00:00 2001 From: Christopher Bowman Date: Thu, 28 Mar 2019 06:53:22 -0400 Subject: [PATCH 07/49] Correct this split This never worked right, but the backend dealt with it until recent changes. This only happens when the dashboard template is loaded directly from the URL --- webapp/content/js/dashboard.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webapp/content/js/dashboard.js b/webapp/content/js/dashboard.js index 66d8331dc..e270daf5b 100644 --- a/webapp/content/js/dashboard.js +++ b/webapp/content/js/dashboard.js @@ -792,7 +792,7 @@ function initDashboard () { if(window.location.hash != '') { if (window.location.hash.indexOf('/') != -1) { - var nameVal = window.location.hash.substr(1).split('#'); + var nameVal = window.location.hash.substr(1).split('/'); sendLoadTemplateRequest(nameVal[0],nameVal[1]); } else { sendLoadRequest(window.location.hash.substr(1)); From 1b0dc68cd6f90269194329f5986e080a2e738361 Mon Sep 17 00:00:00 2001 From: Christopher Bowman Date: Thu, 28 Mar 2019 15:23:29 -0400 Subject: [PATCH 08/49] Add document.body.dataset.baseUrl to these calls --- webapp/content/js/dashboard.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/webapp/content/js/dashboard.js b/webapp/content/js/dashboard.js index e270daf5b..1be7c640b 100644 --- a/webapp/content/js/dashboard.js +++ b/webapp/content/js/dashboard.js @@ -1400,7 +1400,7 @@ function newEmptyGraph() { var record = new GraphRecord({ target: graphTargetString, params: myParams, - url: '/render?' + Ext.urlEncode(urlParams), + url: document.body.dataset.baseUrl + 'render?' + Ext.urlEncode(urlParams), 'width': GraphSize.width, 'height': GraphSize.height, }); @@ -1566,7 +1566,7 @@ function newFromMetric() { var record = new GraphRecord({ target: graphTargetString, params: myParams, - url: '/render?' + Ext.urlEncode(urlParams) + url: document.body.dataset.baseUrl + 'render?' + Ext.urlEncode(urlParams) }); graphStore.add([record]); updateGraphRecords(); From 25f6041da68a7eb280460eda0f14e4b49737b69e Mon Sep 17 00:00:00 2001 From: Edmunt Pienkowsky Date: Wed, 3 Apr 2019 09:49:25 +0200 Subject: [PATCH 09/49] Accept IPv6 addresses in CARBONLINK_HOSTS Using slightly modified functions util.parseDestination(s) from Carbon project. --- webapp/graphite/carbonlink.py | 17 +++-------------- webapp/graphite/util.py | 24 ++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 14 deletions(-) diff --git a/webapp/graphite/carbonlink.py b/webapp/graphite/carbonlink.py index e50d2ec3d..378d0402f 100644 --- a/webapp/graphite/carbonlink.py +++ b/webapp/graphite/carbonlink.py @@ -7,7 +7,7 @@ from graphite.render.hashing import ConsistentHashRing from graphite.logger import log -from graphite.util import load_module, unpickle +from graphite.util import load_module, unpickle, parseHosts from graphite.singleton import ThreadSafeSingleton @@ -83,10 +83,8 @@ def get_connection(self, host): pass #nothing left in the pool, gotta make a new connection log.cache("CarbonLink creating a new socket for %s" % str(host)) - connection = socket.socket() - connection.settimeout(self.timeout) try: - connection.connect((server, port)) + connection = socket.create_connection((server, port), self.timeout) except socket.error: self.last_failure[host] = time.time() raise @@ -193,16 +191,7 @@ def recv_exactly(conn, num_bytes): @ThreadSafeSingleton class GlobalCarbonLinkPool(CarbonLinkPool): def __init__(self): - hosts = [] - for host in settings.CARBONLINK_HOSTS: - parts = host.split(':') - server = parts[0] - port = int(parts[1]) - if len(parts) > 2: - instance = parts[2] - else: - instance = None - hosts.append((server, int(port), instance)) + hosts = parseHosts(settings.CARBONLINK_HOSTS) timeout = settings.CARBONLINK_TIMEOUT CarbonLinkPool.__init__(self, hosts, timeout) diff --git a/webapp/graphite/util.py b/webapp/graphite/util.py index 3adaa6edd..09cd31703 100644 --- a/webapp/graphite/util.py +++ b/webapp/graphite/util.py @@ -366,3 +366,27 @@ def _jsonResponse(data, queryParams, status=200, encoder=None, default=None): def _jsonError(message, queryParams, status=500, encoder=None, default=None): return _jsonResponse( {'error': message}, queryParams, status=status, encoder=encoder, default=default) + +def parseHost(host_string): + s = host_string.strip() + bidx = s.rfind(']:') # find closing bracket and following colon. + cidx = s.find(':') + if s.startswith('[') and bidx is not None: + server = s[1:bidx] + port = s[bidx + 2:] + elif cidx is not None: + server = s[:cidx] + port = s[cidx + 1:] + else: + raise ValueError("Invalid host string \"%s\"" % host_string) + + if ':' in port: + port, _, instance = port.partition(':') + else: + instance = None + + return server, int(port), instance + + +def parseHosts(host_strings): + return [parseHost(host_string) for host_string in host_strings] From baed068b5a56ef6c44b777b8f6d3bd3f81702d40 Mon Sep 17 00:00:00 2001 From: Dan Cech Date: Wed, 3 Apr 2019 08:35:33 -0400 Subject: [PATCH 10/49] lint fix --- webapp/graphite/util.py | 1 + 1 file changed, 1 insertion(+) diff --git a/webapp/graphite/util.py b/webapp/graphite/util.py index 09cd31703..4882bf9c0 100644 --- a/webapp/graphite/util.py +++ b/webapp/graphite/util.py @@ -367,6 +367,7 @@ def _jsonError(message, queryParams, status=500, encoder=None, default=None): return _jsonResponse( {'error': message}, queryParams, status=status, encoder=encoder, default=default) + def parseHost(host_string): s = host_string.strip() bidx = s.rfind(']:') # find closing bracket and following colon. From 2c434070393064f5bda85f31d55cd4c3e98a0119 Mon Sep 17 00:00:00 2001 From: 0xflotus <0xflotus@gmail.com> Date: Tue, 23 Apr 2019 17:39:56 +0200 Subject: [PATCH 11/49] fixed developed --- docs/ceres.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ceres.rst b/docs/ceres.rst index cc3fb6cf2..1ec57980f 100644 --- a/docs/ceres.rst +++ b/docs/ceres.rst @@ -6,7 +6,7 @@ for Graphite. In contrast with Whisper, Ceres is not a fixed-size database and i better support sparse data of arbitrary fixed-size resolutions. This allows Graphite to distribute individual time-series across multiple servers or mounts. -Ceres is not actively developped at the moment. For alternatives to whisper look at :doc:`alternative storage backends `. +Ceres is not actively developed at the moment. For alternatives to whisper look at :doc:`alternative storage backends `. Storage Overview ---------------- From dcb0c3b484c2b9e869cc0eeeffcfdf73aa460f94 Mon Sep 17 00:00:00 2001 From: 0xflotus <0xflotus@gmail.com> Date: Tue, 23 Apr 2019 17:42:21 +0200 Subject: [PATCH 12/49] Update render_api.rst --- docs/render_api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/render_api.rst b/docs/render_api.rst index 171882205..2db88c2ac 100644 --- a/docs/render_api.rst +++ b/docs/render_api.rst @@ -531,7 +531,7 @@ darkgray 111,111,111 darkgrey 111,111,111 ============ ============= -RGB can be passed directly in the format #RRGGBB[AA] where RR, GG, and BB are 2-digit hex vaules for red, green and blue, respectively. AA is an optional addition describing the opacity ("alpha"). Where FF is fully opaque, 00 fully transparent. +RGB can be passed directly in the format #RRGGBB[AA] where RR, GG, and BB are 2-digit hex values for red, green and blue, respectively. AA is an optional addition describing the opacity ("alpha"). Where FF is fully opaque, 00 fully transparent. Examples: From f8ad3e6419597a6aaf5d17bbc3dc961b28954680 Mon Sep 17 00:00:00 2001 From: Christopher Bowman Date: Fri, 24 May 2019 11:37:42 -0400 Subject: [PATCH 13/49] Copy requestContext() and empty prefetch so we don't override prior fetched data --- webapp/graphite/render/functions.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/webapp/graphite/render/functions.py b/webapp/graphite/render/functions.py index 9497e4ff5..f1e08a872 100644 --- a/webapp/graphite/render/functions.py +++ b/webapp/graphite/render/functions.py @@ -3515,7 +3515,10 @@ def useSeriesAbove(requestContext, seriesList, value, search, replace): if not newNames: return [] - newSeries = evaluateTarget(requestContext, 'group(%s)' % ','.join(newNames)) + newContext = requestContext.copy() + newContext['prefetched'] = {} + + newSeries = evaluateTarget(newContext, 'group(%s)' % ','.join(newNames)) return [n for n in newSeries if n is not None and len(n) > 0] From 75c1921e4bfede431788e70ec690f6626bebc7dc Mon Sep 17 00:00:00 2001 From: Christopher Bowman Date: Fri, 24 May 2019 11:49:40 -0400 Subject: [PATCH 14/49] Change all evaluateTarget() calls to include prefetch-less requestContext --- webapp/graphite/render/functions.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/webapp/graphite/render/functions.py b/webapp/graphite/render/functions.py index f1e08a872..370e5e305 100644 --- a/webapp/graphite/render/functions.py +++ b/webapp/graphite/render/functions.py @@ -1239,6 +1239,7 @@ def movingWindow(requestContext, seriesList, windowSize, func='average', xFilesF # ignore original data and pull new, including our preview # data from earlier is needed to calculate the early results newContext = requestContext.copy() + newContext['prefetch'] = {} newContext['startTime'] = requestContext['startTime'] - timedelta(seconds=previewSeconds) previewList = evaluateTarget(newContext, requestContext['args'][0]) result = [] @@ -1324,6 +1325,7 @@ def exponentialMovingAverage(requestContext, seriesList, windowSize): # ignore original data and pull new, including our preview # data from earlier is needed to calculate the early results newContext = requestContext.copy() + newContext['prefetch'] = {} newContext['startTime'] = requestContext['startTime'] - timedelta(seconds=previewSeconds) previewList = evaluateTarget(newContext, requestContext['args'][0]) result = [] @@ -2375,7 +2377,9 @@ def aliasQuery(requestContext, seriesList, search, replace, newName): """ for series in seriesList: newQuery = re.sub(search, replace, series.name) - newSeriesList = evaluateTarget(requestContext, newQuery) + newContext = requestContext.copy() + newContext['prefetch'] = {} + newSeriesList = evaluateTarget(newContext, newQuery) if newSeriesList is None or len(newSeriesList) == 0: raise Exception('No series found with query: ' + newQuery) current = safeLast(newSeriesList[0]) @@ -3517,7 +3521,6 @@ def useSeriesAbove(requestContext, seriesList, value, search, replace): newContext = requestContext.copy() newContext['prefetched'] = {} - newSeries = evaluateTarget(newContext, 'group(%s)' % ','.join(newNames)) return [n for n in newSeries if n is not None and len(n) > 0] @@ -3813,6 +3816,7 @@ def holtWintersForecast(requestContext, seriesList, bootstrapInterval='7d', seas # ignore original data and pull new, including our preview newContext = requestContext.copy() + newContext['prefetch'] = {} newContext['startTime'] = requestContext['startTime'] - timedelta(seconds=previewSeconds) previewList = evaluateTarget(newContext, requestContext['args'][0]) results = [] @@ -3847,6 +3851,7 @@ def holtWintersConfidenceBands(requestContext, seriesList, delta=3, bootstrapInt # ignore original data and pull new, including our preview newContext = requestContext.copy() + newContext['prefetch'] = {} newContext['startTime'] = requestContext['startTime'] - timedelta(seconds=previewSeconds) previewList = evaluateTarget(newContext, requestContext['args'][0]) results = [] @@ -4007,6 +4012,7 @@ def linearRegression(requestContext, seriesList, startSourceAt=None, endSourceAt """ results = [] sourceContext = requestContext.copy() + sourceContext['prefetch'] = {} if startSourceAt is not None: sourceContext['startTime'] = parseATTime(startSourceAt) if endSourceAt is not None: sourceContext['endTime'] = parseATTime(endSourceAt) @@ -4161,6 +4167,7 @@ def timeStack(requestContext, seriesList, timeShiftUnit='1d', timeShiftStart=0, for shft in range(timeShiftStartint,timeShiftEndint): myContext = requestContext.copy() + myContext['prefetch'] = {} innerDelta = delta * shft myContext['startTime'] = requestContext['startTime'] + innerDelta myContext['endTime'] = requestContext['endTime'] + innerDelta @@ -4221,6 +4228,7 @@ def timeShift(requestContext, seriesList, timeShift, resetEnd=True, alignDST=Fal timeShift = '-' + timeShift delta = parseTimeOffset(timeShift) myContext = requestContext.copy() + myContext['prefetch'] = {} myContext['startTime'] = requestContext['startTime'] + delta myContext['endTime'] = requestContext['endTime'] + delta @@ -4805,8 +4813,10 @@ def applyByNode(requestContext, seriesList, nodeNum, templateFunction, newName=N prefix = '.'.join(series.name.split('.')[:nodeNum + 1]) prefixes.add(prefix) results = [] + newContext = requestContext.copy() + newContext['prefetch'] = {} for prefix in sorted(prefixes): - for resultSeries in evaluateTarget(requestContext, templateFunction.replace('%', prefix)): + for resultSeries in evaluateTarget(newContext, templateFunction.replace('%', prefix)): if newName: resultSeries.name = newName.replace('%', prefix) resultSeries.pathExpression = prefix @@ -4972,6 +4982,7 @@ def smartSummarize(requestContext, seriesList, intervalString, func='sum', align if alignTo is not None: alignToUnit = getUnitString(alignTo) requestContext = requestContext.copy() + requestContext['prefetch'] = {} s = requestContext['startTime'] if alignToUnit == YEARS_STRING: requestContext['startTime'] = datetime(s.year, 1, 1, tzinfo = s.tzinfo) @@ -5142,6 +5153,7 @@ def hitcount(requestContext, seriesList, intervalString, alignToInterval = False if alignToInterval: requestContext = requestContext.copy() + requestContext['prefetch'] = {} s = requestContext['startTime'] if interval >= DAY: requestContext['startTime'] = datetime(s.year, s.month, s.day, tzinfo = s.tzinfo) From 3140c1143af63bca8dd5cb7ab92d9973a158c626 Mon Sep 17 00:00:00 2001 From: Dieter Plaetinck Date: Mon, 27 May 2019 12:03:11 +0300 Subject: [PATCH 15/49] update aggregation function docs for aggregate and groupbytags --- webapp/graphite/render/functions.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/webapp/graphite/render/functions.py b/webapp/graphite/render/functions.py index 370e5e305..9c35712d2 100644 --- a/webapp/graphite/render/functions.py +++ b/webapp/graphite/render/functions.py @@ -307,8 +307,9 @@ def aggregate(requestContext, seriesList, func, xFilesFactor=None): &target=sumSeries(host.cpu-[0-7].cpu-{user,system}.value) - This function can be used with aggregation functions ``average``, ``median``, ``sum``, ``min``, - ``max``, ``diff``, ``stddev``, ``count``, ``range``, ``multiply`` & ``last``. + This function can be used with aggregation functions ``average`` (or ``avg``), ``avg_zero``, + ``median``, ``sum`` (or ``total``), ``min``, ``max``, ``diff``, ``stddev``, ``count``, + ``range`` (or ``rangeOf``) , ``multiply`` & ``last`` (or ``current``). """ # strip Series from func if func was passed like sumSeries rawFunc = func @@ -5457,8 +5458,9 @@ def groupByTags(requestContext, seriesList, callback, *tags): averageSeries(seriesByTag("name=cpu","dc=dc1")),averageSeries(seriesByTag("name=cpu","dc=dc2")),... This function can be used with all aggregation functions supported by - :py:func:`aggregate `: ``average``, ``median``, ``sum``, ``min``, ``max``, ``diff``, - ``stddev``, ``range`` & ``multiply``. + :py:func:`aggregate `: ``average`` (or ``avg``), ``avg_zero``, + ``median``, ``sum`` (or ``total``), ``min``, ``max``, ``diff``, ``stddev``, ``count``, + ``range`` (or ``rangeOf``) , ``multiply`` & ``last`` (or ``current``). """ if STORE.tagdb is None: log.info('groupByTags called but no TagDB configured') From 59f78a8ebb9b38cd9e89f761b828b416ea52028b Mon Sep 17 00:00:00 2001 From: Daniel Ziegler Date: Tue, 4 Jun 2019 08:06:25 +0200 Subject: [PATCH 16/49] Add Statusengine to list of integrations (Forwarding) --- docs/tools.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/tools.rst b/docs/tools.rst index cb86d00a7..2c2d22b7f 100644 --- a/docs/tools.rst +++ b/docs/tools.rst @@ -125,6 +125,9 @@ Forwarding `statsd`_ A simple daemon for easy stats aggregation, developed by the folks at Etsy. A list of forks and alternative implementations can be found at +`_Statusengine`_ + A daemon written in PHP to store Nagios and Naemon performance data to Graphite. + Visualization ------------- @@ -399,6 +402,7 @@ Other .. _SqlToGraphite: https://github.com/perryofpeek/SqlToGraphite .. _SSC Serv: https://ssc-serv.com .. _statsd: https://github.com/etsy/statsd +.. _Statusengine: https://github.com/statusengine/worker .. _Tasseo: https://github.com/obfuscurity/tasseo .. _Targets-io: https://github.com/dmoll1974/targets-io .. _telegraf: https://github.com/influxdata/telegraf From 462ad5b8dff7d56e0a4cb2033360952bdfe962d3 Mon Sep 17 00:00:00 2001 From: nook24 Date: Tue, 4 Jun 2019 10:43:13 +0000 Subject: [PATCH 17/49] Resolve failing test --- docs/tools.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tools.rst b/docs/tools.rst index 2c2d22b7f..c82f542db 100644 --- a/docs/tools.rst +++ b/docs/tools.rst @@ -125,7 +125,7 @@ Forwarding `statsd`_ A simple daemon for easy stats aggregation, developed by the folks at Etsy. A list of forks and alternative implementations can be found at -`_Statusengine`_ +`Statusengine`_ A daemon written in PHP to store Nagios and Naemon performance data to Graphite. Visualization From 62ef0e3c1bacb58bd1e02724e8f6c5abcb07ae13 Mon Sep 17 00:00:00 2001 From: Piotr Date: Thu, 4 Jul 2019 23:39:45 +0200 Subject: [PATCH 18/49] Resolving conflicts in tox.ini --- .travis.yml | 16 ++++++---------- requirements.txt | 2 +- setup.py | 2 +- tox.ini | 9 +++------ webapp/graphite/app_settings.py | 1 + 5 files changed, 12 insertions(+), 18 deletions(-) diff --git a/.travis.yml b/.travis.yml index c1ed08a20..5afb211f1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,38 +9,35 @@ matrix: - python: pypy env: - TOXENV=pypy-django111-pyparsing2 - - python: 3.4 - env: - - TOXENV=py34-django20-pyparsing2 - python: 3.5 env: - TOXENV=py35-django21-pyparsing2 - python: 3.5 env: - - TOXENV=py35-django21-pyparsing2 + - TOXENV=py35-django22-pyparsing2 - python: 3.6 env: - - TOXENV=py36-django21-pyparsing2 + - TOXENV=py36-django22-pyparsing2 - python: 3.7 sudo: true dist: xenial env: - - TOXENV=py37-django21-pyparsing2-msgpack + - TOXENV=py37-django22-pyparsing2-msgpack - python: 3.7 sudo: true dist: xenial env: - - TOXENV=py37-django21-pyparsing2-pyhash + - TOXENV=py37-django22-pyparsing2-pyhash - python: 3.7 sudo: true dist: xenial env: - - TOXENV=py37-django21-pyparsing2-mysql TEST_MYSQL_PASSWORD=graphite + - TOXENV=py37-django22-pyparsing2-mysql TEST_MYSQL_PASSWORD=graphite - python: 3.7 sudo: true dist: xenial env: - - TOXENV=py37-django21-pyparsing2-postgresql TEST_POSTGRESQL_PASSWORD=graphite + - TOXENV=py37-django22-pyparsing2-postgresql TEST_POSTGRESQL_PASSWORD=graphite - python: 3.7 sudo: true dist: xenial @@ -48,7 +45,6 @@ matrix: - TOXENV=lint env: - - TOXENV=py27-django18-pyparsing2 - TOXENV=py27-django111-pyparsing2-msgpack - TOXENV=py27-django111-pyparsing2-pyhash - TOXENV=py27-django111-pyparsing2-mysql TEST_MYSQL_PASSWORD=graphite diff --git a/requirements.txt b/requirements.txt index 56f7560c2..63f74f315 100644 --- a/requirements.txt +++ b/requirements.txt @@ -37,7 +37,7 @@ # deactivate # -Django>=1.8,<2.1.99 +Django>=1.8,<2.3 python-memcached==1.58 txAMQP==0.8 django-tagging==0.4.6 diff --git a/setup.py b/setup.py index bf9ac8c62..5d59fb6f9 100644 --- a/setup.py +++ b/setup.py @@ -115,7 +115,7 @@ def read(fname): ['templates/*', 'local_settings.py.example']}, scripts=glob('bin/*'), data_files=list(webapp_content.items()) + storage_dirs + conf_files + examples, - install_requires=['Django>=1.8,<2.1', 'django-tagging==0.4.3', 'pytz', 'pyparsing', 'cairocffi', 'urllib3', 'scandir', 'six'], + install_requires=['Django>=1.8,<2.3', 'django-tagging==0.4.3', 'pytz', 'pyparsing', 'cairocffi', 'urllib3', 'scandir', 'six'], classifiers=[ 'Intended Audience :: Developers', 'Natural Language :: English', diff --git a/tox.ini b/tox.ini index b06adf085..f8ee6e59e 100644 --- a/tox.ini +++ b/tox.ini @@ -1,11 +1,7 @@ [tox] -# Django 21 does not support Python 3.4 -# Django 1.11 has a bug and does not work with Python 3.7 (this will be fixed) envlist = - py{27,py}-django1{8,11}-pyparsing2{,-rrdtool,-msgpack,-pyhash}, - py{34}-django{18,111,20}-pyparsing2{,-rrdtool,-msgpack,-pyhash}, - py{35,36}-django{18,111,20,21}-pyparsing2{,-rrdtool,-msgpack,-pyhash}, - py37-django{20,21}-pyparsing2{,-rrdtool,-msgpack,-pyhash}, + py{27,py}-django111-pyparsing2{,-msgpack,-pyhash}, + py{35,36,37,38}-django{111,21,22}-pyparsing2{,-msgpack,-pyhash}, lint, docs [testenv] @@ -38,6 +34,7 @@ deps = django111: Django>=1.11,<1.11.99 django20: Django>=2.0,<2.0.99 django21: Django>=2.1,<2.1.99 + django22: Django>=2.2,<2.2.99 scandir urllib3 redis diff --git a/webapp/graphite/app_settings.py b/webapp/graphite/app_settings.py index 9b39c990f..a037aa896 100644 --- a/webapp/graphite/app_settings.py +++ b/webapp/graphite/app_settings.py @@ -109,6 +109,7 @@ 'django.contrib.sessions', 'django.contrib.admin', 'django.contrib.contenttypes', + 'django.contrib.messages', 'django.contrib.staticfiles', 'tagging', ) From 7b3550ed2c90fca0c96b071aa45f9a5d376c6bc1 Mon Sep 17 00:00:00 2001 From: Piotr Date: Thu, 4 Jul 2019 23:54:34 +0200 Subject: [PATCH 19/49] Update to Xenial on Travis for a newer SQLite version. django.core.exceptions.ImproperlyConfigured: SQLite 3.8.3 or later is required (found 3.8.2). --- .travis.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.travis.yml b/.travis.yml index 5afb211f1..19e0de611 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,12 +10,18 @@ matrix: env: - TOXENV=pypy-django111-pyparsing2 - python: 3.5 + sudo: true + dist: xenial env: - TOXENV=py35-django21-pyparsing2 - python: 3.5 + sudo: true + dist: xenial env: - TOXENV=py35-django22-pyparsing2 - python: 3.6 + sudo: true + dist: xenial env: - TOXENV=py36-django22-pyparsing2 - python: 3.7 From 6036fe33d666c81bc1b984f174f5f310ee2b3ae9 Mon Sep 17 00:00:00 2001 From: Johnny Lim Date: Sat, 6 Jul 2019 06:39:56 +0900 Subject: [PATCH 20/49] Fix a broken link to structured_metrics in doc --- docs/client-apis.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/client-apis.rst b/docs/client-apis.rst index 2e220abc2..c8011c773 100644 --- a/docs/client-apis.rst +++ b/docs/client-apis.rst @@ -27,5 +27,5 @@ txCarbonClient .. _Cubism.js: http://square.github.io/cubism/ .. _Graphitejs: https://github.com/prestontimmons/graphitejs .. _Scales: https://github.com/Cue/scales -.. _structured_metrics: https://github.com/vimeo/graph-explorer/tree/master/structured_metrics +.. _structured_metrics: https://github.com/vimeo/graph-explorer/tree/master/graph_explorer/structured_metrics .. _txCarbonClient: https://github.com/fdChasm/txCarbonClient From cb7fb763499b74d00fbf519a2c4c912c7a791f57 Mon Sep 17 00:00:00 2001 From: Piotr Date: Mon, 8 Jul 2019 07:07:18 +0200 Subject: [PATCH 21/49] Python 3.8 support Change BufferedHTTPReader from extending IOBase to FileIO Move parse_qs from cgi to six.moves --- .travis.yml | 7 ++++++- setup.py | 1 + tox.ini | 2 +- webapp/graphite/render/views.py | 3 +-- webapp/graphite/util.py | 2 +- 5 files changed, 10 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 19e0de611..e1c85093e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -44,11 +44,16 @@ matrix: dist: xenial env: - TOXENV=py37-django22-pyparsing2-postgresql TEST_POSTGRESQL_PASSWORD=graphite - - python: 3.7 + - python: "3.8-dev" sudo: true dist: xenial env: - TOXENV=lint + - python: "3.8-dev" + sudo: true + dist: xenial + env: + - TOXENV=py38-django22-pyparsing2 env: - TOXENV=py27-django111-pyparsing2-msgpack diff --git a/setup.py b/setup.py index 5d59fb6f9..463192928 100644 --- a/setup.py +++ b/setup.py @@ -128,6 +128,7 @@ def read(fname): 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: Implementation :: CPython', 'Programming Language :: Python :: Implementation :: PyPy', ], diff --git a/tox.ini b/tox.ini index f8ee6e59e..3a279f27d 100644 --- a/tox.ini +++ b/tox.ini @@ -64,7 +64,7 @@ commands = sphinx-build -W -b html -d {envtmpdir}/doctrees . {envtmpdir}/html [testenv:lint] -basepython = python3.7 +basepython = python3.8 deps = flake8 commands = diff --git a/webapp/graphite/render/views.py b/webapp/graphite/render/views.py index 27790b457..f474746e1 100755 --- a/webapp/graphite/render/views.py +++ b/webapp/graphite/render/views.py @@ -19,8 +19,7 @@ from datetime import datetime from time import time from random import shuffle -from six.moves.urllib.parse import urlencode, urlsplit, urlunsplit -from cgi import parse_qs +from six.moves.urllib.parse import urlencode, urlsplit, urlunsplit, parse_qs from graphite.compat import HttpResponse from graphite.user_util import getProfileByUsername diff --git a/webapp/graphite/util.py b/webapp/graphite/util.py index 4882bf9c0..d0651daf4 100644 --- a/webapp/graphite/util.py +++ b/webapp/graphite/util.py @@ -280,7 +280,7 @@ def wrapped_f(*args, **kwargs): return wrapped_f -class BufferedHTTPReader(io.IOBase): +class BufferedHTTPReader(io.FileIO): def __init__(self, response, buffer_size=1048576): self.response = response self.buffer_size = buffer_size From 734c097c5fcf9c4fa5f52f119593e0b3bd4c3b88 Mon Sep 17 00:00:00 2001 From: Piotr Date: Mon, 8 Jul 2019 07:11:37 +0200 Subject: [PATCH 22/49] Fix "SyntaxWarning: invalid escape sequence" --- webapp/graphite/render/functions.py | 8 +-- webapp/tests/test_functions.py | 17 +++--- webapp/tests/test_readers_remote.py | 4 +- webapp/tests/test_render.py | 3 +- webapp/tests/test_render_datalib.py | 4 +- webapp/tests/test_storage.py | 83 ++++++++++++++++++++--------- webapp/tests/test_tags.py | 2 +- webapp/tests/test_util.py | 38 +++++++------ 8 files changed, 100 insertions(+), 59 deletions(-) diff --git a/webapp/graphite/render/functions.py b/webapp/graphite/render/functions.py index 9c35712d2..1b416f73d 100644 --- a/webapp/graphite/render/functions.py +++ b/webapp/graphite/render/functions.py @@ -2341,12 +2341,12 @@ def areaBetween(requestContext, seriesList): def aliasSub(requestContext, seriesList, search, replace): - """ + r""" Runs series names through a regex search/replace. .. code-block:: none - &target=aliasSub(ip.*TCP*,"^.*TCP(\d+)","\\1") + &target=aliasSub(ip.*TCP*,"^.*TCP(\d+)","\1") """ try: seriesList.name = re.sub(search, replace, seriesList.name) @@ -2365,12 +2365,12 @@ def aliasSub(requestContext, seriesList, search, replace): def aliasQuery(requestContext, seriesList, search, replace, newName): - """ + r""" Performs a query to alias the metrics in seriesList. .. code-block:: none - &target=aliasQuery(channel.power.*,"channel\.power\.([0-9]+)","channel.frequency.\\1", "Channel %d MHz") + &target=aliasQuery(channel.power.*,"channel\.power\.([0-9]+)","channel.frequency.\1", "Channel %d MHz") The series in seriesList will be aliased by first translating the series names using the search & replace parameters, then using the last value of the resulting series diff --git a/webapp/tests/test_functions.py b/webapp/tests/test_functions.py index df6ec4441..7c814a8c8 100644 --- a/webapp/tests/test_functions.py +++ b/webapp/tests/test_functions.py @@ -1310,7 +1310,8 @@ def test_divideSeries_error(self): ] ) - with self.assertRaisesRegexp(ValueError, "divideSeries second argument must reference exactly 1 series \(got 2\)"): + message = r"divideSeries second argument must reference exactly 1 series \(got 2\)" + with self.assertRaisesRegexp(ValueError, message): functions.divideSeries({}, seriesList, seriesList2) def test_divideSeries_seriesList2_single(self): @@ -2609,7 +2610,8 @@ def test_vertical_line_before_start(self): endTime=datetime(1971,1,1,1,2,0,0,pytz.timezone(settings.TIME_ZONE)), tzinfo=pytz.utc ) - with self.assertRaisesRegexp(ValueError, "verticalLine\(\): timestamp 3600 exists before start of range"): + message = r"verticalLine\(\): timestamp 3600 exists before start of range" + with self.assertRaisesRegexp(ValueError, message): result = functions.verticalLine(requestContext, "01:0019700101", "foo") def test_vertical_line_after_end(self): @@ -2618,7 +2620,8 @@ def test_vertical_line_after_end(self): endTime=datetime(1970,1,1,1,2,0,0,pytz.timezone(settings.TIME_ZONE)), tzinfo=pytz.utc ) - with self.assertRaisesRegexp(ValueError, "verticalLine\(\): timestamp 31539600 exists after end of range"): + message = r"verticalLine\(\): timestamp 31539600 exists after end of range" + with self.assertRaisesRegexp(ValueError, message): result = functions.verticalLine(requestContext, "01:0019710101", "foo") def test_line_width(self): @@ -2821,7 +2824,7 @@ def test_alias(self): def test_alias_sub(self): seriesList = self._generate_series_list() substitution = "Shrubbery" - results = functions.aliasSub({}, seriesList, "^\w+", substitution) + results = functions.aliasSub({}, seriesList, r"^\w+", substitution) for series in results: self.assertTrue(series.name.startswith(substitution), "aliasSub should replace the name with {0}".format(substitution), @@ -2865,10 +2868,10 @@ def mock_evaluateTarget(requestContext, target): with patch('graphite.render.functions.evaluateTarget', mock_evaluateTarget): # Perform query - this one will not find a matching metric with self.assertRaises(Exception): - functions.aliasQuery({}, seriesList, 'chan\.pow\.([0-9]+)', 'chan.fred.\\1', 'Channel %d MHz') + functions.aliasQuery({}, seriesList, r'chan\.pow\.([0-9]+)', 'chan.fred.\\1', 'Channel %d MHz') # Perform query - this one will find a matching metric - results = functions.aliasQuery({}, seriesList, 'chan\.pow\.([0-9]+)', 'chan.freq.\\1', 'Channel %d MHz') + results = functions.aliasQuery({}, seriesList, r'chan\.pow\.([0-9]+)', 'chan.freq.\\1', 'Channel %d MHz') # Check results self.assertEqual(results, expectedResult) @@ -2880,7 +2883,7 @@ def noneSafeLast(x): # Perform query - this one will fail to return a current value for the matched metric with self.assertRaises(Exception): - functions.aliasQuery({}, seriesList, 'chan\.pow\.([0-9]+)', 'chan.freq.\\1', 'Channel %d MHz') + functions.aliasQuery({}, seriesList, r'chan\.pow\.([0-9]+)', 'chan.freq.\\1', 'Channel %d MHz') # TODO: Add tests for * globbing and {} matching to this def test_alias_by_node(self): diff --git a/webapp/tests/test_readers_remote.py b/webapp/tests/test_readers_remote.py index 17a5589aa..f12132b45 100644 --- a/webapp/tests/test_readers_remote.py +++ b/webapp/tests/test_readers_remote.py @@ -28,7 +28,7 @@ def test_RemoteReader_init_repr_get_intervals(self): bulk_query=['a.b.c.d']) self.assertIsNotNone(reader) - self.assertRegexpMatches(str(reader), "") + self.assertRegexpMatches(str(reader), r"") self.assertEqual(reader.get_intervals(), []) # @@ -163,7 +163,7 @@ def test_RemoteReader_fetch_multi(self, http_request): ) http_request.return_value = responseObject - with self.assertRaisesRegexp(Exception, 'Invalid render response from http://[^ ]+: KeyError\(\'name\',?\)'): + with self.assertRaisesRegexp(Exception, r'Invalid render response from http://[^ ]+: KeyError\(\'name\',?\)'): reader.fetch(startTime, endTime) # non-200 response diff --git a/webapp/tests/test_render.py b/webapp/tests/test_render.py index 3b69d29a4..bc517e95a 100644 --- a/webapp/tests/test_render.py +++ b/webapp/tests/test_render.py @@ -129,7 +129,8 @@ def test_render_evaluateTokens_template(self): 'template(target.$test, test=foo.bar)', ] - with self.assertRaisesRegexp(ValueError, 'invalid template\(\) syntax, only string/numeric arguments are allowed'): + message = r'invalid template\(\) syntax, only string/numeric arguments are allowed' + with self.assertRaisesRegexp(ValueError, message): evaluateTarget({}, test_input) @patch('graphite.render.evaluator.prefetchData', lambda *_: None) diff --git a/webapp/tests/test_render_datalib.py b/webapp/tests/test_render_datalib.py index df1767401..950b7db49 100644 --- a/webapp/tests/test_render_datalib.py +++ b/webapp/tests/test_render_datalib.py @@ -16,9 +16,9 @@ class TimeSeriesTest(TestCase): def test_TimeSeries_init_no_args(self): if sys.version_info[0] >= 3: - msg = '__init__\(\) missing 5 required positional arguments' + msg = r'__init__\(\) missing 5 required positional arguments' else: - msg = '__init__\(\) takes at least 6 arguments \(1 given\)' + msg = r'__init__\(\) takes at least 6 arguments \(1 given\)' with self.assertRaisesRegexp(TypeError, msg): TimeSeries() diff --git a/webapp/tests/test_storage.py b/webapp/tests/test_storage.py index 610674582..5be951abd 100644 --- a/webapp/tests/test_storage.py +++ b/webapp/tests/test_storage.py @@ -70,7 +70,7 @@ def test_fetch_pool_timeout(self): def mock_pool_exec(pool, jobs, timeout): raise PoolTimeoutError() - message = 'Timed out after [-.e0-9]+s for fetch for \[\'a\'\]' + message = r'Timed out after [-.e0-9]+s for fetch for \[\'a\'\]' with patch('graphite.storage.pool_exec', mock_pool_exec): with patch('graphite.storage.log.info') as log_info: with self.assertRaisesRegexp(Exception, message): @@ -84,21 +84,29 @@ def test_fetch_all_failed(self): finders=[TestFinder()] ) + message = r'All requests failed for fetch for \[\'a\'\] \(1\)' with patch('graphite.storage.log.info') as log_info: - with self.assertRaisesRegexp(Exception, 'All requests failed for fetch for \[\'a\'\] \(1\)'): + with self.assertRaisesRegexp(Exception, message): list(store.fetch(['a'], 1, 2, 3, {})) self.assertEqual(log_info.call_count, 1) - self.assertRegexpMatches(log_info.call_args[0][0], 'Exception during fetch for \[\'a\'\] after [-.e0-9]+s: TestFinder.find_nodes') + self.assertRegexpMatches( + log_info.call_args[0][0], + r'Exception during fetch for \[\'a\'\] after [-.e0-9]+s: TestFinder.find_nodes' + ) store = Store( finders=[TestFinder(), TestFinder()] ) + message = r'All requests failed for fetch for \[\'a\'\] \(2\)' with patch('graphite.storage.log.info') as log_info: - with self.assertRaisesRegexp(Exception, 'All requests failed for fetch for \[\'a\'\] \(2\)'): + with self.assertRaisesRegexp(Exception, message): list(store.fetch(['a'], 1, 2, 3, {})) self.assertEqual(log_info.call_count, 2) - self.assertRegexpMatches(log_info.call_args[0][0], 'Exception during fetch for \[\'a\'\] after [-.e0-9]+s: TestFinder.find_nodes') + self.assertRegexpMatches( + log_info.call_args[0][0], + r'Exception during fetch for \[\'a\'\] after [-.e0-9]+s: TestFinder.find_nodes' + ) def test_fetch_some_failed(self): # some finders failed @@ -114,11 +122,15 @@ def test_fetch_some_failed(self): finders=[TestFinder(), TestFinder()] ) + message = r'All requests failed for fetch for \[\'a\'\] \(2\)' with patch('graphite.storage.log.info') as log_info: - with self.assertRaisesRegexp(Exception, 'All requests failed for fetch for \[\'a\'\] \(2\)'): + with self.assertRaisesRegexp(Exception, message): list(store.fetch(['a'], 1, 2, 3, {})) self.assertEqual(log_info.call_count, 2) - self.assertRegexpMatches(log_info.call_args[0][0], 'Exception during fetch for \[\'a\'\] after [-.e0-9]+s: TestFinder.find_nodes') + self.assertRegexpMatches( + log_info.call_args[0][0], + r'Exception during fetch for \[\'a\'\] after [-.e0-9]+s: TestFinder.find_nodes' + ) @override_settings(STORE_FAIL_ON_ERROR=True) def test_fetch_some_failed_hard_fail_enabled(self): @@ -127,21 +139,29 @@ def test_fetch_some_failed_hard_fail_enabled(self): finders=[TestFinder(), RemoteFinder()] ) + message = r'1 request\(s\) failed for fetch for \[\'a\'\] \(2\)' with patch('graphite.storage.log.info') as log_info: - with self.assertRaisesRegexp(Exception, '1 request\(s\) failed for fetch for \[\'a\'\] \(2\)'): + with self.assertRaisesRegexp(Exception, message): list(store.fetch(['a'], 1, 2, 3, {})) self.assertEqual(log_info.call_count, 1) - self.assertRegexpMatches(log_info.call_args[0][0], 'Exception during fetch for \[\'a\'\] after [-.e0-9]+s: TestFinder.find_nodes') + self.assertRegexpMatches( + log_info.call_args[0][0], + r'Exception during fetch for \[\'a\'\] after [-.e0-9]+s: TestFinder.find_nodes' + ) store = Store( finders=[TestFinder(), TestFinder()] ) + message = r'All requests failed for fetch for \[\'a\'\] \(2\)' with patch('graphite.storage.log.info') as log_info: - with self.assertRaisesRegexp(Exception, 'All requests failed for fetch for \[\'a\'\] \(2\)'): + with self.assertRaisesRegexp(Exception, message): list(store.fetch(['a'], 1, 2, 3, {})) self.assertEqual(log_info.call_count, 2) - self.assertRegexpMatches(log_info.call_args[0][0], 'Exception during fetch for \[\'a\'\] after [-.e0-9]+s: TestFinder.find_nodes') + self.assertRegexpMatches( + log_info.call_args[0][0], + r'Exception during fetch for \[\'a\'\] after [-.e0-9]+s: TestFinder.find_nodes' + ) def test_find(self): disabled_finder = DisabledFinder() @@ -174,8 +194,9 @@ def test_find(self): self.assertTrue(node.path in ['a.b.c.d', 'a.b.c.e']) # failure threshold + message = r'Query a yields too many results and failed \(failure threshold is 1\)' with self.settings(METRICS_FIND_FAILURE_THRESHOLD=1): - with self.assertRaisesRegexp(Exception, 'Query a yields too many results and failed \(failure threshold is 1\)'): + with self.assertRaisesRegexp(Exception, message): list(store.find('a')) # warning threshold @@ -197,7 +218,7 @@ def test_find_pool_timeout(self): def mock_pool_exec(pool, jobs, timeout): raise PoolTimeoutError() - message = 'Timed out after [-.e0-9]+s for find ' + message = r'Timed out after [-.e0-9]+s for find ' with patch('graphite.storage.pool_exec', mock_pool_exec): with patch('graphite.storage.log.info') as log_info: with self.assertRaisesRegexp(Exception, message): @@ -211,14 +232,14 @@ def test_find_all_failed(self): finders=[TestFinder()] ) - message = 'All requests failed for find ' + message = r'All requests failed for find ' with patch('graphite.storage.log.info') as log_info: with self.assertRaisesRegexp(Exception, message): list(store.find('a')) self.assertEqual(log_info.call_count, 1) self.assertRegexpMatches( log_info.call_args[0][0], - 'Exception during find after [-.e0-9]+s: TestFinder.find_nodes' + r'Exception during find after [-.e0-9]+s: TestFinder.find_nodes' ) store = Store( @@ -231,7 +252,7 @@ def test_find_all_failed(self): self.assertEqual(log_info.call_count, 2) self.assertRegexpMatches( log_info.call_args[0][0], - 'Exception during find after [-.e0-9]+s: TestFinder.find_nodes' + r'Exception during find after [-.e0-9]+s: TestFinder.find_nodes' ) @override_settings(REMOTE_STORE_FORWARD_HEADERS=['X-Test1', 'X-Test2']) @@ -290,17 +311,23 @@ def test_get_index_all_failed(self): with self.assertRaisesRegexp(Exception, 'All requests failed for get_index'): store.get_index() self.assertEqual(log_info.call_count, 1) - self.assertRegexpMatches(log_info.call_args[0][0], 'Exception during get_index after [-.e0-9]+s: TestFinder.find_nodes') + self.assertRegexpMatches( + log_info.call_args[0][0], + 'Exception during get_index after [-.e0-9]+s: TestFinder.find_nodes' + ) store = Store( finders=[TestFinder(), TestFinder()] ) with patch('graphite.storage.log.info') as log_info: - with self.assertRaisesRegexp(Exception, 'All requests failed for get_index \(2\)'): + with self.assertRaisesRegexp(Exception, r'All requests failed for get_index \(2\)'): store.get_index() self.assertEqual(log_info.call_count, 2) - self.assertRegexpMatches(log_info.call_args[0][0], 'Exception during get_index after [-.e0-9]+s: TestFinder.find_nodes') + self.assertRegexpMatches( + log_info.call_args[0][0], + 'Exception during get_index after [-.e0-9]+s: TestFinder.find_nodes' + ) @override_settings(USE_WORKER_POOL=False) def test_fetch_tag_support(self): @@ -311,7 +338,11 @@ def find_nodes(self, query): pass def fetch(self, patterns, start_time, end_time, now=None, requestContext=None): - if patterns != ['seriesByTag("hello=tiger")', 'seriesByTag("name=notags")', 'seriesByTag("name=testtags")', 'testtags;hello=tiger']: + if patterns != [ + 'seriesByTag("hello=tiger")', + 'seriesByTag("name=notags")', + 'seriesByTag("name=testtags")', + 'testtags;hello=tiger']: raise Exception('Unexpected patterns %s' % str(patterns)) return [ @@ -546,29 +577,29 @@ def mockAutoCompleteValues(exprs, tag, valuePrefix=None, limit=None, requestCont # test exception handling with one finder store = mockStore([TestFinderTagsException()]) - with self.assertRaisesRegexp(Exception, 'All requests failed for tags for \[\'tag1=value1\'\] test.*'): + with self.assertRaisesRegexp(Exception, r'All requests failed for tags for \[\'tag1=value1\'\] test.*'): store.tagdb_auto_complete_tags(['tag1=value1'], 'test', 100, request_context) - with self.assertRaisesRegexp(Exception, 'All requests failed for values for \[\'tag1=value1\'\] tag2 test.*'): + with self.assertRaisesRegexp(Exception, r'All requests failed for values for \[\'tag1=value1\'\] tag2 test.*'): store.tagdb_auto_complete_values(['tag1=value1'], 'tag2', 'test', 100, request_context) # test exception handling with more than one finder store = mockStore([TestFinderTagsException(), TestFinderTagsException()]) - with self.assertRaisesRegexp(Exception, 'All requests failed for tags for \[\'tag1=value1\'\] test'): + with self.assertRaisesRegexp(Exception, r'All requests failed for tags for \[\'tag1=value1\'\] test'): store.tagdb_auto_complete_tags(['tag1=value1'], 'test', 100, request_context) - with self.assertRaisesRegexp(Exception, 'All requests failed for values for \[\'tag1=value1\'\] tag2 test'): + with self.assertRaisesRegexp(Exception, r'All requests failed for values for \[\'tag1=value1\'\] tag2 test'): store.tagdb_auto_complete_values(['tag1=value1'], 'tag2', 'test', 100, request_context) # test pool timeout handling store = mockStore([TestFinderTagsTimeout()]) with self.settings(USE_WORKER_POOL=True, FIND_TIMEOUT=0): - with self.assertRaisesRegexp(Exception, 'Timed out after [-.e0-9]+s for tags for \[\'tag1=value1\'\]'): + with self.assertRaisesRegexp(Exception, r'Timed out after [-.e0-9]+s for tags for \[\'tag1=value1\'\]'): store.tagdb_auto_complete_tags(['tag1=value1'], 'test', 100, request_context) - with self.assertRaisesRegexp(Exception, 'Timed out after [-.e0-9]+s for values for \[\'tag1=value1\'\] tag2 test'): + with self.assertRaisesRegexp(Exception, r'Timed out after [-.e0-9]+s for values for \[\'tag1=value1\'\] tag2 test'): store.tagdb_auto_complete_values(['tag1=value1'], 'tag2', 'test', 100, request_context) # test write_index diff --git a/webapp/tests/test_tags.py b/webapp/tests/test_tags.py index 2b452db65..831a6682a 100644 --- a/webapp/tests/test_tags.py +++ b/webapp/tests/test_tags.py @@ -294,7 +294,7 @@ def test_tagdb_cached(self): self.assertEqual(mockLog.info.call_count, 1) self.assertRegexpMatches( mockLog.info.call_args[0][0], - 'graphite\.tags\.localdatabase\.LocalDatabaseTagDB\.find_series :: completed \(cached\) in [-.e0-9]+s' + r'graphite\.tags\.localdatabase\.LocalDatabaseTagDB\.find_series :: completed \(cached\) in [-.e0-9]+s' ) def test_http_tagdb(self): diff --git a/webapp/tests/test_util.py b/webapp/tests/test_util.py index ca0b08947..a916084c8 100644 --- a/webapp/tests/test_util.py +++ b/webapp/tests/test_util.py @@ -28,14 +28,20 @@ def test_epoch_naive(self, mock_log): self.assertEqual(util.epoch(dt), 600) self.assertEqual(mock_log.call_count, 1) self.assertEqual(len(mock_log.call_args[0]), 1) - self.assertRegexpMatches(mock_log.call_args[0][0], 'epoch\(\) called with non-timezone-aware datetime in test_epoch_naive at .+/webapp/tests/test_util\.py:[0-9]+') + self.assertRegexpMatches( + mock_log.call_args[0][0], + r'epoch\(\) called with non-timezone-aware datetime in test_epoch_naive at .+/webapp/tests/test_util\.py:[0-9]+' + ) with self.settings(TIME_ZONE='Europe/Berlin'): dt = datetime(1970, 1, 1, 1, 10, 0, 0) self.assertEqual(util.epoch(dt), 600) self.assertEqual(mock_log.call_count, 2) self.assertEqual(len(mock_log.call_args[0]), 1) - self.assertRegexpMatches(mock_log.call_args[0][0], 'epoch\(\) called with non-timezone-aware datetime in test_epoch_naive at .+/webapp/tests/test_util\.py:[0-9]+') + self.assertRegexpMatches( + mock_log.call_args[0][0], + r'epoch\(\) called with non-timezone-aware datetime in test_epoch_naive at .+/webapp/tests/test_util\.py:[0-9]+' + ) def test_epoch_to_dt(self): dt = pytz.utc.localize(datetime(1970, 1, 1, 0, 10, 0, 0)) @@ -43,8 +49,8 @@ def test_epoch_to_dt(self): def test_is_local_interface_ipv4(self): addresses = ['127.0.0.1', '127.0.0.1:8080', '8.8.8.8'] - results = [ util.is_local_interface(a) for a in addresses ] - self.assertEqual( results, [True, True, False] ) + results = [util.is_local_interface(a) for a in addresses] + self.assertEqual(results, [True, True, False]) def test_is_local_interface_ipv6(self): # we need to know whether the host provides an ipv6 callback address @@ -69,20 +75,20 @@ def test_is_local_interface_dns(self): self.assertEqual( results, [True, True, False] ) def test_is_escaped_pattern(self): - self.assertFalse( util.is_escaped_pattern('asdf') ) - self.assertTrue( util.is_escaped_pattern('a\*b') ) - self.assertTrue( util.is_escaped_pattern('a\?b') ) - self.assertTrue( util.is_escaped_pattern('a\[b') ) - self.assertTrue( util.is_escaped_pattern('a\{b') ) - self.assertFalse( util.is_escaped_pattern('a*b') ) - self.assertFalse( util.is_escaped_pattern('a?b') ) - self.assertFalse( util.is_escaped_pattern('a[b') ) - self.assertFalse( util.is_escaped_pattern('a{b') ) + self.assertFalse(util.is_escaped_pattern(r'asdf')) + self.assertTrue(util.is_escaped_pattern(r'a\*b')) + self.assertTrue(util.is_escaped_pattern(r'a\?b')) + self.assertTrue(util.is_escaped_pattern(r'a\[b')) + self.assertTrue(util.is_escaped_pattern(r'a\{b')) + self.assertFalse(util.is_escaped_pattern(r'a*b')) + self.assertFalse(util.is_escaped_pattern(r'a?b')) + self.assertFalse(util.is_escaped_pattern(r'a[b')) + self.assertFalse(util.is_escaped_pattern(r'a{b')) def test_find_escaped_pattern_fields(self): - self.assertEqual( list(util.find_escaped_pattern_fields('a.b.c.d')), []) - self.assertEqual( list(util.find_escaped_pattern_fields('a\*.b.c.d')), [0]) - self.assertEqual( list(util.find_escaped_pattern_fields('a.b.\[c].d')), [2]) + self.assertEqual(list(util.find_escaped_pattern_fields(r'a.b.c.d')), []) + self.assertEqual(list(util.find_escaped_pattern_fields(r'a\*.b.c.d')), [0]) + self.assertEqual(list(util.find_escaped_pattern_fields(r'a.b.\[c].d')), [2]) hostcpu = os.path.join(settings.WHISPER_DIR, 'hosts/hostname/cpu.wsp') From 161e9e7ad8b9e737fd56eec8b836637a1cfd2dcd Mon Sep 17 00:00:00 2001 From: Piotr Date: Wed, 10 Jul 2019 14:39:28 +0200 Subject: [PATCH 23/49] New functions: add, sigmoid, logit, exp add: adds a constant to a list of series, easier than sumSeries(serie, constantLine(5)) and add a constant to a list of series without summing them. exp: datapoint to the power of e logit and sigmoid: logistic functions Sort transform functions alphabetically in the composer pulldown. --- webapp/content/js/composer_widgets.js | 34 ++++--- webapp/graphite/render/functions.py | 138 +++++++++++++++++++++++++- webapp/tests/test_functions.py | 54 ++++++++++ 3 files changed, 210 insertions(+), 16 deletions(-) diff --git a/webapp/content/js/composer_widgets.js b/webapp/content/js/composer_widgets.js index aafc0bd8e..e1ce36c7a 100644 --- a/webapp/content/js/composer_widgets.js +++ b/webapp/content/js/composer_widgets.js @@ -1085,28 +1085,32 @@ function createFunctionsMenu() { }, { text: 'Transform', menu: [ - {text: 'Scale', handler: applyFuncToEachWithInput('scale', 'Please enter a scale factor')}, - {text: 'ScaleToSeconds', handler: applyFuncToEachWithInput('scaleToSeconds', 'Please enter a number of seconds to scale to')}, + {text: 'Absolute Value', handler: applyFuncToEach('absolute')}, + {text: 'Add', handler: applyFuncToEachWithInput('add', 'Please enter a constant')}, + {text: 'Delay', handler: applyFuncToEachWithInput('delay', 'Please enter the number of steps to delay')}, + {text: 'Derivative', handler: applyFuncToEach('derivative')}, + {text: 'Exp', handler: applyFuncToEach('exp')}, + {text: 'Hit Count', handler: applyFuncToEachWithInput('hitcount', 'Please enter a summary interval (examples: 10min, 1h, 7d)', {quote: true})}, + {text: 'Interpolate', handler: applyFuncToEach('interpolate')}, + {text: 'Integral', handler: applyFuncToEach('integral')}, + {text: 'Integral by Interval', handler: applyFuncToEachWithInput('integralByInterval', 'Integral this metric with a reset every ___ (examples: 1d, 1h, 10min)', {quote: true})}, + {text: 'Invert', handler: applyFuncToEach('invert')}, + {text: 'Log', handler: applyFuncToEachWithInput('log', 'Please enter a base')}, + {text: 'Logit', handler: applyFuncToEach('logit')}, + {text: 'Non-negative Derivative', handler: applyFuncToEachWithInput('nonNegativeDerivative', 'Please enter a maximum value if this metric is a wrapping counter (or just leave this blank)', {allowBlank: true})}, {text: 'Offset', handler: applyFuncToEachWithInput('offset', 'Please enter the value to offset Y-values by')}, {text: 'OffsetToZero', handler: applyFuncToEach('offsetToZero')}, - {text: 'Interpolate', handler: applyFuncToEach('interpolate')}, - {text: 'Derivative', handler: applyFuncToEach('derivative')}, + {text: 'Percentile Values', handler: applyFuncToEachWithInput('percentileOfSeries', 'Please enter the percentile to use')}, {text: 'Power', handler: applyFuncToEachWithInput('pow', 'Please enter a power factor')}, {text: 'Power Series', handler: applyFuncToEachWithInput('powSeries', 'Please enter at least 2 series')}, + {text: 'Summarize', handler: applyFuncToEachWithInput('summarize', 'Please enter a summary interval (examples: 10min, 1h, 7d)', {quote: true})}, + {text: 'Scale', handler: applyFuncToEachWithInput('scale', 'Please enter a scale factor')}, + {text: 'ScaleToSeconds', handler: applyFuncToEachWithInput('scaleToSeconds', 'Please enter a number of seconds to scale to')}, + {text: 'Sigmoid', handler: applyFuncToEach('sigmoid')}, {text: 'Square Root', handler: applyFuncToEach('squareRoot')}, - {text: 'Time-adjusted Derivative', handler: applyFuncToEachWithInput('perSecond', 'Please enter a maximum value if this metric is a wrapping counter (or just leave this blank)', {allowBlank: true})}, - {text: 'Delay', handler: applyFuncToEachWithInput('delay', 'Please enter the number of steps to delay')}, - {text: 'Integral', handler: applyFuncToEach('integral')}, - {text: 'Integral by Interval', handler: applyFuncToEachWithInput('integralByInterval', 'Integral this metric with a reset every ___ (examples: 1d, 1h, 10min)', {quote: true})}, - {text: 'Percentile Values', handler: applyFuncToEachWithInput('percentileOfSeries', 'Please enter the percentile to use')}, - {text: 'Non-negative Derivative', handler: applyFuncToEachWithInput('nonNegativeDerivative', 'Please enter a maximum value if this metric is a wrapping counter (or just leave this blank)', {allowBlank: true})}, - {text: 'Log', handler: applyFuncToEachWithInput('log', 'Please enter a base')}, - {text: 'Invert', handler: applyFuncToEach('invert')}, - {text: 'Absolute Value', handler: applyFuncToEach('absolute')}, {text: 'timeShift', handler: applyFuncToEachWithInput('timeShift', 'Shift this metric ___ back in time (examples: 10min, 7d, 2w)', {quote: true})}, {text: 'timeSlice', handler: applyFuncToEachWithInput('timeSlice', 'Start showing metric at (example: 14:57 20150115)', {quote: true})}, - {text: 'Summarize', handler: applyFuncToEachWithInput('summarize', 'Please enter a summary interval (examples: 10min, 1h, 7d)', {quote: true})}, - {text: 'Hit Count', handler: applyFuncToEachWithInput('hitcount', 'Please enter a summary interval (examples: 10min, 1h, 7d)', {quote: true})} + {text: 'Time-adjusted Derivative', handler: applyFuncToEachWithInput('perSecond', 'Please enter a maximum value if this metric is a wrapping counter (or just leave this blank)', {allowBlank: true})} ] }, { text: 'Calculate', diff --git a/webapp/graphite/render/functions.py b/webapp/graphite/render/functions.py index 1b416f73d..000403626 100644 --- a/webapp/graphite/render/functions.py +++ b/webapp/graphite/render/functions.py @@ -85,6 +85,13 @@ def safeDiv(a, b): return a / b +def safeExp(a): + try: + return math.exp(a) + except TypeError: + return None + + def safePow(a, b): if a is None: return None if b is None: return None @@ -1021,7 +1028,6 @@ def divideSeriesLists(requestContext, dividendSeriesList, divisorSeriesList): results = [] - #for dividendSeries in dividendSeriesList: for i in range(0, len(dividendSeriesList)): dividendSeries = dividendSeriesList[i] divisorSeries = divisorSeriesList[i] @@ -1456,6 +1462,132 @@ def scaleToSeconds(requestContext, seriesList, seconds): ] +def exp(requestContext, seriesList): + """ + Raise e to the power of the datapoint, + where e = 2.718281... is the base of natural logarithms. + + Example: + + .. code-block:: none + + &target=exp(Server.instance01.threads.busy) + + """ + for series in seriesList: + series.tags['exp'] = 'e' + series.name = "exp(%s)" % (series.name) + series.pathExpression = series.name + for i, value in enumerate(series): + series[i] = safeExp(value) + + return seriesList + + +exp.group = 'Transform' +exp.params = [ + Param('seriesList', ParamTypes.seriesList, required=True), +] + + +def add(requestContext, seriesList, constant): + """ + Takes one metric or a wildcard seriesList followed by a constant, and adds the + constant to each datapoint. Also works for negative numbers. + + Example: + + .. code-block:: none + + &target=add(Server.instance01.threads.busy, 10) + &target=add(Server.instance*.threads.busy, 10) + + """ + for series in seriesList: + series.tags['add'] = constant + series.name = "add(%s,%d)" % (series.name, constant) + series.pathExpression = series.name + for i, value in enumerate(series): + try: + series[i] = value + constant + except TypeError: + series[i] = None + + return seriesList + + +add.group = 'Transform' +add.params = [ + Param('seriesList', ParamTypes.seriesList, required=True), + Param('constant', ParamTypes.float, required=True), +] + + +def sigmoid(requestContext, seriesList): + """ + Takes one metric or a wildcard seriesList and applies the sigmoid + function `1 / (1 + exp(-x))` to each datapoint. + + Example: + + .. code-block:: none + + &target=sigmoid(Server.instance01.threads.busy) + &target=sigmoid(Server.instance*.threads.busy) + + """ + for series in seriesList: + series.tags['sigmoid'] = 'sigmoid' + series.name = "sigmoid(%s)" % series.name + series.pathExpression = series.name + for i, value in enumerate(series): + try: + log.info(value) + series[i] = 1 / (1 + safeExp(-value)) + except (TypeError, ValueError, ZeroDivisionError): + series[i] = None + + return seriesList + + +sigmoid.group = 'Transform' +sigmoid.params = [ + Param('seriesList', ParamTypes.seriesList, required=True), +] + + +def logit(requestContext, seriesList): + """ + Takes one metric or a wildcard seriesList and applies the logit + function `log(x / (1 - x))` to each datapoint. + + Example: + + .. code-block:: none + + &target=logit(Server.instance01.threads.busy) + &target=logit(Server.instance*.threads.busy) + + """ + for series in seriesList: + series.tags['logit'] = 'logit' + series.name = "logit(%s)" % series.name + series.pathExpression = series.name + for i, value in enumerate(series): + try: + series[i] = math.log(value / (1 - value)) + except (TypeError, ValueError, ZeroDivisionError): + series[i] = None + + return seriesList + + +logit.group = 'Transform' +logit.params = [ + Param('seriesList', ParamTypes.seriesList, required=True), +] + + def pow(requestContext, seriesList, factor): """ Takes one metric or a wildcard seriesList followed by a constant, and raises the datapoint @@ -5688,9 +5820,11 @@ def pieMinimum(requestContext, series): 'weightedAverage': weightedAverage, # Transform functions + 'add': add, 'absolute': absolute, 'delay': delay, 'derivative': derivative, + 'exp': exp, 'hitcount': hitcount, 'integral': integral, 'integralByInterval' : integralByInterval, @@ -5698,6 +5832,7 @@ def pieMinimum(requestContext, series): 'invert': invert, 'keepLastValue': keepLastValue, 'log': logarithm, + 'logit': logit, 'minMax': minMax, 'nonNegativeDerivative': nonNegativeDerivative, 'offset': offset, @@ -5708,6 +5843,7 @@ def pieMinimum(requestContext, series): 'round': roundFunction, 'scale': scale, 'scaleToSeconds': scaleToSeconds, + 'sigmoid': sigmoid, 'smartSummarize': smartSummarize, 'squareRoot': squareRoot, 'summarize': summarize, diff --git a/webapp/tests/test_functions.py b/webapp/tests/test_functions.py index 7c814a8c8..e461cc8fd 100644 --- a/webapp/tests/test_functions.py +++ b/webapp/tests/test_functions.py @@ -3789,6 +3789,60 @@ def test_reduceSeries_asPercent(self): results = functions.reduceSeries({}, copy.deepcopy(mappedResult), "asPercent", 2, "bytes_used", "total_bytes") self.assertEqual(results,expectedResult) + def test_add(self): + seriesList = self._generate_series_list() + # Leave the original seriesList undisturbed for verification + results = functions.add({}, copy.deepcopy(seriesList), 1.23) + for i, series in enumerate(results): + for counter, value in enumerate(series): + if value is None: + continue + original_value = seriesList[i][counter] + expected_value = original_value + 1.23 + self.assertEqual(value, expected_value) + + def test_sigmoid(self): + seriesList = self._generate_series_list() + # Leave the original seriesList undisturbed for verification + results = functions.sigmoid({}, copy.deepcopy(seriesList)) + for i, series in enumerate(results): + for counter, value in enumerate(series): + if value is None: + continue + original_value = seriesList[i][counter] + try: + expected_value = 1 / (1 + math.exp(-original_value)) + except (TypeError, ValueError, ZeroDivisionError): + expected_value = None + self.assertEqual(value, expected_value) + + def test_logit(self): + seriesList = self._generate_series_list() + # Leave the original seriesList undisturbed for verification + results = functions.logit({}, copy.deepcopy(seriesList)) + for i, series in enumerate(results): + for counter, value in enumerate(series): + if value is None: + continue + original_value = seriesList[i][counter] + try: + expected_value = math.log(original_value / (1 - original_value)) + except (TypeError, ValueError, ZeroDivisionError): + expected_value = None + self.assertEqual(value, expected_value) + + def test_exp(self): + seriesList = self._generate_series_list() + # Leave the original seriesList undisturbed for verification + results = functions.exp({}, copy.deepcopy(seriesList)) + for i, series in enumerate(results): + for counter, value in enumerate(series): + if value is None: + continue + original_value = seriesList[i][counter] + expected_value = math.exp(original_value) + self.assertEqual(value, expected_value) + def test_pow(self): seriesList = self._generate_series_list() factor = 2 From 8af0548b09b73201bccab5681822cdb9443c687f Mon Sep 17 00:00:00 2001 From: Mauro Stettler Date: Wed, 31 Jul 2019 14:26:28 -0400 Subject: [PATCH 24/49] add functions to validate params based on what functions declare they expect --- webapp/graphite/functions/params.py | 82 +++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) diff --git a/webapp/graphite/functions/params.py b/webapp/graphite/functions/params.py index 68a0c7ab3..0e3923b81 100644 --- a/webapp/graphite/functions/params.py +++ b/webapp/graphite/functions/params.py @@ -1,3 +1,8 @@ +import six + +from graphite.render.attime import parseTimeOffset + + class ParamTypes(object): pass @@ -21,6 +26,41 @@ class ParamTypes(object): setattr(ParamTypes, paramType, paramType) +def validateBoolean(value): + return isinstance(value, bool) + + +def validateFloat(value): + return isinstance(value, float) + + +def validateInteger(value): + return isinstance(value, six.integer_types) + + +def validateInterval(value): + try: + parseTimeOffset(value) + except Exception: + return False + return True + + +def validateSeriesList(value): + return isinstance(value, list) + + +typeValidators = { + 'boolean': validateBoolean, + 'float': validateFloat, + 'integer': validateInteger, + 'interval': validateInterval, + 'node': validateInteger, + 'seriesList': validateSeriesList, + 'seriesLists': validateSeriesList, +} + + class Param(object): __slots__ = ('name', 'type', 'required', 'default', 'multiple', 'options', 'suggestions') @@ -52,3 +92,45 @@ def toJSON(self): if self.suggestions: jsonVal['suggestions'] = self.suggestions return jsonVal + + def validateValue(self, value): + validator = typeValidators.get(self.type, None) + if validator is not None: + return validator(value) + + # if there's no validator for the type we assume True + return True + + +def satisfiesParams(params, args, kwargs): + valid_args = [] + + if len(params) == 0 or params[len(params)-1].multiple is False: + if len(args) + len(kwargs) > len(params): + return False + + for i in range(len(params)): + if len(args) <= i: + # requirement is satisfied from "kwargs" + value = kwargs.get(params[i].name, None) + if value is None: + if params[i].required: + # required parameter is missing + return False + else: + # got multiple values for keyword argument + if params[i].name in valid_args: + return False + else: + # requirement is satisfied from "args" + value = args[i] + + # parameter is restricted to a defined set of values, but value is not in it + if params[i].options and value not in params[i].options: + return False + + params[i].validateValue(value) + + valid_args.append(params[i].name) + + return True From 11c4f93a27190d057b481f23bb4f11119a1d3df0 Mon Sep 17 00:00:00 2001 From: Mauro Stettler Date: Wed, 31 Jul 2019 14:28:59 -0400 Subject: [PATCH 25/49] add tests for the parameter validation functions --- webapp/tests/test_params.py | 152 ++++++++++++++++++++++++++++++++++++ 1 file changed, 152 insertions(+) create mode 100644 webapp/tests/test_params.py diff --git a/webapp/tests/test_params.py b/webapp/tests/test_params.py new file mode 100644 index 000000000..0f85e270f --- /dev/null +++ b/webapp/tests/test_params.py @@ -0,0 +1,152 @@ +import unittest + +from graphite.functions.params import Param, ParamTypes, satisfiesParams + + +class TestParam(unittest.TestCase): + params = [ + Param('one', ParamTypes.string, required=True), + Param('two', ParamTypes.string, required=True), + Param('three', ParamTypes.string, required=True), + ] + + def test_simple_args(self): + self.assertTrue(satisfiesParams( + self.params, + ['arg1', 'arg2', 'arg3'], + {}, + )) + + self.assertFalse(satisfiesParams( + self.params, + ['arg1', 'arg2'], + {}, + )) + + def test_simple_kwargs(self): + self.assertTrue(satisfiesParams( + self.params, + [], + {'one': '1', 'two': '2', 'three': '3'}, + )) + + self.assertFalse(satisfiesParams( + self.params, + [], + {'one': '1', 'two': '2'}, + )) + + self.assertFalse(satisfiesParams( + self.params, + [], + {'one': '1', 'two': '2', 'four': '4'}, + )) + + def test_mixed_cases(self): + self.assertTrue(satisfiesParams( + self.params, + ['one', 'two'], + {'three': '3'}, + )) + + self.assertTrue(satisfiesParams( + self.params, + ['one'], + {'three': '3', 'two': '2'}, + )) + + # positional args don't check the name + self.assertTrue(satisfiesParams( + self.params, + ['one', 'two', 'four'], + {}, + )) + + self.assertFalse(satisfiesParams( + self.params, + [], + {'three': '3', 'two': '2'}, + )) + + self.assertFalse(satisfiesParams( + self.params, + ['one', 'three'], + {'two': '2'}, + )) + + self.assertFalse(satisfiesParams( + self.params, + ['three'], + {'one': '1', 'two': '2'}, + )) + + def test_repeated_args(self): + self.assertFalse(satisfiesParams( + self.params, + ['one'], + {'three': '3', 'one': '1'}, + )) + + self.assertFalse(satisfiesParams( + self.params, + ['one', 'two'], + {'three': '3', 'two': '2'}, + )) + + self.assertFalse(satisfiesParams( + self.params, + ['one', 'two', 'three'], + {'one': '1'}, + )) + + def test_multiple_property(self): + self.assertFalse(satisfiesParams( + [ + Param('one', ParamTypes.string, required=True), + Param('two', ParamTypes.string, required=True), + Param('three', ParamTypes.string, required=True, multiple=False), + ], + ['one', 'two', 'three', 'four'], + {}, + )) + + self.assertTrue(satisfiesParams( + [ + Param('one', ParamTypes.string, required=True), + Param('two', ParamTypes.string, required=True), + Param('three', ParamTypes.string, required=True, multiple=True), + ], + ['one', 'two', 'three', 'four'], + {}, + )) + + def test_options_property(self): + self.assertTrue(satisfiesParams( + [ + Param('one', ParamTypes.string, required=True), + Param('two', ParamTypes.string, required=True), + Param('three', ParamTypes.string, required=True, options=['3', 'three']), + ], + ['one', 'two', '3'], + {}, + )) + + self.assertTrue(satisfiesParams( + [ + Param('one', ParamTypes.string, required=True), + Param('two', ParamTypes.string, required=True), + Param('three', ParamTypes.string, required=True, options=['3', 'three']), + ], + ['one', 'two', 'three'], + {}, + )) + + self.assertFalse(satisfiesParams( + [ + Param('one', ParamTypes.string, required=True), + Param('two', ParamTypes.string, required=True), + Param('three', ParamTypes.string, required=True, options=['3', 'three']), + ], + ['one', 'two', 'foud'], + {}, + )) From 8ecee5d81dbafaaefde8e5734d3191fa88afdc25 Mon Sep 17 00:00:00 2001 From: Mauro Stettler Date: Wed, 31 Jul 2019 14:27:13 -0400 Subject: [PATCH 26/49] add customer input parameter error and handle it in the render view --- webapp/graphite/errors.py | 4 ++++ webapp/graphite/render/views.py | 14 +++++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/webapp/graphite/errors.py b/webapp/graphite/errors.py index 1349153d0..2a11e74ab 100644 --- a/webapp/graphite/errors.py +++ b/webapp/graphite/errors.py @@ -1,3 +1,7 @@ class NormalizeEmptyResultError(Exception): # throw error for normalize() when empty pass + + +class InputParameterError(ValueError): + pass diff --git a/webapp/graphite/render/views.py b/webapp/graphite/render/views.py index f474746e1..367f8eacf 100755 --- a/webapp/graphite/render/views.py +++ b/webapp/graphite/render/views.py @@ -22,6 +22,7 @@ from six.moves.urllib.parse import urlencode, urlsplit, urlunsplit, parse_qs from graphite.compat import HttpResponse +from graphite.errors import InputParameterError from graphite.user_util import getProfileByUsername from graphite.util import json, unpickle, pickle, msgpack, BytesIO from graphite.storage import extractForwardHeaders @@ -33,7 +34,7 @@ from graphite.render.glyph import GraphTypes from graphite.tags.models import Series, Tag, TagValue, SeriesTag # noqa # pylint: disable=unused-import -from django.http import HttpResponseServerError, HttpResponseRedirect +from django.http import HttpResponseServerError, HttpResponseRedirect, HttpResponseBadRequest from django.template import Context, loader from django.core.cache import cache from django.core.exceptions import ObjectDoesNotExist @@ -45,6 +46,17 @@ loadFunctions() +def handleInputParameterError(f): + def new_f(*args, **kwargs): + try: + return f(*args, **kwargs) + except InputParameterError as e: + return HttpResponseBadRequest('Bad Request: {err}'.format(err=e)) + + return new_f + + +@handleInputParameterError def renderView(request): start = time() (graphOptions, requestOptions) = parseOptions(request) From 147f03ca170363d7fa43c5f6dd6d40beb8cd8be0 Mon Sep 17 00:00:00 2001 From: Mauro Stettler Date: Wed, 31 Jul 2019 14:27:51 -0400 Subject: [PATCH 27/49] raise input parameter error if evaluator detects an issue --- webapp/graphite/render/evaluator.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/webapp/graphite/render/evaluator.py b/webapp/graphite/render/evaluator.py index dde13f1a3..4236f9555 100644 --- a/webapp/graphite/render/evaluator.py +++ b/webapp/graphite/render/evaluator.py @@ -1,10 +1,11 @@ import re import six -from graphite.errors import NormalizeEmptyResultError +from graphite.errors import NormalizeEmptyResultError, InputParameterError from graphite.functions import SeriesFunction from graphite.render.grammar import grammar from graphite.render.datalib import fetchData, TimeSeries, prefetchData +from graphite.functions.params import satisfiesParams def evaluateTarget(requestContext, targets): @@ -76,7 +77,7 @@ def evaluateTokens(requestContext, tokens, replacements=None, pipedArg=None): if tokens.call.funcname == 'template': # if template propagates down here, it means the grammar didn't match the invocation # as tokens.template. this generally happens if you try to pass non-numeric/string args - raise ValueError("invalid template() syntax, only string/numeric arguments are allowed") + raise InputParameterError("invalid template() syntax, only string/numeric arguments are allowed") if tokens.call.funcname == 'seriesByTag': return fetchData(requestContext, tokens.call.raw) @@ -89,6 +90,10 @@ def evaluateTokens(requestContext, tokens, replacements=None, pipedArg=None): requestContext['args'] = rawArgs kwargs = dict([(kwarg.argname, evaluateTokens(requestContext, kwarg.args[0], replacements)) for kwarg in tokens.call.kwargs]) + + if hasattr(func, 'params') and not satisfiesParams(func.params, args, kwargs): + raise InputParameterError('Invalid parameters for function "{func}"'.format(func=tokens.call.funcname)) + try: return func(requestContext, *args, **kwargs) except NormalizeEmptyResultError: @@ -106,7 +111,7 @@ def evaluateScalarTokens(tokens): if tokens.number.scientific: return float(tokens.number.scientific[0]) - raise ValueError("unknown numeric type in target evaluator") + raise InputParameterError("unknown numeric type in target evaluator") if tokens.string: return tokens.string[1:-1] @@ -117,7 +122,7 @@ def evaluateScalarTokens(tokens): if tokens.none: return None - raise ValueError("unknown token in target evaluator") + raise InputParameterError("unknown token in target evaluator") def extractPathExpressions(requestContext, targets): From d687fbc7abc4dbdd37fbfddb8a624dd4cb7f147e Mon Sep 17 00:00:00 2001 From: Mauro Stettler Date: Wed, 31 Jul 2019 14:28:43 -0400 Subject: [PATCH 28/49] raise InputParameterError from functions when appropriate, instead of ValueError --- webapp/graphite/render/functions.py | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/webapp/graphite/render/functions.py b/webapp/graphite/render/functions.py index 000403626..6c12d53c8 100644 --- a/webapp/graphite/render/functions.py +++ b/webapp/graphite/render/functions.py @@ -34,8 +34,7 @@ from os import environ from django.conf import settings - -from graphite.errors import NormalizeEmptyResultError +from graphite.errors import NormalizeEmptyResultError, InputParameterError from graphite.events import models from graphite.functions import SeriesFunction, ParamTypes, Param from graphite.logger import log @@ -295,7 +294,7 @@ def getAggFunc(func, rawFunc=None): return aggFuncs[func] if func in aggFuncAliases: return aggFuncAliases[func] - raise ValueError('Unsupported aggregation function: %s' % (rawFunc or func)) + raise InputParameterError('Unsupported aggregation function: %s' % (rawFunc or func)) def aggregate(requestContext, seriesList, func, xFilesFactor=None): @@ -672,7 +671,7 @@ def percentileOfSeries(requestContext, seriesList, n, interpolate=False): supplied series. """ if n <= 0: - raise ValueError('The requested percent is required to be greater than 0') + raise InputParameterError('The requested percent is required to be greater than 0') # if seriesList is empty then just short-circuit if not seriesList: @@ -945,7 +944,7 @@ def asPercent(requestContext, seriesList, total=None, *nodes): totalSeries[key] = TimeSeries(name,start,end,step,totalValues,xFilesFactor=xFilesFactor) # trying to use nodes with a total value, which isn't supported because it has no effect else: - raise ValueError('total must be None or a seriesList') + raise InputParameterError('total must be None or a seriesList') resultList = [] for key in keys: @@ -980,7 +979,7 @@ def asPercent(requestContext, seriesList, total=None, *nodes): totalText = "sumSeries(%s)" % formatPathExpressions(seriesList) elif type(total) is list: if len(total) != 1 and len(total) != len(seriesList): - raise ValueError("asPercent second argument must be missing, a single digit, reference exactly 1 series or reference the same number of series as the first argument") + raise InputParameterError("asPercent second argument must be missing, a single digit, reference exactly 1 series or reference the same number of series as the first argument") if len(total) == 1: normalize([seriesList, total]) @@ -1024,7 +1023,7 @@ def divideSeriesLists(requestContext, dividendSeriesList, divisorSeriesList): """ if len(dividendSeriesList) != len(divisorSeriesList): - raise ValueError("dividendSeriesList and divisorSeriesList argument must have equal length") + raise InputParameterError("dividendSeriesList and divisorSeriesList argument must have equal length") results = [] @@ -1081,7 +1080,7 @@ def divideSeries(requestContext, dividendSeriesList, divisorSeries): series[i] = None return dividendSeriesList if len(divisorSeries) > 1: - raise ValueError("divideSeries second argument must reference exactly 1 series (got {0})".format(len(divisorSeries))) + raise InputParameterError("divideSeries second argument must reference exactly 1 series (got {0})".format(len(divisorSeries))) divisorSeries = divisorSeries[0] results = [] @@ -2514,10 +2513,10 @@ def aliasQuery(requestContext, seriesList, search, replace, newName): newContext['prefetch'] = {} newSeriesList = evaluateTarget(newContext, newQuery) if newSeriesList is None or len(newSeriesList) == 0: - raise Exception('No series found with query: ' + newQuery) + raise InputParameterError('No series found with query: ' + newQuery) current = safeLast(newSeriesList[0]) if current is None: - raise Exception('Cannot get last value of series: ' + newSeriesList[0]) + raise InputParameterError('Cannot get last value of series: ' + newSeriesList[0]) series.name = newName % current return seriesList @@ -2890,7 +2889,7 @@ def filterSeries(requestContext, seriesList, func, operator, threshold): consolidationFunc = getAggFunc(func) if operator not in operatorFuncs: - raise Exception('Unsupported operator: %s' % (operator)) + raise InputParameterError('Unsupported operator: %s' % (operator)) operatorFunc = operatorFuncs[operator] # if seriesList is empty then just short-circuit @@ -4567,9 +4566,9 @@ def verticalLine(requestContext, ts, label=None, color=None): start = int(timestamp( requestContext['startTime'] )) end = int(timestamp( requestContext['endTime'] )) if ts < start: - raise ValueError("verticalLine(): timestamp %s exists before start of range" % ts) + raise InputParameterError("verticalLine(): timestamp %s exists before start of range" % ts) elif ts > end: - raise ValueError("verticalLine(): timestamp %s exists after end of range" % ts) + raise InputParameterError("verticalLine(): timestamp %s exists after end of range" % ts) start = end = ts step = 1.0 series = TimeSeries(label, start, end, step, [1.0, 1.0], xFilesFactor=requestContext.get('xFilesFactor')) @@ -5599,7 +5598,7 @@ def groupByTags(requestContext, seriesList, callback, *tags): return [] if not tags: - raise ValueError("groupByTags(): no tags specified") + raise InputParameterError("groupByTags(): no tags specified") # if all series have the same "name" tag use that for results, otherwise use the callback # if we're grouping by name, then the name is always used (see below) From b115252e0e452e3c075c771f4cd5ebc979cd300e Mon Sep 17 00:00:00 2001 From: Dan Cech Date: Tue, 6 Aug 2019 15:57:34 -0400 Subject: [PATCH 29/49] enforce param checks, handle integer values for float params, handle missing default params, better error messages --- webapp/graphite/functions/params.py | 39 +++++++---- webapp/graphite/render/evaluator.py | 6 +- webapp/tests/test_params.py | 104 +++++++++++++++++++--------- 3 files changed, 103 insertions(+), 46 deletions(-) diff --git a/webapp/graphite/functions/params.py b/webapp/graphite/functions/params.py index 0e3923b81..47630c472 100644 --- a/webapp/graphite/functions/params.py +++ b/webapp/graphite/functions/params.py @@ -1,5 +1,6 @@ import six +from graphite.errors import InputParameterError from graphite.render.attime import parseTimeOffset @@ -31,7 +32,7 @@ def validateBoolean(value): def validateFloat(value): - return isinstance(value, float) + return isinstance(value, float) or validateInteger(value) def validateInteger(value): @@ -94,20 +95,25 @@ def toJSON(self): return jsonVal def validateValue(self, value): - validator = typeValidators.get(self.type, None) - if validator is not None: - return validator(value) + # None is ok for optional params + if not self.required and value is None: + return True + validator = typeValidators.get(self.type, None) # if there's no validator for the type we assume True - return True + if validator is None: + return True + return validator(value) -def satisfiesParams(params, args, kwargs): + +def validateParams(func, params, args, kwargs): valid_args = [] if len(params) == 0 or params[len(params)-1].multiple is False: if len(args) + len(kwargs) > len(params): - return False + raise InputParameterError( + 'Too many parameters specified for function "{func}"'.format(func=func)) for i in range(len(params)): if len(args) <= i: @@ -116,20 +122,29 @@ def satisfiesParams(params, args, kwargs): if value is None: if params[i].required: # required parameter is missing - return False + raise InputParameterError( + 'Missing required parameter "{param}" for function "{func}"'.format( + param=params[i].name, func=func)) else: # got multiple values for keyword argument if params[i].name in valid_args: - return False + raise InputParameterError( + 'Keyword parameter "{param}" specified multiple times for function "{func}"'.format( + param=params[i].name, func=func)) else: # requirement is satisfied from "args" value = args[i] # parameter is restricted to a defined set of values, but value is not in it if params[i].options and value not in params[i].options: - return False - - params[i].validateValue(value) + raise InputParameterError( + 'Invalid option specified for function "{func}" parameter "{param}"'.format( + param=params[i].name, func=func)) + + if not params[i].validateValue(value): + raise InputParameterError( + 'Invalid {type} value specified for function "{func}" parameter "{param}"'.format( + type=params[i].type, param=params[i].name, func=func)) valid_args.append(params[i].name) diff --git a/webapp/graphite/render/evaluator.py b/webapp/graphite/render/evaluator.py index 4236f9555..8115fc52b 100644 --- a/webapp/graphite/render/evaluator.py +++ b/webapp/graphite/render/evaluator.py @@ -5,7 +5,7 @@ from graphite.functions import SeriesFunction from graphite.render.grammar import grammar from graphite.render.datalib import fetchData, TimeSeries, prefetchData -from graphite.functions.params import satisfiesParams +from graphite.functions.params import validateParams def evaluateTarget(requestContext, targets): @@ -91,8 +91,8 @@ def evaluateTokens(requestContext, tokens, replacements=None, pipedArg=None): kwargs = dict([(kwarg.argname, evaluateTokens(requestContext, kwarg.args[0], replacements)) for kwarg in tokens.call.kwargs]) - if hasattr(func, 'params') and not satisfiesParams(func.params, args, kwargs): - raise InputParameterError('Invalid parameters for function "{func}"'.format(func=tokens.call.funcname)) + if hasattr(func, 'params'): + validateParams(tokens.call.funcname, func.params, args, kwargs) try: return func(requestContext, *args, **kwargs) diff --git a/webapp/tests/test_params.py b/webapp/tests/test_params.py index 0f85e270f..a4795085d 100644 --- a/webapp/tests/test_params.py +++ b/webapp/tests/test_params.py @@ -1,6 +1,7 @@ import unittest -from graphite.functions.params import Param, ParamTypes, satisfiesParams +from graphite.errors import InputParameterError +from graphite.functions.params import Param, ParamTypes, validateParams class TestParam(unittest.TestCase): @@ -11,96 +12,131 @@ class TestParam(unittest.TestCase): ] def test_simple_args(self): - self.assertTrue(satisfiesParams( + self.assertTrue(validateParams( + 'TestParam', self.params, ['arg1', 'arg2', 'arg3'], {}, )) - self.assertFalse(satisfiesParams( + self.assertRaises( + InputParameterError, + validateParams, + 'TestParam', self.params, ['arg1', 'arg2'], {}, - )) + ) def test_simple_kwargs(self): - self.assertTrue(satisfiesParams( + self.assertTrue(validateParams( + 'TestParam', self.params, [], {'one': '1', 'two': '2', 'three': '3'}, )) - self.assertFalse(satisfiesParams( + self.assertRaises( + InputParameterError, + validateParams, + 'TestParam', self.params, [], {'one': '1', 'two': '2'}, - )) + ) - self.assertFalse(satisfiesParams( + self.assertRaises( + InputParameterError, + validateParams, + 'TestParam', self.params, [], {'one': '1', 'two': '2', 'four': '4'}, - )) + ) def test_mixed_cases(self): - self.assertTrue(satisfiesParams( + self.assertTrue(validateParams( + 'TestParam', self.params, ['one', 'two'], {'three': '3'}, )) - self.assertTrue(satisfiesParams( + self.assertTrue(validateParams( + 'TestParam', self.params, ['one'], {'three': '3', 'two': '2'}, )) # positional args don't check the name - self.assertTrue(satisfiesParams( + self.assertTrue(validateParams( + 'TestParam', self.params, ['one', 'two', 'four'], {}, )) - self.assertFalse(satisfiesParams( + self.assertRaises( + InputParameterError, + validateParams, + 'TestParam', self.params, [], {'three': '3', 'two': '2'}, - )) + ) - self.assertFalse(satisfiesParams( + self.assertRaises( + InputParameterError, + validateParams, + 'TestParam', self.params, ['one', 'three'], {'two': '2'}, - )) + ) - self.assertFalse(satisfiesParams( + self.assertRaises( + InputParameterError, + validateParams, + 'TestParam', self.params, ['three'], {'one': '1', 'two': '2'}, - )) + ) def test_repeated_args(self): - self.assertFalse(satisfiesParams( + self.assertRaises( + InputParameterError, + validateParams, + 'TestParam', self.params, ['one'], {'three': '3', 'one': '1'}, - )) + ) - self.assertFalse(satisfiesParams( + self.assertRaises( + InputParameterError, + validateParams, + 'TestParam', self.params, ['one', 'two'], {'three': '3', 'two': '2'}, - )) + ) - self.assertFalse(satisfiesParams( + self.assertRaises( + InputParameterError, + validateParams, + 'TestParam', self.params, ['one', 'two', 'three'], {'one': '1'}, - )) + ) def test_multiple_property(self): - self.assertFalse(satisfiesParams( + self.assertRaises( + InputParameterError, + validateParams, + 'TestParam', [ Param('one', ParamTypes.string, required=True), Param('two', ParamTypes.string, required=True), @@ -108,9 +144,10 @@ def test_multiple_property(self): ], ['one', 'two', 'three', 'four'], {}, - )) + ) - self.assertTrue(satisfiesParams( + self.assertTrue(validateParams( + 'TestParam', [ Param('one', ParamTypes.string, required=True), Param('two', ParamTypes.string, required=True), @@ -121,7 +158,8 @@ def test_multiple_property(self): )) def test_options_property(self): - self.assertTrue(satisfiesParams( + self.assertTrue(validateParams( + 'TestParam', [ Param('one', ParamTypes.string, required=True), Param('two', ParamTypes.string, required=True), @@ -131,7 +169,8 @@ def test_options_property(self): {}, )) - self.assertTrue(satisfiesParams( + self.assertTrue(validateParams( + 'TestParam', [ Param('one', ParamTypes.string, required=True), Param('two', ParamTypes.string, required=True), @@ -141,7 +180,10 @@ def test_options_property(self): {}, )) - self.assertFalse(satisfiesParams( + self.assertRaises( + InputParameterError, + validateParams, + 'TestParam', [ Param('one', ParamTypes.string, required=True), Param('two', ParamTypes.string, required=True), @@ -149,4 +191,4 @@ def test_options_property(self): ], ['one', 'two', 'foud'], {}, - )) + ) From 55d5d9b1dc19d18b851cc83be1bc816d090bfa00 Mon Sep 17 00:00:00 2001 From: Mauro Stettler Date: Fri, 9 Aug 2019 19:44:12 -0400 Subject: [PATCH 30/49] make input parameter validation optional --- webapp/graphite/render/evaluator.py | 31 ++++++++++++++++++++++++++++- webapp/graphite/settings.py | 7 +++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/webapp/graphite/render/evaluator.py b/webapp/graphite/render/evaluator.py index 8115fc52b..3630f2b5e 100644 --- a/webapp/graphite/render/evaluator.py +++ b/webapp/graphite/render/evaluator.py @@ -3,10 +3,13 @@ from graphite.errors import NormalizeEmptyResultError, InputParameterError from graphite.functions import SeriesFunction +from graphite.logger import log from graphite.render.grammar import grammar from graphite.render.datalib import fetchData, TimeSeries, prefetchData from graphite.functions.params import validateParams +from django.conf import settings + def evaluateTarget(requestContext, targets): if not isinstance(targets, list): @@ -91,13 +94,39 @@ def evaluateTokens(requestContext, tokens, replacements=None, pipedArg=None): kwargs = dict([(kwarg.argname, evaluateTokens(requestContext, kwarg.args[0], replacements)) for kwarg in tokens.call.kwargs]) + def handleInvalidParameters(e): + if not getattr(handleInvalidParameters, 'alreadyLogged', False): + log.warning( + 'Received invalid parameters ({msg}): {func} ({args})'.format( + msg=str(e.message), + func=tokens.call.funcname, + args=', '.join( + argList + for argList in [ + ', '.join(str(arg) for arg in args), + ', '.join('{k}={v}'.format(k=str(k), v=str(v)) for k, v in kwargs.items()), + ] if argList + ) + )) + + # only log invalid parameters once + setattr(handleInvalidParameters, 'alreadyLogged', True) + + if settings.ENFORCE_INPUT_VALIDATION: + raise + if hasattr(func, 'params'): - validateParams(tokens.call.funcname, func.params, args, kwargs) + try: + validateParams(tokens.call.funcname, func.params, args, kwargs) + except InputParameterError as e: + handleInvalidParameters(e) try: return func(requestContext, *args, **kwargs) except NormalizeEmptyResultError: return [] + except InputParameterError as e: + handleInvalidParameters(e) return evaluateScalarTokens(tokens) diff --git a/webapp/graphite/settings.py b/webapp/graphite/settings.py index 269308478..a5cf02a98 100644 --- a/webapp/graphite/settings.py +++ b/webapp/graphite/settings.py @@ -182,6 +182,13 @@ # Django 1.5 requires this so we set a default but warn the user SECRET_KEY = 'UNSAFE_DEFAULT' +# Input validation +# - When False we still validate the received input parameters, but if validation +# detects an issue it only logs an error and doesn't directly reject the request +# - When True we reject requests of which the input validation detected an issue with the +# provided arguments and return an error message to the user +ENFORCE_INPUT_VALIDATION = False + # Django 1.5 requires this to be set. Here we default to prior behavior and allow all ALLOWED_HOSTS = [ '*' ] From 5c0a55183e52e76069d9589b36f4edf9ab7a75d2 Mon Sep 17 00:00:00 2001 From: Mauro Stettler Date: Fri, 9 Aug 2019 19:44:32 -0400 Subject: [PATCH 31/49] consider an unknown function name an input error --- webapp/graphite/render/evaluator.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/webapp/graphite/render/evaluator.py b/webapp/graphite/render/evaluator.py index 3630f2b5e..abf5db262 100644 --- a/webapp/graphite/render/evaluator.py +++ b/webapp/graphite/render/evaluator.py @@ -85,7 +85,15 @@ def evaluateTokens(requestContext, tokens, replacements=None, pipedArg=None): if tokens.call.funcname == 'seriesByTag': return fetchData(requestContext, tokens.call.raw) - func = SeriesFunction(tokens.call.funcname) + try: + func = SeriesFunction(tokens.call.funcname) + except KeyError: + msg = 'Received request for unknown function: {func}'.format(func=tokens.call.funcname) + log.warning(msg) + + # even if input validation enforcement is disabled, there's nothing else we can do here + raise InputParameterError(msg) + rawArgs = tokens.call.args or [] if pipedArg is not None: rawArgs.insert(0, pipedArg) From 2fc52720926141fe90f4acc920c0a99dcf90c40c Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Tue, 13 Aug 2019 16:34:01 +0200 Subject: [PATCH 32/49] Pass maxDataPoints to the requestContext for Finder --- webapp/graphite/render/views.py | 1 + 1 file changed, 1 insertion(+) diff --git a/webapp/graphite/render/views.py b/webapp/graphite/render/views.py index 367f8eacf..b7ca5cf9b 100755 --- a/webapp/graphite/render/views.py +++ b/webapp/graphite/render/views.py @@ -74,6 +74,7 @@ def renderView(request): 'data' : [], 'prefetched' : {}, 'xFilesFactor' : requestOptions['xFilesFactor'], + 'maxDataPoints' : requestOptions.get('maxDataPoints', None), } data = requestContext['data'] From 9e4baa036d082079945097798c3cd0094f796bcd Mon Sep 17 00:00:00 2001 From: Ahmet DEMIR Date: Thu, 5 Sep 2019 17:19:32 +0200 Subject: [PATCH 33/49] Add redis password support for tagdb --- docs/config-local-settings.rst | 4 ++++ docs/tags.rst | 5 +++-- webapp/graphite/local_settings.py.example | 1 + webapp/graphite/settings.py | 1 + webapp/graphite/tags/redis.py | 1 + 5 files changed, 10 insertions(+), 2 deletions(-) diff --git a/docs/config-local-settings.rst b/docs/config-local-settings.rst index aa1b6928f..f5580100e 100644 --- a/docs/config-local-settings.rst +++ b/docs/config-local-settings.rst @@ -208,6 +208,10 @@ TAGDB_REDIS_DB `Default: 0` Redis database to use with `TAGDB = 'graphite.tags.redis.RedisTagDB'` +TAGDB_REDIS_PASSWORD + `Default: ''` + Redis password to use with `TAGDB = 'graphite.tags.redis.RedisTagDB'` + Configure Webserver (Apache) ---------------------------- There is an example ``example-graphite-vhost.conf`` file in the examples directory of the graphite web source code. You can use this to configure apache. Different distributions have different ways of configuring Apache. Please refer to your distribution's documentation on the subject. diff --git a/docs/tags.rst b/docs/tags.rst index f8ca9af7f..bae542882 100644 --- a/docs/tags.rst +++ b/docs/tags.rst @@ -108,13 +108,14 @@ The Local TagDB stores tag information in tables inside the graphite-web databas Redis TagDB ^^^^^^^^^^^ -The Redis TagDB will store the tag information on a Redis server, and is selected by setting ``TAGDB='graphite.tags.redis.RedisTagDB'`` in `local_settings.py`. There are 3 additional config settings for the Redis TagDB:: +The Redis TagDB will store the tag information on a Redis server, and is selected by setting ``TAGDB='graphite.tags.redis.RedisTagDB'`` in `local_settings.py`. There are 4 additional config settings for the Redis TagDB:: TAGDB_REDIS_HOST = 'localhost' TAGDB_REDIS_PORT = 6379 TAGDB_REDIS_DB = 0 + TAGDB_REDIS_PASSWORD = '' -The default settings (above) will connect to a local Redis server on the default port, and use the default database. +The default settings (above) will connect to a local Redis server on the default port, and use the default database without password. HTTP(S) TagDB ^^^^^^^^^^^^^ diff --git a/webapp/graphite/local_settings.py.example b/webapp/graphite/local_settings.py.example index b5378fae4..ac4ee53ea 100644 --- a/webapp/graphite/local_settings.py.example +++ b/webapp/graphite/local_settings.py.example @@ -374,6 +374,7 @@ DEFAULT_XFILES_FACTOR = 0 #TAGDB_REDIS_HOST = 'localhost' #TAGDB_REDIS_PORT = 6379 #TAGDB_REDIS_DB = 0 +#TAGDB_REDIS_PASSWORD = '' # Settings for HTTP TagDB #TAGDB_HTTP_URL = '' diff --git a/webapp/graphite/settings.py b/webapp/graphite/settings.py index a5cf02a98..9a66c0327 100644 --- a/webapp/graphite/settings.py +++ b/webapp/graphite/settings.py @@ -145,6 +145,7 @@ TAGDB_REDIS_HOST = 'localhost' TAGDB_REDIS_PORT = 6379 TAGDB_REDIS_DB = 0 +TAGDB_REDIS_PASSWORD = '' TAGDB_HTTP_URL = '' TAGDB_HTTP_USER = '' diff --git a/webapp/graphite/tags/redis.py b/webapp/graphite/tags/redis.py index 7ede6ca3d..c034122b7 100644 --- a/webapp/graphite/tags/redis.py +++ b/webapp/graphite/tags/redis.py @@ -32,6 +32,7 @@ def __init__(self, settings, *args, **kwargs): host=settings.TAGDB_REDIS_HOST, port=settings.TAGDB_REDIS_PORT, db=settings.TAGDB_REDIS_DB, + password=settings.TAGDB_REDIS_PASSWORD, decode_responses=(sys.version_info[0] >= 3), ) From 4b1e77973de52d93b85ff07333d229e5e93aa274 Mon Sep 17 00:00:00 2001 From: saikek Date: Mon, 23 Sep 2019 15:11:43 +0200 Subject: [PATCH 34/49] added space before \ I've encountered issues with current code snippet on some instances of centos, which can is fixed by adding extra spaces before \ signs. --- docs/install.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/install.rst b/docs/install.rst index a9c4dea0a..b730783dc 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -8,14 +8,14 @@ Try Graphite in Docker and have it running in seconds: .. code-block:: none - docker run -d\ - --name graphite\ - --restart=always\ - -p 80:80\ - -p 2003-2004:2003-2004\ - -p 2023-2024:2023-2024\ - -p 8125:8125/udp\ - -p 8126:8126\ + docker run -d \ + --name graphite \ + --restart=always \ + -p 80:80 \ + -p 2003-2004:2003-2004 \ + -p 2023-2024:2023-2024 \ + -p 8125:8125/udp \ + -p 8126:8126 \ graphiteapp/graphite-statsd Check `docker repo`_ for details. From 79063eb91fe3a3a0f01127a63c18b7bcd3539fa6 Mon Sep 17 00:00:00 2001 From: Nimish Verma Date: Mon, 23 Sep 2019 13:40:04 -0400 Subject: [PATCH 35/49] Created issue template --- ISSUE_TEMPLATE.md | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 ISSUE_TEMPLATE.md diff --git a/ISSUE_TEMPLATE.md b/ISSUE_TEMPLATE.md new file mode 100644 index 000000000..a8286c0f4 --- /dev/null +++ b/ISSUE_TEMPLATE.md @@ -0,0 +1,39 @@ + + + + +**I'm submitting a ...** + +[ ] bug report +[ ] feature request + + + +**Current behavior:** + + +**Expected behavior:** + + +**Steps to reproduce:** + + +**Related code:** + + + +``` +insert short code snippets here +``` + +**Other information:** + + From 422c2a1dd575bc878bdf64678f0281a2499032ae Mon Sep 17 00:00:00 2001 From: Steven Basgall Date: Thu, 26 Sep 2019 17:11:02 -0600 Subject: [PATCH 36/49] docs: add netdata to 'tools that work with graphite' --- docs/tools.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/tools.rst b/docs/tools.rst index c82f542db..81fb4af7a 100644 --- a/docs/tools.rst +++ b/docs/tools.rst @@ -57,6 +57,9 @@ Collection `metrics-sampler`_ A java program which regularly queries metrics from a configured set of inputs, selects and renames them using regular expressions and sends them to a configured set of outputs. It supports JMX and JDBC as inputs and Graphite as output out of the box. +`netdata`_ + A fast and efficient monitoring agent that supports graphite backends. It has collection, forwarding, visualization and and monitoring features. Netdata collects common system metrics and a variety of other sources through plugins. + `Sensu`_ A monitoring framework that can route metrics to Graphite. Servers subscribe to sets of checks, so getting metrics from a new server to Graphite is as simple as installing the Sensu client and subscribing. @@ -384,6 +387,7 @@ Other .. _metrics-sampler: https://github.com/dimovelev/metrics-sampler .. _metrictank: https://github.com/grafana/metrictank .. _Moira: http://moira.readthedocs.io +.. _netdata: https://github.com/netdata/netdata .. _New Relic: https://newrelic.com/platform .. _Pencil: https://github.com/fetep/pencil .. _pipe-to-graphite: https://github.com/iFixit/pipe-to-graphite From cc67e526ec6d01f11450031e544e95e272216aa2 Mon Sep 17 00:00:00 2001 From: Steven Basgall Date: Thu, 26 Sep 2019 17:35:24 -0600 Subject: [PATCH 37/49] remove duplicate 'and' --- docs/tools.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tools.rst b/docs/tools.rst index 81fb4af7a..4decef334 100644 --- a/docs/tools.rst +++ b/docs/tools.rst @@ -58,7 +58,7 @@ Collection A java program which regularly queries metrics from a configured set of inputs, selects and renames them using regular expressions and sends them to a configured set of outputs. It supports JMX and JDBC as inputs and Graphite as output out of the box. `netdata`_ - A fast and efficient monitoring agent that supports graphite backends. It has collection, forwarding, visualization and and monitoring features. Netdata collects common system metrics and a variety of other sources through plugins. + A fast and efficient monitoring agent that supports graphite backends. It has collection, forwarding, visualization and monitoring features. Netdata collects common system metrics and a variety of other sources through plugins. `Sensu`_ A monitoring framework that can route metrics to Graphite. Servers subscribe to sets of checks, so getting metrics from a new server to Graphite is as simple as installing the Sensu client and subscribing. From 7bf8d14beb95c8339458d4398c545b84223508b9 Mon Sep 17 00:00:00 2001 From: Nimish Verma Date: Sat, 5 Oct 2019 10:36:13 -0400 Subject: [PATCH 38/49] Updated minimumBelow() docstring Issue #2355 suggests that minimumBelow() works on <= rather than <, to avoid confusion I have updated docstring --- webapp/graphite/render/functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webapp/graphite/render/functions.py b/webapp/graphite/render/functions.py index 6c12d53c8..52c274e8d 100644 --- a/webapp/graphite/render/functions.py +++ b/webapp/graphite/render/functions.py @@ -2980,7 +2980,7 @@ def maximumBelow(requestContext, seriesList, n): def minimumBelow(requestContext, seriesList, n): """ Takes one metric or a wildcard seriesList followed by a constant n. - Draws only the metrics with a minimum value below n. + Draws only the metrics with a minimum value below or equal to n. Example: From a967c12cc230f17c5b1c79f10eeddff8ee4c3943 Mon Sep 17 00:00:00 2001 From: Dan Cech Date: Mon, 7 Oct 2019 17:05:51 -0400 Subject: [PATCH 39/49] xFilesFactor is an optional parameter for removeEmptySeries --- webapp/graphite/render/functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webapp/graphite/render/functions.py b/webapp/graphite/render/functions.py index 52c274e8d..e435fd72b 100644 --- a/webapp/graphite/render/functions.py +++ b/webapp/graphite/render/functions.py @@ -5463,7 +5463,7 @@ def removeEmptySeries(requestContext, seriesList, xFilesFactor=None): removeEmptySeries.group = 'Filter Series' removeEmptySeries.params = [ Param('seriesList', ParamTypes.seriesList, required=True), - Param('xFilesFactor', ParamTypes.float, required=True), + Param('xFilesFactor', ParamTypes.float), ] From fa09800df106538656fe2c834cf4305ceaf88938 Mon Sep 17 00:00:00 2001 From: Dieter Plaetinck Date: Tue, 8 Oct 2019 10:02:42 -0400 Subject: [PATCH 40/49] fix functions that aggregate to include the aliases in their params This should fix the validation not being aware of legal aliases. fix #2494 --- webapp/graphite/render/functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webapp/graphite/render/functions.py b/webapp/graphite/render/functions.py index e435fd72b..8d4ef614f 100644 --- a/webapp/graphite/render/functions.py +++ b/webapp/graphite/render/functions.py @@ -286,7 +286,7 @@ def formatPathExpressions(seriesList): 'current': aggFuncs['last'], } -aggFuncNames = sorted(aggFuncs.keys()) +aggFuncNames = sorted(aggFuncs.keys() + aggFuncAliases.keys()) def getAggFunc(func, rawFunc=None): From 62214e1cf5accabf6a89f2fe1d2ea6b26b853791 Mon Sep 17 00:00:00 2001 From: Dieter Plaetinck Date: Tue, 8 Oct 2019 10:25:36 -0400 Subject: [PATCH 41/49] make py3 happy also {}.keys() is a dict_keys type which it doesn't know how to add but casting with list() seems to work in both versions --- webapp/graphite/render/functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webapp/graphite/render/functions.py b/webapp/graphite/render/functions.py index 8d4ef614f..4283936b8 100644 --- a/webapp/graphite/render/functions.py +++ b/webapp/graphite/render/functions.py @@ -286,7 +286,7 @@ def formatPathExpressions(seriesList): 'current': aggFuncs['last'], } -aggFuncNames = sorted(aggFuncs.keys() + aggFuncAliases.keys()) +aggFuncNames = sorted(list(aggFuncs.keys()) + list(aggFuncAliases.keys())) def getAggFunc(func, rawFunc=None): From fea6e59956c9f229ceb1517a4a24aef6995d99ac Mon Sep 17 00:00:00 2001 From: Dan Cech Date: Tue, 8 Oct 2019 11:51:13 -0400 Subject: [PATCH 42/49] the callback parameter for groupByNode is optional --- webapp/graphite/render/functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webapp/graphite/render/functions.py b/webapp/graphite/render/functions.py index 4283936b8..07a7df82a 100644 --- a/webapp/graphite/render/functions.py +++ b/webapp/graphite/render/functions.py @@ -4993,7 +4993,7 @@ def groupByNode(requestContext, seriesList, nodeNum, callback='average'): groupByNode.params = [ Param('seriesList', ParamTypes.seriesList, required=True), Param('nodeNum', ParamTypes.nodeOrTag, required=True), - Param('callback', ParamTypes.aggFunc, default='average', options=aggFuncNames, required=True), + Param('callback', ParamTypes.aggFunc, default='average', options=aggFuncNames), ] From b3cd0b62d187eb6e0ad58fba227e63672001a3d8 Mon Sep 17 00:00:00 2001 From: Dieter Plaetinck Date: Tue, 8 Oct 2019 12:06:26 -0400 Subject: [PATCH 43/49] fix order --- webapp/graphite/render/functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webapp/graphite/render/functions.py b/webapp/graphite/render/functions.py index 07a7df82a..03a041b6e 100644 --- a/webapp/graphite/render/functions.py +++ b/webapp/graphite/render/functions.py @@ -286,7 +286,7 @@ def formatPathExpressions(seriesList): 'current': aggFuncs['last'], } -aggFuncNames = sorted(list(aggFuncs.keys()) + list(aggFuncAliases.keys())) +aggFuncNames = sorted(aggFuncs.keys()) + sorted(aggFuncAliases.keys()) def getAggFunc(func, rawFunc=None): From 31ae812533608bf2346b204843de6c10a5b6830a Mon Sep 17 00:00:00 2001 From: deniszh Date: Fri, 11 Oct 2019 21:39:19 +0200 Subject: [PATCH 44/49] Removing function --- webapp/graphite/composer/urls.py | 1 - webapp/graphite/composer/views.py | 31 ------------------------------- 2 files changed, 32 deletions(-) diff --git a/webapp/graphite/composer/urls.py b/webapp/graphite/composer/urls.py index c741c7b56..d3faefea7 100644 --- a/webapp/graphite/composer/urls.py +++ b/webapp/graphite/composer/urls.py @@ -16,7 +16,6 @@ from . import views urlpatterns = [ - url(r'^/send_email', views.send_email, name='composer_send_email'), url(r'^/mygraph', views.mygraph, name='composer_mygraph'), url(r'^/?$', views.composer, name='composer'), ] diff --git a/webapp/graphite/composer/views.py b/webapp/graphite/composer/views.py index 129c8b187..dddf4aac3 100644 --- a/webapp/graphite/composer/views.py +++ b/webapp/graphite/composer/views.py @@ -91,34 +91,3 @@ def mygraph(request): else: return HttpResponse("Invalid operation '%s'" % action) - -def send_email(request): - try: - recipients = request.GET['to'].split(',') - url = request.GET['url'] - proto, server, path, query, frag = urlsplit(url) - if query: path += '?' + query - conn = HTTPConnection(server) - conn.request('GET',path) - try: # Python 2.7+, use buffering of HTTP responses - resp = conn.getresponse(buffering=True) - except TypeError: # Python 2.6 and older - resp = conn.getresponse() - assert resp.status == 200, "Failed HTTP response %s %s" % (resp.status, resp.reason) - rawData = resp.read() - conn.close() - message = MIMEMultipart() - message['Subject'] = "Graphite Image" - message['To'] = ', '.join(recipients) - message['From'] = 'composer@%s' % gethostname() - text = MIMEText( "Image generated by the following graphite URL at %s\r\n\r\n%s" % (ctime(),url) ) - image = MIMEImage( rawData ) - image.add_header('Content-Disposition', 'attachment', filename="composer_" + strftime("%b%d_%I%M%p.png")) - message.attach(text) - message.attach(image) - s = SMTP(settings.SMTP_SERVER) - s.sendmail('composer@%s' % gethostname(),recipients,message.as_string()) - s.quit() - return HttpResponse( "OK" ) - except Exception: - return HttpResponse(format_exc()) From 91ca9211a4f8b7ec90087c23b203d7eec59ead82 Mon Sep 17 00:00:00 2001 From: deniszh Date: Fri, 11 Oct 2019 21:43:22 +0200 Subject: [PATCH 45/49] Removing test also --- webapp/tests/test_composer.py | 34 ---------------------------------- 1 file changed, 34 deletions(-) delete mode 100644 webapp/tests/test_composer.py diff --git a/webapp/tests/test_composer.py b/webapp/tests/test_composer.py deleted file mode 100644 index f7dcc270d..000000000 --- a/webapp/tests/test_composer.py +++ /dev/null @@ -1,34 +0,0 @@ -import mock - -from urllib3.response import HTTPResponse -from graphite.util import BytesIO - -from .base import TestCase -try: - from django.urls import reverse -except ImportError: # Django < 1.10 - from django.core.urlresolvers import reverse - - -class ComposerTest(TestCase): - @mock.patch('six.moves.http_client.HTTPConnection.request') - @mock.patch('six.moves.http_client.HTTPConnection.getresponse') - @mock.patch('graphite.composer.views.SMTP') - @mock.patch('django.conf.settings.SMTP_SERVER', 'localhost') - def test_send_email(self, mock_smtp, http_response, http_request): - url = reverse('composer_send_email') - request = { "to": "noreply@localhost", - "url": 'https://localhost:8000/render?target=sumSeries(a.b.c.d)&title=Test&width=500&from=-55minutes&until=now&height=400'} - - response = self.client.get(reverse('render'), {'target': 'test'}) - self.assertEqual(response['Content-Type'], 'image/png') - data = response.content - responseObject = HTTPResponse(body=BytesIO(data), status=200, preload_content=False) - http_request.return_value = responseObject - http_response.return_value = responseObject - - instance = mock_smtp.return_value - instance.sendmail.return_value = {} - - response = self.client.get(url, request) - self.assertEqual(response.content, b'OK') From fe933256ccf1de631d1c33811e7514209a253a82 Mon Sep 17 00:00:00 2001 From: deniszh Date: Fri, 11 Oct 2019 22:18:10 +0200 Subject: [PATCH 46/49] Removing blank line --- webapp/graphite/composer/views.py | 1 - 1 file changed, 1 deletion(-) diff --git a/webapp/graphite/composer/views.py b/webapp/graphite/composer/views.py index dddf4aac3..03ae27226 100644 --- a/webapp/graphite/composer/views.py +++ b/webapp/graphite/composer/views.py @@ -90,4 +90,3 @@ def mygraph(request): else: return HttpResponse("Invalid operation '%s'" % action) - From 1be38a901ee138e7f8cf8f02d73fefbaceb7edd3 Mon Sep 17 00:00:00 2001 From: deniszh Date: Fri, 11 Oct 2019 22:35:08 +0200 Subject: [PATCH 47/49] Remove SMTP_SERVER and unused imports --- webapp/graphite/composer/views.py | 9 --------- webapp/graphite/settings.py | 1 - 2 files changed, 10 deletions(-) diff --git a/webapp/graphite/composer/views.py b/webapp/graphite/composer/views.py index 03ae27226..4f61931b4 100644 --- a/webapp/graphite/composer/views.py +++ b/webapp/graphite/composer/views.py @@ -13,15 +13,6 @@ limitations under the License.""" import os -from smtplib import SMTP -from socket import gethostname -from email.mime.multipart import MIMEMultipart -from email.mime.text import MIMEText -from email.mime.image import MIMEImage -from six.moves.http_client import HTTPConnection -from six.moves.urllib.parse import urlsplit -from time import ctime, strftime -from traceback import format_exc from graphite.user_util import getProfile from graphite.logger import log from graphite.account.models import MyGraph diff --git a/webapp/graphite/settings.py b/webapp/graphite/settings.py index 9a66c0327..fe641acc1 100644 --- a/webapp/graphite/settings.py +++ b/webapp/graphite/settings.py @@ -124,7 +124,6 @@ REMOTE_RENDER_CONNECT_TIMEOUT = 1.0 #Miscellaneous settings -SMTP_SERVER = "localhost" DOCUMENTATION_VERSION = 'latest' if 'dev' in WEBAPP_VERSION else WEBAPP_VERSION DOCUMENTATION_URL = 'https://graphite.readthedocs.io/en/{}/'.format(DOCUMENTATION_VERSION) ALLOW_ANONYMOUS_CLI = True From 0845cb71b267d2599bd0ea71c647019f5d93394c Mon Sep 17 00:00:00 2001 From: deniszh Date: Wed, 23 Oct 2019 13:52:01 +0200 Subject: [PATCH 48/49] Upgrading minimal Django version --- docs/requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index caa60a862..137c8c3e5 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,6 +1,6 @@ ## Requirements for documentation -Django>=1.4,<1.8 -django-tagging==0.3.1 +Django>=1.11.19,<2.3 +django-tagging==0.4.6 sphinx sphinx_rtd_theme pytz From 8699b8baef51e9d0d4dcdd0b0c78a2a34b2d37fb Mon Sep 17 00:00:00 2001 From: deniszh Date: Wed, 23 Oct 2019 15:45:05 +0200 Subject: [PATCH 49/49] Cleanup of tox.ini --- tox.ini | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tox.ini b/tox.ini index 3a279f27d..61ea9e74d 100644 --- a/tox.ini +++ b/tox.ini @@ -28,9 +28,6 @@ deps = git+git://github.com/graphite-project/whisper.git#egg=whisper git+git://github.com/graphite-project/ceres.git#egg=ceres pyparsing2: pyparsing - django18: Django>=1.8,<1.8.99 - django19: Django>=1.9,<1.9.99 - django110: Django>=1.10,<1.10.99 django111: Django>=1.11,<1.11.99 django20: Django>=2.0,<2.0.99 django21: Django>=2.1,<2.1.99 @@ -53,7 +50,7 @@ deps = pytz git+git://github.com/graphite-project/whisper.git#egg=whisper git+git://github.com/graphite-project/ceres.git#egg=ceres - Django<2.0 + Django<2.3 pyparsing Sphinx<1.4 sphinx_rtd_theme