From 230e6026340b33ffd558baa1e0ebf47142901f9a Mon Sep 17 00:00:00 2001 From: Joseph Hit Hard Date: Mon, 17 Dec 2018 11:44:10 +0530 Subject: [PATCH] Enhancements (#22) Add support for reading token and global tags from environment variables. --- README.md | 18 ++++ apptuit/apptuit_client.py | 119 ++++++++++++++------- tests/test_send.py | 25 ++--- tests/test_token_and_tags.py | 202 +++++++++++++++++++++++++++++++++++ 4 files changed, 307 insertions(+), 57 deletions(-) create mode 100644 tests/test_token_and_tags.py diff --git a/README.md b/README.md index 45914f7..c09b862 100644 --- a/README.md +++ b/README.md @@ -19,10 +19,28 @@ Supported Python versions: 2.7.x, 3.4, 3.5, 3.6, 3.7 ## Usage ### Contents + - [Configuration](#configuration) - [Sending Data](#sending-data) * [Sending Data using ApptuitReporter](#sending-the-data-using-apptuitreporter) * [Sending Data using `send()` API](#sending-data-using-send-api) - [Querying for Data](#querying-for-data) + +### Configuration + +Apptuit package supports 2 environmental variables `APPTUIT_API_TOKEN` +and `APPTUIT_PY_TAGS`: +* `APPTUIT_API_TOKEN`: If this environmental variable is set then you don't have +to pass token parameter to `Apptuit()` and `ApptuitReporter()`. This will work as +a secondary token, if you pass the token to `Apptuit()` and `ApptuitReporter()` +then that token will take priority. +* `APPTUIT_PY_TAGS`: If this environmental variable is set then these tags will +act as global tags, and these tags will be sent in all send api calls. The tags +have to be specified in key value pair as follows: + ``` + APPTUIT_PY_TAGS: tag_key1: tag_val1, tag_key2: tag_val2 ,..., tag_keyN: tag_valN + ``` + If tag key in environmental variable matches tag key to `DataPoint` then + `DataPoint` tags take priority. ### Sending data diff --git a/apptuit/apptuit_client.py b/apptuit/apptuit_client.py index 79c8e4f..a9c93ea 100644 --- a/apptuit/apptuit_client.py +++ b/apptuit/apptuit_client.py @@ -2,6 +2,7 @@ Client module for Apptuit APIs """ +import os from collections import defaultdict import json from string import ascii_letters, digits @@ -15,25 +16,14 @@ import requests import pandas as pd +APPTUIT_API_TOKEN = "APPTUIT_API_TOKEN" +APPTUIT_PY_TAGS = "APPTUIT_PY_TAGS" VALID_CHARSET = set(ascii_letters + digits + "-_./") INVALID_CHARSET = frozenset(map(chr, range(128))) - VALID_CHARSET def _contains_valid_chars(string): return INVALID_CHARSET.isdisjoint(string) - -def _create_payload(datapoints): - data = [] - for dp in datapoints: - row = {} - row["metric"] = dp.metric - row["timestamp"] = dp.timestamp - row["value"] = dp.value - row["tags"] = dp.tags - data.append(row) - return data - - def _generate_query_string(query_string, start, end): ret = "?start=" + str(start) if end: @@ -69,12 +59,44 @@ def _parse_response(resp, start, end=None): qresult[output_id].series.append(series) return qresult +def _get_tags_from_environment(): + tags_str = os.environ.get(APPTUIT_PY_TAGS) + if not tags_str: + return {} + tags = {} + tags_str = tags_str.strip(", ") + tags_split = tags_str.split(',') + for tag in tags_split: + try: + tag = tag.strip() + if not tag: + continue + key, val = tag.split(":") + tags[key.strip()] = val.strip() + except ValueError: + raise ValueError("Invalid format of " + + APPTUIT_PY_TAGS + + ", failed to parse tag key-value pair '" + + tag + "' format should be like" + "'tag_key1:tag_val1,tag_key2:tag_val2,...,tag_keyN:tag_valN'") + _validate_tags(tags) + return tags + +def _validate_tags(tags): + for tagk, tagv in tags.items(): + if not _contains_valid_chars(tagk): + raise ValueError("Tag key %s contains an invalid character, " + "allowed characters are a-z, A-Z, 0-9, -, _, ., and /" % tagk) + if not _contains_valid_chars(str(tagv)): + raise ValueError("Tag value %s contains an invalid character, " + "allowed characters are a-z, A-Z, 0-9, -, _, ., and /" % tagv) + class Apptuit(object): """ Apptuit is the client object, encapsulating the functionalities provided by Apptuit APIs """ - def __init__(self, token, api_endpoint="https://api.apptuit.ai/", debug=False): + def __init__(self, token=None, api_endpoint="https://api.apptuit.ai/", debug=False): """ Creates an apptuit client object Params: @@ -83,13 +105,45 @@ def __init__(self, token, api_endpoint="https://api.apptuit.ai/", debug=False): port: Port on which the service is running """ - if not token: - raise ValueError("Invalid token") self.token = token + if not self.token: + self.token = os.environ.get(APPTUIT_API_TOKEN) + if not self.token: + raise ValueError("Missing Apptuit API token, " + "either pass it as a parameter or " + "set as value of the environment variable '" + + APPTUIT_API_TOKEN + "'.") self.endpoint = api_endpoint if self.endpoint[-1] == '/': self.endpoint = self.endpoint[:-1] self.debug = debug + self._environ_tags = _get_tags_from_environment() + + def _create_payload(self, datapoints): + data = [] + for dp in datapoints: + if dp.tags and self._environ_tags: + tags = self._environ_tags.copy() + tags.update(dp.tags) + elif dp.tags: + tags = dp.tags + else: + tags = self._environ_tags + if not tags: + raise ValueError("Missing tags for the metric " + + dp.metric + + ". Either pass it as value of the tags" + " parameter to DataPoint or" + " set environment variable '" + + APPTUIT_PY_TAGS + + "' for global tags") + row = {} + row["metric"] = dp.metric + row["timestamp"] = dp.timestamp + row["value"] = dp.value + row["tags"] = tags + data.append(row) + return data def send(self, datapoints): """ @@ -98,8 +152,10 @@ def send(self, datapoints): datapoints: A list of DataPoint objects It raises an ApptuitException in case the backend API responds with an error """ + if not datapoints: + return url = self.endpoint + "/api/put?sync&sync=60000" - data = _create_payload(datapoints) + data = self._create_payload(datapoints) body = json.dumps(data) body = zlib.compress(body.encode("utf-8")) headers = {} @@ -203,18 +259,9 @@ def tags(self): def tags(self, tags): if not isinstance(tags, dict): raise ValueError("tags parameter is expected to be a dict type") - for tagk, tagv in tags.items(): - if not _contains_valid_chars(tagk): - raise ValueError("tag key %s contains a character which is not allowed, " - "only characters [a-z], [A-Z], [0-9] and [-_./] are allowed" - % (tagk)) - if not _contains_valid_chars(str(tagv)): - raise ValueError("tag value %s contains a character which is not allowed, " - "only characters [a-z], [A-Z], [0-9] and [-_./] are allowed" - % (tagv)) + _validate_tags(tags) self._tags = tags - def __repr__(self): repr_str = '%s{' % self.metric for tagk in sorted(self.tags): @@ -300,8 +347,6 @@ def __init__(self, metric, tags, timestamp, value): timestamp: Number of seconds since Unix epoch value: value of the metric at this timestamp (int or float) """ - if tags is None or tags == {}: - raise ValueError("Ivalid tags: Metric: "+metric+" need minimum one tag.") self.metric = metric self.tags = tags self.timestamp = timestamp @@ -324,15 +369,13 @@ def tags(self): @tags.setter def tags(self, tags): - if not isinstance(tags, dict): - raise ValueError("Expected a value of type dict for tags") - for tagk, tagv in tags.items(): - if not _contains_valid_chars(tagk): - raise ValueError("Tag key %s contains an invalid character, " - "allowed characters are a-z, A-Z, 0-9, -, _, ., and /" % tagk) - if not _contains_valid_chars(str(tagv)): - raise ValueError("Tag value %s contains an invalid character, " - "allowed characters are a-z, A-Z, 0-9, -, _, ., and /" % tagv) + if tags is None: + self._tags = tags + return + else: + if not isinstance(tags, dict): + raise ValueError("Expected a value of type dict for tags") + _validate_tags(tags) self._tags = tags @property diff --git a/tests/test_send.py b/tests/test_send.py index 2dcadd8..b5fea13 100644 --- a/tests/test_send.py +++ b/tests/test_send.py @@ -19,24 +19,19 @@ def test_send_positive(mock_post): metric_name = "node.load_avg.1m" tags = {"host": "localhost", "region": "us-east-1", "service": "web-server"} dps = [] + client.send(dps) points_sent = 0 while True: ts = int(time.time()) dps.append(DataPoint(metric_name, tags, ts, random.random())) if len(dps) == 100: - try: - client.send(dps) - except ApptuitException: - ok_(False) + client.send(dps) dps = [] points_sent += 100 if points_sent > 500: break if dps: - try: - client.send(dps) - except ApptuitException: - ok_(False) + client.send(dps) @patch('apptuit.apptuit_client.requests.post') def test_send_server_error(mock_post): @@ -83,18 +78,10 @@ def test_invalid_chars_in_tag_keys(): ts = int(time.time()) with assert_raises(ValueError) as ex: DataPoint(metric_name, tags, ts, random.random()) - -def test_no_tag(): - """ - Test for no tag keys - """ - metric_name = "node.load_avg.1m" - ts = int(time.time()) with assert_raises(ValueError) as ex: - DataPoint(metric_name, None, ts, random.random()) - with assert_raises(ValueError) as ex: - DataPoint(metric_name, {}, ts, random.random()) - + DataPoint(metric_name, "error", ts, random.random()) + dp = DataPoint(metric_name, None, ts, random.random()) + assert_equals(dp.tags,None) def test_invalid_chars_in_tag_values(): """ diff --git a/tests/test_token_and_tags.py b/tests/test_token_and_tags.py new file mode 100644 index 0000000..bb35ae9 --- /dev/null +++ b/tests/test_token_and_tags.py @@ -0,0 +1,202 @@ +""" +Tests on Apptuit class +""" +import os +from nose.tools import assert_equals, assert_raises +from pyformance import MetricsRegistry + +from apptuit.apptuit_client import APPTUIT_API_TOKEN, APPTUIT_PY_TAGS, _get_tags_from_environment + +from apptuit import Apptuit, DataPoint +from apptuit.pyformance import ApptuitReporter + +try: + from unittest.mock import Mock, patch +except ImportError: + from mock import Mock, patch + +def _tester_test_token_positive(token_environ, token_argument, token_test): + """ + to test test_token_positive + """ + mock_environ = patch.dict(os.environ, {APPTUIT_API_TOKEN: token_environ}) + mock_environ.start() + client = Apptuit(token=token_argument) + assert_equals(client.token, token_test) + mock_environ.stop() + +def test_token_positive(): + """ + Test that token is working normally + """ + _tester_test_token_positive("environ_token","","environ_token") + _tester_test_token_positive("environ_token", None, "environ_token") + _tester_test_token_positive("environ_token", "argument_token", "argument_token") + +def _get_tags_from_environ_negative(test_environ_str): + """ + to test test_get_tags_from_environ + """ + mock_environ = patch.dict(os.environ, { + APPTUIT_PY_TAGS: test_environ_str}) + mock_environ.start() + with assert_raises(ValueError): + _get_tags_from_environment() + mock_environ.stop() + +def _get_tags_from_environ_positive(test_tags_str,test_tags_dict): + """ + to test test_get_tags_from_environ + """ + mock_environ = patch.dict(os.environ, { + APPTUIT_PY_TAGS: test_tags_str}) + mock_environ.start() + assert_equals(_get_tags_from_environment(), test_tags_dict) + mock_environ.stop() + +def test_get_tags_from_environ(): + """ + Test that _get_tags_from_environment is working + """ + _get_tags_from_environ_positive(" ", {}) + _get_tags_from_environ_positive('tagk1: 22, tagk2: tagv2', + {"tagk1": "22", "tagk2": "tagv2"}) + _get_tags_from_environ_positive('tagk1: 22, , tagk2: tagv2', + {"tagk1": "22", "tagk2": "tagv2"}) + _get_tags_from_environ_positive(' tagk1 : 22,,tagk2 : tagv2 ', + {"tagk1": "22", "tagk2": "tagv2"}) + _get_tags_from_environ_positive(', , , , tagk1: 22, tagk2: tagv2, , , , ,', + {"tagk1": "22", "tagk2": "tagv2"}) + _get_tags_from_environ_positive(',tagk1: 22, tagk2: tagv2,', + {"tagk1": "22", "tagk2": "tagv2"}) + _get_tags_from_environ_negative('"tagk1":tagv1') + _get_tags_from_environ_negative('tagk1:tagv11:tagv12') + _get_tags_from_environ_negative('tag') + _get_tags_from_environ_negative(' tagk1 : 22,error,tagk2 : tagv2 ') + _get_tags_from_environ_negative(' tagk1 : 22,tagk1:tagv11:tagv12,tagk2 : tagv2 ') + + + +def test_token_negative(): + """ + Test that invalid token raises error + """ + mock_environ = patch.dict(os.environ, {}) + mock_environ.start() + with assert_raises(ValueError): + Apptuit() + with assert_raises(ValueError): + Apptuit(token="") + with assert_raises(ValueError): + Apptuit(token=None) + mock_environ.stop() + +def test_tags_positive(): + """ + Test that tags work normally + """ + mock_environ = patch.dict(os.environ, {APPTUIT_API_TOKEN: "environ_token", + APPTUIT_PY_TAGS: 'tagk1: 22, tagk2: tagv2'}) + mock_environ.start() + client = Apptuit() + assert_equals(client._environ_tags, {"tagk1": "22", "tagk2": "tagv2"}) + mock_environ.stop() + +def test_tags_negative(): + """ + Test that invalid tag raises error + """ + mock_environ = patch.dict(os.environ, {APPTUIT_API_TOKEN: "environ_token"}) + mock_environ.start() + client = Apptuit() + assert_equals({}, client._environ_tags) + mock_environ.stop() + mock_environ = patch.dict(os.environ, {APPTUIT_API_TOKEN: "environ_token", APPTUIT_PY_TAGS: "{InvalidTags"}) + mock_environ.start() + with assert_raises(ValueError): + Apptuit() + mock_environ.stop() + mock_environ = patch.dict(os.environ, {APPTUIT_API_TOKEN: "environ_token", APPTUIT_PY_TAGS: '"tagk1":"tagv1"'}) + mock_environ.start() + with assert_raises(ValueError): + Apptuit() + mock_environ.stop() + +def test_datapoint_tags_take_priority(): + """ + Test that datapoint tags take priority + """ + mock_environ = patch.dict(os.environ, {APPTUIT_API_TOKEN: "environ_token", + APPTUIT_PY_TAGS: 'host: host1, ip: 1.1.1.1'}) + mock_environ.start() + client = Apptuit() + test_val = 123 + dp1 = DataPoint("test_metric", {"host": "host2", "ip": "2.2.2.2", "test": 1}, test_val, test_val) + dp2 = DataPoint("test_metric", {"test": 2}, test_val, test_val) + dp3 = DataPoint("test_metric", {}, test_val, test_val) + payload = client._create_payload([dp1, dp2, dp3]) + assert_equals(len(payload), 3) + assert_equals(payload[0]["tags"], {"host": "host2", "ip": "2.2.2.2", "test": 1}) + assert_equals(payload[1]["tags"], {"host": "host1", "ip": "1.1.1.1", "test": 2}) + assert_equals(payload[2]["tags"], {"host": "host1", "ip": "1.1.1.1"}) + assert_equals(client._environ_tags, {"host": "host1", "ip": "1.1.1.1"}) + mock_environ.stop() + +def test_no_environ_tags(): + """ + Test No Environ tags work + """ + test_val = 123 + mock_environ = patch.dict(os.environ, {APPTUIT_API_TOKEN: "environ_token"}) + mock_environ.start() + client = Apptuit() + dp1 = DataPoint("test_metric", {"host": "host2", "ip": "2.2.2.2", "test": 1}, test_val, test_val) + dp2 = DataPoint("test_metric", {"test": 2}, test_val, test_val) + payload = client._create_payload([dp1, dp2]) + assert_equals(len(payload), 2) + assert_equals(payload[0]["tags"], {"host": "host2", "ip": "2.2.2.2", "test": 1}) + assert_equals(payload[1]["tags"], {"test": 2}) + registry = MetricsRegistry() + counter = registry.counter("counter") + counter.inc(1) + reporter = ApptuitReporter(registry=registry, tags={"host": "reporter", "ip": "2.2.2.2"}) + payload = reporter.client._create_payload(reporter._collect_data_points(reporter.registry)) + assert_equals(len(payload), 1) + assert_equals(payload[0]["tags"], {'host': 'reporter', 'ip': '2.2.2.2'}) + mock_environ.stop() + +def test_reporter_tags_take_priority(): + """ + Test that reporter tags take priority + """ + mock_environ = patch.dict(os.environ, {APPTUIT_API_TOKEN: "environ_token", + APPTUIT_PY_TAGS: 'host: environ, ip: 1.1.1.1'}) + mock_environ.start() + registry = MetricsRegistry() + counter = registry.counter("counter") + counter.inc(1) + reporter = ApptuitReporter(registry=registry, tags={"host": "reporter", "ip": "2.2.2.2"}) + payload = reporter.client._create_payload(reporter._collect_data_points(reporter.registry)) + assert_equals(len(payload), 1) + assert_equals(payload[0]["tags"], {'host': 'reporter', 'ip': '2.2.2.2'}) + reporter = ApptuitReporter(registry=registry) + payload = reporter.client._create_payload(reporter._collect_data_points(reporter.registry)) + assert_equals(len(payload), 1) + assert_equals(payload[0]["tags"], {"host": "environ", "ip": "1.1.1.1"}) + mock_environ.stop() + +def test_tags_of_metric_take_priority(): + """ + Test that metric tags take priority + """ + mock_environ = patch.dict(os.environ, {APPTUIT_API_TOKEN: "environ_token", + APPTUIT_PY_TAGS: 'host: environ, ip: 1.1.1.1'}) + mock_environ.start() + registry = MetricsRegistry() + counter = registry.counter('counter {"host": "metric", "ip": "3.3.3.3"}') + counter.inc(1) + reporter = ApptuitReporter(registry=registry, tags={"host": "reporter", "ip": "2.2.2.2"}) + payload = reporter.client._create_payload(reporter._collect_data_points(reporter.registry)) + assert_equals(len(payload), 1) + assert_equals(payload[0]["tags"], {"host": "metric", "ip": "3.3.3.3"}) + mock_environ.stop()