diff --git a/README.md b/README.md index 3d941e7..c74a9b2 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ ## Installation ``` -pip install apptuit +pip install apptuit --upgrade ``` ## Dependencies @@ -19,10 +19,30 @@ Supported Python versions: 2.7.x, 3.4, 3.5, 3.6, 3.7 ## Usage ### Contents + - [Configuration](#configuration) - [Sending Data](#sending-data) * [Sending Data using ApptuitReporter](#sending-the-data-using-apptuitreporter) * [Sending Data using `send()` API](#sending-data-using-send-api) - [Querying for Data](#querying-for-data) + +### Configuration + +Apptuit supports two environmental variables `APPTUIT_PY_TOKEN` and `APPTUIT_PY_TAGS`: + +* `APPTUIT_PY_TOKEN`: This can be used to set the Apptuit API token. If set, then we don't +need to pass the token as a parameter when working with the apptuit client or the apptuit reporter. +* `APPTUIT_PY_TAGS`: This can be used to set the global tags for apptuit. The apptuit client and reporter +will add these tags with each datapoint they are sending to Apptuit. These tags will work in combination +with any tags set with the reporter as well as set with individual metrics and datapoints. If any metric +shares a tag key in common with the global tags, the value of the tag from the metric takes preference. +The format of the value of this variable is as follows: + ```sh + export APPTUIT_PY_TAGS="tag_key1: tag_val1, tag_key2: tag_val2, tag_key3: tag_val3" + ``` +The spaces after the comma and colon are optional. + +**Note**: Support for these variable has been added in the development version of apptuit-py and is not available +in any of the released versions. ### Sending data diff --git a/apptuit/__init__.py b/apptuit/__init__.py index c7a2607..f261a94 100644 --- a/apptuit/__init__.py +++ b/apptuit/__init__.py @@ -1,6 +1,6 @@ from .apptuit_client import Apptuit, DataPoint, ApptuitException from apptuit import pyformance, timeseries -__version__ = '0.3.0' +__version__ = '0.3.1' __all__ = ['Apptuit', 'DataPoint', 'ApptuitException', 'pyformance', 'timeseries'] diff --git a/apptuit/apptuit_client.py b/apptuit/apptuit_client.py index 79c8e4f..859fcbd 100644 --- a/apptuit/apptuit_client.py +++ b/apptuit/apptuit_client.py @@ -2,6 +2,7 @@ Client module for Apptuit APIs """ +import os from collections import defaultdict import json from string import ascii_letters, digits @@ -15,25 +16,14 @@ import requests import pandas as pd +APPTUIT_PY_TOKEN = "APPTUIT_PY_TOKEN" +APPTUIT_PY_TAGS = "APPTUIT_PY_TAGS" VALID_CHARSET = set(ascii_letters + digits + "-_./") INVALID_CHARSET = frozenset(map(chr, range(128))) - VALID_CHARSET def _contains_valid_chars(string): return INVALID_CHARSET.isdisjoint(string) - -def _create_payload(datapoints): - data = [] - for dp in datapoints: - row = {} - row["metric"] = dp.metric - row["timestamp"] = dp.timestamp - row["value"] = dp.value - row["tags"] = dp.tags - data.append(row) - return data - - def _generate_query_string(query_string, start, end): ret = "?start=" + str(start) if end: @@ -69,27 +59,89 @@ def _parse_response(resp, start, end=None): qresult[output_id].series.append(series) return qresult +def _get_tags_from_environment(): + tags_str = os.environ.get(APPTUIT_PY_TAGS) + if not tags_str: + return {} + tags = {} + tags_str = tags_str.strip(", ") + tags_split = tags_str.split(',') + for tag in tags_split: + tag = tag.strip() + if not tag: + continue + try: + key, val = tag.split(":") + tags[key.strip()] = val.strip() + except ValueError: + raise ValueError("Invalid format for " + + APPTUIT_PY_TAGS + + ", failed to parse tag key-value pair '" + + tag + "', " + APPTUIT_PY_TAGS + " should be in the format - " + "'tag_key1:tag_val1,tag_key2:tag_val2,...,tag_keyN:tag_valN'") + _validate_tags(tags) + return tags + +def _validate_tags(tags): + for tagk, tagv in tags.items(): + if not _contains_valid_chars(tagk): + raise ValueError("Tag key %s contains an invalid character, " + "allowed characters are a-z, A-Z, 0-9, -, _, ., and /" % tagk) + if not _contains_valid_chars(str(tagv)): + raise ValueError("Tag value %s contains an invalid character, " + "allowed characters are a-z, A-Z, 0-9, -, _, ., and /" % tagv) + class Apptuit(object): """ Apptuit is the client object, encapsulating the functionalities provided by Apptuit APIs """ - def __init__(self, token, api_endpoint="https://api.apptuit.ai/", debug=False): + def __init__(self, token=None, api_endpoint="https://api.apptuit.ai", debug=False): """ Creates an apptuit client object Params: token: Token of the tenant to which we wish to connect api_endpoint: Apptuit API End point (including the protocol and port) - port: Port on which the service is running - """ - if not token: - raise ValueError("Invalid token") self.token = token + if not self.token: + self.token = os.environ.get(APPTUIT_PY_TOKEN) + if not self.token: + raise ValueError("Missing Apptuit API token, " + "either pass it as a parameter or " + "set as value of the environment variable '" + + APPTUIT_PY_TOKEN + "'.") self.endpoint = api_endpoint if self.endpoint[-1] == '/': self.endpoint = self.endpoint[:-1] self.debug = debug + self._environ_tags = _get_tags_from_environment() + + def _create_payload(self, datapoints): + data = [] + for dp in datapoints: + if dp.tags and self._environ_tags: + tags = self._environ_tags.copy() + tags.update(dp.tags) + elif dp.tags: + tags = dp.tags + else: + tags = self._environ_tags + if not tags: + raise ValueError("Missing tags for the metric " + + dp.metric + + ". Either pass it as value of the tags" + " parameter to DataPoint or" + " set environment variable '" + + APPTUIT_PY_TAGS + + "' for global tags") + row = {} + row["metric"] = dp.metric + row["timestamp"] = dp.timestamp + row["value"] = dp.value + row["tags"] = tags + data.append(row) + return data def send(self, datapoints): """ @@ -98,8 +150,10 @@ def send(self, datapoints): datapoints: A list of DataPoint objects It raises an ApptuitException in case the backend API responds with an error """ + if not datapoints: + return url = self.endpoint + "/api/put?sync&sync=60000" - data = _create_payload(datapoints) + data = self._create_payload(datapoints) body = json.dumps(data) body = zlib.compress(body.encode("utf-8")) headers = {} @@ -203,18 +257,9 @@ def tags(self): def tags(self, tags): if not isinstance(tags, dict): raise ValueError("tags parameter is expected to be a dict type") - for tagk, tagv in tags.items(): - if not _contains_valid_chars(tagk): - raise ValueError("tag key %s contains a character which is not allowed, " - "only characters [a-z], [A-Z], [0-9] and [-_./] are allowed" - % (tagk)) - if not _contains_valid_chars(str(tagv)): - raise ValueError("tag value %s contains a character which is not allowed, " - "only characters [a-z], [A-Z], [0-9] and [-_./] are allowed" - % (tagv)) + _validate_tags(tags) self._tags = tags - def __repr__(self): repr_str = '%s{' % self.metric for tagk in sorted(self.tags): @@ -300,8 +345,6 @@ def __init__(self, metric, tags, timestamp, value): timestamp: Number of seconds since Unix epoch value: value of the metric at this timestamp (int or float) """ - if tags is None or tags == {}: - raise ValueError("Ivalid tags: Metric: "+metric+" need minimum one tag.") self.metric = metric self.tags = tags self.timestamp = timestamp @@ -324,15 +367,12 @@ def tags(self): @tags.setter def tags(self, tags): + self._tags = None + if tags is None: + return if not isinstance(tags, dict): raise ValueError("Expected a value of type dict for tags") - for tagk, tagv in tags.items(): - if not _contains_valid_chars(tagk): - raise ValueError("Tag key %s contains an invalid character, " - "allowed characters are a-z, A-Z, 0-9, -, _, ., and /" % tagk) - if not _contains_valid_chars(str(tagv)): - raise ValueError("Tag value %s contains an invalid character, " - "allowed characters are a-z, A-Z, 0-9, -, _, ., and /" % tagv) + _validate_tags(tags) self._tags = tags @property @@ -343,7 +383,6 @@ def value(self): def value(self, value): if isinstance(value, (int, float)): self._value = value - elif isinstance(value, str): try: self._value = float(value) diff --git a/setup.py b/setup.py index 190e31c..7eb15a7 100644 --- a/setup.py +++ b/setup.py @@ -2,8 +2,8 @@ setup( name="apptuit", - packages=['apptuit'], - version="0.3.0", + packages=['apptuit', 'apptuit.pyformance'], + version="0.3.1", description="Apptuit Python Client", url="https://github.com/ApptuitAI/apptuit-py", author="Abhinav Upadhyay", diff --git a/tests/test_send.py b/tests/test_send.py index 2dcadd8..b5fea13 100644 --- a/tests/test_send.py +++ b/tests/test_send.py @@ -19,24 +19,19 @@ def test_send_positive(mock_post): metric_name = "node.load_avg.1m" tags = {"host": "localhost", "region": "us-east-1", "service": "web-server"} dps = [] + client.send(dps) points_sent = 0 while True: ts = int(time.time()) dps.append(DataPoint(metric_name, tags, ts, random.random())) if len(dps) == 100: - try: - client.send(dps) - except ApptuitException: - ok_(False) + client.send(dps) dps = [] points_sent += 100 if points_sent > 500: break if dps: - try: - client.send(dps) - except ApptuitException: - ok_(False) + client.send(dps) @patch('apptuit.apptuit_client.requests.post') def test_send_server_error(mock_post): @@ -83,18 +78,10 @@ def test_invalid_chars_in_tag_keys(): ts = int(time.time()) with assert_raises(ValueError) as ex: DataPoint(metric_name, tags, ts, random.random()) - -def test_no_tag(): - """ - Test for no tag keys - """ - metric_name = "node.load_avg.1m" - ts = int(time.time()) with assert_raises(ValueError) as ex: - DataPoint(metric_name, None, ts, random.random()) - with assert_raises(ValueError) as ex: - DataPoint(metric_name, {}, ts, random.random()) - + DataPoint(metric_name, "error", ts, random.random()) + dp = DataPoint(metric_name, None, ts, random.random()) + assert_equals(dp.tags,None) def test_invalid_chars_in_tag_values(): """ diff --git a/tests/test_token_and_tags.py b/tests/test_token_and_tags.py new file mode 100644 index 0000000..aa0a58c --- /dev/null +++ b/tests/test_token_and_tags.py @@ -0,0 +1,216 @@ +""" +Tests for token and global tags environment variables +""" + +import math +import time +import os +from nose.tools import assert_equals, assert_raises +from pyformance import MetricsRegistry + +from apptuit.apptuit_client import APPTUIT_PY_TOKEN, APPTUIT_PY_TAGS +from apptuit import apptuit_client, Apptuit, DataPoint +from apptuit.pyformance import ApptuitReporter + +try: + from unittest.mock import Mock, patch +except ImportError: + from mock import Mock, patch + +def test_token_positive(): + """ + Test various combinations of token argument to apptuit, as parameter and as env variable + """ + test_cases = [ + ("environ_token", "", "environ_token"), + ("environ_token", None, "environ_token"), + ("environ_token", "argument_token", "argument_token") + ] + for env_token, arg_token, expected in test_cases: + mock_environ = patch.dict(os.environ, {APPTUIT_PY_TOKEN: env_token}) + mock_environ.start() + client = Apptuit(token=arg_token) + assert_equals(client.token, expected) + mock_environ.stop() + +def test_tags_env_variable_parsing_negative(): + """ + Test that we fail when the value of global tags env variable is in an invalid format + """ + test_cases = [ + '"tagk1":tagv1', + 'tagk1:tagv11:tagv12', + 'tag', + ' tagk1 : 22,error,tagk2 : tagv2 ', + ' tagk1 : 22,tagk1:tagv11:tagv12,tagk2 : tagv2 ' + ] + for env_tags_value in test_cases: + mock_environ = patch.dict(os.environ, {APPTUIT_PY_TAGS: env_tags_value}) + mock_environ.start() + with assert_raises(ValueError): + apptuit_client._get_tags_from_environment() + mock_environ.stop() + +def test_tags_env_variable_parsing_positive(): + """ + Test that wel are able to parse the global tags from environment variable + """ + test_cases = [ + (" ", {}), + ('tagk1: 22, tagk2: tagv2', {"tagk1": "22", "tagk2": "tagv2"}), + ('tagk1: 22, , tagk2: tagv2', {"tagk1": "22", "tagk2": "tagv2"}), + (' tagk1 : 22,,tagk2 : tagv2 ', {"tagk1": "22", "tagk2": "tagv2"}), + (',tagk1: 22, tagk2: tagv2,', {"tagk1": "22", "tagk2": "tagv2"}), + (', , , , tagk1: 22, tagk2: tagv2, , , , ,', {"tagk1": "22", "tagk2": "tagv2"}) + ] + for env_tags_value, expected_global_tags in test_cases: + mock_environ = patch.dict(os.environ, {APPTUIT_PY_TAGS: env_tags_value}) + mock_environ.start() + assert_equals(apptuit_client._get_tags_from_environment(), expected_global_tags) + mock_environ.stop() + +def test_token_negative(): + """ + Test that if no token parameter is passed and no env variable is set, we get an error + """ + mock_environ = patch.dict(os.environ, {}) + mock_environ.start() + with assert_raises(ValueError): + Apptuit() + with assert_raises(ValueError): + Apptuit(token="") + with assert_raises(ValueError): + Apptuit(token=None) + mock_environ.stop() + +def test_env_global_tags_positive(): + """ + Test that apptuit works with global tags env variable and without them + """ + mock_environ = patch.dict(os.environ, {APPTUIT_PY_TOKEN: "environ_token", + APPTUIT_PY_TAGS: 'tagk1: 22, tagk2: tagv2'}) + mock_environ.start() + client = Apptuit() + assert_equals(client._environ_tags, {"tagk1": "22", "tagk2": "tagv2"}) + mock_environ.stop() + + mock_environ = patch.dict(os.environ, {APPTUIT_PY_TOKEN: "environ_token"}) + mock_environ.start() + client = Apptuit() + assert_equals({}, client._environ_tags) + mock_environ.stop() + +def test_env_global_tags_negative(): + """ + Negative test cases for global tags env variable + """ + mock_environ = patch.dict(os.environ, {APPTUIT_PY_TOKEN: "environ_token", + APPTUIT_PY_TAGS: "{InvalidTags"}) + mock_environ.start() + with assert_raises(ValueError): + Apptuit() + mock_environ.stop() + mock_environ = patch.dict(os.environ, {APPTUIT_PY_TOKEN: "environ_token", + APPTUIT_PY_TAGS: '"tagk1":"tagv1"'}) + mock_environ.start() + with assert_raises(ValueError): + Apptuit() + mock_environ.stop() + +def test_datapoint_tags_and_envtags(): + """ + Test that datapoint tags take priority when global tags env variable is present + """ + mock_environ = patch.dict(os.environ, {APPTUIT_PY_TOKEN: "environ_token", + APPTUIT_PY_TAGS: 'host: host1, ip: 1.1.1.1'}) + mock_environ.start() + client = Apptuit() + timestamp = int(time.time()) + test_val = math.pi + dp1 = DataPoint("test_metric", {"host": "host2", "ip": "2.2.2.2", "test": 1}, + timestamp, test_val) + dp2 = DataPoint("test_metric", {"test": 2}, timestamp, test_val) + dp3 = DataPoint("test_metric", {}, timestamp, test_val) + dp4 = DataPoint("test_metric", None, timestamp, test_val) + payload = client._create_payload([dp1, dp2, dp3, dp4]) + assert_equals(len(payload), 4) + assert_equals(payload[0]["tags"], {"host": "host2", "ip": "2.2.2.2", "test": 1}) + assert_equals(payload[1]["tags"], {"host": "host1", "ip": "1.1.1.1", "test": 2}) + assert_equals(payload[2]["tags"], {"host": "host1", "ip": "1.1.1.1"}) + assert_equals(payload[3]["tags"], {"host": "host1", "ip": "1.1.1.1"}) + assert_equals(client._environ_tags, {"host": "host1", "ip": "1.1.1.1"}) + mock_environ.stop() + +def test_no_environ_tags(): + """ + Test tags work even if no global tags present as env variable + """ + timestamp = int(time.time()) + test_val = math.pi + mock_environ = patch.dict(os.environ, {APPTUIT_PY_TOKEN: "environ_token"}) + mock_environ.start() + client = Apptuit() + dp1 = DataPoint("test_metric", {"host": "host2", "ip": "2.2.2.2", "test": 1}, + timestamp, test_val) + dp2 = DataPoint("test_metric", {"test": 2}, timestamp, test_val) + payload = client._create_payload([dp1, dp2]) + assert_equals(len(payload), 2) + assert_equals(payload[0]["tags"], {"host": "host2", "ip": "2.2.2.2", "test": 1}) + assert_equals(payload[1]["tags"], {"test": 2}) + + registry = MetricsRegistry() + counter = registry.counter("counter") + counter.inc(1) + reporter = ApptuitReporter(registry=registry, tags={"host": "reporter", "ip": "2.2.2.2"}) + payload = reporter.client._create_payload(reporter._collect_data_points(reporter.registry)) + assert_equals(len(payload), 1) + assert_equals(payload[0]["tags"], {'host': 'reporter', 'ip': '2.2.2.2'}) + mock_environ.stop() + +def test_reporter_tags_with_global_env_tags(): + """ + Test that reporter tags take priority + TODO: + We have 8 possible combinations - + 1. global env tags: true, reporter tags: true, metric tags: true + 2. global env tags: true, reporter tags: true, metric tags: false + 3. global env tags: true, reporter tags: false, metric tags: true + 4. global env tags: true, reporter tags: false, metric tags: false + 5. global env tags: false, reporter tags: true, metric tags: true + 6. global env tags: false, reporter tags: true, metric tags: false + 7. global env tags: false, reporter tags: false, metric tags: true + 8. global env tags: false, reporter tags: false, metric tags: false + """ + + + mock_environ = patch.dict(os.environ, {APPTUIT_PY_TOKEN: "environ_token", + APPTUIT_PY_TAGS: 'host: environ, ip: 1.1.1.1'}) + mock_environ.start() + registry = MetricsRegistry() + counter = registry.counter("counter") + counter.inc(1) + reporter = ApptuitReporter(registry=registry, tags={"host": "reporter", "ip": "2.2.2.2"}) + payload = reporter.client._create_payload(reporter._collect_data_points(reporter.registry)) + assert_equals(len(payload), 1) + assert_equals(payload[0]["tags"], {'host': 'reporter', 'ip': '2.2.2.2'}) + reporter = ApptuitReporter(registry=registry) + payload = reporter.client._create_payload(reporter._collect_data_points(reporter.registry)) + assert_equals(len(payload), 1) + assert_equals(payload[0]["tags"], {"host": "environ", "ip": "1.1.1.1"}) + mock_environ.stop() + +def test_tags_of_metric_take_priority(): + """ + Test that metric tags take priority + """ + mock_environ = patch.dict(os.environ, {APPTUIT_PY_TOKEN: "environ_token", + APPTUIT_PY_TAGS: 'host: environ, ip: 1.1.1.1'}) + mock_environ.start() + registry = MetricsRegistry() + counter = registry.counter('counter {"host": "metric", "ip": "3.3.3.3"}') + counter.inc(1) + reporter = ApptuitReporter(registry=registry, tags={"host": "reporter", "ip": "2.2.2.2"}) + payload = reporter.client._create_payload(reporter._collect_data_points(reporter.registry)) + assert_equals(len(payload), 1) + assert_equals(payload[0]["tags"], {"host": "metric", "ip": "3.3.3.3"}) + mock_environ.stop()