[couchbase] migrate couchbase to pytest (DataDog#1514)

* [couchbase] Migrate couchbase to pytest

* removed rake for couchbase
* updated setup.py and created __about__.py
* created tox.ini, requirements-dev.txt, and MANIFEST.in
* renamed imports for datadog_checks/couchbase/couchbase.py
* added integration to constants.py
* re-enabled skipped tests
* add finalizer to setup fixture to clean up container if an exception occurs
* re-work wait_for_... functions that check if the stats have been generated yet
* programmatically get the container's IP and pass that as a fixture to the tests that require it
* container name is now configurable through common.py

couchbase/MANIFEST.in

@@ -0,0 +1,6 @@
+include README.md
+include requirements.in
+include requirements.txt
+include requirements-dev.txt
+graft datadog_checks
+graft tests

couchbase/datadog_checks/__init__.py

@@ -0,0 +1,4 @@
+# (C) Datadog, Inc. 2018
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)
+__path__ = __import__('pkgutil').extend_path(__path__, __name__)

couchbase/datadog_checks/couchbase/__about__.py

@@ -0,0 +1,5 @@
+# (C) Datadog, Inc. 2018
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)
+
+__version__ = "1.3.0"

couchbase/datadog_checks/couchbase/__init__.py

@@ -1,7 +1,10 @@
-from . import couchbase
+# (C) Datadog, Inc. 2018
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)
+from .couchbase import Couchbase
+from .__about__ import __version__
 
-Couchbase = couchbase.Couchbase
-
-__version__ = "1.3.0"
-
-__all__ = ['couchbase']
+__all__ = [
+    '__version__',
+    'Couchbase'
+]

couchbase/datadog_checks/couchbase/couchbase.py

@@ -1,7 +1,7 @@
-# (C) Datadog, Inc. 2013-2017
+# (C) Datadog, Inc. 2018
 # (C) Justin Slattery <[email protected]> 2013
 # All rights reserved
-# Licensed under Simplified BSD License (see LICENSE)
+# Licensed under a 3-clause BSD style license (see LICENSE)
 
 # stdlib
 import re
@@ -10,8 +10,8 @@
 import requests
 
 # project
-from checks import AgentCheck
-from util import headers
+from datadog_checks.checks import AgentCheck
+from datadog_checks.utils.headers import headers
 
 # Constants
 COUCHBASE_STATS_PATH = '/pools/default'
@@ -265,7 +265,8 @@ def _create_metrics(self, data, tags=None):
                     val = self.extract_seconds_value(val)
                 norm_metric_name = self.camel_case_to_joined_lower(metric_name)
                 if norm_metric_name in self.QUERY_STATS:
-                    full_metric_name = '.'.join(['couchbase', 'query', self.camel_case_to_joined_lower(norm_metric_name)])
+                    full_metric_name = '.'.join(['couchbase', 'query',
+                                                self.camel_case_to_joined_lower(norm_metric_name)])
                     self.gauge(full_metric_name, val, tags=tags)
 
     def _get_stats(self, url, instance):
@@ -278,8 +279,7 @@ def _get_stats(self, url, instance):
         if 'user' in instance and 'password' in instance:
             auth = (instance['user'], instance['password'])
 
-        r = requests.get(url, auth=auth, headers=headers(self.agentConfig),
-            timeout=timeout)
+        r = requests.get(url, auth=auth, headers=headers(self.agentConfig), timeout=timeout)
         r.raise_for_status()
         return r.json()
 
@@ -324,15 +324,13 @@ def get_data(self, server, instance):
                 raise Exception("No data returned from couchbase endpoint: %s" % url)
         except requests.exceptions.HTTPError as e:
             self.service_check(self.SERVICE_CHECK_NAME, AgentCheck.CRITICAL,
-                tags=service_check_tags, message=str(e.message))
+                               tags=service_check_tags, message=str(e.message))
             raise
         except Exception as e:
-            self.service_check(self.SERVICE_CHECK_NAME, AgentCheck.CRITICAL,
-                tags=service_check_tags, message=str(e))
+            self.service_check(self.SERVICE_CHECK_NAME, AgentCheck.CRITICAL, tags=service_check_tags, message=str(e))
             raise
         else:
-            self.service_check(self.SERVICE_CHECK_NAME, AgentCheck.OK,
-                tags=service_check_tags)
+            self.service_check(self.SERVICE_CHECK_NAME, AgentCheck.OK, tags=service_check_tags)
 
         couchbase['stats'] = overall_stats
 
@@ -377,7 +375,7 @@ def get_data(self, server, instance):
                     couchbase['query'] = query
             except requests.exceptions.HTTPError:
                 self.log.error("Error accessing the endpoint %s, make sure you're running at least "
-                    "couchbase 4.5 to collect the query monitoring metrics", url)
+                               "couchbase 4.5 to collect the query monitoring metrics", url)
 
         return couchbase
 
@@ -408,4 +406,4 @@ def extract_seconds_value(self, value):
         if unit not in self.TO_SECONDS:
             unit = 'us'
 
-        return float(val)/self.TO_SECONDS[unit]
+        return float(val) / self.TO_SECONDS[unit]

couchbase/manifest.json

@@ -1,8 +1,6 @@
 {
   "maintainer": "[email protected]",
-  "manifest_version": "0.1.1",
-  "max_agent_version": "6.0.0",
-  "min_agent_version": "5.20.0",
+  "manifest_version": "1.0.0",
   "name": "couchbase",
   "short_description": "Track and graph your Couchbase activity and performance metrics.",
   "support": "core",
@@ -11,7 +9,6 @@
     "mac_os",
     "windows"
   ],
-  "version": "1.3.0",
   "guid": "ba7ce7de-4fcb-4418-8c90-329baa6a5d59",
   "public_title": "Datadog-CouchBase Integration",
   "categories":["data store"],

couchbase/requirements-dev.txt

@@ -0,0 +1,2 @@
+mock==2.0.0
+pytest

couchbase/setup.py

@@ -1,71 +1,33 @@
+# (C) Datadog, Inc. 2018
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)
+
 # Always prefer setuptools over distutils
 from setuptools import setup
 # To use a consistent encoding
 from codecs import open
 from os import path
 
-import json
-import re
-
-here = path.abspath(path.dirname(__file__))
-
-def parse_req_line(line):
-    line = line.strip()
-    if not line or line.startswith('--hash') or line[0] == '#':
-        return None
-    req = line.rpartition('#')
-    if len(req[1]) == 0:
-        line = req[2].strip()
-    else:
-        line = req[1].strip()
-
-    if '--hash=' in line:
-        line = line[:line.find('--hash=')].strip()
-    if ';' in line:
-        line = line[:line.find(';')].strip()
-    if '\\' in line:
-        line = line[:line.find('\\')].strip()
-
-    return line
+HERE = path.abspath(path.dirname(__file__))
 
 # Get the long description from the README file
-with open(path.join(here, 'README.md'), encoding='utf-8') as f:
+with open(path.join(HERE, 'README.md'), encoding='utf-8') as f:
     long_description = f.read()
 
-# Parse requirements
-runtime_reqs = ['datadog_checks_base']
-with open(path.join(here, 'requirements.txt'), encoding='utf-8') as f:
-    for line in f.readlines():
-        req = parse_req_line(line)
-        if req:
-            runtime_reqs.append(req)
+# Get version info
+ABOUT = {}
+with open(path.join(HERE, "datadog_checks", "couchbase", "__about__.py")) as f:
+    exec(f.read(), ABOUT)
 
-def read(*parts):
-    with open(path.join(here, *parts), 'r') as fp:
-        return fp.read()
 
-def find_version(*file_paths):
-    version_file = read(*file_paths)
-    version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]",
-                              version_file, re.M)
-    if version_match:
-        return version_match.group(1)
-    raise RuntimeError("Unable to find version string.")
+def get_requirements(fpath):
+    with open(path.join(HERE, fpath), encoding='utf-8') as f:
+        return f.readlines()
 
-# https://packaging.python.org/guides/single-sourcing-package-version/
-version = find_version("datadog_checks", "couchbase", "__init__.py")
-
-manifest_version = None
-with open(path.join(here, 'manifest.json'), encoding='utf-8') as f:
-    manifest = json.load(f)
-    manifest_version = manifest.get('version')
-
-if version != manifest_version:
-    raise Exception("Inconsistent versioning in module and manifest - aborting wheel build")
 
 setup(
     name='datadog-couchbase',
-    version=version,
+    version=ABOUT['__version__'],
     description='The Couchbase check',
     long_description=long_description,
     keywords='datadog agent couchbase check',
@@ -78,15 +40,15 @@ def find_version(*file_paths):
     author_email='[email protected]',
 
     # License
-    license='MIT',
+    license='BSD',
 
     # See https://pypi.python.org/pypi?%3Aaction=list_classifiers
     classifiers=[
         'Development Status :: 5 - Production/Stable',
         'Intended Audience :: Developers',
         'Intended Audience :: System Administrators',
         'Topic :: System :: Monitoring',
-        'License :: OSI Approved :: MIT License',
+        'License :: OSI Approved :: BSD License',
         'Programming Language :: Python :: 2',
         'Programming Language :: Python :: 2.7',
     ],
@@ -95,26 +57,13 @@ def find_version(*file_paths):
     packages=['datadog_checks.couchbase'],
 
     # Run-time dependencies
-    install_requires=list(set(runtime_reqs)),
-
-    # Development dependencies, run with:
-    # $ pip install -e .[dev]
-    extras_require={
-        'dev': [
-            'check-manifest',
-            'datadog_agent_tk>=5.15',
-        ],
-    },
-
-    # Testing setup and dependencies
-    tests_require=[
-        'nose',
-        'coverage',
-        'datadog_agent_tk>=5.15',
+    install_requires=get_requirements('requirements.in') + [
+        'datadog_checks_base',
     ],
-    test_suite='nose.collector',
+    setup_requires=['pytest-runner', ],
+    tests_require=get_requirements('requirements-dev.txt'),
 
     # Extra files to ship with the wheel package
-    package_data={b'datadog_checks.couchbase': ['conf.yaml.example']},
+    package_data={b'datadog_checks.couchbase': ['conf.yaml.example', 'autoconf.yaml']},
     include_package_data=True,
 )