Add hosts endpoint. (#884)

* added api.Hosts.get_all and dogshell.hosts

* resolved flake8 issues

* integration tests passed for test_api.py

* updated test config to pass integration testing. api_client and __ini__.py got formatted by black

* reverted black formatting and added muted hosts filter

* changed muted_hosts and hosts_metadata to flags instead of args

* fixed spaces per flake8

* removed type from argparse when using action=store_true

* added muted_hosts flag to list method

* commented muted_hosts filter

* added totals verb

* fixed totals verb

* updated params for totals method

* added test hosts totals

* added integration tests for hosts totals

* fixed formatting

* fixed formatting

* changed to the expert-services sandbox and added cassettes

* updated cassettes generated with the right key. The IS sandbox had inconsistently tagged hosts.

datadog/api/hosts.py

@@ -1,7 +1,7 @@
 # Unless explicitly stated otherwise all files in this repository are licensed under the BSD-3-Clause License.
 # This product includes software developed at Datadog (https://www.datadoghq.com/).
 # Copyright 2015-Present Datadog, Inc
-from datadog.api.resources import ActionAPIResource, SearchableAPIResource
+from datadog.api.resources import ActionAPIResource, SearchableAPIResource, ListableAPIResource
 
 
 class Host(ActionAPIResource):
@@ -48,7 +48,7 @@ def unmute(cls, host_name):
         return super(Host, cls)._trigger_class_action("POST", "unmute", host_name)
 
 
-class Hosts(ActionAPIResource, SearchableAPIResource):
+class Hosts(ActionAPIResource, SearchableAPIResource, ListableAPIResource):
     """
     A wrapper around Hosts HTTP API.
     """
@@ -82,10 +82,54 @@ def search(cls, **params):
         return super(Hosts, cls)._search(**params)
 
     @classmethod
-    def totals(cls):
+    def totals(cls, **params):
         """
         Get total number of hosts active and up.
 
+        :param from_: Number of seconds since UNIX epoch from which you want to search your hosts.
+        :type from_: integer
+
         :returns: Dictionary representing the API's JSON response
         """
-        return super(Hosts, cls)._trigger_class_action("GET", "totals")
+        return super(Hosts, cls)._trigger_class_action("GET", "totals", **params)
+
+    @classmethod
+    def get_all(cls, **params):
+        """
+        Get all hosts.
+
+        :param filter: query to filter search results
+        :type filter: string
+
+        :param sort_field: field to sort by
+        :type sort_field: string
+
+        :param sort_dir: Direction of sort. Options include asc and desc.
+        :type sort_dir: string
+
+        :param start: Specify the starting point for the host search results.
+            For example, if you set count to 100 and the first 100 results have already been returned,
+            you can set start to 101 to get the next 100 results.
+        :type start: integer
+
+        :param count: number of hosts to return. Max 1000.
+        :type count: integer
+
+        :param from_: Number of seconds since UNIX epoch from which you want to search your hosts.
+        :type from_: integer
+
+        :param include_muted_hosts_data: Include data from muted hosts.
+        :type include_muted_hosts_data: boolean
+
+        :param include_hosts_metadata: Include metadata from the hosts
+            (agent_version, machine, platform, processor, etc.).
+        :type include_hosts_metadata: boolean
+
+        :returns: Dictionary representing the API's JSON response
+        """
+
+        for param in ["filter"]:
+            if param in params and isinstance(params[param], list):
+                params[param] = ",".join(params[param])
+
+        return super(Hosts, cls).get_all(**params)

datadog/dogshell/__init__.py

@@ -17,6 +17,7 @@
 from datadog.dogshell.downtime import DowntimeClient
 from datadog.dogshell.event import EventClient
 from datadog.dogshell.host import HostClient
+from datadog.dogshell.hosts import HostsClient
 from datadog.dogshell.metric import MetricClient
 from datadog.dogshell.monitor import MonitorClient
 from datadog.dogshell.screenboard import ScreenboardClient
@@ -95,6 +96,7 @@ def main():
     ScreenboardClient.setup_parser(subparsers)
     DashboardListClient.setup_parser(subparsers)
     HostClient.setup_parser(subparsers)
+    HostsClient.setup_parser(subparsers)
     DowntimeClient.setup_parser(subparsers)
     ServiceCheckClient.setup_parser(subparsers)
     ServiceLevelObjectiveClient.setup_parser(subparsers)

datadog/dogshell/hosts.py

@@ -0,0 +1,100 @@
+# Unless explicitly stated otherwise all files in this repository are licensed under the BSD-3-Clause License.
+# This product includes software developed at Datadog (https://www.datadoghq.com/).
+# Copyright 2015-Present Datadog, Inc
+# stdlib
+
+import json
+
+# 3p
+from datadog.util.format import pretty_json
+
+# datadog
+from datadog import api
+from datadog.dogshell.common import report_errors, report_warnings
+
+
+class HostsClient(object):
+    @classmethod
+    def setup_parser(cls, subparsers):
+        parser = subparsers.add_parser("hosts", help="Get information about hosts")
+        verb_parsers = parser.add_subparsers(title="Verbs", dest="verb")
+        verb_parsers.required = True
+
+        list_parser = verb_parsers.add_parser("list", help="List all hosts")
+        list_parser.add_argument("--filter", help="String to filter search results", type=str)
+        list_parser.add_argument("--sort_field", help="Sort hosts by this field", type=str)
+        list_parser.add_argument(
+            "--sort_dir",
+            help="Direction of sort. 'asc' or 'desc'",
+            choices=["asc", "desc"],
+            default="asc"
+        )
+        list_parser.add_argument(
+            "--start",
+            help="Specify the starting point for the host search results. \
+                                    For example, if you set count to 100 and the first 100 results  \
+                                    have already been returned, \
+                                    you can set start to 101 to get the next 100 results.",
+            type=int,
+        )
+        list_parser.add_argument("--count", help="Number of hosts to return. Max 1000", type=int, default=100)
+        list_parser.add_argument(
+            "--from",
+            help="Number of seconds since UNIX epoch from which you want to search your hosts.",
+            type=int,
+            dest="from_",
+        )
+        # list_parser.add_argument(
+        #     "--include_muted_hosts_data",
+        #     help="Include information on the muted status of hosts and when the mute expires.",
+        #     action="store_true",
+        # )
+        list_parser.add_argument(
+            "--include_hosts_metadata",
+            help="Include metadata from the hosts \
+                                    (agent_version, machine, platform, processor, etc.).",
+            action="store_true",
+        )
+        list_parser.set_defaults(func=cls._list)
+
+        totals_parser = verb_parsers.add_parser("totals", help="Get the total number of hosts")
+        totals_parser.add_argument("--from",
+                                   help="Number of seconds since UNIX epoch \
+                                    from which you want to search your hosts.",
+                                   type=int,
+                                   dest="from_")
+        totals_parser.set_defaults(func=cls._totals)
+
+    @classmethod
+    def _list(cls, args):
+        api._timeout = args.timeout
+        format = args.format
+        res = api.Hosts.get_all(
+            filter=args.filter,
+            sort_field=args.sort_field,
+            sort_dir=args.sort_dir,
+            start=args.start,
+            count=args.count,
+            from_=args.from_,
+            include_hosts_metadata=args.include_hosts_metadata,
+            # this doesn't seem to actually filter and I don't need it for now.
+            # include_muted_hosts_data=args.include_muted_hosts_data
+        )
+        report_warnings(res)
+        report_errors(res)
+        if format == "pretty":
+            print(pretty_json(res))
+        else:
+            print(json.dumps(res))
+
+    @classmethod
+    def _totals(cls, args):
+        api._timeout = args.timeout
+        format = args.format
+        res = api.Hosts.totals(from_=args.from_)
+        report_warnings(res)
+        report_errors(res)
+        if format == "pretty":
+            print(pretty_json(res))
+        else:
+            print(json.dumps(res))

tests/integration/api/cassettes/TestDatadog.test_hosts_get_all.yaml

@@ -0,0 +1,48 @@
+interactions:
+- request:
+    body: null
+    headers:
+      Accept:
+      - '*/*'
+      Accept-Encoding:
+      - gzip, deflate
+      Connection:
+      - keep-alive
+      User-Agent:
+      - datadogpy/0.50.3-dev (python 3.8.20; os linux; arch aarch64)
+    method: GET
+    uri: https://api.datadoghq.com/api/v1/hosts?count=100&filter=env%3Adev&from_=0&include_hosts_metadata=False&include_muted_hosts_data=True&sort_field=host_name&sort_order=asc&start=0
+  response:
+    body:
+      string: !!binary |
+        H4sIAAAAAAAEA5xSy27EIAy89zM4J1Eg2bzO/YtqhbyEbpEIoGB2W0X595qs1EOlHracYGyPPWM2
+        9uEjSmsisultY2Zmk+jbgfOxq0+8YAjXKC9fMvq0Ks2mjb0CwuyvlM60u02zvrHiYJlMKHkvyoaX
+        vC1FLaoUy7uOWPJK+SUk1JVxqFcHlp33goE1EHXMTKasL506dc0wqlbUnQZFrE8REl8IB5nDQMVw
+        1Q7ZuWCP0Y/ID+ZgITFPNXiIlP+qhHuU2do/dKbAJlyTLpgF2saqg19RzxJNnpL3TTcMXTeO5EiU
+        C/lIVO9gIxXk15HnE23QJWsJ0wi0KXKYbqtRJH1jKiSiqvqh5Q3t1fg7GKqoq7oWQtR92wzU3gNR
+        Z6w5Hafdd3IQPYKlsTCtLvfO/+KAFkD1YRx9BoL0JyiUvwNZ1/7yDQAA//8DAFrb9LBoAgAA
+    headers:
+      Connection:
+      - keep-alive
+      Content-Type:
+      - application/json
+      Date:
+      - Tue, 14 Jan 2025 20:39:44 GMT
+      Transfer-Encoding:
+      - chunked
+      content-encoding:
+      - gzip
+      content-security-policy:
+      - frame-ancestors 'self'; report-uri https://logs.browser-intake-datadoghq.com/api/v2/logs?dd-api-key=pube4f163c23bbf91c16b8f57f56af9fc58&dd-evp-origin=content-security-policy&ddsource=csp-report&ddtags=site%3Adatadoghq.com
+      strict-transport-security:
+      - max-age=31536000; includeSubDomains; preload
+      vary:
+      - Accept-Encoding
+      x-content-type-options:
+      - nosniff
+      x-frame-options:
+      - SAMEORIGIN
+    status:
+      code: 200
+      message: OK
+version: 1

tests/integration/api/cassettes/TestDatadog.test_hosts_totals.yaml

@@ -0,0 +1,40 @@
+interactions:
+- request:
+    body: null
+    headers:
+      Accept:
+      - '*/*'
+      Accept-Encoding:
+      - gzip, deflate
+      Connection:
+      - keep-alive
+      User-Agent:
+      - datadogpy/0.50.3-dev (python 3.12.3; os linux; arch aarch64)
+    method: GET
+    uri: https://api.datadoghq.com/api/v1/hosts/totals
+  response:
+    body:
+      string: '{"total_up":8,"total_active":8}'
+    headers:
+      Connection:
+      - keep-alive
+      Content-Length:
+      - '31'
+      Content-Type:
+      - application/json
+      Date:
+      - Thu, 19 Dec 2024 21:56:20 GMT
+      content-security-policy:
+      - frame-ancestors 'self'; report-uri https://logs.browser-intake-datadoghq.com/api/v2/logs?dd-api-key=pube4f163c23bbf91c16b8f57f56af9fc58&dd-evp-origin=content-security-policy&ddsource=csp-report&ddtags=site%3Adatadoghq.com
+      strict-transport-security:
+      - max-age=31536000; includeSubDomains; preload
+      vary:
+      - Accept-Encoding
+      x-content-type-options:
+      - nosniff
+      x-frame-options:
+      - SAMEORIGIN
+    status:
+      code: 200
+      message: OK
+version: 1

tests/integration/api/test_api.py

@@ -803,6 +803,30 @@ def test_host_muting(self, dog, get_with_retry, freezer):
         unmute = dog.Host.unmute(hostname)
         assert unmute["hostname"] == hostname
         assert unmute["action"] == "Unmuted"
+
+    def test_hosts_get_all(self, dog):
+        params = {
+            "filter": "env:dev",
+            "sort_field": "host_name",
+            "sort_order": "asc",
+            "start": 0,
+            "count": 100,
+            "from_": 0,
+            "include_muted_hosts_data": True,
+            "include_hosts_metadata": False
+            }
+
+        all_hosts = dog.Hosts.get_all(**params)
+        assert "host_list" in all_hosts
+        assert isinstance(all_hosts["host_list"], list)
+
+    def test_hosts_totals(self, dog):
+        params = {
+            "--from": 0,
+        }
+        totals = dog.Hosts.totals()
+        assert "total_active" in totals
+        assert "total_up" in totals
 
     def test_get_all_embeds(self, dog):
         all_embeds = dog.Embed.get_all()

tests/integration/dogshell/cassettes/TestDogshell.test_hosts_list.yaml

@@ -0,0 +1,50 @@
+interactions:
+- request:
+    body: null
+    headers:
+      Accept:
+      - '*/*'
+      Accept-Encoding:
+      - gzip, deflate
+      Connection:
+      - keep-alive
+      User-Agent:
+      - datadogpy/0.50.3-dev (python 3.8.20; os linux; arch aarch64)
+    method: GET
+    uri: https://api.datadoghq.com/api/v1/hosts?count=100&filter=env&from_=0&include_hosts_metadata=False&sort_dir=asc&sort_field=host_name&start=0
+  response:
+    body:
+      string: !!binary |
+        H4sIAAAAAAAEA6xTTW+cMBC992f4DMgfgA23qMmtUe+tIuT1uhtLYCM8zrZa8d87ZrVt2iRVVion
+        e8bvzcd7nMhjiDCMLgLpv56I25Oey1Z0SjFey4KAPsRh92OIIS3Gkv5EbjXofTjgc2L9Ux/DZOHR
+        +QMpNq7+7iO/ub/5Ut5/bgSTn8jDWhA9Oh1tzBhXUsWE6Lik1u6YMY1E5AsQYuZ5A+iD9YBPPMzk
+        oSDnRp5lMOb1hK29IDk3NLyZ1cc45IHf6CnNpIcl2YKMGne02DksYPcDuFyNSdEqVXMqCuLiMCVM
+        kf6bHiMC8m17FxLu1adxxJgFjfvDbeBpcQZHOBEzJ6SqGinrTnBkCkftEEKxaNBImLHrWlyUqRVj
+        XUsb9g5l9vbpoombSyZ5KVjJ6pJTXqVYHm2EklUmTDN2WzkPdvF6fEWvXWuaVqjO4LSt1QZZryL8
+        rWXWEG+bov/W8poCf+p8HfK5B16Z8z0eaNuu+w8ekKpmAnX9ZYGKUs45lbVQFzdQjIlm++p1xQ1C
+        AD2iNSEtPvsPLXQOTRrM9lfmkP2uDQx/J7K31w8/AQAA//8DAI9b56cCBAAA
+    headers:
+      Connection:
+      - keep-alive
+      Content-Type:
+      - application/json
+      Date:
+      - Tue, 14 Jan 2025 20:39:45 GMT
+      Transfer-Encoding:
+      - chunked
+      content-encoding:
+      - gzip
+      content-security-policy:
+      - frame-ancestors 'self'; report-uri https://logs.browser-intake-datadoghq.com/api/v2/logs?dd-api-key=pube4f163c23bbf91c16b8f57f56af9fc58&dd-evp-origin=content-security-policy&ddsource=csp-report&ddtags=site%3Adatadoghq.com
+      strict-transport-security:
+      - max-age=31536000; includeSubDomains; preload
+      vary:
+      - Accept-Encoding
+      x-content-type-options:
+      - nosniff
+      x-frame-options:
+      - SAMEORIGIN
+    status:
+      code: 200
+      message: OK
+version: 1