Skip to content

Commit

Permalink
[PLINT-246] Add pagination support for network networks (#16930)
Browse files Browse the repository at this point in the history 
* adding limit=1 test files

* modified get_network_networks() to use make_paginated_request()

* added unit tests for networks pagination using rest api

* added changelog

* fixed linting error

* fixing additional linting error

* enable capability and added unit tests for networks pagination for sdk api

* asserting that the proper network metrics are still submitted during a pagination call
  • Loading branch information
@rahulkaukuntla @NouemanKHAL
rahulkaukuntla authored and NouemanKHAL committed Mar 4, 2024
1 parent 8308d55 commit b771ada
Show file tree
Hide file tree
Showing 10 changed files with 312 additions and 6 deletions.
1 change: 1 addition & 0 deletions openstack_controller/changelog.d/16930.added
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
[PLINT-246] Add pagination support for network networks
10 changes: 6 additions & 4 deletions openstack_controller/datadog_checks/openstack_controller/api/api_rest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -281,11 +281,13 @@ def get_network_agents(self):

def get_network_networks(self, project_id):
params = {'project_id': project_id}
response = self.http.get(
'{}/v2.0/networks'.format(self._catalog.get_endpoint_by_type(Component.Types.NETWORK.value)), params=params
return self.make_paginated_request(
'{}/v2.0/networks'.format(self._catalog.get_endpoint_by_type(Component.Types.NETWORK.value)),
'networks',
'id',
next_signifier='networks_links',
params=params,
)
response.raise_for_status()
return response.json().get('networks', [])

def get_network_quota(self, project_id):
response = self.http.get(
Expand Down
5 changes: 4 additions & 1 deletion openstack_controller/datadog_checks/openstack_controller/api/api_sdk.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -200,7 +200,10 @@ def get_network_agents(self):

def get_network_networks(self, project_id):
return [
network.to_dict(original_names=True) for network in self.connection.network.networks(project_id=project_id)
network.to_dict(original_names=True)
for network in self.call_paginated_api(
self.connection.network.networks, project_id=project_id, limit=self.config.paginated_limit
)
]

def get_network_quota(self, project_id):
Expand Down
2 changes: 1 addition & 1 deletion openstack_controller/tests/conftest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -513,7 +513,7 @@ def agents():
for agent in mock_responses('GET', '/networking/v2.0/agents')['agents']
]

def networks(project_id):
def networks(project_id, limit=None):
if http_error and 'networks' in http_error and project_id in http_error['networks']:
raise requests.exceptions.HTTPError(response=http_error['networks'])
return [
Expand Down
39 changes: 39 additions & 0 deletions ...37d4d55a50a62b63398ad15/limit=1/marker=f7b6adc8-24ea-490c-9537-5c4eae015cd8/response.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
{
"networks": [
{
"id": "ec38babc-37e8-4bd7-9de0-03009304b2e4",
"name": "public",
"tenant_id": "c09ccb52462d49c89595d2194a367198",
"admin_state_up": true,
"mtu": 1500,
"status": "ACTIVE",
"subnets": [
"4bd0aa9b-bfef-4b64-96ec-4a3c29d694b4",
"51b41dcb-ec03-4036-9845-9b819b1e301f"
],
"shared": false,
"availability_zone_hints": [],
"availability_zones": [],
"ipv4_address_scope": null,
"ipv6_address_scope": null,
"router:external": true,
"description": "",
"port_security_enabled": true,
"is_default": true,
"tags": [],
"created_at": "2023-10-18T15:01:59Z",
"updated_at": "2023-10-18T15:02:14Z",
"revision_number": 3,
"project_id": "c09ccb52462d49c89595d2194a367198",
"provider:network_type": "flat",
"provider:physical_network": "public",
"provider:segmentation_id": null
}
],
"networks_links": [
{
"rel": "previous",
"href": "http://34.91.117.188:9696/networking/v2.0/networks?limit=1&marker=ec38babc-37e8-4bd7-9de0-03009304b2e4&page_reverse=True"
}
]
}
41 changes: 41 additions & 0 deletions ...etworking/v2.0/networks/project_id=1e6e233e637d4d55a50a62b63398ad15/limit=1/response.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
{
"networks": [
{
"id": "f7b6adc8-24ea-490c-9537-5c4eae015cd8",
"name": "shared",
"tenant_id": "1e6e233e637d4d55a50a62b63398ad15",
"admin_state_up": true,
"mtu": 1442,
"status": "ACTIVE",
"subnets": [
"60c84d5f-f884-4fcf-a2de-605662d52ba3"
],
"shared": true,
"availability_zone_hints": [],
"availability_zones": [],
"ipv4_address_scope": null,
"ipv6_address_scope": null,
"router:external": false,
"description": "",
"port_security_enabled": true,
"tags": [],
"created_at": "2023-10-18T15:04:01Z",
"updated_at": "2023-10-18T15:04:03Z",
"revision_number": 2,
"project_id": "1e6e233e637d4d55a50a62b63398ad15",
"provider:network_type": "geneve",
"provider:physical_network": null,
"provider:segmentation_id": 16779
}
],
"networks_links": [
{
"rel": "next",
"href": "http://127.0.0.1:9696/networking/v2.0/networks?limit=1&marker=f7b6adc8-24ea-490c-9537-5c4eae015cd8"
},
{
"rel": "previous",
"href": "http://127.0.0.1:9696/networking/v2.0/networks?limit=1&marker=f7b6adc8-24ea-490c-9537-5c4eae015cd8&page_reverse=True"
}
]
}
66 changes: 66 additions & 0 deletions ...orking/v2.0/networks/project_id=1e6e233e637d4d55a50a62b63398ad15/limit=1000/response.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
{
"networks": [
{
"id": "ec38babc-37e8-4bd7-9de0-03009304b2e4",
"name": "public",
"tenant_id": "c09ccb52462d49c89595d2194a367198",
"admin_state_up": true,
"mtu": 1500,
"status": "ACTIVE",
"subnets": [
"4bd0aa9b-bfef-4b64-96ec-4a3c29d694b4",
"51b41dcb-ec03-4036-9845-9b819b1e301f"
],
"shared": false,
"availability_zone_hints": [],
"availability_zones": [],
"ipv4_address_scope": null,
"ipv6_address_scope": null,
"router:external": true,
"description": "",
"port_security_enabled": true,
"is_default": true,
"tags": [],
"created_at": "2023-10-18T15:01:59Z",
"updated_at": "2023-10-18T15:02:14Z",
"revision_number": 3,
"project_id": "c09ccb52462d49c89595d2194a367198",
"provider:network_type": "flat",
"provider:physical_network": "public",
"provider:segmentation_id": null
},
{
"id": "f7b6adc8-24ea-490c-9537-5c4eae015cd8",
"name": "shared",
"tenant_id": "1e6e233e637d4d55a50a62b63398ad15",
"admin_state_up": true,
"mtu": 1442,
"status": "ACTIVE",
"subnets": [
"60c84d5f-f884-4fcf-a2de-605662d52ba3"
],
"shared": true,
"availability_zone_hints": [],
"availability_zones": [],
"ipv4_address_scope": null,
"ipv6_address_scope": null,
"router:external": false,
"description": "",
"port_security_enabled": true,
"tags": [],
"created_at": "2023-10-18T15:04:01Z",
"updated_at": "2023-10-18T15:04:03Z",
"revision_number": 2,
"project_id": "1e6e233e637d4d55a50a62b63398ad15",
"provider:network_type": "geneve",
"provider:physical_network": null,
"provider:segmentation_id": 16779
}
],
"networks_links": [
{
"rel": "previous",
"href": "http://34.91.117.188:9696/networking/v2.0/networks?limit=1000&marker=ec38babc-37e8-4bd7-9de0-03009304b2e4&page_reverse=True"
}
]
}
38 changes: 38 additions & 0 deletions ...etworking/v2.0/networks/project_id=6e39099cccde4f809b003d9e0dd09304/limit=1/response.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
{
"networks": [
{
"id": "ebdfddd9-14b8-46bd-98d6-10205d13038c",
"name": "private",
"tenant_id": "29b43a58176646e3870a9ea0b60fb7ad",
"admin_state_up": true,
"mtu": 1442,
"status": "ACTIVE",
"subnets": [
"5f1664c6-9b9b-475c-b0d4-95337a84f552",
"ca392b80-175d-4d20-becc-aa2c6f984bd1"
],
"shared": false,
"availability_zone_hints": [],
"availability_zones": [],
"ipv4_address_scope": null,
"ipv6_address_scope": null,
"router:external": false,
"description": "",
"port_security_enabled": true,
"tags": [],
"created_at": "2023-10-18T15:01:45Z",
"updated_at": "2023-10-18T15:01:50Z",
"revision_number": 3,
"project_id": "29b43a58176646e3870a9ea0b60fb7ad",
"provider:network_type": "geneve",
"provider:physical_network": null,
"provider:segmentation_id": 876
}
],
"networks_links": [
{
"rel": "previous",
"href": "http://34.91.117.188:9696/networking/v2.0/networks?limit=1&marker=ebdfddd9-14b8-46bd-98d6-10205d13038c&page_reverse=True"
}
]
}
38 changes: 38 additions & 0 deletions ...orking/v2.0/networks/project_id=6e39099cccde4f809b003d9e0dd09304/limit=1000/response.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
{
"networks": [
{
"id": "ebdfddd9-14b8-46bd-98d6-10205d13038c",
"name": "private",
"tenant_id": "29b43a58176646e3870a9ea0b60fb7ad",
"admin_state_up": true,
"mtu": 1442,
"status": "ACTIVE",
"subnets": [
"5f1664c6-9b9b-475c-b0d4-95337a84f552",
"ca392b80-175d-4d20-becc-aa2c6f984bd1"
],
"shared": false,
"availability_zone_hints": [],
"availability_zones": [],
"ipv4_address_scope": null,
"ipv6_address_scope": null,
"router:external": false,
"description": "",
"port_security_enabled": true,
"tags": [],
"created_at": "2023-10-18T15:01:45Z",
"updated_at": "2023-10-18T15:01:50Z",
"revision_number": 3,
"project_id": "29b43a58176646e3870a9ea0b60fb7ad",
"provider:network_type": "geneve",
"provider:physical_network": null,
"provider:segmentation_id": 876
}
],
"networks_links": [
{
"rel": "previous",
"href": "http://34.91.117.188:9696/networking/v2.0/networks?limit=1000&marker=ebdfddd9-14b8-46bd-98d6-10205d13038c&page_reverse=True"
}
]
}
78 changes: 78 additions & 0 deletions openstack_controller/tests/test_unit_neutron.py
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
# (C) Datadog, Inc. 2023-present
# All rights reserved
# Licensed under a 3-clause BSD style license (see LICENSE)
import copy
import logging
import os

Expand Down Expand Up @@ -1251,6 +1252,83 @@ def test_networks_metrics_with_exclude(aggregator, dd_run_check, instance, opens
)


@pytest.mark.parametrize(
('instance', 'paginated_limit', 'api_type', 'expected_api_calls'),
[
pytest.param(
configs.REST,
1,
ApiType.REST,
3,
id='api rest small limit',
),
pytest.param(
configs.REST,
1000,
ApiType.REST,
2,
id='api rest high limit',
),
pytest.param(
configs.SDK,
1,
ApiType.SDK,
1,
id='api sdk small limit',
),
pytest.param(
configs.SDK,
1000,
ApiType.SDK,
1,
id='api sdk high limit',
),
],
)
@pytest.mark.usefixtures('mock_http_get', 'mock_http_post', 'openstack_connection')
def test_networks_pagination(
aggregator,
instance,
openstack_controller_check,
paginated_limit,
expected_api_calls,
api_type,
dd_run_check,
mock_http_get,
connection_network,
):
paginated_instance = copy.deepcopy(instance)
paginated_instance['paginated_limit'] = paginated_limit
dd_run_check(openstack_controller_check(paginated_instance))
if api_type == ApiType.REST:
args_list = []
for call in mock_http_get.call_args_list:
args, kwargs = call
args_list += list(args)
params = kwargs.get('params', {})
limit = params.get('limit')
args_list += [(args[0], limit)]
assert (
args_list.count(('http://127.0.0.1:9696/networking/v2.0/networks', paginated_limit)) == expected_api_calls
)

else:
assert connection_network.networks.call_count == 2
assert (
connection_network.networks.call_args_list.count(
mock.call(project_id='1e6e233e637d4d55a50a62b63398ad15', limit=paginated_limit)
)
== 1
)
assert (
connection_network.networks.call_args_list.count(
mock.call(project_id='6e39099cccde4f809b003d9e0dd09304', limit=paginated_limit)
)
== 1
)
test_networks_metrics(aggregator, openstack_controller_check(paginated_instance), dd_run_check)


@pytest.mark.parametrize(
('mock_http_get', 'connection_network', 'instance', 'api_type'),
[
Expand Down

0 comments on commit b771ada

Please sign in to comment.