From 1b4b329db0003855024bf9e96b3c8cb0591572d4 Mon Sep 17 00:00:00 2001 From: Jan Snasel Date: Fri, 15 Sep 2023 12:07:29 +0000 Subject: [PATCH 01/12] chore: Added IPFabric --- development/nautobot_config.py | 2 + nautobot_ssot/__init__.py | 6 + .../integrations/ipfabric/__init__.py | 1 + .../integrations/ipfabric/constants.py | 15 + .../ipfabric/diffsync/__init__.py | 4 + .../ipfabric/diffsync/adapter_ipfabric.py | 160 ++++++ .../ipfabric/diffsync/adapter_nautobot.py | 236 +++++++++ .../ipfabric/diffsync/adapters_shared.py | 22 + .../ipfabric/diffsync/diffsync_models.py | 458 ++++++++++++++++++ nautobot_ssot/integrations/ipfabric/jobs.py | 292 +++++++++++ .../integrations/ipfabric/signals.py | 82 ++++ .../ipfabric/utilities/__init__.py | 25 + .../ipfabric/utilities/nbutils.py | 257 ++++++++++ .../ipfabric/utilities/test_utils.py | 21 + .../integrations/ipfabric/workers.py | 150 ++++++ .../nautobot_ssot_ipfabric/ipfabric.png | Bin 0 -> 8451 bytes .../nautobot_ssot_ipfabric/ipfabric_logo.png | Bin 0 -> 11662 bytes nautobot_ssot/tests/ipfabric/__init__.py | 1 + .../tests/ipfabric/fixtures/__init__.py | 17 + .../fixtures/get_device_inventory.json | 68 +++ .../fixtures/get_interface_inventory.json | 50 ++ .../tests/ipfabric/fixtures/get_sites.json | 80 +++ .../tests/ipfabric/fixtures/get_vlans.json | 80 +++ .../tests/ipfabric/test_ipfabric_adapter.py | 89 ++++ nautobot_ssot/tests/ipfabric/test_jobs.py | 72 +++ .../tests/ipfabric/test_nautobot_adapter.py | 79 +++ nautobot_ssot/tests/ipfabric/test_nbutils.py | 127 +++++ poetry.lock | 218 ++++++++- pyproject.toml | 17 + tasks.py | 2 +- 30 files changed, 2624 insertions(+), 7 deletions(-) create mode 100644 nautobot_ssot/integrations/ipfabric/__init__.py create mode 100644 nautobot_ssot/integrations/ipfabric/constants.py create mode 100644 nautobot_ssot/integrations/ipfabric/diffsync/__init__.py create mode 100644 nautobot_ssot/integrations/ipfabric/diffsync/adapter_ipfabric.py create mode 100644 nautobot_ssot/integrations/ipfabric/diffsync/adapter_nautobot.py create mode 100644 nautobot_ssot/integrations/ipfabric/diffsync/adapters_shared.py create mode 100644 nautobot_ssot/integrations/ipfabric/diffsync/diffsync_models.py create mode 100644 nautobot_ssot/integrations/ipfabric/jobs.py create mode 100644 nautobot_ssot/integrations/ipfabric/signals.py create mode 100644 nautobot_ssot/integrations/ipfabric/utilities/__init__.py create mode 100644 nautobot_ssot/integrations/ipfabric/utilities/nbutils.py create mode 100644 nautobot_ssot/integrations/ipfabric/utilities/test_utils.py create mode 100644 nautobot_ssot/integrations/ipfabric/workers.py create mode 100644 nautobot_ssot/static/nautobot_ssot_ipfabric/ipfabric.png create mode 100644 nautobot_ssot/static/nautobot_ssot_ipfabric/ipfabric_logo.png create mode 100644 nautobot_ssot/tests/ipfabric/__init__.py create mode 100644 nautobot_ssot/tests/ipfabric/fixtures/__init__.py create mode 100644 nautobot_ssot/tests/ipfabric/fixtures/get_device_inventory.json create mode 100644 nautobot_ssot/tests/ipfabric/fixtures/get_interface_inventory.json create mode 100644 nautobot_ssot/tests/ipfabric/fixtures/get_sites.json create mode 100644 nautobot_ssot/tests/ipfabric/fixtures/get_vlans.json create mode 100644 nautobot_ssot/tests/ipfabric/test_ipfabric_adapter.py create mode 100644 nautobot_ssot/tests/ipfabric/test_jobs.py create mode 100644 nautobot_ssot/tests/ipfabric/test_nautobot_adapter.py create mode 100644 nautobot_ssot/tests/ipfabric/test_nbutils.py diff --git a/development/nautobot_config.py b/development/nautobot_config.py index 471988aca..f424c5585 100644 --- a/development/nautobot_config.py +++ b/development/nautobot_config.py @@ -135,6 +135,8 @@ PLUGINS = [ "nautobot_ssot", # "nautobot_device_lifecycle_mgmt", + # Enable chatops after Nautobot v2 compatible release + # "nautobot_chatops", ] # Plugins configuration settings. These settings are used by various plugins that the user may have installed. diff --git a/nautobot_ssot/__init__.py b/nautobot_ssot/__init__.py index 4c16ef036..bf0963400 100644 --- a/nautobot_ssot/__init__.py +++ b/nautobot_ssot/__init__.py @@ -55,6 +55,7 @@ class NautobotSSOTPluginConfig(PluginConfig): "enable_aci": False, "enable_aristacv": False, "enable_infoblox": False, + "enable_ipfabric": False, "enable_servicenow": False, "hide_example_jobs": True, "infoblox_default_status": "", @@ -70,6 +71,11 @@ class NautobotSSOTPluginConfig(PluginConfig): "infoblox_username": "", "infoblox_verify_ssl": True, "infoblox_wapi_version": "", + "ipfabric_api_token": "", + "ipfabric_host": "", + "ipfabric_ssl_verify": True, + "ipfabric_timeout": 15, + "nautobot_host": "", "servicenow_instance": "", "servicenow_password": "", "servicenow_username": "", diff --git a/nautobot_ssot/integrations/ipfabric/__init__.py b/nautobot_ssot/integrations/ipfabric/__init__.py new file mode 100644 index 000000000..1d5cf9e79 --- /dev/null +++ b/nautobot_ssot/integrations/ipfabric/__init__.py @@ -0,0 +1 @@ +"""Base module for IPFabric integration.""" diff --git a/nautobot_ssot/integrations/ipfabric/constants.py b/nautobot_ssot/integrations/ipfabric/constants.py new file mode 100644 index 000000000..b7d4dce2a --- /dev/null +++ b/nautobot_ssot/integrations/ipfabric/constants.py @@ -0,0 +1,15 @@ +"""Constants used by IPFabric Integration.""" + +ALLOW_DUPLICATE_ADDRESSES = True +DEFAULT_DEVICE_ROLE = "Network Device" +DEFAULT_DEVICE_ROLE_COLOR = "ff0000" +DEFAULT_DEVICE_STATUS = "Active" +DEFAULT_DEVICE_STATUS_COLOR = "ff0000" +DEFAULT_INTERFACE_MAC = "00:00:00:00:00:01" +DEFAULT_INTERFACE_MTU = 1500 +DEFAULT_INTERFACE_TYPE = "1000base-t" +SAFE_DELETE_DEVICE_STATUS = "Deprecated" +SAFE_DELETE_IPADDRESS_STATUS = "Deprecated" +SAFE_DELETE_SITE_STATUS = "Decommissioning" +SAFE_DELETE_VLAN_STATUS = "Inventory" +SAFE_IPADDRESS_INTERFACES_STATUS = "Deprecated" diff --git a/nautobot_ssot/integrations/ipfabric/diffsync/__init__.py b/nautobot_ssot/integrations/ipfabric/diffsync/__init__.py new file mode 100644 index 000000000..4c6f15145 --- /dev/null +++ b/nautobot_ssot/integrations/ipfabric/diffsync/__init__.py @@ -0,0 +1,4 @@ +"""Diffsync Utilities, Adapters and Models.""" +from .adapters_shared import DiffSyncModelAdapters + +__all__ = ("DiffSyncModelAdapters",) diff --git a/nautobot_ssot/integrations/ipfabric/diffsync/adapter_ipfabric.py b/nautobot_ssot/integrations/ipfabric/diffsync/adapter_ipfabric.py new file mode 100644 index 000000000..5b9ac2dbb --- /dev/null +++ b/nautobot_ssot/integrations/ipfabric/diffsync/adapter_ipfabric.py @@ -0,0 +1,160 @@ +# pylint: disable=duplicate-code +"""DiffSync adapter class for Ip Fabric.""" + +import logging + +from diffsync import ObjectAlreadyExists +from nautobot.dcim.models import Device +from nautobot.ipam.models import VLAN +from netutils.mac import mac_to_format + +from nautobot_ssot.integrations.ipfabric.constants import ( + DEFAULT_INTERFACE_TYPE, + DEFAULT_INTERFACE_MTU, + DEFAULT_INTERFACE_MAC, + DEFAULT_DEVICE_ROLE, + DEFAULT_DEVICE_STATUS, +) +from nautobot_ssot.integrations.ipfabric.diffsync import DiffSyncModelAdapters + +logger = logging.getLogger("nautobot.jobs") + +device_serial_max_length = Device._meta.get_field("serial").max_length +name_max_length = VLAN._meta.get_field("name").max_length + + +class IPFabricDiffSync(DiffSyncModelAdapters): + """Nautobot adapter for DiffSync.""" + + def __init__(self, job, sync, client, *args, **kwargs): + """Initialize the NautobotDiffSync.""" + super().__init__(*args, **kwargs) + self.job = job + self.sync = sync + self.client = client + + def load_sites(self): + """Add IP Fabric Site objects as DiffSync Location models.""" + sites = self.client.inventory.sites.all() + for site in sites: + try: + location = self.location(diffsync=self, name=site["siteName"], site_id=site["id"], status="Active") + self.add(location) + except ObjectAlreadyExists: + self.job.log_debug(message=f"Duplicate Site discovered, {site}") + + def load_device_interfaces(self, device_model, interfaces, device_primary_ip): + """Create and load DiffSync Interface model objects for a specific device.""" + device_interfaces = [iface for iface in interfaces if iface.get("hostname") == device_model.name] + pseudo_interface = pseudo_management_interface(device_model.name, device_interfaces, device_primary_ip) + + if pseudo_interface: + device_interfaces.append(pseudo_interface) + logger.info("Pseudo MGMT Interface: %s", pseudo_interface) + + for iface in device_interfaces: + ip_address = iface.get("primaryIp") + try: + interface = self.interface( + diffsync=self, + name=iface.get("intName"), + device_name=iface.get("hostname"), + description=iface.get("dscr", ""), + enabled=True, + mac_address=mac_to_format(iface.get("mac"), "MAC_COLON_TWO").upper() + if iface.get("mac") + else DEFAULT_INTERFACE_MAC, + mtu=iface.get("mtu") if iface.get("mtu") else DEFAULT_INTERFACE_MTU, + type=DEFAULT_INTERFACE_TYPE, + mgmt_only=iface.get("mgmt_only", False), + ip_address=ip_address, + # TODO: why is only IPv4? and why /32? + subnet_mask="255.255.255.255", + ip_is_primary=ip_address is not None and ip_address == device_primary_ip, + status="Active", + ) + self.add(interface) + device_model.add_child(interface) + except ObjectAlreadyExists: + self.job.log_debug(message=f"Duplicate Interface discovered, {iface}") + + def load(self): + """Load data from IP Fabric.""" + self.load_sites() + devices = self.client.inventory.devices.all() + interfaces = self.client.inventory.interfaces.all() + vlans = self.client.fetch_all("tables/vlan/site-summary") + + for location in self.get_all(self.location): + if location.name is None: + continue + location_vlans = [vlan for vlan in vlans if vlan["siteName"] == location.name] + for vlan in location_vlans: + if not vlan["vlanId"] or (vlan["vlanId"] < 1 or vlan["vlanId"] > 4094): + self.job.log_warning( + message=f"Not syncing VLAN, NAME: {vlan.get('vlanName')} due to invalid VLAN ID: {vlan.get('vlanId')}." + ) + continue + description = vlan.get("dscr") if vlan.get("dscr") else f"VLAN ID: {vlan['vlanId']}" + vlan_name = vlan.get("vlanName") if vlan.get("vlanName") else f"{vlan['siteName']}:{vlan['vlanId']}" + if len(vlan_name) > name_max_length: + self.job.log_warning( + message=f"Not syncing VLAN, {vlan_name} due to character limit exceeding {name_max_length}." + ) + continue + try: + vlan = self.vlan( + diffsync=self, + name=vlan_name, + site=vlan["siteName"], + vid=vlan["vlanId"], + status="Active", + description=description, + ) + self.add(vlan) + location.add_child(vlan) + except ObjectAlreadyExists: + self.job.log_debug(message=f"Duplicate VLAN discovered, {vlan}") + + location_devices = [device for device in devices if device["siteName"] == location.name] + for device in location_devices: + device_primary_ip = device["loginIp"] + sn_length = len(device["sn"]) + serial_number = device["sn"] if sn_length < device_serial_max_length else "" + if not serial_number: + self.job.log_warning( + message=( + f"Serial Number will not be recorded for {device['hostname']} due to character limit. " + f"{sn_length} exceeds {device_serial_max_length}" + ) + ) + try: + device_model = self.device( + diffsync=self, + name=device["hostname"], + location_name=device["siteName"], + model=device.get("model") if device.get("model") else f"Default-{device.get('vendor')}", + vendor=device.get("vendor").capitalize(), + serial_number=serial_number, + role=device.get("devType") if device.get("devType") else DEFAULT_DEVICE_ROLE, + status=DEFAULT_DEVICE_STATUS, + ) + self.add(device_model) + location.add_child(device_model) + self.load_device_interfaces(device_model, interfaces, device_primary_ip) + except ObjectAlreadyExists: + self.job.log_debug(message=f"Duplicate Device discovered, {device}") + + +def pseudo_management_interface(hostname, device_interfaces, device_primary_ip): + """Return a dict for an non-existing interface for NAT management addresses.""" + if any(iface for iface in device_interfaces if iface.get("primaryIp", "") == device_primary_ip): + return None + return { + "hostname": hostname, + "intName": "pseudo_mgmt", + "dscr": "pseudo interface for NAT IP address", + "primaryIp": device_primary_ip, + "type": "virtual", + "mgmt_only": True, + } diff --git a/nautobot_ssot/integrations/ipfabric/diffsync/adapter_nautobot.py b/nautobot_ssot/integrations/ipfabric/diffsync/adapter_nautobot.py new file mode 100644 index 000000000..88fc127a3 --- /dev/null +++ b/nautobot_ssot/integrations/ipfabric/diffsync/adapter_nautobot.py @@ -0,0 +1,236 @@ +# pylint: disable=duplicate-code +# pylint: disable=too-many-arguments +# Load method is packed with conditionals # pylint: disable=too-many-branches +"""DiffSync adapter class for Nautobot as source-of-truth.""" +from collections import defaultdict +from typing import Any, ClassVar, List + +from diffsync import DiffSync +from diffsync.exceptions import ObjectAlreadyExists +from django.db import IntegrityError, transaction +from django.db.models import ProtectedError, Q +from nautobot.dcim.models import Device, Site +from nautobot.extras.models import Tag +from nautobot.ipam.models import VLAN, Interface +from nautobot.utilities.choices import ColorChoices +from netutils.mac import mac_to_format + +from nautobot_ssot.integrations.ipfabric.diffsync import DiffSyncModelAdapters + +from nautobot_ssot.integrations.ipfabric.constants import ( + DEFAULT_INTERFACE_TYPE, + DEFAULT_INTERFACE_MTU, + DEFAULT_INTERFACE_MAC, +) + + +class NautobotDiffSync(DiffSyncModelAdapters): + """Nautobot adapter for DiffSync.""" + + objects_to_delete = defaultdict(list) + + _vlan: ClassVar[Any] = VLAN + _device: ClassVar[Any] = Device + _site: ClassVar[Any] = Site + _interface: ClassVar[Any] = Interface + + def __init__( + self, + job, + sync, + sync_ipfabric_tagged_only: bool, + site_filter: Site, + *args, + **kwargs, + ): + """Initialize the NautobotDiffSync.""" + super().__init__(*args, **kwargs) + self.job = job + self.sync = sync + self.sync_ipfabric_tagged_only = sync_ipfabric_tagged_only + self.site_filter = site_filter + + def sync_complete(self, source: DiffSync, *args, **kwargs): + """Clean up function for DiffSync sync. + + Once the sync is complete, this function runs deleting any objects + from Nautobot that need to be deleted in a specific order. + + Args: + source (DiffSync): DiffSync + """ + for grouping in ( + "_vlan", + "_interface", + "_device", + "_site", + ): + for nautobot_object in self.objects_to_delete[grouping]: + if NautobotDiffSync.safe_delete_mode: + continue + try: + nautobot_object.delete() + except ProtectedError: + self.job.log_failure(obj=nautobot_object, message="Deletion failed protected object") + except IntegrityError: + self.job.log_failure( + obj=nautobot_object, message=f"Deletion failed due to IntegrityError with {nautobot_object}" + ) + + self.objects_to_delete[grouping] = [] + return super().sync_complete(source, *args, **kwargs) + + def load_interfaces(self, device_record: Device, diffsync_device): + """Import a single Nautobot Interface object as a DiffSync Interface model.""" + device_primary_ip = None + if device_record.primary_ip4: + device_primary_ip = device_record.primary_ip4 + elif device_record.primary_ip6: + device_primary_ip = device_record.primary_ip6 + + for interface_record in device_record.interfaces.all(): + interface = self.interface( + diffsync=self, + status=device_record.status.name, + name=interface_record.name, + device_name=device_record.name, + description=interface_record.description if interface_record.description else None, + enabled=True, + mac_address=mac_to_format(str(interface_record.mac_address), "MAC_COLON_TWO").upper() + if interface_record.mac_address + else DEFAULT_INTERFACE_MAC, + subnet_mask="255.255.255.255", + mtu=interface_record.mtu if interface_record.mtu else DEFAULT_INTERFACE_MTU, + type=DEFAULT_INTERFACE_TYPE, + mgmt_only=interface_record.mgmt_only if interface_record.mgmt_only else False, + pk=interface_record.pk, + ip_is_primary=interface_record.ip_addresses.first() == device_primary_ip + if device_primary_ip + else False, + ip_address=str(interface_record.ip_addresses.first().host) + if interface_record.ip_addresses.first() + else None, + ) + self.add(interface) + diffsync_device.add_child(interface) + + def load_device(self, filtered_devices: List, location): + """Load Devices from Nautobot.""" + for device_record in filtered_devices: + self.job.log_debug(message=f"Loading Nautobot Device: {device_record.name}") + device = self.device( + diffsync=self, + name=device_record.name, + model=str(device_record.device_type), + role=str(device_record.device_role.cf.get("ipfabric_type")) + if str(device_record.device_role.cf.get("ipfabric_type")) + else str(device_record.device_role), + location_name=device_record.site.name, + vendor=str(device_record.device_type.manufacturer), + status=device_record.status.name, + serial_number=device_record.serial if device_record.serial else "", + ) + try: + self.add(device) + except ObjectAlreadyExists: + self.job.log_debug(message=f"Duplicate device discovered, {device_record.name}") + continue + + location.add_child(device) + self.load_interfaces(device_record=device_record, diffsync_device=device) + + def load_vlans(self, filtered_vlans: List, location): + """Add Nautobot VLAN objects as DiffSync VLAN models.""" + for vlan_record in filtered_vlans: + if not vlan_record: + continue + vlan = self.vlan( + diffsync=self, + name=vlan_record.name, + site=vlan_record.site.name, + status=vlan_record.status.name if vlan_record.status else "Active", + vid=vlan_record.vid, + vlan_pk=vlan_record.pk, + description=vlan_record.description, + ) + try: + self.add(vlan) + except ObjectAlreadyExists: + self.job.log_debug(message=f"Duplicate VLAN discovered, {vlan_record.name}") + continue + location.add_child(vlan) + + def get_initial_site(self, ssot_tag: Tag): + """Identify the site objects based on user defined job inputs. + + Args: + ssot_tag (Tag): Tag used for filtering + """ + # Simple check / validate Tag is present. + if self.sync_ipfabric_tagged_only: + site_objects = Site.objects.filter(tags__slug=ssot_tag.slug) + if self.site_filter: + site_objects = Site.objects.filter(Q(name=self.site_filter.name) & Q(tags__slug=ssot_tag.slug)) + if not site_objects: + self.job.log_warning( + message=f"{self.site_filter.name} was used to filter, alongside SSoT Tag. {self.site_filter.name} is not tagged." + ) + elif not self.sync_ipfabric_tagged_only: + if self.site_filter: + site_objects = Site.objects.filter(name=self.site_filter.name) + else: + site_objects = Site.objects.all() + return site_objects + + @transaction.atomic + def load_data(self): + """Add Nautobot Site objects as DiffSync Location models.""" + ssot_tag, _ = Tag.objects.get_or_create( + slug="ssot-synced-from-ipfabric", + name="SSoT Synced from IPFabric", + defaults={ + "description": "Object synced at some point from IPFabric to Nautobot", + "color": ColorChoices.COLOR_LIGHT_GREEN, + }, + ) + site_objects = self.get_initial_site(ssot_tag) + # The parent object that stores all children, is the Site. + self.job.log_debug(message=f"Found {site_objects.count()} Nautobot Site objects to start sync from") + + if site_objects: + for site_record in site_objects: + try: + location = self.location( + diffsync=self, + name=site_record.name, + site_id=site_record.custom_field_data.get("ipfabric-site-id"), + status=site_record.status.name, + ) + except AttributeError: + self.job.log_debug( + message=f"Error loading {site_record}, invalid or missing attributes on object. Skipping..." + ) + continue + self.add(location) + try: + # Load Site's Children - Devices with Interfaces, if any. + if self.sync_ipfabric_tagged_only: + nautobot_site_devices = Device.objects.filter(Q(site=site_record) & Q(tags__slug=ssot_tag.slug)) + else: + nautobot_site_devices = Device.objects.filter(site=site_record) + if nautobot_site_devices.exists(): + self.load_device(nautobot_site_devices, location) + + # Load Site Children - Vlans, if any. + nautobot_site_vlans = VLAN.objects.filter(site=site_record) + if not nautobot_site_vlans.exists(): + continue + self.load_vlans(nautobot_site_vlans, location) + except Site.DoesNotExist: + self.job.log_info(message=f"Unable to find Site, {site_record}.") + else: + self.job.log_warning(message="No Nautobot records to load.") + + def load(self): + """Load data from Nautobot.""" + self.load_data() diff --git a/nautobot_ssot/integrations/ipfabric/diffsync/adapters_shared.py b/nautobot_ssot/integrations/ipfabric/diffsync/adapters_shared.py new file mode 100644 index 000000000..5f0d86a44 --- /dev/null +++ b/nautobot_ssot/integrations/ipfabric/diffsync/adapters_shared.py @@ -0,0 +1,22 @@ +"""Diff sync shared adapter class attritbutes to synchronize applications.""" + +from typing import ClassVar + +from diffsync import DiffSync + +from nautobot_ssot.integrations.ipfabric.diffsync import diffsync_models + + +class DiffSyncModelAdapters(DiffSync): + """Nautobot adapter for DiffSync.""" + + safe_delete_mode: ClassVar[bool] = True + + location = diffsync_models.Location + device = diffsync_models.Device + interface = diffsync_models.Interface + vlan = diffsync_models.Vlan + + top_level = [ + "location", + ] diff --git a/nautobot_ssot/integrations/ipfabric/diffsync/diffsync_models.py b/nautobot_ssot/integrations/ipfabric/diffsync/diffsync_models.py new file mode 100644 index 000000000..501704ed2 --- /dev/null +++ b/nautobot_ssot/integrations/ipfabric/diffsync/diffsync_models.py @@ -0,0 +1,458 @@ +# pylint: disable=duplicate-code +# Ignore return statements for updates and deletes, # pylint:disable=R1710 +# Ignore too many args # pylint:disable=too-many-locals +"""DiffSyncModel subclasses for Nautobot-to-IPFabric data sync.""" +from typing import Any, ClassVar, List, Optional +from uuid import UUID + +from diffsync import DiffSyncModel +from django.core.exceptions import ValidationError +from django.db.models import Q +from nautobot.dcim.models import Device as NautobotDevice +from nautobot.dcim.models import DeviceRole, DeviceType, Site +from nautobot.extras.models import Tag +from nautobot.extras.models.statuses import Status +from nautobot.ipam.models import VLAN +from nautobot.utilities.choices import ColorChoices + +import nautobot_ssot.integrations.ipfabric.utilities.nbutils as tonb_nbutils +from nautobot_ssot.integrations.ipfabric.constants import ( + DEFAULT_DEVICE_ROLE, + DEFAULT_DEVICE_ROLE_COLOR, + DEFAULT_DEVICE_STATUS, + DEFAULT_DEVICE_STATUS_COLOR, + DEFAULT_INTERFACE_MAC, + SAFE_DELETE_SITE_STATUS, + SAFE_DELETE_DEVICE_STATUS, + SAFE_DELETE_IPADDRESS_STATUS, + SAFE_DELETE_VLAN_STATUS, +) + + +class DiffSyncExtras(DiffSyncModel): + """Additional components to mix and subclass from with `DiffSyncModel`.""" + + safe_delete_mode: ClassVar[bool] = True + + def safe_delete(self, nautobot_object: Any, safe_delete_status: Optional[str] = None): + """Safe delete an object, by adding tags or changing it's default status. + + Args: + nautobot_object (Any): Any type of Nautobot object + safe_delete_status (Optional[str], optional): Status name, optional as some objects don't have status field. Defaults to None. + """ + update = False + if not self.safe_delete_mode: # This could just check self, refactor. + self.diffsync.job.log_warning( + message=f"{nautobot_object} will be deleted as safe delete mode is not enabled." + ) + # This allows private class naming of nautobot objects to be ordered for delete() + # Example definition in adapter class var: _site = Site + self.diffsync.objects_to_delete[f"_{nautobot_object.__class__.__name__.lower()}"].append( + nautobot_object + ) # pylint: disable=protected-access + super().delete() + else: + if safe_delete_status: + safe_delete_status = Status.objects.get(name=safe_delete_status.capitalize()) + if hasattr(nautobot_object, "status"): + if not nautobot_object.status == safe_delete_status: + nautobot_object.status = safe_delete_status + self.diffsync.job.log_warning( + message=f"{nautobot_object} has changed status to {safe_delete_status}." + ) + update = True + else: + # Not everything has a status. This may come in handy once more models are synced. + self.diffsync.job.log_warning(message=f"{nautobot_object} has no Status attribute.") + if hasattr(nautobot_object, "tags"): + ssot_safe_tag, _ = Tag.objects.get_or_create( + slug="ssot-safe-delete", + name="SSoT Safe Delete", + defaults={ + "description": "Safe Delete Mode tag to flag an object, but not delete from Nautobot.", + "color": ColorChoices.COLOR_RED, + }, + ) + object_tags = nautobot_object.tags.all() + # No exception raised for empty iterator, safe to do this any + if not any(obj_tag for obj_tag in object_tags if obj_tag.name == ssot_safe_tag.name): + nautobot_object.tags.add(ssot_safe_tag) + self.diffsync.job.log_warning(message=f"Tagging {nautobot_object} with `ssot-safe-delete`.") + update = True + if update: + tonb_nbutils.tag_object(nautobot_object=nautobot_object, custom_field="ssot-synced-from-ipfabric") + else: + self.diffsync.job.log_debug( + message=f"{nautobot_object} has previously been tagged with `ssot-safe-delete`. Skipping..." + ) + + return self + + +class Location(DiffSyncExtras): + """Location model.""" + + _modelname = "location" + _identifiers = ("name",) + _attributes = ("site_id", "status") + _children = {"device": "devices", "vlan": "vlans"} + + name: str + site_id: Optional[str] + status: str + devices: List["Device"] = list() # pylint: disable=use-list-literal + vlans: List["Vlan"] = list() # pylint: disable=use-list-literal + + @classmethod + def create(cls, diffsync, ids, attrs): + """Create Site in Nautobot.""" + tonb_nbutils.create_site(site_name=ids["name"], site_id=attrs["site_id"]) + return super().create(ids=ids, diffsync=diffsync, attrs=attrs) + + def delete(self) -> Optional["DiffSyncModel"]: + """Delete Site in Nautobot.""" + site_object = Site.objects.get(name=self.name) + + self.safe_delete( + site_object, + SAFE_DELETE_SITE_STATUS, + ) + return super().delete() + + def update(self, attrs): + """Update Site Object in Nautobot.""" + site = Site.objects.get(name=self.name) + if attrs.get("site_id"): + site.custom_field_data["ipfabric-site-id"] = attrs.get("site_id") + site.validated_save() + if attrs.get("status") == "Active": + safe_delete_tag, _ = Tag.objects.get_or_create(name="SSoT Safe Delete") + if not site.status == "Active": + site.status = Status.objects.get(name="Active") + device_tags = site.tags.filter(pk=safe_delete_tag.pk) + if device_tags.exists(): + site.tags.remove(safe_delete_tag) + tonb_nbutils.tag_object(nautobot_object=site, custom_field="ssot-synced-from-ipfabric") + return super().update(attrs) + + +class Device(DiffSyncExtras): + """Device model.""" + + _modelname = "device" + _identifiers = ("name",) + _attributes = ("location_name", "model", "vendor", "serial_number", "role", "status") + _children = {"interface": "interfaces"} + + name: str + location_name: Optional[str] + model: Optional[str] + vendor: Optional[str] + serial_number: Optional[str] + role: Optional[str] + status: Optional[str] + + mgmt_address: Optional[str] + + interfaces: List["Interface"] = list() # pylint: disable=use-list-literal + + @classmethod + def create(cls, diffsync, ids, attrs): + """Create Device in Nautobot under its parent site.""" + # Get DeviceType + device_type_filter = DeviceType.objects.filter(slug=attrs["model"]) + if device_type_filter.exists(): + device_type_object = device_type_filter.first() + else: + device_type_object = tonb_nbutils.create_device_type_object( + device_type=attrs["model"], vendor_name=attrs["vendor"] + ) + # Get DeviceRole, update if missing cf and create otherwise + role_name = attrs.get("role", DEFAULT_DEVICE_ROLE) + device_role_filter = DeviceRole.objects.filter(name=role_name) + if device_role_filter.exists(): + device_role_object = device_role_filter.first() + device_role_object.cf["ipfabric_type"] = role_name + device_role_object.validated_save() + else: + device_role_object = tonb_nbutils.get_or_create_device_role_object( + role_name=role_name, role_color=DEFAULT_DEVICE_ROLE_COLOR + ) + # Get Status + device_status_filter = Status.objects.filter(name=DEFAULT_DEVICE_STATUS) + if device_status_filter.exists(): + device_status_object = device_status_filter.first() + else: + device_status_object = tonb_nbutils.create_status(DEFAULT_DEVICE_STATUS, DEFAULT_DEVICE_STATUS_COLOR) + # Get Site + site_object_filter = Site.objects.filter(name=attrs["location_name"]) + if site_object_filter.exists(): + site_object = site_object_filter.first() + else: + site_object = tonb_nbutils.create_site(attrs["location_name"]) + + new_device, _ = NautobotDevice.objects.get_or_create( + name=ids["name"], + serial=attrs.get("serial_number", ""), + status=device_status_object, + device_type=device_type_object, + device_role=device_role_object, + site=site_object, + ) + try: + # Validated save happens inside of tag_objet + tonb_nbutils.tag_object(nautobot_object=new_device, custom_field="ssot-synced-from-ipfabric") + except ValidationError as error: + message = f"Unable to create device: {ids['name']}. A validation error occured. Enable debug for more information." + diffsync.job.log_debug(message=error) + diffsync.job.log_failure(message=message) + + return super().create(ids=ids, diffsync=diffsync, attrs=attrs) + + def delete(self) -> Optional["DiffSyncModel"]: + """Delete device in Nautobot.""" + try: + device_object = NautobotDevice.objects.get(name=self.name) + self.safe_delete( + device_object, + SAFE_DELETE_DEVICE_STATUS, + ) + return super().delete() + except NautobotDevice.DoesNotExist: + self.diffsync.job.log_warning(f"Unable to match device by name, {self.name}") + + def update(self, attrs): + """Update devices in Nautobot based on Source.""" + try: + _device = NautobotDevice.objects.get(name=self.name) + if attrs.get("status") == "Active": + safe_delete_tag, _ = Tag.objects.get_or_create(name="SSoT Safe Delete") + if not _device.status == "Active": + _device.status = Status.objects.get(name="Active") + device_tags = _device.tags.filter(pk=safe_delete_tag.pk) + if device_tags.exists(): + _device.tags.remove(safe_delete_tag) + # TODO: If only the "model" is changing, the "vendor" is not available + if attrs.get("model"): + device_type_object = tonb_nbutils.create_device_type_object( + device_type=attrs["model"], vendor_name=attrs["vendor"] + ) + _device.type = device_type_object + if attrs.get("location_name"): + site_object = tonb_nbutils.create_site(attrs["location_name"]) + _device.site = site_object + if attrs.get("serial_number"): + _device.serial = attrs.get("serial_number") + if attrs.get("role"): + device_role_object = tonb_nbutils.get_or_create_device_role_object( + role_name=attrs.get("role", DEFAULT_DEVICE_ROLE), role_color=DEFAULT_DEVICE_ROLE_COLOR + ) + _device.device_role = device_role_object + tonb_nbutils.tag_object(nautobot_object=_device, custom_field="ssot-synced-from-ipfabric") + # Call the super().update() method to update the in-memory DiffSyncModel instance + return super().update(attrs) + except NautobotDevice.DoesNotExist: + self.diffsync.job.log_warning(f"Unable to match device by name, {self.name}") + + +class Interface(DiffSyncExtras): + """Interface model.""" + + _modelname = "interface" + _identifiers = ( + "name", + "device_name", + ) + _shortname = ("name",) + _attributes = ( + "description", + "enabled", + "mac_address", + "mtu", + "type", + "mgmt_only", + "ip_address", + "subnet_mask", + "ip_is_primary", + "status", + ) + + name: str + device_name: str + description: Optional[str] + enabled: Optional[bool] + mac_address: Optional[str] + mtu: Optional[int] + type: Optional[str] + mgmt_only: Optional[bool] + ip_address: Optional[str] + subnet_mask: Optional[str] + ip_is_primary: Optional[bool] + status: str + + @classmethod + def create(cls, diffsync, ids, attrs): + """Create interface in Nautobot under its parent device.""" + ssot_tag, _ = Tag.objects.get_or_create(name="SSoT Synced from IPFabric") + device_obj = NautobotDevice.objects.filter(Q(name=ids["device_name"]) & Q(tags__slug=ssot_tag.slug)).first() + + if not attrs.get("mac_address"): + attrs["mac_address"] = DEFAULT_INTERFACE_MAC + interface_obj = tonb_nbutils.create_interface( + device_obj=device_obj, + interface_details={**ids, **attrs}, + ) + ip_address = attrs["ip_address"] + if ip_address: + if interface_obj.ip_addresses.all().exists(): + interface_obj.ip_addresses.all().delete() + ip_address_obj = tonb_nbutils.create_ip( + ip_address=attrs["ip_address"], + subnet_mask=attrs["subnet_mask"], + status=attrs["status"], + object_pk=interface_obj, + ) + interface_obj.ip_addresses.add(ip_address_obj) + if attrs.get("ip_is_primary"): + if ip_address_obj.family == 4: + device_obj.primary_ip4 = ip_address_obj + device_obj.save() + if ip_address_obj.family == 6: + device_obj.primary_ip6 = ip_address_obj + device_obj.save() + interface_obj.save() + return super().create(ids=ids, diffsync=diffsync, attrs=attrs) + + def delete(self) -> Optional["DiffSyncModel"]: + """Delete Interface Object.""" + try: + ssot_tag, _ = Tag.objects.get_or_create(name="SSoT Synced from IPFabric") + device = NautobotDevice.objects.filter(Q(name=self.device_name) & Q(tags__slug=ssot_tag.slug)).first() + if not device: + return + interface = device.interfaces.get(name=self.name) + # Access the addr within an interface, change the status if necessary + if interface.ip_addresses.first(): + self.safe_delete(interface.ip_addresses.first(), SAFE_DELETE_IPADDRESS_STATUS) + # Then do the parent interface + # Attached interfaces do not have a status to update. + self.safe_delete( + interface, + ) + return super().delete() + except NautobotDevice.DoesNotExist: + self.diffsync.job.log_warning(f"Unable to match device by name, {self.name}") + + def update(self, attrs): # pylint: disable=too-many-branches + """Update Interface object in Nautobot.""" + try: + ssot_tag, _ = Tag.objects.get_or_create(name="SSoT Synced from IPFabric") + device = NautobotDevice.objects.filter(Q(name=self.device_name) & Q(tags__slug=ssot_tag.slug)).first() + interface = device.interfaces.get(name=self.name) + if attrs.get("description"): + interface.description = attrs["description"] + if attrs.get("enabled"): + interface.enabled = attrs["enabled"] + if attrs.get("mac_address"): + interface.mac_address = attrs["mac_address"] + if attrs.get("mtu"): + interface.mtu = attrs["mtu"] + if attrs.get("mode"): + interface.mode = attrs["mode"] + if attrs.get("lag"): + interface.lag = attrs["lag"] + if attrs.get("type"): + interface.type = attrs["type"] + if attrs.get("mgmt_only"): + interface.mgmt_only = attrs["mgmt_only"] + if attrs.get("ip_address"): + if interface.ip_addresses.all().exists(): + self.diffsync.job.log_debug(message=f"Replacing IP from interface {interface} on {device.name}") + interface.ip_addresses.all().delete() + ip_address_obj = tonb_nbutils.create_ip( + ip_address=attrs.get("ip_address"), + subnet_mask=attrs.get("subnet_mask") if attrs.get("subnet_mask") else "255.255.255.255", + status="Active", + object_pk=interface, + ) + interface.ip_addresses.add(ip_address_obj) + if attrs.get("ip_is_primary"): + interface_obj = interface.ip_addresses.first() + if interface_obj: + if interface_obj.family == 4: + device.primary_ip4 = interface_obj + device.save() + if interface_obj.family == 6: + device.primary_ip6 = interface_obj + device.save() + interface.save() + tonb_nbutils.tag_object(nautobot_object=interface, custom_field="ssot-synced-from-ipfabric") + return super().update(attrs) + + except NautobotDevice.DoesNotExist: + self.diffsync.job.log_warning(f"Unable to match device by name, {self.name}") + + +class Vlan(DiffSyncExtras): + """VLAN model.""" + + _modelname = "vlan" + _identifiers = ("name", "site") + _shortname = ("name",) + _attributes = ("vid", "status", "description") + + name: str + vid: int + status: str + site: str + description: Optional[str] + vlan_pk: Optional[UUID] + + @classmethod + def create(cls, diffsync, ids, attrs): + """Create VLANs in Nautobot under the site.""" + status = attrs["status"].lower().capitalize() + site = Site.objects.get(name=ids["site"]) + name = ids["name"] if ids["name"] else f"VLAN{attrs['vid']}" + description = attrs["description"] if attrs["description"] else None + diffsync.job.log_debug(message=f"Creating VLAN: {name} description: {description}") + tonb_nbutils.create_vlan( + vlan_name=name, + vlan_id=attrs["vid"], + vlan_status=status, + site_obj=site, + description=description, + ) + return super().create(ids=ids, diffsync=diffsync, attrs=attrs) + + def delete(self) -> Optional["DiffSyncModel"]: + """Delete.""" + vlan = VLAN.objects.get(name=self.name, pk=self.vlan_pk) + self.safe_delete( + vlan, + SAFE_DELETE_VLAN_STATUS, + ) + return super().delete() + + def update(self, attrs): + """Update VLAN object in Nautobot.""" + vlan = VLAN.objects.get(name=self.name, vid=self.vid, site=Site.objects.get(name=self.site)) + + if attrs.get("status") == "Active": + safe_delete_tag, _ = Tag.objects.get_or_create(name="SSoT Safe Delete") + if not vlan.status == "Active": + vlan.status = Status.objects.get(name="Active") + device_tags = vlan.tags.filter(pk=safe_delete_tag.pk) + if device_tags.exists(): + vlan.tags.remove(safe_delete_tag) + if attrs.get("description"): + vlan.description = vlan.description + + tonb_nbutils.tag_object(nautobot_object=vlan, custom_field="ssot-synced-from-ipfabric") + + +Location.update_forward_refs() +Device.update_forward_refs() +Interface.update_forward_refs() +Vlan.update_forward_refs() diff --git a/nautobot_ssot/integrations/ipfabric/jobs.py b/nautobot_ssot/integrations/ipfabric/jobs.py new file mode 100644 index 000000000..41dd1c87f --- /dev/null +++ b/nautobot_ssot/integrations/ipfabric/jobs.py @@ -0,0 +1,292 @@ +# pylint: disable=keyword-arg-before-vararg +# pylint: disable=too-few-public-methods +# pylint: disable=too-many-locals +"""IP Fabric Data Target Job.""" +import uuid +from diffsync.enum import DiffSyncFlags +from diffsync.exceptions import ObjectNotCreated +from django.conf import settings +from django.templatetags.static import static +from django.urls import reverse +from httpx import ConnectError +from ipfabric import IPFClient +from nautobot.dcim.models import Site +from nautobot.extras.jobs import BooleanVar, Job, ScriptVariable, ChoiceVar +from nautobot.utilities.forms import DynamicModelChoiceField +from nautobot_ssot.jobs.base import DataMapping, DataSource + +from nautobot_ssot.integrations.ipfabric.diffsync.adapter_ipfabric import IPFabricDiffSync +from nautobot_ssot.integrations.ipfabric.diffsync.adapter_nautobot import NautobotDiffSync +from nautobot_ssot.integrations.ipfabric.diffsync.adapters_shared import DiffSyncModelAdapters +from nautobot_ssot.integrations.ipfabric.diffsync.diffsync_models import DiffSyncExtras +from nautobot_ssot.integrations.ipfabric import constants + +CONFIG = settings.PLUGINS_CONFIG.get("nautobot_ssot", {}) +IPFABRIC_HOST = CONFIG["ipfabric_host"] +IPFABRIC_API_TOKEN = CONFIG["ipfabric_api_token"] +IPFABRIC_SSL_VERIFY = CONFIG["ipfabric_ssl_verify"] +IPFABRIC_TIMEOUT = CONFIG["ipfabric_timeout"] +LAST = "$last" +PREV = "$prev" +LAST_LOCKED = "$lastLocked" + +name = "SSoT - IPFabric" # pylint: disable=invalid-name + + +def is_valid_uuid(identifier): + """Return true if the identifier it's a valid UUID.""" + try: + uuid.UUID(str(identifier)) + return True + except ValueError: + return False + + +def get_formatted_snapshots(client: IPFClient): + """Get all loaded snapshots and format them for display in choice menu. + + Returns: + dict: Snapshot objects as dict of tuples {snapshot_ref: (description, snapshot_id)} + """ + formatted_snapshots = {} + snapshot_refs = [] + if client: + for snapshot_ref, snapshot in client.loaded_snapshots.items(): + description = "" + if snapshot_ref in [LAST, PREV, LAST_LOCKED]: + description += f"{snapshot_ref}: " + snapshot_refs.append(snapshot_ref) + if snapshot.name: + description += snapshot.name + " - " + snapshot.end.strftime("%d-%b-%y %H:%M:%S") + else: + description += snapshot.end.strftime("%d-%b-%y %H:%M:%S") + " - " + snapshot.snapshot_id + formatted_snapshots[snapshot_ref] = (description, snapshot.snapshot_id) + for ref in snapshot_refs: + formatted_snapshots.pop(formatted_snapshots[ref][1], None) + + return formatted_snapshots + + +class OptionalObjectVar(ScriptVariable): + """Custom implementation of an Optional ObjectVar. + + An object primary key is returned and accessible in job kwargs. + """ + + form_field = DynamicModelChoiceField + + def __init__( + self, + model=None, + display_field="display", + query_params=None, + null_option=None, + *args, + **kwargs, + ): + """Init.""" + super().__init__(*args, **kwargs) + + if model is not None: + self.field_attrs["queryset"] = model.objects.all() + else: + raise TypeError("ObjectVar must specify a model") + + self.field_attrs.update( + { + "display_field": display_field, + "query_params": query_params, + "null_option": null_option, + } + ) + + +# pylint:disable=too-few-public-methods +class IpFabricDataSource(DataSource, Job): + """Job syncing data from IP Fabric to Nautobot.""" + + client = None + snapshot = None + debug = BooleanVar(description="Enable for more verbose debug logging") + safe_delete_mode = BooleanVar( + description="Records are not deleted. Status fields are updated as necessary.", + default=True, + label="Safe Delete Mode", + ) + sync_ipfabric_tagged_only = BooleanVar( + default=True, + label="Sync Tagged Only", + description="Only sync objects that have the 'ssot-synced-from-ipfabric' tag.", + ) + site_filter = OptionalObjectVar( + description="Only sync Nautobot records belonging to a single Site. This does not filter IPFabric data.", + model=Site, + required=False, + ) + + class Meta: + """Metadata about this Job.""" + + name = "IPFabric ⟹ Nautobot" + data_source = "IP Fabric" + data_source_icon = static("nautobot_ssot_ipfabric/ipfabric.png") + description = "Sync data from IP Fabric into Nautobot." + field_order = ( + "debug", + "snapshot", + "safe_delete_mode", + "sync_ipfabric_tagged_only", + "dry_run", + ) + + @staticmethod + def _init_ipf_client(): + try: + return IPFClient( + base_url=IPFABRIC_HOST, + token=IPFABRIC_API_TOKEN, + verify=IPFABRIC_SSL_VERIFY, + timeout=IPFABRIC_TIMEOUT, + unloaded=False, + ) + except (RuntimeError, ConnectError) as error: + print(f"Got an error {error}") + return None + + @classmethod + def _get_vars(cls): + """Extend JobDataSource._get_vars to include some variables. + + This also initializes them. + """ + got_vars = super()._get_vars() + + if cls.client is None: + cls.client = cls._init_ipf_client() + else: + cls.client.update() + + formatted_snapshots = get_formatted_snapshots(cls.client) + if formatted_snapshots: + default_choice = formatted_snapshots["$last"][::-1] + else: + default_choice = "$last" + + cls.snapshot = ChoiceVar( + description="IPFabric snapshot to sync from. Defaults to $last", + default=default_choice, + choices=[(snapshot_id, snapshot_name) for snapshot_name, snapshot_id in formatted_snapshots.values()], + required=False, + ) + + if hasattr(cls, "snapshot"): + got_vars["snapshot"] = cls.snapshot + + return got_vars + + @classmethod + def data_mappings(cls): + """List describing the data mappings involved in this DataSource.""" + return ( + DataMapping("Device", None, "Device", reverse("dcim:device_list")), + DataMapping("Site", None, "Site", reverse("dcim:site_list")), + DataMapping("Interfaces", None, "Interfaces", reverse("dcim:interface_list")), + DataMapping("IP Addresses", None, "IP Addresses", reverse("ipam:ipaddress_list")), + DataMapping("VLANs", None, "VLANs", reverse("ipam:vlan_list")), + ) + + @classmethod + def config_information(cls): + """Dictionary describing the configuration of this DataSource.""" + return { + "IP Fabric host": CONFIG["ipfabric_host"], + "Nautobot Host URL": CONFIG.get("nautobot_host"), + "Default MAC Address": constants.DEFAULT_INTERFACE_MAC, + "Default Device Role": constants.DEFAULT_DEVICE_ROLE, + "Default Interface Type": constants.DEFAULT_INTERFACE_TYPE, + "Default Device Status": constants.DEFAULT_DEVICE_STATUS, + "Allow Duplicate Addresses": constants.ALLOW_DUPLICATE_ADDRESSES, + "Default MTU": constants.DEFAULT_INTERFACE_MTU, + "Safe Delete Device Status": constants.SAFE_DELETE_DEVICE_STATUS, + "Safe Delete Site Status": constants.SAFE_DELETE_SITE_STATUS, + "Safe Delete IPAddress Status": constants.SAFE_IPADDRESS_INTERFACES_STATUS, + "Safe Delete VLAN status": constants.SAFE_DELETE_VLAN_STATUS, + } + + def log_debug(self, message): + """Conditionally log a debug message.""" + if self.kwargs.get("debug"): + super().log_debug(message) + + def load_source_adapter(self): + """Not used.""" + + def load_target_adapter(self): + """Not used.""" + + def sync_data(self): + """Sync a device data from IP Fabric into Nautobot.""" + if self.client is None: + self.client = self._init_ipf_client() + if self.client is None: + self.log_failure(message="IPFabric client is not ready. Check your config.") + return + + self.client.snapshot_id = self.kwargs["snapshot"] + dry_run = self.kwargs["dry_run"] + safe_mode = self.kwargs["safe_delete_mode"] + tagged_only = self.kwargs["sync_ipfabric_tagged_only"] + site_filter = self.kwargs["site_filter"] + debug_mode = self.kwargs["debug"] + + if site_filter: + site_filter_object = Site.objects.get(pk=site_filter) + else: + site_filter_object = None + options = f"`Snapshot_id`: {self.client.snapshot_id}.`Debug`: {debug_mode}, `Dry Run`: {dry_run}, `Safe Delete Mode`: {safe_mode}, `Sync Tagged Only`: {tagged_only}, `Site Filter`: {site_filter_object}" + self.log_info(message=f"Starting job with the following options: {options}") + + ipfabric_source = IPFabricDiffSync(job=self, sync=self.sync, client=self.client) + self.log_info(message="Loading current data from IP Fabric...") + ipfabric_source.load() + + # Set safe mode either way (Defaults to True) + DiffSyncModelAdapters.safe_delete_mode = safe_mode + DiffSyncExtras.safe_delete_mode = safe_mode + + dest = NautobotDiffSync( + job=self, + sync=self.sync, + sync_ipfabric_tagged_only=tagged_only, + site_filter=site_filter_object, + ) + + self.log_info(message="Loading current data from Nautobot...") + dest.load() + + self.log_info(message="Calculating diffs...") + flags = DiffSyncFlags.CONTINUE_ON_FAILURE + + diff = dest.diff_from(ipfabric_source, flags=flags) + self.log_debug(message=f"Diff: {diff.dict()}") + + self.sync.diff = diff.dict() + self.sync.save() + create = diff.summary().get("create") + update = diff.summary().get("update") + delete = diff.summary().get("delete") + no_change = diff.summary().get("no-change") + self.log_info( + message=f"DiffSync Summary: Create: {create}, Update: {update}, Delete: {delete}, No Change: {no_change}" + ) + if not dry_run: + self.log_info(message="Syncing from IP Fabric to Nautobot") + try: + dest.sync_from(ipfabric_source) + except ObjectNotCreated as err: + self.log_debug(f"Unable to create object. {err}") + + self.log_success(message="Sync complete.") + + +jobs = [IpFabricDataSource] diff --git a/nautobot_ssot/integrations/ipfabric/signals.py b/nautobot_ssot/integrations/ipfabric/signals.py new file mode 100644 index 000000000..cf4d53fdc --- /dev/null +++ b/nautobot_ssot/integrations/ipfabric/signals.py @@ -0,0 +1,82 @@ +# pylint: disable=duplicate-code +"""Signal handlers for IPFabric integration.""" + +from typing import List, Optional + +from nautobot.core.signals import nautobot_database_ready +from nautobot.extras.choices import CustomFieldTypeChoices +from nautobot.utilities.choices import ColorChoices + + +def register_signals(sender): + """Register signals for IPFabric integration.""" + nautobot_database_ready.connect(nautobot_database_ready_callback, sender=sender) + + +def create_custom_field(field_name: str, label: str, models: List, apps, cf_type: Optional[str] = "type_date"): + """Create custom field on a given model instance type. + + Args: + field_name (str): Field Name + label (str): Label description + models (List): List of Django Models + apps: Django Apps + cf_type: (str, optional): Type of Field. Supports 'type_text' or 'type_date'. Defaults to 'type_date'. + """ + ContentType = apps.get_model("contenttypes", "ContentType") # pylint:disable=invalid-name + CustomField = apps.get_model("extras", "CustomField") # pylint:disable=invalid-name + if cf_type == "type_date": + custom_field, _ = CustomField.objects.get_or_create( + type=CustomFieldTypeChoices.TYPE_DATE, + name=field_name, + defaults={ + "label": label, + }, + ) + else: + custom_field, _ = CustomField.objects.get_or_create( + type=CustomFieldTypeChoices.TYPE_TEXT, + name=field_name, + defaults={ + "label": label, + }, + ) + for model in models: + custom_field.content_types.add(ContentType.objects.get_for_model(model)) + custom_field.save() + + +def nautobot_database_ready_callback(sender, *, apps, **kwargs): # pylint: disable=unused-argument + """Callback function triggered by the nautobot_database_ready signal when the Nautobot database is fully ready.""" + # pylint: disable=invalid-name + Device = apps.get_model("dcim", "Device") + DeviceType = apps.get_model("dcim", "DeviceType") + DeviceRole = apps.get_model("dcim", "DeviceRole") + Interface = apps.get_model("dcim", "Interface") + IPAddress = apps.get_model("ipam", "IPAddress") + Manufacturer = apps.get_model("dcim", "Manufacturer") + Site = apps.get_model("dcim", "Site") + VLAN = apps.get_model("ipam", "VLAN") + Tag = apps.get_model("extras", "Tag") + + Tag.objects.get_or_create( + slug="ssot-synced-from-ipfabric", + name="SSoT Synced from IPFabric", + defaults={ + "description": "Object synced at some point from IPFabric to Nautobot", + "color": ColorChoices.COLOR_LIGHT_GREEN, + }, + ) + Tag.objects.get_or_create( + slug="ssot-safe-delete", + name="SSoT Safe Delete", + defaults={ + "description": "Safe Delete Mode tag to flag an object, but not delete from Nautobot.", + "color": ColorChoices.COLOR_RED, + }, + ) + synced_from_models = [Device, DeviceType, Interface, Manufacturer, Site, VLAN, DeviceRole, IPAddress] + create_custom_field("ssot-synced-from-ipfabric", "Last synced from IPFabric on", synced_from_models, apps=apps) + site_model = [Site] + create_custom_field("ipfabric-site-id", "IPFabric Site ID", site_model, apps=apps, cf_type="type_text") + create_custom_field("ipfabric_type", "IPFabric Type", [DeviceRole], apps=apps, cf_type="type_text") diff --git a/nautobot_ssot/integrations/ipfabric/utilities/__init__.py b/nautobot_ssot/integrations/ipfabric/utilities/__init__.py new file mode 100644 index 000000000..87583e875 --- /dev/null +++ b/nautobot_ssot/integrations/ipfabric/utilities/__init__.py @@ -0,0 +1,25 @@ +"""Utilities.""" +from .nbutils import ( + get_or_create_device_role_object, + create_device_type_object, + create_interface, + create_ip, + create_manufacturer, + create_site, + create_status, + create_vlan, +) +from .test_utils import clean_slate, json_fixture + +__all__ = ( + "create_site", + "create_device_type_object", + "create_manufacturer", + "get_or_create_device_role_object", + "create_status", + "create_ip", + "create_interface", + "json_fixture", + "create_vlan", + "clean_slate", +) diff --git a/nautobot_ssot/integrations/ipfabric/utilities/nbutils.py b/nautobot_ssot/integrations/ipfabric/utilities/nbutils.py new file mode 100644 index 000000000..9adbc0a8e --- /dev/null +++ b/nautobot_ssot/integrations/ipfabric/utilities/nbutils.py @@ -0,0 +1,257 @@ +# pylint: disable=duplicate-code +"""Utility functions for Nautobot ORM.""" +import datetime +from typing import Any, Optional + +from django.contrib.contenttypes.models import ContentType +from django.db import IntegrityError +from django.utils.text import slugify +from nautobot.dcim.models import ( + Device, + DeviceRole, + DeviceType, + Interface, + Manufacturer, + Site, +) +from nautobot.extras.choices import CustomFieldTypeChoices +from nautobot.extras.models import CustomField, Tag +from nautobot.extras.models.statuses import Status +from nautobot.ipam.models import VLAN, IPAddress +from nautobot.utilities.choices import ColorChoices +from netutils.ip import netmask_to_cidr + +from nautobot_ssot.integrations.ipfabric.constants import ALLOW_DUPLICATE_ADDRESSES + + +def create_site(site_name, site_id=None): + """Creates a specified site in Nautobot. + + Args: + site_name (str): Name of the site. + site_id (str): ID of the site. + """ + site_obj, _ = Site.objects.get_or_create(name=site_name) + site_obj.slug = slugify(site_name) + site_obj.status = Status.objects.get(name="Active") + if site_id: + # Ensure custom field is available + custom_field_obj, _ = CustomField.objects.get_or_create( + type=CustomFieldTypeChoices.TYPE_TEXT, + name="ipfabric-site-id", + defaults={"label": "IPFabric Site ID"}, + ) + custom_field_obj.content_types.add(ContentType.objects.get_for_model(Site)) + site_obj.cf["ipfabric-site-id"] = site_id + site_obj.validated_save() + tag_object(nautobot_object=site_obj, custom_field="ssot-synced-from-ipfabric") + return site_obj + + +def create_manufacturer(vendor_name): + """Create specified manufacturer in Nautobot.""" + mf_name, _ = Manufacturer.objects.get_or_create(name=vendor_name) + tag_object(nautobot_object=mf_name, custom_field="ssot-synced-from-ipfabric") + return mf_name + + +def create_device_type_object(device_type, vendor_name): + """Create a specified device type in Nautobot. + + Args: + device_type (str): Device model gathered from DiffSync model. + vendor_name (str): Vendor Name + """ + mf_name = create_manufacturer(vendor_name) + device_type_obj, _ = DeviceType.objects.get_or_create( + manufacturer=mf_name, model=device_type, slug=slugify(device_type) + ) + tag_object(nautobot_object=device_type_obj, custom_field="ssot-synced-from-ipfabric") + return device_type_obj + + +def get_or_create_device_role_object(role_name, role_color): + """Create specified device role in Nautobot. + + Args: + role_name (str): Role name. + role_color (str): Role color. + """ + # adds custom field to map custom role names to ipfabric type names + try: + role_obj = DeviceRole.objects.get(_custom_field_data__ipfabric_type=role_name) + except DeviceRole.DoesNotExist: + role_obj = DeviceRole.objects.create(name=role_name, slug=slugify(role_name), color=role_color) + role_obj.cf["ipfabric_type"] = role_name + role_obj.validated_save() + tag_object(nautobot_object=role_obj, custom_field="ssot-synced-from-ipfabric") + return role_obj + + +def create_status(status_name, status_color, description="", app_label="dcim", model="device"): + """Verify status object exists in Nautobot. If not, creates specified status. Defaults to dcim | device. + + Args: + status_name (str): Status name. + status_color (str): Status color. + description (str): Description + app_label (str): App Label ("DCIM") + model (str): Django Model ("DEVICE") + """ + try: + status_obj = Status.objects.get(name=status_name) + except Status.DoesNotExist: + content_type = ContentType.objects.get(app_label=app_label, model=model) + status_obj = Status.objects.create( + name=status_name, + slug=slugify(status_name), + color=status_color, + description=description, + ) + status_obj.content_types.set([content_type]) + return status_obj + + +def create_ip(ip_address, subnet_mask, status="Active", object_pk=None): + """Verify ip address exists in Nautobot. If not, creates specified ip. + + Utility behavior is manipulated by `settings` if duplicate ip's are allowed. + + Args: + ip_address (str): IP address. + subnet_mask (str): Subnet mask used for IP Address. + status (str): Status to assign to IP Address. + object_pk: Object primary key + """ + status_obj = Status.objects.get_for_model(IPAddress).get(slug=slugify(status)) + cidr = netmask_to_cidr(subnet_mask) + if ALLOW_DUPLICATE_ADDRESSES: + addr = IPAddress.objects.filter(host=ip_address) + data = {"address": f"{ip_address}/{cidr}", "status": status_obj} + if addr.exists(): + data["description"] = "Duplicate by IPFabric SSoT" + + ip_obj = IPAddress.objects.create(**data) + + else: + ip_obj, _ = IPAddress.objects.get_or_create(address=f"{ip_address}/{cidr}", status=status_obj) + + if object_pk: + ip_obj.assigned_object_id = object_pk.pk + ip_obj.assigned_object_type = ContentType.objects.get_for_model(type(object_pk)) + # Tag Interface (object_pk) + tag_object(nautobot_object=object_pk, custom_field="ssot-synced-from-ipfabric") + + # Tag IP Addr + tag_object(nautobot_object=ip_obj, custom_field="ssot-synced-from-ipfabric") + return ip_obj + + +def create_interface(device_obj, interface_details): + """Verify interface exists on specified device. If not, creates interface. + + Args: + device_obj (Device): Device object to check interface against. + interface_details (dict): interface details. + """ + interface_fields = ( + "name", + "description", + "enabled", + "mac_address", + "mtu", + "type", + "mgmt_only", + ) + fields = {k: v for k, v in interface_details.items() if k in interface_fields and v} + try: + interface_obj, _ = device_obj.interfaces.get_or_create(**fields) + except IntegrityError: + interface_obj, _ = device_obj.interfaces.get_or_create(name=fields["name"]) + interface_obj.description = fields.get("description", "") + interface_obj.enabled = fields.get("enabled") + interface_obj.mac_address = fields.get("mac_address") + interface_obj.mtu = fields.get("mtu") + interface_obj.type = fields.get("type") + interface_obj.mgmt_only = fields.get("mgmt_only", False) + interface_obj.validated_save() + tag_object(nautobot_object=interface_obj, custom_field="ssot-synced-from-ipfabric") + return interface_obj + + +def create_vlan(vlan_name: str, vlan_id: int, vlan_status: str, site_obj: Site, description: str): + """Creates or obtains VLAN object. + + Args: + vlan_name (str): VLAN Name + vlan_id (int): VLAN ID + vlan_status (str): VLAN Status + site_obj (Site): Site Django Model + description (str): VLAN Description + + Returns: + (VLAN): Returns created or obtained VLAN object. + """ + vlan_obj, _ = site_obj.vlans.get_or_create( + name=vlan_name, vid=vlan_id, status=Status.objects.get(name=vlan_status), description=description + ) + tag_object(nautobot_object=vlan_obj, custom_field="ssot-synced-from-ipfabric") + return vlan_obj + + +def tag_object(nautobot_object: Any, custom_field: str, tag_name: Optional[str] = "SSoT Synced from IPFabric"): + """Apply the given tag and custom field to the identified object. + + Args: + nautobot_object (Any): Nautobot ORM Object + custom_field (str): Name of custom field to update + tag_name (Optional[str], optional): Tag name. Defaults to "SSoT Synced From IPFabric". + """ + if tag_name == "SSoT Synced from IPFabric": + tag, _ = Tag.objects.get_or_create( + slug="ssot-synced-from-ipfabric", + name="SSoT Synced from IPFabric", + defaults={ + "description": "Object synced at some point from IPFabric to Nautobot", + "color": ColorChoices.COLOR_LIGHT_GREEN, + }, + ) + else: + tag, _ = Tag.objects.get_or_create(name=tag_name) + + today = datetime.date.today().isoformat() + + def _tag_object(nautobot_object): + """Apply custom field and tag to object, if applicable.""" + if hasattr(nautobot_object, "tags"): + nautobot_object.tags.add(tag) + if hasattr(nautobot_object, "cf"): + # Ensure that the "ssot-synced-from-ipfabric" custom field is present + if not any(cfield for cfield in CustomField.objects.all() if cfield.name == "ssot-synced-from-ipfabric"): + custom_field_obj, _ = CustomField.objects.get_or_create( + type=CustomFieldTypeChoices.TYPE_DATE, + name="ssot-synced-from-ipfabric", + defaults={ + "label": "Last synced from IPFabric on", + }, + ) + synced_from_models = [ + Device, + DeviceType, + Interface, + Manufacturer, + Site, + VLAN, + DeviceRole, + IPAddress, + ] + for model in synced_from_models: + custom_field_obj.content_types.add(ContentType.objects.get_for_model(model)) + custom_field_obj.validated_save() + + # Update custom field date stamp + nautobot_object.cf[custom_field] = today + nautobot_object.validated_save() + + _tag_object(nautobot_object) + # Ensure proper save diff --git a/nautobot_ssot/integrations/ipfabric/utilities/test_utils.py b/nautobot_ssot/integrations/ipfabric/utilities/test_utils.py new file mode 100644 index 000000000..af159a1be --- /dev/null +++ b/nautobot_ssot/integrations/ipfabric/utilities/test_utils.py @@ -0,0 +1,21 @@ +"""Test Utils.""" +import json + +from nautobot.dcim.models.sites import Site +from nautobot.ipam.models import VLAN, Device + + +def json_fixture(json_file_path): + """Load and return JSON Fixture.""" + with open(json_file_path, "r", encoding="utf-8") as file: + return json.load(file) + + +def clean_slate(): + """Delete all objects synced. + + Use this with caution. Never use in production env. + """ + VLAN.objects.all().delete() + Device.objects.all().delete() + Site.objects.all().delete() diff --git a/nautobot_ssot/integrations/ipfabric/workers.py b/nautobot_ssot/integrations/ipfabric/workers.py new file mode 100644 index 000000000..d2bab1a34 --- /dev/null +++ b/nautobot_ssot/integrations/ipfabric/workers.py @@ -0,0 +1,150 @@ +# Disable dispatcher from chatops unused. # pylint: disable=unused-argument +"""Chat Ops Worker.""" +import uuid + +from django.conf import settings +from django.contrib.contenttypes.models import ContentType +from django_rq import job +from nautobot.core.settings_funcs import is_truthy +from nautobot.extras.models import JobResult + +# pylint: disable-next=import-error +from nautobot_chatops.choices import CommandStatusChoices + +# pylint: disable-next=import-error +from nautobot_chatops.dispatchers import Dispatcher + +# pylint: disable-next=import-error +from nautobot_chatops.workers import handle_subcommands, subcommand_of + +from nautobot_ssot.integrations.ipfabric.jobs import IpFabricDataSource + +# from nautobot.dcim.models import Site + +CONFIG = settings.PLUGINS_CONFIG.get("nautobot_ssot", {}) +NAUTOBOT_HOST = CONFIG.get("nautobot_host") + +BASE_CMD = "ipfabric" +IPFABRIC_LOGO_PATH = "nautobot_ssot_ipfabric/ipfabric_logo.png" +IPFABRIC_LOGO_ALT = "IPFabric Logo" + + +def prompt_for_bool(dispatcher: Dispatcher, action_id: str, help_text: str): + """Prompt the user to select a True or False choice.""" + choices = [("Yes", "True"), ("No", "False")] + dispatcher.prompt_from_menu(action_id, help_text, choices, default=("Yes", "True")) + return False + + +# def prompt_for_site(dispatcher: Dispatcher, action_id: str, help_text: str, sites=None, offset=0): +# """Prompt the user to select a valid site from a drop-down menu.""" +# if sites is None: +# sites = Site.objects.all().order_by("name") +# if not sites: +# dispatcher.send_error("No sites were found") +# return (CommandStatusChoices.STATUS_FAILED, "No sites found") +# choices = [(f"{site.name}: {site.name}", site.name) for site in sites] +# return dispatcher.prompt_from_menu(action_id, help_text, choices, offset=offset) + + +def ipfabric_logo(dispatcher): + """Construct an image_element containing the locally hosted IP Fabric logo.""" + return dispatcher.image_element(dispatcher.static_url(IPFABRIC_LOGO_PATH), alt_text=IPFABRIC_LOGO_ALT) + + +@job("default") +def ipfabric(subcommand, **kwargs): + """Interact with ipfabric plugin.""" + return handle_subcommands("ipfabric", subcommand, **kwargs) + + +@subcommand_of("ipfabric") +def ssot_sync_to_nautobot( + dispatcher, + dry_run=None, + safe_delete_mode=None, + sync_ipfabric_tagged_only=None, +): + """Start an SSoT sync from IPFabric to Nautobot.""" + if dry_run is None: + prompt_for_bool(dispatcher, f"{BASE_CMD} ssot-sync-to-nautobot", "Do you want to run a `Dry Run`?") + return (CommandStatusChoices.STATUS_SUCCEEDED, "Success") + + if safe_delete_mode is None: + prompt_for_bool( + dispatcher, f"{BASE_CMD} ssot-sync-to-nautobot {dry_run}", "Do you want to run in `Safe Delete Mode`?" + ) + return (CommandStatusChoices.STATUS_SUCCEEDED, "Success") + + if sync_ipfabric_tagged_only is None: + prompt_for_bool( + dispatcher, + f"{BASE_CMD} ssot-sync-to-nautobot {dry_run} {safe_delete_mode}", + "Do you want to sync against `ssot-tagged-from-ipfabric` tagged objects only?", + ) + return (CommandStatusChoices.STATUS_SUCCEEDED, "Success") + + # if site_filter is None: + # prompt_for_site( + # dispatcher, + # f"{BASE_CMD} ssot-sync-to-nautobot {dry_run} {safe_delete_mode} {sync_ipfabric_tagged_only}", + # "Select a Site to use as an optional filter?", + # ) + # return (CommandStatusChoices.STATUS_SUCCEEDED, "Success") + + # Implement filter in future release + site_filter = False + + data = { + "debug": False, + "dry_run": is_truthy(dry_run), + "safe_delete_mode": is_truthy(safe_delete_mode), + "sync_ipfabric_tagged_only": is_truthy(sync_ipfabric_tagged_only), + "site_filter": site_filter, + } + + sync_job = IpFabricDataSource() + + sync_job.job_result = JobResult( + name=sync_job.class_path, + obj_type=ContentType.objects.get( + app_label="extras", + model="job", + ), + job_id=uuid.uuid4(), + ) + sync_job.job_result.validated_save() + + dispatcher.send_markdown( + f"Stand by {dispatcher.user_mention()}, I'm running your sync with options set to `Dry Run`: {dry_run}, `Safe Delete Mode`: {safe_delete_mode}. `Sync Tagged Only`: {sync_ipfabric_tagged_only}", + ephemeral=True, + ) + + sync_job.run(data, commit=True) + sync_job.post_run() + sync_job.job_result.set_status(status="completed" if not sync_job.failed else "failed") + sync_job.job_result.validated_save() + + blocks = [ + *dispatcher.command_response_header( + "ipfabric", + "ssot-sync-to-nautobot", + [ + ("Dry Run", str(dry_run)), + ("Safe Delete Mode", str(safe_delete_mode)), + ("Sync IPFabric Tagged Only", str(sync_ipfabric_tagged_only)), + ], + "sync job", + ipfabric_logo(dispatcher), + ), + ] + dispatcher.send_blocks(blocks) + if sync_job.job_result.status == "completed": + dispatcher.send_markdown( + f"Sync completed succesfully. Here is the link to your job: {NAUTOBOT_HOST}{sync_job.sync.get_absolute_url()}." + ) + else: + dispatcher.send_warning( + f"Sync failed. Here is the link to your job: {NAUTOBOT_HOST}{sync_job.sync.get_absolute_url()}" + ) + return CommandStatusChoices.STATUS_SUCCEEDED diff --git a/nautobot_ssot/static/nautobot_ssot_ipfabric/ipfabric.png b/nautobot_ssot/static/nautobot_ssot_ipfabric/ipfabric.png new file mode 100644 index 0000000000000000000000000000000000000000..5d1073934cc881765208c6f34aea98198c55bb5d GIT binary patch literal 8451 zcmV+eA^hHnP) zaB^>EX>4U6ba`-PAZ2)IW&i+q+O3&emL;chhTj>5hrm5xI1s>Eo`Hw&@6T$rj&!vA zV7p4)Rob096C&cjvBUkZ|9s&;_^Ns2P@9cDdyjn8S!Z7SGUA_K_t(!EGxGB{N*41_$~4MFR$nK>t8dk6n>VU|NZ^Eq8pWexl#Z9GyT)+sr~Ks^84}mFDn}T z_mlHqR&-MQ%h&kx&wFP)_8fF|UA0r6{%cczZ1MYz3pa%+-`hIh#-GOT_I*#jM_kcE zs_{gRn{Q7uDYvsig9q2^5sQfmy$qc^2=nlrMV&3i0;*XEXgUIs!mqLp5Z z*37c#RjsY|P^}JTXtmO6YppkHqs_K@>9x1syLDl4S+Q!(x|t1|wniCsw9$u+G3Hod z=2^gMn|<0GbI!GR=;F?ccdxN{Vdq_T-EH@6d+fQFGd^Cud3W>S(^ok8lv7VT{n#02 zo^{h@ly1H4_G@?CdDmmqUU~Jk*Pp%d=3BqJ_H{MHUToWBPb%24+>RBAA)~wD2^(@W|Dw<}msX2h1j?uw-Q*JN)?%kiA z`?tOYaR0yUTmIM11-0(~qjN#6`-jf`d*A+{Ye)RU9fFWlMYF0r!- zZdCKjWw()x_Zd%TH*be#wXE0oTlTz+!yV2&=iO7*I_%zeqO(xz+f2_Lt2{bv5a7P0 z)6;fGvlB|&UJ_NI;XB8U?!4pXK+p?PVTz-6^iub&$m#-nTY3 zpGQx}FH3(bl`?78=Jut(`_|p9>`U$X%e_yU(K{Dkx0%_=>tJm`9$armg~lnBw@w<} zuANgE_Z1l*R^qWYYNH2_8>u79K6HLm)Tmm>2gSLtc-JLw6?S;?T+OMMzho5Ztx0Y2 z;FUHU2okU@OYLE1q_#!I;m<|Q*}i?Vt%ifHG>J>&P& z60gyQw0B@x?VV!p?kmYe1$Qj>gScLt@6ne1sH`;{#)L=#z^P2~p#^!~S+aSk%;EHw zxA(VBjerwK(A&gs_eLuWNkuE)Xw>>b$v4uvTDg zlY*9ZN^|CmVzmSGd|N3OL1yG(2rl1#Y8fq}YxTu*XFDl-I(Y!C))~3B>Vp|5d~9Be zwYdEBS1N0)=Om^v`m$~GFELZmcwxqUfV#jupDie!2t`a z31_G%UC(*XkoQRjVZc74LIDX}@4IPJA5f>2zNYovnFa37eMW-K2BWXGAh$R5@Fc3r!MzNd|eE)I4Kyo~BMaAKH_ z$+nF#hoT`rI-98(bzqRv!`$fRHSNKpUKsX0EBCz^!`I@5;UEmQyxB*E?4aqEYy8Yr z9R>39v>s1kTW#5JeiHuk`AF*vp5;G(Em zFm_a+-&kOj>DW@PDviz-VBE9YLLhj7iNf^3HLQQv;#B?N{AfQO9o%ln4=Rp+Tjk#( z^ROuLXsyE4RZ?@10@l0?nrla}hLN*<*v}`dEbOB-ANCjHk=jqaez)e2#;7mq%AAjp zdaU-|$xoEUUU*1$V_=}RVY|V>5eD_Pzz8^lr-776t)V-wpk8r z?Fzqv#_L==Sk^S4fAwZO1XpIC02UWADFlQ$f+$3z0bWvq1W*-5oL_B}jaknNZOXe! z1sx6q!z{DINEj8FM&XUC0%DGfGFagA2e8-tRvAJ)zI!9&}ZS5}<^8-?wHbUB(9;NBceU77!LC2|N*OdVK~s zE;`b-1f07c(#ufHxlF96I6bT_JpR&T%CN`dhih0Q!v}ZVUE072FZ@f=qg`2G=xqv2*P-myE7`+NF&K{UciYl{$ ziM=DQ4pQMohh}O`dC_Wcifm&>;T3Tdg3h}?TG@L?(6cr^dzOm{K)0eQ>IvZ)lV;GF zPW|K{kO2l3?Pp*f+o65)v1W(|sn__VFd_mbzfg+PXu_sZ;l8?rU;uX2Vr40gbU+Z zX@Zo3O>T&J9sH$q0%alaXa}MQwqb7Urvp1ySr+|+DwW07!T)!09WBYzLzlrt4rpka z;GY6O9O=M^PJ>hml61a_2Sw_GpTyw|yxNe+E=?0GMDTl2K!#Q71b`SU+!6NS;G#F+ zQ^OrDPGibdL6AO^(N=I=BLj*aF$820CX&GEPzE= zP?ehgcK5&D98*if&N@KF30s34z>U~FdIpKgT-(G%c)#zl?^n;mF$%gXrbP=z{%l3k z-{go#VZT?v;OB0C8I8|5hQBzCU*?fU{<4oT{x*=%ul^se%nfPh`EJz`1ArsQNH!&< zmOAMm%w5bDPM!uI7K{ecDr^joW_E~8_>p>6{l<`BKv&VuQnC}XkZ-t=x*}Utn;3_M z!$iox@>X|(BdliE)kQ#lnYe;5F%xkqgVBN;ppSQEaVK|dxDsi>XsAmLf3FE3d@$}C z9Tt{#0&pHfv4@(xo9W8jx%Mjh7qqnO9Z4+!jB9a}M|BfY_1IAI({c*>NjX~SvVr(!U}K7EUt)mU52;{6eG4;B%F(BqL2_r5UVkD7Zxi1LkmN`#GqFZ)+ zA};&}rE^#MpfeS=+~1scy!hhz$lcqd{z}TL5-Lc&V%D4mcQL>^9rdc*s615Mt4*x# zY}-1?S@@=#$Xp|wyRMGnozbrmuMq1D)Dfd%pbV?bjD!PD84O zlOhm-I6N{tB*k`}+VMn)B<}MzF82-4-*9hL^iD8C>~UOh^1_gi!iX}3bpSzvMJIs3 z&U9y@2m}=}@9MG*AT#{kRQ`C%zOi{P4L>Rql*_7!0EOM4S_v0rRMV5D@DWk~`%Fj@DrIcPu6eb9+#64jGMk}MjXygJ_LOm~IDiL~__#DM#TvgyJljKL2&f?#VT zQ<{ucIm6dU0h>s8_H`ZMq9nUt0hwMvH;PMPP*s9=XR_O4M3_71efTAB4-= z#S#xE&bGN!#}ryJ=p?9*9_D(4WSyamihMo1ITFc`@#6K^loc>bSRh8BGzEEsXO^Hs z)Fe-`UtD;gBdnI#D83VglH|1B%Vej$7NW`YAjh)en4e39*%LrP{1I!K_*xM zM7Xd(olxhrCyxXm7Lp0n(KoQ=IA3)I{tAp19%fOJtb0Kj?79wNUaPkq+2%YzF?`* z_1U?}>yz$e7YTub7zx|v#%FmVX9jxv( z6haj_uXLm6BnC1podzSpR`_CADz{evBR&p=QIrtuDE3!ca|`ZpNAU+0$K`=EA! zy6GJHtcW+*3?A`EhjG~Css3RWab-Z;PlytJyTBpq{I!|T4UKToSqc_|D2a7Ika-oS zhOyC^xh50v+K#Lneho3#AIO4U`peF=CT7K+x}6bObZ*A{t@UCnf0kPPXQ_Sqx76_W zwI;y5GzDm6Ukl=0z!2{?$gdS?LCAXNN%q&3h6ATESql@km-UHjVj$6I#x(BlI56pB z+!b+xPmTo3mz1>Foet$r#55Y279aL0t;CGSeDRN!;euA4hry$Lg43wVy==r2)XC~6 ziBneLBpwn2JE!Mwim8(fs5~;mjQbioY+=a6R~r@CAhwi5A(+5S+-27WT<)wZ2ZGVS zRYG6~(jnRbcd}SFihI$#)}Q*8kR2V7>cXDyq#Cwntx_#Y1lgE3DF$-LL4&V)Gt z1J{#Gj|dp$*J&vx7z(09O=rktu63^tETBTr89FKo!Q;>&9jY>;oV}SGK7NA$iKwI1 z7vUJFU8(xW2KwF&CkUF~(uW>3;JJrJqL=w8hz@8NIx!!@M;v0a7guzPuL&5r)2uq| zn#iXJbN)aIdra+}ks|$2#zkah;)r4Rr{X73>hrM2Sdryzqm`%wxj9ERZ~CMNU8${x zqr+;SG-Cx7hin$y66bhS4`}g>Vh8(K+HM1WEdaTOZ~c6G~o_H7+-w3X&-+ zyVoHS4A@w+gTI89%j_{S;y)mW@U5fI7wIWaL|fp{pwlk+5?!N{BpRGPjvJ#jK*en& zZ`N9154B1k6zZh+)*_D-rLv?P(uoD0eNR{X#q$juF1=?lj7ic9cniR+Y*LfTp>v21 znUauDGoIasKGL#D-dlX~gJSb|!RKU;O zCyyg#GYPB?yaH0(IWq)T(xIi(lA=`Bxa%2{Nic`GzH@#^ojAN3NSx654AAw^$@%(D z_tw;M`_YHl6)af773hctzpf7nqmD$74Ul)XJOI8Dx&Ci{p&r-z2^A~=000JJOGiWi z{{a60|De66lK=n!32;bRa{vG?BLDy{BLR4&KXw2B00(qQO+^Ri1rQD&8fD6Zn*aa~ zMM*?KRCwC$oqLoWMV-e#eKTZY9!Q+`KE(y9T2&`w}LhBwB zLI)Jm9-$Hu_dxByjQG!uqS0E5cm?pb`s~FxPnP!y`!9yerc>sCM-1d*Ik;zS0 z-|l<6XC_R4&*?vIS69{VcJ;R&zhC`+WuQzux7MQ|B4o z)dX1$cwt#%bxoQ&&*%xMs}b2vA@>Ik1r7pQ)$I+ypMYmQ*V!8RH%jHodBCkP>)Q}LZ%O(>sFphR8`ELZRH z&oSzH7MPTgoc#+bXW_d(*axUBFUx=>$GLMUrQUF`D~(C3vSs}Ws|0UWHZQ-OJ=kUs%td#=;f$p4DK2-P`E z0?ze))|x`j@?7Uv4Sr8!Vvzh|U~bIWZ;Se#+I^T~3R(1AXL%!Q^dHz(@r+uM$Ly^? zNn<9op`s%>7kJVX@`&9??wNJcXAFQ$!+?KN@AB}B>pBp)+Z1yCq^a|avVx(+)d=Iw?r6k3VGN7$@UNGU4Ez^>biJa zclCQIll%(cf)qNvTTCH87nkVUfUoM51E!FZO(FLSOP?U*Z~_zBn4HJ4urI;x#6b95L_ETM#nL>(`-mM9ICndQI;5JjpJ>wAlZrdvUxrV^4rjQGiQt9`= zUrZryogidu-{(4AXS-53<%kbq3ORL<$nsW$)An9`et_lwd_OLSu1Tq!D}X)fJoCEJ ze>?$n#TYy;h(q*yYj5~)9f1Ri(WmD)&xh~F$v;GJnE}IVu2$eMU>*>qu{|$~r9p^1 zkypVmbQ<^5V%9l1rRnoU2DBP5qW4m3*MC$5>rQD_@g}fm?Ry|la!vTBIQYp3HroUeKI!U;moRwH>Y zRt{03zY%8QATR~ZQKqS%Y4T1To)6Qwr~Ld&vD~fM-l0 zHvrda@_sZyJ@0s~vpMAc24Jx=y*{Qo7+t%qZe}(Iyib$12@tjBLuHR&YzjFIc#?PR z>wPiN6tb<}h`zaIH2;vMFo0HnR~OeU`D}bwc&>A2Z713Xdn-Z>MocH!Zp!|czW+EP zZT}S8>>c2{5syr2vWjCP(r%90BjCBt3&5g?wYpia(RQ*Q7Yt(!fnRtSx(bF-JB7#< zaeDk;e|(@RLuWSAeG^kmA;)GZPzw7GlHi$$wD&hVVTwq5AV&VjBGL}5NA#VhDXTl? zR3j7*Ql;;-cFFmyMbwtXVS`OrQ^@xL(;`OhwVMC&*^K2nI3oEk{n%!EG+FkCBhsFa zk^e5WR0_Km*IT2XUOP4NT<2O-$SuI-zyZK3z;`{@`B6yznK_@eh|&me8iewGMze<8 zWEec=`z#^c{yP)6Gefz)5s{X5%!2U|X&ag>TemyVk27mg`h{QX((4iZbER@+qHPtQ zuPO9K;Krzohquo)x~qOQ=X2;NbR2NJZq(Rgn6YLMimmIkJ32d@9#cpi(LXam$ktBR z+14nzOd)#-)}uz%;RO1kbI@-t@qu5Y{0=UTNNcvuTC!}sLX7m=O6AIkqy;OD;AesV z9fDjQsUi4iQ^>CX_w%lKvKEi7T$ZkvTTj&=*-ZXJr^rIY6#O^6ga^Vp_EElm${z<; z0iL@4NMK5ONqtvQJILO8{~a-u2ZFNqF#^5|gxAmj8R#{dgU z<;q#MRot3TQS!DaEhi2kqTgT&S-UmTAfEHa!VKhDYYMqJ>Lc}B=WJbK)kgq2 zOTZG<)*DSBAJeq+4B)PT$vp8P^nyL4I?{&chRH3Z-H6BCkZx>91VO4xC40SJw)^Z!KRMBR~(YRXk@dV z{Q$upaNSkU2}EZmd(m^9zEf762jld^+UFMf@%>NtTxVkg`9_4aTO1|Zce0K(&>tgu zFOC9!Un*CYtKRu(r8L@$cm2^vmC6-iTgAWDlKs#r@}$)rPF15YS!4=%Iq(DEUxw(? zeJgN|W^>3uLuXcdFk~8kb;xsj_f-8834Ese1NWOkp5nRAEB#Q{y`JkV*d0cmq&NvE z0m}&Xx<5o%oOrt=uJNl z_5CrdU_1e=PdF>c0!ILgYyB@Wz*2&s5WE+O9_J-LTq;)@3gGCujszAAA(Hflq}za_kIXQUAX z8&JaHE$pc}k*iZ|Bb^tXG1!UzUzc8ZzF$M~nFJd4UJM5=R3rBnYx3W>8xwt110z>v z7*;TrH|V`qZ@QiIoW2j`(i2Dro&@`eVr`(b!H zLQk@<=Q``^=u-A3+%4A!evzfgBkV)JT=VP~O6AHSHQiW#0kp7L@|t8a*r{|w+qAy*lgmL<`{HfTiIYsc42%eLn{8#tIk?gN~V zLD5+Rjy$r#P9PGV>ugd)(P36^sHPMU<;x!D2(ptR(q7Jz=wSgNqqZczw3DJ`ifqa0 zHsGVVkwLkqrCFFM4qSFqaSo~ z^U>?uR(P(nF-xM)xc-(~XV1FeU4@)&6@OYPSH==(E8nkrn?-~>C!CgZZNR4Oqk)q( z$xnN(b5x8nW)Z{}tVKahMQ@KO&Rlghf+d~lSU9c&iiGbw8B1`Q%J-V=P_~*vE{y0< zJ_s!GeYTrIz62Ef;CQBbTXcT{*&VV7FtS?pcd>)pR`EuH(@-Y?ZMIcB)3%D6Yd(@O zfR$5jz;FuskYV@7EMsv$uhYZu{WJbcIAtcc!A@>LB^8^5WhNjU}pN(p+3CV5lANaT0DM00-x zLByJ@aHnB#Nty*J8+%8cQGHYj_53h}naEb)%bx2z(s#X-CjJVv+g9;~1_<7!4jjBD zr`y26_i0D3-0DcU?K5h7Zx&(3UI#4Cb9*5GC3bTM~=Y?Q^*kn!CLksuvqmd-n|-UFMk>% zx|ZaVrp`0Abysg8n`4G@jo>#p96{s^ZWIdb$9S_t2h z9tUqIlJDZYqC!32N^mfJ7Fd@zU5*^dM)YM3kiD1W&kO*`tH3>Z)8)vKfatbW>>@Z? zHSU3kUBFb^Dn6|tc_m|#e_b6amm^1xnE1QlrBtpQNT9~A7wc`2Z52b60;Z53NAE?` lEH)6BPgds5mLo?p{0+!%eDdfXNrM0Y002ovPDHLkV1nx~Pe}j( literal 0 HcmV?d00001 diff --git a/nautobot_ssot/static/nautobot_ssot_ipfabric/ipfabric_logo.png b/nautobot_ssot/static/nautobot_ssot_ipfabric/ipfabric_logo.png new file mode 100644 index 0000000000000000000000000000000000000000..7c83b27aae4c88186843151afa1a8199bfa1e934 GIT binary patch literal 11662 zcmZWuWmFtZvtHa63-0bNVQ~#C1b26Lcd|gR;ENO7A-HSUgy0qk?gR}UoM7L+_wPM( zdZerRR84jFQ}xWmX{agUV3K130011YlDroDI`UsbM}?p34C)I201Ck%9RnXNOMhAq zFLygf7h77NKo475+W<#903cvx{>rZ;%d3jUV99KQJ=GFcb$0fY^P_aVu;r6)AF!(xjw;{Oi zK(gLze;jvX6ZG|K@a7eH0M7Jhnma6V){uYFF9E)redn|4=P}8v_dEhM=i%b!g&Tzf zIFAqN&wuaobaAI$e0)#%S|lHCG|m;fHa+Pge|Sh`KCnNjhM1q}?|fNttMr`V7CYT? z?%waky6rs7JU;6THLv{qp6}l&R>Oum%~`O}o6RR|;Z8i&Si0^V6$i^G@{sJX7+BAX zHTT&@$NH+Oe%D_wlRs3EAW6^uiO0_$?v7W(r*e$whkx}iehLh1;1#K}`q^)yAiWlN zpxKS}^z!ckH{3R5MAvA449QQdfbN}_lov-f#UU&zZt*^(vvv`{iGQc|eh7u$q6KTz z);|7j5VX)lP&hRa$~%KAyQOs*Z}^W?mRV%}cq62uGq_IuCGax=Js#Vx2D&O%zQJHN zeVd*YQ%n#hTSdATdv!%-5D>JV?#*r3wxq(^R0dUHYipf#u4<~tb*^q}Us1i{()d!n zr)T6^cQd6b#^5ix?tGBAtw`!wma0hR+d9MYM_}i@z%>7{OUvgyeFM+(41Gi2w)u|L zve_iD)86s=p3^X;HbOP=KfBs$5~tV6RqIu`JbaHrULTTjgB21XITS4;3g=aS{n*aw z^0pY+?BpjD6@SFjOUniiPUw-8>hC64*?q2LhuUX?zZt6GGemfb6(&8#t|yN?6TFpM ze|6v4mr|T!^?8jOA&WZuJn~H5;a<}I@m$;20YNyUn0Ov_>^wbRK>hGQrz|2lQAWk8 z*sgZMvBJ#CWgt99GHcTMOncq1;^W92T@yQKk8qX%fqy=>*K_crKD83U~^2#y<8mI+&7O1cwYyw@s9QzZ65;vZPH z_j&P!yeeDx-+a8!Yu1~>x>ofLG<~67T%lnYrr8N4pPZNvZ7aPg8y{VpV5i=Wq53(n z)brw308opSyJ{K_$;W;DI%KtNZmo$OoSfp^@iKlD8Yt==)6)`|Y=5e*et#VDyWxxN z8N|Hce3eXhar#V5kAmR*U%;hNwQo@#B9rQbCi?KTxMMNuDCXCQNj^$~fhez&KT|Qf zY8>%SBkye*1qI4NSi=PoN7r#O_T-$KCiZ1N2X{#TZQts>_Xw@%4J;R)RSqr4);n}O zX)?BLvj$FP0a?gN?I&%GMe(J{;*6H#`OPd3nuc7ydmb1Adq_H*$iC<(W}xyHaER{p zhL}`jDY$9B25^sPg1C zvP7zSNtL8NIiH2N}mEx`oikJH_*^`#Df=(x|zkd5ct+%tlwM?*2>L0W<930^v zv|lkNFyLpBqeK`XpD|P;O1m4{E#qN#u(_Si4f`=|?Jj^Xb1JP`8$!l?BLRCmTYKMl zYQz`OdV|bAbH|S^RP=k6y6-I9phTQg)1zY9Kw0W1b}{Gb8R9(@PlA6>$Wkie7?`6W z-GfYgnoF7&(N@y}#8tzYi`)tORlg2+Y0FLNtwZ(g`kr?@f+v-uHs}H7sIELUgzdBE zcnUkJ89`FEC=tpQME&XFM;H5Va*1kNwfJfBH?ybn{yp}rJA&x_2njG( z{xcUy9v~KmI*%=hA&YiCW1GM@wJww=@w5~PVnQzS(a0>$Zz(}}GXg2!;b0_W!?D!0>$ zOpNI!@`NuMN0`q<{PF~Lf77N2r_9Ye+IS&*Z8FFyzF~`>Es`uO`6E}=e$=uNOi%Lf zdE@2Zm^a6Q*rfc?DIq?FM_%HH^REb=pafC?O2;!u;hq@Gu=@url%8hW5_wJtJ|P@`w>XWO*+t&y(&ALsn|(VfgnlS^mLN&%5h@ zH`v>>oNj5aacgzXbz6D}ZxmGsPZXqZ`tA5)Krmg_aM)zF(%?940yB~7rL9!9lfSFL z0nl1kP*syCfv8N7*O0Ienz$YZbEodK#HZ9ytGJwi5o)<4fY5AY5iOi69I_~_)d~Q5 z3+sB}n_~k-Ik5aRq;8^|=0u9w4h=%=x22kU;mvQ&1BWB>mt*;&u0BFuZ(+p0eD6{HDSeH3z3B4TBZ!Su^ z;z&%dX6JlyqweC)o^GFlHuJ-LMso#RNR-V|uo1y?XtBzH9yyWb5`RDiwTjAFNbL_= zEdJamgUwlS2&}G*D2{iix+US{W66@1B^g3&fZx)F0Y$cQgMpHV@}W8?D8z9_4fhB7 zy9e$1qJlAu6Z$J7q7#_0Z}H}zsI|VRTiK_Ro$Qs*q$SM>d)43RZf6_qZNE6Jlo7eN}Sb8NKK%4zTiwhMvpMDv7;!0p6X zk*&L--vN{R!Nu)a*uFqF6`ctTA^;_ zKk+sxvGq%)Wii9HrKS-mx(wry8XM@_^mKZjP6qlDbCgtcI}8t5y!DubXC&sGBuAK0 zwFd$rw~PSDY(k7uL5Qq|wRMsyV;n%a8!&KYpzI1Y0|LfR@{V?{skvh=N z85JGGGJ%o){czVmZTqhUxfhBP%HCsF4Jk1Mbg&SKaD~f{^D5o@KsfZ8a8vb69CgJ-@N_U#FGc$v) z-hyh-ZcBH;&g~;{M7vO1`~phTJ*ldrX^7Uq_bpK+VL@*`x)fj$QGs_XUvv3_&+%wm zwN?@#$#@-Gb1`Yj2P8v(G1$labcTFlq(nxs{MGjp7-Yy*ebihE>Zhe*ynLKUqrKl# zk_MwSYqX}=*$TuI>^<0>J{I$(5X5r(pi^#t6JQ3mmSOa3I|SNXkae}G6d#R`1QSQY z4ys;Al>|-?8@S}|CTv)UQCj6cS%6y_WEOnTVIE`D=S)TfV|!{fva@a!Za=AY6Sl16 zB`0jv3vH_6_?p7=Efu7s(;8h050DHjU??d^qhwmjO2lR>Z>@1(+ZF$D4o{L?WpVaPeDmT3WQg4u5V*%ClP83DbHILvY> z;b`1RYuky^>uqanfWt)SHMoRYWJITzknLkAlO?hY1z$1)nGm>QXG+Q2&8;{PV}!Vy zo~R@@QPT5j4N`4^SR`*YF8t`Cx%vLhj{&sbBc=*Bl#~Jw{F>8CyBR6Wkv0rn^GS&P z7;}b0C3Efz0ST@DjLbMc##l{q=0zeF11(+NKgm9qlVJdU_0Et}HwfXTTl^gOO)dY{ z&XVq6#HUQsf%FYgYy`?M1ZWBP%`I|sYtJA=tMPq4o{?CgM+}Uq4n9^=@GU@4{KZOY z%#$fgBx&|J0!6zJH!ec(Ivo*+IThoV8z%y)aQ?<9l&T&-nOaA5KR&nijfC(!^4n51 zDnDU=UJ86z!VK}0wxh?CAg8ABGS|)bpElNXcxuU23d(MD>TYS3W2%~;+12alr#)I5 zvc!QD7t5=h@>g~FHpk}8uvfah&9dC+NeuOve@(tT5~T5)RKULzu@*dQqP}e>yq|~& z$;WFV<=Mc$QTxmH2{DmvtjuJn4cxFg(?ALukMeVCvkFdj20CQYK(;~!gc5T4w^Qkl zoUFxr;@JF+1Jv>Rf#!QqKDoz^g!N7|yG!sXu zAQpRFd-Q=;{jckn}N6g=fk(Y zZ-c(8^h5tlrb~zgMkdcnP@;sgkKJ!EZ9619$fQ{838>*2gBj;qTh+KWAZy3y8Ec?%00MC}zyj?@-NtTLdqUTk5Br1*%?kig#D$tQ2u+de^oSCh}7 zJd+dliZ6j)=y@SU1sys@&OBuq-rNICYs2z@gHL!DJpo-Kmq^r#G=I#1AO} zHd#HX`1Z9T0ZX;ANYxw{0mgYhaZ)!Eotqd8CnM_T2iiO1W>-vuah=3KX%uHK_;m%k zYGnqy64knT?%y(WfC9~j@29o2K;N*K!g)pd@jzWVC;uuTzBkWK8|f-@Y7A+}pVI?- zA|mU%jav;ygZo6o(4A*n&ujuLh&3NO+?U!hhfP-@3Gq7ajo?a=iC{sQM>XRTN+tlHq%=#L$E>L|Kni_33u9Tl zFkd7wKeDq+)H2JT2nXCs6=)^O`(!G!4PF&xi0ZUj&>#)1s`c*vBU5d@|ELeIIGv;? zC421}dz^WEyJ-cvv%_0nb0bX|#e-t~FN;lRQXJq#|5oW^_^mv zTjamu+RQ_753yRS7vfhcqav79bR}#P!UsP2NxkijkewqFg=nhrHqAE3;S^x&7~jx0 zYe8+0)*=+43Q&vJXo6HczE=z;=$YE&Tjkw5y9(?>Gh&ZC{Fn+(jql8al)2U>26QC`! z*=oe1=`fb>@ACP&`~{esQH_+4Ol`i=U^;YEMCHZ4-NX-!o%qN(@FRjQ8zU@2D8gJE zOHTOImp80j>7Iuf{4yMjc(QC?Y@><&vKMFL?ORaoFpYbxp%*^WO^Qb8{w)APSO;mK zZ%m_9TjI)(J37kzb5V)?QdZ*ErmFRo_D|(UQMEBAV{vVXx!hP8p-(k6D|c%h(`vT- z9%xl@{GbDN!d0in=H55=!TQKGA;Mx4h5lkMFLp5xS8l^JZ^gcze9xA0n-<|eO2K)f zM|rem+|p1u$@wj2Eq7PiC)CcYsb=_VGR+cyRa@N*=N748O;iDNn=}1#Vh$9Ega*BBMEovX(iy$77^*KW}pYIT|EnZtoqF=M21ex)h^VPue%+_Z-pVq_m z&oX6CQe&n5FDF$`+kS~w%5@wce$O^l{y-7G0e0e;{|#uNAeiVQd@me)+^NZwz9wws z+HE*%<@ifz8u<2WSb>>Te_RyR+mM$Gp}#wT(kv?*%Sw;JGHYz1Ba68dw=anqFlPb9 zR&K2D2yZ)^35FGXX|4ho_Zh$shm{QFk5g4J?}tmXI-+wv!IhQKr5l`o^K=w3TLOdi zGC_!!qG!<-T(PSq8$_KyvOC2BTCj$)QTb4Q5h`D{uZ8~$y?#9Up4IiY zeyjYJ&SZAbSZp!OK}!9A6DrJ<(aQ1mk5@)5vKhyhI9dr0Dwr<;`McVUw^@ODk zHL4Ve0MP+H)PruzT{F(sE?e_QERkXh(-OvZTe*|nJjldBE$~hR~0$zb!C+?$&{}S$+(Ih|qfNV(fQ){rg1%0(Z!ipHgaeXH1;-3c8?Q zwqFGP{G3aMJ_SE6?-yq5`)4eB(2DX+wWbLt_&VWnIfi~IN=YzVwbjKW?>KZ0xB1It^xEmr9?4Ck?Tp=hYq*d_6Me{-l4ngKvYdO8 zWpC0vA_|3S^@-Qhx#G|1`cVWR(lgO9Yg11-J!tgm~!OnS3#vyO)^77L8~VkYZ0MoNqC= zPdN=Nl)1eDwV6&m&iNVMnFv%2A!e)O&(hHAv6Rw~VNFjN|GpKfhnQ$3Wc@bJYt`D-UORg*sjeqS&O5w5}y9F16jN z;`AA|IjYuJ0Ut~nMszY4%Qcrpp9oH-4hH>;*~6@fsVX_Vl_=f_sG;; zj{m|f?I&h!4**>zoiS&lvz-=cmHyhAN!XIG&$aXh6s2yhK=k(m@kJe~E8>JRhkfBW4T(0P+|;ukZ%UCx1SP4>kzYo?{^DhD|RPyK-rjN1yY# zNE;6~#^p+eJ+7Y{_qqM}-cN#rqEV#*Ztzk!FjonS-+LC{KR^o_+wRI&-s>t>juI1X zGxCV4i>s01*%s{oP_>En*s~`}2R&OkKRwKz5jdaC3y6>Q(}$uRuqI zPEe&>nf}jcymU-=O%7rCZfBhH^TB>f&WHBwNg=(2fY4w6!lf6!Fw<~lnlKA0B(l=8bUz1lpnkk%?GTifOdd_ zhl7I6;OnIX0N}5JyG_#Q zCaV5Q#98zH;(7SJS)x?a-5AgjXJ#N$zZS z_5Nj4R`_GM7`ttMd4+@$A1e+DGZGs3)7%)J+f(*T@$P+)A>fDtAy45}bjUe?6NmB-BPHm>=VDgc;v_WZs1`APv+c8DNHVXx*A0*P`Wcf*6Mv z-;eZMIloB4egK10tTa{#1Aw6iDU|u+sVHBO{c6&mJ67*#E0utwsh%KOBVk`HMY-Bk zr+6sCj#sPKoReH_!1gx%4J}m83vdy|=dv(+(%^C`gQ@}?r32|Tfk~UKSLqfOoYLS5 zwL`+LC^KQeQ5kOYPROBH4CL!6%OCnxulLB9uT0=`Poxp(?pEWHV08ta$fZmD!7FK1>Vune)Gx2pZr7Bz;fDZny=pYa=}?#k zJh*d_wWE8mz=C6)mM2&=rT8ePIhh)Ah>$r3;*4>Pd0@S_!UByR=ddjdS;!P)mVFph zaBRW_59U6tvsc_MgbqzTlajy&$4y}9{X{Qi%zNQ+eLbcaWW&Ttgtyg96pof1a?GFi zIN;$hga?37V$m?y7nBJGV$32~ODqPO{x>wzDcrM{OwU2nzLGVIgQ!WY*O+p*XfFkaAwyIU zRR|Fb4dx6pu}~{N;i>FlI9yGlLWJ_qK3^Z*!?Jco7LJy}E0-rCBer3>h%H~rLP9#$h&DgCjLJ2y zEMGRq5U=0Qag}+Yv}djLJ#S*!nybbR4d>t*eCf-zZ%1j$$obLW-IX-5hxKR_`xxxdm-QFe`FUc7~C1Q84V!7yhYVHkUYBxwHn8!0hDVRf9HJ|2a0-X%uH z-e5RP(98)Y{YPX&EQ)!N4T7z4 zi@*(~V5+zwf*(Wds|SRf(=EhSZq>}WrZO&!tO1n zZXcUA4$051J^eX%>O&2kF~X(>;bc8+&w{8$`AI+OM71|ambMYk-(P6j1*^HTyff|h zJ7XR@{jrx{I66`#nnC};*$-KsOOcCJ(F{wr3l)_nT%D_lcG^COWu{efxnz#aR9*V> zLmQBsf3?nDQpr#} zy|tTAw7EfT*3O(D6FRI2Zs}JEV=}@fnv)Q=5klT(%V^#P910sfr~h6YAq%Nu%#N`M zN5Xv*FR6{qYO(xceSkQ?Zu>eLh-+$b%4JCEMLKrLEwkd`#K(Hm#yM&ZPXma&=$+B^ z?(#+$f=I){WG&|9CsNum|3u^_`6bHedLA=9c`hkdN#Gcb0$WD9io&3y1Ux}lKP+^_ zCP`Hx#zXYu7W5{o)1Up1bn)Tq9o-{4w;b7V`lFRb_n>!vH>bWk+v9i~bU^~zeDGJh zRS^#Dz0>KD zst|zbWkf5`JdF8RUD?5mamatNDLrV>LJ!v&u9U@Az1$$`#v`p{epQX_y#@((YRua2?g;|FS8@j2$oB zl3o)&ZyjkmHwS8Y0o;qZ_;*e&s3Igw9i-`Ma1=hRRe_F^Br%CVrGjQX0d zbjjv{%eziW-PWWklEs4!NJ3C!TCx6RDtSR|#`f4kVTu=9-dhy%iKyx(@_Cp zLG@QUn2Zks9z-7YaMY{ck-(-??()H$34cexsKA*#?ooFjI=!HoAOp*RIsRRG3Yhk_ zV`3dG9=ecL@j-Xcl34MB1;R+l?-J77Sd@$l(9V$I*w+G!cw>wt_348)^^%)GlOPzZ zC49Y#(n&6&6;YOk7XT{Alf)25;k-7f98dej;nq9)(Q-+uAb}b{xe^su&1h(vP8fdd zPjqzAco(!u@)x5TnM$9v|OZ;cc-Ixm%T{*0MNBkJ{Moc+$V7)XO<_qhR#VfdiF|?L$^!cv`M>?%6)Q5BQ z6vjj#-Nh{|GIsLEL|9uEAtf)1na(+YWtFgVvHeg(4@Yt%k@) zl$El~q&lZEw#0pHSHWvq$+PKZG#o@O3FRb^ueD+TH`kB^CLeZ+W6+DH+kDVPS;TY2 zSIa91BCfUqT`2c)6Z7nCB8gBg#k2BU>Us9uzX>3`XSg8c9rn)WIhuxdm@gR8Q!2!Rhq~Cxi%55b zuE@sbK?wcG`^1s6^_j=Je(=C`t(H)RhCjwmSv1VQE0=-o3>LK();$VG$3SoyW6^gu zPlR_5Wm@z9p<``7BH_YEx(T02OzqqhA>)5&P7P(+r{8jjzxxW}Z6Qq;Zh7;e9T{J5 z>WLM#yjb2n!lgw(qxH?BCI)>$tuO38Vfah6dC8w&@iNu>alApoIUTXh(@H7WS0R%u zLY4t|n$!(J?d7?5dRNrlw8&;q+GSfdP40Cg0p{>#N>gm*RijuA8U6{`Y2UB4f7h(7 zq2RmaG}d3ib%CZ4Hd*}!v7k3gBE^^39e7u!7*NxV7QCxqjLhMoHmMZa<-wu-B6lW( zg8n9tq+KhxIMX1u%MqTM6i)TGxKrT@?+6ac2x&gzXebht$u#oiyl{TIqXghNib~K$snMs4tcbZ z_Am9{-0iL@E$FCk53<;!kssVjImygfuaUPkqHWGu*VHia?=xnZX4T1l$>91c^a~HF z@kYvQt>dD|cW;7}@cxFXcutpuv4}i>IS_4F5Ool;LHFCZ2B)CX)3|%o`0`vTZkZo> zys9ImaSl=!>_(>2{;{a!#jY#+$v0$zu;%o4V-t{cWiq@Fp2xrx3uuIP2EnNZ-3$mK z?Df76G2y~wcr0_t?7bXWnj3J97BBBU__E-g|M_?EXCZj?bywv+8wb?%yYul3;bYBr=ZiC04IJgP(Boqb zB^1G1LR&&uqIV-H#CJst!w6U!;#h0&2I1hs_hr19yt`|~LLE`}l%o8&+?l!x6C6}! zU>J^_Bsfj8gMwR|Iymo@$9dc_pxhwJY2O;d`i~}Gc_~#IdYNK(9UM^ivHEtl%TsA( zy;>QNY3|)}&%hB%Nv+s*FF+9<*E=Q|Mn&@7rRc+tb<_m%fHhi>-slWkCyAJGGLsS# z|2(OHo)-+}ATUP-Yi!*pMXv6r+!;zH%MS#hCbC|m%ekQ@sb7U2sdVoul^h%z;^d&_ zL2rXIXrZMW-brDWBkS_FDyF7ZuFGmbqhX(ddIaIV8BiH|F%#$uGyqg$E@CkFx`xIS)p7o3%<5iHUO zmi*Mg0SRO0D2*-Xg7HQ@i9LuKIkR}yP)y9(9(m^etZV|*>Z22oQ2n#e*kzoC=NLEj z+ENGbsKSLYNi)Vt_7VLN1mIS+Cy z;Uv}o^~n!MMtm^`OFS$VG#n{};sDbV^gp9D5+D zPCzxVn!oo`QF?}q5MH-fRz|eGFm18H&F~Lh7f8JS`GPA(iAEARyC6-r1b03On|Bq) z$7TD$@$q$L+lX`8cLOb&(00q&#waV9ku4@Ox$JWf&QlcwNR)^%a~G+k_e($XSnFb} zM2y+qrzr#MrxJVItCq~>{@RxQx$+ozQPzWmQsTWwZi#Y1G?Z-)2F@U;gd2y0;VbEO z@p;cT2m;RsSRV8qjE!iTFM4p#W~VCUhx^yjo`T^O$kCCM$$5+cwsIfD9SI4Ns6E@l zDKI1sO{e_p2^e5fWH*pWk9|C49xyhYwBq?_fc8R@Giy2@QSgKW|MCL>E2zoW%UVSI E53***7XSbN literal 0 HcmV?d00001 diff --git a/nautobot_ssot/tests/ipfabric/__init__.py b/nautobot_ssot/tests/ipfabric/__init__.py new file mode 100644 index 000000000..033383d79 --- /dev/null +++ b/nautobot_ssot/tests/ipfabric/__init__.py @@ -0,0 +1 @@ +"""Unit tests for nautobot_ssot.integrations.ipfabric plugin.""" diff --git a/nautobot_ssot/tests/ipfabric/fixtures/__init__.py b/nautobot_ssot/tests/ipfabric/fixtures/__init__.py new file mode 100644 index 000000000..13092a6d2 --- /dev/null +++ b/nautobot_ssot/tests/ipfabric/fixtures/__init__.py @@ -0,0 +1,17 @@ +"""Fixtures. + +In your test file, simply import: +``` +from nautobot_ssot.integrations.ipfabric.utilities import json_fixture +from nautobot_ssot.integrations.ipfabric.tests.fixtures import real_path +``` +Then you can simply load fixtures that you have added to the fixtures directory +and assign them to your mocks by using the json_fixture utility: + +json_fixture(f"{FIXTURES}/get_projects.json") + +This will return a loaded json object. +""" +import os + +real_path = os.path.dirname(os.path.realpath(__file__)) diff --git a/nautobot_ssot/tests/ipfabric/fixtures/get_device_inventory.json b/nautobot_ssot/tests/ipfabric/fixtures/get_device_inventory.json new file mode 100644 index 000000000..64da0491a --- /dev/null +++ b/nautobot_ssot/tests/ipfabric/fixtures/get_device_inventory.json @@ -0,0 +1,68 @@ +[ + { + "hostname": "nyc-spine-02", + "siteName": "NYC-SPINE-02", + "vendor": "arista", + "platform": "veos", + "model": "CVX", + "memoryUtilization": 40.11, + "version": "4.22.4M", + "sn": "5254.00d3.a91d", + "loginIp": "172.18.0.22" + }, + { + "hostname": "jcy-spine-01.infra.ntc.com", + "siteName": "JCY-SPINE-01.INFRA.NTC.COM_1", + "vendor": "cisco", + "platform": "nx9000", + "model": "N9K-C9300v", + "memoryUtilization": 58.22, + "version": "9.3(3)", + "sn": "a000a04", + "loginIp": "172.18.0.9" + }, + { + "hostname": "jcy-rtr-01", + "siteName": "JCY-RTR-01_1", + "vendor": "cisco", + "platform": "csr1000", + "model": "CSR1000V", + "memoryUtilization": 11.97, + "version": "17.1.1", + "sn": "a000a01", + "loginIp": "172.18.0.3" + }, + { + "hostname": "nyc-rtr-01", + "siteName": "NYC-RTR-01", + "vendor": "juniper", + "platform": "vmx", + "model": "", + "memoryUtilization": 15, + "version": "18.2R1.9", + "sn": "VM60D5EE2211", + "loginIp": "172.18.0.14" + }, + { + "hostname": "nyc-leaf-01", + "siteName": "NYC-LEAF-01", + "vendor": "arista", + "platform": "veos", + "model": "CVX", + "memoryUtilization": 39.6, + "version": "4.22.4M", + "sn": "5254.0029.fbf2", + "loginIp": "172.18.0.25" + }, + { + "hostname": "jcy-rtr-02", + "siteName": "JCY-RTR-02_1", + "vendor": "cisco", + "platform": "csr1000", + "model": "CSR1000V", + "memoryUtilization": 11.97, + "version": "17.1.1", + "sn": "a000a02", + "loginIp": "172.18.0.5" + } +] \ No newline at end of file diff --git a/nautobot_ssot/tests/ipfabric/fixtures/get_interface_inventory.json b/nautobot_ssot/tests/ipfabric/fixtures/get_interface_inventory.json new file mode 100644 index 000000000..44fae79e7 --- /dev/null +++ b/nautobot_ssot/tests/ipfabric/fixtures/get_interface_inventory.json @@ -0,0 +1,50 @@ +[ + { + "id":"19941192", + "hostname":"nyc-rtr-01", + "intName":"ipip", + "dscr":null, + "mac":null, + "duplex":null, + "speed":"unlimited", + "media":null, + "mtu":null, + "primaryIp":null + }, + { + "id":"19952051", + "hostname":"nyc-leaf-01", + "intName":"Et15", + "dscr":null, + "mac":"5254.009d.f816", + "duplex":"full", + "speed":"unconfigured", + "media":null, + "mtu":9214, + "primaryIp":null + }, + { + "id":"19957897", + "hostname":"jcy-rtr-02", + "intName":"Gi4", + "dscr":null, + "mac":"5254.0090.4b0a", + "duplex":"full", + "speed":1000000000, + "media":"Virtual", + "mtu":1500, + "primaryIp":"10.10.0.10" + }, + { + "id":"19941192", + "hostname":"nyc-rtr-01", + "intName":"eth1", + "dscr":null, + "mac":null, + "duplex":null, + "speed":"unlimited", + "media":null, + "mtu":null, + "primaryIp":null + } +] \ No newline at end of file diff --git a/nautobot_ssot/tests/ipfabric/fixtures/get_sites.json b/nautobot_ssot/tests/ipfabric/fixtures/get_sites.json new file mode 100644 index 000000000..dffb0c3fc --- /dev/null +++ b/nautobot_ssot/tests/ipfabric/fixtures/get_sites.json @@ -0,0 +1,80 @@ +[ + { + "id": "55467784", + "siteName": "JCY-RTR-01_1", + "siteKey": "55467784", + "devicesCount": 1, + "usersCount": 0, + "stpDCount": 0, + "switchesCount": 0, + "vlanCount": 0, + "rDCount": 1, + "routersCount": 1, + "networksCount": 5 + }, + { + "id": "55467890", + "siteName": "JCY-RTR-02_1", + "siteKey": "55467890", + "devicesCount": 1, + "usersCount": 0, + "stpDCount": 0, + "switchesCount": 0, + "vlanCount": 0, + "rDCount": 1, + "routersCount": 1, + "networksCount": 5 + }, + { + "id": "55467964", + "siteName": "JCY-SPINE-01.INFRA.NTC.COM_1", + "siteKey": "55467964", + "devicesCount": 1, + "usersCount": 3, + "stpDCount": 0, + "switchesCount": 1, + "vlanCount": 11, + "rDCount": 1, + "routersCount": 1, + "networksCount": 7 + }, + { + "id": "55467605", + "siteName": "NYC-LEAF-01", + "siteKey": "55467605", + "devicesCount": 1, + "usersCount": 0, + "stpDCount": 0, + "switchesCount": 1, + "vlanCount": 1, + "rDCount": 1, + "routersCount": 1, + "networksCount": 4 + }, + { + "id": "55468036", + "siteName": "NYC-RTR-01", + "siteKey": "55468036", + "devicesCount": 1, + "usersCount": 0, + "stpDCount": 0, + "switchesCount": 0, + "vlanCount": 0, + "rDCount": 1, + "routersCount": 1, + "networksCount": 7 + }, + { + "id": "55467678", + "siteName": "NYC-SPINE-02", + "siteKey": "55467678", + "devicesCount": 1, + "usersCount": 0, + "stpDCount": 0, + "switchesCount": 1, + "vlanCount": 1, + "rDCount": 1, + "routersCount": 1, + "networksCount": 5 + } +] \ No newline at end of file diff --git a/nautobot_ssot/tests/ipfabric/fixtures/get_vlans.json b/nautobot_ssot/tests/ipfabric/fixtures/get_vlans.json new file mode 100644 index 000000000..806fdca6e --- /dev/null +++ b/nautobot_ssot/tests/ipfabric/fixtures/get_vlans.json @@ -0,0 +1,80 @@ +[ + { + "siteName":"NYC-LEAF-01", + "vlanName":"default", + "vlanId":1, + "dscr":"None" + }, + { + "siteName":"NYC-SPINE-02", + "vlanName":"default", + "vlanId":1, + "dscr":"None" + }, + { + "siteName":"JCY-SPINE-01.INFRA.NTC.COM_1", + "vlanName":"default", + "vlanId":1, + "dscr":"None" + }, + { + "siteName":"JCY-SPINE-01.INFRA.NTC.COM_1", + "vlanName":"autovlan", + "vlanId":101, + "dscr":"None" + }, + { + "siteName":"JCY-SPINE-01.INFRA.NTC.COM_1", + "vlanName":"vlan109", + "vlanId":109, + "dscr":"None" + }, + { + "siteName":"JCY-SPINE-01.INFRA.NTC.COM_1", + "vlanName":"webvlan0001", + "vlanId":194, + "dscr":"None" + }, + { + "siteName":"JCY-SPINE-01.INFRA.NTC.COM_1", + "vlanName":"vlan202", + "vlanId":202, + "dscr":"None" + }, + { + "siteName":"JCY-SPINE-01.INFRA.NTC.COM_1", + "vlanName":"vlan203", + "vlanId":203, + "dscr":"None" + }, + { + "siteName":"JCY-SPINE-01.INFRA.NTC.COM_1", + "vlanName":"vlan204", + "vlanId":204, + "dscr":"None" + }, + { + "siteName":"JCY-SPINE-01.INFRA.NTC.COM_1", + "vlanName":"vlan205", + "vlanId":205, + "dscr":"None" + }, + { + "siteName":"JCY-SPINE-01.INFRA.NTC.COM_1", + "vlanName":"vlan280", + "vlanId":280, + "dscr":"None" + }, + { + "siteName":"JCY-SPINE-01.INFRA.NTC.COM_1", + "vlanName":"appvlan0004", + "vlanId":292, + "dscr":"None" + }, + { + "siteName":"JCY-SPINE-01.INFRA.NTC.COM_1", + "vlanName":"dbvlan0001", + "vlanId":393, + "dscr":"None" + } +] \ No newline at end of file diff --git a/nautobot_ssot/tests/ipfabric/test_ipfabric_adapter.py b/nautobot_ssot/tests/ipfabric/test_ipfabric_adapter.py new file mode 100644 index 000000000..e19402b0b --- /dev/null +++ b/nautobot_ssot/tests/ipfabric/test_ipfabric_adapter.py @@ -0,0 +1,89 @@ +"""Unit tests for the IPFabric DiffSync adapter class.""" +import json +import uuid +from unittest.mock import MagicMock + +from django.contrib.contenttypes.models import ContentType +from django.test import TestCase +from nautobot.extras.models import Job, JobResult + +from nautobot_ssot.integrations.ipfabric.diffsync.adapter_ipfabric import IPFabricDiffSync +from nautobot_ssot.integrations.ipfabric.jobs import IpFabricDataSource + + +def load_json(path): + """Load a json file.""" + with open(path, encoding="utf-8") as file: + return json.loads(file.read()) + + +SITE_FIXTURE = load_json("./nautobot_ssot/tests/ipfabric/fixtures/get_sites.json") +DEVICE_INVENTORY_FIXTURE = load_json("./nautobot_ssot/tests/ipfabric/fixtures/get_device_inventory.json") +VLAN_FIXTURE = load_json("./nautobot_ssot/tests/ipfabric/fixtures/get_vlans.json") +INTERFACE_FIXTURE = load_json("./nautobot_ssot/tests/ipfabric/fixtures/get_interface_inventory.json") + + +class IPFabricDiffSyncTestCase(TestCase): + """Test the IPFabricDiffSync adapter class.""" + + def test_data_loading(self): + """Test the load() function.""" + + # Create a mock client + ipfabric_client = MagicMock() + ipfabric_client.inventory.sites.all.return_value = SITE_FIXTURE + ipfabric_client.inventory.devices.all.return_value = DEVICE_INVENTORY_FIXTURE + ipfabric_client.fetch_all = MagicMock( + side_effect=(lambda x: VLAN_FIXTURE if x == "tables/vlan/site-summary" else "") + ) + ipfabric_client.inventory.interfaces.all.return_value = INTERFACE_FIXTURE + + job = IpFabricDataSource() + job.job_result = JobResult.objects.create( + name=job.class_path, obj_type=ContentType.objects.get_for_model(Job), user=None, job_id=uuid.uuid4() + ) + ipfabric = IPFabricDiffSync(job=job, sync=None, client=ipfabric_client) + ipfabric.load() + self.assertEqual( + {site["siteName"] for site in SITE_FIXTURE}, + {site.get_unique_id() for site in ipfabric.get_all("location")}, + ) + self.assertEqual( + {dev["hostname"] for dev in DEVICE_INVENTORY_FIXTURE}, + {dev.get_unique_id() for dev in ipfabric.get_all("device")}, + ) + self.assertEqual( + {f"{vlan['vlanName']}__{vlan['siteName']}" for vlan in VLAN_FIXTURE}, + {vlan.get_unique_id() for vlan in ipfabric.get_all("vlan")}, + ) + + # Assert each site has a device tied to it. + for site in ipfabric.get_all("location"): + self.assertEqual(len(site.devices), 1, f"{site} does not have the expected single device tied to it.") + self.assertTrue(hasattr(site, "vlans")) + + # Assert each device has the necessary attributes + for device in ipfabric.get_all("device"): + self.assertTrue(hasattr(device, "location_name")) + self.assertTrue(hasattr(device, "model")) + self.assertTrue(hasattr(device, "vendor")) + self.assertTrue(hasattr(device, "serial_number")) + self.assertTrue(hasattr(device, "interfaces")) + + # Assert each vlan has the necessary attributes + for vlan in ipfabric.get_all("vlan"): + self.assertTrue(hasattr(vlan, "name")) + self.assertTrue(hasattr(vlan, "vid")) + self.assertTrue(hasattr(vlan, "status")) + self.assertTrue(hasattr(vlan, "site")) + self.assertTrue(hasattr(vlan, "description")) + + # Assert each interface has the necessary attributes + for interface in ipfabric.get_all("interface"): + self.assertTrue(hasattr(interface, "name")) + self.assertTrue(hasattr(interface, "device_name")) + self.assertTrue(hasattr(interface, "mac_address")) + self.assertTrue(hasattr(interface, "mtu")) + self.assertTrue(hasattr(interface, "ip_address")) + self.assertTrue(hasattr(interface, "subnet_mask")) + self.assertTrue(hasattr(interface, "type")) diff --git a/nautobot_ssot/tests/ipfabric/test_jobs.py b/nautobot_ssot/tests/ipfabric/test_jobs.py new file mode 100644 index 000000000..1cc354212 --- /dev/null +++ b/nautobot_ssot/tests/ipfabric/test_jobs.py @@ -0,0 +1,72 @@ +"""Test IPFabric Jobs.""" +from copy import deepcopy + +from django.conf import settings +from django.test import TestCase +from django.urls import reverse + +from nautobot_ssot.integrations.ipfabric import jobs + +CONFIG = settings.PLUGINS_CONFIG.get("nautobot_ssot", {}) +BACKUP_CONFIG = deepcopy(CONFIG) + + +class IPFabricJobTest(TestCase): + """Test the IPFabric job.""" + + def test_metadata(self): + """Verify correctness of the Job Meta attributes.""" + self.assertEqual("IPFabric ⟹ Nautobot", jobs.IpFabricDataSource.name) + self.assertEqual("IPFabric ⟹ Nautobot", jobs.IpFabricDataSource.Meta.name) + self.assertEqual("IP Fabric", jobs.IpFabricDataSource.Meta.data_source) + self.assertEqual("Sync data from IP Fabric into Nautobot.", jobs.IpFabricDataSource.Meta.description) + + def test_data_mapping(self): + """Verify correctness of the data_mappings() API.""" + mappings = jobs.IpFabricDataSource.data_mappings() + + self.assertEqual("Device", mappings[0].source_name) + self.assertIsNone(mappings[0].source_url) + self.assertEqual("Device", mappings[0].target_name) + self.assertEqual(reverse("dcim:device_list"), mappings[0].target_url) + + self.assertEqual("Site", mappings[1].source_name) + self.assertIsNone(mappings[1].source_url) + self.assertEqual("Site", mappings[1].target_name) + self.assertEqual(reverse("dcim:site_list"), mappings[1].target_url) + + self.assertEqual("Interfaces", mappings[2].source_name) + self.assertIsNone(mappings[2].source_url) + self.assertEqual("Interfaces", mappings[2].target_name) + self.assertEqual(reverse("dcim:interface_list"), mappings[2].target_url) + + self.assertEqual("IP Addresses", mappings[3].source_name) + self.assertIsNone(mappings[3].source_url) + self.assertEqual("IP Addresses", mappings[3].target_name) + self.assertEqual(reverse("ipam:ipaddress_list"), mappings[3].target_url) + + self.assertEqual("VLANs", mappings[4].source_name) + self.assertIsNone(mappings[4].source_url) + self.assertEqual("VLANs", mappings[4].target_name) + self.assertEqual(reverse("ipam:vlan_list"), mappings[4].target_url) + + # @override_settings( + # PLUGINS_CONFIG={ + # "nautobot_ssot": { + # "IPFABRIC_HOST": "https://ipfabric.networktocode.com", + # "IPFABRIC_API_TOKEN": "1234", + # } + # } + # ) + # def test_config_information(self): + # """Verify the config_information() API.""" + # CONFIG["ipfabric_host"] = "https://ipfabric.networktocode.com" + # config_information = jobs.IpFabricDataSource.config_information() + # self.assertContains( + # config_information, + # { + # "IP Fabric host": "https://ipfabric.networktocode.com", + # }, + # ) + # # CLEANUP + # CONFIG["ipfabric_host"] = BACKUP_CONFIG["ipfabric_host"] diff --git a/nautobot_ssot/tests/ipfabric/test_nautobot_adapter.py b/nautobot_ssot/tests/ipfabric/test_nautobot_adapter.py new file mode 100644 index 000000000..15963bf79 --- /dev/null +++ b/nautobot_ssot/tests/ipfabric/test_nautobot_adapter.py @@ -0,0 +1,79 @@ +# """Unit tests for the IPFabric DiffSync adapter class.""" + +# import uuid + +# from django.contrib.contenttypes.models import ContentType +# from django.test import TestCase +# from nautobot.dcim.models import Device, DeviceRole, DeviceType, Manufacturer, Site +# from nautobot.extras.models import Job, JobResult, Status +# from nautobot.ipam.models import VLAN + +# from nautobot_ssot_ipfabric.diffsync.adapter_nautobot import NautobotDiffSync +# from nautobot_ssot_ipfabric.jobs import IpFabricDataSource + + +# class IPFabricDiffSyncTestCase(TestCase): +# """Test the NautobotDiffSync adapter class.""" + +# def setUp(self): +# """Create Nautobot objects to load and test with.""" +# status_active = Status.objects.get(slug="active") + +# site_1 = Site.objects.create(name="Site 1", slug="site-1", status=status_active) +# site_2 = Site.objects.create(name="Site 2", slug="site-2", status=status_active) + +# manufacturer = Manufacturer.objects.create(name="Cisco", slug="cisco") +# device_type = DeviceType.objects.create(manufacturer=manufacturer, model="CSR 1000v", slug="csr1000v") +# device_role = DeviceRole.objects.create(name="Router", slug="router") + +# Device.objects.create( +# name="csr1", device_type=device_type, device_role=device_role, site=site_1, status=status_active +# ) +# Device.objects.create( +# name="csr2", device_type=device_type, device_role=device_role, site=site_2, status=status_active +# ) + +# VLAN.objects.create(name="VLAN101", vid=101, status=status_active, site=site_1) + +# def test_data_loading(self): +# """Test the load() function.""" + +# job = IpFabricDataSource() +# job.job_result = JobResult.objects.create( +# name=job.class_path, obj_type=ContentType.objects.get_for_model(Job), user=None, job_id=uuid.uuid4() +# ) + +# nautobot = NautobotDiffSync( +# job=job, +# sync=None, +# safe_delete_mode=True, +# sync_ipfabric_tagged_only=True, +# ) +# nautobot.load() + +# self.assertEqual( +# set(["Site 1", "Site 2"]), +# {site.get_unique_id() for site in nautobot.get_all("location")}, +# ) +# self.assertEqual( +# set(["csr1", "csr2"]), +# {dev.get_unique_id() for dev in nautobot.get_all("device")}, +# ) + +# # Assert each site has a device tied to it. +# for device in nautobot.get_all("device"): +# self.assertTrue(hasattr(device, "location_name"), f"{device} is missing location_name") +# self.assertTrue(hasattr(device, "model"), f"{device} is missing model") +# self.assertTrue(hasattr(device, "name"), f"{device} is missing name") +# # These attributes don't exist on our Device DiffSyncModel yet but we may want them there in the future +# # self.assertTrue(hasattr(device, "platform")) +# # self.assertTrue(hasattr(device, "role")) +# # self.assertTrue(hasattr(device, "status")) +# self.assertTrue(hasattr(device, "serial_number"), f"{device} is missing serial_number") + +# # Assert each vlan has the necessary attributes +# for vlan in nautobot.get_all("vlan"): +# self.assertTrue(hasattr(vlan, "name")) +# self.assertTrue(hasattr(vlan, "vid")) +# self.assertTrue(hasattr(vlan, "status")) +# self.assertTrue(hasattr(vlan, "site")) diff --git a/nautobot_ssot/tests/ipfabric/test_nbutils.py b/nautobot_ssot/tests/ipfabric/test_nbutils.py new file mode 100644 index 000000000..80f71b66d --- /dev/null +++ b/nautobot_ssot/tests/ipfabric/test_nbutils.py @@ -0,0 +1,127 @@ +"""Test Nautobot Utilities.""" +from django.contrib.contenttypes.models import ContentType +from django.test import TestCase +from django.utils.text import slugify +from nautobot.dcim.models import DeviceRole, DeviceType, Manufacturer, Site +from nautobot.dcim.models.devices import Device +from nautobot.extras.models.statuses import Status +from nautobot.ipam.models import VLAN, IPAddress +from nautobot.utilities.choices import ColorChoices + +from nautobot_ssot.integrations.ipfabric.utilities import ( # create_ip,; create_interface,; create_site, + get_or_create_device_role_object, + create_device_type_object, + create_manufacturer, + create_status, + create_vlan, +) + + +# pylint: disable=too-many-instance-attributes +class TestNautobotUtils(TestCase): + """Test Nautobot Utility.""" + + def setUp(self): + """Setup.""" + self.site = Site.objects.create( + name="Test-Site", + slug="test-site", + status=Status.objects.get(name="Active"), + ) + + self.manufacturer = Manufacturer.objects.create(name="Test-Manufacturer", slug="test-manufacturer") + self.device_type = DeviceType.objects.create( + model="Test-DeviceType", slug="test-devicetype", manufacturer=self.manufacturer + ) + self.device_role = DeviceRole.objects.create(name="Test-Role", slug="test-role", color=ColorChoices.COLOR_RED) + self.device_role.cf["ipfabric_type"] = "Test-Role" + self.device_role.validated_save() + self.content_type = ContentType.objects.get(app_label="dcim", model="device") + self.status = Status.objects.create( + name="Test-Status", + slug=slugify("Test-Status"), + color=ColorChoices.COLOR_AMBER, + description="Test-Description", + ) + self.status.content_types.set([self.content_type]) + self.status_obj = Status.objects.get_for_model(IPAddress).get(slug=slugify("Active")) + self.ip_address = IPAddress.objects.create(address="192.168.0.1/32", status=self.status_obj) + + self.device = Device.objects.create( + name="Test-Device", site=self.site, device_type=self.device_type, device_role=self.device_role + ) + + self.device.interfaces.create(name="Test-Interface") + self.vlan_content_type = ContentType.objects.get(app_label="ipam", model="vlan") + self.vlan_status = Status.objects.create( + name="Test-Vlan-Status", + slug=slugify("Test-Vlan-Status"), + color=ColorChoices.COLOR_AMBER, + description="Test-Description", + ) + self.vlan_status.content_types.set([self.vlan_content_type]) + + def test_create_vlan(self): + """Test `create_vlan` Utility.""" + vlan = create_vlan( + vlan_name="Test-Vlan", + vlan_id=100, + vlan_status="Test-Vlan-Status", + site_obj=self.site, + description="Test-Vlan", + ) + self.assertEqual(VLAN.objects.get(name="Test-Vlan").pk, vlan.pk) + + # def test_create_site(self): + # """Test `create_site` Utility.""" + # test_site = create_site(site_name="Test-Site") + # self.assertEqual(test_site.id, self.site.id) + + # def test_create_site_exception(self): + # """Test `create_site` Utility exception.""" + # site = create_site( + # site_name="Test-Site-100", + # site_id=123456, + # ) + # self.assertEqual(Site.objects.get(name="Test-Site-100").pk, site.pk) + + def test_create_device_type_object(self): + """Test `create_device_type_object` Utility.""" + test_device_type = create_device_type_object(device_type="Test-DeviceType", vendor_name="Test-Manufacturer") + self.assertEqual(test_device_type.id, self.device_type.id) + + def test_create_manufacturer(self): + """Test `create_manufacturer` Utility.""" + test_manufacturer = create_manufacturer(vendor_name="Test-Manufacturer") + self.assertEqual(test_manufacturer.id, self.manufacturer.id) + + def test_get_or_create_device_role(self): + """Test `get_or_create_device_role` Utility.""" + test_device_role = get_or_create_device_role_object("Test-Role", role_color=ColorChoices.COLOR_RED) + self.assertEqual(test_device_role.id, self.device_role.id) + + def test_create_status(self): + """Test `create_status` Utility.""" + test_status = create_status(status_name="Test-Status", status_color=ColorChoices.COLOR_AMBER) + self.assertEqual(test_status.id, self.status.id) + + def test_create_status_doesnt_exist(self): + """Test `create_status` Utility.""" + test_status = create_status(status_name="Test-Status-100", status_color=ColorChoices.COLOR_AMBER) + self.assertEqual(test_status.id, Status.objects.get(name="Test-Status-100").id) + + # def test_create_ip(self): + # """Test `create_ip` Utility.""" + # test_ip = create_ip("192.168.0.1", "255.255.255.255") + # self.assertEqual(test_ip.id, self.ip_address.id) + + # def test_create_ip_device_add(self): + # """Test `create_ip` adding to device Utility.""" + # test_ip = create_ip("192.168.0.1", "255.255.255.255", object_pk=self.device.id) + # self.assertEqual(test_ip.id, self.ip_address.id) + + # def test_create_interface(self): + # """Test `create_interface` Utility.""" + # interface_details = {"name": "Test-Interface"} + # test_interface = create_interface(self.device, interface_details) + # self.assertEqual(test_interface.id, self.device.interfaces.get(name="Test-Interface").id) diff --git a/poetry.lock b/poetry.lock index 1a6065fa2..d3e904025 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. [[package]] name = "amqp" @@ -25,6 +25,27 @@ files = [ {file = "aniso8601-7.0.0.tar.gz", hash = "sha256:513d2b6637b7853806ae79ffaca6f3e8754bdd547048f5ccc1420aec4b714f1e"}, ] +[[package]] +name = "anyio" +version = "4.0.0" +description = "High level compatibility layer for multiple asynchronous event loop implementations" +optional = true +python-versions = ">=3.8" +files = [ + {file = "anyio-4.0.0-py3-none-any.whl", hash = "sha256:cfdb2b588b9fc25ede96d8db56ed50848b0b649dca3dd1df0b11f683bb9e0b5f"}, + {file = "anyio-4.0.0.tar.gz", hash = "sha256:f7ed51751b2c2add651e5747c891b47e26d2a21be5d32d9311dfe9692f3e5d7a"}, +] + +[package.dependencies] +exceptiongroup = {version = ">=1.0.2", markers = "python_version < \"3.11\""} +idna = ">=2.8" +sniffio = ">=1.1" + +[package.extras] +doc = ["Sphinx (>=7)", "packaging", "sphinx-autodoc-typehints (>=1.2.0)"] +test = ["anyio[trio]", "coverage[toml] (>=7)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "uvloop (>=0.17)"] +trio = ["trio (>=0.22)"] + [[package]] name = "appnope" version = "0.1.3" @@ -712,6 +733,24 @@ files = [ {file = "decorator-5.1.1.tar.gz", hash = "sha256:637996211036b6385ef91435e4fae22989472f9d571faba8927ba8253acbc330"}, ] +[[package]] +name = "deepdiff" +version = "6.5.0" +description = "Deep Difference and Search of any Python object/data. Recreate objects by adding adding deltas to each other." +optional = true +python-versions = ">=3.7" +files = [ + {file = "deepdiff-6.5.0-py3-none-any.whl", hash = "sha256:acdc1651a3e802415e0337b7e1192df5cd7c17b72fbab480466fdd799b9a72e7"}, + {file = "deepdiff-6.5.0.tar.gz", hash = "sha256:080b1359d6128f3f5f1738c6be3064f0ad9b0cc41994aa90a028065f6ad11f25"}, +] + +[package.dependencies] +ordered-set = ">=4.0.2,<4.2.0" + +[package.extras] +cli = ["click (==8.1.3)", "pyyaml (==6.0.1)"] +optimize = ["orjson"] + [[package]] name = "defusedxml" version = "0.7.1" @@ -1412,6 +1451,61 @@ files = [ [package.extras] protobuf = ["grpcio-tools (>=1.57.0)"] +[[package]] +name = "h11" +version = "0.14.0" +description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" +optional = true +python-versions = ">=3.7" +files = [ + {file = "h11-0.14.0-py3-none-any.whl", hash = "sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761"}, + {file = "h11-0.14.0.tar.gz", hash = "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d"}, +] + +[[package]] +name = "httpcore" +version = "0.16.3" +description = "A minimal low-level HTTP client." +optional = true +python-versions = ">=3.7" +files = [ + {file = "httpcore-0.16.3-py3-none-any.whl", hash = "sha256:da1fb708784a938aa084bde4feb8317056c55037247c787bd7e19eb2c2949dc0"}, + {file = "httpcore-0.16.3.tar.gz", hash = "sha256:c5d6f04e2fc530f39e0c077e6a30caa53f1451096120f1f38b954afd0b17c0cb"}, +] + +[package.dependencies] +anyio = ">=3.0,<5.0" +certifi = "*" +h11 = ">=0.13,<0.15" +sniffio = "==1.*" + +[package.extras] +http2 = ["h2 (>=3,<5)"] +socks = ["socksio (==1.*)"] + +[[package]] +name = "httpx" +version = "0.23.3" +description = "The next generation HTTP client." +optional = true +python-versions = ">=3.7" +files = [ + {file = "httpx-0.23.3-py3-none-any.whl", hash = "sha256:a211fcce9b1254ea24f0cd6af9869b3d29aba40154e947d2a07bb499b3e310d6"}, + {file = "httpx-0.23.3.tar.gz", hash = "sha256:9818458eb565bb54898ccb9b8b251a28785dd4a55afbc23d0eb410754fe7d0f9"}, +] + +[package.dependencies] +certifi = "*" +httpcore = ">=0.15.0,<0.17.0" +rfc3986 = {version = ">=1.3,<2", extras = ["idna2008"]} +sniffio = "*" + +[package.extras] +brotli = ["brotli", "brotlicffi"] +cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<13)"] +http2 = ["h2 (>=3,<5)"] +socks = ["socksio (==1.*)"] + [[package]] name = "idna" version = "3.4" @@ -1580,6 +1674,64 @@ files = [ {file = "invoke-2.2.0.tar.gz", hash = "sha256:ee6cbb101af1a859c7fe84f2a264c059020b0cb7fe3535f9424300ab568f6bd5"}, ] +[[package]] +name = "ipfabric" +version = "6.0.10" +description = "Python package for interacting with IP Fabric" +optional = true +python-versions = ">=3.7.1,<4.0.0" +files = [ + {file = "ipfabric-6.0.10-py3-none-any.whl", hash = "sha256:22a45ac36199eb69226dc9213d0d3f4df8d0e156ff7982c9a7dd6ff80409b598"}, + {file = "ipfabric-6.0.10.tar.gz", hash = "sha256:7dca65f77b14b5bda04ba76abbed0685b81d934e6b1f53a67c54bc8509ba9807"}, +] + +[package.dependencies] +deepdiff = ">=6.2.2,<7.0.0" +httpx = ">=0.23.2,<0.24.0" +ipfabric-httpx-auth = ">=6.0.0,<7.0.0" +macaddress = ">=2.0.2,<2.1.0" +pydantic = ">=1.8.2,<2.0.0" +python-dateutil = ">=2.8.2,<3.0.0" +python-dotenv = ">=0.21,<0.22" +pytz = ">=2022.4,<2023.0" + +[package.extras] +examples = ["openpyxl (>=3.0.9,<4.0.0)", "pandas (>=1.3.0,<2.0.0)", "python-json-logger (>=2.0.4,<3.0.0)", "pyyaml (>=6.0,<7.0)", "tabulate (>=0.8.9,<0.10.0)"] + +[[package]] +name = "ipfabric-diagrams" +version = "6.0.2" +description = "Python package for interacting with IP Fabric Diagrams" +optional = true +python-versions = ">=3.7.1,<4.0.0" +files = [ + {file = "ipfabric_diagrams-6.0.2-py3-none-any.whl", hash = "sha256:933e58250d7bd09ee5557667fe8bf4add1e9e3e1ec7dd137345f97bdd928b255"}, + {file = "ipfabric_diagrams-6.0.2.tar.gz", hash = "sha256:a76791e8547f48d70b59e5d653271f224b7482e5d77f995eca3be36886118900"}, +] + +[package.dependencies] +ipfabric = ">=6.0.7,<6.1.0" +pydantic = ">=1.8.2,<2.0.0" +typing-extensions = ">=4.1.1,<5.0.0" + +[package.extras] +examples = ["rich (>=12.5.1,<13.0.0)"] + +[[package]] +name = "ipfabric-httpx-auth" +version = "6.0.1" +description = "DEPRECATED: Authentication plugin for IP Fabric" +optional = true +python-versions = ">=3.7.1,<4.0.0" +files = [ + {file = "ipfabric_httpx_auth-6.0.1-py3-none-any.whl", hash = "sha256:7e47559d7a0f575f77b32ddefbeaff416973e36fe5728493c86a85f64ba918ba"}, + {file = "ipfabric_httpx_auth-6.0.1.tar.gz", hash = "sha256:32c713ed7a2326e9b11c581379e748b8893e24ea97e0388378a95e82b392bc0c"}, +] + +[package.dependencies] +httpx = ">=0.23.0,<0.24.0" +PyJWT = ">=2.4.0,<3.0.0" + [[package]] name = "ipython" version = "8.12.2" @@ -1789,6 +1941,17 @@ files = [ {file = "lazy_object_proxy-1.9.0-cp39-cp39-win_amd64.whl", hash = "sha256:db1c1722726f47e10e0b5fdbf15ac3b8adb58c091d12b3ab713965795036985f"}, ] +[[package]] +name = "macaddress" +version = "2.0.2" +description = "Like ``ipaddress``, but for hardware identifiers such as MAC addresses." +optional = true +python-versions = "*" +files = [ + {file = "macaddress-2.0.2-py3-none-any.whl", hash = "sha256:6f4a0430f9b5af6d98a582b8d527ba2cd3f0825fce5503a9ce5c73acb772c30f"}, + {file = "macaddress-2.0.2.tar.gz", hash = "sha256:1400ccdc28d747102d57ae61e5b78d8985872930810ceb8860cd49abd1e1fa37"}, +] + [[package]] name = "markdown" version = "3.3.7" @@ -2265,6 +2428,20 @@ rsa = ["cryptography (>=3.0.0)"] signals = ["blinker (>=1.4.0)"] signedtoken = ["cryptography (>=3.0.0)", "pyjwt (>=2.0.0,<3)"] +[[package]] +name = "ordered-set" +version = "4.1.0" +description = "An OrderedSet is a custom MutableSet that remembers its order, so that every" +optional = true +python-versions = ">=3.7" +files = [ + {file = "ordered-set-4.1.0.tar.gz", hash = "sha256:694a8e44c87657c59292ede72891eb91d34131f6531463aab3009191c77364a8"}, + {file = "ordered_set-4.1.0-py3-none-any.whl", hash = "sha256:046e1132c71fcf3330438a539928932caf51ddbc582496833e23de611de14562"}, +] + +[package.extras] +dev = ["black", "mypy", "pytest"] + [[package]] name = "packaging" version = "23.1" @@ -2972,13 +3149,13 @@ postgresql = ["psycopg2"] [[package]] name = "pytz" -version = "2023.3.post1" +version = "2022.7.1" description = "World timezone definitions, modern and historical" optional = false python-versions = "*" files = [ - {file = "pytz-2023.3.post1-py2.py3-none-any.whl", hash = "sha256:ce42d816b81b68506614c11e8937d3aa9e41007ceb50bfdcb0749b921bf646c7"}, - {file = "pytz-2023.3.post1.tar.gz", hash = "sha256:7b4fddbeb94a1eba4b557da24f19fdf9db575192544270a9101d8509f9f43d7b"}, + {file = "pytz-2022.7.1-py2.py3-none-any.whl", hash = "sha256:78f4f37d8198e0627c5f1143240bb0206b8691d8d7ac6d78fee88b78733f8c4a"}, + {file = "pytz-2022.7.1.tar.gz", hash = "sha256:01a0681c4b9684a28304615eba55d1ab31ae00bf68ec157ec3708a8182dbbcd0"}, ] [[package]] @@ -3285,6 +3462,23 @@ requests = ">=2.0.0" [package.extras] rsa = ["oauthlib[signedtoken] (>=3.0.0)"] +[[package]] +name = "rfc3986" +version = "1.5.0" +description = "Validating URI References per RFC 3986" +optional = true +python-versions = "*" +files = [ + {file = "rfc3986-1.5.0-py2.py3-none-any.whl", hash = "sha256:a86d6e1f5b1dc238b218b012df0aa79409667bb209e58da56d0b94704e712a97"}, + {file = "rfc3986-1.5.0.tar.gz", hash = "sha256:270aaf10d87d0d4e095063c65bf3ddbc6ee3d0b226328ce21e036f946e421835"}, +] + +[package.dependencies] +idna = {version = "*", optional = true, markers = "extra == \"idna2008\""} + +[package.extras] +idna2008 = ["idna"] + [[package]] name = "rich" version = "13.5.2" @@ -3457,6 +3651,17 @@ files = [ {file = "smmap-5.0.0.tar.gz", hash = "sha256:c840e62059cd3be204b0c9c9f74be2c09d5648eddd4580d9314c3ecde0b30936"}, ] +[[package]] +name = "sniffio" +version = "1.3.0" +description = "Sniff out which async library your code is running under" +optional = true +python-versions = ">=3.7" +files = [ + {file = "sniffio-1.3.0-py3-none-any.whl", hash = "sha256:eecefdce1e5bbfb7ad2eeaabf7c1eeb404d7757c379bd1f7e5cce9d8bf425384"}, + {file = "sniffio-1.3.0.tar.gz", hash = "sha256:e60305c5e5d314f5389259b7f22aaa33d8f7dee49763119234af3755c55b9101"}, +] + [[package]] name = "snowballstemmer" version = "2.2.0" @@ -3923,9 +4128,10 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [extras] aci = ["PyYAML"] -all = ["Jinja2", "PyYAML", "cloudvision", "cvprac", "dnspython", "ijson", "oauthlib", "python-magic", "pytz", "requests", "requests-oauthlib", "six"] +all = ["Jinja2", "PyYAML", "cloudvision", "cvprac", "dnspython", "ijson", "ipfabric", "ipfabric-diagrams", "netutils", "oauthlib", "python-magic", "pytz", "requests", "requests-oauthlib", "six"] aristacv = ["cloudvision", "cvprac"] infoblox = ["dnspython"] +ipfabric = ["ipfabric", "ipfabric-diagrams", "netutils"] nautobot-device-lifecycle-mgmt = [] pysnow = ["ijson", "oauthlib", "python-magic", "pytz", "requests", "requests-oauthlib", "six"] servicenow = ["Jinja2", "PyYAML", "ijson", "oauthlib", "python-magic", "pytz", "requests", "requests-oauthlib", "six"] @@ -3933,4 +4139,4 @@ servicenow = ["Jinja2", "PyYAML", "ijson", "oauthlib", "python-magic", "pytz", " [metadata] lock-version = "2.0" python-versions = "^3.8,<3.12" -content-hash = "15c124872e02b12f84f614cfd564d39883075b091c0268ed11b81f17ca1d14a0" +content-hash = "74bb9f18d7c7ca540f53cab1d8f0168ae3c5140ce27abb47554aa68998becbe2" diff --git a/pyproject.toml b/pyproject.toml index 83781f150..0e0b13e86 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,6 +39,9 @@ dnspython = { version = "^2.1.0", optional = true } packaging = ">=21.3, <24" prometheus-client = "~0.17.1" ijson = { version = ">=2.5.1", optional = true } +ipfabric = { version = "~6.0.9", optional = true } +ipfabric-diagrams = { version = "~6.0.2", optional = true } +netutils = { version = "^1.0.0", optional = true } oauthlib = { version = ">=3.1.0", optional = true } python-magic = { version = ">=0.4.15", optional = true } pytz = { version = ">=2019.3", optional = true } @@ -79,6 +82,8 @@ mkdocstrings = "0.22.0" mkdocstrings-python = "1.1.2" requests-mock = "^1.10.0" parameterized = "^0.8.1" +# Enable chatops after dropping Python 3.7 support +# nautobot-chatops = { version = "^2.0.2", extras = ["ipfabric"] } [tool.poetry.plugins."nautobot_ssot.data_sources"] "example" = "nautobot_ssot.sync.example:ExampleSyncWorker" @@ -86,6 +91,10 @@ parameterized = "^0.8.1" [tool.poetry.plugins."nautobot_ssot.data_targets"] "example" = "nautobot_ssot.sync.example:ExampleSyncWorker" +# Enable chatops after dropping Python 3.7 support +# [tool.poetry.plugins."nautobot.workers"] +# "ipfabric" = "nautobot_ssot.integrations.ipfabric.workers:ipfabric" + [tool.poetry.extras] aci = [ "PyYAML", @@ -97,7 +106,10 @@ all = [ "cvprac", "dnspython", "ijson", + "ipfabric", + "ipfabric-diagrams", "nautobot-device-lifecycle-mgmt", + "netutils", "oauthlib", "python-magic", "pytz", @@ -112,6 +124,11 @@ aristacv = [ infoblox = [ "dnspython", ] +ipfabric = [ + "ipfabric", + "ipfabric-diagrams", + "netutils", +] # pysnow = "^0.7.17" # PySNow is currently pinned to an older version of pytz as a dependency, which blocks compatibility with newer # versions of Nautobot. See https://github.com/rbw/pysnow/pull/186 diff --git a/tasks.py b/tasks.py index d56d05f2a..5ebbf77b0 100644 --- a/tasks.py +++ b/tasks.py @@ -45,7 +45,7 @@ def is_truthy(arg): namespace.configure( { "nautobot_ssot": { - "nautobot_ver": "1.5.1", + "nautobot_ver": "2.0.0-rc.2", "project_name": "nautobot_ssot", "python_ver": "3.8", "local": False, From 1e3463ef1a2590a7f4f9d9298ced22488bad62a5 Mon Sep 17 00:00:00 2001 From: Jan Snasel Date: Mon, 25 Sep 2023 09:55:11 +0000 Subject: [PATCH 02/12] chore: Migrate ipfabric --- development/creds.example.env | 2 + development/development.env | 10 ++- development/nautobot_config.py | 15 ++++ .../integrations/ipfabric/constants.py | 2 +- .../ipfabric/diffsync/adapter_ipfabric.py | 4 +- .../ipfabric/diffsync/adapter_nautobot.py | 87 +++++++++--------- .../ipfabric/diffsync/diffsync_models.py | 77 ++++++++-------- nautobot_ssot/integrations/ipfabric/jobs.py | 89 ++++++++++++------- .../integrations/ipfabric/signals.py | 20 ++--- .../ipfabric/utilities/__init__.py | 4 +- .../ipfabric/utilities/nbutils.py | 66 +++++++------- .../ipfabric/utilities/test_utils.py | 6 +- .../integrations/ipfabric/workers.py | 24 ++--- .../tests/ipfabric/test_ipfabric_adapter.py | 8 +- nautobot_ssot/tests/ipfabric/test_jobs.py | 6 +- nautobot_ssot/tests/ipfabric/test_nbutils.py | 69 +++++++------- poetry.lock | 4 +- pyproject.toml | 2 + 18 files changed, 268 insertions(+), 227 deletions(-) diff --git a/development/creds.example.env b/development/creds.example.env index 74b9371af..780d04b29 100644 --- a/development/creds.example.env +++ b/development/creds.example.env @@ -44,3 +44,5 @@ NAUTOBOT_APIC_VERIFY_DEVNET=False # NAUTOBOT_APIC_SITE_DEVNET="DevNet Sandbox" SERVICENOW_PASSWORD="changeme" + +IPFABRIC_API_TOKEN=secrettoken diff --git a/development/development.env b/development/development.env index 72e993940..19a0d075e 100644 --- a/development/development.env +++ b/development/development.env @@ -39,7 +39,10 @@ MYSQL_USER=${NAUTOBOT_DB_USER} MYSQL_DATABASE=${NAUTOBOT_DB_NAME} MYSQL_ROOT_HOST=% -NAUTOBOT_CELERY_TASK_SOFT_TIME_LIMIT=600 +NAUTOBOT_HOST="http://nautobot:8080" + +NAUTOBOT_CELERY_TASK_SOFT_TIME_LIMIT=3600 +NAUTOBOT_CELERY_TASK_TIME_LIMIT=3600 NAUTOBOT_SSOT_HIDE_EXAMPLE_JOBS="False" @@ -83,3 +86,8 @@ NAUTOBOT_SSOT_INFOBLOX_VERIFY_SSL="True" NAUTOBOT_SSOT_ENABLE_SERVICENOW="True" SERVICENOW_INSTANCE="" SERVICENOW_USERNAME="" + +NAUTOBOT_SSOT_ENABLE_IPFABRIC="True" +IPFABRIC_HOST="https://ipfabric.example.com" +IPFABRIC_SSL_VERIFY="True" +IPFABRIC_TIMEOUT=15 diff --git a/development/nautobot_config.py b/development/nautobot_config.py index f424c5585..264c29352 100644 --- a/development/nautobot_config.py +++ b/development/nautobot_config.py @@ -142,6 +142,15 @@ # Plugins configuration settings. These settings are used by various plugins that the user may have installed. # Each key in the dictionary is the name of an installed plugin and its value is a dictionary of settings. PLUGINS_CONFIG = { + # Enable chatops after Nautobot v2 compatible release + # "nautobot_chatops": { + # "enable_slack": True, + # "slack_api_token": os.getenv("SLACK_API_TOKEN"), + # "slack_signing_secret": os.getenv("SLACK_SIGNING_SECRET"), + # "session_cache_timeout": 3600, + # "ipfabric_api_token": os.getenv("IPFABRIC_API_TOKEN"), + # "ipfabric_host": os.getenv("IPFABRIC_HOST"), + # }, "nautobot_ssot": { # URL and credentials should be configured as environment variables on the host system "aci_apics": {x: os.environ[x] for x in os.environ if "APIC" in x}, @@ -192,6 +201,7 @@ "enable_aci": is_truthy(os.getenv("NAUTOBOT_SSOT_ENABLE_ACI")), "enable_aristacv": is_truthy(os.getenv("NAUTOBOT_SSOT_ENABLE_ARISTACV")), "enable_infoblox": is_truthy(os.getenv("NAUTOBOT_SSOT_ENABLE_INFOBLOX")), + "enable_ipfabric": is_truthy(os.getenv("NAUTOBOT_SSOT_ENABLE_IPFABRIC")), "enable_servicenow": is_truthy(os.getenv("NAUTOBOT_SSOT_ENABLE_SERVICENOW")), "hide_example_jobs": is_truthy(os.getenv("NAUTOBOT_SSOT_HIDE_EXAMPLE_JOBS")), "infoblox_default_status": os.getenv("NAUTOBOT_SSOT_INFOBLOX_DEFAULT_STATUS", "active"), @@ -208,6 +218,11 @@ "infoblox_username": os.getenv("NAUTOBOT_SSOT_INFOBLOX_USERNAME"), "infoblox_verify_ssl": is_truthy(os.getenv("NAUTOBOT_SSOT_INFOBLOX_VERIFY_SSL", True)), "infoblox_wapi_version": os.getenv("NAUTOBOT_SSOT_INFOBLOX_WAPI_VERSION", "v2.12"), + "ipfabric_api_token": os.getenv("IPFABRIC_API_TOKEN"), + "ipfabric_host": os.getenv("IPFABRIC_HOST"), + "ipfabric_ssl_verify": is_truthy(os.getenv("IPFABRIC_VERIFY", "False")), + "ipfabric_timeout": int(os.getenv("IPFABRIC_TIMEOUT", "15")), + "nautobot_host": os.getenv("NAUTOBOT_HOST"), "servicenow_instance": os.getenv("SERVICENOW_INSTANCE", ""), "servicenow_password": os.getenv("SERVICENOW_PASSWORD", ""), "servicenow_username": os.getenv("SERVICENOW_USERNAME", ""), diff --git a/nautobot_ssot/integrations/ipfabric/constants.py b/nautobot_ssot/integrations/ipfabric/constants.py index b7d4dce2a..bf8e32e6d 100644 --- a/nautobot_ssot/integrations/ipfabric/constants.py +++ b/nautobot_ssot/integrations/ipfabric/constants.py @@ -10,6 +10,6 @@ DEFAULT_INTERFACE_TYPE = "1000base-t" SAFE_DELETE_DEVICE_STATUS = "Deprecated" SAFE_DELETE_IPADDRESS_STATUS = "Deprecated" -SAFE_DELETE_SITE_STATUS = "Decommissioning" +SAFE_DELETE_LOCATION_STATUS = "Decommissioning" SAFE_DELETE_VLAN_STATUS = "Inventory" SAFE_IPADDRESS_INTERFACES_STATUS = "Deprecated" diff --git a/nautobot_ssot/integrations/ipfabric/diffsync/adapter_ipfabric.py b/nautobot_ssot/integrations/ipfabric/diffsync/adapter_ipfabric.py index 5b9ac2dbb..7c3afa2ca 100644 --- a/nautobot_ssot/integrations/ipfabric/diffsync/adapter_ipfabric.py +++ b/nautobot_ssot/integrations/ipfabric/diffsync/adapter_ipfabric.py @@ -34,14 +34,14 @@ def __init__(self, job, sync, client, *args, **kwargs): self.client = client def load_sites(self): - """Add IP Fabric Site objects as DiffSync Location models.""" + """Add IP Fabric Location objects as DiffSync Location models.""" sites = self.client.inventory.sites.all() for site in sites: try: location = self.location(diffsync=self, name=site["siteName"], site_id=site["id"], status="Active") self.add(location) except ObjectAlreadyExists: - self.job.log_debug(message=f"Duplicate Site discovered, {site}") + self.job.log_debug(message=f"Duplicate Location discovered, {site}") def load_device_interfaces(self, device_model, interfaces, device_primary_ip): """Create and load DiffSync Interface model objects for a specific device.""" diff --git a/nautobot_ssot/integrations/ipfabric/diffsync/adapter_nautobot.py b/nautobot_ssot/integrations/ipfabric/diffsync/adapter_nautobot.py index 88fc127a3..7bfa170f3 100644 --- a/nautobot_ssot/integrations/ipfabric/diffsync/adapter_nautobot.py +++ b/nautobot_ssot/integrations/ipfabric/diffsync/adapter_nautobot.py @@ -3,16 +3,16 @@ # Load method is packed with conditionals # pylint: disable=too-many-branches """DiffSync adapter class for Nautobot as source-of-truth.""" from collections import defaultdict -from typing import Any, ClassVar, List +from typing import Any, ClassVar, List, Optional from diffsync import DiffSync from diffsync.exceptions import ObjectAlreadyExists from django.db import IntegrityError, transaction from django.db.models import ProtectedError, Q -from nautobot.dcim.models import Device, Site +from nautobot.dcim.models import Device, Location from nautobot.extras.models import Tag from nautobot.ipam.models import VLAN, Interface -from nautobot.utilities.choices import ColorChoices +from nautobot.core.choices import ColorChoices from netutils.mac import mac_to_format from nautobot_ssot.integrations.ipfabric.diffsync import DiffSyncModelAdapters @@ -31,7 +31,7 @@ class NautobotDiffSync(DiffSyncModelAdapters): _vlan: ClassVar[Any] = VLAN _device: ClassVar[Any] = Device - _site: ClassVar[Any] = Site + _location: ClassVar[Any] = Location _interface: ClassVar[Any] = Interface def __init__( @@ -39,7 +39,7 @@ def __init__( job, sync, sync_ipfabric_tagged_only: bool, - site_filter: Site, + location_filter: Optional[Location], *args, **kwargs, ): @@ -48,7 +48,7 @@ def __init__( self.job = job self.sync = sync self.sync_ipfabric_tagged_only = sync_ipfabric_tagged_only - self.site_filter = site_filter + self.location_filter = location_filter def sync_complete(self, source: DiffSync, *args, **kwargs): """Clean up function for DiffSync sync. @@ -63,7 +63,7 @@ def sync_complete(self, source: DiffSync, *args, **kwargs): "_vlan", "_interface", "_device", - "_site", + "_location", ): for nautobot_object in self.objects_to_delete[grouping]: if NautobotDiffSync.safe_delete_mode: @@ -125,7 +125,7 @@ def load_device(self, filtered_devices: List, location): role=str(device_record.device_role.cf.get("ipfabric_type")) if str(device_record.device_role.cf.get("ipfabric_type")) else str(device_record.device_role), - location_name=device_record.site.name, + location_name=device_record.location.name, vendor=str(device_record.device_type.manufacturer), status=device_record.status.name, serial_number=device_record.serial if device_record.serial else "", @@ -147,7 +147,7 @@ def load_vlans(self, filtered_vlans: List, location): vlan = self.vlan( diffsync=self, name=vlan_record.name, - site=vlan_record.site.name, + location=vlan_record.location.name, status=vlan_record.status.name if vlan_record.status else "Active", vid=vlan_record.vid, vlan_pk=vlan_record.pk, @@ -160,74 +160,77 @@ def load_vlans(self, filtered_vlans: List, location): continue location.add_child(vlan) - def get_initial_site(self, ssot_tag: Tag): - """Identify the site objects based on user defined job inputs. + def get_initial_location(self, ssot_tag: Tag): + """Identify the location objects based on user defined job inputs. Args: ssot_tag (Tag): Tag used for filtering """ # Simple check / validate Tag is present. if self.sync_ipfabric_tagged_only: - site_objects = Site.objects.filter(tags__slug=ssot_tag.slug) - if self.site_filter: - site_objects = Site.objects.filter(Q(name=self.site_filter.name) & Q(tags__slug=ssot_tag.slug)) - if not site_objects: + location_objects = Location.objects.filter(tags__name=ssot_tag.name) + if self.location_filter: + location_objects = Location.objects.filter( + Q(name=self.location_filter.name) & Q(tags__name=ssot_tag.name) + ) + if not location_objects: self.job.log_warning( - message=f"{self.site_filter.name} was used to filter, alongside SSoT Tag. {self.site_filter.name} is not tagged." + message=f"{self.location_filter.name} was used to filter, alongside SSoT Tag. {self.location_filter.name} is not tagged." ) elif not self.sync_ipfabric_tagged_only: - if self.site_filter: - site_objects = Site.objects.filter(name=self.site_filter.name) + if self.location_filter: + location_objects = Location.objects.filter(name=self.location_filter.name) else: - site_objects = Site.objects.all() - return site_objects + location_objects = Location.objects.all() + return location_objects @transaction.atomic def load_data(self): - """Add Nautobot Site objects as DiffSync Location models.""" + """Add Nautobot Location objects as DiffSync Location models.""" ssot_tag, _ = Tag.objects.get_or_create( - slug="ssot-synced-from-ipfabric", name="SSoT Synced from IPFabric", defaults={ "description": "Object synced at some point from IPFabric to Nautobot", "color": ColorChoices.COLOR_LIGHT_GREEN, }, ) - site_objects = self.get_initial_site(ssot_tag) - # The parent object that stores all children, is the Site. - self.job.log_debug(message=f"Found {site_objects.count()} Nautobot Site objects to start sync from") + location_objects = self.get_initial_location(ssot_tag) + # The parent object that stores all children, is the Location. + self.job.log_debug(message=f"Found {location_objects.count()} Nautobot Location objects to start sync from") - if site_objects: - for site_record in site_objects: + if location_objects: + for location_record in location_objects: try: location = self.location( diffsync=self, - name=site_record.name, - site_id=site_record.custom_field_data.get("ipfabric-site-id"), - status=site_record.status.name, + name=location_record.name, + site_id=location_record.custom_field_data.get("ipfabric-site-id"), + status=location_record.status.name, ) except AttributeError: self.job.log_debug( - message=f"Error loading {site_record}, invalid or missing attributes on object. Skipping..." + message=f"Error loading {location_record}, invalid or missing attributes on object. Skipping..." ) continue self.add(location) try: - # Load Site's Children - Devices with Interfaces, if any. + # Load Location's Children - Devices with Interfaces, if any. if self.sync_ipfabric_tagged_only: - nautobot_site_devices = Device.objects.filter(Q(site=site_record) & Q(tags__slug=ssot_tag.slug)) + nautobot_location_devices = Device.objects.filter( + Q(location=location_record) & Q(tags__name=ssot_tag.name) + ) else: - nautobot_site_devices = Device.objects.filter(site=site_record) - if nautobot_site_devices.exists(): - self.load_device(nautobot_site_devices, location) + nautobot_location_devices = Device.objects.filter(location=location_record) + if nautobot_location_devices.exists(): + self.load_device(nautobot_location_devices, location) - # Load Site Children - Vlans, if any. - nautobot_site_vlans = VLAN.objects.filter(site=site_record) - if not nautobot_site_vlans.exists(): + # Load Location Children - Vlans, if any. + nautobot_location_vlans = VLAN.objects.filter(location=location_record) + if not nautobot_location_vlans.exists(): continue - self.load_vlans(nautobot_site_vlans, location) - except Site.DoesNotExist: - self.job.log_info(message=f"Unable to find Site, {site_record}.") + self.load_vlans(nautobot_location_vlans, location) + except Location.DoesNotExist: + self.job.log_info(message=f"Unable to find Location, {location_record}.") else: self.job.log_warning(message="No Nautobot records to load.") diff --git a/nautobot_ssot/integrations/ipfabric/diffsync/diffsync_models.py b/nautobot_ssot/integrations/ipfabric/diffsync/diffsync_models.py index 501704ed2..ef2024e38 100644 --- a/nautobot_ssot/integrations/ipfabric/diffsync/diffsync_models.py +++ b/nautobot_ssot/integrations/ipfabric/diffsync/diffsync_models.py @@ -9,11 +9,11 @@ from django.core.exceptions import ValidationError from django.db.models import Q from nautobot.dcim.models import Device as NautobotDevice -from nautobot.dcim.models import DeviceRole, DeviceType, Site -from nautobot.extras.models import Tag +from nautobot.dcim.models import DeviceType, Location as NautobotLocation +from nautobot.extras.models import Role, Tag from nautobot.extras.models.statuses import Status from nautobot.ipam.models import VLAN -from nautobot.utilities.choices import ColorChoices +from nautobot.core.choices import ColorChoices import nautobot_ssot.integrations.ipfabric.utilities.nbutils as tonb_nbutils from nautobot_ssot.integrations.ipfabric.constants import ( @@ -22,7 +22,7 @@ DEFAULT_DEVICE_STATUS, DEFAULT_DEVICE_STATUS_COLOR, DEFAULT_INTERFACE_MAC, - SAFE_DELETE_SITE_STATUS, + SAFE_DELETE_LOCATION_STATUS, SAFE_DELETE_DEVICE_STATUS, SAFE_DELETE_IPADDRESS_STATUS, SAFE_DELETE_VLAN_STATUS, @@ -47,7 +47,7 @@ def safe_delete(self, nautobot_object: Any, safe_delete_status: Optional[str] = message=f"{nautobot_object} will be deleted as safe delete mode is not enabled." ) # This allows private class naming of nautobot objects to be ordered for delete() - # Example definition in adapter class var: _site = Site + # Example definition in adapter class var: _site = Location self.diffsync.objects_to_delete[f"_{nautobot_object.__class__.__name__.lower()}"].append( nautobot_object ) # pylint: disable=protected-access @@ -67,7 +67,6 @@ def safe_delete(self, nautobot_object: Any, safe_delete_status: Optional[str] = self.diffsync.job.log_warning(message=f"{nautobot_object} has no Status attribute.") if hasattr(nautobot_object, "tags"): ssot_safe_tag, _ = Tag.objects.get_or_create( - slug="ssot-safe-delete", name="SSoT Safe Delete", defaults={ "description": "Safe Delete Mode tag to flag an object, but not delete from Nautobot.", @@ -106,34 +105,34 @@ class Location(DiffSyncExtras): @classmethod def create(cls, diffsync, ids, attrs): - """Create Site in Nautobot.""" - tonb_nbutils.create_site(site_name=ids["name"], site_id=attrs["site_id"]) + """Create Location in Nautobot.""" + tonb_nbutils.create_location(location_name=ids["name"], location_id=attrs["site_id"]) return super().create(ids=ids, diffsync=diffsync, attrs=attrs) def delete(self) -> Optional["DiffSyncModel"]: - """Delete Site in Nautobot.""" - site_object = Site.objects.get(name=self.name) + """Delete Location in Nautobot.""" + location = NautobotLocation.objects.get(name=self.name) self.safe_delete( - site_object, - SAFE_DELETE_SITE_STATUS, + location, + SAFE_DELETE_LOCATION_STATUS, ) return super().delete() def update(self, attrs): - """Update Site Object in Nautobot.""" - site = Site.objects.get(name=self.name) + """Update Location Object in Nautobot.""" + location = NautobotLocation.objects.get(name=self.name) if attrs.get("site_id"): - site.custom_field_data["ipfabric-site-id"] = attrs.get("site_id") - site.validated_save() + location.custom_field_data["ipfabric-site-id"] = attrs.get("site_id") + location.validated_save() if attrs.get("status") == "Active": safe_delete_tag, _ = Tag.objects.get_or_create(name="SSoT Safe Delete") - if not site.status == "Active": - site.status = Status.objects.get(name="Active") - device_tags = site.tags.filter(pk=safe_delete_tag.pk) + if not location.status == "Active": + location.status = Status.objects.get(name="Active") + device_tags = location.tags.filter(pk=safe_delete_tag.pk) if device_tags.exists(): - site.tags.remove(safe_delete_tag) - tonb_nbutils.tag_object(nautobot_object=site, custom_field="ssot-synced-from-ipfabric") + location.tags.remove(safe_delete_tag) + tonb_nbutils.tag_object(nautobot_object=location, custom_field="ssot-synced-from-ipfabric") return super().update(attrs) @@ -159,18 +158,18 @@ class Device(DiffSyncExtras): @classmethod def create(cls, diffsync, ids, attrs): - """Create Device in Nautobot under its parent site.""" + """Create Device in Nautobot under its parent location.""" # Get DeviceType - device_type_filter = DeviceType.objects.filter(slug=attrs["model"]) + device_type_filter = DeviceType.objects.filter(name=attrs["model"]) if device_type_filter.exists(): device_type_object = device_type_filter.first() else: device_type_object = tonb_nbutils.create_device_type_object( device_type=attrs["model"], vendor_name=attrs["vendor"] ) - # Get DeviceRole, update if missing cf and create otherwise + # Get Role, update if missing cf and create otherwise role_name = attrs.get("role", DEFAULT_DEVICE_ROLE) - device_role_filter = DeviceRole.objects.filter(name=role_name) + device_role_filter = Role.objects.filter(name=role_name) if device_role_filter.exists(): device_role_object = device_role_filter.first() device_role_object.cf["ipfabric_type"] = role_name @@ -185,12 +184,12 @@ def create(cls, diffsync, ids, attrs): device_status_object = device_status_filter.first() else: device_status_object = tonb_nbutils.create_status(DEFAULT_DEVICE_STATUS, DEFAULT_DEVICE_STATUS_COLOR) - # Get Site - site_object_filter = Site.objects.filter(name=attrs["location_name"]) - if site_object_filter.exists(): - site_object = site_object_filter.first() + # Get Location + location_object_filter = NautobotLocation.objects.filter(name=attrs["location_name"]) + if location_object_filter.exists(): + location = location_object_filter.first() else: - site_object = tonb_nbutils.create_site(attrs["location_name"]) + location = tonb_nbutils.create_location(attrs["location_name"]) new_device, _ = NautobotDevice.objects.get_or_create( name=ids["name"], @@ -198,7 +197,7 @@ def create(cls, diffsync, ids, attrs): status=device_status_object, device_type=device_type_object, device_role=device_role_object, - site=site_object, + location=location, ) try: # Validated save happens inside of tag_objet @@ -240,8 +239,8 @@ def update(self, attrs): ) _device.type = device_type_object if attrs.get("location_name"): - site_object = tonb_nbutils.create_site(attrs["location_name"]) - _device.site = site_object + location = tonb_nbutils.create_location(attrs["location_name"]) + _device.location = location if attrs.get("serial_number"): _device.serial = attrs.get("serial_number") if attrs.get("role"): @@ -295,7 +294,7 @@ class Interface(DiffSyncExtras): def create(cls, diffsync, ids, attrs): """Create interface in Nautobot under its parent device.""" ssot_tag, _ = Tag.objects.get_or_create(name="SSoT Synced from IPFabric") - device_obj = NautobotDevice.objects.filter(Q(name=ids["device_name"]) & Q(tags__slug=ssot_tag.slug)).first() + device_obj = NautobotDevice.objects.filter(Q(name=ids["device_name"]) & Q(tags__name=ssot_tag.name)).first() if not attrs.get("mac_address"): attrs["mac_address"] = DEFAULT_INTERFACE_MAC @@ -328,7 +327,7 @@ def delete(self) -> Optional["DiffSyncModel"]: """Delete Interface Object.""" try: ssot_tag, _ = Tag.objects.get_or_create(name="SSoT Synced from IPFabric") - device = NautobotDevice.objects.filter(Q(name=self.device_name) & Q(tags__slug=ssot_tag.slug)).first() + device = NautobotDevice.objects.filter(Q(name=self.device_name) & Q(tags__name=ssot_tag.name)).first() if not device: return interface = device.interfaces.get(name=self.name) @@ -348,7 +347,7 @@ def update(self, attrs): # pylint: disable=too-many-branches """Update Interface object in Nautobot.""" try: ssot_tag, _ = Tag.objects.get_or_create(name="SSoT Synced from IPFabric") - device = NautobotDevice.objects.filter(Q(name=self.device_name) & Q(tags__slug=ssot_tag.slug)).first() + device = NautobotDevice.objects.filter(Q(name=self.device_name) & Q(tags__name=ssot_tag.name)).first() interface = device.interfaces.get(name=self.name) if attrs.get("description"): interface.description = attrs["description"] @@ -413,7 +412,7 @@ class Vlan(DiffSyncExtras): def create(cls, diffsync, ids, attrs): """Create VLANs in Nautobot under the site.""" status = attrs["status"].lower().capitalize() - site = Site.objects.get(name=ids["site"]) + location = NautobotLocation.objects.get(name=ids["site"]) name = ids["name"] if ids["name"] else f"VLAN{attrs['vid']}" description = attrs["description"] if attrs["description"] else None diffsync.job.log_debug(message=f"Creating VLAN: {name} description: {description}") @@ -421,7 +420,7 @@ def create(cls, diffsync, ids, attrs): vlan_name=name, vlan_id=attrs["vid"], vlan_status=status, - site_obj=site, + location_obj=location, description=description, ) return super().create(ids=ids, diffsync=diffsync, attrs=attrs) @@ -437,7 +436,7 @@ def delete(self) -> Optional["DiffSyncModel"]: def update(self, attrs): """Update VLAN object in Nautobot.""" - vlan = VLAN.objects.get(name=self.name, vid=self.vid, site=Site.objects.get(name=self.site)) + vlan = VLAN.objects.get(name=self.name, vid=self.vid, location=NautobotLocation.objects.get(name=self.site)) if attrs.get("status") == "Active": safe_delete_tag, _ = Tag.objects.get_or_create(name="SSoT Safe Delete") diff --git a/nautobot_ssot/integrations/ipfabric/jobs.py b/nautobot_ssot/integrations/ipfabric/jobs.py index 41dd1c87f..609231931 100644 --- a/nautobot_ssot/integrations/ipfabric/jobs.py +++ b/nautobot_ssot/integrations/ipfabric/jobs.py @@ -10,9 +10,9 @@ from django.urls import reverse from httpx import ConnectError from ipfabric import IPFClient -from nautobot.dcim.models import Site +from nautobot.dcim.models import Location from nautobot.extras.jobs import BooleanVar, Job, ScriptVariable, ChoiceVar -from nautobot.utilities.forms import DynamicModelChoiceField +from nautobot.core.forms import DynamicModelChoiceField from nautobot_ssot.jobs.base import DataMapping, DataSource from nautobot_ssot.integrations.ipfabric.diffsync.adapter_ipfabric import IPFabricDiffSync @@ -73,6 +73,7 @@ class OptionalObjectVar(ScriptVariable): An object primary key is returned and accessible in job kwargs. """ + kwargs = {} form_field = DynamicModelChoiceField def __init__( @@ -118,11 +119,12 @@ class IpFabricDataSource(DataSource, Job): label="Sync Tagged Only", description="Only sync objects that have the 'ssot-synced-from-ipfabric' tag.", ) - site_filter = OptionalObjectVar( - description="Only sync Nautobot records belonging to a single Site. This does not filter IPFabric data.", - model=Site, + location_filter = OptionalObjectVar( + description="Only sync Nautobot records belonging to a single Location. This does not filter IPFabric data.", + model=Location, required=False, ) + kwargs = {} class Meta: """Metadata about this Job.""" @@ -136,7 +138,7 @@ class Meta: "snapshot", "safe_delete_mode", "sync_ipfabric_tagged_only", - "dry_run", + "dryrun", ) @staticmethod @@ -189,7 +191,7 @@ def data_mappings(cls): """List describing the data mappings involved in this DataSource.""" return ( DataMapping("Device", None, "Device", reverse("dcim:device_list")), - DataMapping("Site", None, "Site", reverse("dcim:site_list")), + DataMapping("Location", None, "Location", reverse("dcim:location_list")), DataMapping("Interfaces", None, "Interfaces", reverse("dcim:interface_list")), DataMapping("IP Addresses", None, "IP Addresses", reverse("ipam:ipaddress_list")), DataMapping("VLANs", None, "VLANs", reverse("ipam:vlan_list")), @@ -208,15 +210,35 @@ def config_information(cls): "Allow Duplicate Addresses": constants.ALLOW_DUPLICATE_ADDRESSES, "Default MTU": constants.DEFAULT_INTERFACE_MTU, "Safe Delete Device Status": constants.SAFE_DELETE_DEVICE_STATUS, - "Safe Delete Site Status": constants.SAFE_DELETE_SITE_STATUS, + "Safe Delete Location Status": constants.SAFE_DELETE_LOCATION_STATUS, "Safe Delete IPAddress Status": constants.SAFE_IPADDRESS_INTERFACES_STATUS, "Safe Delete VLAN status": constants.SAFE_DELETE_VLAN_STATUS, } - def log_debug(self, message): - """Conditionally log a debug message.""" - if self.kwargs.get("debug"): - super().log_debug(message) + # pylint: disable-next=too-many-arguments + def run( + self, + dryrun, + memory_profiling, + snapshot=None, + safe_delete_mode=True, + sync_ipfabric_tagged_only=True, + location_filter=None, + debug=False, + *args, + **kwargs, + ): + """Run the job.""" + self.kwargs = { + "snapshot": snapshot, + "dryrun": dryrun, + "safe_delete_mode": safe_delete_mode, + "sync_ipfabric_tagged_only": sync_ipfabric_tagged_only, + "location_filter": location_filter, + "debug": debug, + } + + super().run(dryrun=dryrun, memory_profiling=memory_profiling, *args, **kwargs) def load_source_adapter(self): """Not used.""" @@ -224,30 +246,30 @@ def load_source_adapter(self): def load_target_adapter(self): """Not used.""" - def sync_data(self): + def sync_data(self, *_args, **_kwargs): """Sync a device data from IP Fabric into Nautobot.""" if self.client is None: self.client = self._init_ipf_client() if self.client is None: - self.log_failure(message="IPFabric client is not ready. Check your config.") + self.logger.error("IPFabric client is not ready. Check your config.") return self.client.snapshot_id = self.kwargs["snapshot"] - dry_run = self.kwargs["dry_run"] + dryrun = self.kwargs["dryrun"] safe_mode = self.kwargs["safe_delete_mode"] tagged_only = self.kwargs["sync_ipfabric_tagged_only"] - site_filter = self.kwargs["site_filter"] + location_filter = self.kwargs["location_filter"] debug_mode = self.kwargs["debug"] - if site_filter: - site_filter_object = Site.objects.get(pk=site_filter) + if location_filter: + location_filter_object = Location.objects.get(pk=location_filter) else: - site_filter_object = None - options = f"`Snapshot_id`: {self.client.snapshot_id}.`Debug`: {debug_mode}, `Dry Run`: {dry_run}, `Safe Delete Mode`: {safe_mode}, `Sync Tagged Only`: {tagged_only}, `Site Filter`: {site_filter_object}" - self.log_info(message=f"Starting job with the following options: {options}") + location_filter_object = None + options = f"`Snapshot_id`: {self.client.snapshot_id}.`Debug`: {debug_mode}, `Dry Run`: {dryrun}, `Safe Delete Mode`: {safe_mode}, `Sync Tagged Only`: {tagged_only}, `Location Filter`: {location_filter_object}" + self.logger.info(f"Starting job with the following options: {options}") ipfabric_source = IPFabricDiffSync(job=self, sync=self.sync, client=self.client) - self.log_info(message="Loading current data from IP Fabric...") + self.logger.info("Loading current data from IP Fabric...") ipfabric_source.load() # Set safe mode either way (Defaults to True) @@ -258,17 +280,18 @@ def sync_data(self): job=self, sync=self.sync, sync_ipfabric_tagged_only=tagged_only, - site_filter=site_filter_object, + location_filter=location_filter_object, ) - self.log_info(message="Loading current data from Nautobot...") + self.logger.info("Loading current data from Nautobot...") dest.load() - self.log_info(message="Calculating diffs...") + self.logger.info("Calculating diffs...") flags = DiffSyncFlags.CONTINUE_ON_FAILURE diff = dest.diff_from(ipfabric_source, flags=flags) - self.log_debug(message=f"Diff: {diff.dict()}") + # pylint: disable-next=logging-fstring-interpolation + self.logger.debug(f"Diff: {diff.dict()}") self.sync.diff = diff.dict() self.sync.save() @@ -276,17 +299,17 @@ def sync_data(self): update = diff.summary().get("update") delete = diff.summary().get("delete") no_change = diff.summary().get("no-change") - self.log_info( - message=f"DiffSync Summary: Create: {create}, Update: {update}, Delete: {delete}, No Change: {no_change}" + self.logger.info( + f"DiffSync Summary: Create: {create}, Update: {update}, Delete: {delete}, No Change: {no_change}" ) - if not dry_run: - self.log_info(message="Syncing from IP Fabric to Nautobot") + if not dryrun: + self.logger.info("Syncing from IP Fabric to Nautobot") try: dest.sync_from(ipfabric_source) - except ObjectNotCreated as err: - self.log_debug(f"Unable to create object. {err}") + except ObjectNotCreated: + self.logger.debug("Unable to create object.", exc_info=True) - self.log_success(message="Sync complete.") + self.logger.info("Sync complete.") jobs = [IpFabricDataSource] diff --git a/nautobot_ssot/integrations/ipfabric/signals.py b/nautobot_ssot/integrations/ipfabric/signals.py index cf4d53fdc..8eb22c15c 100644 --- a/nautobot_ssot/integrations/ipfabric/signals.py +++ b/nautobot_ssot/integrations/ipfabric/signals.py @@ -5,7 +5,7 @@ from nautobot.core.signals import nautobot_database_ready from nautobot.extras.choices import CustomFieldTypeChoices -from nautobot.utilities.choices import ColorChoices +from nautobot.core.choices import ColorChoices def register_signals(sender): @@ -28,7 +28,7 @@ def create_custom_field(field_name: str, label: str, models: List, apps, cf_type if cf_type == "type_date": custom_field, _ = CustomField.objects.get_or_create( type=CustomFieldTypeChoices.TYPE_DATE, - name=field_name, + label=field_name, defaults={ "label": label, }, @@ -36,7 +36,7 @@ def create_custom_field(field_name: str, label: str, models: List, apps, cf_type else: custom_field, _ = CustomField.objects.get_or_create( type=CustomFieldTypeChoices.TYPE_TEXT, - name=field_name, + label=field_name, defaults={ "label": label, }, @@ -51,16 +51,15 @@ def nautobot_database_ready_callback(sender, *, apps, **kwargs): # pylint: disa # pylint: disable=invalid-name Device = apps.get_model("dcim", "Device") DeviceType = apps.get_model("dcim", "DeviceType") - DeviceRole = apps.get_model("dcim", "DeviceRole") + Role = apps.get_model("extras", "Role") Interface = apps.get_model("dcim", "Interface") IPAddress = apps.get_model("ipam", "IPAddress") Manufacturer = apps.get_model("dcim", "Manufacturer") - Site = apps.get_model("dcim", "Site") + Location = apps.get_model("dcim", "Location") VLAN = apps.get_model("ipam", "VLAN") Tag = apps.get_model("extras", "Tag") Tag.objects.get_or_create( - slug="ssot-synced-from-ipfabric", name="SSoT Synced from IPFabric", defaults={ "description": "Object synced at some point from IPFabric to Nautobot", @@ -68,15 +67,14 @@ def nautobot_database_ready_callback(sender, *, apps, **kwargs): # pylint: disa }, ) Tag.objects.get_or_create( - slug="ssot-safe-delete", name="SSoT Safe Delete", defaults={ "description": "Safe Delete Mode tag to flag an object, but not delete from Nautobot.", "color": ColorChoices.COLOR_RED, }, ) - synced_from_models = [Device, DeviceType, Interface, Manufacturer, Site, VLAN, DeviceRole, IPAddress] + synced_from_models = [Device, DeviceType, Interface, Manufacturer, Location, VLAN, Role, IPAddress] create_custom_field("ssot-synced-from-ipfabric", "Last synced from IPFabric on", synced_from_models, apps=apps) - site_model = [Site] - create_custom_field("ipfabric-site-id", "IPFabric Site ID", site_model, apps=apps, cf_type="type_text") - create_custom_field("ipfabric_type", "IPFabric Type", [DeviceRole], apps=apps, cf_type="type_text") + location_model = [Location] + create_custom_field("ipfabric-site-id", "IPFabric Location ID", location_model, apps=apps, cf_type="type_text") + create_custom_field("ipfabric_type", "IPFabric Type", [Role], apps=apps, cf_type="type_text") diff --git a/nautobot_ssot/integrations/ipfabric/utilities/__init__.py b/nautobot_ssot/integrations/ipfabric/utilities/__init__.py index 87583e875..5db34ba1f 100644 --- a/nautobot_ssot/integrations/ipfabric/utilities/__init__.py +++ b/nautobot_ssot/integrations/ipfabric/utilities/__init__.py @@ -5,14 +5,14 @@ create_interface, create_ip, create_manufacturer, - create_site, + create_location, create_status, create_vlan, ) from .test_utils import clean_slate, json_fixture __all__ = ( - "create_site", + "create_location", "create_device_type_object", "create_manufacturer", "get_or_create_device_role_object", diff --git a/nautobot_ssot/integrations/ipfabric/utilities/nbutils.py b/nautobot_ssot/integrations/ipfabric/utilities/nbutils.py index 9adbc0a8e..c430cb03b 100644 --- a/nautobot_ssot/integrations/ipfabric/utilities/nbutils.py +++ b/nautobot_ssot/integrations/ipfabric/utilities/nbutils.py @@ -5,47 +5,44 @@ from django.contrib.contenttypes.models import ContentType from django.db import IntegrityError -from django.utils.text import slugify from nautobot.dcim.models import ( Device, - DeviceRole, DeviceType, Interface, Manufacturer, - Site, + Location, ) from nautobot.extras.choices import CustomFieldTypeChoices -from nautobot.extras.models import CustomField, Tag +from nautobot.extras.models import CustomField, Role, Tag from nautobot.extras.models.statuses import Status from nautobot.ipam.models import VLAN, IPAddress -from nautobot.utilities.choices import ColorChoices +from nautobot.core.choices import ColorChoices from netutils.ip import netmask_to_cidr from nautobot_ssot.integrations.ipfabric.constants import ALLOW_DUPLICATE_ADDRESSES -def create_site(site_name, site_id=None): - """Creates a specified site in Nautobot. +def create_location(location_name, location_id=None): + """Creates a specified location in Nautobot. Args: - site_name (str): Name of the site. - site_id (str): ID of the site. + location_name (str): Name of the location. + location_id (str): ID of the location. """ - site_obj, _ = Site.objects.get_or_create(name=site_name) - site_obj.slug = slugify(site_name) - site_obj.status = Status.objects.get(name="Active") - if site_id: + location_obj, _ = Location.objects.get_or_create(name=location_name) + location_obj.status = Status.objects.get(name="Active") + if location_id: # Ensure custom field is available custom_field_obj, _ = CustomField.objects.get_or_create( type=CustomFieldTypeChoices.TYPE_TEXT, - name="ipfabric-site-id", - defaults={"label": "IPFabric Site ID"}, + label="ipfabric-site-id", + defaults={"label": "IPFabric Location ID"}, ) - custom_field_obj.content_types.add(ContentType.objects.get_for_model(Site)) - site_obj.cf["ipfabric-site-id"] = site_id - site_obj.validated_save() - tag_object(nautobot_object=site_obj, custom_field="ssot-synced-from-ipfabric") - return site_obj + custom_field_obj.content_types.add(ContentType.objects.get_for_model(Location)) + location_obj.cf["ipfabric-site-id"] = location_id + location_obj.validated_save() + tag_object(nautobot_object=location_obj, custom_field="ssot-synced-from-ipfabric") + return location_obj def create_manufacturer(vendor_name): @@ -63,9 +60,7 @@ def create_device_type_object(device_type, vendor_name): vendor_name (str): Vendor Name """ mf_name = create_manufacturer(vendor_name) - device_type_obj, _ = DeviceType.objects.get_or_create( - manufacturer=mf_name, model=device_type, slug=slugify(device_type) - ) + device_type_obj, _ = DeviceType.objects.get_or_create(manufacturer=mf_name, model=device_type) tag_object(nautobot_object=device_type_obj, custom_field="ssot-synced-from-ipfabric") return device_type_obj @@ -79,11 +74,12 @@ def get_or_create_device_role_object(role_name, role_color): """ # adds custom field to map custom role names to ipfabric type names try: - role_obj = DeviceRole.objects.get(_custom_field_data__ipfabric_type=role_name) - except DeviceRole.DoesNotExist: - role_obj = DeviceRole.objects.create(name=role_name, slug=slugify(role_name), color=role_color) + role_obj = Role.objects.get(_custom_field_data__ipfabric_type=role_name) + except Role.DoesNotExist: + role_obj = Role.objects.create(name=role_name, color=role_color) role_obj.cf["ipfabric_type"] = role_name role_obj.validated_save() + role_obj.content_types.set([ContentType.objects.get_for_model(Device)]) tag_object(nautobot_object=role_obj, custom_field="ssot-synced-from-ipfabric") return role_obj @@ -104,7 +100,6 @@ def create_status(status_name, status_color, description="", app_label="dcim", m content_type = ContentType.objects.get(app_label=app_label, model=model) status_obj = Status.objects.create( name=status_name, - slug=slugify(status_name), color=status_color, description=description, ) @@ -123,7 +118,7 @@ def create_ip(ip_address, subnet_mask, status="Active", object_pk=None): status (str): Status to assign to IP Address. object_pk: Object primary key """ - status_obj = Status.objects.get_for_model(IPAddress).get(slug=slugify(status)) + status_obj = Status.objects.get_for_model(IPAddress).get(name=status) cidr = netmask_to_cidr(subnet_mask) if ALLOW_DUPLICATE_ADDRESSES: addr = IPAddress.objects.filter(host=ip_address) @@ -179,20 +174,20 @@ def create_interface(device_obj, interface_details): return interface_obj -def create_vlan(vlan_name: str, vlan_id: int, vlan_status: str, site_obj: Site, description: str): +def create_vlan(vlan_name: str, vlan_id: int, vlan_status: str, location_obj: Location, description: str): """Creates or obtains VLAN object. Args: vlan_name (str): VLAN Name vlan_id (int): VLAN ID vlan_status (str): VLAN Status - site_obj (Site): Site Django Model + location_obj (Location): Location Django Model description (str): VLAN Description Returns: (VLAN): Returns created or obtained VLAN object. """ - vlan_obj, _ = site_obj.vlans.get_or_create( + vlan_obj, _ = location_obj.vlans.get_or_create( name=vlan_name, vid=vlan_id, status=Status.objects.get(name=vlan_status), description=description ) tag_object(nautobot_object=vlan_obj, custom_field="ssot-synced-from-ipfabric") @@ -209,7 +204,6 @@ def tag_object(nautobot_object: Any, custom_field: str, tag_name: Optional[str] """ if tag_name == "SSoT Synced from IPFabric": tag, _ = Tag.objects.get_or_create( - slug="ssot-synced-from-ipfabric", name="SSoT Synced from IPFabric", defaults={ "description": "Object synced at some point from IPFabric to Nautobot", @@ -227,10 +221,10 @@ def _tag_object(nautobot_object): nautobot_object.tags.add(tag) if hasattr(nautobot_object, "cf"): # Ensure that the "ssot-synced-from-ipfabric" custom field is present - if not any(cfield for cfield in CustomField.objects.all() if cfield.name == "ssot-synced-from-ipfabric"): + if not any(cfield for cfield in CustomField.objects.all() if cfield.label == "ssot-synced-from-ipfabric"): custom_field_obj, _ = CustomField.objects.get_or_create( type=CustomFieldTypeChoices.TYPE_DATE, - name="ssot-synced-from-ipfabric", + label="ssot-synced-from-ipfabric", defaults={ "label": "Last synced from IPFabric on", }, @@ -240,9 +234,9 @@ def _tag_object(nautobot_object): DeviceType, Interface, Manufacturer, - Site, + Location, VLAN, - DeviceRole, + Role, IPAddress, ] for model in synced_from_models: diff --git a/nautobot_ssot/integrations/ipfabric/utilities/test_utils.py b/nautobot_ssot/integrations/ipfabric/utilities/test_utils.py index af159a1be..fb672dbfc 100644 --- a/nautobot_ssot/integrations/ipfabric/utilities/test_utils.py +++ b/nautobot_ssot/integrations/ipfabric/utilities/test_utils.py @@ -1,8 +1,8 @@ """Test Utils.""" import json -from nautobot.dcim.models.sites import Site -from nautobot.ipam.models import VLAN, Device +from nautobot.dcim.models import Device, Location +from nautobot.ipam.models import VLAN def json_fixture(json_file_path): @@ -18,4 +18,4 @@ def clean_slate(): """ VLAN.objects.all().delete() Device.objects.all().delete() - Site.objects.all().delete() + Location.objects.all().delete() diff --git a/nautobot_ssot/integrations/ipfabric/workers.py b/nautobot_ssot/integrations/ipfabric/workers.py index d2bab1a34..c9dbeb9b2 100644 --- a/nautobot_ssot/integrations/ipfabric/workers.py +++ b/nautobot_ssot/integrations/ipfabric/workers.py @@ -4,7 +4,6 @@ from django.conf import settings from django.contrib.contenttypes.models import ContentType -from django_rq import job from nautobot.core.settings_funcs import is_truthy from nautobot.extras.models import JobResult @@ -52,7 +51,6 @@ def ipfabric_logo(dispatcher): return dispatcher.image_element(dispatcher.static_url(IPFABRIC_LOGO_PATH), alt_text=IPFABRIC_LOGO_ALT) -@job("default") def ipfabric(subcommand, **kwargs): """Interact with ipfabric plugin.""" return handle_subcommands("ipfabric", subcommand, **kwargs) @@ -84,7 +82,7 @@ def ssot_sync_to_nautobot( ) return (CommandStatusChoices.STATUS_SUCCEEDED, "Success") - # if site_filter is None: + # if location_filter is None: # prompt_for_site( # dispatcher, # f"{BASE_CMD} ssot-sync-to-nautobot {dry_run} {safe_delete_mode} {sync_ipfabric_tagged_only}", @@ -93,15 +91,7 @@ def ssot_sync_to_nautobot( # return (CommandStatusChoices.STATUS_SUCCEEDED, "Success") # Implement filter in future release - site_filter = False - - data = { - "debug": False, - "dry_run": is_truthy(dry_run), - "safe_delete_mode": is_truthy(safe_delete_mode), - "sync_ipfabric_tagged_only": is_truthy(sync_ipfabric_tagged_only), - "site_filter": site_filter, - } + location_filter = False sync_job = IpFabricDataSource() @@ -120,9 +110,13 @@ def ssot_sync_to_nautobot( ephemeral=True, ) - sync_job.run(data, commit=True) - sync_job.post_run() - sync_job.job_result.set_status(status="completed" if not sync_job.failed else "failed") + sync_job.run( + dryrun=is_truthy(dry_run), + memory_profiling=False, + safe_delete_mode=is_truthy(safe_delete_mode), + sync_ipfabric_tagged_only=is_truthy(sync_ipfabric_tagged_only), + location_filter=location_filter, + ) sync_job.job_result.validated_save() blocks = [ diff --git a/nautobot_ssot/tests/ipfabric/test_ipfabric_adapter.py b/nautobot_ssot/tests/ipfabric/test_ipfabric_adapter.py index e19402b0b..1e5f7f536 100644 --- a/nautobot_ssot/tests/ipfabric/test_ipfabric_adapter.py +++ b/nautobot_ssot/tests/ipfabric/test_ipfabric_adapter.py @@ -1,11 +1,9 @@ """Unit tests for the IPFabric DiffSync adapter class.""" import json -import uuid from unittest.mock import MagicMock -from django.contrib.contenttypes.models import ContentType from django.test import TestCase -from nautobot.extras.models import Job, JobResult +from nautobot.extras.models import JobResult from nautobot_ssot.integrations.ipfabric.diffsync.adapter_ipfabric import IPFabricDiffSync from nautobot_ssot.integrations.ipfabric.jobs import IpFabricDataSource @@ -39,9 +37,7 @@ def test_data_loading(self): ipfabric_client.inventory.interfaces.all.return_value = INTERFACE_FIXTURE job = IpFabricDataSource() - job.job_result = JobResult.objects.create( - name=job.class_path, obj_type=ContentType.objects.get_for_model(Job), user=None, job_id=uuid.uuid4() - ) + job.job_result = JobResult.objects.create(name=job.class_path, task_name="fake task", worker="default") ipfabric = IPFabricDiffSync(job=job, sync=None, client=ipfabric_client) ipfabric.load() self.assertEqual( diff --git a/nautobot_ssot/tests/ipfabric/test_jobs.py b/nautobot_ssot/tests/ipfabric/test_jobs.py index 1cc354212..ac355629d 100644 --- a/nautobot_ssot/tests/ipfabric/test_jobs.py +++ b/nautobot_ssot/tests/ipfabric/test_jobs.py @@ -30,10 +30,10 @@ def test_data_mapping(self): self.assertEqual("Device", mappings[0].target_name) self.assertEqual(reverse("dcim:device_list"), mappings[0].target_url) - self.assertEqual("Site", mappings[1].source_name) + self.assertEqual("Location", mappings[1].source_name) self.assertIsNone(mappings[1].source_url) - self.assertEqual("Site", mappings[1].target_name) - self.assertEqual(reverse("dcim:site_list"), mappings[1].target_url) + self.assertEqual("Location", mappings[1].target_name) + self.assertEqual(reverse("dcim:location_list"), mappings[1].target_url) self.assertEqual("Interfaces", mappings[2].source_name) self.assertIsNone(mappings[2].source_url) diff --git a/nautobot_ssot/tests/ipfabric/test_nbutils.py b/nautobot_ssot/tests/ipfabric/test_nbutils.py index 80f71b66d..3ac2ce7e5 100644 --- a/nautobot_ssot/tests/ipfabric/test_nbutils.py +++ b/nautobot_ssot/tests/ipfabric/test_nbutils.py @@ -1,14 +1,14 @@ """Test Nautobot Utilities.""" from django.contrib.contenttypes.models import ContentType from django.test import TestCase -from django.utils.text import slugify -from nautobot.dcim.models import DeviceRole, DeviceType, Manufacturer, Site +from nautobot.dcim.models import DeviceType, Manufacturer, Location, LocationType from nautobot.dcim.models.devices import Device from nautobot.extras.models.statuses import Status -from nautobot.ipam.models import VLAN, IPAddress -from nautobot.utilities.choices import ColorChoices +from nautobot.ipam.models import VLAN, IPAddress, Prefix, get_default_namespace +from nautobot.core.choices import ColorChoices +from nautobot.extras.models import Role -from nautobot_ssot.integrations.ipfabric.utilities import ( # create_ip,; create_interface,; create_site, +from nautobot_ssot.integrations.ipfabric.utilities import ( # create_ip,; create_interface,; create_location, get_or_create_device_role_object, create_device_type_object, create_manufacturer, @@ -23,39 +23,46 @@ class TestNautobotUtils(TestCase): def setUp(self): """Setup.""" - self.site = Site.objects.create( - name="Test-Site", - slug="test-site", + reg_loctype = LocationType.objects.update_or_create(name="Region")[0] + reg_loctype.content_types.set([ContentType.objects.get_for_model(VLAN)]) + self.location = Location.objects.create( + name="Test-Location", status=Status.objects.get(name="Active"), + location_type=reg_loctype, ) - self.manufacturer = Manufacturer.objects.create(name="Test-Manufacturer", slug="test-manufacturer") - self.device_type = DeviceType.objects.create( - model="Test-DeviceType", slug="test-devicetype", manufacturer=self.manufacturer - ) - self.device_role = DeviceRole.objects.create(name="Test-Role", slug="test-role", color=ColorChoices.COLOR_RED) + status_active = Status.objects.get(name="Active") + + self.manufacturer = Manufacturer.objects.create(name="Test-Manufacturer") + self.device_type = DeviceType.objects.create(model="Test-DeviceType", manufacturer=self.manufacturer) + self.content_type = ContentType.objects.get_for_model(Device) + self.device_role = Role.objects.create(name="Test-Role", color=ColorChoices.COLOR_RED) + self.device_role.content_types.set([self.content_type]) self.device_role.cf["ipfabric_type"] = "Test-Role" self.device_role.validated_save() - self.content_type = ContentType.objects.get(app_label="dcim", model="device") self.status = Status.objects.create( name="Test-Status", - slug=slugify("Test-Status"), color=ColorChoices.COLOR_AMBER, description="Test-Description", ) self.status.content_types.set([self.content_type]) - self.status_obj = Status.objects.get_for_model(IPAddress).get(slug=slugify("Active")) - self.ip_address = IPAddress.objects.create(address="192.168.0.1/32", status=self.status_obj) + prefix = Prefix.objects.get_or_create( + prefix="192.168.0.0/16", namespace=get_default_namespace(), status=status_active + )[0] + self.ip_address = IPAddress.objects.create(address="192.168.0.1/32", status=status_active, parent=prefix) self.device = Device.objects.create( - name="Test-Device", site=self.site, device_type=self.device_type, device_role=self.device_role + name="Test-Device", + location=self.location, + device_type=self.device_type, + role=self.device_role, + status=status_active, ) - self.device.interfaces.create(name="Test-Interface") + self.device.interfaces.create(name="Test-Interface", status=status_active) self.vlan_content_type = ContentType.objects.get(app_label="ipam", model="vlan") self.vlan_status = Status.objects.create( name="Test-Vlan-Status", - slug=slugify("Test-Vlan-Status"), color=ColorChoices.COLOR_AMBER, description="Test-Description", ) @@ -67,23 +74,23 @@ def test_create_vlan(self): vlan_name="Test-Vlan", vlan_id=100, vlan_status="Test-Vlan-Status", - site_obj=self.site, + location_obj=self.location, description="Test-Vlan", ) self.assertEqual(VLAN.objects.get(name="Test-Vlan").pk, vlan.pk) - # def test_create_site(self): - # """Test `create_site` Utility.""" - # test_site = create_site(site_name="Test-Site") - # self.assertEqual(test_site.id, self.site.id) + # def test_create_location(self): + # """Test `create_location` Utility.""" + # test_location = create_location(location_name="Test-Location") + # self.assertEqual(test_location.id, self.location.id) - # def test_create_site_exception(self): - # """Test `create_site` Utility exception.""" - # site = create_site( - # site_name="Test-Site-100", - # site_id=123456, + # def test_create_location_exception(self): + # """Test `create_location` Utility exception.""" + # location = create_location( + # location_name="Test-Location-100", + # location_id=123456, # ) - # self.assertEqual(Site.objects.get(name="Test-Site-100").pk, site.pk) + # self.assertEqual(Location.objects.get(name="Test-Location-100").pk, location.pk) def test_create_device_type_object(self): """Test `create_device_type_object` Utility.""" diff --git a/poetry.lock b/poetry.lock index d3e904025..51cd6983c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -4131,7 +4131,7 @@ aci = ["PyYAML"] all = ["Jinja2", "PyYAML", "cloudvision", "cvprac", "dnspython", "ijson", "ipfabric", "ipfabric-diagrams", "netutils", "oauthlib", "python-magic", "pytz", "requests", "requests-oauthlib", "six"] aristacv = ["cloudvision", "cvprac"] infoblox = ["dnspython"] -ipfabric = ["ipfabric", "ipfabric-diagrams", "netutils"] +ipfabric = ["httpx", "ipfabric", "ipfabric-diagrams", "netutils"] nautobot-device-lifecycle-mgmt = [] pysnow = ["ijson", "oauthlib", "python-magic", "pytz", "requests", "requests-oauthlib", "six"] servicenow = ["Jinja2", "PyYAML", "ijson", "oauthlib", "python-magic", "pytz", "requests", "requests-oauthlib", "six"] @@ -4139,4 +4139,4 @@ servicenow = ["Jinja2", "PyYAML", "ijson", "oauthlib", "python-magic", "pytz", " [metadata] lock-version = "2.0" python-versions = "^3.8,<3.12" -content-hash = "74bb9f18d7c7ca540f53cab1d8f0168ae3c5140ce27abb47554aa68998becbe2" +content-hash = "814f81470e03cc59f97b56eb8d33760113e3b2ffd011c6ebd135adec74b75b39" diff --git a/pyproject.toml b/pyproject.toml index 0e0b13e86..947606bde 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,6 +49,7 @@ requests = { version = ">=2.21.0", optional = true } requests-oauthlib = { version = ">=1.3.0", optional = true } six = { version = ">=1.13.0", optional = true } drf-spectacular = "0.26.3" +httpx = { version = ">=0.23.3", optional = true } [tool.poetry.dev-dependencies] bandit = "*" @@ -125,6 +126,7 @@ infoblox = [ "dnspython", ] ipfabric = [ + "httpx", "ipfabric", "ipfabric-diagrams", "netutils", From 46ff59c895531c0e8a8a4752212b5e4c75af145b Mon Sep 17 00:00:00 2001 From: alhogan <98360253+alhogan@users.noreply.github.com> Date: Wed, 27 Sep 2023 19:25:43 -0500 Subject: [PATCH 03/12] Updates for 2.0 --- .../ipfabric/diffsync/adapter_ipfabric.py | 16 +++---- .../ipfabric/diffsync/adapter_nautobot.py | 37 ++++++++-------- .../ipfabric/diffsync/diffsync_models.py | 42 +++++++++++-------- .../integrations/ipfabric/signals.py | 19 ++++----- .../ipfabric/utilities/nbutils.py | 10 +++-- 5 files changed, 66 insertions(+), 58 deletions(-) diff --git a/nautobot_ssot/integrations/ipfabric/diffsync/adapter_ipfabric.py b/nautobot_ssot/integrations/ipfabric/diffsync/adapter_ipfabric.py index 7c3afa2ca..992bc6622 100644 --- a/nautobot_ssot/integrations/ipfabric/diffsync/adapter_ipfabric.py +++ b/nautobot_ssot/integrations/ipfabric/diffsync/adapter_ipfabric.py @@ -41,7 +41,7 @@ def load_sites(self): location = self.location(diffsync=self, name=site["siteName"], site_id=site["id"], status="Active") self.add(location) except ObjectAlreadyExists: - self.job.log_debug(message=f"Duplicate Location discovered, {site}") + logger.warning(f"Duplicate Location discovered, {site}") def load_device_interfaces(self, device_model, interfaces, device_primary_ip): """Create and load DiffSync Interface model objects for a specific device.""" @@ -76,7 +76,7 @@ def load_device_interfaces(self, device_model, interfaces, device_primary_ip): self.add(interface) device_model.add_child(interface) except ObjectAlreadyExists: - self.job.log_debug(message=f"Duplicate Interface discovered, {iface}") + logger.warning(f"Duplicate Interface discovered, {iface}") def load(self): """Load data from IP Fabric.""" @@ -91,14 +91,14 @@ def load(self): location_vlans = [vlan for vlan in vlans if vlan["siteName"] == location.name] for vlan in location_vlans: if not vlan["vlanId"] or (vlan["vlanId"] < 1 or vlan["vlanId"] > 4094): - self.job.log_warning( - message=f"Not syncing VLAN, NAME: {vlan.get('vlanName')} due to invalid VLAN ID: {vlan.get('vlanId')}." + logger.warning( + f"Not syncing VLAN, NAME: {vlan.get('vlanName')} due to invalid VLAN ID: {vlan.get('vlanId')}." ) continue description = vlan.get("dscr") if vlan.get("dscr") else f"VLAN ID: {vlan['vlanId']}" vlan_name = vlan.get("vlanName") if vlan.get("vlanName") else f"{vlan['siteName']}:{vlan['vlanId']}" if len(vlan_name) > name_max_length: - self.job.log_warning( + logger.warning( message=f"Not syncing VLAN, {vlan_name} due to character limit exceeding {name_max_length}." ) continue @@ -114,7 +114,7 @@ def load(self): self.add(vlan) location.add_child(vlan) except ObjectAlreadyExists: - self.job.log_debug(message=f"Duplicate VLAN discovered, {vlan}") + logger.warning(message=f"Duplicate VLAN discovered, {vlan}") location_devices = [device for device in devices if device["siteName"] == location.name] for device in location_devices: @@ -122,7 +122,7 @@ def load(self): sn_length = len(device["sn"]) serial_number = device["sn"] if sn_length < device_serial_max_length else "" if not serial_number: - self.job.log_warning( + logger.warning( message=( f"Serial Number will not be recorded for {device['hostname']} due to character limit. " f"{sn_length} exceeds {device_serial_max_length}" @@ -143,7 +143,7 @@ def load(self): location.add_child(device_model) self.load_device_interfaces(device_model, interfaces, device_primary_ip) except ObjectAlreadyExists: - self.job.log_debug(message=f"Duplicate Device discovered, {device}") + logger.warning(message=f"Duplicate Device discovered, {device}") def pseudo_management_interface(hostname, device_interfaces, device_primary_ip): diff --git a/nautobot_ssot/integrations/ipfabric/diffsync/adapter_nautobot.py b/nautobot_ssot/integrations/ipfabric/diffsync/adapter_nautobot.py index 7bfa170f3..fd34b37be 100644 --- a/nautobot_ssot/integrations/ipfabric/diffsync/adapter_nautobot.py +++ b/nautobot_ssot/integrations/ipfabric/diffsync/adapter_nautobot.py @@ -4,6 +4,7 @@ """DiffSync adapter class for Nautobot as source-of-truth.""" from collections import defaultdict from typing import Any, ClassVar, List, Optional +import logging from diffsync import DiffSync from diffsync.exceptions import ObjectAlreadyExists @@ -23,6 +24,8 @@ DEFAULT_INTERFACE_MAC, ) +logger = logging.getLogger("nautobot.ssot.ipfabric") + class NautobotDiffSync(DiffSyncModelAdapters): """Nautobot adapter for DiffSync.""" @@ -71,11 +74,9 @@ def sync_complete(self, source: DiffSync, *args, **kwargs): try: nautobot_object.delete() except ProtectedError: - self.job.log_failure(obj=nautobot_object, message="Deletion failed protected object") + logger.warning("Deletion failed protected object", extra={"object": nautobot_object}) except IntegrityError: - self.job.log_failure( - obj=nautobot_object, message=f"Deletion failed due to IntegrityError with {nautobot_object}" - ) + logger.warning(f"Deletion failed due to IntegrityError with {nautobot_object}") self.objects_to_delete[grouping] = [] return super().sync_complete(source, *args, **kwargs) @@ -117,14 +118,15 @@ def load_interfaces(self, device_record: Device, diffsync_device): def load_device(self, filtered_devices: List, location): """Load Devices from Nautobot.""" for device_record in filtered_devices: - self.job.log_debug(message=f"Loading Nautobot Device: {device_record.name}") + if self.job.debug: + logger.debug(f"Loading Nautobot Device: {device_record.name}") device = self.device( diffsync=self, name=device_record.name, model=str(device_record.device_type), - role=str(device_record.device_role.cf.get("ipfabric_type")) - if str(device_record.device_role.cf.get("ipfabric_type")) - else str(device_record.device_role), + role=str(device_record.role.cf.get("ipfabric_type")) + if str(device_record.role.cf.get("ipfabric_type")) + else str(device_record.role), location_name=device_record.location.name, vendor=str(device_record.device_type.manufacturer), status=device_record.status.name, @@ -133,7 +135,7 @@ def load_device(self, filtered_devices: List, location): try: self.add(device) except ObjectAlreadyExists: - self.job.log_debug(message=f"Duplicate device discovered, {device_record.name}") + logger.warning(f"Duplicate device discovered, {device_record.name}") continue location.add_child(device) @@ -156,7 +158,7 @@ def load_vlans(self, filtered_vlans: List, location): try: self.add(vlan) except ObjectAlreadyExists: - self.job.log_debug(message=f"Duplicate VLAN discovered, {vlan_record.name}") + logger.warning(f"Duplicate VLAN discovered, {vlan_record.name}") continue location.add_child(vlan) @@ -174,8 +176,8 @@ def get_initial_location(self, ssot_tag: Tag): Q(name=self.location_filter.name) & Q(tags__name=ssot_tag.name) ) if not location_objects: - self.job.log_warning( - message=f"{self.location_filter.name} was used to filter, alongside SSoT Tag. {self.location_filter.name} is not tagged." + logger.warning( + f"{self.location_filter.name} was used to filter, alongside SSoT Tag. {self.location_filter.name} is not tagged." ) elif not self.sync_ipfabric_tagged_only: if self.location_filter: @@ -196,7 +198,8 @@ def load_data(self): ) location_objects = self.get_initial_location(ssot_tag) # The parent object that stores all children, is the Location. - self.job.log_debug(message=f"Found {location_objects.count()} Nautobot Location objects to start sync from") + if self.job.debug: + logger.debug(f"Found {location_objects.count()} Nautobot Location objects to start sync from") if location_objects: for location_record in location_objects: @@ -208,8 +211,8 @@ def load_data(self): status=location_record.status.name, ) except AttributeError: - self.job.log_debug( - message=f"Error loading {location_record}, invalid or missing attributes on object. Skipping..." + logger.error( + f"Error loading {location_record}, invalid or missing attributes on object. Skipping..." ) continue self.add(location) @@ -230,9 +233,9 @@ def load_data(self): continue self.load_vlans(nautobot_location_vlans, location) except Location.DoesNotExist: - self.job.log_info(message=f"Unable to find Location, {location_record}.") + logger.error(f"Unable to find Location, {location_record}.") else: - self.job.log_warning(message="No Nautobot records to load.") + logger.warning("No Nautobot records to load.") def load(self): """Load data from Nautobot.""" diff --git a/nautobot_ssot/integrations/ipfabric/diffsync/diffsync_models.py b/nautobot_ssot/integrations/ipfabric/diffsync/diffsync_models.py index ef2024e38..d265740ce 100644 --- a/nautobot_ssot/integrations/ipfabric/diffsync/diffsync_models.py +++ b/nautobot_ssot/integrations/ipfabric/diffsync/diffsync_models.py @@ -4,6 +4,7 @@ """DiffSyncModel subclasses for Nautobot-to-IPFabric data sync.""" from typing import Any, ClassVar, List, Optional from uuid import UUID +import logging from diffsync import DiffSyncModel from django.core.exceptions import ValidationError @@ -28,6 +29,8 @@ SAFE_DELETE_VLAN_STATUS, ) +logger = logging.getLogger(__name__) + class DiffSyncExtras(DiffSyncModel): """Additional components to mix and subclass from with `DiffSyncModel`.""" @@ -43,8 +46,8 @@ def safe_delete(self, nautobot_object: Any, safe_delete_status: Optional[str] = """ update = False if not self.safe_delete_mode: # This could just check self, refactor. - self.diffsync.job.log_warning( - message=f"{nautobot_object} will be deleted as safe delete mode is not enabled." + logger.warning( + f"{nautobot_object} will be deleted as safe delete mode is not enabled." ) # This allows private class naming of nautobot objects to be ordered for delete() # Example definition in adapter class var: _site = Location @@ -58,13 +61,13 @@ def safe_delete(self, nautobot_object: Any, safe_delete_status: Optional[str] = if hasattr(nautobot_object, "status"): if not nautobot_object.status == safe_delete_status: nautobot_object.status = safe_delete_status - self.diffsync.job.log_warning( - message=f"{nautobot_object} has changed status to {safe_delete_status}." + logger.warning( + f"{nautobot_object} has changed status to {safe_delete_status}." ) update = True else: # Not everything has a status. This may come in handy once more models are synced. - self.diffsync.job.log_warning(message=f"{nautobot_object} has no Status attribute.") + logger.warning(f"{nautobot_object} has no Status attribute.") if hasattr(nautobot_object, "tags"): ssot_safe_tag, _ = Tag.objects.get_or_create( name="SSoT Safe Delete", @@ -77,13 +80,13 @@ def safe_delete(self, nautobot_object: Any, safe_delete_status: Optional[str] = # No exception raised for empty iterator, safe to do this any if not any(obj_tag for obj_tag in object_tags if obj_tag.name == ssot_safe_tag.name): nautobot_object.tags.add(ssot_safe_tag) - self.diffsync.job.log_warning(message=f"Tagging {nautobot_object} with `ssot-safe-delete`.") + logger.warning(f"Tagging {nautobot_object} with `ssot-safe-delete`.") update = True if update: tonb_nbutils.tag_object(nautobot_object=nautobot_object, custom_field="ssot-synced-from-ipfabric") else: - self.diffsync.job.log_debug( - message=f"{nautobot_object} has previously been tagged with `ssot-safe-delete`. Skipping..." + logger.warning( + f"{nautobot_object} has previously been tagged with `ssot-safe-delete`. Skipping..." ) return self @@ -160,7 +163,7 @@ class Device(DiffSyncExtras): def create(cls, diffsync, ids, attrs): """Create Device in Nautobot under its parent location.""" # Get DeviceType - device_type_filter = DeviceType.objects.filter(name=attrs["model"]) + device_type_filter = DeviceType.objects.filter(model=attrs["model"]) if device_type_filter.exists(): device_type_object = device_type_filter.first() else: @@ -196,7 +199,7 @@ def create(cls, diffsync, ids, attrs): serial=attrs.get("serial_number", ""), status=device_status_object, device_type=device_type_object, - device_role=device_role_object, + role=device_role_object, location=location, ) try: @@ -204,8 +207,10 @@ def create(cls, diffsync, ids, attrs): tonb_nbutils.tag_object(nautobot_object=new_device, custom_field="ssot-synced-from-ipfabric") except ValidationError as error: message = f"Unable to create device: {ids['name']}. A validation error occured. Enable debug for more information." - diffsync.job.log_debug(message=error) - diffsync.job.log_failure(message=message) + if diffsync.job.debug: + logger.debug(error) + logger.error(message) + raise Exception("A validation error occured.") return super().create(ids=ids, diffsync=diffsync, attrs=attrs) @@ -219,7 +224,7 @@ def delete(self) -> Optional["DiffSyncModel"]: ) return super().delete() except NautobotDevice.DoesNotExist: - self.diffsync.job.log_warning(f"Unable to match device by name, {self.name}") + logger.warning(f"Unable to match device by name, {self.name}") def update(self, attrs): """Update devices in Nautobot based on Source.""" @@ -252,7 +257,7 @@ def update(self, attrs): # Call the super().update() method to update the in-memory DiffSyncModel instance return super().update(attrs) except NautobotDevice.DoesNotExist: - self.diffsync.job.log_warning(f"Unable to match device by name, {self.name}") + logger.warning(f"Unable to match device by name, {self.name}") class Interface(DiffSyncExtras): @@ -341,7 +346,7 @@ def delete(self) -> Optional["DiffSyncModel"]: ) return super().delete() except NautobotDevice.DoesNotExist: - self.diffsync.job.log_warning(f"Unable to match device by name, {self.name}") + logger.warning(f"Unable to match device by name, {self.name}") def update(self, attrs): # pylint: disable=too-many-branches """Update Interface object in Nautobot.""" @@ -367,7 +372,7 @@ def update(self, attrs): # pylint: disable=too-many-branches interface.mgmt_only = attrs["mgmt_only"] if attrs.get("ip_address"): if interface.ip_addresses.all().exists(): - self.diffsync.job.log_debug(message=f"Replacing IP from interface {interface} on {device.name}") + logger.info(f"Replacing IP from interface {interface} on {device.name}") interface.ip_addresses.all().delete() ip_address_obj = tonb_nbutils.create_ip( ip_address=attrs.get("ip_address"), @@ -390,7 +395,7 @@ def update(self, attrs): # pylint: disable=too-many-branches return super().update(attrs) except NautobotDevice.DoesNotExist: - self.diffsync.job.log_warning(f"Unable to match device by name, {self.name}") + logger.warning(f"Unable to match device by name, {self.name}") class Vlan(DiffSyncExtras): @@ -415,7 +420,8 @@ def create(cls, diffsync, ids, attrs): location = NautobotLocation.objects.get(name=ids["site"]) name = ids["name"] if ids["name"] else f"VLAN{attrs['vid']}" description = attrs["description"] if attrs["description"] else None - diffsync.job.log_debug(message=f"Creating VLAN: {name} description: {description}") + if diffsync.job.debug: + logger.debug(f"Creating VLAN: {name} description: {description}") tonb_nbutils.create_vlan( vlan_name=name, vlan_id=attrs["vid"], diff --git a/nautobot_ssot/integrations/ipfabric/signals.py b/nautobot_ssot/integrations/ipfabric/signals.py index 8eb22c15c..6f389433c 100644 --- a/nautobot_ssot/integrations/ipfabric/signals.py +++ b/nautobot_ssot/integrations/ipfabric/signals.py @@ -13,11 +13,11 @@ def register_signals(sender): nautobot_database_ready.connect(nautobot_database_ready_callback, sender=sender) -def create_custom_field(field_name: str, label: str, models: List, apps, cf_type: Optional[str] = "type_date"): +def create_custom_field(key: str, label: str, models: List, apps, cf_type: Optional[str] = "type_date"): """Create custom field on a given model instance type. Args: - field_name (str): Field Name + key (str): Natural key label (str): Label description models (List): List of Django Models apps: Django Apps @@ -27,19 +27,15 @@ def create_custom_field(field_name: str, label: str, models: List, apps, cf_type CustomField = apps.get_model("extras", "CustomField") # pylint:disable=invalid-name if cf_type == "type_date": custom_field, _ = CustomField.objects.get_or_create( + key=key, type=CustomFieldTypeChoices.TYPE_DATE, - label=field_name, - defaults={ - "label": label, - }, + label=label, ) else: custom_field, _ = CustomField.objects.get_or_create( + key=key, type=CustomFieldTypeChoices.TYPE_TEXT, - label=field_name, - defaults={ - "label": label, - }, + label=label, ) for model in models: custom_field.content_types.add(ContentType.objects.get_for_model(model)) @@ -75,6 +71,5 @@ def nautobot_database_ready_callback(sender, *, apps, **kwargs): # pylint: disa ) synced_from_models = [Device, DeviceType, Interface, Manufacturer, Location, VLAN, Role, IPAddress] create_custom_field("ssot-synced-from-ipfabric", "Last synced from IPFabric on", synced_from_models, apps=apps) - location_model = [Location] - create_custom_field("ipfabric-site-id", "IPFabric Location ID", location_model, apps=apps, cf_type="type_text") + create_custom_field("ipfabric_site_id", "IPFabric Location ID", [Location], apps=apps, cf_type="type_text") create_custom_field("ipfabric_type", "IPFabric Type", [Role], apps=apps, cf_type="type_text") diff --git a/nautobot_ssot/integrations/ipfabric/utilities/nbutils.py b/nautobot_ssot/integrations/ipfabric/utilities/nbutils.py index c430cb03b..20cfcc52e 100644 --- a/nautobot_ssot/integrations/ipfabric/utilities/nbutils.py +++ b/nautobot_ssot/integrations/ipfabric/utilities/nbutils.py @@ -11,11 +11,12 @@ Interface, Manufacturer, Location, + LocationType, ) from nautobot.extras.choices import CustomFieldTypeChoices from nautobot.extras.models import CustomField, Role, Tag from nautobot.extras.models.statuses import Status -from nautobot.ipam.models import VLAN, IPAddress +from nautobot.ipam.models import VLAN, IPAddress, Namespace from nautobot.core.choices import ColorChoices from netutils.ip import netmask_to_cidr @@ -29,8 +30,7 @@ def create_location(location_name, location_id=None): location_name (str): Name of the location. location_id (str): ID of the location. """ - location_obj, _ = Location.objects.get_or_create(name=location_name) - location_obj.status = Status.objects.get(name="Active") + location_obj, _ = Location.objects.get_or_create(name=location_name, location_type=LocationType.objects.get(name="Site"), status=Status.objects.get(name="Active")) if location_id: # Ensure custom field is available custom_field_obj, _ = CustomField.objects.get_or_create( @@ -119,6 +119,7 @@ def create_ip(ip_address, subnet_mask, status="Active", object_pk=None): object_pk: Object primary key """ status_obj = Status.objects.get_for_model(IPAddress).get(name=status) + namespace_obj = Namespace.objects.get(name="Global") cidr = netmask_to_cidr(subnet_mask) if ALLOW_DUPLICATE_ADDRESSES: addr = IPAddress.objects.filter(host=ip_address) @@ -157,9 +158,11 @@ def create_interface(device_obj, interface_details): "mtu", "type", "mgmt_only", + "status", ) fields = {k: v for k, v in interface_details.items() if k in interface_fields and v} try: + fields["status"] = Status.objects.get_for_model(Interface).get(name=fields.get(fields["status"], "Active")) interface_obj, _ = device_obj.interfaces.get_or_create(**fields) except IntegrityError: interface_obj, _ = device_obj.interfaces.get_or_create(name=fields["name"]) @@ -169,6 +172,7 @@ def create_interface(device_obj, interface_details): interface_obj.mtu = fields.get("mtu") interface_obj.type = fields.get("type") interface_obj.mgmt_only = fields.get("mgmt_only", False) + interface_obj.status = Status.objects.get_for_model(Interface).get(name=fields.get("status", "Active")) interface_obj.validated_save() tag_object(nautobot_object=interface_obj, custom_field="ssot-synced-from-ipfabric") return interface_obj From 746c9b59e8afd43f8514b8ff4a1162af3c914a18 Mon Sep 17 00:00:00 2001 From: alhogan <98360253+alhogan@users.noreply.github.com> Date: Fri, 29 Sep 2023 12:30:23 -0500 Subject: [PATCH 04/12] fix job formating for 2.0 --- nautobot_ssot/integrations/ipfabric/jobs.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/nautobot_ssot/integrations/ipfabric/jobs.py b/nautobot_ssot/integrations/ipfabric/jobs.py index 609231931..4ad3d57dd 100644 --- a/nautobot_ssot/integrations/ipfabric/jobs.py +++ b/nautobot_ssot/integrations/ipfabric/jobs.py @@ -3,7 +3,6 @@ # pylint: disable=too-many-locals """IP Fabric Data Target Job.""" import uuid -from diffsync.enum import DiffSyncFlags from diffsync.exceptions import ObjectNotCreated from django.conf import settings from django.templatetags.static import static @@ -11,7 +10,7 @@ from httpx import ConnectError from ipfabric import IPFClient from nautobot.dcim.models import Location -from nautobot.extras.jobs import BooleanVar, Job, ScriptVariable, ChoiceVar +from nautobot.extras.jobs import BooleanVar, ScriptVariable, ChoiceVar from nautobot.core.forms import DynamicModelChoiceField from nautobot_ssot.jobs.base import DataMapping, DataSource @@ -103,7 +102,7 @@ def __init__( # pylint:disable=too-few-public-methods -class IpFabricDataSource(DataSource, Job): +class IpFabricDataSource(DataSource): """Job syncing data from IP Fabric to Nautobot.""" client = None @@ -220,11 +219,11 @@ def run( self, dryrun, memory_profiling, + debug, snapshot=None, safe_delete_mode=True, sync_ipfabric_tagged_only=True, location_filter=None, - debug=False, *args, **kwargs, ): @@ -285,13 +284,12 @@ def sync_data(self, *_args, **_kwargs): self.logger.info("Loading current data from Nautobot...") dest.load() - self.logger.info("Calculating diffs...") - flags = DiffSyncFlags.CONTINUE_ON_FAILURE - diff = dest.diff_from(ipfabric_source, flags=flags) + diff = dest.diff_from(ipfabric_source) # pylint: disable-next=logging-fstring-interpolation - self.logger.debug(f"Diff: {diff.dict()}") + if debug_mode: + self.logger.debug(f"Diff: {diff.dict()}") self.sync.diff = diff.dict() self.sync.save() From 9a9ae5454e5a5ceb69b93c1f468ec5b4872b1fc9 Mon Sep 17 00:00:00 2001 From: alhogan <98360253+alhogan@users.noreply.github.com> Date: Fri, 29 Sep 2023 12:31:14 -0500 Subject: [PATCH 05/12] fix location type creation for site --- nautobot_ssot/integrations/ipfabric/signals.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/nautobot_ssot/integrations/ipfabric/signals.py b/nautobot_ssot/integrations/ipfabric/signals.py index 6f389433c..5f874d249 100644 --- a/nautobot_ssot/integrations/ipfabric/signals.py +++ b/nautobot_ssot/integrations/ipfabric/signals.py @@ -54,6 +54,10 @@ def nautobot_database_ready_callback(sender, *, apps, **kwargs): # pylint: disa Location = apps.get_model("dcim", "Location") VLAN = apps.get_model("ipam", "VLAN") Tag = apps.get_model("extras", "Tag") + ContentType = apps.get_model("contenttypes", "ContentType") + Device = apps.get_model("dcim", "Device") + Prefix = apps.get_model("ipam", "Prefix") + location_type = apps.get_model("dcim", "LocationType") Tag.objects.get_or_create( name="SSoT Synced from IPFabric", @@ -69,7 +73,11 @@ def nautobot_database_ready_callback(sender, *, apps, **kwargs): # pylint: disa "color": ColorChoices.COLOR_RED, }, ) + loc_type, _ = location_type.objects.update_or_create(name="Site") + loc_type.content_types.add(ContentType.objects.get_for_model(Device)) + loc_type.content_types.add(ContentType.objects.get_for_model(Prefix)) + loc_type.content_types.add(ContentType.objects.get_for_model(VLAN)) synced_from_models = [Device, DeviceType, Interface, Manufacturer, Location, VLAN, Role, IPAddress] create_custom_field("ssot-synced-from-ipfabric", "Last synced from IPFabric on", synced_from_models, apps=apps) - create_custom_field("ipfabric_site_id", "IPFabric Location ID", [Location], apps=apps, cf_type="type_text") + create_custom_field("ipfabric-site-id", "IPFabric Location ID", [Location], apps=apps, cf_type="type_text") create_custom_field("ipfabric_type", "IPFabric Type", [Role], apps=apps, cf_type="type_text") From 82f39c7e201deb69e7134feeda3d224a4d7e162c Mon Sep 17 00:00:00 2001 From: alhogan <98360253+alhogan@users.noreply.github.com> Date: Fri, 29 Sep 2023 12:31:42 -0500 Subject: [PATCH 06/12] fix logging and change site to location --- .../integrations/ipfabric/diffsync/adapter_ipfabric.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/nautobot_ssot/integrations/ipfabric/diffsync/adapter_ipfabric.py b/nautobot_ssot/integrations/ipfabric/diffsync/adapter_ipfabric.py index 992bc6622..10cf05cca 100644 --- a/nautobot_ssot/integrations/ipfabric/diffsync/adapter_ipfabric.py +++ b/nautobot_ssot/integrations/ipfabric/diffsync/adapter_ipfabric.py @@ -99,14 +99,14 @@ def load(self): vlan_name = vlan.get("vlanName") if vlan.get("vlanName") else f"{vlan['siteName']}:{vlan['vlanId']}" if len(vlan_name) > name_max_length: logger.warning( - message=f"Not syncing VLAN, {vlan_name} due to character limit exceeding {name_max_length}." + f"Not syncing VLAN, {vlan_name} due to character limit exceeding {name_max_length}." ) continue try: vlan = self.vlan( diffsync=self, name=vlan_name, - site=vlan["siteName"], + location=vlan["siteName"], vid=vlan["vlanId"], status="Active", description=description, @@ -114,7 +114,7 @@ def load(self): self.add(vlan) location.add_child(vlan) except ObjectAlreadyExists: - logger.warning(message=f"Duplicate VLAN discovered, {vlan}") + logger.warning(f"Duplicate VLAN discovered, {vlan}") location_devices = [device for device in devices if device["siteName"] == location.name] for device in location_devices: @@ -123,7 +123,7 @@ def load(self): serial_number = device["sn"] if sn_length < device_serial_max_length else "" if not serial_number: logger.warning( - message=( + ( f"Serial Number will not be recorded for {device['hostname']} due to character limit. " f"{sn_length} exceeds {device_serial_max_length}" ) @@ -143,7 +143,7 @@ def load(self): location.add_child(device_model) self.load_device_interfaces(device_model, interfaces, device_primary_ip) except ObjectAlreadyExists: - logger.warning(message=f"Duplicate Device discovered, {device}") + logger.warning(f"Duplicate Device discovered, {device}") def pseudo_management_interface(hostname, device_interfaces, device_primary_ip): From 1525af1c9a6d7b8d7905cd5066a129f5fe4f60ed Mon Sep 17 00:00:00 2001 From: alhogan <98360253+alhogan@users.noreply.github.com> Date: Fri, 29 Sep 2023 12:32:10 -0500 Subject: [PATCH 07/12] fix ip version lookup --- .../ipfabric/diffsync/diffsync_models.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/nautobot_ssot/integrations/ipfabric/diffsync/diffsync_models.py b/nautobot_ssot/integrations/ipfabric/diffsync/diffsync_models.py index d265740ce..b6f839d50 100644 --- a/nautobot_ssot/integrations/ipfabric/diffsync/diffsync_models.py +++ b/nautobot_ssot/integrations/ipfabric/diffsync/diffsync_models.py @@ -319,10 +319,10 @@ def create(cls, diffsync, ids, attrs): ) interface_obj.ip_addresses.add(ip_address_obj) if attrs.get("ip_is_primary"): - if ip_address_obj.family == 4: + if ip_address_obj.ip_version == 4: device_obj.primary_ip4 = ip_address_obj device_obj.save() - if ip_address_obj.family == 6: + if ip_address_obj.ip_version == 6: device_obj.primary_ip6 = ip_address_obj device_obj.save() interface_obj.save() @@ -384,10 +384,10 @@ def update(self, attrs): # pylint: disable=too-many-branches if attrs.get("ip_is_primary"): interface_obj = interface.ip_addresses.first() if interface_obj: - if interface_obj.family == 4: + if interface_obj.ip_version == 4: device.primary_ip4 = interface_obj device.save() - if interface_obj.family == 6: + if interface_obj.ip_version == 6: device.primary_ip6 = interface_obj device.save() interface.save() @@ -402,14 +402,14 @@ class Vlan(DiffSyncExtras): """VLAN model.""" _modelname = "vlan" - _identifiers = ("name", "site") + _identifiers = ("name", "location") _shortname = ("name",) _attributes = ("vid", "status", "description") name: str vid: int status: str - site: str + location: str description: Optional[str] vlan_pk: Optional[UUID] @@ -417,7 +417,7 @@ class Vlan(DiffSyncExtras): def create(cls, diffsync, ids, attrs): """Create VLANs in Nautobot under the site.""" status = attrs["status"].lower().capitalize() - location = NautobotLocation.objects.get(name=ids["site"]) + location = NautobotLocation.objects.get(name=ids["location"]) name = ids["name"] if ids["name"] else f"VLAN{attrs['vid']}" description = attrs["description"] if attrs["description"] else None if diffsync.job.debug: From 4caf019a9c59098b0df20afb12fa9bb292681a2e Mon Sep 17 00:00:00 2001 From: alhogan <98360253+alhogan@users.noreply.github.com> Date: Fri, 29 Sep 2023 12:33:07 -0500 Subject: [PATCH 08/12] 2.0 updates --- .../ipfabric/utilities/nbutils.py | 29 +++++++------------ 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/nautobot_ssot/integrations/ipfabric/utilities/nbutils.py b/nautobot_ssot/integrations/ipfabric/utilities/nbutils.py index 20cfcc52e..3cdb2681c 100644 --- a/nautobot_ssot/integrations/ipfabric/utilities/nbutils.py +++ b/nautobot_ssot/integrations/ipfabric/utilities/nbutils.py @@ -5,6 +5,7 @@ from django.contrib.contenttypes.models import ContentType from django.db import IntegrityError +from django.core.exceptions import ValidationError from nautobot.dcim.models import ( Device, DeviceType, @@ -16,12 +17,11 @@ from nautobot.extras.choices import CustomFieldTypeChoices from nautobot.extras.models import CustomField, Role, Tag from nautobot.extras.models.statuses import Status -from nautobot.ipam.models import VLAN, IPAddress, Namespace +from nautobot.ipam.models import VLAN, IPAddress, Namespace, Prefix +from nautobot.ipam.choices import PrefixTypeChoices from nautobot.core.choices import ColorChoices from netutils.ip import netmask_to_cidr -from nautobot_ssot.integrations.ipfabric.constants import ALLOW_DUPLICATE_ADDRESSES - def create_location(location_name, location_id=None): """Creates a specified location in Nautobot. @@ -121,16 +121,11 @@ def create_ip(ip_address, subnet_mask, status="Active", object_pk=None): status_obj = Status.objects.get_for_model(IPAddress).get(name=status) namespace_obj = Namespace.objects.get(name="Global") cidr = netmask_to_cidr(subnet_mask) - if ALLOW_DUPLICATE_ADDRESSES: - addr = IPAddress.objects.filter(host=ip_address) - data = {"address": f"{ip_address}/{cidr}", "status": status_obj} - if addr.exists(): - data["description"] = "Duplicate by IPFabric SSoT" - - ip_obj = IPAddress.objects.create(**data) - - else: + try: ip_obj, _ = IPAddress.objects.get_or_create(address=f"{ip_address}/{cidr}", status=status_obj) + except ValidationError: + parent, _ = Prefix.objects.get_or_create(network="0.0.0.0", prefix_length=0, type=PrefixTypeChoices.TYPE_NETWORK, status=Status.objects.get_for_model(Prefix).get(name="Active"), namespace=namespace_obj) + ip_obj, _ = IPAddress.objects.get_or_create(address=f"{ip_address}/{cidr}", status=status_obj, parent=parent) if object_pk: ip_obj.assigned_object_id = object_pk.pk @@ -225,13 +220,9 @@ def _tag_object(nautobot_object): nautobot_object.tags.add(tag) if hasattr(nautobot_object, "cf"): # Ensure that the "ssot-synced-from-ipfabric" custom field is present - if not any(cfield for cfield in CustomField.objects.all() if cfield.label == "ssot-synced-from-ipfabric"): - custom_field_obj, _ = CustomField.objects.get_or_create( - type=CustomFieldTypeChoices.TYPE_DATE, - label="ssot-synced-from-ipfabric", - defaults={ - "label": "Last synced from IPFabric on", - }, + if not any(cfield for cfield in CustomField.objects.all() if cfield.key == "ssot-synced-from-ipfabric"): + custom_field_obj, _ = CustomField.objects.get( + key="ssot-synced-from-ipfabric", ) synced_from_models = [ Device, From eb659f4991a6862538944393cfe074c253b5f2fa Mon Sep 17 00:00:00 2001 From: alhogan <98360253+alhogan@users.noreply.github.com> Date: Fri, 29 Sep 2023 14:19:36 -0500 Subject: [PATCH 09/12] linting --- .../ipfabric/diffsync/adapter_ipfabric.py | 4 +--- .../ipfabric/diffsync/diffsync_models.py | 12 +++--------- .../integrations/ipfabric/utilities/nbutils.py | 14 ++++++++++++-- 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/nautobot_ssot/integrations/ipfabric/diffsync/adapter_ipfabric.py b/nautobot_ssot/integrations/ipfabric/diffsync/adapter_ipfabric.py index 10cf05cca..d3e761e9b 100644 --- a/nautobot_ssot/integrations/ipfabric/diffsync/adapter_ipfabric.py +++ b/nautobot_ssot/integrations/ipfabric/diffsync/adapter_ipfabric.py @@ -98,9 +98,7 @@ def load(self): description = vlan.get("dscr") if vlan.get("dscr") else f"VLAN ID: {vlan['vlanId']}" vlan_name = vlan.get("vlanName") if vlan.get("vlanName") else f"{vlan['siteName']}:{vlan['vlanId']}" if len(vlan_name) > name_max_length: - logger.warning( - f"Not syncing VLAN, {vlan_name} due to character limit exceeding {name_max_length}." - ) + logger.warning(f"Not syncing VLAN, {vlan_name} due to character limit exceeding {name_max_length}.") continue try: vlan = self.vlan( diff --git a/nautobot_ssot/integrations/ipfabric/diffsync/diffsync_models.py b/nautobot_ssot/integrations/ipfabric/diffsync/diffsync_models.py index b6f839d50..63ba3c285 100644 --- a/nautobot_ssot/integrations/ipfabric/diffsync/diffsync_models.py +++ b/nautobot_ssot/integrations/ipfabric/diffsync/diffsync_models.py @@ -46,9 +46,7 @@ def safe_delete(self, nautobot_object: Any, safe_delete_status: Optional[str] = """ update = False if not self.safe_delete_mode: # This could just check self, refactor. - logger.warning( - f"{nautobot_object} will be deleted as safe delete mode is not enabled." - ) + logger.warning(f"{nautobot_object} will be deleted as safe delete mode is not enabled.") # This allows private class naming of nautobot objects to be ordered for delete() # Example definition in adapter class var: _site = Location self.diffsync.objects_to_delete[f"_{nautobot_object.__class__.__name__.lower()}"].append( @@ -61,9 +59,7 @@ def safe_delete(self, nautobot_object: Any, safe_delete_status: Optional[str] = if hasattr(nautobot_object, "status"): if not nautobot_object.status == safe_delete_status: nautobot_object.status = safe_delete_status - logger.warning( - f"{nautobot_object} has changed status to {safe_delete_status}." - ) + logger.warning(f"{nautobot_object} has changed status to {safe_delete_status}.") update = True else: # Not everything has a status. This may come in handy once more models are synced. @@ -85,9 +81,7 @@ def safe_delete(self, nautobot_object: Any, safe_delete_status: Optional[str] = if update: tonb_nbutils.tag_object(nautobot_object=nautobot_object, custom_field="ssot-synced-from-ipfabric") else: - logger.warning( - f"{nautobot_object} has previously been tagged with `ssot-safe-delete`. Skipping..." - ) + logger.warning(f"{nautobot_object} has previously been tagged with `ssot-safe-delete`. Skipping...") return self diff --git a/nautobot_ssot/integrations/ipfabric/utilities/nbutils.py b/nautobot_ssot/integrations/ipfabric/utilities/nbutils.py index 3cdb2681c..7feff5f2d 100644 --- a/nautobot_ssot/integrations/ipfabric/utilities/nbutils.py +++ b/nautobot_ssot/integrations/ipfabric/utilities/nbutils.py @@ -30,7 +30,11 @@ def create_location(location_name, location_id=None): location_name (str): Name of the location. location_id (str): ID of the location. """ - location_obj, _ = Location.objects.get_or_create(name=location_name, location_type=LocationType.objects.get(name="Site"), status=Status.objects.get(name="Active")) + location_obj, _ = Location.objects.get_or_create( + name=location_name, + location_type=LocationType.objects.get(name="Site"), + status=Status.objects.get(name="Active"), + ) if location_id: # Ensure custom field is available custom_field_obj, _ = CustomField.objects.get_or_create( @@ -124,7 +128,13 @@ def create_ip(ip_address, subnet_mask, status="Active", object_pk=None): try: ip_obj, _ = IPAddress.objects.get_or_create(address=f"{ip_address}/{cidr}", status=status_obj) except ValidationError: - parent, _ = Prefix.objects.get_or_create(network="0.0.0.0", prefix_length=0, type=PrefixTypeChoices.TYPE_NETWORK, status=Status.objects.get_for_model(Prefix).get(name="Active"), namespace=namespace_obj) + parent, _ = Prefix.objects.get_or_create( + network="0.0.0.0", + prefix_length=0, + type=PrefixTypeChoices.TYPE_NETWORK, + status=Status.objects.get_for_model(Prefix).get(name="Active"), + namespace=namespace_obj, + ) ip_obj, _ = IPAddress.objects.get_or_create(address=f"{ip_address}/{cidr}", status=status_obj, parent=parent) if object_pk: From a929e054e6927fd83a4a445f008d9f3cea427c32 Mon Sep 17 00:00:00 2001 From: alhogan <98360253+alhogan@users.noreply.github.com> Date: Fri, 29 Sep 2023 14:51:07 -0500 Subject: [PATCH 10/12] linting --- .../integrations/ipfabric/diffsync/adapter_nautobot.py | 8 ++++---- .../integrations/ipfabric/diffsync/diffsync_models.py | 5 ++--- nautobot_ssot/integrations/ipfabric/jobs.py | 4 ++-- nautobot_ssot/integrations/ipfabric/signals.py | 6 ++---- nautobot_ssot/integrations/ipfabric/utilities/nbutils.py | 2 +- nautobot_ssot/integrations/ipfabric/workers.py | 1 + 6 files changed, 12 insertions(+), 14 deletions(-) diff --git a/nautobot_ssot/integrations/ipfabric/diffsync/adapter_nautobot.py b/nautobot_ssot/integrations/ipfabric/diffsync/adapter_nautobot.py index fd34b37be..e2d45a681 100644 --- a/nautobot_ssot/integrations/ipfabric/diffsync/adapter_nautobot.py +++ b/nautobot_ssot/integrations/ipfabric/diffsync/adapter_nautobot.py @@ -119,7 +119,7 @@ def load_device(self, filtered_devices: List, location): """Load Devices from Nautobot.""" for device_record in filtered_devices: if self.job.debug: - logger.debug(f"Loading Nautobot Device: {device_record.name}") + logger.debug("Loading Nautobot Device: %s", device_record.name) device = self.device( diffsync=self, name=device_record.name, @@ -199,7 +199,7 @@ def load_data(self): location_objects = self.get_initial_location(ssot_tag) # The parent object that stores all children, is the Location. if self.job.debug: - logger.debug(f"Found {location_objects.count()} Nautobot Location objects to start sync from") + logger.debug("Found %s Nautobot Location objects to start sync from", location_objects.count()) if location_objects: for location_record in location_objects: @@ -212,7 +212,7 @@ def load_data(self): ) except AttributeError: logger.error( - f"Error loading {location_record}, invalid or missing attributes on object. Skipping..." + "Error loading %s, invalid or missing attributes on object. Skipping...", location_record ) continue self.add(location) @@ -233,7 +233,7 @@ def load_data(self): continue self.load_vlans(nautobot_location_vlans, location) except Location.DoesNotExist: - logger.error(f"Unable to find Location, {location_record}.") + logger.error("Unable to find Location, %s.", location_record) else: logger.warning("No Nautobot records to load.") diff --git a/nautobot_ssot/integrations/ipfabric/diffsync/diffsync_models.py b/nautobot_ssot/integrations/ipfabric/diffsync/diffsync_models.py index 63ba3c285..104cc1fb4 100644 --- a/nautobot_ssot/integrations/ipfabric/diffsync/diffsync_models.py +++ b/nautobot_ssot/integrations/ipfabric/diffsync/diffsync_models.py @@ -204,7 +204,6 @@ def create(cls, diffsync, ids, attrs): if diffsync.job.debug: logger.debug(error) logger.error(message) - raise Exception("A validation error occured.") return super().create(ids=ids, diffsync=diffsync, attrs=attrs) @@ -415,7 +414,7 @@ def create(cls, diffsync, ids, attrs): name = ids["name"] if ids["name"] else f"VLAN{attrs['vid']}" description = attrs["description"] if attrs["description"] else None if diffsync.job.debug: - logger.debug(f"Creating VLAN: {name} description: {description}") + logger.debug("Creating VLAN: %s description: %s", name, description) tonb_nbutils.create_vlan( vlan_name=name, vlan_id=attrs["vid"], @@ -436,7 +435,7 @@ def delete(self) -> Optional["DiffSyncModel"]: def update(self, attrs): """Update VLAN object in Nautobot.""" - vlan = VLAN.objects.get(name=self.name, vid=self.vid, location=NautobotLocation.objects.get(name=self.site)) + vlan = VLAN.objects.get(name=self.name, vid=self.vid, location=NautobotLocation.objects.get(name=self.location)) if attrs.get("status") == "Active": safe_delete_tag, _ = Tag.objects.get_or_create(name="SSoT Safe Delete") diff --git a/nautobot_ssot/integrations/ipfabric/jobs.py b/nautobot_ssot/integrations/ipfabric/jobs.py index 4ad3d57dd..b98b65060 100644 --- a/nautobot_ssot/integrations/ipfabric/jobs.py +++ b/nautobot_ssot/integrations/ipfabric/jobs.py @@ -214,7 +214,7 @@ def config_information(cls): "Safe Delete VLAN status": constants.SAFE_DELETE_VLAN_STATUS, } - # pylint: disable-next=too-many-arguments + # pylint: disable-next=too-many-arguments, arguments-differ def run( self, dryrun, @@ -289,7 +289,7 @@ def sync_data(self, *_args, **_kwargs): diff = dest.diff_from(ipfabric_source) # pylint: disable-next=logging-fstring-interpolation if debug_mode: - self.logger.debug(f"Diff: {diff.dict()}") + self.logger.debug("Diff: %s", diff.dict()) self.sync.diff = diff.dict() self.sync.save() diff --git a/nautobot_ssot/integrations/ipfabric/signals.py b/nautobot_ssot/integrations/ipfabric/signals.py index 5f874d249..3fadb9284 100644 --- a/nautobot_ssot/integrations/ipfabric/signals.py +++ b/nautobot_ssot/integrations/ipfabric/signals.py @@ -44,7 +44,7 @@ def create_custom_field(key: str, label: str, models: List, apps, cf_type: Optio def nautobot_database_ready_callback(sender, *, apps, **kwargs): # pylint: disable=unused-argument """Callback function triggered by the nautobot_database_ready signal when the Nautobot database is fully ready.""" - # pylint: disable=invalid-name + # pylint: disable=invalid-name, too-many-locals Device = apps.get_model("dcim", "Device") DeviceType = apps.get_model("dcim", "DeviceType") Role = apps.get_model("extras", "Role") @@ -55,8 +55,6 @@ def nautobot_database_ready_callback(sender, *, apps, **kwargs): # pylint: disa VLAN = apps.get_model("ipam", "VLAN") Tag = apps.get_model("extras", "Tag") ContentType = apps.get_model("contenttypes", "ContentType") - Device = apps.get_model("dcim", "Device") - Prefix = apps.get_model("ipam", "Prefix") location_type = apps.get_model("dcim", "LocationType") Tag.objects.get_or_create( @@ -75,7 +73,7 @@ def nautobot_database_ready_callback(sender, *, apps, **kwargs): # pylint: disa ) loc_type, _ = location_type.objects.update_or_create(name="Site") loc_type.content_types.add(ContentType.objects.get_for_model(Device)) - loc_type.content_types.add(ContentType.objects.get_for_model(Prefix)) + loc_type.content_types.add(ContentType.objects.get_for_model(apps.get_model("ipam", "Prefix"))) loc_type.content_types.add(ContentType.objects.get_for_model(VLAN)) synced_from_models = [Device, DeviceType, Interface, Manufacturer, Location, VLAN, Role, IPAddress] create_custom_field("ssot-synced-from-ipfabric", "Last synced from IPFabric on", synced_from_models, apps=apps) diff --git a/nautobot_ssot/integrations/ipfabric/utilities/nbutils.py b/nautobot_ssot/integrations/ipfabric/utilities/nbutils.py index 7feff5f2d..554f18539 100644 --- a/nautobot_ssot/integrations/ipfabric/utilities/nbutils.py +++ b/nautobot_ssot/integrations/ipfabric/utilities/nbutils.py @@ -129,7 +129,7 @@ def create_ip(ip_address, subnet_mask, status="Active", object_pk=None): ip_obj, _ = IPAddress.objects.get_or_create(address=f"{ip_address}/{cidr}", status=status_obj) except ValidationError: parent, _ = Prefix.objects.get_or_create( - network="0.0.0.0", + network="0.0.0.0", # nosec B104 prefix_length=0, type=PrefixTypeChoices.TYPE_NETWORK, status=Status.objects.get_for_model(Prefix).get(name="Active"), diff --git a/nautobot_ssot/integrations/ipfabric/workers.py b/nautobot_ssot/integrations/ipfabric/workers.py index c9dbeb9b2..dad7cf6ea 100644 --- a/nautobot_ssot/integrations/ipfabric/workers.py +++ b/nautobot_ssot/integrations/ipfabric/workers.py @@ -116,6 +116,7 @@ def ssot_sync_to_nautobot( safe_delete_mode=is_truthy(safe_delete_mode), sync_ipfabric_tagged_only=is_truthy(sync_ipfabric_tagged_only), location_filter=location_filter, + debug=False, ) sync_job.job_result.validated_save() From 5cd14f76e162b667ce1c31662e267ae0cdd03a17 Mon Sep 17 00:00:00 2001 From: alhogan <98360253+alhogan@users.noreply.github.com> Date: Fri, 29 Sep 2023 14:58:47 -0500 Subject: [PATCH 11/12] fix unittest for vlan --- nautobot_ssot/tests/ipfabric/test_ipfabric_adapter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nautobot_ssot/tests/ipfabric/test_ipfabric_adapter.py b/nautobot_ssot/tests/ipfabric/test_ipfabric_adapter.py index 1e5f7f536..18d2d5b28 100644 --- a/nautobot_ssot/tests/ipfabric/test_ipfabric_adapter.py +++ b/nautobot_ssot/tests/ipfabric/test_ipfabric_adapter.py @@ -71,7 +71,7 @@ def test_data_loading(self): self.assertTrue(hasattr(vlan, "name")) self.assertTrue(hasattr(vlan, "vid")) self.assertTrue(hasattr(vlan, "status")) - self.assertTrue(hasattr(vlan, "site")) + self.assertTrue(hasattr(vlan, "location")) self.assertTrue(hasattr(vlan, "description")) # Assert each interface has the necessary attributes From 54ed4c74fd5729de4395d3a4d109d617272622a5 Mon Sep 17 00:00:00 2001 From: Justin Drew <2396364+jdrew82@users.noreply.github.com> Date: Fri, 29 Sep 2023 15:27:30 -0500 Subject: [PATCH 12/12] Prefix setting with ipfabric Update setting to have ipfabric prefix to follow pattern for integrations. --- nautobot_ssot/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nautobot_ssot/__init__.py b/nautobot_ssot/__init__.py index bf0963400..f827a082e 100644 --- a/nautobot_ssot/__init__.py +++ b/nautobot_ssot/__init__.py @@ -75,7 +75,7 @@ class NautobotSSOTPluginConfig(PluginConfig): "ipfabric_host": "", "ipfabric_ssl_verify": True, "ipfabric_timeout": 15, - "nautobot_host": "", + "ipfabric_nautobot_host": "", "servicenow_instance": "", "servicenow_password": "", "servicenow_username": "",