Add IPFabric integration

.github/workflows/ci.yml

@@ -119,14 +119,14 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        python-version: ["3.7", "3.8", "3.9", "3.10"]
+        python-version: ["3.8", "3.9", "3.10"]
         db-backend: ["postgresql"]
         nautobot-version: ["latest"]
         include:
           - python-version: "3.10"
             db-backend: "postgresql"
             nautobot-version: "1.5.13"
-          - python-version: "3.7"
+          - python-version: "3.8"
             db-backend: "mysql"
             nautobot-version: "1.5.13"
           - python-version: "3.10"

development/Dockerfile

@@ -68,11 +68,13 @@ RUN sort poetry_freeze_base.txt poetry_freeze_all.txt | uniq -u > poetry_freeze_
 
 # Install all local project as editable, constrained on Nautobot version, to get any additional
 #   direct dependencies of the app
-RUN pip install -c constraints.txt -e .[all]
+RUN --mount=type=cache,target=/root/.cache/pip \
+    pip install -c constraints.txt -e .[all]
 
 # Install any dev dependencies frozen from Poetry
 # Can be improved in Poetry 1.2 which allows `poetry install --only dev`
-RUN pip install -c constraints.txt -r poetry_freeze_dev.txt
+RUN --mount=type=cache,target=/root/.cache/pip \
+    pip install -c constraints.txt -r poetry_freeze_dev.txt
 
 COPY development/nautobot_config.py ${NAUTOBOT_ROOT}/nautobot_config.py
 # !!! USE CAUTION WHEN MODIFYING LINES ABOVE

development/creds.example.env

@@ -44,3 +44,5 @@ NAUTOBOT_APIC_VERIFY_DEVNET=False
 NAUTOBOT_APIC_SITE_DEVNET="DevNet Sandbox"
 
 SERVICENOW_PASSWORD="changeme"
+
+IPFABRIC_API_TOKEN=secrettoken

development/development.env

@@ -39,7 +39,10 @@ MYSQL_USER=${NAUTOBOT_DB_USER}
 MYSQL_DATABASE=${NAUTOBOT_DB_NAME}
 MYSQL_ROOT_HOST=%
 
-NAUTOBOT_CELERY_TASK_SOFT_TIME_LIMIT=600
+NAUTOBOT_HOST="http://nautobot:8080"
+
+NAUTOBOT_CELERY_TASK_SOFT_TIME_LIMIT=3600
+NAUTOBOT_CELERY_TASK_TIME_LIMIT=3600
 
 NAUTOBOT_SSOT_HIDE_EXAMPLE_JOBS="False"
 
@@ -84,3 +87,8 @@ NAUTOBOT_SSOT_INFOBLOX_VERIFY_SSL="True"
 NAUTOBOT_SSOT_ENABLE_SERVICENOW="True"
 SERVICENOW_INSTANCE=""
 SERVICENOW_USERNAME=""
+
+NAUTOBOT_SSOT_ENABLE_IPFABRIC="True"
+IPFABRIC_HOST="https://ipfabric.example.com"
+IPFABRIC_SSL_VERIFY="True"
+IPFABRIC_TIMEOUT=15

development/nautobot_config.py

@@ -133,13 +133,24 @@
 
 # Enable installed plugins. Add the name of each plugin to the list.
 PLUGINS = [
-    "nautobot_ssot",
+    # Enable chatops after dropping Python 3.7 support
+    # "nautobot_chatops",
     "nautobot_device_lifecycle_mgmt",
+    "nautobot_ssot",
 ]
 
 # Plugins configuration settings. These settings are used by various plugins that the user may have installed.
 # Each key in the dictionary is the name of an installed plugin and its value is a dictionary of settings.
 PLUGINS_CONFIG = {
+    # Enable chatops after dropping Python 3.7 support
+    # "nautobot_chatops": {
+    #     "enable_slack": True,
+    #     "slack_api_token": os.getenv("SLACK_API_TOKEN"),
+    #     "slack_signing_secret": os.getenv("SLACK_SIGNING_SECRET"),
+    #     "session_cache_timeout": 3600,
+    #     "ipfabric_api_token": os.getenv("IPFABRIC_API_TOKEN"),
+    #     "ipfabric_host": os.getenv("IPFABRIC_HOST"),
+    # },
     "nautobot_ssot": {
         # URL and credentials should be configured as environment variables on the host system
         "aci_apics": {x: os.environ[x] for x in os.environ if "APIC" in x},
@@ -190,6 +201,7 @@
         "enable_aci": is_truthy(os.getenv("NAUTOBOT_SSOT_ENABLE_ACI")),
         "enable_aristacv": is_truthy(os.getenv("NAUTOBOT_SSOT_ENABLE_ARISTACV")),
         "enable_infoblox": is_truthy(os.getenv("NAUTOBOT_SSOT_ENABLE_INFOBLOX")),
+        "enable_ipfabric": is_truthy(os.getenv("NAUTOBOT_SSOT_ENABLE_IPFABRIC")),
         "enable_servicenow": is_truthy(os.getenv("NAUTOBOT_SSOT_ENABLE_SERVICENOW")),
         "hide_example_jobs": is_truthy(os.getenv("NAUTOBOT_SSOT_HIDE_EXAMPLE_JOBS")),
         "infoblox_default_status": os.getenv("NAUTOBOT_SSOT_INFOBLOX_DEFAULT_STATUS", "active"),
@@ -209,6 +221,11 @@
         "infoblox_username": os.getenv("NAUTOBOT_SSOT_INFOBLOX_USERNAME"),
         "infoblox_verify_ssl": is_truthy(os.getenv("NAUTOBOT_SSOT_INFOBLOX_VERIFY_SSL", True)),
         "infoblox_wapi_version": os.getenv("NAUTOBOT_SSOT_INFOBLOX_WAPI_VERSION", "v2.12"),
+        "ipfabric_api_token": os.getenv("IPFABRIC_API_TOKEN"),
+        "ipfabric_host": os.getenv("IPFABRIC_HOST"),
+        "ipfabric_ssl_verify": is_truthy(os.getenv("IPFABRIC_VERIFY", "False")),
+        "ipfabric_timeout": int(os.getenv("IPFABRIC_TIMEOUT", "15")),
+        "nautobot_host": os.getenv("NAUTOBOT_HOST"),
         "servicenow_instance": os.getenv("SERVICENOW_INSTANCE", ""),
         "servicenow_password": os.getenv("SERVICENOW_PASSWORD", ""),
         "servicenow_username": os.getenv("SERVICENOW_USERNAME", ""),

nautobot_ssot/__init__.py

@@ -60,6 +60,7 @@ class NautobotSSOTPluginConfig(PluginConfig):
         "enable_aci": False,
         "enable_aristacv": False,
         "enable_infoblox": False,
+        "enable_ipfabric": False,
         "enable_servicenow": False,
         "hide_example_jobs": True,
         "infoblox_default_status": "",
@@ -75,6 +76,11 @@ class NautobotSSOTPluginConfig(PluginConfig):
         "infoblox_username": "",
         "infoblox_verify_ssl": True,
         "infoblox_wapi_version": "",
+        "ipfabric_api_token": "",
+        "ipfabric_host": "",
+        "ipfabric_ssl_verify": True,
+        "ipfabric_timeout": 15,
+        "nautobot_host": "",
         "servicenow_instance": "",
         "servicenow_password": "",
         "servicenow_username": "",

nautobot_ssot/integrations/ipfabric/__init__.py

@@ -0,0 +1 @@
+"""Base module for IPFabric integration."""

nautobot_ssot/integrations/ipfabric/constants.py

@@ -0,0 +1,15 @@
+"""Constants used by IPFabric Integration."""
+
+ALLOW_DUPLICATE_ADDRESSES = True
+DEFAULT_DEVICE_ROLE = "Network Device"
+DEFAULT_DEVICE_ROLE_COLOR = "ff0000"
+DEFAULT_DEVICE_STATUS = "Active"
+DEFAULT_DEVICE_STATUS_COLOR = "ff0000"
+DEFAULT_INTERFACE_MAC = "00:00:00:00:00:01"
+DEFAULT_INTERFACE_MTU = 1500
+DEFAULT_INTERFACE_TYPE = "1000base-t"
+SAFE_DELETE_DEVICE_STATUS = "Deprecated"
+SAFE_DELETE_IPADDRESS_STATUS = "Deprecated"
+SAFE_DELETE_SITE_STATUS = "Decommissioning"
+SAFE_DELETE_VLAN_STATUS = "Inventory"
+SAFE_IPADDRESS_INTERFACES_STATUS = "Deprecated"

nautobot_ssot/integrations/ipfabric/diffsync/__init__.py

@@ -0,0 +1,4 @@
+"""Diffsync Utilities, Adapters and Models."""
+from .adapters_shared import DiffSyncModelAdapters
+
+__all__ = ("DiffSyncModelAdapters",)

nautobot_ssot/integrations/ipfabric/diffsync/adapter_ipfabric.py

@@ -0,0 +1,160 @@
+# pylint: disable=duplicate-code
+"""DiffSync adapter class for Ip Fabric."""
+
+import logging
+
+from diffsync import ObjectAlreadyExists
+from nautobot.dcim.models import Device
+from nautobot.ipam.models import VLAN
+from netutils.mac import mac_to_format
+
+from nautobot_ssot.integrations.ipfabric.constants import (
+    DEFAULT_INTERFACE_TYPE,
+    DEFAULT_INTERFACE_MTU,
+    DEFAULT_INTERFACE_MAC,
+    DEFAULT_DEVICE_ROLE,
+    DEFAULT_DEVICE_STATUS,
+)
+from nautobot_ssot.integrations.ipfabric.diffsync import DiffSyncModelAdapters
+
+logger = logging.getLogger("nautobot.jobs")
+
+device_serial_max_length = Device._meta.get_field("serial").max_length
+name_max_length = VLAN._meta.get_field("name").max_length
+
+
+class IPFabricDiffSync(DiffSyncModelAdapters):
+    """Nautobot adapter for DiffSync."""
+
+    def __init__(self, job, sync, client, *args, **kwargs):
+        """Initialize the NautobotDiffSync."""
+        super().__init__(*args, **kwargs)
+        self.job = job
+        self.sync = sync
+        self.client = client
+
+    def load_sites(self):
+        """Add IP Fabric Site objects as DiffSync Location models."""
+        sites = self.client.inventory.sites.all()
+        for site in sites:
+            try:
+                location = self.location(diffsync=self, name=site["siteName"], site_id=site["id"], status="Active")
+                self.add(location)
+            except ObjectAlreadyExists:
+                self.job.log_debug(message=f"Duplicate Site discovered, {site}")
+
+    def load_device_interfaces(self, device_model, interfaces, device_primary_ip):
+        """Create and load DiffSync Interface model objects for a specific device."""
+        device_interfaces = [iface for iface in interfaces if iface.get("hostname") == device_model.name]
+        pseudo_interface = pseudo_management_interface(device_model.name, device_interfaces, device_primary_ip)
+
+        if pseudo_interface:
+            device_interfaces.append(pseudo_interface)
+            logger.info("Pseudo MGMT Interface: %s", pseudo_interface)
+
+        for iface in device_interfaces:
+            ip_address = iface.get("primaryIp")
+            try:
+                interface = self.interface(
+                    diffsync=self,
+                    name=iface.get("intName"),
+                    device_name=iface.get("hostname"),
+                    description=iface.get("dscr", ""),
+                    enabled=True,
+                    mac_address=mac_to_format(iface.get("mac"), "MAC_COLON_TWO").upper()
+                    if iface.get("mac")
+                    else DEFAULT_INTERFACE_MAC,
+                    mtu=iface.get("mtu") if iface.get("mtu") else DEFAULT_INTERFACE_MTU,
+                    type=DEFAULT_INTERFACE_TYPE,
+                    mgmt_only=iface.get("mgmt_only", False),
+                    ip_address=ip_address,
+                    # TODO: why is only IPv4? and why /32?
+                    subnet_mask="255.255.255.255",
+                    ip_is_primary=ip_address is not None and ip_address == device_primary_ip,
+                    status="Active",
+                )
+                self.add(interface)
+                device_model.add_child(interface)
+            except ObjectAlreadyExists:
+                self.job.log_debug(message=f"Duplicate Interface discovered, {iface}")
+
+    def load(self):
+        """Load data from IP Fabric."""
+        self.load_sites()
+        devices = self.client.inventory.devices.all()
+        interfaces = self.client.inventory.interfaces.all()
+        vlans = self.client.fetch_all("tables/vlan/site-summary")
+
+        for location in self.get_all(self.location):
+            if location.name is None:
+                continue
+            location_vlans = [vlan for vlan in vlans if vlan["siteName"] == location.name]
+            for vlan in location_vlans:
+                if not vlan["vlanId"] or (vlan["vlanId"] < 1 or vlan["vlanId"] > 4094):
+                    self.job.log_warning(
+                        message=f"Not syncing VLAN, NAME: {vlan.get('vlanName')} due to invalid VLAN ID: {vlan.get('vlanId')}."
+                    )
+                    continue
+                description = vlan.get("dscr") if vlan.get("dscr") else f"VLAN ID: {vlan['vlanId']}"
+                vlan_name = vlan.get("vlanName") if vlan.get("vlanName") else f"{vlan['siteName']}:{vlan['vlanId']}"
+                if len(vlan_name) > name_max_length:
+                    self.job.log_warning(
+                        message=f"Not syncing VLAN, {vlan_name} due to character limit exceeding {name_max_length}."
+                    )
+                    continue
+                try:
+                    vlan = self.vlan(
+                        diffsync=self,
+                        name=vlan_name,
+                        site=vlan["siteName"],
+                        vid=vlan["vlanId"],
+                        status="Active",
+                        description=description,
+                    )
+                    self.add(vlan)
+                    location.add_child(vlan)
+                except ObjectAlreadyExists:
+                    self.job.log_debug(message=f"Duplicate VLAN discovered, {vlan}")
+
+            location_devices = [device for device in devices if device["siteName"] == location.name]
+            for device in location_devices:
+                device_primary_ip = device["loginIp"]
+                sn_length = len(device["sn"])
+                serial_number = device["sn"] if sn_length < device_serial_max_length else ""
+                if not serial_number:
+                    self.job.log_warning(
+                        message=(
+                            f"Serial Number will not be recorded for {device['hostname']} due to character limit. "
+                            f"{sn_length} exceeds {device_serial_max_length}"
+                        )
+                    )
+                try:
+                    device_model = self.device(
+                        diffsync=self,
+                        name=device["hostname"],
+                        location_name=device["siteName"],
+                        model=device.get("model") if device.get("model") else f"Default-{device.get('vendor')}",
+                        vendor=device.get("vendor").capitalize(),
+                        serial_number=serial_number,
+                        role=device.get("devType") if device.get("devType") else DEFAULT_DEVICE_ROLE,
+                        status=DEFAULT_DEVICE_STATUS,
+                    )
+                    self.add(device_model)
+                    location.add_child(device_model)
+                    self.load_device_interfaces(device_model, interfaces, device_primary_ip)
+                except ObjectAlreadyExists:
+                    self.job.log_debug(message=f"Duplicate Device discovered, {device}")
+
+
+def pseudo_management_interface(hostname, device_interfaces, device_primary_ip):
+    """Return a dict for an non-existing interface for NAT management addresses."""
+    if any(iface for iface in device_interfaces if iface.get("primaryIp", "") == device_primary_ip):
+        return None
+    return {
+        "hostname": hostname,
+        "intName": "pseudo_mgmt",
+        "dscr": "pseudo interface for NAT IP address",
+        "primaryIp": device_primary_ip,
+        "type": "virtual",
+        "mgmt_only": True,
+    }