diff --git a/changes/479.fixed b/changes/479.fixed new file mode 100644 index 000000000..284c238d3 --- /dev/null +++ b/changes/479.fixed @@ -0,0 +1 @@ +Corrected the attribute used to reference the ControllerManagedDeviceGroup off a Controller object. \ No newline at end of file diff --git a/changes/542.documentation b/changes/542.documentation new file mode 100644 index 000000000..87eab84f7 --- /dev/null +++ b/changes/542.documentation @@ -0,0 +1 @@ +Correct documentation for ACI integration and add missing DNA Center installation documentation. \ No newline at end of file diff --git a/changes/546.added b/changes/546.added new file mode 100644 index 000000000..1beb7a1b8 --- /dev/null +++ b/changes/546.added @@ -0,0 +1,2 @@ +Added support for specifying LocationType for Areas, Buildings, and Floors in DNA Center integration. +Added support for specifying LocationType for Buildings in Device42 integration. \ No newline at end of file diff --git a/changes/546.documentation b/changes/546.documentation new file mode 100644 index 000000000..9c0b3dda6 --- /dev/null +++ b/changes/546.documentation @@ -0,0 +1,2 @@ +Added documentation on how to use DNA Center integration along with screenshots of the steps. +Updated documentation for Device42 integration and updated Job form screenshot to update for Building LocationType Job form change. \ No newline at end of file diff --git a/changes/548.fixed b/changes/548.fixed new file mode 100644 index 000000000..05924edd1 --- /dev/null +++ b/changes/548.fixed @@ -0,0 +1 @@ +Fixed SSoT jobs not respecting DryRun variable. \ No newline at end of file diff --git a/changes/558.fixed b/changes/558.fixed new file mode 100644 index 000000000..930e1aba2 --- /dev/null +++ b/changes/558.fixed @@ -0,0 +1 @@ +Fixed VRF attribute for Prefix create() to be ids instead of attrs. \ No newline at end of file diff --git a/changes/561.fixed b/changes/561.fixed new file mode 100644 index 000000000..0853ce390 --- /dev/null +++ b/changes/561.fixed @@ -0,0 +1 @@ +Bug in IP Fabric that causes some network columns to return host bits set; changed `ip_network` to use `strict=False`. \ No newline at end of file diff --git a/docs/admin/integrations/aci_setup.md b/docs/admin/integrations/aci_setup.md index 9a8cbeb56..2d84e8339 100644 --- a/docs/admin/integrations/aci_setup.md +++ b/docs/admin/integrations/aci_setup.md @@ -56,7 +56,7 @@ PLUGINS_CONFIG = { All APIC specific settings have been updated to use the Controller and related ExternalIntegration objects. The ExternalIntegration object that is assigned to the Controller will define the APIC base URL, user credentials, and SSL verification. It will also have a `tenant_prefix` key in the `extra_config` section of the ExternalIntegration to define the Tenant prefix. -The `aci_apics` setting from the `nautobot_config.py` file is no longer used and any configuration found for it will be automatically migrated into a Controller and an ExternalIntegration object. +The `aci_apics` setting from the `nautobot_config.py` file is no longer used. Any configuration found for the APICs that were defined in `aci_apics` will need to be manually input into the Nautobot UI to create the required ExternalIntegration and Controller objects. ## Nautobot Objects Affected by Settings @@ -82,14 +82,19 @@ There are example YAML files for a few common switch models in `nautobot_ssot/in When upgrading from `nautobot-plugin-ssot-aci` app, it's necessary to [avoid conflicts](../upgrade.md#potential-apps-conflicts). - Uninstall the old app: + ```shell pip uninstall nautobot-plugin-ssot-aci ``` + - Upgrade the app with required extras: + ```shell pip install --upgrade nautobot-ssot[aci] ``` + - Fix `nautobot_config.py` by removing `nautobot_ssot_aci` from `PLUGINS` and merging app configuration into `nautobot_ssot`: + ```python PLUGINS = [ "nautobot_ssot", diff --git a/docs/admin/integrations/dna_center_setup.md b/docs/admin/integrations/dna_center_setup.md index 6dc18d058..c3de6eb9b 100644 --- a/docs/admin/integrations/dna_center_setup.md +++ b/docs/admin/integrations/dna_center_setup.md @@ -12,7 +12,7 @@ pip install nautobot-ssot[dna_center] ## Configuration -Connecting to a DNA Center instance is handled through the Nautobot [Controller](https://docs.nautobot.com/projects/core/en/stable/development/core/controllers/) object. There is an expectation that you will create an [ExternalIntegration](https://docs.nautobot.com/projects/core/en/stable/user-guide/platform-functionality/externalintegration/) with the requisite connection information for your DNA Center instance attached to that Controller object. All imported Devices will be associated to a [ControllerManagedDeviceGroup](https://docs.nautobot.com/projects/core/en/stable/user-guide/core-data-model/dcim/controllermanageddevicegroup/) that is found or created during each Job run. It will update the group name to be "\ Managed Devices" if it exists. When running the Sync Job you will specify which DNA Center Controller instance you wish to synchronize with. Other behaviors for the integration can be controlled with the following settings: +Connecting to a DNA Center instance is handled through the Nautobot [Controller](https://docs.nautobot.com/projects/core/en/stable/development/core/controllers/) object. There is an expectation that you will create an [ExternalIntegration](https://docs.nautobot.com/projects/core/en/stable/user-guide/platform-functionality/externalintegration/) with the requisite connection information for your DNA Center instance attached to that Controller object. All imported Devices will be associated to a [ControllerManagedDeviceGroup](https://docs.nautobot.com/projects/core/en/stable/user-guide/core-data-model/dcim/controllermanageddevicegroup/) that is found or created during each Job run. It will update the group name to be " Managed Devices" if it exists. When running the Sync Job you will specify which DNA Center Controller instance you wish to synchronize with. Other behaviors for the integration can be controlled with the following settings: | Configuration Variable | Type | Usage | Default | | --------------------------------------------------- | ------- | ---------------------------------------------------------- | -------------------- | diff --git a/docs/admin/integrations/index.md b/docs/admin/integrations/index.md index 0353e67b4..ce610d824 100644 --- a/docs/admin/integrations/index.md +++ b/docs/admin/integrations/index.md @@ -6,6 +6,7 @@ This Nautobot app supports the following integrations: - [Bootstrap](./bootstrap_setup.md) - [Arista CloudVision](./aristacv_setup.md) - [Device42](./device42_setup.md) +- [Cisco DNA Center](./dna_center_setup.md) - [Infoblox](./infoblox_setup.md) - [IPFabric](./ipfabric_setup.md) - [Itential](./itential_setup.md) diff --git a/docs/images/device42_job-form.png b/docs/images/device42_job-form.png index 2da6f005c..d53cba9a9 100644 Binary files a/docs/images/device42_job-form.png and b/docs/images/device42_job-form.png differ diff --git a/docs/images/dnac_controller.png b/docs/images/dnac_controller.png new file mode 100644 index 000000000..f3797e2d3 Binary files /dev/null and b/docs/images/dnac_controller.png differ diff --git a/docs/images/dnac_dashboard.png b/docs/images/dnac_dashboard.png new file mode 100644 index 000000000..1a97d9d6b Binary files /dev/null and b/docs/images/dnac_dashboard.png differ diff --git a/docs/images/dnac_detail-view.png b/docs/images/dnac_detail-view.png new file mode 100644 index 000000000..28b8e6bae Binary files /dev/null and b/docs/images/dnac_detail-view.png differ diff --git a/docs/images/dnac_enabled_job.png b/docs/images/dnac_enabled_job.png new file mode 100644 index 000000000..d038bee8f Binary files /dev/null and b/docs/images/dnac_enabled_job.png differ diff --git a/docs/images/dnac_external_integration.png b/docs/images/dnac_external_integration.png new file mode 100644 index 000000000..a58956d4c Binary files /dev/null and b/docs/images/dnac_external_integration.png differ diff --git a/docs/images/dnac_external_integration_adv.png b/docs/images/dnac_external_integration_adv.png new file mode 100644 index 000000000..3e2752fd5 Binary files /dev/null and b/docs/images/dnac_external_integration_adv.png differ diff --git a/docs/images/dnac_job_form.png b/docs/images/dnac_job_form.png new file mode 100644 index 000000000..33275af38 Binary files /dev/null and b/docs/images/dnac_job_form.png differ diff --git a/docs/images/dnac_job_list.png b/docs/images/dnac_job_list.png new file mode 100644 index 000000000..e24d9d885 Binary files /dev/null and b/docs/images/dnac_job_list.png differ diff --git a/docs/images/dnac_job_settings.png b/docs/images/dnac_job_settings.png new file mode 100644 index 000000000..2615b8ff9 Binary files /dev/null and b/docs/images/dnac_job_settings.png differ diff --git a/docs/images/dnac_jobresult.png b/docs/images/dnac_jobresult.png new file mode 100644 index 000000000..b4931debe Binary files /dev/null and b/docs/images/dnac_jobresult.png differ diff --git a/docs/images/dnac_password_secret.png b/docs/images/dnac_password_secret.png new file mode 100644 index 000000000..8f0900157 Binary files /dev/null and b/docs/images/dnac_password_secret.png differ diff --git a/docs/images/dnac_secretsgroup.png b/docs/images/dnac_secretsgroup.png new file mode 100644 index 000000000..565ea23f1 Binary files /dev/null and b/docs/images/dnac_secretsgroup.png differ diff --git a/docs/images/dnac_ssot-sync-details.png b/docs/images/dnac_ssot-sync-details.png new file mode 100644 index 000000000..33be053e7 Binary files /dev/null and b/docs/images/dnac_ssot-sync-details.png differ diff --git a/docs/images/dnac_username_secret.png b/docs/images/dnac_username_secret.png new file mode 100644 index 000000000..29ed1d3fc Binary files /dev/null and b/docs/images/dnac_username_secret.png differ diff --git a/docs/user/integrations/device42.md b/docs/user/integrations/device42.md index 00f027a2e..2e449a05b 100644 --- a/docs/user/integrations/device42.md +++ b/docs/user/integrations/device42.md @@ -6,7 +6,7 @@ From Device42 into Nautobot, it synchronizes the following objects: | Device42 | Nautobot | | ----------------------- | ---------------------------- | -| Buildings | Sites | +| Buildings | Locations | | Rooms | RackGroups | | Racks | Racks | | Vendors | Manufacturers | @@ -36,7 +36,9 @@ To start the synchronization, simply select the ExternalIntegration that corresp ![Job Form](../../images/device42_job-form.png) -If you wish to just test the synchronization but not have any data created in Nautobot you'll want to select the `Dry run` toggle. Clicking the `Debug` toggle will enable more verbose logging to inform you of what is occuring behind the scenes. Finally, the `Bulk import` option will enable bulk create and update operations to be used when the synchronization is complete. This can improve performance times for the App by forsaking validation of the imported data. Be aware that this could potentially cause bad data to be pushed into Nautobot. +> As of SSoT 3.2.0 you now have the option to define the LocationType to use for imported Buildings. If unspecifed in the Job form it will resort to using the Site LocationType as it did previously. + +If you wish to just test the synchronization but not have any data created in Nautobot you'll want to select the `Dryrun` toggle. Clicking the `Debug` toggle will enable more verbose logging to inform you of what is occuring behind the scenes. Finally, the `Bulk import` option will enable bulk create and update operations to be used when the synchronization is complete. This can improve performance times for the App by forsaking validation of the imported data. Be aware that this could potentially cause bad data to be pushed into Nautobot. Running this Job will redirect you to a `Nautobot Job Result` view. diff --git a/docs/user/integrations/dna_center.md b/docs/user/integrations/dna_center.md new file mode 100644 index 000000000..07606b2bd --- /dev/null +++ b/docs/user/integrations/dna_center.md @@ -0,0 +1,85 @@ +# Cisco DNA Center SSoT Integration + +The Cisco DNA Center SSoT integration is built as part of the [Nautobot Single Source of Truth (SSoT)](https://github.com/nautobot/nautobot-app-ssot) app. The SSoT app enables Nautobot to be the aggregation point for data coming from multiple systems of record (SoR). + +From Cisco DNA Center into Nautobot, it synchronizes the following objects: + +| DNA Center | Nautobot | +| ----------------------- | ---------------------------- | +| Areas | Location* | +| Buildings | Location* | +| Floors | Location* | +| Devices | Devices** | +| Ports | Interfaces | +| Prefixes | Prefixes | +| IP Addresses | IP Addresses | + +`*` As of SSoT 3.2.0 the LocationType for Areas, Buildings, and Floors can be defined in the Job form. Prior to SSoT 3.2.0 the DNA Center integration creates a Region, Site, and Floor LocationType and imports Areas as Region Locations, Buildings as Site Locations, and Floors as Floor Locations. + +`**` If the [Device Lifecycle Nautobot app](https://github.com/nautobot/nautobot-app-device-lifecycle-mgmt) is found to be installed, a matching Version will be created with a RelationshipAssociation connecting the Device and that Version. + +## Usage + +Once the app is installed and configured, you will be able to perform an inventory ingestion from DNA Center into Nautobot. From the Nautobot SSoT Dashboard view (`/plugins/ssot/`), DNA Center will show as a Data Source. + +![Dashboard View](../../images/dnac_dashboard.png) + +From the Dashboard, you can also view more information about the App by clicking on the `DNA Center to Nautobot` link and see the Detail view. This view will show the mappings of DNA Center objects to Nautobot objects, the sync history, and other configuration details for the App: + +![Detail View](../../images/dnac_detail-view.png) + +In order to utilize this integration you must first enable the Job. You can find the available installed Jobs under Jobs -> Jobs: + +![Job List](../../images/dnac_job_list.png) + +To enable the Job you must click on the orange pencil icon to the right of the `DNA Center to Nautobot` Job. You will be presented with the settings for the Job as shown below: + +![Job Settings](../../images/dnac_job_settings.png) + +You'll need to check the `Enabled` checkbox and then the `Update` button at the bottom of the page. You will then see that the play button next to the Job changes to blue and becomes functional, linking to the Job form. + +![Enabled Job](../../images/dnac_enabled_job.png) + +Once the Job is enabled, you'll need to manually create a few objects in Nautobot to use with the Job. First, you'll need to create a Secret that contains the username and password for authenticating to your desired DNA Center instance: + +![Username Secret](../../images/dnac_username_secret.png) + +![Password Secret](../../images/dnac_password_secret.png) + +Once the required Secrets are created, you'll need to create a SecretsGroup that pairs them together and defines the Access Type of HTTP(S) like shown below: + +![DNAC SecretsGroup](../../images/dnac_secretsgroup.png) + +With the SecretsGroup defined containing your instance credentials you'll then need to create an ExternalIntegration object to store the information about the DNA Center instance you wish to synchronize with. + +![DNAC ExternalIntegration](../../images/dnac_external_integration.png) + +> The only required portions are the Name, Remote URL, Verify SSL, and Secrets Group. The `Extra Config` section allows you to specify the port that DNA Center is running on. It will default to 443 if unspecified. + +![DNAC ExternalIntegration](../../images/dnac_external_integration_adv.png) + +The final step before running the Job is to create a Controller that references the ExternalIntegration that you just created. You can attach a `Managed Device Group` to the Controller for all imported Devices to be placed in. If you don't create a Managed Device Group, one will be created automatically and associated to the specified Controller with the name of ` Managed Devices`. + +![DNAC Controller](../../images/dnac_controller.png) + +> You can utilize multiple DNA Center Controllers with this integration as long as you specify a unique Tenant per Controller. The failure to use differing Tenants will have the Devices, Prefixes, and IPAddresses potentially removed if they are non-existent on the additional Controller. Locations should remain unaffected. + +With those configured, you will then need to define a LocationType to use for each DNA Center location type of Areas, Buildings, and Floors. With those created, you can run the Job to start the synchronization: + +> When creating the Area LocationType you must check the "Nestable" option. + +![Job Form](../../images/dnac_job_form.png) + +If you wish to just test the synchronization but not have any data created in Nautobot you'll want to select the `Dryrun` toggle. Clicking the `Debug` toggle will enable more verbose logging to inform you of what is occuring behind the scenes. Finally, the `Bulk import` option will enable bulk create and update operations to be used when the synchronization is complete. This can improve performance times for the integration by forsaking validation of the imported data. Be aware that this could potentially cause bad data to be pushed into Nautobot. After those toggles there are also dropdowns that allow you to specify the DNA Center Controller to synchronize with and to define the LocationTypes to use for the imported Areas, Buildings, and Floors from DNA Center. In addition, there are also some optional settings on the Job form: + +- The Location Mapping allows you to define a dictionary of Location mappings. This feature is intended for specifying parent Locations for the Areas and Building locations in DNA Center. This is useful if this information is missing from DNA Center but required for Nautobot or to allow you to change the information as it's imported to match information from another System of Record. The expected pattern for this field is `{"": {"parent": ""}}`. + +- Finally there is an option to specify a Tenant to be assigned to the imported Devices, Prefixes, and IPAddreses. This is handy for cases where you have multiple DNA Center instances that are used by differing business units. + +Running this Job will redirect you to a `Nautobot Job Result` view. + +![JobResult View](../../images/dnac_jobresult.png) + +Once the Job has finished you can click on the `SSoT Sync Details` button at the top right of the Job Result page to see detailed information about the data that was synchronized from DNA Center and the outcome of the sync Job. + +![SSoT Sync Details](../../images/dnac_ssot-sync-details.png) diff --git a/docs/user/integrations/index.md b/docs/user/integrations/index.md index ebd3a8df9..19dd72b71 100644 --- a/docs/user/integrations/index.md +++ b/docs/user/integrations/index.md @@ -6,6 +6,7 @@ This Nautobot app supports the following integrations: - [Arista CloudVision](./aristacv.md) - [Bootstrap](./bootstrap.md) - [Device42](./device42.md) +- [Cisco DNA Center](./dna_center.md) - [Infoblox](./infoblox.md) - [IPFabric](./ipfabric.md) - [Itential](./itential.md) diff --git a/mkdocs.yml b/mkdocs.yml index c40d947b0..51264d4f7 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -112,6 +112,7 @@ nav: - Arista CloudVision: "user/integrations/aristacv.md" - Bootstrap: "user/integrations/bootstrap.md" - Device42: "user/integrations/device42.md" + - DNA Center: "user/integrations/dna_center.md" - Infoblox: "user/integrations/infoblox.md" - IPFabric: "user/integrations/ipfabric.md" - Itential: "user/integrations/itential.md" @@ -128,6 +129,7 @@ nav: - Arista CloudVision: "admin/integrations/aristacv_setup.md" - Bootstrap: "admin/integrations/bootstrap_setup.md" - Device42: "admin/integrations/device42_setup.md" + - DNA Center: "admin/integrations/dna_center_setup.md" - Infoblox: "admin/integrations/infoblox_setup.md" - IPFabric: "admin/integrations/ipfabric_setup.md" - Itential: "admin/integrations/itential_setup.md" diff --git a/nautobot_ssot/exceptions.py b/nautobot_ssot/exceptions.py new file mode 100644 index 000000000..105ad16b1 --- /dev/null +++ b/nautobot_ssot/exceptions.py @@ -0,0 +1,67 @@ +"""Custom Exceptions to be used with SSoT integrations.""" + + +class AdapterLoadException(Exception): + """Raised when there's an error while loading data.""" + + +class AuthFailure(Exception): + """Exception raised when authenticating to endpoint fails.""" + + def __init__(self, error_code, message): + """Populate exception information.""" + self.expression = error_code + self.message = message + super().__init__(self.message) + + +class ConfigurationError(Exception): + """Exception thrown when Job configuration is wrong.""" + + +class JobException(Exception): + """Exception raised when failure loading integration Job.""" + + def __init__(self, message): + """Populate exception information.""" + self.message = message + super().__init__(self.message) + + +class InvalidUrlScheme(Exception): + """Exception raised for wrong scheme being passed for URL. + + Attributes: + message (str): Returned explanation of Error. + """ + + def __init__(self, scheme): + """Initialize Exception with wrong scheme in message.""" + self.message = f"Invalid URL scheme '{scheme}' found!" + super().__init__(self.message) + + +class MissingConfigSetting(Exception): + """Exception raised for missing configuration settings. + + Attributes: + message (str): Returned explanation of Error. + """ + + def __init__(self, setting): + """Initialize Exception with Setting that is missing and message.""" + self.setting = setting + self.message = f"Missing configuration setting - {setting}!" + super().__init__(self.message) + + +class MissingSecretsGroupException(Exception): + """Custom Exception in case SecretsGroup is not found on ExternalIntegration.""" + + +class RequestConnectError(Exception): + """Exception class to be raised upon requests module connection errors.""" + + +class RequestHTTPError(Exception): + """Exception class to be raised upon requests module HTTP errors.""" diff --git a/nautobot_ssot/integrations/aci/diffsync/adapters/aci.py b/nautobot_ssot/integrations/aci/diffsync/adapters/aci.py index 9ff461070..f8d9ab89f 100644 --- a/nautobot_ssot/integrations/aci/diffsync/adapters/aci.py +++ b/nautobot_ssot/integrations/aci/diffsync/adapters/aci.py @@ -435,8 +435,8 @@ def load_devices(self): site=self.site, site_tag=self.site, controller_group=( - self.job.apic.controller_managed_device_group.name - if self.job.apic.controller_managed_device_group + self.job.apic.controller_managed_device_groups.name + if self.job.apic.controller_managed_device_groups else "" ), ) diff --git a/nautobot_ssot/integrations/aci/diffsync/client.py b/nautobot_ssot/integrations/aci/diffsync/client.py index c51b73440..9eb220428 100644 --- a/nautobot_ssot/integrations/aci/diffsync/client.py +++ b/nautobot_ssot/integrations/aci/diffsync/client.py @@ -11,6 +11,8 @@ import requests import urllib3 +from nautobot_ssot.exceptions import RequestConnectError, RequestHTTPError + from .utils import ( ap_from_dn, bd_from_dn, @@ -26,14 +28,6 @@ logger = logging.getLogger(__name__) -class RequestConnectError(Exception): - """Exception class to be raised upon requests module connection errors.""" - - -class RequestHTTPError(Exception): - """Exception class to be raised upon requests module HTTP errors.""" - - class AciApi: """Representation and methods for interacting with aci.""" diff --git a/nautobot_ssot/integrations/aci/diffsync/models/nautobot.py b/nautobot_ssot/integrations/aci/diffsync/models/nautobot.py index 479678f8e..737ea1024 100644 --- a/nautobot_ssot/integrations/aci/diffsync/models/nautobot.py +++ b/nautobot_ssot/integrations/aci/diffsync/models/nautobot.py @@ -449,7 +449,7 @@ def create(cls, adapter, ids, attrs): try: vrf_tenant = OrmTenant.objects.get(name=attrs["vrf_tenant"]) except OrmTenant.DoesNotExist: - adapter.job.logger.warning(f"Tenant {attrs['vrf_tenant']} not found for VRF {attrs['vrf']}") + adapter.job.logger.warning(f"Tenant {attrs['vrf_tenant']} not found for VRF {ids['vrf']}") vrf_tenant = None return None diff --git a/nautobot_ssot/integrations/aci/jobs.py b/nautobot_ssot/integrations/aci/jobs.py index 046cd647b..c851ea4f1 100644 --- a/nautobot_ssot/integrations/aci/jobs.py +++ b/nautobot_ssot/integrations/aci/jobs.py @@ -5,6 +5,7 @@ from nautobot.dcim.models import Controller, Location from nautobot.extras.jobs import BooleanVar, Job, ObjectVar +from nautobot_ssot.exceptions import ConfigurationError from nautobot_ssot.integrations.aci.diffsync.adapters.aci import AciAdapter from nautobot_ssot.integrations.aci.diffsync.adapters.nautobot import NautobotAdapter from nautobot_ssot.integrations.aci.diffsync.client import AciApi @@ -14,10 +15,6 @@ name = "Cisco ACI SSoT" # pylint: disable=invalid-name, abstract-method -class ConfigurationError(Exception): - """Exception thrown when Job configuration is wrong.""" - - class AciDataSource(DataSource, Job): # pylint: disable=abstract-method, too-many-instance-attributes """ACI SSoT Data Source.""" diff --git a/nautobot_ssot/integrations/aristacv/jobs.py b/nautobot_ssot/integrations/aristacv/jobs.py index 735318059..6162dbecc 100644 --- a/nautobot_ssot/integrations/aristacv/jobs.py +++ b/nautobot_ssot/integrations/aristacv/jobs.py @@ -7,6 +7,7 @@ from nautobot.dcim.models import DeviceType from nautobot.extras.jobs import BooleanVar, Job +from nautobot_ssot.exceptions import MissingConfigSetting from nautobot_ssot.integrations.aristacv.diffsync.adapters.cloudvision import CloudvisionAdapter from nautobot_ssot.integrations.aristacv.diffsync.adapters.nautobot import NautobotAdapter from nautobot_ssot.integrations.aristacv.utils.cloudvision import CloudvisionApi @@ -16,20 +17,6 @@ name = "SSoT - Arista CloudVision" # pylint: disable=invalid-name -class MissingConfigSetting(Exception): - """Exception raised for missing configuration settings. - - Attributes: - message (str): Returned explanation of Error. - """ - - def __init__(self, setting): - """Initialize Exception with Setting that is missing and message.""" - self.setting = setting - self.message = f"Missing configuration setting - {setting}!" - super().__init__(self.message) - - class CloudVisionDataSource(DataSource, Job): # pylint: disable=abstract-method """CloudVision SSoT Data Source.""" diff --git a/nautobot_ssot/integrations/aristacv/utils/cloudvision.py b/nautobot_ssot/integrations/aristacv/utils/cloudvision.py index 0477a7a85..2e242e50b 100644 --- a/nautobot_ssot/integrations/aristacv/utils/cloudvision.py +++ b/nautobot_ssot/integrations/aristacv/utils/cloudvision.py @@ -22,6 +22,7 @@ from cvprac.cvp_client import CvpClient, CvpLoginError from google.protobuf.wrappers_pb2 import StringValue # pylint: disable=no-name-in-module +from nautobot_ssot.exceptions import AuthFailure from nautobot_ssot.integrations.aristacv.constants import PORT_TYPE_MAP from nautobot_ssot.integrations.aristacv.types import CloudVisionAppConfig @@ -31,16 +32,6 @@ UPDATES_TYPE = List[UPDATE_TYPE] -class AuthFailure(Exception): - """Exception raised when authenticating to on-prem CVP fails.""" - - def __init__(self, error_code, message): - """Populate exception information.""" - self.expression = error_code - self.message = message - super().__init__(self.message) - - class CloudvisionApi: # pylint: disable=too-many-instance-attributes, too-many-arguments """Arista CloudVision gRPC client.""" diff --git a/nautobot_ssot/integrations/device42/diffsync/adapters/device42.py b/nautobot_ssot/integrations/device42/diffsync/adapters/device42.py index 8b617b3d9..25e7dbdd7 100644 --- a/nautobot_ssot/integrations/device42/diffsync/adapters/device42.py +++ b/nautobot_ssot/integrations/device42/diffsync/adapters/device42.py @@ -201,6 +201,7 @@ def load_buildings(self): _tags.sort() building = self.building( name=record["name"], + location_type=self.job.building_loctype.name, address=sanitize_string(record["address"]) if record.get("address") else "", latitude=float(round(Decimal(record["latitude"] if record["latitude"] else 0.0), 6)), longitude=float(round(Decimal(record["longitude"] if record["longitude"] else 0.0), 6)), @@ -235,6 +236,7 @@ def load_rooms(self): room = self.room( name=record["name"], building=record["building"], + building_loctype=self.job.building_loctype.name, notes=record["notes"] if record.get("notes") else "", custom_fields=get_custom_field_dict(record["custom_fields"]), tags=_tags, @@ -242,7 +244,9 @@ def load_rooms(self): ) try: self.add(room) - _site = self.get(self.building, record.get("building")) + _site = self.get( + self.building, {"name": record.get("building"), "location_type": self.job.building_loctype.name} + ) _site.add_child(child=room) except ObjectAlreadyExists as err: if self.job.debug: @@ -275,7 +279,13 @@ def load_racks(self): try: self.add(rack) _room = self.get( - self.room, {"name": record["room"], "building": record["building"], "room": record["room"]} + self.room, + { + "name": record["room"], + "building": record["building"], + "building_loctype": self.job.building_loctype.name, + "room": record["room"], + }, ) _room.add_child(child=rack) except ObjectAlreadyExists as err: diff --git a/nautobot_ssot/integrations/device42/diffsync/adapters/nautobot.py b/nautobot_ssot/integrations/device42/diffsync/adapters/nautobot.py index d4495787a..184e51575 100644 --- a/nautobot_ssot/integrations/device42/diffsync/adapters/nautobot.py +++ b/nautobot_ssot/integrations/device42/diffsync/adapters/nautobot.py @@ -14,7 +14,6 @@ FrontPort, Interface, Location, - LocationType, Manufacturer, Platform, Rack, @@ -161,11 +160,12 @@ def sync_complete(self, source: Adapter, *args, **kwargs): def load_sites(self): """Add Nautobot Site objects as DiffSync Building models.""" - for site in Location.objects.filter(location_type=LocationType.objects.get_or_create(name="Site")[0]): + for site in Location.objects.filter(location_type=self.job.building_loctype.name): self.site_map[site.name] = site.id try: building = self.building( name=site.name, + location_type=self.job.building_loctype.name, address=site.physical_address, latitude=site.latitude, longitude=site.longitude, @@ -191,12 +191,15 @@ def load_rackgroups(self): room = self.room( name=_rg.name, building=_rg.location.name, + building_loctype=self.job.building_loctype.name, notes=_rg.description, custom_fields=nautobot.get_custom_field_dict(_rg.get_custom_fields()), uuid=_rg.id, ) self.add(room) - _site = self.get(self.building, _rg.location.name) + _site = self.get( + self.building, {"name": _rg.location.name, "location_type": self.job.building_loctype.name} + ) _site.add_child(child=room) def load_racks(self): diff --git a/nautobot_ssot/integrations/device42/diffsync/models/__init__.py b/nautobot_ssot/integrations/device42/diffsync/models/__init__.py index 132b83161..f09843ad5 100644 --- a/nautobot_ssot/integrations/device42/diffsync/models/__init__.py +++ b/nautobot_ssot/integrations/device42/diffsync/models/__init__.py @@ -35,6 +35,12 @@ NautobotRoom, NautobotVendor, ) +from nautobot_ssot.integrations.device42.diffsync.models.nautobot.ipam import ( + NautobotIPAddress, + NautobotSubnet, + NautobotVLAN, + NautobotVRFGroup, +) __all__ = ( "PatchPanel", @@ -65,8 +71,12 @@ "NautobotConnection", "NautobotDevice", "NautobotHardware", + "NautobotIPAddress", "NautobotPort", "NautobotRack", "NautobotRoom", + "NautobotSubnet", "NautobotVendor", + "NautobotVLAN", + "NautobotVRFGroup", ) diff --git a/nautobot_ssot/integrations/device42/diffsync/models/base/dcim.py b/nautobot_ssot/integrations/device42/diffsync/models/base/dcim.py index 9515bc10b..3867daec2 100644 --- a/nautobot_ssot/integrations/device42/diffsync/models/base/dcim.py +++ b/nautobot_ssot/integrations/device42/diffsync/models/base/dcim.py @@ -10,10 +10,11 @@ class Building(DiffSyncModel): """Base Building model.""" _modelname = "building" - _identifiers = ("name",) + _identifiers = ("name", "location_type") _attributes = ("address", "latitude", "longitude", "contact_name", "contact_phone", "tags", "custom_fields") _children = {"room": "rooms"} name: str + location_type: str address: Optional[str] = None latitude: Optional[float] = None longitude: Optional[float] = None @@ -29,11 +30,12 @@ class Room(DiffSyncModel): """Base Room model.""" _modelname = "room" - _identifiers = ("name", "building") + _identifiers = ("name", "building", "building_loctype") _attributes = ("notes", "custom_fields") _children = {"rack": "racks"} name: str building: str + building_loctype: str notes: Optional[str] = None racks: List["Rack"] = [] custom_fields: Optional[dict] = None diff --git a/nautobot_ssot/integrations/device42/diffsync/models/nautobot/dcim.py b/nautobot_ssot/integrations/device42/diffsync/models/nautobot/dcim.py index 2ac93bb94..90d2d062a 100644 --- a/nautobot_ssot/integrations/device42/diffsync/models/nautobot/dcim.py +++ b/nautobot_ssot/integrations/device42/diffsync/models/nautobot/dcim.py @@ -14,7 +14,6 @@ from nautobot.dcim.models import FrontPort as OrmFrontPort from nautobot.dcim.models import Interface as OrmInterface from nautobot.dcim.models import Location as OrmSite -from nautobot.dcim.models import LocationType as OrmLocationType from nautobot.dcim.models import Manufacturer as OrmManufacturer from nautobot.dcim.models import Rack as OrmRack from nautobot.dcim.models import RackGroup as OrmRackGroup @@ -52,16 +51,15 @@ class NautobotBuilding(Building): @classmethod def create(cls, adapter, ids, attrs): """Create Site object in Nautobot.""" - adapter.job.logger.info(f"Creating Site {ids['name']}.") + adapter.job.logger.info(f"Creating {ids['location_type']} {ids['name']}.") def_site_status = adapter.status_map[DEFAULTS.get("site_status")] - loc_type = OrmLocationType.objects.get_or_create(name="Site")[0] new_site = OrmSite( name=ids["name"], status_id=def_site_status, physical_address=attrs["address"] if attrs.get("address") else "", latitude=round(Decimal(attrs["latitude"] if attrs["latitude"] else 0.0), 6), longitude=round(Decimal(attrs["longitude"] if attrs["longitude"] else 0.0), 6), - location_type=loc_type, + location_type=adapter.job.building_loctype, contact_name=attrs["contact_name"] if attrs.get("contact_name") else "", contact_phone=attrs["contact_phone"] if attrs.get("contact_phone") else "", ) diff --git a/nautobot_ssot/integrations/device42/jobs.py b/nautobot_ssot/integrations/device42/jobs.py index 3cf996049..5dbe369bc 100644 --- a/nautobot_ssot/integrations/device42/jobs.py +++ b/nautobot_ssot/integrations/device42/jobs.py @@ -3,12 +3,14 @@ from django.templatetags.static import static from django.urls import reverse +from nautobot.dcim.models import LocationType from nautobot.extras.jobs import BooleanVar, ObjectVar from nautobot.extras.models import ExternalIntegration from nautobot_ssot.integrations.device42.diffsync.adapters.device42 import Device42Adapter from nautobot_ssot.integrations.device42.diffsync.adapters.nautobot import NautobotAdapter from nautobot_ssot.integrations.device42.utils.device42 import Device42API +from nautobot_ssot.integrations.device42.utils.nautobot import ensure_contenttypes_on_location_type from nautobot_ssot.jobs.base import DataMapping, DataSource from nautobot_ssot.utils import get_username_password_https_from_secretsgroup @@ -18,6 +20,11 @@ class Device42DataSource(DataSource): # pylint: disable=too-many-instance-attributes """Device42 SSoT Data Source.""" + debug = BooleanVar(description="Enable for more verbose debug logging", default=False) + bulk_import = BooleanVar( + description="Perform bulk operations when importing data. CAUTION! Might cause bad data to be pushed to Nautobot.", + default=False, + ) integration = ObjectVar( model=ExternalIntegration, queryset=ExternalIntegration.objects.all(), @@ -25,8 +32,14 @@ class Device42DataSource(DataSource): # pylint: disable=too-many-instance-attri required=True, label="Device42 Instance", ) - debug = BooleanVar(description="Enable for more verbose debug logging", default=False) - bulk_import = BooleanVar(description="Enable using bulk create option for object creation.", default=False) + building_loctype = ObjectVar( + model=LocationType, + queryset=LocationType.objects.all(), + display_field="name", + required=False, + label="Building LocationType", + description="LocationType to use for imported Buildings from Device42. If unspecified, will revert to Site LocationType.", + ) class Meta: """Meta data for Device42.""" @@ -46,7 +59,7 @@ def data_mappings(cls): """List describing the data mappings involved in this DataSource.""" return ( DataMapping( - "Buildings", "/admin/rackraj/building/", "Sites", reverse("dcim:site_list") + "Buildings", "/admin/rackraj/building/", "Locations", reverse("dcim:location_list") ), DataMapping( "Rooms", @@ -134,11 +147,15 @@ def load_target_adapter(self): self.target_adapter.load() def run( # pylint: disable=arguments-differ, too-many-arguments - self, dryrun, memory_profiling, integration, debug, bulk_import, *args, **kwargs + self, dryrun, memory_profiling, integration, debug, bulk_import, building_loctype, *args, **kwargs ): """Perform data synchronization.""" self.integration = integration self.bulk_import = bulk_import + self.building_loctype = building_loctype + if not self.building_loctype: + self.building_loctype = LocationType.objects.get_or_create(name="Site")[0] + ensure_contenttypes_on_location_type(location_type=self.building_loctype) self.debug = debug self.dryrun = dryrun self.memory_profiling = memory_profiling diff --git a/nautobot_ssot/integrations/device42/signals.py b/nautobot_ssot/integrations/device42/signals.py deleted file mode 100644 index 5692db849..000000000 --- a/nautobot_ssot/integrations/device42/signals.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Signals for Device42 integration.""" - -from nautobot.core.signals import nautobot_database_ready - - -def register_signals(sender): - """Register signals for IPFabric integration.""" - nautobot_database_ready.connect(nautobot_database_ready_callback, sender=sender) - - -# pylint: disable=unused-argument, invalid-name -def nautobot_database_ready_callback(sender, *, apps, **kwargs): - """Ensure Site LocationType created and configured correctly. - - Callback function triggered by the nautobot_database_ready signal when the Nautobot database is fully ready. - """ - ContentType = apps.get_model("contenttypes", "ContentType") - Device = apps.get_model("dcim", "Device") - Site = apps.get_model("dcim", "Location") - RackGroup = apps.get_model("dcim", "RackGroup") - Rack = apps.get_model("dcim", "Rack") - Prefix = apps.get_model("ipam", "Prefix") - VLAN = apps.get_model("ipam", "VLAN") - LocationType = apps.get_model("dcim", "LocationType") - VirtualChassis = apps.get_model("dcim", "VirtualChassis") - - loc_type = LocationType.objects.update_or_create(name="Site")[0] - for obj_type in [Site, RackGroup, Rack, Device, VirtualChassis, Prefix, VLAN]: - loc_type.content_types.add(ContentType.objects.get_for_model(obj_type)) - loc_type.save() diff --git a/nautobot_ssot/integrations/device42/utils/device42.py b/nautobot_ssot/integrations/device42/utils/device42.py index ec2917fb2..4eeb4289f 100644 --- a/nautobot_ssot/integrations/device42/utils/device42.py +++ b/nautobot_ssot/integrations/device42/utils/device42.py @@ -13,20 +13,6 @@ from nautobot_ssot.integrations.device42.diffsync.models.base.ipam import VLAN -class MissingConfigSetting(Exception): - """Exception raised for missing configuration settings. - - Attributes: - message (str): Returned explanation of Error. - """ - - def __init__(self, setting): - """Initialize Exception with Setting that is missing and message.""" - self.setting = setting - self.message = f"Missing configuration setting - {setting}!" - super().__init__(self.message) - - def merge_offset_dicts(orig_dict: dict, offset_dict: dict) -> dict: """Method to merge two dicts and merge a list if found. diff --git a/nautobot_ssot/integrations/device42/utils/nautobot.py b/nautobot_ssot/integrations/device42/utils/nautobot.py index 36e023c73..f0b437f72 100644 --- a/nautobot_ssot/integrations/device42/utils/nautobot.py +++ b/nautobot_ssot/integrations/device42/utils/nautobot.py @@ -9,9 +9,10 @@ from diffsync.exceptions import ObjectNotFound from django.contrib.contenttypes.models import ContentType from nautobot.circuits.models import CircuitType -from nautobot.dcim.models import Device, Interface, Platform +from nautobot.dcim.models import Device, Interface, Location, LocationType, Platform, Rack, RackGroup, VirtualChassis from nautobot.extras.choices import CustomFieldTypeChoices from nautobot.extras.models import CustomField, Relationship, Role, Tag +from nautobot.ipam.models import VLAN, Prefix from netutils.lib_mapper import ANSIBLE_LIB_MAPPER_REVERSE, NAPALM_LIB_MAPPER_REVERSE from taggit.managers import TaggableManager @@ -362,3 +363,13 @@ def apply_vlans_to_port(adapter, device_name: str, mode: str, vlans: list, port: tagged_vlans.append(tagged_vlan) port.tagged_vlans.set(tagged_vlans) port.validated_save() + + +def ensure_contenttypes_on_location_type(location_type: LocationType): + """Ensure that the required ContentTypes are on the specified Building LocationType. + + Args: + location_type (LocationType): The specified LocationType to use when importing Building Locations. + """ + for obj_type in [Location, RackGroup, Rack, Device, VirtualChassis, Prefix, VLAN]: + location_type.content_types.add(ContentType.objects.get_for_model(obj_type)) diff --git a/nautobot_ssot/integrations/dna_center/diffsync/adapters/dna_center.py b/nautobot_ssot/integrations/dna_center/diffsync/adapters/dna_center.py index abccda16a..b101af86c 100644 --- a/nautobot_ssot/integrations/dna_center/diffsync/adapters/dna_center.py +++ b/nautobot_ssot/integrations/dna_center/diffsync/adapters/dna_center.py @@ -72,13 +72,44 @@ def load_locations(self): def load_controller_locations(self): """Load location data for Controller specified in Job form.""" - if self.job.dnac.location.location_type.name == "Site": + if self.job.dnac.location.location_type == self.job.floor_loctype: + self.get_or_instantiate( + self.floor, + ids={ + "name": self.job.dnac.location.name, + "building": self.job.dnac.location.parent.name, + }, + attrs={ + "tenant": self.job.dnac.location.tenant.name if self.job.dnac.location.tenant else None, + "uuid": None, + }, + ) + if ( + self.job.dnac.location.parent.parent + and self.job.dnac.location.parent.parent.location_type == self.job.building_loctype + ): self.get_or_instantiate( self.building, - ids={"name": self.job.dnac.location.name}, + ids={ + "name": self.job.dnac.location.parent.parent.name, + "parent": ( + self.job.dnac.location.parent.parent.parent.name + if self.job.dnac.location.parent.parent.parent + else None + ), + }, + attrs={"uuid": None}, + ) + + if self.job.dnac.location.location_type == self.job.building_loctype: + self.get_or_instantiate( + self.building, + ids={ + "name": self.job.dnac.location.name, + "area": self.job.dnac.location.parent.name if self.job.dnac.location.parent else None, + }, attrs={ "address": self.job.dnac.location.physical_address, - "area": self.job.dnac.location.parent.name if self.job.dnac.location.parent else None, "area_parent": ( self.job.dnac.location.parent.parent.name if self.job.dnac.location.parent and self.job.dnac.location.parent.parent @@ -90,7 +121,7 @@ def load_controller_locations(self): "uuid": None, }, ) - if self.job.dnac.location.parent.location_type.name == "Region": + if self.job.dnac.location.parent.location_type == self.job.area_loctype: self.get_or_instantiate( self.area, ids={ @@ -101,7 +132,10 @@ def load_controller_locations(self): }, attrs={"uuid": None}, ) - if self.job.dnac.location.parent.parent and self.job.dnac.location.parent.parent.location_type.name == "Region": + if ( + self.job.dnac.location.parent.parent + and self.job.dnac.location.parent.parent.location_type == self.job.area_loctype + ): self.get_or_instantiate( self.area, ids={ @@ -138,9 +172,11 @@ def load_areas(self, areas: List[dict]): ) if loaded: if self.job.debug: - self.job.logger.info(f"Loaded area {location['name']}. {location}") + self.job.logger.info(f"Loaded {self.job.area_loctype.name} {location['name']}. {location}") else: - self.job.logger.warning(f"Duplicate area {location['name']} attempting to be loaded.") + self.job.logger.warning( + f"Duplicate {self.job.area_loctype.name} {location['name']} attempting to be loaded." + ) def load_buildings(self, buildings: List[dict]): """Load building data from DNAC into DiffSync model. @@ -149,19 +185,21 @@ def load_buildings(self, buildings: List[dict]): buildings (List[dict]): List of dictionaries containing location information about a building. """ for location in buildings: + if location["parentId"] in self.dnac_location_map: + _area = self.dnac_location_map[location["parentId"]] + else: + _area = {"name": "Global", "parent": None} try: - self.get(self.building, location["name"]) - self.job.logger.warning(f"Building {location['name']} already loaded so skipping.") + self.get(self.building, {"name": location["name"], "area": _area["name"]}) + self.job.logger.warning( + f"{self.job.building_loctype.name} {location['name']} already loaded so skipping." + ) continue except ObjectNotFound: if self.job.debug: - self.job.logger.info(f"Loading building {location['name']}. {location}") + self.job.logger.info(f"Loading {self.job.building_loctype.name} {location['name']}. {location}") address, _ = self.conn.find_address_and_type(info=location["additionalInfo"]) latitude, longitude = self.conn.find_latitude_and_longitude(info=location["additionalInfo"]) - if location["parentId"] in self.dnac_location_map: - _area = self.dnac_location_map[location["parentId"]] - else: - _area = {"name": "Global", "parent": None} new_building = self.building( name=location["name"], address=address if address else "", @@ -193,7 +231,10 @@ def load_floors(self, floors: List[dict]): continue floor_name = f"{_building['name']} - {location['name']}" try: - self.get(self.floor, {"name": floor_name, "building": _building["name"]}) + self.get( + self.floor, + {"name": floor_name, "building": _building["name"]}, + ) self.job.logger.warning(f"Duplicate Floor {floor_name} attempting to be loaded.") except ObjectNotFound: new_floor = self.floor( @@ -205,11 +246,14 @@ def load_floors(self, floors: List[dict]): try: self.add(new_floor) try: - parent = self.get(self.building, _building["name"]) + parent = self.get( + self.building, + {"name": _building["name"], "area": self.dnac_location_map[location["parentId"]]["parent"]}, + ) parent.add_child(new_floor) except ObjectNotFound as err: self.job.logger.warning( - f"Unable to find building {_building['name']} for floor {floor_name}. {err}" + f"Unable to find {self.job.building_loctype.name} {_building['name']} for {self.job.floor_loctype.name} {floor_name}. {err}" ) except ValidationError as err: self.job.logger.warning(f"Unable to load floor {floor_name}. {err}") diff --git a/nautobot_ssot/integrations/dna_center/diffsync/adapters/nautobot.py b/nautobot_ssot/integrations/dna_center/diffsync/adapters/nautobot.py index 973b8ec22..542c0d359 100644 --- a/nautobot_ssot/integrations/dna_center/diffsync/adapters/nautobot.py +++ b/nautobot_ssot/integrations/dna_center/diffsync/adapters/nautobot.py @@ -40,7 +40,6 @@ NautobotPort, NautobotPrefix, ) -from nautobot_ssot.jobs.base import DataTarget class NautobotAdapter(Adapter): @@ -69,13 +68,11 @@ class NautobotAdapter(Adapter): prefix_map = {} ipaddr_map = {} - def __init__( - self, *args, job: Optional[DataTarget] = None, sync=None, tenant: Optional[OrmTenant] = None, **kwargs - ): + def __init__(self, *args, job, sync=None, tenant: Optional[OrmTenant] = None, **kwargs): """Initialize Nautobot. Args: - job (DataTarget, optional): Nautobot job. Defaults to None. + job (DataSource): Nautobot job. sync (object, optional): Nautobot DiffSync. Defaults to None. tenant (OrmTenant, optional): Tenant defined in Job form that all non-location objects should belong to. """ @@ -86,85 +83,82 @@ def __init__( self.objects_to_create = defaultdict(list) self.objects_to_delete = defaultdict(list) - def load_regions(self): - """Load Region data from Nautobt into DiffSync models.""" - try: - locations = OrmLocation.objects.filter(location_type=self.locationtype_map["Region"]).select_related( - "parent" - ) - for region in locations: - parent = None - if region.parent: - parent = region.parent.name - if parent not in self.region_map: - self.region_map[parent] = {} - self.region_map[parent][region.name] = region.id - try: - self.get(self.area, {"name": region.name, "parent": parent}) - self.job.logger.warning(f"Region {region.name} already loaded so skipping duplicate.") - except ObjectNotFound: - new_region = self.area( - name=region.name, - parent=parent, - uuid=region.id, - ) - if not PLUGIN_CFG.get("dna_center_delete_locations"): - new_region.model_flags = DiffSyncModelFlags.SKIP_UNMATCHED_DST - self.add(new_region) - except OrmLocationType.DoesNotExist as err: - self.job.logger.warning( - f"Unable to find LocationType: Region so can't find region Locations to load. {err}" - ) + def load_areas(self): + """Load Location data from Nautobot for specified Area LocationType into DiffSync models.""" + areas = OrmLocation.objects.filter(location_type=self.job.area_loctype).select_related("parent") + for area in areas: + parent = None + if area.parent: + parent = area.parent.name + if parent not in self.region_map: + self.region_map[parent] = {} + self.region_map[parent][area.name] = area.id + try: + self.get(self.area, {"name": area.name, "parent": parent}) + self.job.logger.warning( + f"{self.job.area_loctype.name} {area.name} already loaded so skipping duplicate." + ) + except ObjectNotFound: + new_region = self.area( + name=area.name, + parent=parent, + uuid=area.id, + ) + if not PLUGIN_CFG.get("dna_center_delete_locations"): + new_region.model_flags = DiffSyncModelFlags.SKIP_UNMATCHED_DST + self.add(new_region) - def load_sites(self): - """Load Site data from Nautobot into DiffSync models.""" - try: - locations = OrmLocation.objects.filter(location_type=self.locationtype_map["Site"]) - for site in locations: - self.site_map[site.name] = site.id - try: - self.get(self.building, {"name": site.name, "area": site.parent.name if site.parent else None}) - except ObjectNotFound: - new_building = self.building( - name=site.name, - address=site.physical_address, - area=site.parent.name if site.parent else "", - area_parent=site.parent.parent.name if site.parent and site.parent.parent else None, - latitude=str(site.latitude).rstrip("0"), - longitude=str(site.longitude).rstrip("0"), - tenant=site.tenant.name if site.tenant else None, - uuid=site.id, - ) - if not PLUGIN_CFG.get("dna_center_delete_locations"): - new_building.model_flags = DiffSyncModelFlags.SKIP_UNMATCHED_DST - self.add(new_building) - except OrmLocationType.DoesNotExist as err: - self.job.logger.warning(f"Unable to find LocationType: Site so can't find site Locations to load. {err}") + def load_buildings(self): + """Load Location data from Nautobot for specified Building LocationType into DiffSync models.""" + buildings = OrmLocation.objects.filter(location_type=self.job.building_loctype) + for building in buildings: + self.site_map[building.name] = building.id + try: + self.get( + self.building, + { + "name": building.name, + "area": building.parent.name if building.parent else None, + }, + ) + except ObjectNotFound: + new_building = self.building( + name=building.name, + address=building.physical_address, + area=building.parent.name if building.parent else "", + area_parent=building.parent.parent.name if building.parent and building.parent.parent else None, + latitude=str(building.latitude).rstrip("0"), + longitude=str(building.longitude).rstrip("0"), + tenant=building.tenant.name if building.tenant else None, + uuid=building.id, + ) + if not PLUGIN_CFG.get("dna_center_delete_locations"): + new_building.model_flags = DiffSyncModelFlags.SKIP_UNMATCHED_DST + self.add(new_building) def load_floors(self): """Load LocationType floors from Nautobot into DiffSync models.""" - try: - loc_type = OrmLocationType.objects.get(name="Floor") - locations = OrmLocation.objects.filter(location_type=loc_type) - for location in locations: - self.floor_map[location.name] = location.id - new_floor = self.floor( - name=location.name, - building=location.parent.name if location.parent else "", - tenant=location.tenant.name if location.tenant else None, - uuid=location.id, - ) - self.add(new_floor) - try: - if location.parent: - building = self.get(self.building, location.parent.name) - building.add_child(new_floor) - except ObjectNotFound as err: - self.job.logger.warning( - f"Unable to load building {location.parent.name} for floor {location.name}. {err}" + floors = OrmLocation.objects.filter(location_type=self.job.floor_loctype) + for floor in floors: + self.floor_map[floor.name] = floor.id + new_floor = self.floor( + name=floor.name, + building=floor.parent.name if floor.parent else "", + tenant=floor.tenant.name if floor.tenant else None, + uuid=floor.id, + ) + self.add(new_floor) + try: + if floor.parent: + building = self.get( + self.building, + {"name": floor.parent.name, "area": floor.parent.parent.name}, ) - except OrmLocationType.DoesNotExist as err: - self.job.logger.warning(f"Unable to find LocationType: Floor so can't find floor Locations to load. {err}") + building.add_child(new_floor) + except ObjectNotFound as err: + self.job.logger.warning( + f"Unable to load {self.job.building_loctype.name} {floor.parent.name} for {self.job.floor_loctype.name} {floor.name}. {err}" + ) def load_devices(self): """Load Device data from Nautobot into DiffSync models.""" @@ -392,8 +386,8 @@ def load(self): self.tenant_map = {tenant.name: tenant.id for tenant in OrmTenant.objects.only("id", "name")} self.namespace_map = {ns.name: ns.id for ns in Namespace.objects.only("id", "name")} - self.load_regions() - self.load_sites() + self.load_areas() + self.load_buildings() self.load_floors() self.load_devices() self.load_ports() diff --git a/nautobot_ssot/integrations/dna_center/diffsync/models/base.py b/nautobot_ssot/integrations/dna_center/diffsync/models/base.py index d2f3e70a2..9da609aa7 100644 --- a/nautobot_ssot/integrations/dna_center/diffsync/models/base.py +++ b/nautobot_ssot/integrations/dna_center/diffsync/models/base.py @@ -24,8 +24,8 @@ class Building(DiffSyncModel): """DiffSync model for DNA Center buildings.""" _modelname = "building" - _identifiers = ("name",) - _attributes = ("address", "area", "area_parent", "latitude", "longitude", "tenant") + _identifiers = ("name", "area") + _attributes = ("address", "area_parent", "latitude", "longitude", "tenant") _children = {"floor": "floors"} name: str diff --git a/nautobot_ssot/integrations/dna_center/diffsync/models/nautobot.py b/nautobot_ssot/integrations/dna_center/diffsync/models/nautobot.py index 9ff2d6f18..e712b538e 100644 --- a/nautobot_ssot/integrations/dna_center/diffsync/models/nautobot.py +++ b/nautobot_ssot/integrations/dna_center/diffsync/models/nautobot.py @@ -10,7 +10,6 @@ DeviceType, Interface, Location, - LocationType, Manufacturer, ) from nautobot.extras.models import Role @@ -38,23 +37,25 @@ class NautobotArea(base.Area): def create(cls, adapter, ids, attrs): """Create Region in Nautobot from Area object.""" if adapter.job.debug: - adapter.job.logger.info(f"Creating Region {ids['name']} in {ids['parent']}.") - new_region = Location( + adapter.job.logger.info(f"Creating {adapter.job.area_loctype.name} {ids['name']} in {ids['parent']}.") + new_area = Location( name=ids["name"], - location_type_id=adapter.locationtype_map["Region"], + location_type=adapter.job.area_loctype, status_id=adapter.status_map["Active"], ) try: parents_parent = "Global" if ids["parent"] == "Global": parents_parent = None - new_region.parent_id = adapter.region_map[parents_parent][ids["parent"]] + new_area.parent_id = adapter.region_map[parents_parent][ids["parent"]] except KeyError: - adapter.job.logger.warning(f"Unable to find Region {ids['parent']} for {ids['name']}.") - new_region.validated_save() + adapter.job.logger.warning( + f"Unable to find {adapter.job.area_loctype.name} {ids['parent']} for {ids['name']}." + ) + new_area.validated_save() if ids["parent"] not in adapter.region_map: adapter.region_map[ids["parent"]] = {} - adapter.region_map[ids["parent"]][ids["name"]] = new_region.id + adapter.region_map[ids["parent"]][ids["name"]] = new_area.id return super().create(adapter=adapter, ids=ids, attrs=attrs) def delete(self): @@ -66,7 +67,7 @@ def delete(self): return None area = Location.objects.get(id=self.uuid) if self.adapter.job.debug: - self.adapter.job.logger.info(f"Deleting Region {area.name}.") + self.adapter.job.logger.info(f"Deleting {self.job.area_loctype.name} {area.name}.") self.adapter.objects_to_delete["regions"].append(area) return self @@ -79,19 +80,19 @@ def create(cls, adapter, ids, attrs): """Create Site in Nautobot from Building object.""" if adapter.job.debug: adapter.job.logger.info(f"Creating Site {ids['name']}.") - new_site = Location( + new_building = Location( name=ids["name"], - location_type_id=adapter.locationtype_map["Site"], - parent_id=adapter.region_map[attrs["area_parent"]][attrs["area"]], + location_type=adapter.job.building_loctype, + parent_id=adapter.region_map[attrs["area_parent"]][ids["area"]], physical_address=attrs["address"] if attrs.get("address") else "", status_id=adapter.status_map["Active"], latitude=attrs["latitude"], longitude=attrs["longitude"], ) if attrs.get("tenant"): - new_site.tenant_id = adapter.tenant_map[attrs["tenant"]] - new_site.validated_save() - adapter.site_map[ids["name"]] = new_site.id + new_building.tenant_id = adapter.tenant_map[attrs["tenant"]] + new_building.validated_save() + adapter.site_map[ids["name"]] = new_building.id return super().create(adapter=adapter, ids=ids, attrs=attrs) def update(self, attrs): @@ -129,7 +130,7 @@ def delete(self): return None site = Location.objects.get(id=self.uuid) if self.adapter.job.debug: - self.adapter.job.logger.info(f"Deleting Site {site.name}.") + self.adapter.job.logger.info(f"Deleting {self.adapter.job.building_loctype.name} {site.name}.") self.adapter.objects_to_delete["sites"].append(site) return self @@ -141,12 +142,12 @@ class NautobotFloor(base.Floor): def create(cls, adapter, ids, attrs): """Create LocationType: Floor in Nautobot from Floor object.""" if adapter.job.debug: - adapter.job.logger.info(f"Creating Floor {ids['name']}.") + adapter.job.logger.info(f"Creating {adapter.job.floor_loctype.name} {ids['name']}.") new_floor = Location( name=ids["name"], status_id=adapter.status_map["Active"], parent_id=adapter.site_map[ids["building"]], - location_type_id=adapter.locationtype_map["Floor"], + location_type=adapter.job.floor_loctype, ) if attrs.get("tenant"): new_floor.tenant_id = adapter.tenant_map[attrs["tenant"]] @@ -156,9 +157,9 @@ def create(cls, adapter, ids, attrs): def update(self, attrs): """Update LocationType: Floor in Nautobot from Floor object.""" - floor = Location.objects.get(name=self.name, location_type=LocationType.objects.get(name="Floor")) + floor = Location.objects.get(name=self.name, location_type=self.adapter.job.floor_loctype) if self.adapter.job.debug: - self.adapter.job.logger.info(f"Updating Floor {floor.name} with {attrs}") + self.adapter.job.logger.info(f"Updating {self.adapter.job.floor_loctype.name} {floor.name} with {attrs}") if "tenant" in attrs: if attrs.get("tenant"): floor.tenant_id = self.adapter.tenant_map[attrs["tenant"]] @@ -176,7 +177,9 @@ def delete(self): return None floor = Location.objects.get(id=self.uuid) if self.adapter.job.debug: - self.adapter.job.logger.info(f"Deleting Floor {floor.name} in {floor.parent.name}.") + self.adapter.job.logger.info( + f"Deleting {self.adapter.job.floor_loctype.name} {floor.name} in {floor.parent.name}." + ) self.adapter.objects_to_delete["floors"].append(floor) return self diff --git a/nautobot_ssot/integrations/dna_center/jobs.py b/nautobot_ssot/integrations/dna_center/jobs.py index e52af5189..282770ff7 100644 --- a/nautobot_ssot/integrations/dna_center/jobs.py +++ b/nautobot_ssot/integrations/dna_center/jobs.py @@ -2,12 +2,13 @@ from django.templatetags.static import static from django.urls import reverse +from nautobot.apps.jobs import BooleanVar, JSONVar, ObjectVar from nautobot.core.celery import register_jobs -from nautobot.dcim.models import Controller, ControllerManagedDeviceGroup +from nautobot.dcim.models import Controller, ControllerManagedDeviceGroup, LocationType from nautobot.extras.choices import SecretsGroupAccessTypeChoices, SecretsGroupSecretTypeChoices -from nautobot.extras.jobs import BooleanVar, ObjectVar from nautobot.tenancy.models import Tenant +from nautobot_ssot.exceptions import ConfigurationError from nautobot_ssot.integrations.dna_center.diffsync.adapters import dna_center, nautobot from nautobot_ssot.integrations.dna_center.utils.dna_center import DnaCenterClient from nautobot_ssot.jobs.base import DataMapping, DataSource @@ -25,8 +26,42 @@ class DnaCenterDataSource(DataSource): # pylint: disable=too-many-instance-attr required=True, label="DNA Center Controller", ) + area_loctype = ObjectVar( + model=LocationType, + queryset=LocationType.objects.all(), + display_field="display", + required=True, + label="Area LocationType", + description="LocationType to use for imported DNA Center Areas. Must allow nesting.", + ) + building_loctype = ObjectVar( + model=LocationType, + queryset=LocationType.objects.all(), + display_field="display", + required=True, + label="Building LocationType", + description="LocationType to use for imported DNA Center Buildings.", + ) + floor_loctype = ObjectVar( + model=LocationType, + queryset=LocationType.objects.all(), + display_field="display", + required=True, + label="Floor LocationType", + description="LocationType to use for imported DNA Center Floors.", + ) + location_map = JSONVar( + label="Location Mapping", + required=False, + default={}, + description="Map of information regarding Locations in DNA Center. Ex: {'': {'parent': ''}}", + ) + debug = BooleanVar(description="Enable for more verbose debug logging", default=False) - bulk_import = BooleanVar(description="Perform bulk operations when importing data", default=False) + bulk_import = BooleanVar( + description="Perform bulk operations when importing data. CAUTION! Might cause bad data to be pushed to Nautobot.", + default=False, + ) tenant = ObjectVar(model=Tenant, label="Tenant", required=False) class Meta: # pylint: disable=too-few-public-methods @@ -37,6 +72,18 @@ class Meta: # pylint: disable=too-few-public-methods data_target = "Nautobot" description = "Sync information from DNA Center to Nautobot" data_source_icon = static("nautobot_ssot_dna_center/dna_center_logo.png") + has_sensitive_variables = False + field_order = [ + "dryrun", + "bulk_import", + "debug", + "dnac", + "area_loctype", + "building_loctype", + "floor_loctype", + "location_map", + "tenant", + ] def __init__(self): """Initiailize Job vars.""" @@ -99,12 +146,40 @@ def load_target_adapter(self): self.target_adapter = nautobot.NautobotAdapter(job=self, sync=self.sync, tenant=self.tenant) self.target_adapter.load() + def validate_locationtypes(self): + """Validate the LocationTypes specified are related and configured correctly.""" + if not self.area_loctype.nestable: + self.logger.error("Area LocationType is not nestable.") + raise ConfigurationError(f"{self.area_loctype.name} LocationType is not nestable.") + if self.building_loctype.parent != self.area_loctype: + self.logger.error( + "LocationType %s is not the parent of %s LocationType. The Area and Building LocationTypes specified must be related.", + self.area_loctype.name, + self.building_loctype.name, + ) + raise ConfigurationError( + f"{self.area_loctype.name} is not parent to {self.building_loctype.name}. Please correct.", + ) + if self.floor_loctype.parent != self.building_loctype: + self.logger.error( + "LocationType %s is not the parent of %s LocationType. The Building and Floor LocationTypes specified must be related.", + self.building_loctype.name, + self.floor_loctype.name, + ) + raise ConfigurationError( + f"{self.building_loctype.name} is not parent to {self.floor_loctype.name}. Please correct.", + ) + def run( self, dryrun, memory_profiling, debug, dnac, + area_loctype, + building_loctype, + floor_loctype, + location_map, bulk_import, tenant, *args, @@ -112,6 +187,11 @@ def run( ): """Perform data synchronization.""" self.dnac = dnac + self.area_loctype = area_loctype + self.building_loctype = building_loctype + self.floor_loctype = floor_loctype + self.validate_locationtypes() + self.location_map = location_map self.tenant = tenant self.debug = debug self.bulk_import = bulk_import diff --git a/nautobot_ssot/integrations/dna_center/signals.py b/nautobot_ssot/integrations/dna_center/signals.py index 8310af49f..cf7643f4c 100644 --- a/nautobot_ssot/integrations/dna_center/signals.py +++ b/nautobot_ssot/integrations/dna_center/signals.py @@ -17,7 +17,6 @@ def nautobot_database_ready_callback(sender, *, apps, **kwargs): # pylint: disa # pylint: disable=invalid-name ContentType = apps.get_model("contenttypes", "ContentType") CustomField = apps.get_model("extras", "CustomField") - LocationType = apps.get_model("dcim", "LocationType") Device = apps.get_model("dcim", "Device") Rack = apps.get_model("dcim", "Rack") RackGroup = apps.get_model("dcim", "RackGroup") @@ -25,12 +24,6 @@ def nautobot_database_ready_callback(sender, *, apps, **kwargs): # pylint: disa IPAddress = apps.get_model("ipam", "IPAddress") Prefix = apps.get_model("ipam", "Prefix") - region = LocationType.objects.update_or_create(name="Region", defaults={"nestable": True})[0] - site = LocationType.objects.update_or_create(name="Site", defaults={"nestable": False, "parent": region})[0] - site.content_types.add(ContentType.objects.get_for_model(Device)) - floor = LocationType.objects.update_or_create(name="Floor", defaults={"nestable": False, "parent": site})[0] - floor.content_types.add(ContentType.objects.get_for_model(Device)) - ver_dict = { "key": "os_version", "type": CustomFieldTypeChoices.TYPE_TEXT, diff --git a/nautobot_ssot/integrations/infoblox/diffsync/adapters/infoblox.py b/nautobot_ssot/integrations/infoblox/diffsync/adapters/infoblox.py index 75368306b..29f20010e 100644 --- a/nautobot_ssot/integrations/infoblox/diffsync/adapters/infoblox.py +++ b/nautobot_ssot/integrations/infoblox/diffsync/adapters/infoblox.py @@ -8,6 +8,7 @@ from diffsync.exceptions import ObjectAlreadyExists from nautobot.extras.plugins.exceptions import PluginImproperlyConfigured +from nautobot_ssot.exceptions import AdapterLoadException from nautobot_ssot.integrations.infoblox.choices import FixedAddressTypeChoices from nautobot_ssot.integrations.infoblox.diffsync.models.infoblox import ( InfobloxDnsARecord, @@ -27,10 +28,6 @@ ) -class AdapterLoadException(Exception): - """Raised when there's an error while loading data.""" - - class InfobloxAdapter(Adapter): """DiffSync adapter using requests to communicate to Infoblox server.""" diff --git a/nautobot_ssot/integrations/infoblox/utils/client.py b/nautobot_ssot/integrations/infoblox/utils/client.py index 6f43442f5..d6ade64c3 100644 --- a/nautobot_ssot/integrations/infoblox/utils/client.py +++ b/nautobot_ssot/integrations/infoblox/utils/client.py @@ -17,6 +17,7 @@ from requests.compat import urljoin from requests.exceptions import HTTPError +from nautobot_ssot.exceptions import InvalidUrlScheme from nautobot_ssot.integrations.infoblox.utils.diffsync import get_ext_attr_dict logger = logging.getLogger("nautobot.ssot.infoblox") @@ -86,19 +87,6 @@ def get_dns_name(possible_fqdn: str) -> str: return dns_name -class InvalidUrlScheme(Exception): - """Exception raised for wrong scheme being passed for URL. - - Attributes: - message (str): Returned explanation of Error. - """ - - def __init__(self, scheme): - """Initialize Exception with wrong scheme in message.""" - self.message = f"Invalid URL scheme '{scheme}' found for Infoblox URL. Please correct to use HTTPS." - super().__init__(self.message) - - class InfobloxApi: # pylint: disable=too-many-public-methods, too-many-instance-attributes """Representation and methods for interacting with Infoblox.""" diff --git a/nautobot_ssot/integrations/ipfabric/diffsync/adapter_ipfabric.py b/nautobot_ssot/integrations/ipfabric/diffsync/adapter_ipfabric.py index e701060df..9a1e1dccf 100644 --- a/nautobot_ssot/integrations/ipfabric/diffsync/adapter_ipfabric.py +++ b/nautobot_ssot/integrations/ipfabric/diffsync/adapter_ipfabric.py @@ -120,7 +120,8 @@ def load(self): # pylint: disable=too-many-locals,too-many-statements filters={"net": ["empty", False], "siteName": ["empty", False]}, columns=["net", "siteName"], ): - networks[network["siteName"]].append(ipaddress.ip_network(network["net"])) + # IPF bug NIM-15635 Fix Version 7.0: 'net' column has host bits set. + networks[network["siteName"]].append(ipaddress.ip_network(network["net"], strict=False)) for location in self.get_all(self.location): if location.name is None: continue diff --git a/nautobot_ssot/jobs/__init__.py b/nautobot_ssot/jobs/__init__.py index 2d7ae96fd..ea85964f9 100644 --- a/nautobot_ssot/jobs/__init__.py +++ b/nautobot_ssot/jobs/__init__.py @@ -9,6 +9,7 @@ from nautobot.core.settings_funcs import is_truthy from nautobot.extras.models import Job +from nautobot_ssot.exceptions import JobException from nautobot_ssot.integrations.utils import each_enabled_integration, each_enabled_integration_module from nautobot_ssot.jobs.base import DataSource, DataTarget from nautobot_ssot.jobs.examples import ExampleDataSource, ExampleDataTarget @@ -28,15 +29,6 @@ jobs = [ExampleDataSource, ExampleDataTarget] -class JobException(Exception): - """Exception raised when failure loading integration Job.""" - - def __init__(self, message): - """Populate exception information.""" - self.message = message - super().__init__(self.message) - - def _check_min_nautobot_version_met(): incompatible_apps_msg = [] nautobot_version = metadata.version("nautobot") diff --git a/nautobot_ssot/jobs/base.py b/nautobot_ssot/jobs/base.py index 9e087baed..972f8952d 100644 --- a/nautobot_ssot/jobs/base.py +++ b/nautobot_ssot/jobs/base.py @@ -45,7 +45,10 @@ class DataSyncBaseJob(Job): # pylint: disable=too-many-instance-attributes - `data_source_icon` and `data_target_icon` """ - dryrun = DryRunVar(description="Perform a dry-run, making no actual changes to Nautobot data.", default=True) + dryrun = DryRunVar( + description="Perform a dry-run, making no actual changes to Nautobot data.", + default=True, + ) memory_profiling = BooleanVar(description="Perform a memory profiling analysis.", default=False) def load_source_adapter(self): @@ -96,7 +99,7 @@ def execute_sync(self): else: self.logger.warning("Not both adapters were properly initialized prior to synchronization.") - def sync_data(self, memory_profiling): + def sync_data(self, memory_profiling): # pylint: disable=too-many-statements """Method to load data from adapters, calculate diffs and sync (if not dry-run). It is composed by 4 methods: @@ -117,10 +120,16 @@ def format_size(size): # pylint: disable=inconsistent-return-statements for unit in ("B", "KiB", "MiB", "GiB", "TiB"): if abs(size) < 100 and unit != "B": # 3 digits (xx.x UNIT) - return "%.1f %s" % (size, unit) # pylint: disable=consider-using-f-string + return "%.1f %s" % ( # pylint: disable=consider-using-f-string + size, + unit, + ) if abs(size) < 10 * 1024 or unit == "TiB": # 4 or 5 digits (xxxx UNIT) - return "%.0f %s" % (size, unit) # pylint: disable=consider-using-f-string + return "%.0f %s" % ( # pylint: disable=consider-using-f-string + size, + unit, + ) size /= 1024 def record_memory_trace(step: str): @@ -150,7 +159,11 @@ def record_memory_trace(step: str): load_source_adapter_time = datetime.now() self.sync.source_load_time = load_source_adapter_time - start_time self.sync.save() - self.logger.info("Source Load Time from %s: %s", self.source_adapter, self.sync.source_load_time) + self.logger.info( + "Source Load Time from %s: %s", + self.source_adapter, + self.sync.source_load_time, + ) if memory_profiling: record_memory_trace("source_load") @@ -159,7 +172,11 @@ def record_memory_trace(step: str): load_target_adapter_time = datetime.now() self.sync.target_load_time = load_target_adapter_time - load_source_adapter_time self.sync.save() - self.logger.info("Target Load Time from %s: %s", self.target_adapter, self.sync.target_load_time) + self.logger.info( + "Target Load Time from %s: %s", + self.target_adapter, + self.sync.target_load_time, + ) if memory_profiling: record_memory_trace("target_load") @@ -172,10 +189,11 @@ def record_memory_trace(step: str): if memory_profiling: record_memory_trace("diff") - if self.dryrun: + if self.sync.dry_run: self.logger.info("As `dryrun` is set, skipping the actual data sync.") else: self.logger.info("Syncing from %s to %s...", self.source_adapter, self.target_adapter) + print("I'm executing the sync now") self.execute_sync() execute_sync_time = datetime.now() self.sync.sync_time = execute_sync_time - calculate_diff_time @@ -185,7 +203,11 @@ def record_memory_trace(step: str): if memory_profiling: record_memory_trace("sync") - def lookup_object(self, model_name, unique_id) -> Optional[BaseModel]: # pylint: disable=unused-argument + def lookup_object( # pylint: disable=unused-argument + self, + model_name, + unique_id, + ) -> Optional[BaseModel]: """Look up the Nautobot record, if any, identified by the args. Optional helper method used to build more detailed/accurate SyncLogEntry records from DiffSync logs. @@ -321,7 +343,10 @@ def run(self, dryrun, memory_profiling, *args, **kwargs): # pylint:disable=argu # Add _structlog_to_sync_log_entry as a processor for structlog calls from DiffSync structlog.configure( - processors=[self._structlog_to_sync_log_entry, structlog.stdlib.render_to_log_kwargs], + processors=[ + self._structlog_to_sync_log_entry, + structlog.stdlib.render_to_log_kwargs, + ], context_class=dict, logger_factory=structlog.stdlib.LoggerFactory(), wrapper_class=structlog.stdlib.BoundLogger, diff --git a/nautobot_ssot/jobs/examples.py b/nautobot_ssot/jobs/examples.py index f206ad410..2c460d2f6 100644 --- a/nautobot_ssot/jobs/examples.py +++ b/nautobot_ssot/jobs/examples.py @@ -26,6 +26,7 @@ from nautobot.tenancy.models import Tenant from nautobot_ssot.contrib import NautobotAdapter, NautobotModel +from nautobot_ssot.exceptions import MissingSecretsGroupException from nautobot_ssot.jobs.base import DataMapping, DataSource, DataTarget from nautobot_ssot.tests.contrib_base_classes import ContentTypeDict @@ -34,10 +35,6 @@ name = "SSoT Examples" # pylint: disable=invalid-name -class MissingSecretsGroupException(Exception): - """Custom Exception in case SecretsGroup is not found on ExternalIntegration.""" - - class LocationTypeModel(NautobotModel): """Shared data model representing a LocationType in either of the local or remote Nautobot instances.""" diff --git a/nautobot_ssot/tests/device42/unit/test_device42_adapter.py b/nautobot_ssot/tests/device42/unit/test_device42_adapter.py index 4f6129708..441f3c837 100644 --- a/nautobot_ssot/tests/device42/unit/test_device42_adapter.py +++ b/nautobot_ssot/tests/device42/unit/test_device42_adapter.py @@ -5,6 +5,7 @@ from diffsync.exceptions import ObjectAlreadyExists, ObjectNotFound from nautobot.core.testing import TransactionTestCase +from nautobot.dcim.models import LocationType from nautobot.extras.models import JobResult from parameterized import parameterized @@ -75,6 +76,7 @@ def setUp(self): self.d42_client.get_ipaddr_default_custom_fields.return_value = {} self.job = self.job_class() + self.job.building_loctype = LocationType.objects.get_or_create(name="Site")[0] self.job.logger = MagicMock() self.job.logger.info = MagicMock() self.job.logger.warning = MagicMock() @@ -105,12 +107,12 @@ def test_data_loading(self): """Test the load() function.""" self.device42.load_buildings() self.assertEqual( - {site["name"] for site in BUILDING_FIXTURE}, + {f"{site['name']}__{self.job.building_loctype.name}" for site in BUILDING_FIXTURE}, {site.get_unique_id() for site in self.device42.get_all("building")}, ) self.device42.load_rooms() self.assertEqual( - {f"{room['name']}__{room['building']}" for room in ROOM_FIXTURE}, + {f"{room['name']}__{room['building']}__{self.job.building_loctype.name}" for room in ROOM_FIXTURE}, {room.get_unique_id() for room in self.device42.get_all("room")}, ) self.device42.load_racks() @@ -157,7 +159,7 @@ def test_load_buildings_duplicate_site(self): self.device42.load_buildings() self.device42.load_buildings() self.job.logger.warning.assert_called_with( - "Microsoft HQ is already loaded. ('Object Microsoft HQ already present', building \"Microsoft HQ\")" + "Microsoft HQ is already loaded. ('Object Microsoft HQ__Site already present', building \"Microsoft HQ__Site\")" ) def test_load_rooms_duplicate_room(self): @@ -166,7 +168,7 @@ def test_load_rooms_duplicate_room(self): self.device42.load_rooms() self.device42.load_rooms() self.job.logger.warning.assert_called_with( - "Secondary IDF is already loaded. ('Object Secondary IDF__Microsoft HQ already present', room \"Secondary IDF__Microsoft HQ\")" + "Secondary IDF is already loaded. ('Object Secondary IDF__Microsoft HQ__Site already present', room \"Secondary IDF__Microsoft HQ__Site\")" ) def test_load_rooms_missing_building(self): diff --git a/nautobot_ssot/tests/device42/unit/test_utils_device42.py b/nautobot_ssot/tests/device42/unit/test_utils_device42.py index 80c8f3da1..08def18a2 100644 --- a/nautobot_ssot/tests/device42/unit/test_utils_device42.py +++ b/nautobot_ssot/tests/device42/unit/test_utils_device42.py @@ -7,6 +7,7 @@ from nautobot.core.testing import TestCase from parameterized import parameterized +from nautobot_ssot.exceptions import MissingConfigSetting from nautobot_ssot.integrations.device42.jobs import Device42DataSource from nautobot_ssot.integrations.device42.utils import device42 @@ -23,7 +24,7 @@ class TestMissingConfigSetting(TestCase): def setUp(self): """Setup MissingConfigSetting instance.""" self.setting = "D42_URL" - self.missing_setting = device42.MissingConfigSetting(setting=self.setting) + self.missing_setting = MissingConfigSetting(setting=self.setting) def test_missingconfigsetting(self): self.assertTrue(self.missing_setting.setting == "D42_URL") diff --git a/nautobot_ssot/tests/dna_center/test_adapters_dna_center.py b/nautobot_ssot/tests/dna_center/test_adapters_dna_center.py index ee3e6cfaf..36442cfa8 100644 --- a/nautobot_ssot/tests/dna_center/test_adapters_dna_center.py +++ b/nautobot_ssot/tests/dna_center/test_adapters_dna_center.py @@ -81,6 +81,9 @@ def setUp(self): ) self.hq_site.validated_save() + self.floor_loc_type = LocationType.objects.get_or_create(name="Floor", parent=self.site_loc_type)[0] + self.floor_loc_type.content_types.add(ContentType.objects.get_for_model(Device)) + cisco_manu = Manufacturer.objects.get_or_create(name="Cisco")[0] catalyst_devicetype = DeviceType.objects.get_or_create(model="WS-C3850-24P-L", manufacturer=cisco_manu)[0] core_role, created = Role.objects.get_or_create(name="CORE") @@ -120,6 +123,9 @@ def setUp(self): dnac = Controller.objects.get_or_create(name="DNA Center", status=self.status_active, location=self.hq_site)[0] self.job = DnaCenterDataSource() + self.job.area_loctype = self.reg_loc_type + self.job.building_loctype = self.site_loc_type + self.job.floor_loctype = self.floor_loc_type self.job.dnac = dnac self.job.controller_group = ControllerManagedDeviceGroup.objects.get_or_create( name="DNA Center Managed Devices", controller=dnac @@ -187,7 +193,7 @@ def test_load_areas_w_global(self): area_actual = sorted([area.get_unique_id() for area in self.dna_center.get_all("area")]) self.assertEqual(area_actual, area_expected) self.dna_center.job.logger.info.assert_called_with( - "Loaded area Sydney. {'parentId': '262696b1-aa87-432b-8a21-db9a77c51f23', 'additionalInfo': [{'nameSpace': 'Location', 'attributes': {'addressInheritedFrom': '262696b1-aa87-432b-8a21-db9a77c51f23', 'type': 'area'}}], 'name': 'Sydney', 'instanceTenantId': '623f029857259506a56ad9bd', 'id': '6e404051-4c06-4dab-adaa-72c5eeac577b', 'siteHierarchy': '9e5f9fc2-032e-45e8-994c-4a00629648e8/262696b1-aa87-432b-8a21-db9a77c51f23/6e404051-4c06-4dab-adaa-72c5eeac577b', 'siteNameHierarchy': 'Global/Australia/Sydney'}" + "Loaded Region Sydney. {'parentId': '262696b1-aa87-432b-8a21-db9a77c51f23', 'additionalInfo': [{'nameSpace': 'Location', 'attributes': {'addressInheritedFrom': '262696b1-aa87-432b-8a21-db9a77c51f23', 'type': 'area'}}], 'name': 'Sydney', 'instanceTenantId': '623f029857259506a56ad9bd', 'id': '6e404051-4c06-4dab-adaa-72c5eeac577b', 'siteHierarchy': '9e5f9fc2-032e-45e8-994c-4a00629648e8/262696b1-aa87-432b-8a21-db9a77c51f23/6e404051-4c06-4dab-adaa-72c5eeac577b', 'siteNameHierarchy': 'Global/Australia/Sydney'}" ) @override_settings(PLUGINS_CONFIG={"nautobot_ssot": {"dna_center_import_global": False}}) @@ -216,7 +222,9 @@ def test_load_buildings_w_global(self): ("", "building"), ] self.dna_center.load_buildings(buildings=EXPECTED_BUILDINGS) - building_expected = [x["name"] for x in EXPECTED_DNAC_LOCATION_MAP.values() if x["loc_type"] == "building"] + building_expected = [ + f"{x['name']}__{x['parent']}" for x in EXPECTED_DNAC_LOCATION_MAP.values() if x["loc_type"] == "building" + ] building_actual = [building.get_unique_id() for building in self.dna_center.get_all("building")] self.assertEqual(building_actual, building_expected) @@ -235,7 +243,9 @@ def test_load_buildings_wo_global(self): self.dna_center.dnac_location_map = EXPECTED_DNAC_LOCATION_MAP_WO_GLOBAL self.dna_center.load_buildings(buildings=EXPECTED_BUILDINGS) building_expected = [ - x["name"] for x in EXPECTED_DNAC_LOCATION_MAP_WO_GLOBAL.values() if x["loc_type"] == "building" + f"{x['name']}__{x['parent']}" + for x in EXPECTED_DNAC_LOCATION_MAP_WO_GLOBAL.values() + if x["loc_type"] == "building" ] building_actual = [building.get_unique_id() for building in self.dna_center.get_all("building")] self.assertEqual(sorted(building_actual), sorted(building_expected)) @@ -254,7 +264,7 @@ def test_load_buildings_duplicate(self): ] self.dna_center.load_buildings(buildings=EXPECTED_BUILDINGS) self.dna_center.load_buildings(buildings=[EXPECTED_BUILDINGS[0]]) - self.dna_center.job.logger.warning.assert_called_with("Building Building1 already loaded so skipping.") + self.dna_center.job.logger.warning.assert_called_with("Site Building1 already loaded so skipping.") def test_load_buildings_with_validation_error(self): """Test Nautobot SSoT for Cisco DNA Center load_buildings() function with a ValidationError.""" diff --git a/nautobot_ssot/tests/dna_center/test_adapters_nautobot.py b/nautobot_ssot/tests/dna_center/test_adapters_nautobot.py index 51a1eda5a..596939f56 100644 --- a/nautobot_ssot/tests/dna_center/test_adapters_nautobot.py +++ b/nautobot_ssot/tests/dna_center/test_adapters_nautobot.py @@ -48,10 +48,12 @@ def setUp(self): # pylint: disable=too-many-locals job.job_result = JobResult.objects.create( name=job.class_path, task_name="fake task", user=None, id=uuid.uuid4() ) + job.logger.info = MagicMock() + job.logger.warning = MagicMock() + job.area_loctype = self.reg_loc_type + job.building_loctype = self.site_loc_type + job.floor_loctype = self.floor_loc_type self.nb_adapter = NautobotAdapter(job=job, sync=None) - self.nb_adapter.job = MagicMock() - self.nb_adapter.job.logger.info = MagicMock() - self.nb_adapter.job.logger.warning = MagicMock() def build_nautobot_objects(self): # pylint: disable=too-many-locals, too-many-statements """Build out Nautobot objects to test loading.""" @@ -259,7 +261,7 @@ def test_data_loading(self): sorted(loc.get_unique_id() for loc in self.nb_adapter.get_all("area")), ) self.assertEqual( - ["HQ"], + ["HQ__NY"], sorted(site.get_unique_id() for site in self.nb_adapter.get_all("building")), ) self.assertEqual( @@ -289,11 +291,11 @@ def test_data_loading(self): sorted(ipaddr.get_unique_id() for ipaddr in self.nb_adapter.get_all("ipaddress")), ) - def test_load_regions_failure(self): - """Test the load_regions method failing with loading duplicate Regions.""" + def test_load_areas_failure(self): + """Test the load_areas method failing with loading duplicate Areas.""" self.build_nautobot_objects() self.nb_adapter.load() - self.nb_adapter.load_regions() + self.nb_adapter.load_areas() self.nb_adapter.job.logger.warning.assert_called_with("Region NY already loaded so skipping duplicate.") @patch("nautobot_ssot.integrations.dna_center.diffsync.adapters.nautobot.OrmLocationType") @@ -311,9 +313,7 @@ def test_load_floors_missing_building(self, mock_floors, mock_loc_type): self.nb_adapter.get = MagicMock() self.nb_adapter.get.side_effect = [ObjectNotFound()] self.nb_adapter.load_floors() - self.nb_adapter.job.logger.warning.assert_called_with( - "Unable to load building Missing for floor HQ - Floor 1. " - ) + self.nb_adapter.job.logger.warning.assert_called_with("Unable to load Site Missing for Floor HQ - Floor 1. ") def test_sync_complete(self): """Test the sync_complete() method in the NautobotAdapter.""" diff --git a/nautobot_ssot/tests/dna_center/test_models_nautobot.py b/nautobot_ssot/tests/dna_center/test_models_nautobot.py index 12d5afa6a..39f7c30a8 100644 --- a/nautobot_ssot/tests/dna_center/test_models_nautobot.py +++ b/nautobot_ssot/tests/dna_center/test_models_nautobot.py @@ -30,11 +30,12 @@ class TestNautobotArea(TransactionTestCase): def setUp(self): super().setUp() + self.region_type = LocationType.objects.get_or_create(name="Region", nestable=True)[0] self.adapter = Adapter() self.adapter.job = MagicMock() + self.adapter.job.area_loctype = self.region_type self.adapter.job.logger.info = MagicMock() self.adapter.region_map = {} - self.region_type = LocationType.objects.get_or_create(name="Region", nestable=True)[0] self.adapter.locationtype_map = {"Region": self.region_type.id} self.adapter.status_map = {"Active": Status.objects.get(name="Active").id} @@ -72,17 +73,21 @@ class TestNautobotBuilding(TransactionTestCase): def setUp(self): super().setUp() + self.reg_loc = LocationType.objects.get_or_create(name="Region", nestable=True)[0] + loc_type = LocationType.objects.get_or_create(name="Site", parent=self.reg_loc)[0] self.adapter = Adapter() self.adapter.job = MagicMock() self.adapter.job.debug = True + self.adapter.job.area_loctype = self.reg_loc + self.adapter.job.building_loctype = loc_type self.adapter.job.logger.info = MagicMock() self.adapter.status_map = {"Active": Status.objects.get(name="Active").id} ga_tenant = Tenant.objects.create(name="G&A") self.adapter.tenant_map = {"G&A": ga_tenant.id} - reg_loc = LocationType.objects.get_or_create(name="Region", nestable=True)[0] - ny_region = Location.objects.create(name="NY", location_type=reg_loc, status=Status.objects.get(name="Active")) - loc_type = LocationType.objects.get_or_create(name="Site", parent=reg_loc)[0] - self.adapter.locationtype_map = {"Region": reg_loc.id, "Site": loc_type.id} + ny_region = Location.objects.create( + name="NY", location_type=self.reg_loc, status=Status.objects.get(name="Active") + ) + self.adapter.locationtype_map = {"Region": self.reg_loc.id, "Site": loc_type.id} self.sec_site = Location.objects.create( name="Site 2", parent=ny_region, status=Status.objects.get(name="Active"), location_type=loc_type ) @@ -102,10 +107,9 @@ def setUp(self): def test_create(self): """Validate the NautobotBuilding create() method creates a Site.""" - ids = {"name": "HQ"} + ids = {"name": "HQ", "area": "NY"} attrs = { "address": "123 Main St", - "area": "NY", "area_parent": None, "latitude": "12.345", "longitude": "-67.890", @@ -121,7 +125,7 @@ def test_create(self): self.adapter.job.logger.info.assert_called_once_with("Creating Site HQ.") site_obj = Location.objects.get(name=ids["name"], location_type__name="Site") self.assertEqual(site_obj.name, ids["name"]) - self.assertEqual(site_obj.parent.name, attrs["area"]) + self.assertEqual(site_obj.parent.name, ids["area"]) self.assertEqual(site_obj.physical_address, attrs["address"]) self.assertEqual(site_obj.tenant.name, attrs["tenant"]) @@ -157,15 +161,19 @@ def test_update_wo_tenant(self): def test_delete(self): """Validate the NautobotBuilding delete() method deletes a Site.""" ds_mock_site = MagicMock(spec=Location) + ds_mock_site.location_type = "Site" ds_mock_site.uuid = "1234567890" ds_mock_site.adapter = MagicMock() + ds_mock_site.adapter.job.building_loctype = self.adapter.job.building_loctype ds_mock_site.adapter.job.logger.info = MagicMock() mock_site = MagicMock(spec=Location) mock_site.name = "Test" site_get_mock = MagicMock(return_value=mock_site) with patch.object(Location.objects, "get", site_get_mock): result = NautobotBuilding.delete(ds_mock_site) - ds_mock_site.adapter.job.logger.info.assert_called_once_with("Deleting Site Test.") + ds_mock_site.adapter.job.logger.info.assert_called_once_with( + f"Deleting {self.adapter.job.building_loctype.name} Test." + ) self.assertEqual(ds_mock_site, result) @@ -177,13 +185,16 @@ class TestNautobotFloor(TransactionTestCase): def setUp(self): super().setUp() + site_loc_type = LocationType.objects.get_or_create(name="Site")[0] + self.floor_loc_type = LocationType.objects.get_or_create(name="Floor", parent=site_loc_type)[0] self.adapter = Adapter() self.adapter.job = MagicMock() + self.adapter.job.building_loctype = site_loc_type + self.adapter.job.floor_loctype = self.floor_loc_type self.adapter.job.logger.info = MagicMock() ga_tenant = Tenant.objects.create(name="G&A") self.adapter.tenant_map = {"G&A": ga_tenant.id} - site_loc_type = LocationType.objects.get_or_create(name="Site")[0] - self.floor_loc_type = LocationType.objects.get_or_create(name="Floor", parent=site_loc_type)[0] + self.adapter.locationtype_map = {"Site": site_loc_type.id, "Floor": self.floor_loc_type.id} self.hq_site, _ = Location.objects.get_or_create( name="HQ", location_type=site_loc_type, status=Status.objects.get(name="Active") @@ -191,6 +202,7 @@ def setUp(self): self.adapter.site_map = {"HQ": self.hq_site.id} self.adapter.floor_map = {} self.adapter.status_map = {"Active": Status.objects.get(name="Active").id} + self.adapter.objects_to_delete = {"floors": []} def test_create(self): """Test the NautobotFloor create() method creates a LocationType: Floor.""" @@ -206,10 +218,9 @@ def test_create(self): def test_update_w_tenant(self): """Test the NautobotFloor update() method updates a LocationType: Floor with tenant.""" - floor_type = LocationType.objects.get(name="Floor") mock_floor = Location.objects.create( name="HQ - Floor 2", - location_type=floor_type, + location_type=self.floor_loc_type, parent=self.hq_site, status=Status.objects.get(name="Active"), ) @@ -236,8 +247,7 @@ def test_update_wo_tenant(self): ) mock_floor.validated_save() test_floor = NautobotFloor(name="HQ - Floor 2", building="HQ", tenant="", uuid=mock_floor.id) - test_floor.adapter = MagicMock() - test_floor.adapter.job.logger.info = MagicMock() + test_floor.adapter = self.adapter update_attrs = { "tenant": None, } @@ -249,16 +259,18 @@ def test_update_wo_tenant(self): def test_delete(self): """Validate the NautobotFloor delete() method deletes a LocationType: Floor.""" ds_mock_floor = MagicMock(spec=Location) + ds_mock_floor.location_type = "Floor" ds_mock_floor.uuid = "1234567890" - ds_mock_floor.adapter = MagicMock() - ds_mock_floor.adapter.job.logger.info = MagicMock() + ds_mock_floor.adapter = self.adapter mock_floor = MagicMock(spec=Location) mock_floor.name = "Test" mock_floor.parent.name = "HQ" floor_get_mock = MagicMock(return_value=mock_floor) with patch.object(Location.objects, "get", floor_get_mock): result = NautobotFloor.delete(ds_mock_floor) - ds_mock_floor.adapter.job.logger.info.assert_called_once_with("Deleting Floor Test in HQ.") + ds_mock_floor.adapter.job.logger.info.assert_called_once_with( + f"Deleting {self.adapter.job.floor_loctype.name} Test in HQ." + ) self.assertEqual(ds_mock_floor, result) diff --git a/nautobot_ssot/tests/test_jobs.py b/nautobot_ssot/tests/test_jobs.py index b4052b242..de2f09cca 100644 --- a/nautobot_ssot/tests/test_jobs.py +++ b/nautobot_ssot/tests/test_jobs.py @@ -1,7 +1,7 @@ """Test the Job classes in nautobot_ssot.""" import os.path -from unittest.mock import Mock, call +from unittest.mock import Mock, call, patch from django.db.utils import IntegrityError, OperationalError from django.test import override_settings @@ -90,9 +90,53 @@ def test_run(self): self.assertIsNone(self.job.sync.sync_time) self.assertEqual(self.job.sync.source, self.job.data_source) self.assertEqual(self.job.sync.target, self.job.data_target) - self.assertTrue(self.job.dryrun) + self.assertTrue(self.job.sync.dry_run) self.assertEqual(self.job.job_result, self.job.sync.job_result) + def test_job_dryrun_false(self): + """Test the job is not ran in dryrun mode.""" + with patch.object(DataSyncBaseJob, "execute_sync") as mock_execute_sync: + isolated_job = DataSyncBaseJob() + + isolated_job.job_result = JobResult.objects.create( + name="fake job no dryrun", + task_name="fake job no dryrun", + worker="default", + ) + isolated_job.load_source_adapter = lambda *x, **y: None + isolated_job.load_target_adapter = lambda *x, **y: None + isolated_job.run(dryrun=False, memory_profiling=False) + self.assertFalse(isolated_job.sync.dry_run) + mock_execute_sync.assert_called() + + def test_job_dryrun_true(self): + """Test the job is ran in dryrun mode.""" + with patch.object(DataSyncBaseJob, "execute_sync") as mock_execute_sync: + isolated_job = DataSyncBaseJob() + + isolated_job.job_result = JobResult.objects.create( + name="fake job", + task_name="fake job", + worker="default", + ) + isolated_job.load_source_adapter = lambda *x, **y: None + isolated_job.load_target_adapter = lambda *x, **y: None + isolated_job.run(dryrun=True, memory_profiling=False) + self.assertTrue(isolated_job.sync.dry_run) + mock_execute_sync.assert_not_called() + + @patch("tracemalloc.start") + def test_job_memory_profiling_true(self, mock_malloc_start): + """Test the job is ran in dryrun mode.""" + self.job.run(dryrun=False, memory_profiling=True) + mock_malloc_start.assert_called() + + @patch("tracemalloc.start") + def test_job_memory_profiling_false(self, mock_malloc_start): + """Test the job is ran in dryrun mode.""" + self.job.run(dryrun=False, memory_profiling=False) + mock_malloc_start.assert_not_called() + def test_calculate_diff(self): """Test calculate_diff() method.""" self.job.sync = Mock()