Merge pull request #586 from nautobot/patch-fix_585

Fix Bootstrap for DLM 3.0 and Correct Docs

README.md

@@ -82,6 +82,7 @@ Full documentation for this app can be found over on the [Nautobot Docs](https:/
 The SSoT framework includes a number of integrations with external Systems of Record:
 
 * Cisco ACI
+* Bootstrap
 * Arista CloudVision
 * Device42
 * Cisco DNA Center

changes/585.documentation

@@ -0,0 +1 @@
+Fix documentation for Bootstrap installation.

changes/585.fixed

@@ -0,0 +1 @@
+Fixed use of DLM classes with Bootstrap integration.

changes/585.housekeeping

@@ -0,0 +1 @@
+Disabled the BootstrapDataTarget Job as it's not usable at this time.

development/creds.example.env

@@ -49,9 +49,5 @@ SERVICENOW_PASSWORD="changeme"
 
 IPFABRIC_API_TOKEN=secrettoken
 
-NAUTOBOT_SSOT_ENABLE_BOOTSTRAP="False"
-NAUTOBOT_BOOTSTRAP_SSOT_ENVIRONMENT_BRANCH=develop
-NAUTOBOT_BOOTSTRAP_SSOT_LOAD_SOURCE=file # or git
-
 MERAKI_ORG_ID='123456'
 MERAKI_TOKEN='vtx01710aa0fn452740055y1hs60ns8c107ho168'

development/development.env

@@ -69,6 +69,10 @@ NAUTOBOT_ARISTACV_IMPORT_ACTIVE="False"
 NAUTOBOT_ARISTACV_IMPORT_TAG="False"
 NAUTOBOT_ARISTACV_VERIFY=True
 
+NAUTOBOT_SSOT_ENABLE_BOOTSTRAP="False"
+NAUTOBOT_BOOTSTRAP_SSOT_ENVIRONMENT_BRANCH=develop
+NAUTOBOT_BOOTSTRAP_SSOT_LOAD_SOURCE=file # or git
+
 NAUTOBOT_SSOT_ENABLE_DEVICE42="False"
 NAUTOBOT_SSOT_DEVICE42_HOST=""
 NAUTOBOT_SSOT_DEVICE42_USERNAME=""

docs/admin/integrations/bootstrap_setup.md

@@ -1,6 +1,5 @@
 # Bootstrap
 
-
 ## Description
 
 This App will sync data from YAML files into Nautobot to create baseline environments. Most items will receive a custom field associated with them called "System of Record", which will be set to "Bootstrap". These items are then the only ones managed by the Bootstrap SSoT App. Other items within the Nautobot instance will not be affected unless there's items with overlapping names. There is currently two exceptions to this and those are the ComputedField, and GraphQLQuery models since they can't have a custom field associated. If you choose to manage ComputedField or GraphQLQuery objects with the Bootstrap SSoT App, make sure to define them all within the YAML file, since any "locally defined" Computed Fields and GraphQL Queries within Nautobot will end up getting deleted when the job runs. If an item exists in Nautobot by it's identifiers but it does not have the "System of Record" custom field on it, the item will be updated with "Bootstrap" (or `SYSTEM_OF_RECORD` environment variable value) when the App runs. This way no duplicates are created, and the App will not delete any items that are not defined in the Bootstrap data but were manually created in Nautobot.
@@ -13,17 +12,17 @@ Before configuring the integration, please ensure, that `nautobot-ssot` app was
 pip install nautobot-ssot[bootstrap]
 ```
 
+## Configuration
 
-### nautobot_config.py
-
-The settings here are pretty straightforward, `nautobot_environment_branch` will be loaded from the environment variable `NAUTOBOT_BOOTSTRAP_SSOT_ENVIRONMENT_BRANCH`, or default to develop. The rest of the settings define which models/objects you want to have the App sync to Nautobot. There are a couple of caveats to these. For example, for DynamicGroup objects to sync, the filter criteria need to already exist in Nautobot. So, if you are going to have groups that are filtered on platforms/regions/sites/etc make sure not to include DynamicGroup objects in the "models_to_sync" until those items exist. Same for Git Repositories when you want to sync Golden Config-related repositories. The Golden Config App needs to be installed, for the `provided_contents` items to be able to be found. This also goes for the Lifecycle Management app with `Software/ValidatedSoftware` models.
+Once the SSoT package has been installed you simply need to enable the integration by setting `enable_bootstrap` to True. There are additional settings that allow you to control which Nautobot objects are defined in your data. The settings are pretty straightforward. Assuming that you're copying the example settings below, the `bootstrap_nautobot_environment_branch` setting will be loaded from the environment variable `NAUTOBOT_BOOTSTRAP_SSOT_ENVIRONMENT_BRANCH`, or default to develop. The `bootstrap_models_to_sync` setting defines which models/objects you want to have the App sync to Nautobot. There are a couple of caveats to this functionality. For example, for DynamicGroup objects to sync, the filter criteria need to already exist in Nautobot. So, if you are going to have groups that are filtered on Platforms, Locations, etc, make sure not to include DynamicGroup objects in your Bootstrap Data until those items exist in Nautobot. If these items are also being synchronized in your Bootstrap Data, they will be created in the correct order. The same goes for Golden Config-related Git Repositories. It should go without saying, but the [Golden Config App](https://github.com/nautobot/nautobot-app-golden-config) must be installed, for the backup, intended, and template `provided_contents` items to be available options in your Bootstrap Data along with support of the `SoftwareLCM, SoftwareImageLCM, and ValidatedSoftware` models from the [Device Lifecycle Management app](https://github.com/nautobot/nautobot-app-device-lifecycle-mgmt).
 
 ```python
 PLUGINS = ["nautobot_ssot"]
 
 PLUGINS_CONFIG = {
   "nautobot_ssot": {
         # Other nautobot_ssot settings ommitted.
+        "enable_bootstrap": is_truthy(os.getenv("NAUTOBOT_SSOT_ENABLE_BOOTSTRAP", "true")),
         "bootstrap_nautobot_environment_branch": os.getenv("NAUTOBOT_BOOTSTRAP_SSOT_ENVIRONMENT_BRANCH", "develop"),
         "bootstrap_models_to_sync": {
             "secret": True,
@@ -57,20 +56,17 @@ PLUGINS_CONFIG = {
             "vrf": True,
             "prefix": True,
         },
-        "enable_bootstrap": is_truthy(os.getenv("NAUTOBOT_SSOT_ENABLE_BOOTSTRAP", "false")),
   }
 }
 ```
 
-## Configuration
-
-### Bootstrap data
+### Bootstrap Data
 
 Bootstrap data can be stored in 2 fashions.
 
-1. (Recommended) Bootstrap data can be stored in a Git Repository and referenced in the app as a Git Datasource. A user should create a Git Repository in Nautobot (including any necessary Secrets and SecretsGroups for access) with the word "Bootstrap" in the name, and with a provided content type of `config contexts`. This is how the App will locate the correct repository. The data structure is flat files, and there is a naming scheme to these files. The first one required is `global_settings.yml`. This contains the main data structures of what data can be loaded `Secrets,SecretsGroups,GitRepository,DynamicGroup,Tag,etc`. You can then create additional `.yml` files with naming of your CI environments, i.e. production, development, etc for default values for specific items. This is where the environment variables described below would be matched to pull in additional data from the other YAML files defined in the directory.
+1. __Recommended__ Bootstrap data can be stored in a Git Repository and referenced in the app as a Git Datasource. A user should create a Git Repository in Nautobot (including any necessary Secrets and SecretsGroups for access) with the word "Bootstrap" in the name, and with a provided content type of `config contexts`. This is how the App will locate the correct repository. The data structure is flat files, and there is a naming scheme to these files. The first one required is `global_settings.yml`. This contains the main data structures of what data can be loaded. ie `Secrets, SecretsGroups, GitRepository, DynamicGroup, Tag, etc`. You can then create additional `.yml` files with naming of your CI environments, ie `production`, `development`, etc for default values for specific items. This is where the environment variables described below would be matched to pull in additional data from the other YAML files defined in the directory.
 
-2. Bootstrap data can be stored within the `nautobot_ssot/bootstrap/fixtures` directory. Using local files is not recommended as this requires a fork of the plugin and locally editing the YAML data files in the fixtures folder.
+2. Bootstrap data can be stored within the `nautobot_ssot/integrations/bootstrap/fixtures` directory. Using local files is not recommended as this requires a fork of the plugin and locally editing the YAML data files in the fixtures folder.
 
 A simple structure would look something like this:
 

nautobot_ssot/integrations/bootstrap/diffsync/adapters/nautobot.py

@@ -82,32 +82,40 @@
 )
 
 try:
-    import nautobot_device_lifecycle_mgmt  # noqa: F401
+    # noqa: F401
+    from nautobot_device_lifecycle_mgmt.models import (
+        SoftwareLCM as ORMSoftware,
+    )
 
-    LIFECYCLE_MGMT = True
-except ImportError:
-    LIFECYCLE_MGMT = False
+    from nautobot_ssot.integrations.bootstrap.diffsync.models.nautobot import NautobotSoftware
 
-if LIFECYCLE_MGMT:
+    SOFTWARE_LIFECYCLE_MGMT = True
+except (ImportError, RuntimeError):
+    SOFTWARE_LIFECYCLE_MGMT = False
+
+try:
     # noqa: F401
     from nautobot_device_lifecycle_mgmt.models import (
         SoftwareImageLCM as ORMSoftwareImage,
     )
-    from nautobot_device_lifecycle_mgmt.models import (
-        SoftwareLCM as ORMSoftware,
-    )
 
+    from nautobot_ssot.integrations.bootstrap.diffsync.models.nautobot import NautobotSoftwareImage
+
+    SOFTWARE_IMAGE_LIFECYCLE_MGMT = True
+except (ImportError, RuntimeError):
+    SOFTWARE_IMAGE_LIFECYCLE_MGMT = False
+
+try:
     # noqa: F401
     from nautobot_device_lifecycle_mgmt.models import (
         ValidatedSoftwareLCM as ORMValidatedSoftware,
     )
 
-    # noqa: F401
-    from nautobot_ssot.integrations.bootstrap.diffsync.models.nautobot import (  # noqa: F401
-        NautobotSoftware,
-        NautobotSoftwareImage,
-        NautobotValidatedSoftware,
-    )
+    from nautobot_ssot.integrations.bootstrap.diffsync.models.nautobot import NautobotValidatedSoftware
+
+    VALID_SOFTWARE_LIFECYCLE_MGMT = True
+except (ImportError, RuntimeError):
+    VALID_SOFTWARE_LIFECYCLE_MGMT = False
 
 
 class NautobotAdapter(Adapter):
@@ -141,9 +149,11 @@ class NautobotAdapter(Adapter):
     tag = NautobotTag
     graph_ql_query = NautobotGraphQLQuery
 
-    if LIFECYCLE_MGMT:
+    if SOFTWARE_LIFECYCLE_MGMT:
         software = NautobotSoftware
+    if SOFTWARE_IMAGE_LIFECYCLE_MGMT:
         software_image = NautobotSoftwareImage
+    if VALID_SOFTWARE_LIFECYCLE_MGMT:
         validated_software = NautobotValidatedSoftware
 
     top_level = [
@@ -175,9 +185,11 @@ class NautobotAdapter(Adapter):
         "graph_ql_query",
     ]
 
-    if LIFECYCLE_MGMT:
+    if SOFTWARE_LIFECYCLE_MGMT:
         top_level.append("software")
+    if SOFTWARE_IMAGE_LIFECYCLE_MGMT:
         top_level.append("software_image")
+    if VALID_SOFTWARE_LIFECYCLE_MGMT:
         top_level.append("validated_software")
 
     def __init__(self, *args, job=None, sync=None, **kwargs):  # noqa: D417
@@ -1350,10 +1362,12 @@ def load(self):
             self.load_tag()
         if settings.PLUGINS_CONFIG["nautobot_ssot"]["bootstrap_models_to_sync"]["graph_ql_query"]:
             self.load_graph_ql_query()
-        if LIFECYCLE_MGMT:
+        if SOFTWARE_LIFECYCLE_MGMT:
             if settings.PLUGINS_CONFIG["nautobot_ssot"]["bootstrap_models_to_sync"]["software"]:
                 self.load_software()
+        if SOFTWARE_IMAGE_LIFECYCLE_MGMT:
             if settings.PLUGINS_CONFIG["nautobot_ssot"]["bootstrap_models_to_sync"]["software_image"]:
                 self.load_software_image()
+        if VALID_SOFTWARE_LIFECYCLE_MGMT:
             if settings.PLUGINS_CONFIG["nautobot_ssot"]["bootstrap_models_to_sync"]["validated_software"]:
                 self.load_validated_software()

nautobot_ssot/integrations/bootstrap/diffsync/models/nautobot.py

@@ -78,33 +78,40 @@
 )
 
 try:
-    import nautobot_device_lifecycle_mgmt  # noqa: F401
-
-    LIFECYCLE_MGMT = True
-except ImportError:
-    LIFECYCLE_MGMT = False
-
-if LIFECYCLE_MGMT:
     # noqa: F401
     from nautobot_device_lifecycle_mgmt.models import (
-        SoftwareImageLCM as ORMSoftwareImage,
+        SoftwareLCM as ORMSoftware,
     )
 
+    from nautobot_ssot.integrations.bootstrap.diffsync.models.base import Software
+
+    SOFTWARE_LIFECYCLE_MGMT = True
+except (ImportError, RuntimeError):
+    SOFTWARE_LIFECYCLE_MGMT = False
+
+try:
     # noqa: F401
     from nautobot_device_lifecycle_mgmt.models import (
-        SoftwareLCM as ORMSoftware,
+        SoftwareImageLCM as ORMSoftwareImage,
     )
 
+    from nautobot_ssot.integrations.bootstrap.diffsync.models.base import SoftwareImage
+
+    SOFTWARE_IMAGE_LIFECYCLE_MGMT = True
+except (ImportError, RuntimeError):
+    SOFTWARE_IMAGE_LIFECYCLE_MGMT = False
+
+try:
     # noqa: F401
     from nautobot_device_lifecycle_mgmt.models import (
         ValidatedSoftwareLCM as ORMValidatedSoftware,
     )
 
-    from nautobot_ssot.integrations.bootstrap.diffsync.models.base import (
-        Software,
-        SoftwareImage,
-        ValidatedSoftware,
-    )
+    from nautobot_ssot.integrations.bootstrap.diffsync.models.base import ValidatedSoftware
+
+    VALID_SOFTWARE_LIFECYCLE_MGMT = True
+except (ImportError, RuntimeError):
+    VALID_SOFTWARE_LIFECYCLE_MGMT = False
 
 
 class NautobotTenantGroup(TenantGroup):
@@ -2176,7 +2183,7 @@ def delete(self):
             self.adapter.job.logger.warning(f"Unable to find GraphQLQuery {self.name} for deletion. {err}")
 
 
-if LIFECYCLE_MGMT:
+if SOFTWARE_LIFECYCLE_MGMT:
 
     class NautobotSoftware(Software):
         """Nautobot implementation of Bootstrap Software model."""
@@ -2254,6 +2261,9 @@ def delete(self):
                     f"Unable to find Software {self.platform} - {self.version} for deletion. {err}"
                 )
 
+
+if SOFTWARE_IMAGE_LIFECYCLE_MGMT:
+
     class NautobotSoftwareImage(SoftwareImage):
         """Nautobot implementation of Bootstrap SoftwareImage model."""
 
@@ -2329,6 +2339,9 @@ def delete(self):
             except ORMSoftwareImage.DoesNotExist as err:
                 self.adapter.job.logger.warning(f"Unable to find SoftwareImage {self.software} for deletion. {err}")
 
+
+if VALID_SOFTWARE_LIFECYCLE_MGMT:
+
     class NautobotValidatedSoftware(ValidatedSoftware):
         """Nautobot implementation of Bootstrap ValidatedSoftware model."""
 

nautobot_ssot/integrations/bootstrap/jobs.py

@@ -157,4 +157,4 @@ def run(self, read_destination, dryrun, memory_profiling, debug, *args, **kwargs
         super().run(dryrun=self.dryrun, memory_profiling=self.memory_profiling, *args, **kwargs)
 
 
-jobs = [BootstrapDataSource, BootstrapDataTarget]
+jobs = [BootstrapDataSource]

nautobot_ssot/integrations/bootstrap/signals.py

@@ -16,7 +16,7 @@ def register_signals(sender):
     nautobot_database_ready.connect(nautobot_database_ready_callback, sender=sender)
 
 
-def nautobot_database_ready_callback(sender, *, apps, **kwargs):  # pylint: disable=unused-argument
+def nautobot_database_ready_callback(sender, *, apps, **kwargs):  # pylint: disable=unused-argument, too-many-statements
     """Adds OS Version and Physical Address CustomField to Devices and System of Record and Last Sync'd to Device, and IPAddress.
 
     Callback function triggered by the nautobot_database_ready signal when the Nautobot database is fully ready.
@@ -49,11 +49,6 @@ def nautobot_database_ready_callback(sender, *, apps, **kwargs):  # pylint: disa
     GitRepository = apps.get_model("extras", "GitRepository")
     Role = apps.get_model("extras", "Role")
 
-    if LIFECYCLE_MGMT:
-        SoftwareLCM = apps.get_model("nautobot_device_lifecycle_mgmt", "SoftwareLCM")
-        SoftwareImageLCM = apps.get_model("nautobot_device_lifecycle_mgmt", "SoftwareImageLCM")
-        ValidatedSoftwareLCM = apps.get_model("nautobot_device_lifecycle_mgmt", "ValidatedSoftwareLCM")
-
     signal_to_model_mapping = {
         "manufacturer": Manufacturer,
         "platform": Platform,
@@ -83,13 +78,21 @@ def nautobot_database_ready_callback(sender, *, apps, **kwargs):  # pylint: disa
     }
 
     if LIFECYCLE_MGMT:
-        signal_to_model_mapping.update(
-            {
-                "software": SoftwareLCM,
-                "software_image": SoftwareImageLCM,
-                "validated_software": ValidatedSoftwareLCM,
-            }
-        )
+        try:
+            SoftwareLCM = apps.get_model("nautobot_device_lifecycle_mgmt", "SoftwareLCM")
+            signal_to_model_mapping["software"] = SoftwareLCM
+        except LookupError as err:
+            print(f"Unable to find SoftwareLCM model from Device Lifecycle Management App. {err}")
+        try:
+            SoftwareImageLCM = apps.get_model("nautobot_device_lifecycle_mgmt", "SoftwareImageLCM")
+            signal_to_model_mapping["software_image"] = SoftwareImageLCM
+        except LookupError as err:
+            print(f"Unable to find SoftwareImageLCM model from Device Lifecycle Management App. {err}")
+        try:
+            ValidatedSoftwareLCM = apps.get_model("nautobot_device_lifecycle_mgmt", "ValidatedSoftwareLCM")
+            signal_to_model_mapping["validated_software"] = ValidatedSoftwareLCM
+        except LookupError as err:
+            print(f"Unable to find ValidatedSoftwareLCM model from Device Lifecycle Management App. {err}")
 
     sync_custom_field, _ = create_or_update_custom_field(
         key="last_synced_from_sor",