From 72a4206aafd5654e419b9fc4971a28e2e13b4bee Mon Sep 17 00:00:00 2001 From: kr3ator <48438188+kr3ator@users.noreply.github.com> Date: Tue, 5 Apr 2022 08:34:08 +0200 Subject: [PATCH] feat: Make startup scripts idempotent --- initializers/users.yml | 1 + startup_scripts/000_users.py | 20 +++++++++++-------- startup_scripts/020_object_permissions.py | 8 +++++--- startup_scripts/040_custom_links.py | 6 ++++-- startup_scripts/050_tags.py | 5 +++-- startup_scripts/060_webhooks.py | 6 ++++-- startup_scripts/070_tenant_groups.py | 5 +++-- startup_scripts/080_tenants.py | 10 ++++++++-- startup_scripts/090_regions.py | 5 +++-- startup_scripts/110_sites.py | 10 ++++++++-- startup_scripts/120_locations.py | 6 ++++-- startup_scripts/130_rack_roles.py | 5 +++-- startup_scripts/140_racks.py | 12 ++++++++--- startup_scripts/150_power_panels.py | 12 ++++++++--- startup_scripts/160_power_feeds.py | 12 ++++++++--- startup_scripts/170_manufacturers.py | 5 +++-- startup_scripts/180_device_roles.py | 5 +++-- startup_scripts/190_device_types.py | 12 ++++++++--- startup_scripts/200_devices.py | 14 +++++++++---- startup_scripts/210_dcim_interfaces.py | 11 ++++++++-- startup_scripts/220_platforms.py | 5 +++-- startup_scripts/230_route_targets.py | 10 ++++++++-- startup_scripts/240_vrfs.py | 11 ++++++++-- startup_scripts/250_rirs.py | 5 +++-- startup_scripts/260_asns.py | 7 ++++--- startup_scripts/270_aggregates.py | 12 ++++++++--- startup_scripts/280_prefix_vlan_roles.py | 5 +++-- startup_scripts/290_cluster_types.py | 5 +++-- startup_scripts/300_cluster_groups.py | 7 +++++-- startup_scripts/310_clusters.py | 12 ++++++++--- startup_scripts/320_vlan_groups.py | 11 ++++++++-- startup_scripts/330_vlans.py | 11 ++++++++-- startup_scripts/340_virtual_machines.py | 14 ++++++++++--- .../350_virtualization_interfaces.py | 11 ++++++++-- startup_scripts/360_prefixes.py | 11 ++++++++-- startup_scripts/370_ip_addresses.py | 11 ++++++++-- startup_scripts/400_services.py | 4 +++- startup_scripts/420_providers.py | 10 ++++++++-- startup_scripts/440_circuit_types.py | 10 ++++++++-- startup_scripts/450_circuits.py | 12 ++++++++--- .../startup_script_utils/__init__.py | 1 + startup_scripts/startup_script_utils/utils.py | 15 ++++++++++++++ 42 files changed, 275 insertions(+), 95 deletions(-) create mode 100644 startup_scripts/startup_script_utils/utils.py diff --git a/initializers/users.yml b/initializers/users.yml index c163d5043..2dca92d6f 100644 --- a/initializers/users.yml +++ b/initializers/users.yml @@ -4,6 +4,7 @@ # password: reader # writer: # password: writer +# api_token: # leave empty to not generate any token # jdoe: # first_name: John # last_name: Doe diff --git a/startup_scripts/000_users.py b/startup_scripts/000_users.py index 1435d8128..7ca63052f 100644 --- a/startup_scripts/000_users.py +++ b/startup_scripts/000_users.py @@ -9,13 +9,17 @@ sys.exit() for username, user_details in users.items(): - if not User.objects.filter(username=username): - user = User.objects.create_user( - username=username, - password=user_details.get("password", 0) or User.objects.make_random_password(), - ) - print("πŸ‘€ Created user", username) + api_token = user_details.pop("api_token", Token.generate_key()) + password = user_details.pop("password", User.objects.make_random_password()) + + user, created = User.objects.get_or_create(username=username, defaults=user_details) + + if created: + user.set_password(password) + user.save() - if user_details.get("api_token", 0): - Token.objects.create(user=user, key=user_details["api_token"]) + if api_token: + Token.objects.get_or_create(user=user, key=api_token) + + print("πŸ‘€ Created user", username) diff --git a/startup_scripts/020_object_permissions.py b/startup_scripts/020_object_permissions.py index 8a5ecd40b..562a09c26 100644 --- a/startup_scripts/020_object_permissions.py +++ b/startup_scripts/020_object_permissions.py @@ -14,9 +14,11 @@ object_permission, created = ObjectPermission.objects.get_or_create( name=permission_name, - description=permission_details["description"], - enabled=permission_details["enabled"], - actions=permission_details["actions"], + defaults={ + "description": permission_details["description"], + "enabled": permission_details["enabled"], + "actions": permission_details["actions"], + }, ) if permission_details.get("object_types", 0): diff --git a/startup_scripts/040_custom_links.py b/startup_scripts/040_custom_links.py index 40144bd13..d8c0bba32 100644 --- a/startup_scripts/040_custom_links.py +++ b/startup_scripts/040_custom_links.py @@ -2,7 +2,7 @@ from django.contrib.contenttypes.models import ContentType from extras.models import CustomLink -from startup_script_utils import load_yaml +from startup_script_utils import load_yaml, split_params custom_links = load_yaml("/opt/netbox/initializers/custom_links.yml") @@ -28,6 +28,8 @@ def get_content_type_id(content_type): ) continue - custom_link, created = CustomLink.objects.get_or_create(**link) + matching_params, defaults = split_params(link) + custom_link, created = CustomLink.objects.get_or_create(**matching_params, defaults=defaults) + if created: print("πŸ”— Created Custom Link '{0}'".format(custom_link.name)) diff --git a/startup_scripts/050_tags.py b/startup_scripts/050_tags.py index e50a000cc..06a18cb9f 100644 --- a/startup_scripts/050_tags.py +++ b/startup_scripts/050_tags.py @@ -1,7 +1,7 @@ import sys from extras.models import Tag -from startup_script_utils import load_yaml +from startup_script_utils import load_yaml, split_params from utilities.choices import ColorChoices tags = load_yaml("/opt/netbox/initializers/tags.yml") @@ -17,7 +17,8 @@ if color in color_tpl: params["color"] = color_tpl[0] - tag, created = Tag.objects.get_or_create(**params) + matching_params, defaults = split_params(params) + tag, created = Tag.objects.get_or_create(**matching_params, defaults=defaults) if created: print("🎨 Created Tag", tag.name) diff --git a/startup_scripts/060_webhooks.py b/startup_scripts/060_webhooks.py index 8787f5f40..9de6e7e31 100644 --- a/startup_scripts/060_webhooks.py +++ b/startup_scripts/060_webhooks.py @@ -2,7 +2,7 @@ from django.contrib.contenttypes.models import ContentType from extras.models import Webhook -from startup_script_utils import load_yaml +from startup_script_utils import load_yaml, split_params webhooks = load_yaml("/opt/netbox/initializers/webhooks.yml") @@ -26,7 +26,9 @@ def get_content_type_id(hook_name, content_type): except ContentType.DoesNotExist: continue - webhook, created = Webhook.objects.get_or_create(**hook) + matching_params, defaults = split_params(hook) + webhook, created = Webhook.objects.get_or_create(**matching_params, defaults=defaults) + if created: webhook.content_types.set(obj_type_ids) webhook.save() diff --git a/startup_scripts/070_tenant_groups.py b/startup_scripts/070_tenant_groups.py index 65cf15541..0bfebadbe 100644 --- a/startup_scripts/070_tenant_groups.py +++ b/startup_scripts/070_tenant_groups.py @@ -1,6 +1,6 @@ import sys -from startup_script_utils import load_yaml +from startup_script_utils import load_yaml, split_params from tenancy.models import TenantGroup tenant_groups = load_yaml("/opt/netbox/initializers/tenant_groups.yml") @@ -9,7 +9,8 @@ sys.exit() for params in tenant_groups: - tenant_group, created = TenantGroup.objects.get_or_create(**params) + matching_params, defaults = split_params(params) + tenant_group, created = TenantGroup.objects.get_or_create(**matching_params, defaults=defaults) if created: print("πŸ”³ Created Tenant Group", tenant_group.name) diff --git a/startup_scripts/080_tenants.py b/startup_scripts/080_tenants.py index 7b1a629f0..9f34ec9d6 100644 --- a/startup_scripts/080_tenants.py +++ b/startup_scripts/080_tenants.py @@ -1,6 +1,11 @@ import sys -from startup_script_utils import load_yaml, pop_custom_fields, set_custom_fields_values +from startup_script_utils import ( + load_yaml, + pop_custom_fields, + set_custom_fields_values, + split_params, +) from tenancy.models import Tenant, TenantGroup tenants = load_yaml("/opt/netbox/initializers/tenants.yml") @@ -20,7 +25,8 @@ params[assoc] = model.objects.get(**query) - tenant, created = Tenant.objects.get_or_create(**params) + matching_params, defaults = split_params(params) + tenant, created = Tenant.objects.get_or_create(**matching_params, defaults=defaults) if created: set_custom_fields_values(tenant, custom_field_data) diff --git a/startup_scripts/090_regions.py b/startup_scripts/090_regions.py index 9d5c91f92..592c4c5ac 100644 --- a/startup_scripts/090_regions.py +++ b/startup_scripts/090_regions.py @@ -1,7 +1,7 @@ import sys from dcim.models import Region -from startup_script_utils import load_yaml +from startup_script_utils import load_yaml, split_params regions = load_yaml("/opt/netbox/initializers/regions.yml") @@ -19,7 +19,8 @@ params[assoc] = model.objects.get(**query) - region, created = Region.objects.get_or_create(**params) + matching_params, defaults = split_params(params) + region, created = Region.objects.get_or_create(**matching_params, defaults=defaults) if created: print("🌐 Created region", region.name) diff --git a/startup_scripts/110_sites.py b/startup_scripts/110_sites.py index f7851391a..a8b9bc417 100644 --- a/startup_scripts/110_sites.py +++ b/startup_scripts/110_sites.py @@ -1,7 +1,12 @@ import sys from dcim.models import Region, Site -from startup_script_utils import load_yaml, pop_custom_fields, set_custom_fields_values +from startup_script_utils import ( + load_yaml, + pop_custom_fields, + set_custom_fields_values, + split_params, +) from tenancy.models import Tenant sites = load_yaml("/opt/netbox/initializers/sites.yml") @@ -21,7 +26,8 @@ params[assoc] = model.objects.get(**query) - site, created = Site.objects.get_or_create(**params) + matching_params, defaults = split_params(params) + site, created = Site.objects.get_or_create(**matching_params, defaults=defaults) if created: set_custom_fields_values(site, custom_field_data) diff --git a/startup_scripts/120_locations.py b/startup_scripts/120_locations.py index d8a2c5a11..6a269b24e 100644 --- a/startup_scripts/120_locations.py +++ b/startup_scripts/120_locations.py @@ -1,13 +1,14 @@ import sys from dcim.models import Location, Site -from startup_script_utils import load_yaml +from startup_script_utils import load_yaml, split_params rack_groups = load_yaml("/opt/netbox/initializers/locations.yml") if rack_groups is None: sys.exit() +match_params = ["name", "slug", "site"] required_assocs = {"site": (Site, "name")} for params in rack_groups: @@ -17,7 +18,8 @@ query = {field: params.pop(assoc)} params[assoc] = model.objects.get(**query) - location, created = Location.objects.get_or_create(**params) + matching_params, defaults = split_params(params, match_params) + location, created = Location.objects.get_or_create(**matching_params, defaults=defaults) if created: print("🎨 Created location", location.name) diff --git a/startup_scripts/130_rack_roles.py b/startup_scripts/130_rack_roles.py index 585040558..8d43237be 100644 --- a/startup_scripts/130_rack_roles.py +++ b/startup_scripts/130_rack_roles.py @@ -1,7 +1,7 @@ import sys from dcim.models import RackRole -from startup_script_utils import load_yaml +from startup_script_utils import load_yaml, split_params from utilities.choices import ColorChoices rack_roles = load_yaml("/opt/netbox/initializers/rack_roles.yml") @@ -17,7 +17,8 @@ if color in color_tpl: params["color"] = color_tpl[0] - rack_role, created = RackRole.objects.get_or_create(**params) + matching_params, defaults = split_params(params) + rack_role, created = RackRole.objects.get_or_create(**matching_params, defaults=defaults) if created: print("🎨 Created rack role", rack_role.name) diff --git a/startup_scripts/140_racks.py b/startup_scripts/140_racks.py index b2cfc801c..8029ed4b6 100644 --- a/startup_scripts/140_racks.py +++ b/startup_scripts/140_racks.py @@ -1,7 +1,12 @@ import sys from dcim.models import Location, Rack, RackRole, Site -from startup_script_utils import load_yaml, pop_custom_fields, set_custom_fields_values +from startup_script_utils import ( + load_yaml, + pop_custom_fields, + set_custom_fields_values, + split_params, +) from tenancy.models import Tenant racks = load_yaml("/opt/netbox/initializers/racks.yml") @@ -9,8 +14,8 @@ if racks is None: sys.exit() +match_params = ["name", "site"] required_assocs = {"site": (Site, "name")} - optional_assocs = { "role": (RackRole, "name"), "tenant": (Tenant, "name"), @@ -33,7 +38,8 @@ params[assoc] = model.objects.get(**query) - rack, created = Rack.objects.get_or_create(**params) + matching_params, defaults = split_params(params, match_params) + rack, created = Rack.objects.get_or_create(**matching_params, defaults=defaults) if created: set_custom_fields_values(rack, custom_field_data) diff --git a/startup_scripts/150_power_panels.py b/startup_scripts/150_power_panels.py index 8542435bc..34d48ff8f 100644 --- a/startup_scripts/150_power_panels.py +++ b/startup_scripts/150_power_panels.py @@ -1,15 +1,20 @@ import sys from dcim.models import Location, PowerPanel, Site -from startup_script_utils import load_yaml, pop_custom_fields, set_custom_fields_values +from startup_script_utils import ( + load_yaml, + pop_custom_fields, + set_custom_fields_values, + split_params, +) power_panels = load_yaml("/opt/netbox/initializers/power_panels.yml") if power_panels is None: sys.exit() +match_params = ["name", "site"] required_assocs = {"site": (Site, "name")} - optional_assocs = {"location": (Location, "name")} for params in power_panels: @@ -28,7 +33,8 @@ params[assoc] = model.objects.get(**query) - power_panel, created = PowerPanel.objects.get_or_create(**params) + matching_params, defaults = split_params(params, match_params) + power_panel, created = PowerPanel.objects.get_or_create(**matching_params, defaults=defaults) if created: set_custom_fields_values(power_panel, custom_field_data) diff --git a/startup_scripts/160_power_feeds.py b/startup_scripts/160_power_feeds.py index f5aa5b570..77e9937aa 100644 --- a/startup_scripts/160_power_feeds.py +++ b/startup_scripts/160_power_feeds.py @@ -1,15 +1,20 @@ import sys from dcim.models import PowerFeed, PowerPanel, Rack -from startup_script_utils import load_yaml, pop_custom_fields, set_custom_fields_values +from startup_script_utils import ( + load_yaml, + pop_custom_fields, + set_custom_fields_values, + split_params, +) power_feeds = load_yaml("/opt/netbox/initializers/power_feeds.yml") if power_feeds is None: sys.exit() +match_params = ["name", "power_panel"] required_assocs = {"power_panel": (PowerPanel, "name")} - optional_assocs = {"rack": (Rack, "name")} for params in power_feeds: @@ -28,7 +33,8 @@ params[assoc] = model.objects.get(**query) - power_feed, created = PowerFeed.objects.get_or_create(**params) + matching_params, defaults = split_params(params, match_params) + power_feed, created = PowerFeed.objects.get_or_create(**matching_params, defaults=defaults) if created: set_custom_fields_values(power_feed, custom_field_data) diff --git a/startup_scripts/170_manufacturers.py b/startup_scripts/170_manufacturers.py index d11b44092..c105a4651 100644 --- a/startup_scripts/170_manufacturers.py +++ b/startup_scripts/170_manufacturers.py @@ -1,7 +1,7 @@ import sys from dcim.models import Manufacturer -from startup_script_utils import load_yaml +from startup_script_utils import load_yaml, split_params manufacturers = load_yaml("/opt/netbox/initializers/manufacturers.yml") @@ -9,7 +9,8 @@ sys.exit() for params in manufacturers: - manufacturer, created = Manufacturer.objects.get_or_create(**params) + matching_params, defaults = split_params(params) + manufacturer, created = Manufacturer.objects.get_or_create(**matching_params, defaults=defaults) if created: print("🏭 Created Manufacturer", manufacturer.name) diff --git a/startup_scripts/180_device_roles.py b/startup_scripts/180_device_roles.py index 635acff54..3cea0f10d 100644 --- a/startup_scripts/180_device_roles.py +++ b/startup_scripts/180_device_roles.py @@ -1,7 +1,7 @@ import sys from dcim.models import DeviceRole -from startup_script_utils import load_yaml +from startup_script_utils import load_yaml, split_params from utilities.choices import ColorChoices device_roles = load_yaml("/opt/netbox/initializers/device_roles.yml") @@ -18,7 +18,8 @@ if color in color_tpl: params["color"] = color_tpl[0] - device_role, created = DeviceRole.objects.get_or_create(**params) + matching_params, defaults = split_params(params) + device_role, created = DeviceRole.objects.get_or_create(**matching_params, defaults=defaults) if created: print("🎨 Created device role", device_role.name) diff --git a/startup_scripts/190_device_types.py b/startup_scripts/190_device_types.py index 0d3050b45..98abd05a0 100644 --- a/startup_scripts/190_device_types.py +++ b/startup_scripts/190_device_types.py @@ -1,7 +1,12 @@ import sys from dcim.models import DeviceType, Manufacturer, Region -from startup_script_utils import load_yaml, pop_custom_fields, set_custom_fields_values +from startup_script_utils import ( + load_yaml, + pop_custom_fields, + set_custom_fields_values, + split_params, +) from tenancy.models import Tenant device_types = load_yaml("/opt/netbox/initializers/device_types.yml") @@ -9,8 +14,8 @@ if device_types is None: sys.exit() +match_params = ["manufacturer", "model", "slug"] required_assocs = {"manufacturer": (Manufacturer, "name")} - optional_assocs = {"region": (Region, "name"), "tenant": (Tenant, "name")} for params in device_types: @@ -29,7 +34,8 @@ params[assoc] = model.objects.get(**query) - device_type, created = DeviceType.objects.get_or_create(**params) + matching_params, defaults = split_params(params, match_params) + device_type, created = DeviceType.objects.get_or_create(**matching_params, defaults=defaults) if created: set_custom_fields_values(device_type, custom_field_data) diff --git a/startup_scripts/200_devices.py b/startup_scripts/200_devices.py index 423b7c9de..079fa1f30 100644 --- a/startup_scripts/200_devices.py +++ b/startup_scripts/200_devices.py @@ -1,7 +1,12 @@ import sys from dcim.models import Device, DeviceRole, DeviceType, Location, Platform, Rack, Site -from startup_script_utils import load_yaml, pop_custom_fields, set_custom_fields_values +from startup_script_utils import ( + load_yaml, + pop_custom_fields, + set_custom_fields_values, + split_params, +) from tenancy.models import Tenant from virtualization.models import Cluster @@ -10,12 +15,12 @@ if devices is None: sys.exit() +match_params = ["device_type", "name", "site"] required_assocs = { "device_role": (DeviceRole, "name"), "device_type": (DeviceType, "model"), "site": (Site, "name"), } - optional_assocs = { "tenant": (Tenant, "name"), "platform": (Platform, "name"), @@ -27,7 +32,7 @@ for params in devices: custom_field_data = pop_custom_fields(params) - # primary ips are handled later in `270_primary_ips.py` + # primary ips are handled later in `380_primary_ips.py` params.pop("primary_ip4", None) params.pop("primary_ip6", None) @@ -44,7 +49,8 @@ params[assoc] = model.objects.get(**query) - device, created = Device.objects.get_or_create(**params) + matching_params, defaults = split_params(params, match_params) + device, created = Device.objects.get_or_create(**matching_params, defaults=defaults) if created: set_custom_fields_values(device, custom_field_data) diff --git a/startup_scripts/210_dcim_interfaces.py b/startup_scripts/210_dcim_interfaces.py index a8026282f..f8adbd585 100644 --- a/startup_scripts/210_dcim_interfaces.py +++ b/startup_scripts/210_dcim_interfaces.py @@ -1,13 +1,19 @@ import sys from dcim.models import Device, Interface -from startup_script_utils import load_yaml, pop_custom_fields, set_custom_fields_values +from startup_script_utils import ( + load_yaml, + pop_custom_fields, + set_custom_fields_values, + split_params, +) interfaces = load_yaml("/opt/netbox/initializers/dcim_interfaces.yml") if interfaces is None: sys.exit() +match_params = ["device", "name"] required_assocs = {"device": (Device, "name")} for params in interfaces: @@ -19,7 +25,8 @@ params[assoc] = model.objects.get(**query) - interface, created = Interface.objects.get_or_create(**params) + matching_params, defaults = split_params(params, match_params) + interface, created = Interface.objects.get_or_create(**matching_params, defaults=defaults) if created: set_custom_fields_values(interface, custom_field_data) diff --git a/startup_scripts/220_platforms.py b/startup_scripts/220_platforms.py index 633b89ffc..73905d4e1 100644 --- a/startup_scripts/220_platforms.py +++ b/startup_scripts/220_platforms.py @@ -1,7 +1,7 @@ import sys from dcim.models import Manufacturer, Platform -from startup_script_utils import load_yaml +from startup_script_utils import load_yaml, split_params platforms = load_yaml("/opt/netbox/initializers/platforms.yml") @@ -21,7 +21,8 @@ params[assoc] = model.objects.get(**query) - platform, created = Platform.objects.get_or_create(**params) + matching_params, defaults = split_params(params) + platform, created = Platform.objects.get_or_create(**matching_params, defaults=defaults) if created: print("πŸ’Ύ Created platform", platform.name) diff --git a/startup_scripts/230_route_targets.py b/startup_scripts/230_route_targets.py index e1c82217e..3782391d6 100644 --- a/startup_scripts/230_route_targets.py +++ b/startup_scripts/230_route_targets.py @@ -1,7 +1,12 @@ import sys from ipam.models import RouteTarget -from startup_script_utils import load_yaml, pop_custom_fields, set_custom_fields_values +from startup_script_utils import ( + load_yaml, + pop_custom_fields, + set_custom_fields_values, + split_params, +) from tenancy.models import Tenant route_targets = load_yaml("/opt/netbox/initializers/route_targets.yml") @@ -21,7 +26,8 @@ params[assoc] = model.objects.get(**query) - route_target, created = RouteTarget.objects.get_or_create(**params) + matching_params, defaults = split_params(params) + route_target, created = RouteTarget.objects.get_or_create(**matching_params, defaults=defaults) if created: set_custom_fields_values(route_target, custom_field_data) diff --git a/startup_scripts/240_vrfs.py b/startup_scripts/240_vrfs.py index a67c8c8cb..061f11e46 100644 --- a/startup_scripts/240_vrfs.py +++ b/startup_scripts/240_vrfs.py @@ -1,7 +1,12 @@ import sys from ipam.models import VRF -from startup_script_utils import load_yaml, pop_custom_fields, set_custom_fields_values +from startup_script_utils import ( + load_yaml, + pop_custom_fields, + set_custom_fields_values, + split_params, +) from tenancy.models import Tenant vrfs = load_yaml("/opt/netbox/initializers/vrfs.yml") @@ -9,6 +14,7 @@ if vrfs is None: sys.exit() +match_params = ["name", "rd"] optional_assocs = {"tenant": (Tenant, "name")} for params in vrfs: @@ -21,7 +27,8 @@ params[assoc] = model.objects.get(**query) - vrf, created = VRF.objects.get_or_create(**params) + matching_params, defaults = split_params(params) + vrf, created = VRF.objects.get_or_create(**matching_params, defaults=defaults) if created: set_custom_fields_values(vrf, custom_field_data) diff --git a/startup_scripts/250_rirs.py b/startup_scripts/250_rirs.py index 0e0df2008..08d1703af 100644 --- a/startup_scripts/250_rirs.py +++ b/startup_scripts/250_rirs.py @@ -1,7 +1,7 @@ import sys from ipam.models import RIR -from startup_script_utils import load_yaml +from startup_script_utils import load_yaml, split_params rirs = load_yaml("/opt/netbox/initializers/rirs.yml") @@ -9,7 +9,8 @@ sys.exit() for params in rirs: - rir, created = RIR.objects.get_or_create(**params) + matching_params, defaults = split_params(params) + rir, created = RIR.objects.get_or_create(**matching_params, defaults=defaults) if created: print("πŸ—ΊοΈ Created RIR", rir.name) diff --git a/startup_scripts/260_asns.py b/startup_scripts/260_asns.py index 893f3ba91..93776d247 100644 --- a/startup_scripts/260_asns.py +++ b/startup_scripts/260_asns.py @@ -1,7 +1,7 @@ import sys from ipam.models import ASN, RIR -from startup_script_utils import load_yaml +from startup_script_utils import load_yaml, split_params from tenancy.models import Tenant asns = load_yaml("/opt/netbox/initializers/asns.yml") @@ -9,8 +9,8 @@ if asns is None: sys.exit() +match_params = ["asn", "rir"] required_assocs = {"rir": (RIR, "name")} - optional_assocs = {"tenant": (Tenant, "name")} for params in asns: @@ -27,7 +27,8 @@ params[assoc] = model.objects.get(**query) - asn, created = ASN.objects.get_or_create(**params) + matching_params, defaults = split_params(params, match_params) + asn, created = ASN.objects.get_or_create(**matching_params, defaults=defaults) if created: print(f"πŸ”‘ Created ASN {asn.asn}") diff --git a/startup_scripts/270_aggregates.py b/startup_scripts/270_aggregates.py index c638e6f81..d1682ade3 100644 --- a/startup_scripts/270_aggregates.py +++ b/startup_scripts/270_aggregates.py @@ -2,7 +2,12 @@ from ipam.models import RIR, Aggregate from netaddr import IPNetwork -from startup_script_utils import load_yaml, pop_custom_fields, set_custom_fields_values +from startup_script_utils import ( + load_yaml, + pop_custom_fields, + set_custom_fields_values, + split_params, +) from tenancy.models import Tenant aggregates = load_yaml("/opt/netbox/initializers/aggregates.yml") @@ -10,8 +15,8 @@ if aggregates is None: sys.exit() +match_params = ["prefix", "rir"] required_assocs = {"rir": (RIR, "name")} - optional_assocs = { "tenant": (Tenant, "name"), } @@ -34,7 +39,8 @@ params[assoc] = model.objects.get(**query) - aggregate, created = Aggregate.objects.get_or_create(**params) + matching_params, defaults = split_params(params, match_params) + aggregate, created = Aggregate.objects.get_or_create(**matching_params, defaults=defaults) if created: set_custom_fields_values(aggregate, custom_field_data) diff --git a/startup_scripts/280_prefix_vlan_roles.py b/startup_scripts/280_prefix_vlan_roles.py index ec359fbc0..c17feb853 100644 --- a/startup_scripts/280_prefix_vlan_roles.py +++ b/startup_scripts/280_prefix_vlan_roles.py @@ -1,7 +1,7 @@ import sys from ipam.models import Role -from startup_script_utils import load_yaml +from startup_script_utils import load_yaml, split_params roles = load_yaml("/opt/netbox/initializers/prefix_vlan_roles.yml") @@ -9,7 +9,8 @@ sys.exit() for params in roles: - role, created = Role.objects.get_or_create(**params) + matching_params, defaults = split_params(params) + role, created = Role.objects.get_or_create(**matching_params, defaults=defaults) if created: print("⛹️‍ Created Prefix/VLAN Role", role.name) diff --git a/startup_scripts/290_cluster_types.py b/startup_scripts/290_cluster_types.py index 9f361b1cf..642bc7a5c 100644 --- a/startup_scripts/290_cluster_types.py +++ b/startup_scripts/290_cluster_types.py @@ -1,6 +1,6 @@ import sys -from startup_script_utils import load_yaml +from startup_script_utils import load_yaml, split_params from virtualization.models import ClusterType cluster_types = load_yaml("/opt/netbox/initializers/cluster_types.yml") @@ -9,7 +9,8 @@ sys.exit() for params in cluster_types: - cluster_type, created = ClusterType.objects.get_or_create(**params) + matching_params, defaults = split_params(params) + cluster_type, created = ClusterType.objects.get_or_create(**matching_params, defaults=defaults) if created: print("🧰 Created Cluster Type", cluster_type.name) diff --git a/startup_scripts/300_cluster_groups.py b/startup_scripts/300_cluster_groups.py index fedd292ec..a8e357337 100644 --- a/startup_scripts/300_cluster_groups.py +++ b/startup_scripts/300_cluster_groups.py @@ -1,6 +1,6 @@ import sys -from startup_script_utils import load_yaml +from startup_script_utils import load_yaml, split_params from virtualization.models import ClusterGroup cluster_groups = load_yaml("/opt/netbox/initializers/cluster_groups.yml") @@ -9,7 +9,10 @@ sys.exit() for params in cluster_groups: - cluster_group, created = ClusterGroup.objects.get_or_create(**params) + matching_params, defaults = split_params(params) + cluster_group, created = ClusterGroup.objects.get_or_create( + **matching_params, defaults=defaults + ) if created: print("πŸ—„οΈ Created Cluster Group", cluster_group.name) diff --git a/startup_scripts/310_clusters.py b/startup_scripts/310_clusters.py index 2748f2038..5da600597 100644 --- a/startup_scripts/310_clusters.py +++ b/startup_scripts/310_clusters.py @@ -1,7 +1,12 @@ import sys from dcim.models import Site -from startup_script_utils import load_yaml, pop_custom_fields, set_custom_fields_values +from startup_script_utils import ( + load_yaml, + pop_custom_fields, + set_custom_fields_values, + split_params, +) from tenancy.models import Tenant from virtualization.models import Cluster, ClusterGroup, ClusterType @@ -10,8 +15,8 @@ if clusters is None: sys.exit() +match_params = ["name", "type"] required_assocs = {"type": (ClusterType, "name")} - optional_assocs = { "site": (Site, "name"), "group": (ClusterGroup, "name"), @@ -34,7 +39,8 @@ params[assoc] = model.objects.get(**query) - cluster, created = Cluster.objects.get_or_create(**params) + matching_params, defaults = split_params(params, match_params) + cluster, created = Cluster.objects.get_or_create(**matching_params, defaults=defaults) if created: set_custom_fields_values(cluster, custom_field_data) diff --git a/startup_scripts/320_vlan_groups.py b/startup_scripts/320_vlan_groups.py index 2a4a33dec..19ba9ed05 100644 --- a/startup_scripts/320_vlan_groups.py +++ b/startup_scripts/320_vlan_groups.py @@ -2,7 +2,12 @@ from django.contrib.contenttypes.models import ContentType from ipam.models import VLANGroup -from startup_script_utils import load_yaml, pop_custom_fields, set_custom_fields_values +from startup_script_utils import ( + load_yaml, + pop_custom_fields, + set_custom_fields_values, + split_params, +) vlan_groups = load_yaml("/opt/netbox/initializers/vlan_groups.yml") @@ -32,7 +37,9 @@ ) continue params["scope_id"] = ct.model_class().objects.get(**query).id - vlan_group, created = VLANGroup.objects.get_or_create(**params) + + matching_params, defaults = split_params(params) + vlan_group, created = VLANGroup.objects.get_or_create(**matching_params, defaults=defaults) if created: set_custom_fields_values(vlan_group, custom_field_data) diff --git a/startup_scripts/330_vlans.py b/startup_scripts/330_vlans.py index e8ebb94f1..11243a7ba 100644 --- a/startup_scripts/330_vlans.py +++ b/startup_scripts/330_vlans.py @@ -2,7 +2,12 @@ from dcim.models import Site from ipam.models import VLAN, Role, VLANGroup -from startup_script_utils import load_yaml, pop_custom_fields, set_custom_fields_values +from startup_script_utils import ( + load_yaml, + pop_custom_fields, + set_custom_fields_values, + split_params, +) from tenancy.models import Tenant, TenantGroup vlans = load_yaml("/opt/netbox/initializers/vlans.yml") @@ -10,6 +15,7 @@ if vlans is None: sys.exit() +match_params = ["name", "vid"] optional_assocs = { "site": (Site, "name"), "tenant": (Tenant, "name"), @@ -28,7 +34,8 @@ params[assoc] = model.objects.get(**query) - vlan, created = VLAN.objects.get_or_create(**params) + matching_params, defaults = split_params(params, match_params) + vlan, created = VLAN.objects.get_or_create(**matching_params, defaults=defaults) if created: set_custom_fields_values(vlan, custom_field_data) diff --git a/startup_scripts/340_virtual_machines.py b/startup_scripts/340_virtual_machines.py index 2e3f42885..ebde15709 100644 --- a/startup_scripts/340_virtual_machines.py +++ b/startup_scripts/340_virtual_machines.py @@ -1,7 +1,12 @@ import sys from dcim.models import DeviceRole, Platform -from startup_script_utils import load_yaml, pop_custom_fields, set_custom_fields_values +from startup_script_utils import ( + load_yaml, + pop_custom_fields, + set_custom_fields_values, + split_params, +) from tenancy.models import Tenant from virtualization.models import Cluster, VirtualMachine @@ -10,8 +15,8 @@ if virtual_machines is None: sys.exit() +match_params = ["cluster", "name"] required_assocs = {"cluster": (Cluster, "name")} - optional_assocs = { "tenant": (Tenant, "name"), "platform": (Platform, "name"), @@ -38,7 +43,10 @@ params[assoc] = model.objects.get(**query) - virtual_machine, created = VirtualMachine.objects.get_or_create(**params) + matching_params, defaults = split_params(params, match_params) + virtual_machine, created = VirtualMachine.objects.get_or_create( + **matching_params, defaults=defaults + ) if created: set_custom_fields_values(virtual_machine, custom_field_data) diff --git a/startup_scripts/350_virtualization_interfaces.py b/startup_scripts/350_virtualization_interfaces.py index 6ee63474c..b2407d843 100644 --- a/startup_scripts/350_virtualization_interfaces.py +++ b/startup_scripts/350_virtualization_interfaces.py @@ -1,6 +1,11 @@ import sys -from startup_script_utils import load_yaml, pop_custom_fields, set_custom_fields_values +from startup_script_utils import ( + load_yaml, + pop_custom_fields, + set_custom_fields_values, + split_params, +) from virtualization.models import VirtualMachine, VMInterface interfaces = load_yaml("/opt/netbox/initializers/virtualization_interfaces.yml") @@ -8,6 +13,7 @@ if interfaces is None: sys.exit() +match_params = ["name", "virtual_machine"] required_assocs = {"virtual_machine": (VirtualMachine, "name")} for params in interfaces: @@ -19,7 +25,8 @@ params[assoc] = model.objects.get(**query) - interface, created = VMInterface.objects.get_or_create(**params) + matching_params, defaults = split_params(params, match_params) + interface, created = VMInterface.objects.get_or_create(**matching_params, defaults=defaults) if created: set_custom_fields_values(interface, custom_field_data) diff --git a/startup_scripts/360_prefixes.py b/startup_scripts/360_prefixes.py index 4e2b0d007..b50ff3c09 100644 --- a/startup_scripts/360_prefixes.py +++ b/startup_scripts/360_prefixes.py @@ -3,7 +3,12 @@ from dcim.models import Site from ipam.models import VLAN, VRF, Prefix, Role from netaddr import IPNetwork -from startup_script_utils import load_yaml, pop_custom_fields, set_custom_fields_values +from startup_script_utils import ( + load_yaml, + pop_custom_fields, + set_custom_fields_values, + split_params, +) from tenancy.models import Tenant, TenantGroup prefixes = load_yaml("/opt/netbox/initializers/prefixes.yml") @@ -11,6 +16,7 @@ if prefixes is None: sys.exit() +match_params = ["prefix", "site", "vrf", "vlan"] optional_assocs = { "site": (Site, "name"), "tenant": (Tenant, "name"), @@ -31,7 +37,8 @@ query = {field: params.pop(assoc)} params[assoc] = model.objects.get(**query) - prefix, created = Prefix.objects.get_or_create(**params) + matching_params, defaults = split_params(params, match_params) + prefix, created = Prefix.objects.get_or_create(**matching_params, defaults=defaults) if created: set_custom_fields_values(prefix, custom_field_data) diff --git a/startup_scripts/370_ip_addresses.py b/startup_scripts/370_ip_addresses.py index 7f166f0bb..89dd8e051 100644 --- a/startup_scripts/370_ip_addresses.py +++ b/startup_scripts/370_ip_addresses.py @@ -5,7 +5,12 @@ from django.db.models import Q from ipam.models import VRF, IPAddress from netaddr import IPNetwork -from startup_script_utils import load_yaml, pop_custom_fields, set_custom_fields_values +from startup_script_utils import ( + load_yaml, + pop_custom_fields, + set_custom_fields_values, + split_params, +) from tenancy.models import Tenant from virtualization.models import VirtualMachine, VMInterface @@ -14,6 +19,7 @@ if ip_addresses is None: sys.exit() +match_params = ["address", "vrf"] optional_assocs = { "tenant": (Tenant, "name"), "vrf": (VRF, "name"), @@ -55,7 +61,8 @@ params[assoc] = model.objects.get(**query) - ip_address, created = IPAddress.objects.get_or_create(**params) + matching_params, defaults = split_params(params, match_params) + ip_address, created = IPAddress.objects.get_or_create(**matching_params, defaults=defaults) if created: set_custom_fields_values(ip_address, custom_field_data) diff --git a/startup_scripts/400_services.py b/startup_scripts/400_services.py index a28eb09f7..a92d82b1d 100644 --- a/startup_scripts/400_services.py +++ b/startup_scripts/400_services.py @@ -2,7 +2,7 @@ from dcim.models import Device from ipam.models import Service -from startup_script_utils import load_yaml +from startup_script_utils import load_yaml, split_params from virtualization.models import VirtualMachine services = load_yaml("/opt/netbox/initializers/services.yml") @@ -10,6 +10,7 @@ if services is None: sys.exit() +match_params = ["name", "device", "virtual_machine"] optional_assocs = { "device": (Device, "name"), "virtual_machine": (VirtualMachine, "name"), @@ -24,6 +25,7 @@ params[assoc] = model.objects.get(**query) + matching_params, defaults = split_params(params, match_params) service, created = Service.objects.get_or_create(**params) if created: diff --git a/startup_scripts/420_providers.py b/startup_scripts/420_providers.py index 5c4330ad3..e97315284 100644 --- a/startup_scripts/420_providers.py +++ b/startup_scripts/420_providers.py @@ -1,7 +1,12 @@ import sys from circuits.models import Provider -from startup_script_utils import load_yaml, pop_custom_fields, set_custom_fields_values +from startup_script_utils import ( + load_yaml, + pop_custom_fields, + set_custom_fields_values, + split_params, +) providers = load_yaml("/opt/netbox/initializers/providers.yml") @@ -11,7 +16,8 @@ for params in providers: custom_field_data = pop_custom_fields(params) - provider, created = Provider.objects.get_or_create(**params) + matching_params, defaults = split_params(params) + provider, created = Provider.objects.get_or_create(**matching_params, defaults=defaults) if created: set_custom_fields_values(provider, custom_field_data) diff --git a/startup_scripts/440_circuit_types.py b/startup_scripts/440_circuit_types.py index 071793c6f..285b61281 100644 --- a/startup_scripts/440_circuit_types.py +++ b/startup_scripts/440_circuit_types.py @@ -1,7 +1,12 @@ import sys from circuits.models import CircuitType -from startup_script_utils import load_yaml, pop_custom_fields, set_custom_fields_values +from startup_script_utils import ( + load_yaml, + pop_custom_fields, + set_custom_fields_values, + split_params, +) circuit_types = load_yaml("/opt/netbox/initializers/circuit_types.yml") @@ -11,7 +16,8 @@ for params in circuit_types: custom_field_data = pop_custom_fields(params) - circuit_type, created = CircuitType.objects.get_or_create(**params) + matching_params, defaults = split_params(params) + circuit_type, created = CircuitType.objects.get_or_create(**matching_params, defaults=defaults) if created: set_custom_fields_values(circuit_type, custom_field_data) diff --git a/startup_scripts/450_circuits.py b/startup_scripts/450_circuits.py index f82d3b71d..00de7dd76 100644 --- a/startup_scripts/450_circuits.py +++ b/startup_scripts/450_circuits.py @@ -1,7 +1,12 @@ import sys from circuits.models import Circuit, CircuitType, Provider -from startup_script_utils import load_yaml, pop_custom_fields, set_custom_fields_values +from startup_script_utils import ( + load_yaml, + pop_custom_fields, + set_custom_fields_values, + split_params, +) from tenancy.models import Tenant circuits = load_yaml("/opt/netbox/initializers/circuits.yml") @@ -9,8 +14,8 @@ if circuits is None: sys.exit() +match_params = ["cid", "provider", "type"] required_assocs = {"provider": (Provider, "name"), "type": (CircuitType, "name")} - optional_assocs = {"tenant": (Tenant, "name")} for params in circuits: @@ -29,7 +34,8 @@ params[assoc] = model.objects.get(**query) - circuit, created = Circuit.objects.get_or_create(**params) + matching_params, defaults = split_params(params, match_params) + circuit, created = Circuit.objects.get_or_create(**matching_params, defaults=defaults) if created: set_custom_fields_values(circuit, custom_field_data) diff --git a/startup_scripts/startup_script_utils/__init__.py b/startup_scripts/startup_script_utils/__init__.py index 290b87b4f..4c25c6bee 100644 --- a/startup_scripts/startup_script_utils/__init__.py +++ b/startup_scripts/startup_script_utils/__init__.py @@ -1,2 +1,3 @@ from .custom_fields import pop_custom_fields, set_custom_fields_values from .load_yaml import load_yaml +from .utils import split_params diff --git a/startup_scripts/startup_script_utils/utils.py b/startup_scripts/startup_script_utils/utils.py new file mode 100644 index 000000000..afcdf75b8 --- /dev/null +++ b/startup_scripts/startup_script_utils/utils.py @@ -0,0 +1,15 @@ +from typing import Any, Dict + + +def split_params(params: dict, unique_params: list = None) -> Dict[Any, Any]: + """Split params dict into dict with matching params and a dict with default values""" + + if unique_params is None: + unique_params = ["name", "slug"] + + matching_params = {} + for unique_param in unique_params: + param = params.pop(unique_param, None) + if param: + matching_params[unique_param] = param + return matching_params, params