Merge branch 'test' into feature/separate_default_params

build.sh

@@ -6,10 +6,9 @@ echo "▶️ $0 $*"
 set -e
 
 if [ "${1}x" == "x" ] || [ "${1}" == "--help" ] || [ "${1}" == "-h" ]; then
-  echo "Usage: ${0} <branch> [--push|--push-only]"
+  echo "Usage: ${0} <branch> [--push]"
   echo "  branch       The branch or tag to build. Required."
   echo "  --push       Pushes the built Docker image to the registry."
-  echo "  --push-only  Only pushes the Docker image to the registry, but does not build it."
   echo ""
   echo "You can use the following ENV variables to customize the build:"
   echo "  SRC_ORG     Which fork of netbox to use (i.e. github.com/\${SRC_ORG}/\${SRC_REPO})."
@@ -30,6 +29,9 @@ if [ "${1}x" == "x" ] || [ "${1}" == "--help" ] || [ "${1}" == "-h" ]; then
   echo "                When <branch>=master:  latest"
   echo "                When <branch>=develop: snapshot"
   echo "                Else:                  same as <branch>"
+  echo "  DOCKER_BUILD_PLATFORMS platforms the docker buildx will build the image for"
+  echo "              Used for specifying multiple arches."
+  echo "              Default: linux/amd64,linux/arm64"
   echo "  DOCKER_REGISTRY The Docker repository's registry (i.e. '\${DOCKER_REGISTRY}/\${DOCKER_ORG}/\${DOCKER_REPO}'')"
   echo "              Used for tagging the image."
   echo "              Default: docker.io"
@@ -95,8 +97,6 @@ if [ "${1}x" == "x" ] || [ "${1}" == "--help" ] || [ "${1}" == "-h" ]; then
   fi
 fi
 
-source ./build-functions/gh-functions.sh
-
 ###
 # Enabling dry-run mode
 ###
@@ -201,6 +201,12 @@ if [ -d "${NETBOX_PATH}/.git" ]; then
   )
 fi
 
+
+###
+# Variables for arch target
+###
+DOCKER_BUILD_PLATFORMS="${DOCKER_BUILD_PLATFORMS-linux/amd64,linux/arm64}"
+
 ###
 # Variables for tagging the docker image
 ###
@@ -375,38 +381,23 @@ for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do
     if [ -n "${NO_PROXY}" ]; then
       DOCKER_BUILD_ARGS+=(--build-arg "no_proxy=${NO_PROXY}")
     fi
+    if [ "${2}" == "--push" ]; then
+      DOCKER_BUILD_ARGS+=(--push)
+    fi
 
     ###
     # Building the docker image
     ###
     if [ "${SHOULD_BUILD}" == "true" ]; then
       echo "🐳 Building the Docker image '${TARGET_DOCKER_TAG_PROJECT}'."
       echo "    Build reason set to: ${BUILD_REASON}"
-      $DRY docker build "${DOCKER_BUILD_ARGS[@]}" .
+      $DRY docker buildx build --platform ${DOCKER_BUILD_PLATFORMS} "${DOCKER_BUILD_ARGS[@]}" .
       echo "✅ Finished building the Docker images '${TARGET_DOCKER_TAG_PROJECT}'"
-      echo "🔎 Inspecting labels on '${TARGET_DOCKER_TAG_PROJECT}'"
-      $DRY docker inspect "${TARGET_DOCKER_TAG_PROJECT}" --format "{{json .Config.Labels}}"
     else
       echo "Build skipped because sources didn't change"
       echo "::set-output name=skipped::true"
     fi
   fi
 
-  ###
-  # Pushing the docker images if either `--push` or `--push-only` are passed
-  ###
-  if [ "${2}" == "--push" ] || [ "${2}" == "--push-only" ]; then
-    source ./build-functions/docker-functions.sh
-    push_image_to_registry "${TARGET_DOCKER_TAG}"
-    push_image_to_registry "${TARGET_DOCKER_TAG_PROJECT}"
-
-    if [ -n "${TARGET_DOCKER_SHORT_TAG}" ]; then
-      push_image_to_registry "${TARGET_DOCKER_SHORT_TAG}"
-      push_image_to_registry "${TARGET_DOCKER_SHORT_TAG_PROJECT}"
-      push_image_to_registry "${TARGET_DOCKER_LATEST_TAG}"
-      push_image_to_registry "${TARGET_DOCKER_LATEST_TAG_PROJECT}"
-    fi
-  fi
-
   gh_echo "::endgroup::"
-done
+done

initializers/cables.yml

@@ -0,0 +1,53 @@
+## Required parameters for termination X ('a' or 'b'):
+##   termination_X_name - name of interface
+##   termination_X_device - name of the device interface belongs to
+##   termination_X_class - required if different than Interface which is the default
+## Supported termination classes: Interface, ConsolePort, ConsoleServerPort, FrontPort, RearPort
+##
+## If termination is a circuit then the required parameter is termination_x_circuit.
+## Required parameters for a circuit termination:
+## termination_x_circuit:
+##   term_side - termination side of a circuit. Must be A or B
+##   cid - circuit ID value
+##   site OR provider_network - name of Site or ProviderNetwork respectively. If both provided, Site takes precedence
+##
+## Any other Cable parameters supported by Netbox are supported as the top level keys, e.g. 'type', 'status', etc.
+##
+## - termination_a_name: console
+##   termination_a_device: spine
+##   termination_a_class: ConsolePort
+##   termination_b_name: tty9
+##   termination_b_device: console-server
+##   termination_b_class: ConsoleServerPort
+##   type: cat6
+##
+# - termination_a_name: to-server02
+#   termination_a_device: server01
+#   termination_b_name: to-server01
+#   termination_b_device: server02
+#   status: planned
+#   type: mmf
+
+# - termination_a_name: eth0
+#   termination_a_device: server02
+#   termination_b_circuit:
+#     term_side: A
+#     cid: Circuit_ID-1
+#     site: AMS 1
+#   type: cat6
+
+# - termination_a_name: psu0
+#   termination_a_device: server04
+#   termination_a_class: PowerPort
+#   termination_b_feed:
+#     name: power feed 1
+#     power_panel:
+#       name: power panel AMS 1
+#       site: AMS 1
+
+# - termination_a_name: outlet1
+#   termination_a_device: server04
+#   termination_a_class: PowerOutlet
+#   termination_b_name: psu1
+#   termination_b_device: server04
+#   termination_b_class: PowerPort

initializers/dcim_interfaces.yml

@@ -10,9 +10,13 @@
 
 # - device: server01
 #   enabled: true
-#   type: virtual
+#   type: 1000base-x-sfp
 #   name: to-server02
 # - device: server02
 #   enabled: true
-#   type: virtual
+#   type: 1000base-x-sfp
 #   name: to-server01
+# - device: server02
+#   enabled: true
+#   type: 1000base-t
+#   name: eth0

initializers/device_types.yml

@@ -21,3 +21,35 @@
 #   slug: other
 #   custom_field_data:
 #     text_field: Description
+#   interfaces:
+#     - name: eth0
+#       type: 1000base-t
+#       mgmt_only: True
+#     - name: eth1
+#       type: 1000base-t
+#   console_server_ports:
+#     - name_template: ttyS[1-48]
+#       type: rj-45
+#   power_ports:
+#     - name_template: psu[0,1]
+#       type: iec-60320-c14
+#       maximum_draw: 35
+#       allocated_draw: 35
+#   front_ports:
+#     - name_template: front[1,2]
+#       type: 8p8c
+#       rear_port_template: rear[0,1]
+#       rear_port_position_template: "[1,2]"
+#   rear_ports:
+#     - name_template: rear[0,1]
+#       type: 8p8c
+#       positions_template: "[3,2]"
+#   device_bays:
+#     - name_template: bay[0-9]
+#       label_template: test[0-5,9,6-8]
+#       description: Test description
+#   power_outlets:
+#     - name_template: outlet[0,1]
+#       type: iec-60320-c5
+#       power_port: psu0
+#       feed_leg: B

startup_scripts/190_device_types.py

@@ -1,13 +1,72 @@
 import sys
+from typing import List
 
 from dcim.models import DeviceType, Manufacturer, Region
+from dcim.models.device_component_templates import (
+    ConsolePortTemplate,
+    ConsoleServerPortTemplate,
+    DeviceBayTemplate,
+    FrontPortTemplate,
+    InterfaceTemplate,
+    PowerOutletTemplate,
+    PowerPortTemplate,
+    RearPortTemplate,
+)
+
 from startup_script_utils import (
     load_yaml,
     pop_custom_fields,
     set_custom_fields_values,
     split_params,
 )
 from tenancy.models import Tenant
+from utilities.forms.utils import expand_alphanumeric_pattern
+
+
+def expand_templates(params: List[dict], device_type: DeviceType) -> List[dict]:
+    templateable_fields = ["name", "label", "positions", "rear_port", "rear_port_position"]
+
+    expanded = []
+    for param in params:
+        param["device_type"] = device_type
+        expanded_fields = {}
+        has_plain_fields = False
+
+        for field in templateable_fields:
+            template_value = param.pop(f"{field}_template", None)
+
+            if field in param:
+                has_plain_fields = True
+                expanded.append(param)
+            elif template_value:
+                expanded_fields[field] = list(expand_alphanumeric_pattern(template_value))
+
+        if expanded_fields and has_plain_fields:
+            raise ValueError(f"Mix of plain and template keys provided for {templateable_fields}")
+
+        if not expanded_fields:
+            continue
+
+        elements = list(expanded_fields.values())
+        master_len = len(elements[0])
+        if not all([len(elem) == master_len for elem in elements]):
+            raise ValueError(
+                f"Number of elements in template fields "
+                f"{list(expanded_fields.keys())} must be equal"
+            )
+
+        for idx in range(master_len):
+            tmp = param.copy()
+            for field, value in expanded_fields.items():
+                if field in nested_assocs:
+                    model, match_key = nested_assocs[field]
+                    query = {match_key: value[idx], "device_type": device_type}
+                    tmp[field] = model.objects.get(**query)
+                else:
+                    tmp[field] = value[idx]
+            expanded.append(tmp)
+    return expanded
+
 
 device_types = load_yaml("/opt/netbox/initializers/device_types.yml")
 
@@ -17,9 +76,22 @@
 match_params = ["manufacturer", "model", "slug"]
 required_assocs = {"manufacturer": (Manufacturer, "name")}
 optional_assocs = {"region": (Region, "name"), "tenant": (Tenant, "name")}
+nested_assocs = {"rear_port": (RearPortTemplate, "name"), "power_port": (PowerPortTemplate, "name")}
+
+supported_components = {
+    "interfaces": (InterfaceTemplate, ["name"]),
+    "console_ports": (ConsolePortTemplate, ["name"]),
+    "console_server_ports": (ConsoleServerPortTemplate, ["name"]),
+    "power_ports": (PowerPortTemplate, ["name"]),
+    "power_outlets": (PowerOutletTemplate, ["name"]),
+    "rear_ports": (RearPortTemplate, ["name"]),
+    "front_ports": (FrontPortTemplate, ["name"]),
+    "device_bays": (DeviceBayTemplate, ["name"]),
+}
 
 for params in device_types:
     custom_field_data = pop_custom_fields(params)
+    components = [(v[0], v[1], params.pop(k, [])) for k, v in supported_components.items()]
 
     for assoc, details in required_assocs.items():
         model, field = details
@@ -41,3 +113,29 @@
         print("🔡 Created device type", device_type.manufacturer, device_type.model)
 
     set_custom_fields_values(device_type, custom_field_data)
+
+    for component in components:
+        c_model, c_match_params, c_params = component
+        c_match_params.append("device_type")
+
+        if not c_params:
+            continue
+
+        expanded_c_params = expand_templates(c_params, device_type)
+
+        for n_assoc, n_details in nested_assocs.items():
+            n_model, n_field = n_details
+            for c_param in expanded_c_params:
+                if n_assoc in c_param:
+                    n_query = {n_field: c_param[n_assoc], "device_type": device_type}
+                    c_param[n_assoc] = n_model.objects.get(**n_query)
+
+        for new_param in expanded_c_params:
+            new_matching_params, new_defaults = split_params(new_param, c_match_params)
+            new_obj, new_obj_created = c_model.objects.get_or_create(
+                **new_matching_params, defaults=new_defaults
+            )
+            if new_obj_created:
+                print(
+                    f"🧷  Created {c_model._meta} {new_obj} component for device type {device_type}"
+                )