From a311002141708ac6e1d987e7f35e467b9d9434e1 Mon Sep 17 00:00:00 2001 From: John Anderson Date: Sun, 9 Feb 2020 03:20:59 -0500 Subject: [PATCH 01/30] initial work on dynamic lookup expressions --- netbox/circuits/filters.py | 12 +++-- netbox/dcim/filters.py | 44 ++++++++--------- netbox/extras/filters.py | 13 ++--- netbox/tenancy/filters.py | 2 +- netbox/utilities/constants.py | 38 +++++++++++++++ netbox/utilities/filters.py | 91 +++++++++++++++++++++++++++++++++++ 6 files changed, 166 insertions(+), 34 deletions(-) diff --git a/netbox/circuits/filters.py b/netbox/circuits/filters.py index c27ffb8d7bd..b0c4cacbd78 100644 --- a/netbox/circuits/filters.py +++ b/netbox/circuits/filters.py @@ -4,7 +4,9 @@ from dcim.models import Region, Site from extras.filters import CustomFieldFilterSet, CreatedUpdatedFilterSet from tenancy.filters import TenancyFilterSet -from utilities.filters import NameSlugSearchFilterSet, NumericInFilter, TagFilter, TreeNodeMultipleChoiceFilter +from utilities.filters import ( + BaseFilterSet, NameSlugSearchFilterSet, NumericInFilter, TagFilter, TreeNodeMultipleChoiceFilter +) from .choices import * from .models import Circuit, CircuitTermination, CircuitType, Provider @@ -16,7 +18,7 @@ ) -class ProviderFilterSet(CustomFieldFilterSet, CreatedUpdatedFilterSet): +class ProviderFilterSet(BaseFilterSet, CustomFieldFilterSet, CreatedUpdatedFilterSet): id__in = NumericInFilter( field_name='id', lookup_expr='in' @@ -65,14 +67,14 @@ def search(self, queryset, name, value): ) -class CircuitTypeFilterSet(NameSlugSearchFilterSet): +class CircuitTypeFilterSet(BaseFilterSet, NameSlugSearchFilterSet): class Meta: model = CircuitType fields = ['id', 'name', 'slug'] -class CircuitFilterSet(CustomFieldFilterSet, TenancyFilterSet, CreatedUpdatedFilterSet): +class CircuitFilterSet(BaseFilterSet, CustomFieldFilterSet, TenancyFilterSet, CreatedUpdatedFilterSet): id__in = NumericInFilter( field_name='id', lookup_expr='in' @@ -146,7 +148,7 @@ def search(self, queryset, name, value): ).distinct() -class CircuitTerminationFilterSet(django_filters.FilterSet): +class CircuitTerminationFilterSet(BaseFilterSet): q = django_filters.CharFilter( method='search', label='Search', diff --git a/netbox/dcim/filters.py b/netbox/dcim/filters.py index 7b278ca0e09..d9658a28e76 100644 --- a/netbox/dcim/filters.py +++ b/netbox/dcim/filters.py @@ -6,8 +6,8 @@ from tenancy.models import Tenant from utilities.constants import COLOR_CHOICES from utilities.filters import ( - MultiValueCharFilter, MultiValueMACAddressFilter, MultiValueNumberFilter, NameSlugSearchFilterSet, NumericInFilter, - TagFilter, TreeNodeMultipleChoiceFilter, + BaseFilterSet, MultiValueCharFilter, MultiValueMACAddressFilter, MultiValueNumberFilter, + BaseFilterSet, NameSlugSearchFilterSet, NumericInFilter, TagFilter, TreeNodeMultipleChoiceFilter, ) from virtualization.models import Cluster from .choices import * @@ -60,7 +60,7 @@ ) -class RegionFilterSet(NameSlugSearchFilterSet): +class RegionFilterSet(BaseFilterSet, NameSlugSearchFilterSet): parent_id = django_filters.ModelMultipleChoiceFilter( queryset=Region.objects.all(), label='Parent region (ID)', @@ -77,7 +77,7 @@ class Meta: fields = ['id', 'name', 'slug'] -class SiteFilterSet(TenancyFilterSet, CustomFieldFilterSet, CreatedUpdatedFilterSet): +class SiteFilterSet(BaseFilterSet, CustomFieldFilterSet, CreatedUpdatedFilterSet): id__in = NumericInFilter( field_name='id', lookup_expr='in' @@ -131,7 +131,7 @@ def search(self, queryset, name, value): return queryset.filter(qs_filter) -class RackGroupFilterSet(NameSlugSearchFilterSet): +class RackGroupFilterSet(BaseFilterSet, NameSlugSearchFilterSet): region_id = TreeNodeMultipleChoiceFilter( queryset=Region.objects.all(), field_name='site__region__in', @@ -159,14 +159,14 @@ class Meta: fields = ['id', 'name', 'slug'] -class RackRoleFilterSet(NameSlugSearchFilterSet): +class RackRoleFilterSet(BaseFilterSet, NameSlugSearchFilterSet): class Meta: model = RackRole fields = ['id', 'name', 'slug', 'color'] -class RackFilterSet(TenancyFilterSet, CustomFieldFilterSet, CreatedUpdatedFilterSet): +class RackFilterSet(BaseFilterSet, CustomFieldFilterSet, CreatedUpdatedFilterSet): id__in = NumericInFilter( field_name='id', lookup_expr='in' @@ -244,7 +244,7 @@ def search(self, queryset, name, value): ) -class RackReservationFilterSet(TenancyFilterSet): +class RackReservationFilterSet(BaseFilterSet, TenancyFilterSet): id__in = NumericInFilter( field_name='id', lookup_expr='in' @@ -305,14 +305,14 @@ def search(self, queryset, name, value): ) -class ManufacturerFilterSet(NameSlugSearchFilterSet): +class ManufacturerFilterSet(BaseFilterSet, NameSlugSearchFilterSet): class Meta: model = Manufacturer fields = ['id', 'name', 'slug'] -class DeviceTypeFilterSet(CustomFieldFilterSet, CreatedUpdatedFilterSet): +class DeviceTypeFilterSet(BaseFilterSet, CustomFieldFilterSet, CreatedUpdatedFilterSet): id__in = NumericInFilter( field_name='id', lookup_expr='in' @@ -402,7 +402,7 @@ def _device_bays(self, queryset, name, value): return queryset.exclude(device_bay_templates__isnull=value) -class DeviceTypeComponentFilterSet(NameSlugSearchFilterSet): +class DeviceTypeComponentFilterSet(BaseFilterSet, NameSlugSearchFilterSet): devicetype_id = django_filters.ModelMultipleChoiceFilter( queryset=DeviceType.objects.all(), field_name='device_type_id', @@ -466,14 +466,14 @@ class Meta: fields = ['id', 'name'] -class DeviceRoleFilterSet(NameSlugSearchFilterSet): +class DeviceRoleFilterSet(BaseFilterSet, NameSlugSearchFilterSet): class Meta: model = DeviceRole fields = ['id', 'name', 'slug', 'color', 'vm_role'] -class PlatformFilterSet(NameSlugSearchFilterSet): +class PlatformFilterSet(BaseFilterSet, NameSlugSearchFilterSet): manufacturer_id = django_filters.ModelMultipleChoiceFilter( field_name='manufacturer', queryset=Manufacturer.objects.all(), @@ -491,7 +491,7 @@ class Meta: fields = ['id', 'name', 'slug', 'napalm_driver'] -class DeviceFilterSet(LocalConfigContextFilterSet, TenancyFilterSet, CustomFieldFilterSet, CreatedUpdatedFilterSet): +class DeviceFilterSet(BaseFilterSet, LocalConfigContextFilterSet, CustomFieldFilterSet, CreatedUpdatedFilterSet): id__in = NumericInFilter( field_name='id', lookup_expr='in' @@ -690,7 +690,7 @@ def _device_bays(self, queryset, name, value): return queryset.exclude(device_bays__isnull=value) -class DeviceComponentFilterSet(django_filters.FilterSet): +class DeviceComponentFilterSet(BaseFilterSet): q = django_filters.CharFilter( method='search', label='Search', @@ -1002,7 +1002,7 @@ def search(self, queryset, name, value): return queryset.filter(qs_filter) -class VirtualChassisFilterSet(django_filters.FilterSet): +class VirtualChassisFilterSet(BaseFilterSet): q = django_filters.CharFilter( method='search', label='Search', @@ -1056,7 +1056,7 @@ def search(self, queryset, name, value): return queryset.filter(qs_filter) -class CableFilterSet(django_filters.FilterSet): +class CableFilterSet(BaseFilterSet): q = django_filters.CharFilter( method='search', label='Search', @@ -1119,7 +1119,7 @@ def filter_device(self, queryset, name, value): return queryset -class ConsoleConnectionFilterSet(django_filters.FilterSet): +class ConsoleConnectionFilterSet(BaseFilterSet): site = django_filters.CharFilter( method='filter_site', label='Site (slug)', @@ -1150,7 +1150,7 @@ def filter_device(self, queryset, name, value): ) -class PowerConnectionFilterSet(django_filters.FilterSet): +class PowerConnectionFilterSet(BaseFilterSet): site = django_filters.CharFilter( method='filter_site', label='Site (slug)', @@ -1181,7 +1181,7 @@ def filter_device(self, queryset, name, value): ) -class InterfaceConnectionFilterSet(django_filters.FilterSet): +class InterfaceConnectionFilterSet(BaseFilterSet): site = django_filters.CharFilter( method='filter_site', label='Site (slug)', @@ -1215,7 +1215,7 @@ def filter_device(self, queryset, name, value): ) -class PowerPanelFilterSet(django_filters.FilterSet): +class PowerPanelFilterSet(BaseFilterSet): id__in = NumericInFilter( field_name='id', lookup_expr='in' @@ -1264,7 +1264,7 @@ def search(self, queryset, name, value): return queryset.filter(qs_filter) -class PowerFeedFilterSet(CustomFieldFilterSet, CreatedUpdatedFilterSet): +class PowerFeedFilterSet(BaseFilterSet, CustomFieldFilterSet, CreatedUpdatedFilterSet): id__in = NumericInFilter( field_name='id', lookup_expr='in' diff --git a/netbox/extras/filters.py b/netbox/extras/filters.py index dcd4f3edef0..ccd723a0c94 100644 --- a/netbox/extras/filters.py +++ b/netbox/extras/filters.py @@ -4,6 +4,7 @@ from dcim.models import DeviceRole, Platform, Region, Site from tenancy.models import Tenant, TenantGroup +from utilities.filters import BaseFilterSet from virtualization.models import Cluster, ClusterGroup from .choices import * from .models import ConfigContext, CustomField, Graph, ExportTemplate, ObjectChange, Tag @@ -89,21 +90,21 @@ def __init__(self, *args, **kwargs): self.filters['cf_{}'.format(cf.name)] = CustomFieldFilter(field_name=cf.name, custom_field=cf) -class GraphFilterSet(django_filters.FilterSet): +class GraphFilterSet(BaseFilterSet): class Meta: model = Graph fields = ['type', 'name', 'template_language'] -class ExportTemplateFilterSet(django_filters.FilterSet): +class ExportTemplateFilterSet(BaseFilterSet): class Meta: model = ExportTemplate fields = ['content_type', 'name', 'template_language'] -class TagFilterSet(django_filters.FilterSet): +class TagFilterSet(BaseFilterSet): q = django_filters.CharFilter( method='search', label='Search', @@ -122,7 +123,7 @@ def search(self, queryset, name, value): ) -class ConfigContextFilterSet(django_filters.FilterSet): +class ConfigContextFilterSet(BaseFilterSet): q = django_filters.CharFilter( method='search', label='Search', @@ -234,7 +235,7 @@ def search(self, queryset, name, value): # Filter for Local Config Context Data # -class LocalConfigContextFilterSet(django_filters.FilterSet): +class LocalConfigContextFilterSet(BaseFilterSet): local_context_data = django_filters.BooleanFilter( method='_local_context_data', label='Has local config context data', @@ -244,7 +245,7 @@ def _local_context_data(self, queryset, name, value): return queryset.exclude(local_context_data__isnull=value) -class ObjectChangeFilterSet(django_filters.FilterSet): +class ObjectChangeFilterSet(BaseFilterSet): q = django_filters.CharFilter( method='search', label='Search', diff --git a/netbox/tenancy/filters.py b/netbox/tenancy/filters.py index 001cf29e7be..fa40e398696 100644 --- a/netbox/tenancy/filters.py +++ b/netbox/tenancy/filters.py @@ -2,7 +2,7 @@ from django.db.models import Q from extras.filters import CustomFieldFilterSet, CreatedUpdatedFilterSet -from utilities.filters import NameSlugSearchFilterSet, NumericInFilter, TagFilter +from utilities.filters import BaseFilterSet, NameSlugSearchFilterSet, NumericInFilter, TagFilter from .models import Tenant, TenantGroup diff --git a/netbox/utilities/constants.py b/netbox/utilities/constants.py index ad6e8fd90b6..d7f819e8c78 100644 --- a/netbox/utilities/constants.py +++ b/netbox/utilities/constants.py @@ -27,3 +27,41 @@ ('111111', 'Black'), ('ffffff', 'White'), ) + + +# +# Filter lookup expressions +# + +FILTER_CHAR_BASED_LOOKUP_MAP = dict( + n='exact', + ic='icontains', + nic='icontains', + iew='iendswith', + niew='iendswith', + isw='istartswith', + nisw='istartswith', + ie='iexact', + nie='iexact' +) + +FILTER_NUMERIC_BASED_LOOKUP_MAP = dict( + n='exact', + lte='lte', + lt='lt', + gte='gte', + gt='gt' +) + +FILTER_LOOKUP_HELP_TEXT_MAP = dict( + icontains='case insensitive contains', + iendswith='case insensitive ends with', + istartswith='case insensitive starts with', + iexact='case insensitive exact', + exact='case sensitive exact', + lt='less than', + lte='less than or equal', + gt='greater than', + gte='greater than or equal', + n='negated' +) diff --git a/netbox/utilities/filters.py b/netbox/utilities/filters.py index 957020e4074..edb7bafa0bd 100644 --- a/netbox/utilities/filters.py +++ b/netbox/utilities/filters.py @@ -3,7 +3,12 @@ from django import forms from django.conf import settings from django.db import models +from django_filters.utils import get_model_field + from extras.models import Tag +from utilities.constants import ( + FILTER_CHAR_BASED_LOOKUP_MAP, FILTER_LOOKUP_HELP_TEXT_MAP, FILTER_NUMERIC_BASED_LOOKUP_MAP +) def multivalue_field_factory(field_class): @@ -111,6 +116,92 @@ def __init__(self, *args, **kwargs): # FilterSets # +class BaseFilterSet(django_filters.FilterSet): + """ + A base filterset which provides common functionaly to all NetBox filtersets + """ + @classmethod + def get_filters(cls): + """ + Override filter generation to support dynamic lookup expressions for certain filter types. + + For specific filter types, new filters are created based on defined lookup expressions in + the form `_` + """ + filters = super().get_filters() + + new_filters = {} + for existing_filter_name, existing_filter in filters.items(): + # Loop over existing filters to extract metadata by which to create new filters + + # It the filter makes use of a custom filter method or lookup expression skip it + # as we cannot sanely handle these cases in a generic mannor + if existing_filter.method is not None or existing_filter.lookup_expr != 'exact': + continue + + # Choose the lookup expression map based on the filter type + if isinstance(existing_filter, ( + django_filters.filters.CharFilter, + django_filters.MultipleChoiceFilter, + MultiValueCharFilter, + MultiValueMACAddressFilter, + TagFilter + )): + lookup_map = FILTER_CHAR_BASED_LOOKUP_MAP + + elif isinstance(existing_filter, ( + MultiValueDateFilter, + MultiValueDateTimeFilter, + MultiValueNumberFilter, + MultiValueTimeFilter + )): + lookup_map = FILTER_NUMERIC_BASED_LOOKUP_MAP + + elif isinstance(existing_filter, ( + django_filters.ModelChoiceFilter, + django_filters.ModelMultipleChoiceFilter, + NumericInFilter, + TreeNodeMultipleChoiceFilter, + )): + # These filter types support only negation + lookup_map = dict( + n='exact' + ) + + else: + # Do no augment any other filter types with more lookup expressions + continue + + # Get properties of the existing filter for later use + field_name = existing_filter.field_name + field = get_model_field(cls._meta.model, field_name) + + # Create new filters for each lookup expression in the map + for lookup_name, lookup_expr in lookup_map.items(): + new_filter_name = '{}_{}'.format(existing_filter_name, lookup_name) + + try: + print(existing_filter_name) + new_filter = cls.filter_for_field(field, field_name, lookup_expr) + except django_filters.exceptions.FieldLookupError: + # The filter could not be created because the lookup expression is not supported + continue + + help_text = FILTER_LOOKUP_HELP_TEXT_MAP[lookup_expr] + + if lookup_name.startswith('n'): + # This is a negation filter which requires a queryselt.exclud() clause + new_filter.exclude = True + help_text = 'negated {}'.format(help_text) + + new_filter.extra = existing_filter.extra + new_filter.extra['help_text'] = '{} - {}'.format(field_name, help_text) + new_filters[new_filter_name] = new_filter + + filters.update(new_filters) + return filters + + class NameSlugSearchFilterSet(django_filters.FilterSet): """ A base class for adding the search method to models which only expose the `name` and `slug` fields From a6b43b30e9f59f30a3365fb6a0e80a6edb4954af Mon Sep 17 00:00:00 2001 From: John Anderson Date: Sun, 9 Feb 2020 17:46:21 -0500 Subject: [PATCH 02/30] functional dynamic filter lookups --- netbox/circuits/filters.py | 12 ++- netbox/dcim/filters.py | 94 +++++++++++-------- netbox/extras/filters.py | 2 +- netbox/extras/tests/test_filters.py | 12 +-- netbox/ipam/filters.py | 39 ++++---- netbox/secrets/filters.py | 6 +- netbox/tenancy/filters.py | 4 +- netbox/utilities/filters.py | 134 ++++++++++++++-------------- netbox/virtualization/filters.py | 24 +++-- 9 files changed, 183 insertions(+), 144 deletions(-) diff --git a/netbox/circuits/filters.py b/netbox/circuits/filters.py index b0c4cacbd78..4bd5fa15847 100644 --- a/netbox/circuits/filters.py +++ b/netbox/circuits/filters.py @@ -29,12 +29,14 @@ class ProviderFilterSet(BaseFilterSet, CustomFieldFilterSet, CreatedUpdatedFilte ) region_id = TreeNodeMultipleChoiceFilter( queryset=Region.objects.all(), - field_name='circuits__terminations__site__region__in', + field_name='circuits__terminations__site__region', + lookup_expr='in', label='Region (ID)', ) region = TreeNodeMultipleChoiceFilter( queryset=Region.objects.all(), - field_name='circuits__terminations__site__region__in', + field_name='circuits__terminations__site__region', + lookup_expr='in', to_field_name='slug', label='Region (slug)', ) @@ -120,12 +122,14 @@ class CircuitFilterSet(BaseFilterSet, CustomFieldFilterSet, TenancyFilterSet, Cr ) region_id = TreeNodeMultipleChoiceFilter( queryset=Region.objects.all(), - field_name='terminations__site__region__in', + field_name='terminations__site__region', + lookup_expr='in', label='Region (ID)', ) region = TreeNodeMultipleChoiceFilter( queryset=Region.objects.all(), - field_name='terminations__site__region__in', + field_name='terminations__site__region', + lookup_expr='in', to_field_name='slug', label='Region (slug)', ) diff --git a/netbox/dcim/filters.py b/netbox/dcim/filters.py index d9658a28e76..0d7c440c264 100644 --- a/netbox/dcim/filters.py +++ b/netbox/dcim/filters.py @@ -7,7 +7,7 @@ from utilities.constants import COLOR_CHOICES from utilities.filters import ( BaseFilterSet, MultiValueCharFilter, MultiValueMACAddressFilter, MultiValueNumberFilter, - BaseFilterSet, NameSlugSearchFilterSet, NumericInFilter, TagFilter, TreeNodeMultipleChoiceFilter, + NameSlugSearchFilterSet, NumericInFilter, TagFilter, TreeNodeMultipleChoiceFilter, ) from virtualization.models import Cluster from .choices import * @@ -92,12 +92,14 @@ class SiteFilterSet(BaseFilterSet, CustomFieldFilterSet, CreatedUpdatedFilterSet ) region_id = TreeNodeMultipleChoiceFilter( queryset=Region.objects.all(), - field_name='region__in', + field_name='region', + lookup_expr='in', label='Region (ID)', ) region = TreeNodeMultipleChoiceFilter( queryset=Region.objects.all(), - field_name='region__in', + field_name='region', + lookup_expr='in', to_field_name='slug', label='Region (slug)', ) @@ -134,12 +136,14 @@ def search(self, queryset, name, value): class RackGroupFilterSet(BaseFilterSet, NameSlugSearchFilterSet): region_id = TreeNodeMultipleChoiceFilter( queryset=Region.objects.all(), - field_name='site__region__in', + field_name='site__region', + lookup_expr='in', label='Region (ID)', ) region = TreeNodeMultipleChoiceFilter( queryset=Region.objects.all(), - field_name='site__region__in', + field_name='site__region', + lookup_expr='in', to_field_name='slug', label='Region (slug)', ) @@ -177,12 +181,14 @@ class RackFilterSet(BaseFilterSet, CustomFieldFilterSet, CreatedUpdatedFilterSet ) region_id = TreeNodeMultipleChoiceFilter( queryset=Region.objects.all(), - field_name='site__region__in', + field_name='site__region', + lookup_expr='in', label='Region (ID)', ) region = TreeNodeMultipleChoiceFilter( queryset=Region.objects.all(), - field_name='site__region__in', + field_name='site__region', + lookup_expr='in', to_field_name='slug', label='Region (slug)', ) @@ -402,7 +408,7 @@ def _device_bays(self, queryset, name, value): return queryset.exclude(device_bay_templates__isnull=value) -class DeviceTypeComponentFilterSet(BaseFilterSet, NameSlugSearchFilterSet): +class DeviceTypeComponentFilterSet(NameSlugSearchFilterSet): devicetype_id = django_filters.ModelMultipleChoiceFilter( queryset=DeviceType.objects.all(), field_name='device_type_id', @@ -410,56 +416,56 @@ class DeviceTypeComponentFilterSet(BaseFilterSet, NameSlugSearchFilterSet): ) -class ConsolePortTemplateFilterSet(DeviceTypeComponentFilterSet): +class ConsolePortTemplateFilterSet(BaseFilterSet, DeviceTypeComponentFilterSet): class Meta: model = ConsolePortTemplate fields = ['id', 'name', 'type'] -class ConsoleServerPortTemplateFilterSet(DeviceTypeComponentFilterSet): +class ConsoleServerPortTemplateFilterSet(BaseFilterSet, DeviceTypeComponentFilterSet): class Meta: model = ConsoleServerPortTemplate fields = ['id', 'name', 'type'] -class PowerPortTemplateFilterSet(DeviceTypeComponentFilterSet): +class PowerPortTemplateFilterSet(BaseFilterSet, DeviceTypeComponentFilterSet): class Meta: model = PowerPortTemplate fields = ['id', 'name', 'type', 'maximum_draw', 'allocated_draw'] -class PowerOutletTemplateFilterSet(DeviceTypeComponentFilterSet): +class PowerOutletTemplateFilterSet(BaseFilterSet, DeviceTypeComponentFilterSet): class Meta: model = PowerOutletTemplate fields = ['id', 'name', 'type', 'feed_leg'] -class InterfaceTemplateFilterSet(DeviceTypeComponentFilterSet): +class InterfaceTemplateFilterSet(BaseFilterSet, DeviceTypeComponentFilterSet): class Meta: model = InterfaceTemplate fields = ['id', 'name', 'type', 'mgmt_only'] -class FrontPortTemplateFilterSet(DeviceTypeComponentFilterSet): +class FrontPortTemplateFilterSet(BaseFilterSet, DeviceTypeComponentFilterSet): class Meta: model = FrontPortTemplate fields = ['id', 'name', 'type'] -class RearPortTemplateFilterSet(DeviceTypeComponentFilterSet): +class RearPortTemplateFilterSet(BaseFilterSet, DeviceTypeComponentFilterSet): class Meta: model = RearPortTemplate fields = ['id', 'name', 'type', 'positions'] -class DeviceBayTemplateFilterSet(DeviceTypeComponentFilterSet): +class DeviceBayTemplateFilterSet(BaseFilterSet, DeviceTypeComponentFilterSet): class Meta: model = DeviceBayTemplate @@ -538,12 +544,14 @@ class DeviceFilterSet(BaseFilterSet, LocalConfigContextFilterSet, CustomFieldFil ) region_id = TreeNodeMultipleChoiceFilter( queryset=Region.objects.all(), - field_name='site__region__in', + field_name='site__region', + lookup_expr='in', label='Region (ID)', ) region = TreeNodeMultipleChoiceFilter( queryset=Region.objects.all(), - field_name='site__region__in', + field_name='site__region', + lookup_expr='in', to_field_name='slug', label='Region (slug)', ) @@ -690,19 +698,21 @@ def _device_bays(self, queryset, name, value): return queryset.exclude(device_bays__isnull=value) -class DeviceComponentFilterSet(BaseFilterSet): +class DeviceComponentFilterSet(django_filters.FilterSet): q = django_filters.CharFilter( method='search', label='Search', ) region_id = TreeNodeMultipleChoiceFilter( queryset=Region.objects.all(), - field_name='device__site__region__in', + field_name='device__site__region', + lookup_expr='in', label='Region (ID)', ) region = TreeNodeMultipleChoiceFilter( queryset=Region.objects.all(), - field_name='device__site__region__in', + field_name='device__site__region', + lookup_expr='in', to_field_name='slug', label='Region (slug)', ) @@ -738,7 +748,7 @@ def search(self, queryset, name, value): ) -class ConsolePortFilterSet(DeviceComponentFilterSet): +class ConsolePortFilterSet(BaseFilterSet, DeviceComponentFilterSet): type = django_filters.MultipleChoiceFilter( choices=ConsolePortTypeChoices, null_value=None @@ -754,7 +764,7 @@ class Meta: fields = ['id', 'name', 'description', 'connection_status'] -class ConsoleServerPortFilterSet(DeviceComponentFilterSet): +class ConsoleServerPortFilterSet(BaseFilterSet, DeviceComponentFilterSet): type = django_filters.MultipleChoiceFilter( choices=ConsolePortTypeChoices, null_value=None @@ -770,7 +780,7 @@ class Meta: fields = ['id', 'name', 'description', 'connection_status'] -class PowerPortFilterSet(DeviceComponentFilterSet): +class PowerPortFilterSet(BaseFilterSet, DeviceComponentFilterSet): type = django_filters.MultipleChoiceFilter( choices=PowerPortTypeChoices, null_value=None @@ -786,7 +796,7 @@ class Meta: fields = ['id', 'name', 'maximum_draw', 'allocated_draw', 'description', 'connection_status'] -class PowerOutletFilterSet(DeviceComponentFilterSet): +class PowerOutletFilterSet(BaseFilterSet, DeviceComponentFilterSet): type = django_filters.MultipleChoiceFilter( choices=PowerOutletTypeChoices, null_value=None @@ -802,7 +812,7 @@ class Meta: fields = ['id', 'name', 'feed_leg', 'description', 'connection_status'] -class InterfaceFilterSet(DeviceComponentFilterSet): +class InterfaceFilterSet(BaseFilterSet, DeviceComponentFilterSet): q = django_filters.CharFilter( method='search', label='Search', @@ -900,7 +910,7 @@ def filter_kind(self, queryset, name, value): }.get(value, queryset.none()) -class FrontPortFilterSet(DeviceComponentFilterSet): +class FrontPortFilterSet(BaseFilterSet, DeviceComponentFilterSet): cabled = django_filters.BooleanFilter( field_name='cable', lookup_expr='isnull', @@ -912,7 +922,7 @@ class Meta: fields = ['id', 'name', 'type', 'description'] -class RearPortFilterSet(DeviceComponentFilterSet): +class RearPortFilterSet(BaseFilterSet, DeviceComponentFilterSet): cabled = django_filters.BooleanFilter( field_name='cable', lookup_expr='isnull', @@ -924,26 +934,28 @@ class Meta: fields = ['id', 'name', 'type', 'positions', 'description'] -class DeviceBayFilterSet(DeviceComponentFilterSet): +class DeviceBayFilterSet(BaseFilterSet, DeviceComponentFilterSet): class Meta: model = DeviceBay fields = ['id', 'name', 'description'] -class InventoryItemFilterSet(DeviceComponentFilterSet): +class InventoryItemFilterSet(BaseFilterSet, DeviceComponentFilterSet): q = django_filters.CharFilter( method='search', label='Search', ) region_id = TreeNodeMultipleChoiceFilter( queryset=Region.objects.all(), - field_name='device__site__region__in', + field_name='device__site__region', + lookup_expr='in', label='Region (ID)', ) region = TreeNodeMultipleChoiceFilter( queryset=Region.objects.all(), - field_name='device__site__region__in', + field_name='device__site__region', + lookup_expr='in', to_field_name='slug', label='Region (slug)', ) @@ -1009,12 +1021,14 @@ class VirtualChassisFilterSet(BaseFilterSet): ) region_id = TreeNodeMultipleChoiceFilter( queryset=Region.objects.all(), - field_name='master__site__region__in', + field_name='master__site__region', + lookup_expr='in', label='Region (ID)', ) region = TreeNodeMultipleChoiceFilter( queryset=Region.objects.all(), - field_name='master__site__region__in', + field_name='master__site__region', + lookup_expr='in', to_field_name='slug', label='Region (slug)', ) @@ -1226,12 +1240,14 @@ class PowerPanelFilterSet(BaseFilterSet): ) region_id = TreeNodeMultipleChoiceFilter( queryset=Region.objects.all(), - field_name='site__region__in', + field_name='site__region', + lookup_expr='in', label='Region (ID)', ) region = TreeNodeMultipleChoiceFilter( queryset=Region.objects.all(), - field_name='site__region__in', + field_name='site__region', + lookup_expr='in', to_field_name='slug', label='Region (slug)', ) @@ -1275,12 +1291,14 @@ class PowerFeedFilterSet(BaseFilterSet, CustomFieldFilterSet, CreatedUpdatedFilt ) region_id = TreeNodeMultipleChoiceFilter( queryset=Region.objects.all(), - field_name='power_panel__site__region__in', + field_name='power_panel__site__region', + lookup_expr='in', label='Region (ID)', ) region = TreeNodeMultipleChoiceFilter( queryset=Region.objects.all(), - field_name='power_panel__site__region__in', + field_name='power_panel__site__region', + lookup_expr='in', to_field_name='slug', label='Region (slug)', ) diff --git a/netbox/extras/filters.py b/netbox/extras/filters.py index ccd723a0c94..ad414a691f3 100644 --- a/netbox/extras/filters.py +++ b/netbox/extras/filters.py @@ -235,7 +235,7 @@ def search(self, queryset, name, value): # Filter for Local Config Context Data # -class LocalConfigContextFilterSet(BaseFilterSet): +class LocalConfigContextFilterSet(django_filters.FilterSet): local_context_data = django_filters.BooleanFilter( method='_local_context_data', label='Has local config context data', diff --git a/netbox/extras/tests/test_filters.py b/netbox/extras/tests/test_filters.py index 5ef96faa255..ab559cf73c9 100644 --- a/netbox/extras/tests/test_filters.py +++ b/netbox/extras/tests/test_filters.py @@ -28,8 +28,8 @@ def setUpTestData(cls): Graph.objects.bulk_create(graphs) def test_name(self): - params = {'name': 'Graph 1'} - self.assertEqual(self.filterset(params, self.queryset).qs.count(), 1) + params = {'name': ['Graph 1', 'Graph 2']} + self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2) def test_type(self): content_type = ContentType.objects.filter(GRAPH_MODELS).first() @@ -59,8 +59,8 @@ def setUpTestData(cls): ExportTemplate.objects.bulk_create(export_templates) def test_name(self): - params = {'name': 'Export Template 1'} - self.assertEqual(self.filterset(params, self.queryset).qs.count(), 1) + params = {'name': ['Export Template 1', 'Export Template 2']} + self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2) def test_content_type(self): params = {'content_type': ContentType.objects.get(model='site').pk} @@ -154,8 +154,8 @@ def setUpTestData(cls): c.tenants.set([tenants[i]]) def test_name(self): - params = {'name': 'Config Context 1'} - self.assertEqual(self.filterset(params, self.queryset).qs.count(), 1) + params = {'name': ['Config Context 1', 'Config Context 2']} + self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2) def test_is_active(self): params = {'is_active': True} diff --git a/netbox/ipam/filters.py b/netbox/ipam/filters.py index 67ad769cc5b..f0b3275f0ed 100644 --- a/netbox/ipam/filters.py +++ b/netbox/ipam/filters.py @@ -8,7 +8,8 @@ from extras.filters import CustomFieldFilterSet, CreatedUpdatedFilterSet from tenancy.filters import TenancyFilterSet from utilities.filters import ( - MultiValueCharFilter, NameSlugSearchFilterSet, NumericInFilter, TagFilter, TreeNodeMultipleChoiceFilter, + BaseFilterSet, MultiValueCharFilter, NameSlugSearchFilterSet, NumericInFilter, TagFilter, + TreeNodeMultipleChoiceFilter, ) from virtualization.models import VirtualMachine from .choices import * @@ -28,7 +29,7 @@ ) -class VRFFilterSet(TenancyFilterSet, CustomFieldFilterSet, CreatedUpdatedFilterSet): +class VRFFilterSet(BaseFilterSet, TenancyFilterSet, CustomFieldFilterSet, CreatedUpdatedFilterSet): id__in = NumericInFilter( field_name='id', lookup_expr='in' @@ -53,7 +54,7 @@ class Meta: fields = ['name', 'rd', 'enforce_unique'] -class RIRFilterSet(NameSlugSearchFilterSet): +class RIRFilterSet(BaseFilterSet, NameSlugSearchFilterSet): id__in = NumericInFilter( field_name='id', lookup_expr='in' @@ -64,7 +65,7 @@ class Meta: fields = ['name', 'slug', 'is_private'] -class AggregateFilterSet(CustomFieldFilterSet, CreatedUpdatedFilterSet): +class AggregateFilterSet(BaseFilterSet, CustomFieldFilterSet, CreatedUpdatedFilterSet): id__in = NumericInFilter( field_name='id', lookup_expr='in' @@ -114,7 +115,7 @@ def filter_prefix(self, queryset, name, value): return queryset.none() -class RoleFilterSet(NameSlugSearchFilterSet): +class RoleFilterSet(BaseFilterSet, NameSlugSearchFilterSet): q = django_filters.CharFilter( method='search', label='Search', @@ -125,7 +126,7 @@ class Meta: fields = ['id', 'name', 'slug'] -class PrefixFilterSet(TenancyFilterSet, CustomFieldFilterSet, CreatedUpdatedFilterSet): +class PrefixFilterSet(BaseFilterSet, TenancyFilterSet, CustomFieldFilterSet, CreatedUpdatedFilterSet): id__in = NumericInFilter( field_name='id', lookup_expr='in' @@ -166,12 +167,14 @@ class PrefixFilterSet(TenancyFilterSet, CustomFieldFilterSet, CreatedUpdatedFilt ) region_id = TreeNodeMultipleChoiceFilter( queryset=Region.objects.all(), - field_name='site__region__in', + field_name='site__region', + lookup_expr='in', label='Region (ID)', ) region = TreeNodeMultipleChoiceFilter( queryset=Region.objects.all(), - field_name='site__region__in', + field_name='site__region', + lookup_expr='in', to_field_name='slug', label='Region (slug)', ) @@ -273,7 +276,7 @@ def filter_mask_length(self, queryset, name, value): return queryset.filter(prefix__net_mask_length=value) -class IPAddressFilterSet(TenancyFilterSet, CustomFieldFilterSet, CreatedUpdatedFilterSet): +class IPAddressFilterSet(BaseFilterSet, TenancyFilterSet, CustomFieldFilterSet, CreatedUpdatedFilterSet): id__in = NumericInFilter( field_name='id', lookup_expr='in' @@ -395,15 +398,17 @@ def _assigned_to_interface(self, queryset, name, value): return queryset.exclude(interface__isnull=value) -class VLANGroupFilterSet(NameSlugSearchFilterSet): +class VLANGroupFilterSet(BaseFilterSet, NameSlugSearchFilterSet): region_id = TreeNodeMultipleChoiceFilter( queryset=Region.objects.all(), - field_name='site__region__in', + field_name='site__region', + lookup_expr='in', label='Region (ID)', ) region = TreeNodeMultipleChoiceFilter( queryset=Region.objects.all(), - field_name='site__region__in', + field_name='site__region', + lookup_expr='in', to_field_name='slug', label='Region (slug)', ) @@ -423,7 +428,7 @@ class Meta: fields = ['id', 'name', 'slug'] -class VLANFilterSet(TenancyFilterSet, CustomFieldFilterSet, CreatedUpdatedFilterSet): +class VLANFilterSet(BaseFilterSet, TenancyFilterSet, CustomFieldFilterSet, CreatedUpdatedFilterSet): id__in = NumericInFilter( field_name='id', lookup_expr='in' @@ -434,12 +439,14 @@ class VLANFilterSet(TenancyFilterSet, CustomFieldFilterSet, CreatedUpdatedFilter ) region_id = TreeNodeMultipleChoiceFilter( queryset=Region.objects.all(), - field_name='site__region__in', + field_name='site__region', + lookup_expr='in', label='Region (ID)', ) region = TreeNodeMultipleChoiceFilter( queryset=Region.objects.all(), - field_name='site__region__in', + field_name='site__region', + lookup_expr='in', to_field_name='slug', label='Region (slug)', ) @@ -494,7 +501,7 @@ def search(self, queryset, name, value): return queryset.filter(qs_filter) -class ServiceFilterSet(CreatedUpdatedFilterSet): +class ServiceFilterSet(BaseFilterSet, CreatedUpdatedFilterSet): q = django_filters.CharFilter( method='search', label='Search', diff --git a/netbox/secrets/filters.py b/netbox/secrets/filters.py index 0c2b01f4df9..f32ac1c557c 100644 --- a/netbox/secrets/filters.py +++ b/netbox/secrets/filters.py @@ -3,7 +3,7 @@ from dcim.models import Device from extras.filters import CustomFieldFilterSet, CreatedUpdatedFilterSet -from utilities.filters import NameSlugSearchFilterSet, NumericInFilter, TagFilter +from utilities.filters import BaseFilterSet, NameSlugSearchFilterSet, NumericInFilter, TagFilter from .models import Secret, SecretRole @@ -13,14 +13,14 @@ ) -class SecretRoleFilterSet(NameSlugSearchFilterSet): +class SecretRoleFilterSet(BaseFilterSet, NameSlugSearchFilterSet): class Meta: model = SecretRole fields = ['id', 'name', 'slug'] -class SecretFilterSet(CustomFieldFilterSet, CreatedUpdatedFilterSet): +class SecretFilterSet(BaseFilterSet, CustomFieldFilterSet, CreatedUpdatedFilterSet): id__in = NumericInFilter( field_name='id', lookup_expr='in' diff --git a/netbox/tenancy/filters.py b/netbox/tenancy/filters.py index fa40e398696..8ba3054aab2 100644 --- a/netbox/tenancy/filters.py +++ b/netbox/tenancy/filters.py @@ -13,14 +13,14 @@ ) -class TenantGroupFilterSet(NameSlugSearchFilterSet): +class TenantGroupFilterSet(BaseFilterSet, NameSlugSearchFilterSet): class Meta: model = TenantGroup fields = ['id', 'name', 'slug'] -class TenantFilterSet(CustomFieldFilterSet, CreatedUpdatedFilterSet): +class TenantFilterSet(BaseFilterSet, CustomFieldFilterSet, CreatedUpdatedFilterSet): id__in = NumericInFilter( field_name='id', lookup_expr='in' diff --git a/netbox/utilities/filters.py b/netbox/utilities/filters.py index edb7bafa0bd..9dacc55eb77 100644 --- a/netbox/utilities/filters.py +++ b/netbox/utilities/filters.py @@ -1,9 +1,10 @@ import django_filters +from copy import deepcopy from dcim.forms import MACAddressField from django import forms from django.conf import settings from django.db import models -from django_filters.utils import get_model_field +from django_filters.utils import get_model_field, resolve_field from extras.models import Tag from utilities.constants import ( @@ -120,13 +121,62 @@ class BaseFilterSet(django_filters.FilterSet): """ A base filterset which provides common functionaly to all NetBox filtersets """ + FILTER_DEFAULTS = deepcopy(django_filters.filterset.FILTER_FOR_DBFIELD_DEFAULTS) + FILTER_DEFAULTS.update({ + models.AutoField: { + 'filter_class': MultiValueNumberFilter + }, + models.CharField: { + 'filter_class': MultiValueCharFilter + }, + models.DateField: { + 'filter_class': MultiValueDateFilter + }, + models.DateTimeField: { + 'filter_class': MultiValueDateTimeFilter + }, + models.DecimalField: { + 'filter_class': MultiValueNumberFilter + }, + models.EmailField: { + 'filter_class': MultiValueCharFilter + }, + models.FloatField: { + 'filter_class': MultiValueNumberFilter + }, + models.IntegerField: { + 'filter_class': MultiValueNumberFilter + }, + models.PositiveIntegerField: { + 'filter_class': MultiValueNumberFilter + }, + models.PositiveSmallIntegerField: { + 'filter_class': MultiValueNumberFilter + }, + models.SlugField: { + 'filter_class': MultiValueCharFilter + }, + models.SmallIntegerField: { + 'filter_class': MultiValueNumberFilter + }, + models.TimeField: { + 'filter_class': MultiValueTimeFilter + }, + models.URLField: { + 'filter_class': MultiValueCharFilter + }, + MACAddressField: { + 'filter_class': MultiValueMACAddressFilter + }, + }) + @classmethod def get_filters(cls): """ Override filter generation to support dynamic lookup expressions for certain filter types. For specific filter types, new filters are created based on defined lookup expressions in - the form `_` + the form `__` """ filters = super().get_filters() @@ -136,7 +186,7 @@ def get_filters(cls): # It the filter makes use of a custom filter method or lookup expression skip it # as we cannot sanely handle these cases in a generic mannor - if existing_filter.method is not None or existing_filter.lookup_expr != 'exact': + if existing_filter.method is not None or existing_filter.lookup_expr not in ['exact', 'in']: continue # Choose the lookup expression map based on the filter type @@ -169,7 +219,7 @@ def get_filters(cls): ) else: - # Do no augment any other filter types with more lookup expressions + # Do not augment any other filter types with more lookup expressions continue # Get properties of the existing filter for later use @@ -178,24 +228,29 @@ def get_filters(cls): # Create new filters for each lookup expression in the map for lookup_name, lookup_expr in lookup_map.items(): - new_filter_name = '{}_{}'.format(existing_filter_name, lookup_name) - + new_filter_name = '{}__{}'.format(existing_filter_name, lookup_name) + try: - print(existing_filter_name) - new_filter = cls.filter_for_field(field, field_name, lookup_expr) + if existing_filter_name in cls.declared_filters: + resolve_field(field, lookup_expr) # Will raise FieldLookupError if the lookup is invalid + new_filter = type(existing_filter)( + field_name=field_name, + lookup_expr=lookup_expr, + label=existing_filter.label, + exclude=existing_filter.exclude, + distinct=existing_filter.distinct, + **existing_filter.extra + ) + else: + new_filter = cls.filter_for_field(field, field_name, lookup_expr) except django_filters.exceptions.FieldLookupError: - # The filter could not be created because the lookup expression is not supported + # The filter could not be created because the lookup expression is not supported on the field continue - help_text = FILTER_LOOKUP_HELP_TEXT_MAP[lookup_expr] - if lookup_name.startswith('n'): # This is a negation filter which requires a queryselt.exclud() clause new_filter.exclude = True - help_text = 'negated {}'.format(help_text) - new_filter.extra = existing_filter.extra - new_filter.extra['help_text'] = '{} - {}'.format(field_name, help_text) new_filters[new_filter_name] = new_filter filters.update(new_filters) @@ -218,54 +273,3 @@ def search(self, queryset, name, value): models.Q(name__icontains=value) | models.Q(slug__icontains=value) ) - - -# -# Update default filters -# - -FILTER_DEFAULTS = django_filters.filterset.FILTER_FOR_DBFIELD_DEFAULTS -FILTER_DEFAULTS.update({ - models.AutoField: { - 'filter_class': MultiValueNumberFilter - }, - models.CharField: { - 'filter_class': MultiValueCharFilter - }, - models.DateField: { - 'filter_class': MultiValueDateFilter - }, - models.DateTimeField: { - 'filter_class': MultiValueDateTimeFilter - }, - models.DecimalField: { - 'filter_class': MultiValueNumberFilter - }, - models.EmailField: { - 'filter_class': MultiValueCharFilter - }, - models.FloatField: { - 'filter_class': MultiValueNumberFilter - }, - models.IntegerField: { - 'filter_class': MultiValueNumberFilter - }, - models.PositiveIntegerField: { - 'filter_class': MultiValueNumberFilter - }, - models.PositiveSmallIntegerField: { - 'filter_class': MultiValueNumberFilter - }, - models.SlugField: { - 'filter_class': MultiValueCharFilter - }, - models.SmallIntegerField: { - 'filter_class': MultiValueNumberFilter - }, - models.TimeField: { - 'filter_class': MultiValueTimeFilter - }, - models.URLField: { - 'filter_class': MultiValueCharFilter - }, -}) diff --git a/netbox/virtualization/filters.py b/netbox/virtualization/filters.py index 0d817ef47de..d5af9e3d4d0 100644 --- a/netbox/virtualization/filters.py +++ b/netbox/virtualization/filters.py @@ -6,7 +6,8 @@ from tenancy.filters import TenancyFilterSet from tenancy.models import Tenant from utilities.filters import ( - MultiValueMACAddressFilter, NameSlugSearchFilterSet, NumericInFilter, TagFilter, TreeNodeMultipleChoiceFilter, + BaseFilterSet, MultiValueMACAddressFilter, NameSlugSearchFilterSet, NumericInFilter, TagFilter, + TreeNodeMultipleChoiceFilter, ) from .choices import * from .models import Cluster, ClusterGroup, ClusterType, VirtualMachine @@ -20,21 +21,21 @@ ) -class ClusterTypeFilterSet(NameSlugSearchFilterSet): +class ClusterTypeFilterSet(BaseFilterSet, NameSlugSearchFilterSet): class Meta: model = ClusterType fields = ['id', 'name', 'slug'] -class ClusterGroupFilterSet(NameSlugSearchFilterSet): +class ClusterGroupFilterSet(BaseFilterSet, NameSlugSearchFilterSet): class Meta: model = ClusterGroup fields = ['id', 'name', 'slug'] -class ClusterFilterSet(CustomFieldFilterSet, CreatedUpdatedFilterSet): +class ClusterFilterSet(BaseFilterSet, CustomFieldFilterSet, CreatedUpdatedFilterSet): id__in = NumericInFilter( field_name='id', lookup_expr='in' @@ -45,12 +46,14 @@ class ClusterFilterSet(CustomFieldFilterSet, CreatedUpdatedFilterSet): ) region_id = TreeNodeMultipleChoiceFilter( queryset=Region.objects.all(), - field_name='site__region__in', + field_name='site__region', + lookup_expr='in', label='Region (ID)', ) region = TreeNodeMultipleChoiceFilter( queryset=Region.objects.all(), - field_name='site__region__in', + field_name='site__region', + lookup_expr='in', to_field_name='slug', label='Region (slug)', ) @@ -104,6 +107,7 @@ def search(self, queryset, name, value): class VirtualMachineFilterSet( + BaseFilterSet, LocalConfigContextFilterSet, TenancyFilterSet, CustomFieldFilterSet, @@ -149,12 +153,14 @@ class VirtualMachineFilterSet( ) region_id = TreeNodeMultipleChoiceFilter( queryset=Region.objects.all(), - field_name='cluster__site__region__in', + field_name='cluster__site__region', + lookup_expr='in', label='Region (ID)', ) region = TreeNodeMultipleChoiceFilter( queryset=Region.objects.all(), - field_name='cluster__site__region__in', + field_name='cluster__site__region', + lookup_expr='in', to_field_name='slug', label='Region (slug)', ) @@ -208,7 +214,7 @@ def search(self, queryset, name, value): ) -class InterfaceFilterSet(django_filters.FilterSet): +class InterfaceFilterSet(BaseFilterSet): q = django_filters.CharFilter( method='search', label='Search', From 9284e8327025a1df129768f46da4945d468a0ff5 Mon Sep 17 00:00:00 2001 From: John Anderson Date: Sun, 9 Feb 2020 21:32:45 -0500 Subject: [PATCH 03/30] py3.5 compatibility --- netbox/utilities/filters.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/netbox/utilities/filters.py b/netbox/utilities/filters.py index 9dacc55eb77..d1364cf4b86 100644 --- a/netbox/utilities/filters.py +++ b/netbox/utilities/filters.py @@ -178,13 +178,15 @@ def get_filters(cls): For specific filter types, new filters are created based on defined lookup expressions in the form `__` """ - filters = super().get_filters() + # TODO: once 3.6 is the minimum required version of python, change this to a bare super() call + # We have to do it this way in py3.5 becuase of django_filters.FilterSet's use of a metaclass + filters = super(django_filters.FilterSet, cls).get_filters() new_filters = {} for existing_filter_name, existing_filter in filters.items(): # Loop over existing filters to extract metadata by which to create new filters - # It the filter makes use of a custom filter method or lookup expression skip it + # If the filter makes use of a custom filter method or lookup expression skip it # as we cannot sanely handle these cases in a generic mannor if existing_filter.method is not None or existing_filter.lookup_expr not in ['exact', 'in']: continue @@ -232,6 +234,9 @@ def get_filters(cls): try: if existing_filter_name in cls.declared_filters: + # The filter field has been explicity defined on the filterset class so we must manually + # create the new filter with the same type because there is no guarantee the defined type + # is the same as the default type for the field resolve_field(field, lookup_expr) # Will raise FieldLookupError if the lookup is invalid new_filter = type(existing_filter)( field_name=field_name, @@ -242,13 +247,14 @@ def get_filters(cls): **existing_filter.extra ) else: + # The filter field is listed in Meta.fields so we can safely rely on default behaviour new_filter = cls.filter_for_field(field, field_name, lookup_expr) except django_filters.exceptions.FieldLookupError: # The filter could not be created because the lookup expression is not supported on the field continue if lookup_name.startswith('n'): - # This is a negation filter which requires a queryselt.exclud() clause + # This is a negation filter which requires a queryset.exclude() clause new_filter.exclude = True new_filters[new_filter_name] = new_filter From a136a0788c649d5947202bf0780f004bfb5a5232 Mon Sep 17 00:00:00 2001 From: John Anderson Date: Tue, 18 Feb 2020 00:32:58 -0500 Subject: [PATCH 04/30] #4121 - dynamic filter lookup expressions --- netbox/utilities/constants.py | 4 ++ netbox/utilities/filters.py | 33 ++++++------ netbox/utilities/tests/test_filters.py | 74 +++++++++++++++++++++++++- 3 files changed, 94 insertions(+), 17 deletions(-) diff --git a/netbox/utilities/constants.py b/netbox/utilities/constants.py index d7f819e8c78..1f8e135534d 100644 --- a/netbox/utilities/constants.py +++ b/netbox/utilities/constants.py @@ -53,6 +53,10 @@ gt='gt' ) +FILTER_NEGATION_LOOKUP_MAP = dict( + n='exact' +) + FILTER_LOOKUP_HELP_TEXT_MAP = dict( icontains='case insensitive contains', iendswith='case insensitive ends with', diff --git a/netbox/utilities/filters.py b/netbox/utilities/filters.py index d1364cf4b86..5e9e1f4c1b2 100644 --- a/netbox/utilities/filters.py +++ b/netbox/utilities/filters.py @@ -8,7 +8,8 @@ from extras.models import Tag from utilities.constants import ( - FILTER_CHAR_BASED_LOOKUP_MAP, FILTER_LOOKUP_HELP_TEXT_MAP, FILTER_NUMERIC_BASED_LOOKUP_MAP + FILTER_CHAR_BASED_LOOKUP_MAP, FILTER_LOOKUP_HELP_TEXT_MAP, FILTER_NEGATION_LOOKUP_MAP, + FILTER_NUMERIC_BASED_LOOKUP_MAP ) @@ -193,15 +194,6 @@ def get_filters(cls): # Choose the lookup expression map based on the filter type if isinstance(existing_filter, ( - django_filters.filters.CharFilter, - django_filters.MultipleChoiceFilter, - MultiValueCharFilter, - MultiValueMACAddressFilter, - TagFilter - )): - lookup_map = FILTER_CHAR_BASED_LOOKUP_MAP - - elif isinstance(existing_filter, ( MultiValueDateFilter, MultiValueDateTimeFilter, MultiValueNumberFilter, @@ -212,13 +204,19 @@ def get_filters(cls): elif isinstance(existing_filter, ( django_filters.ModelChoiceFilter, django_filters.ModelMultipleChoiceFilter, - NumericInFilter, TreeNodeMultipleChoiceFilter, - )): + TagFilter + )) or existing_filter.extra.get('choices'): # These filter types support only negation - lookup_map = dict( - n='exact' - ) + lookup_map = FILTER_NEGATION_LOOKUP_MAP + + elif isinstance(existing_filter, ( + django_filters.filters.CharFilter, + django_filters.MultipleChoiceFilter, + MultiValueCharFilter, + MultiValueMACAddressFilter + )): + lookup_map = FILTER_CHAR_BASED_LOOKUP_MAP else: # Do not augment any other filter types with more lookup expressions @@ -231,6 +229,8 @@ def get_filters(cls): # Create new filters for each lookup expression in the map for lookup_name, lookup_expr in lookup_map.items(): new_filter_name = '{}__{}'.format(existing_filter_name, lookup_name) + if existing_filter.lookup_expr == 'in': + lookup_expr = 'in' # 'in' lookups must remain to avoid unwanted slicing on certain querysets try: if existing_filter_name in cls.declared_filters: @@ -255,7 +255,8 @@ def get_filters(cls): if lookup_name.startswith('n'): # This is a negation filter which requires a queryset.exclude() clause - new_filter.exclude = True + # Of course setting the negation of the existing filter's exclude attribute handles both cases + new_filter.exclude = not existing_filter.exclude new_filters[new_filter_name] = new_filter diff --git a/netbox/utilities/tests/test_filters.py b/netbox/utilities/tests/test_filters.py index 513e11bcaa8..a1cb771a142 100644 --- a/netbox/utilities/tests/test_filters.py +++ b/netbox/utilities/tests/test_filters.py @@ -2,8 +2,9 @@ from django.test import TestCase import django_filters +from dcim.filters import SiteFilterSet from dcim.models import Region, Site -from utilities.filters import TreeNodeMultipleChoiceFilter +from utilities.filters import BaseFilterSet, TreeNodeMultipleChoiceFilter class TreeNodeMultipleChoiceFilterTest(TestCase): @@ -60,3 +61,74 @@ def test_filter_combined(self): self.assertEqual(qs.count(), 2) self.assertEqual(qs[0], self.site1) self.assertEqual(qs[1], self.site3) + + +class DynamicFilterLookupExpressionTest(TestCase): + """ + These tests ensure of the utilities.filters.BaseFilterSet.get_filters() method + correctly generates dynamic filter expressions + """ + + def setUp(self): + + super().setUp() + + self.region1 = Region.objects.create(name='Test Region 1', slug='test-region-1') + self.region2 = Region.objects.create(name='Test Region 2', slug='test-region-2') + self.site1 = Site.objects.create(region=self.region1, name='Test Site 1', slug='ABC-test-site1-ABC', asn=65001) + self.site2 = Site.objects.create(region=self.region2, name='Test Site 2', slug='def-test-site2-def', asn=65101) + self.site3 = Site.objects.create(region=None, name='Test Site 3', slug='ghi-test-site3-ghi', asn=65201) + + self.queryset = Site.objects.all() + + def test_site_name_negation(self): + params = {'name__n': ['Test Site 1']} + self.assertEqual(SiteFilterSet(params, self.queryset).qs.count(), 2) + + def test_site_slug_icontains(self): + params = {'slug__ic': ['abc']} + self.assertEqual(SiteFilterSet(params, self.queryset).qs.count(), 1) + + def test_site_slug_icontains_negation(self): + params = {'slug__nic': ['abc']} + self.assertEqual(SiteFilterSet(params, self.queryset).qs.count(), 2) + + def test_site_slug_startswith(self): + params = {'slug__isw': ['abc']} + self.assertEqual(SiteFilterSet(params, self.queryset).qs.count(), 1) + + def test_site_slug_startswith_negation(self): + params = {'slug__nisw': ['abc']} + self.assertEqual(SiteFilterSet(params, self.queryset).qs.count(), 2) + + def test_site_slug_endswith(self): + params = {'slug__iew': ['abc']} + self.assertEqual(SiteFilterSet(params, self.queryset).qs.count(), 1) + + def test_site_slug_endswith_negation(self): + params = {'slug__niew': ['abc']} + self.assertEqual(SiteFilterSet(params, self.queryset).qs.count(), 2) + + def test_site_asn_lt(self): + params = {'asn__lt': [65101]} + self.assertEqual(SiteFilterSet(params, self.queryset).qs.count(), 1) + + def test_site_asn_lte(self): + params = {'asn__lte': [65101]} + self.assertEqual(SiteFilterSet(params, self.queryset).qs.count(), 2) + + def test_site_asn_gt(self): + params = {'asn__lt': [65101]} + self.assertEqual(SiteFilterSet(params, self.queryset).qs.count(), 1) + + def test_site_asn_gte(self): + params = {'asn__gte': [65101]} + self.assertEqual(SiteFilterSet(params, self.queryset).qs.count(), 2) + + def test_site_region_negation(self): + params = {'region__n': ['test-region-1']} + self.assertEqual(SiteFilterSet(params, self.queryset).qs.count(), 2) + + def test_site_region_id_negation(self): + params = {'region_id__n': [self.region1.pk]} + self.assertEqual(SiteFilterSet(params, self.queryset).qs.count(), 2) From 67565ca191a1976b6a4938b2bbfde578997b6264 Mon Sep 17 00:00:00 2001 From: John Anderson Date: Mon, 24 Feb 2020 15:03:07 -0500 Subject: [PATCH 05/30] added docs and more tests --- docs/api/filtering.md | 71 ++++++++++ docs/api/overview.md | 2 + mkdocs.yml | 1 + netbox/utilities/constants.py | 13 -- netbox/utilities/filters.py | 3 +- netbox/utilities/tests/test_filters.py | 186 +++++++++++++++++++++---- 6 files changed, 231 insertions(+), 45 deletions(-) create mode 100644 docs/api/filtering.md diff --git a/docs/api/filtering.md b/docs/api/filtering.md new file mode 100644 index 00000000000..e7b51d303e7 --- /dev/null +++ b/docs/api/filtering.md @@ -0,0 +1,71 @@ +# API Filtering + +The NetBox API supports robust filtering of results based on the fields of each model. +Generally speaking you are able to filter based on the attributes (fields) present in +the response body. Please note however that certain read-only or metadata fields are not +filterable. + +Filtering is achieved by passing HTTP query parameters and the parameter name is the +name of the field you wish to filter on and the value is the field value. + +E.g. filtering based on a device's name: +``` +/api/dcim/devices/?name=DC-SPINE-1 +``` + +## Multi Value Logic + +While you are able to filter based on an arbitrary number of fields, you are also able to +pass multiple values for the same field. In most cases filtering on multiple values is +implemented as a logical OR operation. A notible exception is the `tag` filter which +is a logical AND. Passing multiple values for one field, can be combined with other fields. + +For example, filtering for devices with either the name of DC-SPINE-1 _or_ DC-LEAF-4: +``` +/api/dcim/devices/?name=DC-SPINE-1&name=DC-LEAF-4 +``` + +Filtering for devices with tag `router` and `customer-a` will return only devices with +_both_ of those tags applied: +``` +/api/dcim/devices/?tag=router&tag=customer-a +``` + +## Lookup Expressions + +Certain model fields also support filtering using additonal lookup expressions. This allows +for negation and other context specific filtering. + +These lookup expressions can be applied by adding a suffix to the desired field's name. +E.g. `mac_address__n`. In this case, the filter expression is for negation and it is seperated +by two underscores. Below are the lookup expressions that are supported across different field +types. + +### Numeric Fields + +Numeric based fields (ASN, VLAN ID, etc) support these lookup expressions: + +- `n` - not equal (negation) +- `lt` - less than +- `lte` - less than or equal +- `gt` - greater than +- `gte` - greater than or equal + +### String Fields + +String based (char) fields (Name, Address, etc) support these lookup expressions: + +- `n` - not equal (negation) +- `ic` - case insensitive contains +- `nic` - negated case insensitive contains +- `isw` - case insensitive starts with +- `nisw` - negated case insensitive starts with +- `iew` - case insensitive ends with +- `niew` - negated case insensitive ends with +- `ie` - case sensitive exact match +- `nie` - negated case sensitive exact match + +### Foreign Keys & Other Fields + +Certain other fields, namely foreign key relationships support just the negation +expression: `n`. diff --git a/docs/api/overview.md b/docs/api/overview.md index 3841e8bbf39..daa4f7c6335 100644 --- a/docs/api/overview.md +++ b/docs/api/overview.md @@ -62,6 +62,8 @@ Lists of objects can be filtered using a set of query parameters. For example, t GET /api/dcim/interfaces/?device_id=123 ``` +See [filtering](filtering.md) for more details. + # Serialization The NetBox API employs three types of serializers to represent model data: diff --git a/mkdocs.yml b/mkdocs.yml index 4ba91dfe52a..9dc8b857844 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -50,6 +50,7 @@ pages: - Authentication: 'api/authentication.md' - Working with Secrets: 'api/working-with-secrets.md' - Examples: 'api/examples.md' + - Filtering: 'api/filtering.md' - Development: - Introduction: 'development/index.md' - Style Guide: 'development/style-guide.md' diff --git a/netbox/utilities/constants.py b/netbox/utilities/constants.py index 1665b45c2b0..bf2cba592b6 100644 --- a/netbox/utilities/constants.py +++ b/netbox/utilities/constants.py @@ -57,19 +57,6 @@ n='exact' ) -FILTER_LOOKUP_HELP_TEXT_MAP = dict( - icontains='case insensitive contains', - iendswith='case insensitive ends with', - istartswith='case insensitive starts with', - iexact='case insensitive exact', - exact='case sensitive exact', - lt='less than', - lte='less than or equal', - gt='greater than', - gte='greater than or equal', - n='negated' -) - # Keys for PostgreSQL advisory locks. These are arbitrary bigints used by # the advisory_lock contextmanager. When a lock is acquired, diff --git a/netbox/utilities/filters.py b/netbox/utilities/filters.py index 5e9e1f4c1b2..2a9f0431638 100644 --- a/netbox/utilities/filters.py +++ b/netbox/utilities/filters.py @@ -8,8 +8,7 @@ from extras.models import Tag from utilities.constants import ( - FILTER_CHAR_BASED_LOOKUP_MAP, FILTER_LOOKUP_HELP_TEXT_MAP, FILTER_NEGATION_LOOKUP_MAP, - FILTER_NUMERIC_BASED_LOOKUP_MAP + FILTER_CHAR_BASED_LOOKUP_MAP, FILTER_NEGATION_LOOKUP_MAP, FILTER_NUMERIC_BASED_LOOKUP_MAP ) diff --git a/netbox/utilities/tests/test_filters.py b/netbox/utilities/tests/test_filters.py index a1cb771a142..ca1c10faffe 100644 --- a/netbox/utilities/tests/test_filters.py +++ b/netbox/utilities/tests/test_filters.py @@ -2,8 +2,12 @@ from django.test import TestCase import django_filters -from dcim.filters import SiteFilterSet -from dcim.models import Region, Site +from dcim.filters import DeviceFilterSet, SiteFilterSet +from dcim.choices import * +from dcim.models import ( + Device, DeviceRole, DeviceType, Interface, Manufacturer, Platform, Rack, Region, Site +) +from ipam.models import IPAddress from utilities.filters import BaseFilterSet, TreeNodeMultipleChoiceFilter @@ -68,67 +72,189 @@ class DynamicFilterLookupExpressionTest(TestCase): These tests ensure of the utilities.filters.BaseFilterSet.get_filters() method correctly generates dynamic filter expressions """ + device_queryset = Device.objects.all() + device_filterset = DeviceFilterSet + site_queryset = Site.objects.all() + site_filterset = SiteFilterSet + + @classmethod + def setUpTestData(cls): + + manufacturers = ( + Manufacturer(name='Manufacturer 1', slug='manufacturer-1'), + Manufacturer(name='Manufacturer 2', slug='manufacturer-2'), + Manufacturer(name='Manufacturer 3', slug='manufacturer-3'), + ) + Manufacturer.objects.bulk_create(manufacturers) - def setUp(self): + device_types = ( + DeviceType(manufacturer=manufacturers[0], model='Model 1', slug='model-1', is_full_depth=True), + DeviceType(manufacturer=manufacturers[1], model='Model 2', slug='model-2', is_full_depth=True), + DeviceType(manufacturer=manufacturers[2], model='Model 3', slug='model-3', is_full_depth=False), + ) + DeviceType.objects.bulk_create(device_types) - super().setUp() + device_roles = ( + DeviceRole(name='Device Role 1', slug='device-role-1'), + DeviceRole(name='Device Role 2', slug='device-role-2'), + DeviceRole(name='Device Role 3', slug='device-role-3'), + ) + DeviceRole.objects.bulk_create(device_roles) - self.region1 = Region.objects.create(name='Test Region 1', slug='test-region-1') - self.region2 = Region.objects.create(name='Test Region 2', slug='test-region-2') - self.site1 = Site.objects.create(region=self.region1, name='Test Site 1', slug='ABC-test-site1-ABC', asn=65001) - self.site2 = Site.objects.create(region=self.region2, name='Test Site 2', slug='def-test-site2-def', asn=65101) - self.site3 = Site.objects.create(region=None, name='Test Site 3', slug='ghi-test-site3-ghi', asn=65201) + platforms = ( + Platform(name='Platform 1', slug='platform-1'), + Platform(name='Platform 2', slug='platform-2'), + Platform(name='Platform 3', slug='platform-3'), + ) + Platform.objects.bulk_create(platforms) - self.queryset = Site.objects.all() + regions = ( + Region(name='Region 1', slug='region-1'), + Region(name='Region 2', slug='region-2'), + Region(name='Region 3', slug='region-3'), + ) + for region in regions: + region.save() + + sites = ( + Site(name='Site 1', slug='abc-site-1', region=regions[0], asn=65001), + Site(name='Site 2', slug='def-site-2', region=regions[1], asn=65101), + Site(name='Site 3', slug='ghi-site-3', region=regions[2], asn=65201), + ) + Site.objects.bulk_create(sites) + + racks = ( + Rack(name='Rack 1', site=sites[0]), + Rack(name='Rack 2', site=sites[1]), + Rack(name='Rack 3', site=sites[2]), + ) + Rack.objects.bulk_create(racks) + + devices = ( + Device(name='Device 1', device_type=device_types[0], device_role=device_roles[0], platform=platforms[0], serial='ABC', asset_tag='1001', site=sites[0], rack=racks[0], position=1, face=DeviceFaceChoices.FACE_FRONT, status=DeviceStatusChoices.STATUS_ACTIVE, local_context_data={"foo": 123}), + Device(name='Device 2', device_type=device_types[1], device_role=device_roles[1], platform=platforms[1], serial='DEF', asset_tag='1002', site=sites[1], rack=racks[1], position=2, face=DeviceFaceChoices.FACE_FRONT, status=DeviceStatusChoices.STATUS_STAGED), + Device(name='Device 3', device_type=device_types[2], device_role=device_roles[2], platform=platforms[2], serial='GHI', asset_tag='1003', site=sites[2], rack=racks[2], position=3, face=DeviceFaceChoices.FACE_REAR, status=DeviceStatusChoices.STATUS_FAILED), + ) + Device.objects.bulk_create(devices) + + interfaces = ( + Interface(device=devices[0], name='Interface 1', mac_address='00-00-00-00-00-01'), + Interface(device=devices[0], name='Interface 2', mac_address='aa-00-00-00-00-01'), + Interface(device=devices[1], name='Interface 3', mac_address='00-00-00-00-00-02'), + Interface(device=devices[1], name='Interface 4', mac_address='bb-00-00-00-00-02'), + Interface(device=devices[2], name='Interface 5', mac_address='00-00-00-00-00-03'), + Interface(device=devices[2], name='Interface 6', mac_address='cc-00-00-00-00-03'), + ) + Interface.objects.bulk_create(interfaces) def test_site_name_negation(self): - params = {'name__n': ['Test Site 1']} - self.assertEqual(SiteFilterSet(params, self.queryset).qs.count(), 2) + params = {'name__n': ['Site 1']} + self.assertEqual(SiteFilterSet(params, self.site_queryset).qs.count(), 2) def test_site_slug_icontains(self): - params = {'slug__ic': ['abc']} - self.assertEqual(SiteFilterSet(params, self.queryset).qs.count(), 1) + params = {'slug__ic': ['-1']} + self.assertEqual(SiteFilterSet(params, self.site_queryset).qs.count(), 1) def test_site_slug_icontains_negation(self): - params = {'slug__nic': ['abc']} - self.assertEqual(SiteFilterSet(params, self.queryset).qs.count(), 2) + params = {'slug__nic': ['-1']} + self.assertEqual(SiteFilterSet(params, self.site_queryset).qs.count(), 2) def test_site_slug_startswith(self): params = {'slug__isw': ['abc']} - self.assertEqual(SiteFilterSet(params, self.queryset).qs.count(), 1) + self.assertEqual(SiteFilterSet(params, self.site_queryset).qs.count(), 1) def test_site_slug_startswith_negation(self): params = {'slug__nisw': ['abc']} - self.assertEqual(SiteFilterSet(params, self.queryset).qs.count(), 2) + self.assertEqual(SiteFilterSet(params, self.site_queryset).qs.count(), 2) def test_site_slug_endswith(self): - params = {'slug__iew': ['abc']} - self.assertEqual(SiteFilterSet(params, self.queryset).qs.count(), 1) + params = {'slug__iew': ['-1']} + self.assertEqual(SiteFilterSet(params, self.site_queryset).qs.count(), 1) def test_site_slug_endswith_negation(self): - params = {'slug__niew': ['abc']} - self.assertEqual(SiteFilterSet(params, self.queryset).qs.count(), 2) + params = {'slug__niew': ['-1']} + self.assertEqual(SiteFilterSet(params, self.site_queryset).qs.count(), 2) def test_site_asn_lt(self): params = {'asn__lt': [65101]} - self.assertEqual(SiteFilterSet(params, self.queryset).qs.count(), 1) + self.assertEqual(SiteFilterSet(params, self.site_queryset).qs.count(), 1) def test_site_asn_lte(self): params = {'asn__lte': [65101]} - self.assertEqual(SiteFilterSet(params, self.queryset).qs.count(), 2) + self.assertEqual(SiteFilterSet(params, self.site_queryset).qs.count(), 2) def test_site_asn_gt(self): params = {'asn__lt': [65101]} - self.assertEqual(SiteFilterSet(params, self.queryset).qs.count(), 1) + self.assertEqual(SiteFilterSet(params, self.site_queryset).qs.count(), 1) def test_site_asn_gte(self): params = {'asn__gte': [65101]} - self.assertEqual(SiteFilterSet(params, self.queryset).qs.count(), 2) + self.assertEqual(SiteFilterSet(params, self.site_queryset).qs.count(), 2) def test_site_region_negation(self): - params = {'region__n': ['test-region-1']} - self.assertEqual(SiteFilterSet(params, self.queryset).qs.count(), 2) + params = {'region__n': ['region-1']} + self.assertEqual(SiteFilterSet(params, self.site_queryset).qs.count(), 2) def test_site_region_id_negation(self): - params = {'region_id__n': [self.region1.pk]} - self.assertEqual(SiteFilterSet(params, self.queryset).qs.count(), 2) + params = {'region_id__n': [Region.objects.first().pk]} + self.assertEqual(SiteFilterSet(params, self.site_queryset).qs.count(), 2) + + def test_device_name_eq(self): + params = {'name': ['Device 1']} + self.assertEqual(DeviceFilterSet(params, self.device_queryset).qs.count(), 1) + + def test_device_name_negation(self): + params = {'name__n': ['Device 1']} + self.assertEqual(DeviceFilterSet(params, self.device_queryset).qs.count(), 2) + + def test_device_name_startswith(self): + params = {'name__isw': ['Device']} + self.assertEqual(DeviceFilterSet(params, self.device_queryset).qs.count(), 3) + + def test_device_name_startswith_negation(self): + params = {'name__nisw': ['Device 1']} + self.assertEqual(DeviceFilterSet(params, self.device_queryset).qs.count(), 2) + + def test_device_name_endswith(self): + params = {'name__iew': [' 1']} + self.assertEqual(DeviceFilterSet(params, self.device_queryset).qs.count(), 1) + + def test_device_name_endswith_negation(self): + params = {'name__niew': [' 1']} + self.assertEqual(DeviceFilterSet(params, self.device_queryset).qs.count(), 2) + + def test_device_name_icontains(self): + params = {'name__ic': [' 2']} + self.assertEqual(DeviceFilterSet(params, self.device_queryset).qs.count(), 1) + + def test_device_name_icontains_negation(self): + params = {'name__nic': [' ']} + self.assertEqual(DeviceFilterSet(params, self.device_queryset).qs.count(), 0) + + def test_device_mac_address_negation(self): + params = {'mac_address__n': ['00-00-00-00-00-01', 'aa-00-00-00-00-01']} + self.assertEqual(DeviceFilterSet(params, self.device_queryset).qs.count(), 2) + + def test_device_mac_address_startswith(self): + params = {'mac_address__isw': ['aa:']} + self.assertEqual(DeviceFilterSet(params, self.device_queryset).qs.count(), 1) + + def test_device_mac_address_startswith_negation(self): + params = {'mac_address__nisw': ['aa:']} + self.assertEqual(DeviceFilterSet(params, self.device_queryset).qs.count(), 2) + + def test_device_mac_address_endswith(self): + params = {'mac_address__iew': [':02']} + self.assertEqual(DeviceFilterSet(params, self.device_queryset).qs.count(), 1) + + def test_device_mac_address_endswith_negation(self): + params = {'mac_address__niew': [':02']} + self.assertEqual(DeviceFilterSet(params, self.device_queryset).qs.count(), 2) + + def test_device_mac_address_icontains(self): + params = {'mac_address__ic': ['aa:', 'bb']} + self.assertEqual(DeviceFilterSet(params, self.device_queryset).qs.count(), 2) + + def test_device_mac_address_icontains_negation(self): + params = {'mac_address__nic': ['aa:', 'bb']} + self.assertEqual(DeviceFilterSet(params, self.device_queryset).qs.count(), 1) From afc8c9bfe9fb4dadb5220804a87733ef2be0c447 Mon Sep 17 00:00:00 2001 From: John Anderson Date: Tue, 25 Feb 2020 13:50:31 -0500 Subject: [PATCH 06/30] fix tenancy filterset bases --- netbox/dcim/filters.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/netbox/dcim/filters.py b/netbox/dcim/filters.py index 0d7c440c264..7b98359c8cb 100644 --- a/netbox/dcim/filters.py +++ b/netbox/dcim/filters.py @@ -77,7 +77,7 @@ class Meta: fields = ['id', 'name', 'slug'] -class SiteFilterSet(BaseFilterSet, CustomFieldFilterSet, CreatedUpdatedFilterSet): +class SiteFilterSet(BaseFilterSet, TenancyFilterSet, CustomFieldFilterSet, CreatedUpdatedFilterSet): id__in = NumericInFilter( field_name='id', lookup_expr='in' @@ -170,7 +170,7 @@ class Meta: fields = ['id', 'name', 'slug', 'color'] -class RackFilterSet(BaseFilterSet, CustomFieldFilterSet, CreatedUpdatedFilterSet): +class RackFilterSet(BaseFilterSet, TenancyFilterSet, CustomFieldFilterSet, CreatedUpdatedFilterSet): id__in = NumericInFilter( field_name='id', lookup_expr='in' @@ -497,7 +497,13 @@ class Meta: fields = ['id', 'name', 'slug', 'napalm_driver'] -class DeviceFilterSet(BaseFilterSet, LocalConfigContextFilterSet, CustomFieldFilterSet, CreatedUpdatedFilterSet): +class DeviceFilterSet( + BaseFilterSet, + TenancyFilterSet, + LocalConfigContextFilterSet, + CustomFieldFilterSet, + CreatedUpdatedFilterSet +): id__in = NumericInFilter( field_name='id', lookup_expr='in' From 3b4607d30dd831dc54d87426eeaacd59baf8283b Mon Sep 17 00:00:00 2001 From: John Anderson Date: Tue, 25 Feb 2020 15:16:27 -0500 Subject: [PATCH 07/30] refactor lookup map logic --- netbox/utilities/filters.py | 64 +++++++++++++++++++++---------------- 1 file changed, 37 insertions(+), 27 deletions(-) diff --git a/netbox/utilities/filters.py b/netbox/utilities/filters.py index 2a9f0431638..d88366d24e0 100644 --- a/netbox/utilities/filters.py +++ b/netbox/utilities/filters.py @@ -170,6 +170,39 @@ class BaseFilterSet(django_filters.FilterSet): }, }) + @staticmethod + def _get_filter_lookup_dict(existing_filter): + # Choose the lookup expression map based on the filter type + if isinstance(existing_filter, ( + MultiValueDateFilter, + MultiValueDateTimeFilter, + MultiValueNumberFilter, + MultiValueTimeFilter + )): + lookup_map = FILTER_NUMERIC_BASED_LOOKUP_MAP + + elif isinstance(existing_filter, ( + django_filters.ModelChoiceFilter, + django_filters.ModelMultipleChoiceFilter, + TreeNodeMultipleChoiceFilter, + TagFilter + )) or existing_filter.extra.get('choices'): + # These filter types support only negation + lookup_map = FILTER_NEGATION_LOOKUP_MAP + + elif isinstance(existing_filter, ( + django_filters.filters.CharFilter, + django_filters.MultipleChoiceFilter, + MultiValueCharFilter, + MultiValueMACAddressFilter + )): + lookup_map = FILTER_CHAR_BASED_LOOKUP_MAP + + else: + lookup_map = None + + return lookup_map + @classmethod def get_filters(cls): """ @@ -192,33 +225,9 @@ def get_filters(cls): continue # Choose the lookup expression map based on the filter type - if isinstance(existing_filter, ( - MultiValueDateFilter, - MultiValueDateTimeFilter, - MultiValueNumberFilter, - MultiValueTimeFilter - )): - lookup_map = FILTER_NUMERIC_BASED_LOOKUP_MAP - - elif isinstance(existing_filter, ( - django_filters.ModelChoiceFilter, - django_filters.ModelMultipleChoiceFilter, - TreeNodeMultipleChoiceFilter, - TagFilter - )) or existing_filter.extra.get('choices'): - # These filter types support only negation - lookup_map = FILTER_NEGATION_LOOKUP_MAP - - elif isinstance(existing_filter, ( - django_filters.filters.CharFilter, - django_filters.MultipleChoiceFilter, - MultiValueCharFilter, - MultiValueMACAddressFilter - )): - lookup_map = FILTER_CHAR_BASED_LOOKUP_MAP - - else: - # Do not augment any other filter types with more lookup expressions + lookup_map = cls._get_filter_lookup_dict(existing_filter) + if lookup_map is None: + # Do not augment this filter type with more lookup expressions continue # Get properties of the existing filter for later use @@ -247,6 +256,7 @@ def get_filters(cls): ) else: # The filter field is listed in Meta.fields so we can safely rely on default behaviour + # Will raise FieldLookupError if the lookup is invalid new_filter = cls.filter_for_field(field, field_name, lookup_expr) except django_filters.exceptions.FieldLookupError: # The filter could not be created because the lookup expression is not supported on the field From e5f8f1529382cb833b3ffd380ff7a23eafd3f2e0 Mon Sep 17 00:00:00 2001 From: John Anderson Date: Fri, 28 Feb 2020 19:58:06 -0500 Subject: [PATCH 08/30] added lookup map for treenode filter --- netbox/utilities/constants.py | 4 ++++ netbox/utilities/filters.py | 12 ++++++++---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/netbox/utilities/constants.py b/netbox/utilities/constants.py index bf2cba592b6..bdcdeef1153 100644 --- a/netbox/utilities/constants.py +++ b/netbox/utilities/constants.py @@ -57,6 +57,10 @@ n='exact' ) +FILTER_TREENODE_NEGATION_LOOKUP_MAP = dict( + n='in' +) + # Keys for PostgreSQL advisory locks. These are arbitrary bigints used by # the advisory_lock contextmanager. When a lock is acquired, diff --git a/netbox/utilities/filters.py b/netbox/utilities/filters.py index d88366d24e0..ff34a60118b 100644 --- a/netbox/utilities/filters.py +++ b/netbox/utilities/filters.py @@ -8,7 +8,8 @@ from extras.models import Tag from utilities.constants import ( - FILTER_CHAR_BASED_LOOKUP_MAP, FILTER_NEGATION_LOOKUP_MAP, FILTER_NUMERIC_BASED_LOOKUP_MAP + FILTER_CHAR_BASED_LOOKUP_MAP, FILTER_NEGATION_LOOKUP_MAP, FILTER_TREENODE_NEGATION_LOOKUP_MAP, + FILTER_NUMERIC_BASED_LOOKUP_MAP ) @@ -181,10 +182,15 @@ def _get_filter_lookup_dict(existing_filter): )): lookup_map = FILTER_NUMERIC_BASED_LOOKUP_MAP + elif isinstance(existing_filter, ( + TreeNodeMultipleChoiceFilter, + )): + # TreeNodeMultipleChoiceFilter only support negation but must maintain the `in` lookup expression + lookup_map = FILTER_TREENODE_NEGATION_LOOKUP_MAP + elif isinstance(existing_filter, ( django_filters.ModelChoiceFilter, django_filters.ModelMultipleChoiceFilter, - TreeNodeMultipleChoiceFilter, TagFilter )) or existing_filter.extra.get('choices'): # These filter types support only negation @@ -237,8 +243,6 @@ def get_filters(cls): # Create new filters for each lookup expression in the map for lookup_name, lookup_expr in lookup_map.items(): new_filter_name = '{}__{}'.format(existing_filter_name, lookup_name) - if existing_filter.lookup_expr == 'in': - lookup_expr = 'in' # 'in' lookups must remain to avoid unwanted slicing on certain querysets try: if existing_filter_name in cls.declared_filters: From 1e1c6526b2270b8dca291d888008e9ead97b5128 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Mon, 2 Mar 2020 12:25:21 -0500 Subject: [PATCH 09/30] Add BaseFilterSetTest to validate automatic generation of filters --- netbox/utilities/tests/test_filters.py | 270 ++++++++++++++++++++++++- 1 file changed, 264 insertions(+), 6 deletions(-) diff --git a/netbox/utilities/tests/test_filters.py b/netbox/utilities/tests/test_filters.py index ca1c10faffe..f70d7e1db52 100644 --- a/netbox/utilities/tests/test_filters.py +++ b/netbox/utilities/tests/test_filters.py @@ -1,14 +1,21 @@ +import django_filters from django.conf import settings +from django.db import models from django.test import TestCase -import django_filters +from mptt.fields import TreeForeignKey +from taggit.managers import TaggableManager -from dcim.filters import DeviceFilterSet, SiteFilterSet from dcim.choices import * +from dcim.fields import MACAddressField +from dcim.filters import DeviceFilterSet, SiteFilterSet from dcim.models import ( Device, DeviceRole, DeviceType, Interface, Manufacturer, Platform, Rack, Region, Site ) -from ipam.models import IPAddress -from utilities.filters import BaseFilterSet, TreeNodeMultipleChoiceFilter +from extras.models import TaggedItem +from utilities.filters import ( + BaseFilterSet, MACAddressFilter, MultiValueCharFilter, MultiValueDateFilter, MultiValueDateTimeFilter, + MultiValueNumberFilter, MultiValueTimeFilter, TagFilter, TreeNodeMultipleChoiceFilter, +) class TreeNodeMultipleChoiceFilterTest(TestCase): @@ -67,10 +74,261 @@ def test_filter_combined(self): self.assertEqual(qs[1], self.site3) +class DummyModel(models.Model): + """ + Dummy model used by BaseFilterSetTest for filter validation. Should never appear in a schema migration. + """ + charfield = models.CharField( + max_length=10 + ) + choicefield = models.IntegerField( + choices=(('A', 1), ('B', 2), ('C', 3)) + ) + datefield = models.DateField() + datetimefield = models.DateTimeField() + integerfield = models.IntegerField() + macaddressfield = MACAddressField() + timefield = models.TimeField() + treeforeignkeyfield = TreeForeignKey( + to='self', + on_delete=models.CASCADE + ) + + tags = TaggableManager(through=TaggedItem) + + +class BaseFilterSetTest(TestCase): + """ + Ensure that a BaseFilterSet automatically creates the expected set of filters for each filter type. + """ + class DummyFilterSet(BaseFilterSet): + charfield = django_filters.CharFilter() + macaddressfield = MACAddressFilter() + modelchoicefield = django_filters.ModelChoiceFilter( + field_name='integerfield', # We're pretending this is a ForeignKey field + queryset=Site.objects.all() + ) + modelmultiplechoicefield = django_filters.ModelMultipleChoiceFilter( + field_name='integerfield', # We're pretending this is a ForeignKey field + queryset=Site.objects.all() + ) + multiplechoicefield = django_filters.MultipleChoiceFilter( + field_name='choicefield' + ) + multivaluecharfield = MultiValueCharFilter( + field_name='charfield' + ) + tagfield = TagFilter() + treeforeignkeyfield = TreeNodeMultipleChoiceFilter( + queryset=DummyModel.objects.all() + ) + + class Meta: + model = DummyModel + fields = ( + 'charfield', + 'choicefield', + 'datefield', + 'datetimefield', + 'integerfield', + 'macaddressfield', + 'modelchoicefield', + 'modelmultiplechoicefield', + 'multiplechoicefield', + 'tagfield', + 'timefield', + 'treeforeignkeyfield', + ) + + @classmethod + def setUpTestData(cls): + cls.filters = cls.DummyFilterSet().filters + + def test_char_filter(self): + self.assertIsInstance(self.filters['charfield'], django_filters.CharFilter) + self.assertEqual(self.filters['charfield'].lookup_expr, 'exact') + self.assertEqual(self.filters['charfield'].exclude, False) + self.assertEqual(self.filters['charfield__n'].lookup_expr, 'exact') + self.assertEqual(self.filters['charfield__n'].exclude, True) + self.assertEqual(self.filters['charfield__ie'].lookup_expr, 'iexact') + self.assertEqual(self.filters['charfield__ie'].exclude, False) + self.assertEqual(self.filters['charfield__nie'].lookup_expr, 'iexact') + self.assertEqual(self.filters['charfield__nie'].exclude, True) + self.assertEqual(self.filters['charfield__ic'].lookup_expr, 'icontains') + self.assertEqual(self.filters['charfield__ic'].exclude, False) + self.assertEqual(self.filters['charfield__nic'].lookup_expr, 'icontains') + self.assertEqual(self.filters['charfield__nic'].exclude, True) + self.assertEqual(self.filters['charfield__isw'].lookup_expr, 'istartswith') + self.assertEqual(self.filters['charfield__isw'].exclude, False) + self.assertEqual(self.filters['charfield__nisw'].lookup_expr, 'istartswith') + self.assertEqual(self.filters['charfield__nisw'].exclude, True) + self.assertEqual(self.filters['charfield__iew'].lookup_expr, 'iendswith') + self.assertEqual(self.filters['charfield__iew'].exclude, False) + self.assertEqual(self.filters['charfield__niew'].lookup_expr, 'iendswith') + self.assertEqual(self.filters['charfield__niew'].exclude, True) + + def test_mac_address_filter(self): + self.assertIsInstance(self.filters['macaddressfield'], MACAddressFilter) + self.assertEqual(self.filters['macaddressfield'].lookup_expr, 'exact') + self.assertEqual(self.filters['macaddressfield'].exclude, False) + self.assertEqual(self.filters['macaddressfield__n'].lookup_expr, 'exact') + self.assertEqual(self.filters['macaddressfield__n'].exclude, True) + self.assertEqual(self.filters['macaddressfield__ie'].lookup_expr, 'iexact') + self.assertEqual(self.filters['macaddressfield__ie'].exclude, False) + self.assertEqual(self.filters['macaddressfield__nie'].lookup_expr, 'iexact') + self.assertEqual(self.filters['macaddressfield__nie'].exclude, True) + self.assertEqual(self.filters['macaddressfield__ic'].lookup_expr, 'icontains') + self.assertEqual(self.filters['macaddressfield__ic'].exclude, False) + self.assertEqual(self.filters['macaddressfield__nic'].lookup_expr, 'icontains') + self.assertEqual(self.filters['macaddressfield__nic'].exclude, True) + self.assertEqual(self.filters['macaddressfield__isw'].lookup_expr, 'istartswith') + self.assertEqual(self.filters['macaddressfield__isw'].exclude, False) + self.assertEqual(self.filters['macaddressfield__nisw'].lookup_expr, 'istartswith') + self.assertEqual(self.filters['macaddressfield__nisw'].exclude, True) + self.assertEqual(self.filters['macaddressfield__iew'].lookup_expr, 'iendswith') + self.assertEqual(self.filters['macaddressfield__iew'].exclude, False) + self.assertEqual(self.filters['macaddressfield__niew'].lookup_expr, 'iendswith') + self.assertEqual(self.filters['macaddressfield__niew'].exclude, True) + + def test_model_choice_filter(self): + self.assertIsInstance(self.filters['modelchoicefield'], django_filters.ModelChoiceFilter) + self.assertEqual(self.filters['modelchoicefield'].lookup_expr, 'exact') + self.assertEqual(self.filters['modelchoicefield'].exclude, False) + self.assertEqual(self.filters['modelchoicefield__n'].lookup_expr, 'exact') + self.assertEqual(self.filters['modelchoicefield__n'].exclude, True) + + def test_model_multiple_choice_filter(self): + self.assertIsInstance(self.filters['modelmultiplechoicefield'], django_filters.ModelMultipleChoiceFilter) + self.assertEqual(self.filters['modelmultiplechoicefield'].lookup_expr, 'exact') + self.assertEqual(self.filters['modelmultiplechoicefield'].exclude, False) + self.assertEqual(self.filters['modelmultiplechoicefield__n'].lookup_expr, 'exact') + self.assertEqual(self.filters['modelmultiplechoicefield__n'].exclude, True) + + def test_multi_value_char_filter(self): + self.assertIsInstance(self.filters['multivaluecharfield'], MultiValueCharFilter) + self.assertEqual(self.filters['multivaluecharfield'].lookup_expr, 'exact') + self.assertEqual(self.filters['multivaluecharfield'].exclude, False) + self.assertEqual(self.filters['multivaluecharfield__n'].lookup_expr, 'exact') + self.assertEqual(self.filters['multivaluecharfield__n'].exclude, True) + self.assertEqual(self.filters['multivaluecharfield__ie'].lookup_expr, 'iexact') + self.assertEqual(self.filters['multivaluecharfield__ie'].exclude, False) + self.assertEqual(self.filters['multivaluecharfield__nie'].lookup_expr, 'iexact') + self.assertEqual(self.filters['multivaluecharfield__nie'].exclude, True) + self.assertEqual(self.filters['multivaluecharfield__ic'].lookup_expr, 'icontains') + self.assertEqual(self.filters['multivaluecharfield__ic'].exclude, False) + self.assertEqual(self.filters['multivaluecharfield__nic'].lookup_expr, 'icontains') + self.assertEqual(self.filters['multivaluecharfield__nic'].exclude, True) + self.assertEqual(self.filters['multivaluecharfield__isw'].lookup_expr, 'istartswith') + self.assertEqual(self.filters['multivaluecharfield__isw'].exclude, False) + self.assertEqual(self.filters['multivaluecharfield__nisw'].lookup_expr, 'istartswith') + self.assertEqual(self.filters['multivaluecharfield__nisw'].exclude, True) + self.assertEqual(self.filters['multivaluecharfield__iew'].lookup_expr, 'iendswith') + self.assertEqual(self.filters['multivaluecharfield__iew'].exclude, False) + self.assertEqual(self.filters['multivaluecharfield__niew'].lookup_expr, 'iendswith') + self.assertEqual(self.filters['multivaluecharfield__niew'].exclude, True) + + def test_multi_value_date_filter(self): + self.assertIsInstance(self.filters['datefield'], MultiValueDateFilter) + self.assertEqual(self.filters['datefield'].lookup_expr, 'exact') + self.assertEqual(self.filters['datefield'].exclude, False) + self.assertEqual(self.filters['datefield__n'].lookup_expr, 'exact') + self.assertEqual(self.filters['datefield__n'].exclude, True) + self.assertEqual(self.filters['datefield__lt'].lookup_expr, 'lt') + self.assertEqual(self.filters['datefield__lt'].exclude, False) + self.assertEqual(self.filters['datefield__lte'].lookup_expr, 'lte') + self.assertEqual(self.filters['datefield__lte'].exclude, False) + self.assertEqual(self.filters['datefield__gt'].lookup_expr, 'gt') + self.assertEqual(self.filters['datefield__gt'].exclude, False) + self.assertEqual(self.filters['datefield__gte'].lookup_expr, 'gte') + self.assertEqual(self.filters['datefield__gte'].exclude, False) + + def test_multi_value_datetime_filter(self): + self.assertIsInstance(self.filters['datetimefield'], MultiValueDateTimeFilter) + self.assertEqual(self.filters['datetimefield'].lookup_expr, 'exact') + self.assertEqual(self.filters['datetimefield'].exclude, False) + self.assertEqual(self.filters['datetimefield__n'].lookup_expr, 'exact') + self.assertEqual(self.filters['datetimefield__n'].exclude, True) + self.assertEqual(self.filters['datetimefield__lt'].lookup_expr, 'lt') + self.assertEqual(self.filters['datetimefield__lt'].exclude, False) + self.assertEqual(self.filters['datetimefield__lte'].lookup_expr, 'lte') + self.assertEqual(self.filters['datetimefield__lte'].exclude, False) + self.assertEqual(self.filters['datetimefield__gt'].lookup_expr, 'gt') + self.assertEqual(self.filters['datetimefield__gt'].exclude, False) + self.assertEqual(self.filters['datetimefield__gte'].lookup_expr, 'gte') + self.assertEqual(self.filters['datetimefield__gte'].exclude, False) + + def test_multi_value_number_filter(self): + self.assertIsInstance(self.filters['integerfield'], MultiValueNumberFilter) + self.assertEqual(self.filters['integerfield'].lookup_expr, 'exact') + self.assertEqual(self.filters['integerfield'].exclude, False) + self.assertEqual(self.filters['integerfield__n'].lookup_expr, 'exact') + self.assertEqual(self.filters['integerfield__n'].exclude, True) + self.assertEqual(self.filters['integerfield__lt'].lookup_expr, 'lt') + self.assertEqual(self.filters['integerfield__lt'].exclude, False) + self.assertEqual(self.filters['integerfield__lte'].lookup_expr, 'lte') + self.assertEqual(self.filters['integerfield__lte'].exclude, False) + self.assertEqual(self.filters['integerfield__gt'].lookup_expr, 'gt') + self.assertEqual(self.filters['integerfield__gt'].exclude, False) + self.assertEqual(self.filters['integerfield__gte'].lookup_expr, 'gte') + self.assertEqual(self.filters['integerfield__gte'].exclude, False) + + def test_multi_value_time_filter(self): + self.assertIsInstance(self.filters['timefield'], MultiValueTimeFilter) + self.assertEqual(self.filters['timefield'].lookup_expr, 'exact') + self.assertEqual(self.filters['timefield'].exclude, False) + self.assertEqual(self.filters['timefield__n'].lookup_expr, 'exact') + self.assertEqual(self.filters['timefield__n'].exclude, True) + self.assertEqual(self.filters['timefield__lt'].lookup_expr, 'lt') + self.assertEqual(self.filters['timefield__lt'].exclude, False) + self.assertEqual(self.filters['timefield__lte'].lookup_expr, 'lte') + self.assertEqual(self.filters['timefield__lte'].exclude, False) + self.assertEqual(self.filters['timefield__gt'].lookup_expr, 'gt') + self.assertEqual(self.filters['timefield__gt'].exclude, False) + self.assertEqual(self.filters['timefield__gte'].lookup_expr, 'gte') + self.assertEqual(self.filters['timefield__gte'].exclude, False) + + def test_multiple_choice_filter(self): + self.assertIsInstance(self.filters['multiplechoicefield'], django_filters.MultipleChoiceFilter) + self.assertEqual(self.filters['multiplechoicefield'].lookup_expr, 'exact') + self.assertEqual(self.filters['multiplechoicefield'].exclude, False) + self.assertEqual(self.filters['multiplechoicefield__n'].lookup_expr, 'exact') + self.assertEqual(self.filters['multiplechoicefield__n'].exclude, True) + self.assertEqual(self.filters['multiplechoicefield__ie'].lookup_expr, 'iexact') + self.assertEqual(self.filters['multiplechoicefield__ie'].exclude, False) + self.assertEqual(self.filters['multiplechoicefield__nie'].lookup_expr, 'iexact') + self.assertEqual(self.filters['multiplechoicefield__nie'].exclude, True) + self.assertEqual(self.filters['multiplechoicefield__ic'].lookup_expr, 'icontains') + self.assertEqual(self.filters['multiplechoicefield__ic'].exclude, False) + self.assertEqual(self.filters['multiplechoicefield__nic'].lookup_expr, 'icontains') + self.assertEqual(self.filters['multiplechoicefield__nic'].exclude, True) + self.assertEqual(self.filters['multiplechoicefield__isw'].lookup_expr, 'istartswith') + self.assertEqual(self.filters['multiplechoicefield__isw'].exclude, False) + self.assertEqual(self.filters['multiplechoicefield__nisw'].lookup_expr, 'istartswith') + self.assertEqual(self.filters['multiplechoicefield__nisw'].exclude, True) + self.assertEqual(self.filters['multiplechoicefield__iew'].lookup_expr, 'iendswith') + self.assertEqual(self.filters['multiplechoicefield__iew'].exclude, False) + self.assertEqual(self.filters['multiplechoicefield__niew'].lookup_expr, 'iendswith') + self.assertEqual(self.filters['multiplechoicefield__niew'].exclude, True) + + def test_tag_filter(self): + self.assertIsInstance(self.filters['tagfield'], TagFilter) + self.assertEqual(self.filters['tagfield'].lookup_expr, 'exact') + self.assertEqual(self.filters['tagfield'].exclude, False) + self.assertEqual(self.filters['tagfield__n'].lookup_expr, 'exact') + self.assertEqual(self.filters['tagfield__n'].exclude, True) + + def test_tree_node_multiple_choice_filter(self): + self.assertIsInstance(self.filters['treeforeignkeyfield'], TreeNodeMultipleChoiceFilter) + # TODO: lookup_expr different for negation? + self.assertEqual(self.filters['treeforeignkeyfield'].lookup_expr, 'exact') + self.assertEqual(self.filters['treeforeignkeyfield'].exclude, False) + self.assertEqual(self.filters['treeforeignkeyfield__n'].lookup_expr, 'in') + self.assertEqual(self.filters['treeforeignkeyfield__n'].exclude, True) + + class DynamicFilterLookupExpressionTest(TestCase): """ - These tests ensure of the utilities.filters.BaseFilterSet.get_filters() method - correctly generates dynamic filter expressions + Validate function of automatically generated filters using the Device model as an example. """ device_queryset = Device.objects.all() device_filterset = DeviceFilterSet From 406708218bf41d5fb72db814399b2011d9e48856 Mon Sep 17 00:00:00 2001 From: Dan Starner Date: Mon, 2 Mar 2020 13:34:01 -0500 Subject: [PATCH 10/30] set local_context_data serializer on device and vm to method --- netbox/dcim/api/serializers.py | 6 ++++++ netbox/virtualization/api/serializers.py | 7 +++++++ 2 files changed, 13 insertions(+) diff --git a/netbox/dcim/api/serializers.py b/netbox/dcim/api/serializers.py index 5483904f522..7a7feb8e11a 100644 --- a/netbox/dcim/api/serializers.py +++ b/netbox/dcim/api/serializers.py @@ -376,6 +376,7 @@ class DeviceSerializer(TaggitSerializer, CustomFieldModelSerializer): cluster = NestedClusterSerializer(required=False, allow_null=True) virtual_chassis = NestedVirtualChassisSerializer(required=False, allow_null=True) tags = TagListSerializerField(required=False) + local_context_data = serializers.SerializerMethodField() class Meta: model = Device @@ -411,6 +412,11 @@ def get_parent_device(self, obj): data['device_bay'] = NestedDeviceBaySerializer(instance=device_bay, context=context).data return data + @swagger_serializer_method(serializer_or_field=serializers.DictField) + def get_local_context_data(self, obj): + """Used to strongly type the local_context_data field for Swagger generation + """ + return obj.local_context_data class DeviceWithConfigContextSerializer(DeviceSerializer): config_context = serializers.SerializerMethodField() diff --git a/netbox/virtualization/api/serializers.py b/netbox/virtualization/api/serializers.py index a294cdb6faa..ecddfbdaece 100644 --- a/netbox/virtualization/api/serializers.py +++ b/netbox/virtualization/api/serializers.py @@ -67,6 +67,7 @@ class VirtualMachineSerializer(TaggitSerializer, CustomFieldModelSerializer): primary_ip4 = NestedIPAddressSerializer(required=False, allow_null=True) primary_ip6 = NestedIPAddressSerializer(required=False, allow_null=True) tags = TagListSerializerField(required=False) + local_context_data = serializers.SerializerMethodField() class Meta: model = VirtualMachine @@ -77,6 +78,12 @@ class Meta: ] validators = [] + @swagger_serializer_method(serializer_or_field=serializers.DictField) + def get_local_context_data(self, obj): + """Used to strongly type the local_context_data field for Swagger generation + """ + return obj.local_context_data + class VirtualMachineWithConfigContextSerializer(VirtualMachineSerializer): config_context = serializers.SerializerMethodField() From 3070c7e9911568637bc0a9a8ca8ea777dcb14425 Mon Sep 17 00:00:00 2001 From: Daniel Starner Date: Mon, 2 Mar 2020 15:45:58 -0500 Subject: [PATCH 11/30] fix linting mistake --- netbox/dcim/api/serializers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/netbox/dcim/api/serializers.py b/netbox/dcim/api/serializers.py index 7a7feb8e11a..03cf8dde3e1 100644 --- a/netbox/dcim/api/serializers.py +++ b/netbox/dcim/api/serializers.py @@ -418,6 +418,7 @@ def get_local_context_data(self, obj): """ return obj.local_context_data + class DeviceWithConfigContextSerializer(DeviceSerializer): config_context = serializers.SerializerMethodField() From fa992853b0602318abd99c14e91e52a9cb7955aa Mon Sep 17 00:00:00 2001 From: kobayashi Date: Tue, 3 Mar 2020 10:52:56 -0500 Subject: [PATCH 12/30] Describe supported Python version --- .travis.yml | 2 ++ docs/index.md | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/.travis.yml b/.travis.yml index 872121c2148..8bd352dd5fb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,6 +7,8 @@ addons: language: python python: - "3.5" + - "3.6" + - "3.7" install: - pip install -r requirements.txt - pip install pycodestyle diff --git a/docs/index.md b/docs/index.md index a68d5a6bf44..b3c3671653e 100644 --- a/docs/index.md +++ b/docs/index.md @@ -53,6 +53,10 @@ NetBox is built on the [Django](https://djangoproject.com/) Python framework and | Task queuing | Redis/django-rq | | Live device access | NAPALM | +## Supported Python Version + +NetBox supports Python 3.5, 3.6, and 3.7 environments currently. Python 3.5 is scheduled to be unsupported in NetBox v2.8. + # Getting Started See the [installation guide](installation/index.md) for help getting NetBox up and running quickly. From e6ee9803d49276b5a31ae6d977b32fb4cd486a27 Mon Sep 17 00:00:00 2001 From: Dan Starner Date: Tue, 3 Mar 2020 12:04:46 -0500 Subject: [PATCH 13/30] use FieldInspector for JSONField type --- netbox/dcim/api/serializers.py | 7 ------- netbox/netbox/settings.py | 1 + netbox/utilities/custom_inspectors.py | 10 ++++++++++ netbox/virtualization/api/serializers.py | 7 ------- 4 files changed, 11 insertions(+), 14 deletions(-) diff --git a/netbox/dcim/api/serializers.py b/netbox/dcim/api/serializers.py index 03cf8dde3e1..5483904f522 100644 --- a/netbox/dcim/api/serializers.py +++ b/netbox/dcim/api/serializers.py @@ -376,7 +376,6 @@ class DeviceSerializer(TaggitSerializer, CustomFieldModelSerializer): cluster = NestedClusterSerializer(required=False, allow_null=True) virtual_chassis = NestedVirtualChassisSerializer(required=False, allow_null=True) tags = TagListSerializerField(required=False) - local_context_data = serializers.SerializerMethodField() class Meta: model = Device @@ -412,12 +411,6 @@ def get_parent_device(self, obj): data['device_bay'] = NestedDeviceBaySerializer(instance=device_bay, context=context).data return data - @swagger_serializer_method(serializer_or_field=serializers.DictField) - def get_local_context_data(self, obj): - """Used to strongly type the local_context_data field for Swagger generation - """ - return obj.local_context_data - class DeviceWithConfigContextSerializer(DeviceSerializer): config_context = serializers.SerializerMethodField() diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index 89958bc13fb..7f4f44b1aeb 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -506,6 +506,7 @@ def _setting(name, default=None): SWAGGER_SETTINGS = { 'DEFAULT_AUTO_SCHEMA_CLASS': 'utilities.custom_inspectors.NetBoxSwaggerAutoSchema', 'DEFAULT_FIELD_INSPECTORS': [ + 'utilities.custom_inspectors.JSONFieldInspector', 'utilities.custom_inspectors.NullableBooleanFieldInspector', 'utilities.custom_inspectors.CustomChoiceFieldInspector', 'utilities.custom_inspectors.TagListFieldInspector', diff --git a/netbox/utilities/custom_inspectors.py b/netbox/utilities/custom_inspectors.py index 3eaf1ccf1e3..68fe57d82bf 100644 --- a/netbox/utilities/custom_inspectors.py +++ b/netbox/utilities/custom_inspectors.py @@ -1,3 +1,4 @@ +from django.contrib.postgres.fields import JSONField from drf_yasg import openapi from drf_yasg.inspectors import FieldInspector, NotHandled, PaginatorInspector, FilterInspector, SwaggerAutoSchema from drf_yasg.utils import get_serializer_ref_name @@ -119,6 +120,15 @@ def process_result(self, result, method_name, obj, **kwargs): return result +class JSONFieldInspector(FieldInspector): + """Required because by default, Swagger sees a JSONField as a string and not dict + """ + def process_result(self, result, method_name, obj, **kwargs): + if isinstance(result, openapi.Schema) and isinstance(obj, JSONField): + result.type = 'dict' + return result + + class IdInFilterInspector(FilterInspector): def process_result(self, result, method_name, obj, **kwargs): if isinstance(result, list): diff --git a/netbox/virtualization/api/serializers.py b/netbox/virtualization/api/serializers.py index ecddfbdaece..a294cdb6faa 100644 --- a/netbox/virtualization/api/serializers.py +++ b/netbox/virtualization/api/serializers.py @@ -67,7 +67,6 @@ class VirtualMachineSerializer(TaggitSerializer, CustomFieldModelSerializer): primary_ip4 = NestedIPAddressSerializer(required=False, allow_null=True) primary_ip6 = NestedIPAddressSerializer(required=False, allow_null=True) tags = TagListSerializerField(required=False) - local_context_data = serializers.SerializerMethodField() class Meta: model = VirtualMachine @@ -78,12 +77,6 @@ class Meta: ] validators = [] - @swagger_serializer_method(serializer_or_field=serializers.DictField) - def get_local_context_data(self, obj): - """Used to strongly type the local_context_data field for Swagger generation - """ - return obj.local_context_data - class VirtualMachineWithConfigContextSerializer(VirtualMachineSerializer): config_context = serializers.SerializerMethodField() From e4abfd192ef71fbd413aa0c99e6085cbb11e758a Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Tue, 3 Mar 2020 14:54:48 -0500 Subject: [PATCH 14/30] Introduce CustomFieldDefaultValues class to handle default custom field values --- netbox/extras/api/customfields.py | 87 ++++++++++++++++++------------- 1 file changed, 50 insertions(+), 37 deletions(-) diff --git a/netbox/extras/api/customfields.py b/netbox/extras/api/customfields.py index 9a3041238da..3f436970dec 100644 --- a/netbox/extras/api/customfields.py +++ b/netbox/extras/api/customfields.py @@ -4,6 +4,7 @@ from django.db import transaction from rest_framework import serializers from rest_framework.exceptions import ValidationError +from rest_framework.fields import CreateOnlyDefault from extras.choices import * from extras.models import CustomField, CustomFieldChoice, CustomFieldValue @@ -14,6 +15,36 @@ # Custom fields # +class CustomFieldDefaultValues: + """ + Return a dictionary of all CustomFields assigned to the parent model and their default values. + """ + def __call__(self): + + # Retrieve the CustomFields for the parent model + content_type = ContentType.objects.get_for_model(self.model) + fields = CustomField.objects.filter(obj_type=content_type) + + # Populate the default value for each CustomField + value = {} + for field in fields: + if field.default: + if field.type == CustomFieldTypeChoices.TYPE_SELECT: + field_value = field.choices.get(value=field.default).pk + elif field.type == CustomFieldTypeChoices.TYPE_BOOLEAN: + field_value = bool(field.default) + else: + field_value = field.default + value[field.name] = field_value + else: + value[field.name] = None + + return value + + def set_context(self, serializer_field): + self.model = serializer_field.parent.Meta.model + + class CustomFieldsSerializer(serializers.BaseSerializer): def to_representation(self, obj): @@ -94,53 +125,35 @@ class CustomFieldModelSerializer(ValidatedModelSerializer): """ Extends ModelSerializer to render any CustomFields and their values associated with an object. """ - custom_fields = CustomFieldsSerializer(required=False) + custom_fields = CustomFieldsSerializer( + required=False, + default=CreateOnlyDefault(CustomFieldDefaultValues()) + ) def __init__(self, *args, **kwargs): - - def _populate_custom_fields(instance, fields): - instance.custom_fields = {} - for field in fields: - value = instance.cf.get(field.name) - if field.type == CustomFieldTypeChoices.TYPE_SELECT and value is not None: - instance.custom_fields[field.name] = CustomFieldChoiceSerializer(value).data - else: - instance.custom_fields[field.name] = value - super().__init__(*args, **kwargs) - # Retrieve the set of CustomFields which apply to this type of object - content_type = ContentType.objects.get_for_model(self.Meta.model) - fields = CustomField.objects.filter(obj_type=content_type) - if self.instance is not None: + # Retrieve the set of CustomFields which apply to this type of object + content_type = ContentType.objects.get_for_model(self.Meta.model) + fields = CustomField.objects.filter(obj_type=content_type) + # Populate CustomFieldValues for each instance from database try: for obj in self.instance: - _populate_custom_fields(obj, fields) + self._populate_custom_fields(obj, fields) except TypeError: - _populate_custom_fields(self.instance, fields) - - else: - - if not hasattr(self, 'initial_data'): - self.initial_data = {} - - # Populate default values - if fields and 'custom_fields' not in self.initial_data: - self.initial_data['custom_fields'] = {} - - # Populate initial data using custom field default values - for field in fields: - if field.name not in self.initial_data['custom_fields'] and field.default: - if field.type == CustomFieldTypeChoices.TYPE_SELECT: - field_value = field.choices.get(value=field.default).pk - elif field.type == CustomFieldTypeChoices.TYPE_BOOLEAN: - field_value = bool(field.default) - else: - field_value = field.default - self.initial_data['custom_fields'][field.name] = field_value + self._populate_custom_fields(self.instance, fields) + + def _populate_custom_fields(self, instance, custom_fields): + instance.custom_fields = {} + for field in custom_fields: + value = instance.cf.get(field.name) + if field.type == CustomFieldTypeChoices.TYPE_SELECT and value is not None: + instance.custom_fields[field.name] = CustomFieldChoiceSerializer(value).data + else: + instance.custom_fields[field.name] = value def _save_custom_fields(self, instance, custom_fields): content_type = ContentType.objects.get_for_model(self.Meta.model) From 4611536ca97ef925a2b10c7a56a61806720ea195 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Tue, 3 Mar 2020 17:07:43 -0500 Subject: [PATCH 15/30] Revise custom field API tests to check for single/multiple objects with/without custom field values --- netbox/extras/api/customfields.py | 14 +- netbox/extras/tests/test_customfields.py | 434 ++++++++++++----------- 2 files changed, 245 insertions(+), 203 deletions(-) diff --git a/netbox/extras/api/customfields.py b/netbox/extras/api/customfields.py index 3f436970dec..5bb1f033dec 100644 --- a/netbox/extras/api/customfields.py +++ b/netbox/extras/api/customfields.py @@ -1,6 +1,7 @@ from datetime import datetime from django.contrib.contenttypes.models import ContentType +from django.core.exceptions import ObjectDoesNotExist from django.db import transaction from rest_framework import serializers from rest_framework.exceptions import ValidationError @@ -29,10 +30,17 @@ def __call__(self): value = {} for field in fields: if field.default: - if field.type == CustomFieldTypeChoices.TYPE_SELECT: - field_value = field.choices.get(value=field.default).pk + if field.type == CustomFieldTypeChoices.TYPE_INTEGER: + field_value = int(field.default) elif field.type == CustomFieldTypeChoices.TYPE_BOOLEAN: - field_value = bool(field.default) + # TODO: Fix default value assignment for boolean custom fields + field_value = False if field.default.lower() == 'false' else bool(field.default) + elif field.type == CustomFieldTypeChoices.TYPE_SELECT: + try: + field_value = field.choices.get(value=field.default).pk + except ObjectDoesNotExist: + # Invalid default value + field_value = None else: field_value = field.default value[field.name] = field_value diff --git a/netbox/extras/tests/test_customfields.py b/netbox/extras/tests/test_customfields.py index a6e2bfcec02..33ce9cca252 100644 --- a/netbox/extras/tests/test_customfields.py +++ b/netbox/extras/tests/test_customfields.py @@ -101,240 +101,274 @@ def test_select_field(self): class CustomFieldAPITest(APITestCase): - def setUp(self): - - super().setUp() + @classmethod + def setUpTestData(cls): content_type = ContentType.objects.get_for_model(Site) # Text custom field - self.cf_text = CustomField(type=CustomFieldTypeChoices.TYPE_TEXT, name='magic_word') - self.cf_text.save() - self.cf_text.obj_type.set([content_type]) - self.cf_text.save() + cls.cf_text = CustomField(type=CustomFieldTypeChoices.TYPE_TEXT, name='text_field', default='foo') + cls.cf_text.save() + cls.cf_text.obj_type.set([content_type]) # Integer custom field - self.cf_integer = CustomField(type=CustomFieldTypeChoices.TYPE_INTEGER, name='magic_number') - self.cf_integer.save() - self.cf_integer.obj_type.set([content_type]) - self.cf_integer.save() + cls.cf_integer = CustomField(type=CustomFieldTypeChoices.TYPE_INTEGER, name='number_field', default=123) + cls.cf_integer.save() + cls.cf_integer.obj_type.set([content_type]) # Boolean custom field - self.cf_boolean = CustomField(type=CustomFieldTypeChoices.TYPE_BOOLEAN, name='is_magic') - self.cf_boolean.save() - self.cf_boolean.obj_type.set([content_type]) - self.cf_boolean.save() + cls.cf_boolean = CustomField(type=CustomFieldTypeChoices.TYPE_BOOLEAN, name='boolean_field', default=False) + cls.cf_boolean.save() + cls.cf_boolean.obj_type.set([content_type]) # Date custom field - self.cf_date = CustomField(type=CustomFieldTypeChoices.TYPE_DATE, name='magic_date') - self.cf_date.save() - self.cf_date.obj_type.set([content_type]) - self.cf_date.save() + cls.cf_date = CustomField(type=CustomFieldTypeChoices.TYPE_DATE, name='date_field', default='2020-01-01') + cls.cf_date.save() + cls.cf_date.obj_type.set([content_type]) # URL custom field - self.cf_url = CustomField(type=CustomFieldTypeChoices.TYPE_URL, name='magic_url') - self.cf_url.save() - self.cf_url.obj_type.set([content_type]) - self.cf_url.save() + cls.cf_url = CustomField(type=CustomFieldTypeChoices.TYPE_URL, name='url_field', default='http://example.com/1') + cls.cf_url.save() + cls.cf_url.obj_type.set([content_type]) # Select custom field - self.cf_select = CustomField(type=CustomFieldTypeChoices.TYPE_SELECT, name='magic_choice') - self.cf_select.save() - self.cf_select.obj_type.set([content_type]) - self.cf_select.save() - self.cf_select_choice1 = CustomFieldChoice(field=self.cf_select, value='Foo') - self.cf_select_choice1.save() - self.cf_select_choice2 = CustomFieldChoice(field=self.cf_select, value='Bar') - self.cf_select_choice2.save() - self.cf_select_choice3 = CustomFieldChoice(field=self.cf_select, value='Baz') - self.cf_select_choice3.save() - - self.site = Site.objects.create(name='Test Site 1', slug='test-site-1') - - def test_get_obj_without_custom_fields(self): - - url = reverse('dcim-api:site-detail', kwargs={'pk': self.site.pk}) - response = self.client.get(url, **self.header) - - self.assertEqual(response.data['name'], self.site.name) - self.assertEqual(response.data['custom_fields'], { - 'magic_word': None, - 'magic_number': None, - 'is_magic': None, - 'magic_date': None, - 'magic_url': None, - 'magic_choice': None, - }) - - def test_get_obj_with_custom_fields(self): - - CUSTOM_FIELD_VALUES = [ - (self.cf_text, 'Test string'), - (self.cf_integer, 1234), - (self.cf_boolean, True), - (self.cf_date, date(2016, 6, 23)), - (self.cf_url, 'http://example.com/'), - (self.cf_select, self.cf_select_choice1.pk), - ] - for field, value in CUSTOM_FIELD_VALUES: - cfv = CustomFieldValue(field=field, obj=self.site) - cfv.value = value - cfv.save() - - url = reverse('dcim-api:site-detail', kwargs={'pk': self.site.pk}) - response = self.client.get(url, **self.header) - - self.assertEqual(response.data['name'], self.site.name) - self.assertEqual(response.data['custom_fields'].get('magic_word'), CUSTOM_FIELD_VALUES[0][1]) - self.assertEqual(response.data['custom_fields'].get('magic_number'), CUSTOM_FIELD_VALUES[1][1]) - self.assertEqual(response.data['custom_fields'].get('is_magic'), CUSTOM_FIELD_VALUES[2][1]) - self.assertEqual(response.data['custom_fields'].get('magic_date'), CUSTOM_FIELD_VALUES[3][1]) - self.assertEqual(response.data['custom_fields'].get('magic_url'), CUSTOM_FIELD_VALUES[4][1]) - self.assertEqual(response.data['custom_fields'].get('magic_choice'), { - 'value': self.cf_select_choice1.pk, 'label': 'Foo' - }) - - def test_set_custom_field_text(self): - - data = { - 'name': 'Test Site 1', - 'slug': 'test-site-1', - 'custom_fields': { - 'magic_word': 'Foo bar baz', - } - } - - url = reverse('dcim-api:site-detail', kwargs={'pk': self.site.pk}) - response = self.client.put(url, data, format='json', **self.header) - - self.assertHttpStatus(response, status.HTTP_200_OK) - self.assertEqual(response.data['custom_fields'].get('magic_word'), data['custom_fields']['magic_word']) - cfv = self.site.custom_field_values.get(field=self.cf_text) - self.assertEqual(cfv.value, data['custom_fields']['magic_word']) - - def test_set_custom_field_integer(self): - + cls.cf_select = CustomField(type=CustomFieldTypeChoices.TYPE_SELECT, name='choice_field') + cls.cf_select.save() + cls.cf_select.obj_type.set([content_type]) + cls.cf_select_choice1 = CustomFieldChoice(field=cls.cf_select, value='Foo') + cls.cf_select_choice1.save() + cls.cf_select_choice2 = CustomFieldChoice(field=cls.cf_select, value='Bar') + cls.cf_select_choice2.save() + cls.cf_select_choice3 = CustomFieldChoice(field=cls.cf_select, value='Baz') + cls.cf_select_choice3.save() + + cls.cf_select.default = cls.cf_select_choice1.value + cls.cf_select.save() + + # def test_get_obj_without_custom_fields(self): + # + # url = reverse('dcim-api:site-detail', kwargs={'pk': self.site.pk}) + # response = self.client.get(url, **self.header) + # + # self.assertEqual(response.data['name'], self.site.name) + # self.assertEqual(response.data['custom_fields'], { + # 'text_field': None, + # 'number_field': None, + # 'boolean_field': None, + # 'date_field': None, + # 'url_field': None, + # 'choice_field': None, + # }) + + # def test_get_single_object_with_custom_fields(self): + # + # CUSTOM_FIELD_VALUES = [ + # (self.cf_text, 'Test string'), + # (self.cf_integer, 1234), + # (self.cf_boolean, True), + # (self.cf_date, date(2016, 6, 23)), + # (self.cf_url, 'http://example.com/'), + # (self.cf_select, self.cf_select_choice1.pk), + # ] + # for field, value in CUSTOM_FIELD_VALUES: + # cfv = CustomFieldValue(field=field, obj=self.site) + # cfv.value = value + # cfv.save() + # + # url = reverse('dcim-api:site-detail', kwargs={'pk': self.site.pk}) + # response = self.client.get(url, **self.header) + # + # self.assertEqual(response.data['name'], self.site.name) + # self.assertEqual(response.data['custom_fields'].get('text_field'), CUSTOM_FIELD_VALUES[0][1]) + # self.assertEqual(response.data['custom_fields'].get('number_field'), CUSTOM_FIELD_VALUES[1][1]) + # self.assertEqual(response.data['custom_fields'].get('boolean_field'), CUSTOM_FIELD_VALUES[2][1]) + # self.assertEqual(response.data['custom_fields'].get('date_field'), CUSTOM_FIELD_VALUES[3][1]) + # self.assertEqual(response.data['custom_fields'].get('url_field'), CUSTOM_FIELD_VALUES[4][1]) + # self.assertEqual(response.data['custom_fields'].get('choice_field'), { + # 'value': self.cf_select_choice1.pk, 'label': 'Foo' + # }) + + def test_create_single_object_with_defaults(self): + """ + Create a new site and check that it received the default custom field values. + """ data = { - 'name': 'Test Site 1', - 'slug': 'test-site-1', - 'custom_fields': { - 'magic_number': 42, - } + 'name': 'Site 2', + 'slug': 'site-2', } - url = reverse('dcim-api:site-detail', kwargs={'pk': self.site.pk}) - response = self.client.put(url, data, format='json', **self.header) - - self.assertHttpStatus(response, status.HTTP_200_OK) - self.assertEqual(response.data['custom_fields'].get('magic_number'), data['custom_fields']['magic_number']) - cfv = self.site.custom_field_values.get(field=self.cf_integer) - self.assertEqual(cfv.value, data['custom_fields']['magic_number']) - - def test_set_custom_field_boolean(self): + url = reverse('dcim-api:site-list') + response = self.client.post(url, data, format='json', **self.header) + self.assertHttpStatus(response, status.HTTP_201_CREATED) - data = { - 'name': 'Test Site 1', - 'slug': 'test-site-1', - 'custom_fields': { - 'is_magic': 0, - } + # Validate response data + response_cf = response.data['custom_fields'] + self.assertEqual(response_cf['text_field'], self.cf_text.default) + self.assertEqual(response_cf['number_field'], self.cf_integer.default) + self.assertEqual(response_cf['boolean_field'], self.cf_boolean.default) + self.assertEqual(response_cf['date_field'], self.cf_date.default) + self.assertEqual(response_cf['url_field'], self.cf_url.default) + self.assertEqual(response_cf['choice_field'], self.cf_select_choice1.pk) + + # Validate database data + site = Site.objects.get(pk=response.data['id']) + cfvs = { + cfv.field.name: cfv.value for cfv in site.custom_field_values.all() } - - url = reverse('dcim-api:site-detail', kwargs={'pk': self.site.pk}) - response = self.client.put(url, data, format='json', **self.header) - - self.assertHttpStatus(response, status.HTTP_200_OK) - self.assertEqual(response.data['custom_fields'].get('is_magic'), data['custom_fields']['is_magic']) - cfv = self.site.custom_field_values.get(field=self.cf_boolean) - self.assertEqual(cfv.value, data['custom_fields']['is_magic']) - - def test_set_custom_field_date(self): - + self.assertEqual(cfvs['text_field'], self.cf_text.default) + self.assertEqual(cfvs['number_field'], self.cf_integer.default) + self.assertEqual(cfvs['boolean_field'], self.cf_boolean.default) + self.assertEqual(str(cfvs['date_field']), self.cf_date.default) + self.assertEqual(cfvs['url_field'], self.cf_url.default) + self.assertEqual(cfvs['choice_field'].pk, self.cf_select_choice1.pk) + + def test_create_single_object_with_values(self): + """ + Create a single new site with a value for each type of custom field. + """ data = { - 'name': 'Test Site 1', - 'slug': 'test-site-1', + 'name': 'Site 2', + 'slug': 'site-2', 'custom_fields': { - 'magic_date': '2017-04-25', - } + 'text_field': 'bar', + 'number_field': 456, + 'boolean_field': True, + 'date_field': '2020-01-02', + 'url_field': 'http://example.com/2', + 'choice_field': self.cf_select_choice2.pk, + }, } - url = reverse('dcim-api:site-detail', kwargs={'pk': self.site.pk}) - response = self.client.put(url, data, format='json', **self.header) - - self.assertHttpStatus(response, status.HTTP_200_OK) - self.assertEqual(response.data['custom_fields'].get('magic_date'), data['custom_fields']['magic_date']) - cfv = self.site.custom_field_values.get(field=self.cf_date) - self.assertEqual(cfv.value.isoformat(), data['custom_fields']['magic_date']) - - def test_set_custom_field_url(self): + url = reverse('dcim-api:site-list') + response = self.client.post(url, data, format='json', **self.header) + self.assertHttpStatus(response, status.HTTP_201_CREATED) - data = { - 'name': 'Test Site 1', - 'slug': 'test-site-1', - 'custom_fields': { - 'magic_url': 'http://example.com/2/', - } + # Validate response data + response_cf = response.data['custom_fields'] + data_cf = data['custom_fields'] + self.assertEqual(response_cf['text_field'], data_cf['text_field']) + self.assertEqual(response_cf['number_field'], data_cf['number_field']) + self.assertEqual(response_cf['boolean_field'], data_cf['boolean_field']) + self.assertEqual(response_cf['date_field'], data_cf['date_field']) + self.assertEqual(response_cf['url_field'], data_cf['url_field']) + self.assertEqual(response_cf['choice_field'], data_cf['choice_field']) + + # Validate database data + site = Site.objects.get(pk=response.data['id']) + cfvs = { + cfv.field.name: cfv.value for cfv in site.custom_field_values.all() } + self.assertEqual(cfvs['text_field'], data_cf['text_field']) + self.assertEqual(cfvs['number_field'], data_cf['number_field']) + self.assertEqual(cfvs['boolean_field'], data_cf['boolean_field']) + self.assertEqual(str(cfvs['date_field']), data_cf['date_field']) + self.assertEqual(cfvs['url_field'], data_cf['url_field']) + self.assertEqual(cfvs['choice_field'].pk, data_cf['choice_field']) + + def test_create_multiple_objects_with_defaults(self): + """ + Create three news sites and check that each received the default custom field values. + """ + data = ( + { + 'name': 'Site 2', + 'slug': 'site-2', + }, + { + 'name': 'Site 3', + 'slug': 'site-3', + }, + { + 'name': 'Site 4', + 'slug': 'site-4', + }, + ) - url = reverse('dcim-api:site-detail', kwargs={'pk': self.site.pk}) - response = self.client.put(url, data, format='json', **self.header) - - self.assertHttpStatus(response, status.HTTP_200_OK) - self.assertEqual(response.data['custom_fields'].get('magic_url'), data['custom_fields']['magic_url']) - cfv = self.site.custom_field_values.get(field=self.cf_url) - self.assertEqual(cfv.value, data['custom_fields']['magic_url']) - - def test_set_custom_field_select(self): - - data = { - 'name': 'Test Site 1', - 'slug': 'test-site-1', - 'custom_fields': { - 'magic_choice': self.cf_select_choice2.pk, + url = reverse('dcim-api:site-list') + response = self.client.post(url, data, format='json', **self.header) + self.assertHttpStatus(response, status.HTTP_201_CREATED) + self.assertEqual(len(response.data), len(data)) + + for i, obj in enumerate(data): + + # Validate response data + response_cf = response.data[i]['custom_fields'] + self.assertEqual(response_cf['text_field'], self.cf_text.default) + self.assertEqual(response_cf['number_field'], self.cf_integer.default) + self.assertEqual(response_cf['boolean_field'], self.cf_boolean.default) + self.assertEqual(response_cf['date_field'], self.cf_date.default) + self.assertEqual(response_cf['url_field'], self.cf_url.default) + self.assertEqual(response_cf['choice_field'], self.cf_select_choice1.pk) + + # Validate database data + site = Site.objects.get(pk=response.data[i]['id']) + cfvs = { + cfv.field.name: cfv.value for cfv in site.custom_field_values.all() } - } - - url = reverse('dcim-api:site-detail', kwargs={'pk': self.site.pk}) - response = self.client.put(url, data, format='json', **self.header) - - self.assertHttpStatus(response, status.HTTP_200_OK) - self.assertEqual(response.data['custom_fields'].get('magic_choice'), data['custom_fields']['magic_choice']) - cfv = self.site.custom_field_values.get(field=self.cf_select) - self.assertEqual(cfv.value.pk, data['custom_fields']['magic_choice']) - - def test_set_custom_field_defaults(self): + self.assertEqual(cfvs['text_field'], self.cf_text.default) + self.assertEqual(cfvs['number_field'], self.cf_integer.default) + self.assertEqual(cfvs['boolean_field'], self.cf_boolean.default) + self.assertEqual(str(cfvs['date_field']), self.cf_date.default) + self.assertEqual(cfvs['url_field'], self.cf_url.default) + self.assertEqual(cfvs['choice_field'].pk, self.cf_select_choice1.pk) + + def test_create_multiple_objects_with_values(self): """ - Create a new object with no custom field data. Custom field values should be created using the custom fields' - default values. + Create a three new sites, each with custom fields defined. """ - CUSTOM_FIELD_DEFAULTS = { - 'magic_word': 'foobar', - 'magic_number': '123', - 'is_magic': 'true', - 'magic_date': '2019-12-13', - 'magic_url': 'http://example.com/', - 'magic_choice': self.cf_select_choice1.value, - } - - # Update CustomFields to set default values - for field_name, default_value in CUSTOM_FIELD_DEFAULTS.items(): - CustomField.objects.filter(name=field_name).update(default=default_value) - - data = { - 'name': 'Test Site X', - 'slug': 'test-site-x', + custom_field_data = { + 'text_field': 'bar', + 'number_field': 456, + 'boolean_field': True, + 'date_field': '2020-01-02', + 'url_field': 'http://example.com/2', + 'choice_field': self.cf_select_choice2.pk, } + data = ( + { + 'name': 'Site 2', + 'slug': 'site-2', + 'custom_fields': custom_field_data, + }, + { + 'name': 'Site 3', + 'slug': 'site-3', + 'custom_fields': custom_field_data, + }, + { + 'name': 'Site 4', + 'slug': 'site-4', + 'custom_fields': custom_field_data, + }, + ) url = reverse('dcim-api:site-list') response = self.client.post(url, data, format='json', **self.header) - self.assertHttpStatus(response, status.HTTP_201_CREATED) - self.assertEqual(response.data['custom_fields']['magic_word'], CUSTOM_FIELD_DEFAULTS['magic_word']) - self.assertEqual(response.data['custom_fields']['magic_number'], str(CUSTOM_FIELD_DEFAULTS['magic_number'])) - self.assertEqual(response.data['custom_fields']['is_magic'], bool(CUSTOM_FIELD_DEFAULTS['is_magic'])) - self.assertEqual(response.data['custom_fields']['magic_date'], CUSTOM_FIELD_DEFAULTS['magic_date']) - self.assertEqual(response.data['custom_fields']['magic_url'], CUSTOM_FIELD_DEFAULTS['magic_url']) - self.assertEqual(response.data['custom_fields']['magic_choice'], self.cf_select_choice1.pk) + self.assertEqual(len(response.data), len(data)) + + for i, obj in enumerate(data): + + # Validate response data + response_cf = response.data[i]['custom_fields'] + self.assertEqual(response_cf['text_field'], custom_field_data['text_field']) + self.assertEqual(response_cf['number_field'], custom_field_data['number_field']) + self.assertEqual(response_cf['boolean_field'], custom_field_data['boolean_field']) + self.assertEqual(response_cf['date_field'], custom_field_data['date_field']) + self.assertEqual(response_cf['url_field'], custom_field_data['url_field']) + self.assertEqual(response_cf['choice_field'], custom_field_data['choice_field']) + + # Validate database data + site = Site.objects.get(pk=response.data[i]['id']) + cfvs = { + cfv.field.name: cfv.value for cfv in site.custom_field_values.all() + } + self.assertEqual(cfvs['text_field'], custom_field_data['text_field']) + self.assertEqual(cfvs['number_field'], custom_field_data['number_field']) + self.assertEqual(cfvs['boolean_field'], custom_field_data['boolean_field']) + self.assertEqual(str(cfvs['date_field']), custom_field_data['date_field']) + self.assertEqual(cfvs['url_field'], custom_field_data['url_field']) + self.assertEqual(cfvs['choice_field'].pk, custom_field_data['choice_field']) class CustomFieldChoiceAPITest(APITestCase): From 57b6ac7cb1bc2136c34b9160a099f2f60c47821e Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Wed, 4 Mar 2020 09:53:49 -0500 Subject: [PATCH 16/30] Finish work on improved custom fields API tests --- netbox/extras/tests/test_customfields.py | 171 +++++++++++++++-------- 1 file changed, 113 insertions(+), 58 deletions(-) diff --git a/netbox/extras/tests/test_customfields.py b/netbox/extras/tests/test_customfields.py index 33ce9cca252..d7653243792 100644 --- a/netbox/extras/tests/test_customfields.py +++ b/netbox/extras/tests/test_customfields.py @@ -103,7 +103,6 @@ class CustomFieldAPITest(APITestCase): @classmethod def setUpTestData(cls): - content_type = ContentType.objects.get_for_model(Site) # Text custom field @@ -145,56 +144,70 @@ def setUpTestData(cls): cls.cf_select.default = cls.cf_select_choice1.value cls.cf_select.save() - # def test_get_obj_without_custom_fields(self): - # - # url = reverse('dcim-api:site-detail', kwargs={'pk': self.site.pk}) - # response = self.client.get(url, **self.header) - # - # self.assertEqual(response.data['name'], self.site.name) - # self.assertEqual(response.data['custom_fields'], { - # 'text_field': None, - # 'number_field': None, - # 'boolean_field': None, - # 'date_field': None, - # 'url_field': None, - # 'choice_field': None, - # }) - - # def test_get_single_object_with_custom_fields(self): - # - # CUSTOM_FIELD_VALUES = [ - # (self.cf_text, 'Test string'), - # (self.cf_integer, 1234), - # (self.cf_boolean, True), - # (self.cf_date, date(2016, 6, 23)), - # (self.cf_url, 'http://example.com/'), - # (self.cf_select, self.cf_select_choice1.pk), - # ] - # for field, value in CUSTOM_FIELD_VALUES: - # cfv = CustomFieldValue(field=field, obj=self.site) - # cfv.value = value - # cfv.save() - # - # url = reverse('dcim-api:site-detail', kwargs={'pk': self.site.pk}) - # response = self.client.get(url, **self.header) - # - # self.assertEqual(response.data['name'], self.site.name) - # self.assertEqual(response.data['custom_fields'].get('text_field'), CUSTOM_FIELD_VALUES[0][1]) - # self.assertEqual(response.data['custom_fields'].get('number_field'), CUSTOM_FIELD_VALUES[1][1]) - # self.assertEqual(response.data['custom_fields'].get('boolean_field'), CUSTOM_FIELD_VALUES[2][1]) - # self.assertEqual(response.data['custom_fields'].get('date_field'), CUSTOM_FIELD_VALUES[3][1]) - # self.assertEqual(response.data['custom_fields'].get('url_field'), CUSTOM_FIELD_VALUES[4][1]) - # self.assertEqual(response.data['custom_fields'].get('choice_field'), { - # 'value': self.cf_select_choice1.pk, 'label': 'Foo' - # }) + # Create some sites + cls.sites = ( + Site(name='Site 1', slug='site-1'), + Site(name='Site 2', slug='site-2'), + ) + Site.objects.bulk_create(cls.sites) + + # Assign custom field values for site 2 + site2_cfvs = { + cls.cf_text: 'bar', + cls.cf_integer: 456, + cls.cf_boolean: True, + cls.cf_date: '2020-01-02', + cls.cf_url: 'http://example.com/2', + cls.cf_select: cls.cf_select_choice2.pk, + } + for field, value in site2_cfvs.items(): + cfv = CustomFieldValue(field=field, obj=cls.sites[1]) + cfv.value = value + cfv.save() + + def test_get_single_object_without_custom_field_values(self): + """ + Validate that custom fields are present on an object even if it has no values defined. + """ + url = reverse('dcim-api:site-detail', kwargs={'pk': self.sites[0].pk}) + response = self.client.get(url, **self.header) + + self.assertEqual(response.data['name'], self.sites[0].name) + self.assertEqual(response.data['custom_fields'], { + 'text_field': None, + 'number_field': None, + 'boolean_field': None, + 'date_field': None, + 'url_field': None, + 'choice_field': None, + }) + + def test_get_single_object_with_custom_field_values(self): + """ + Validate that custom fields are present and correctly set for an object with values defined. + """ + site2_cfvs = { + cfv.field.name: cfv.value for cfv in self.sites[1].custom_field_values.all() + } + + url = reverse('dcim-api:site-detail', kwargs={'pk': self.sites[1].pk}) + response = self.client.get(url, **self.header) + + self.assertEqual(response.data['name'], self.sites[1].name) + self.assertEqual(response.data['custom_fields']['text_field'], site2_cfvs['text_field']) + self.assertEqual(response.data['custom_fields']['number_field'], site2_cfvs['number_field']) + self.assertEqual(response.data['custom_fields']['boolean_field'], site2_cfvs['boolean_field']) + self.assertEqual(response.data['custom_fields']['date_field'], site2_cfvs['date_field']) + self.assertEqual(response.data['custom_fields']['url_field'], site2_cfvs['url_field']) + self.assertEqual(response.data['custom_fields']['choice_field']['label'], self.cf_select_choice2.value) def test_create_single_object_with_defaults(self): """ - Create a new site and check that it received the default custom field values. + Create a new site with no specified custom field values and check that it received the default values. """ data = { - 'name': 'Site 2', - 'slug': 'site-2', + 'name': 'Site 3', + 'slug': 'site-3', } url = reverse('dcim-api:site-list') @@ -227,8 +240,8 @@ def test_create_single_object_with_values(self): Create a single new site with a value for each type of custom field. """ data = { - 'name': 'Site 2', - 'slug': 'site-2', + 'name': 'Site 3', + 'slug': 'site-3', 'custom_fields': { 'text_field': 'bar', 'number_field': 456, @@ -267,13 +280,10 @@ def test_create_single_object_with_values(self): def test_create_multiple_objects_with_defaults(self): """ - Create three news sites and check that each received the default custom field values. + Create three news sites with no specified custom field values and check that each received + the default custom field values. """ data = ( - { - 'name': 'Site 2', - 'slug': 'site-2', - }, { 'name': 'Site 3', 'slug': 'site-3', @@ -282,6 +292,10 @@ def test_create_multiple_objects_with_defaults(self): 'name': 'Site 4', 'slug': 'site-4', }, + { + 'name': 'Site 5', + 'slug': 'site-5', + }, ) url = reverse('dcim-api:site-list') @@ -325,11 +339,6 @@ def test_create_multiple_objects_with_values(self): 'choice_field': self.cf_select_choice2.pk, } data = ( - { - 'name': 'Site 2', - 'slug': 'site-2', - 'custom_fields': custom_field_data, - }, { 'name': 'Site 3', 'slug': 'site-3', @@ -340,6 +349,11 @@ def test_create_multiple_objects_with_values(self): 'slug': 'site-4', 'custom_fields': custom_field_data, }, + { + 'name': 'Site 5', + 'slug': 'site-5', + 'custom_fields': custom_field_data, + }, ) url = reverse('dcim-api:site-list') @@ -370,6 +384,47 @@ def test_create_multiple_objects_with_values(self): self.assertEqual(cfvs['url_field'], custom_field_data['url_field']) self.assertEqual(cfvs['choice_field'].pk, custom_field_data['choice_field']) + def test_update_single_object_with_values(self): + """ + Update an object with existing custom field values. Ensure that only the updated custom field values are + modified. + """ + site2_original_cfvs = { + cfv.field.name: cfv.value for cfv in self.sites[1].custom_field_values.all() + } + data = { + 'custom_fields': { + 'text_field': 'ABCD', + 'number_field': 1234, + }, + } + + url = reverse('dcim-api:site-detail', kwargs={'pk': self.sites[1].pk}) + response = self.client.patch(url, data, format='json', **self.header) + self.assertHttpStatus(response, status.HTTP_200_OK) + + # Validate response data + response_cf = response.data['custom_fields'] + data_cf = data['custom_fields'] + self.assertEqual(response_cf['text_field'], data_cf['text_field']) + self.assertEqual(response_cf['number_field'], data_cf['number_field']) + # TODO: Non-updated fields are missing from the response data + # self.assertEqual(response_cf['boolean_field'], site2_original_cfvs['boolean_field']) + # self.assertEqual(response_cf['date_field'], site2_original_cfvs['date_field']) + # self.assertEqual(response_cf['url_field'], site2_original_cfvs['url_field']) + # self.assertEqual(response_cf['choice_field']['label'], site2_original_cfvs['choice_field'].value) + + # Validate database data + site2_updated_cfvs = { + cfv.field.name: cfv.value for cfv in self.sites[1].custom_field_values.all() + } + self.assertEqual(site2_updated_cfvs['text_field'], data_cf['text_field']) + self.assertEqual(site2_updated_cfvs['number_field'], data_cf['number_field']) + self.assertEqual(site2_updated_cfvs['boolean_field'], site2_original_cfvs['boolean_field']) + self.assertEqual(site2_updated_cfvs['date_field'], site2_original_cfvs['date_field']) + self.assertEqual(site2_updated_cfvs['url_field'], site2_original_cfvs['url_field']) + self.assertEqual(site2_updated_cfvs['choice_field'], site2_original_cfvs['choice_field']) + class CustomFieldChoiceAPITest(APITestCase): def setUp(self): From 31e5d9ffe67b6d5edbb3a70e0100856b547e0354 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Wed, 4 Mar 2020 10:03:39 -0500 Subject: [PATCH 17/30] Changelog for #4298 --- docs/release-notes/version-2.7.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/release-notes/version-2.7.md b/docs/release-notes/version-2.7.md index cdc13953ce1..eea53489e0d 100644 --- a/docs/release-notes/version-2.7.md +++ b/docs/release-notes/version-2.7.md @@ -18,6 +18,7 @@ * [#4282](https://github.com/netbox-community/netbox/issues/4282) - Fix label on export button for device types * [#4285](https://github.com/netbox-community/netbox/issues/4285) - Include A/Z termination sites in provider circuits table * [#4295](https://github.com/netbox-community/netbox/issues/4295) - Fix assignment of parent LAG during interface bulk edit +* [#4298](https://github.com/netbox-community/netbox/issues/4298) - Fix bulk creation of objects with custom fields via REST API * [#4300](https://github.com/netbox-community/netbox/issues/4300) - Pass "commit" argument when executing scripts via REST API * [#4301](https://github.com/netbox-community/netbox/issues/4301) - Fix exception when deleting device type with components * [#4306](https://github.com/netbox-community/netbox/issues/4306) - Fix toggling of device images for all racks in elevations view From 10dee9b57b31328cc06240fdb0a39568d8c7e77b Mon Sep 17 00:00:00 2001 From: Jeremy Parker Date: Thu, 5 Mar 2020 02:07:58 +1100 Subject: [PATCH 18/30] Fix typo in caching.md (#4310) Co-authored-by: Jeremy Stretch --- docs/additional-features/caching.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/additional-features/caching.md b/docs/additional-features/caching.md index 32ac77cbf5c..0e6513602d7 100644 --- a/docs/additional-features/caching.md +++ b/docs/additional-features/caching.md @@ -3,7 +3,7 @@ To improve performance, NetBox supports caching for most object and list views. Caching is implemented using Redis, and [django-cacheops](https://github.com/Suor/django-cacheops) -Several management commands are avaliable for administrators to manaully invalidate cache entries in extenuating circumstances. +Several management commands are avaliable for administrators to manually invalidate cache entries in extenuating circumstances. To invalidate a specifc model instance (for example a Device with ID 34): ``` From 746bfd8bca93d601982eefa0566d0d330ffeb160 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Wed, 4 Mar 2020 10:29:22 -0500 Subject: [PATCH 19/30] Closes #4119: Extend upgrade script to clear expired user sessions --- docs/installation/upgrading.md | 3 +++ docs/release-notes/version-2.7.md | 1 + upgrade.sh | 5 +++++ 3 files changed, 9 insertions(+) diff --git a/docs/installation/upgrading.md b/docs/installation/upgrading.md index bf6497f6d62..79f94fd1724 100644 --- a/docs/installation/upgrading.md +++ b/docs/installation/upgrading.md @@ -74,6 +74,9 @@ This script: * Installs all required Python packages * Applies any database migrations that were included in the release * Collects all static files to be served by the HTTP service +* Deletes stale content types from the database +* Deletes all expired user sessions from the database +* Clears all cached data to prevent conflicts with the new release !!! note It's possible that the upgrade script will display a notice warning of unreflected database migrations: diff --git a/docs/release-notes/version-2.7.md b/docs/release-notes/version-2.7.md index eea53489e0d..b884aba822d 100644 --- a/docs/release-notes/version-2.7.md +++ b/docs/release-notes/version-2.7.md @@ -5,6 +5,7 @@ ## Enhancements * [#3949](https://github.com/netbox-community/netbox/issues/3949) - Revised the installation docs and upgrade script to employ a Python virtual environment +* [#4119](https://github.com/netbox-community/netbox/issues/4119) - Extend upgrade script to clear expired user sessions * [#4218](https://github.com/netbox-community/netbox/issues/4218) - Allow negative voltage for DC power feeds * [#4281](https://github.com/netbox-community/netbox/issues/4281) - Allow filtering device component list views by type * [#4284](https://github.com/netbox-community/netbox/issues/4284) - Add MRJ21 port and cable types diff --git a/upgrade.sh b/upgrade.sh index 977d9684d45..26fa5d1d633 100755 --- a/upgrade.sh +++ b/upgrade.sh @@ -48,6 +48,11 @@ COMMAND="python3 netbox/manage.py remove_stale_contenttypes --no-input" echo "Removing stale content types ($COMMAND)..." eval $COMMAND +# Delete any expired user sessions +COMMAND="python3 netbox/manage.py clearsessions" +echo "Removing expired user sessions ($COMMAND)..." +eval $COMMAND + # Clear all cached data COMMAND="python3 netbox/manage.py invalidate all" echo "Clearing cache data ($COMMAND)..." From 9e925202660a0b7a8c74fa8790cba0f417561aca Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Wed, 4 Mar 2020 11:53:53 -0500 Subject: [PATCH 20/30] Changelog for #4121 --- docs/release-notes/version-2.7.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/release-notes/version-2.7.md b/docs/release-notes/version-2.7.md index b884aba822d..72d88b74383 100644 --- a/docs/release-notes/version-2.7.md +++ b/docs/release-notes/version-2.7.md @@ -6,6 +6,7 @@ * [#3949](https://github.com/netbox-community/netbox/issues/3949) - Revised the installation docs and upgrade script to employ a Python virtual environment * [#4119](https://github.com/netbox-community/netbox/issues/4119) - Extend upgrade script to clear expired user sessions +* [#4121](https://github.com/netbox-community/netbox/issues/4121) - Add dynamic lookup expressions for all filters * [#4218](https://github.com/netbox-community/netbox/issues/4218) - Allow negative voltage for DC power feeds * [#4281](https://github.com/netbox-community/netbox/issues/4281) - Allow filtering device component list views by type * [#4284](https://github.com/netbox-community/netbox/issues/4284) - Add MRJ21 port and cable types From f7c6df6e6add402b5265a7252ec5eaef9bcb0b9f Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Wed, 4 Mar 2020 12:20:06 -0500 Subject: [PATCH 21/30] Improved verbosity of upgrade script --- upgrade.sh | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/upgrade.sh b/upgrade.sh index 26fa5d1d633..9fc54693e88 100755 --- a/upgrade.sh +++ b/upgrade.sh @@ -20,7 +20,8 @@ echo "Creating a new virtual environment at ${VIRTUALENV}..." eval $COMMAND || { echo "--------------------------------------------------------------------" echo "ERROR: Failed to create the virtual environment. Check that you have" - echo "the required system packages installed." + echo "the required system packages installed and the following path is" + echo "writable: ${VIRTUALENV}" echo "--------------------------------------------------------------------" exit 1 } @@ -62,9 +63,13 @@ if [ WARN_MISSING_VENV ]; then echo "--------------------------------------------------------------------" echo "WARNING: No existing virtual environment was detected. A new one has" echo "been created. Update your systemd service files to reflect the new" - echo "executables." - echo " Python: ${VIRTUALENV}/bin/python" - echo " gunicorn: ${VIRTUALENV}/bin/gunicorn" + echo "Python and gunicorn executables." + echo "" + echo "netbox.service ExecStart:" + echo " ${VIRTUALENV}/bin/gunicorn" + echo "" + echo "netbox-rq.service ExecStart:" + echo " ${VIRTUALENV}/bin/python" echo "--------------------------------------------------------------------" fi From e4e4af1b2d0856f41620b6f83f929aa9ab698f9a Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Wed, 4 Mar 2020 16:07:19 -0500 Subject: [PATCH 22/30] Delete obsolete old_requirements.txt --- old_requirements.txt | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 old_requirements.txt diff --git a/old_requirements.txt b/old_requirements.txt deleted file mode 100644 index b3f7b3c4790..00000000000 --- a/old_requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -django-rest-swagger -psycopg2 -pycrypto From e4cfeb1977fad15f939ee256fbf4a94bf285fc2b Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Wed, 4 Mar 2020 17:19:16 -0500 Subject: [PATCH 23/30] Add reminder to reload systemctl daemon if needed --- upgrade.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/upgrade.sh b/upgrade.sh index 9fc54693e88..17438248224 100755 --- a/upgrade.sh +++ b/upgrade.sh @@ -70,8 +70,11 @@ if [ WARN_MISSING_VENV ]; then echo "" echo "netbox-rq.service ExecStart:" echo " ${VIRTUALENV}/bin/python" + echo "" + echo "After modifying these files, reload the systemctl daemon:" + echo " > systemctl daemon-reload" echo "--------------------------------------------------------------------" fi echo "Upgrade complete! Don't forget to restart the NetBox services:" -echo " sudo systemctl restart netbox netbox-rq" +echo " > sudo systemctl restart netbox netbox-rq" From aefed0df894d7e5fc5026c4298957264374cd407 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Thu, 5 Mar 2020 08:51:03 -0500 Subject: [PATCH 24/30] Add user survey note to README --- README.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 5167c53c445..be69a9e520b 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ ![NetBox](docs/netbox_logo.svg "NetBox logo") +**The [2020 NetBox user survey](https://docs.google.com/forms/d/1OVZuC4kQ-6kJbVf0bDB6vgkL9H96xF6phvYzby23elk/edit) is open!** Your feedback helps guide the project's long-term development. + NetBox is an IP address management (IPAM) and data center infrastructure management (DCIM) tool. Initially conceived by the network engineering team at [DigitalOcean](https://www.digitalocean.com/), NetBox was developed specifically @@ -22,7 +24,7 @@ or join us in the #netbox Slack channel on [NetworkToCode](https://networktocode | **master** | [![Build Status](https://travis-ci.org/netbox-community/netbox.svg?branch=master)](https://travis-ci.com/netbox-community/netbox/) | | **develop** | [![Build Status](https://travis-ci.org/netbox-community/netbox.svg?branch=develop)](https://travis-ci.com/netbox-community/netbox/) | -## Screenshots +### Screenshots ![Screenshot of main page](docs/media/screenshot1.png "Main page") @@ -34,13 +36,13 @@ or join us in the #netbox Slack channel on [NetworkToCode](https://networktocode ![Screenshot of prefix hierarchy](docs/media/screenshot3.png "Prefix hierarchy") -# Installation +## Installation Please see [the documentation](http://netbox.readthedocs.io/en/stable/) for instructions on installing NetBox. To upgrade NetBox, please download the [latest release](https://github.com/netbox-community/netbox/releases) and run `upgrade.sh`. -# Providing Feedback +## Providing Feedback Feature requests and bug reports must be submitted as GiHub issues. (Please be sure to use the [appropriate template](https://github.com/netbox-community/netbox/issues/new/choose).) @@ -49,6 +51,6 @@ For general discussion, please consider joining our [mailing list](https://group If you are interested in contributing to the development of NetBox, please read our [contributing guide](CONTRIBUTING.md) prior to beginning any work. -# Related projects +## Related projects Please see [our wiki](https://github.com/netbox-community/netbox/wiki/Community-Contributions) for a list of relevant community projects. From 4136a5fd5e7bde6bb091a82de34d5661235bbfc8 Mon Sep 17 00:00:00 2001 From: Tomas Slusny Date: Fri, 31 Jan 2020 15:33:43 +0100 Subject: [PATCH 25/30] List choices for choice fields as enums Fixes #4062 Signed-off-by: Tomas Slusny --- netbox/utilities/custom_inspectors.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/netbox/utilities/custom_inspectors.py b/netbox/utilities/custom_inspectors.py index 68fe57d82bf..553d989820c 100644 --- a/netbox/utilities/custom_inspectors.py +++ b/netbox/utilities/custom_inspectors.py @@ -76,26 +76,28 @@ def field_to_swagger_object(self, field, swagger_object_type, use_references, ** SwaggerType, _ = self._get_partial_types(field, swagger_object_type, use_references, **kwargs) if isinstance(field, ChoiceField): - value_schema = openapi.Schema(type=openapi.TYPE_STRING) + choices = field._choices + choice_value = list(choices.keys()) + choice_label = list(choices.values()) + value_schema = openapi.Schema(type=openapi.TYPE_STRING, enum=choice_value) - choices = list(field._choices.keys()) - if set([None] + choices) == {None, True, False}: + if set([None] + choice_value) == {None, True, False}: # DeviceType.subdevice_role, Device.face and InterfaceConnection.connection_status all need to be # differentiated since they each have subtly different values in their choice keys. # - subdevice_role and connection_status are booleans, although subdevice_role includes None # - face is an integer set {0, 1} which is easily confused with {False, True} schema_type = openapi.TYPE_STRING - if all(type(x) == bool for x in [c for c in choices if c is not None]): + if all(type(x) == bool for x in [c for c in choice_value if c is not None]): schema_type = openapi.TYPE_BOOLEAN - value_schema = openapi.Schema(type=schema_type) + value_schema = openapi.Schema(type=schema_type, enum=choice_value) value_schema['x-nullable'] = True - if isinstance(choices[0], int): + if isinstance(choice_value[0], int): # Change value_schema for IPAddressFamilyChoices, RackWidthChoices - value_schema = openapi.Schema(type=openapi.TYPE_INTEGER) + value_schema = openapi.Schema(type=openapi.TYPE_INTEGER, enum=choice_value) schema = SwaggerType(type=openapi.TYPE_OBJECT, required=["label", "value"], properties={ - "label": openapi.Schema(type=openapi.TYPE_STRING), + "label": openapi.Schema(type=openapi.TYPE_STRING, enum=choice_label), "value": value_schema }) From 920078a738242a21b7c283d8b25569e1c3646743 Mon Sep 17 00:00:00 2001 From: kobayashi Date: Thu, 5 Mar 2020 23:52:33 -0500 Subject: [PATCH 26/30] set fix#4062 to release note --- docs/release-notes/version-2.7.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/release-notes/version-2.7.md b/docs/release-notes/version-2.7.md index 72d88b74383..581d395e13d 100644 --- a/docs/release-notes/version-2.7.md +++ b/docs/release-notes/version-2.7.md @@ -5,6 +5,7 @@ ## Enhancements * [#3949](https://github.com/netbox-community/netbox/issues/3949) - Revised the installation docs and upgrade script to employ a Python virtual environment +* [#4062](https://github.com/netbox-community/netbox/issues/4062) - Enumerate ChoiceField type and value in API * [#4119](https://github.com/netbox-community/netbox/issues/4119) - Extend upgrade script to clear expired user sessions * [#4121](https://github.com/netbox-community/netbox/issues/4121) - Add dynamic lookup expressions for all filters * [#4218](https://github.com/netbox-community/netbox/issues/4218) - Allow negative voltage for DC power feeds From 12bedac28a810ffbecc07e184fdf2320a2c1e09b Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Fri, 6 Mar 2020 10:26:12 -0500 Subject: [PATCH 27/30] Tweak upgrade script to exit immediately if any individual tasks fail --- upgrade.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/upgrade.sh b/upgrade.sh index 17438248224..bd1a06f6788 100755 --- a/upgrade.sh +++ b/upgrade.sh @@ -32,32 +32,32 @@ source "${VIRTUALENV}/bin/activate" # Install Python packages COMMAND="pip3 install -r requirements.txt" echo "Installing Python packages ($COMMAND)..." -eval $COMMAND +eval $COMMAND || exit 1 # Apply any database migrations COMMAND="python3 netbox/manage.py migrate" echo "Applying database migrations ($COMMAND)..." -eval $COMMAND +eval $COMMAND || exit 1 # Collect static files COMMAND="python3 netbox/manage.py collectstatic --no-input" echo "Collecting static files ($COMMAND)..." -eval $COMMAND +eval $COMMAND || exit 1 # Delete any stale content types COMMAND="python3 netbox/manage.py remove_stale_contenttypes --no-input" echo "Removing stale content types ($COMMAND)..." -eval $COMMAND +eval $COMMAND || exit 1 # Delete any expired user sessions COMMAND="python3 netbox/manage.py clearsessions" echo "Removing expired user sessions ($COMMAND)..." -eval $COMMAND +eval $COMMAND || exit 1 # Clear all cached data COMMAND="python3 netbox/manage.py invalidate all" echo "Clearing cache data ($COMMAND)..." -eval $COMMAND +eval $COMMAND || exit 1 if [ WARN_MISSING_VENV ]; then echo "--------------------------------------------------------------------" From afcd9b801f82c2604f6a2cd8ab6538dc3d4f44f5 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Fri, 6 Mar 2020 10:32:12 -0500 Subject: [PATCH 28/30] Delete squashed migration to avoid 'pending trigger events' exception under certain conditions --- ...squashed_0088_powerfeed_available_power.py | 839 ------------------ 1 file changed, 839 deletions(-) delete mode 100644 netbox/dcim/migrations/0071_device_components_add_description_squashed_0088_powerfeed_available_power.py diff --git a/netbox/dcim/migrations/0071_device_components_add_description_squashed_0088_powerfeed_available_power.py b/netbox/dcim/migrations/0071_device_components_add_description_squashed_0088_powerfeed_available_power.py deleted file mode 100644 index f74572c6f50..00000000000 --- a/netbox/dcim/migrations/0071_device_components_add_description_squashed_0088_powerfeed_available_power.py +++ /dev/null @@ -1,839 +0,0 @@ -import sys - -import django.core.validators -import django.db.models.deletion -import taggit.managers -from django.db import migrations, models - -SITE_STATUS_CHOICES = ( - (1, 'active'), - (2, 'planned'), - (4, 'retired'), -) - -RACK_TYPE_CHOICES = ( - (100, '2-post-frame'), - (200, '4-post-frame'), - (300, '4-post-cabinet'), - (1000, 'wall-frame'), - (1100, 'wall-cabinet'), -) - -RACK_STATUS_CHOICES = ( - (0, 'reserved'), - (1, 'available'), - (2, 'planned'), - (3, 'active'), - (4, 'deprecated'), -) - -RACK_DIMENSION_CHOICES = ( - (1000, 'mm'), - (2000, 'in'), -) - -SUBDEVICE_ROLE_CHOICES = ( - ('true', 'parent'), - ('false', 'child'), -) - -DEVICE_FACE_CHOICES = ( - (0, 'front'), - (1, 'rear'), -) - -DEVICE_STATUS_CHOICES = ( - (0, 'offline'), - (1, 'active'), - (2, 'planned'), - (3, 'staged'), - (4, 'failed'), - (5, 'inventory'), - (6, 'decommissioning'), -) - -INTERFACE_TYPE_CHOICES = ( - (0, 'virtual'), - (200, 'lag'), - (800, '100base-tx'), - (1000, '1000base-t'), - (1050, '1000base-x-gbic'), - (1100, '1000base-x-sfp'), - (1120, '2.5gbase-t'), - (1130, '5gbase-t'), - (1150, '10gbase-t'), - (1170, '10gbase-cx4'), - (1200, '10gbase-x-sfpp'), - (1300, '10gbase-x-xfp'), - (1310, '10gbase-x-xenpak'), - (1320, '10gbase-x-x2'), - (1350, '25gbase-x-sfp28'), - (1400, '40gbase-x-qsfpp'), - (1420, '50gbase-x-sfp28'), - (1500, '100gbase-x-cfp'), - (1510, '100gbase-x-cfp2'), - (1520, '100gbase-x-cfp4'), - (1550, '100gbase-x-cpak'), - (1600, '100gbase-x-qsfp28'), - (1650, '200gbase-x-cfp2'), - (1700, '200gbase-x-qsfp56'), - (1750, '400gbase-x-qsfpdd'), - (1800, '400gbase-x-osfp'), - (2600, 'ieee802.11a'), - (2610, 'ieee802.11g'), - (2620, 'ieee802.11n'), - (2630, 'ieee802.11ac'), - (2640, 'ieee802.11ad'), - (2810, 'gsm'), - (2820, 'cdma'), - (2830, 'lte'), - (6100, 'sonet-oc3'), - (6200, 'sonet-oc12'), - (6300, 'sonet-oc48'), - (6400, 'sonet-oc192'), - (6500, 'sonet-oc768'), - (6600, 'sonet-oc1920'), - (6700, 'sonet-oc3840'), - (3010, '1gfc-sfp'), - (3020, '2gfc-sfp'), - (3040, '4gfc-sfp'), - (3080, '8gfc-sfpp'), - (3160, '16gfc-sfpp'), - (3320, '32gfc-sfp28'), - (3400, '128gfc-sfp28'), - (7010, 'inifiband-sdr'), - (7020, 'inifiband-ddr'), - (7030, 'inifiband-qdr'), - (7040, 'inifiband-fdr10'), - (7050, 'inifiband-fdr'), - (7060, 'inifiband-edr'), - (7070, 'inifiband-hdr'), - (7080, 'inifiband-ndr'), - (7090, 'inifiband-xdr'), - (4000, 't1'), - (4010, 'e1'), - (4040, 't3'), - (4050, 'e3'), - (5000, 'cisco-stackwise'), - (5050, 'cisco-stackwise-plus'), - (5100, 'cisco-flexstack'), - (5150, 'cisco-flexstack-plus'), - (5200, 'juniper-vcp'), - (5300, 'extreme-summitstack'), - (5310, 'extreme-summitstack-128'), - (5320, 'extreme-summitstack-256'), - (5330, 'extreme-summitstack-512'), -) - -INTERFACE_MODE_CHOICES = ( - (100, 'access'), - (200, 'tagged'), - (300, 'tagged-all'), -) - -PORT_TYPE_CHOICES = ( - (1000, '8p8c'), - (1100, '110-punch'), - (1200, 'bnc'), - (2000, 'st'), - (2100, 'sc'), - (2110, 'sc-apc'), - (2200, 'fc'), - (2300, 'lc'), - (2310, 'lc-apc'), - (2400, 'mtrj'), - (2500, 'mpo'), - (2600, 'lsh'), - (2610, 'lsh-apc'), -) - -CABLE_TYPE_CHOICES = ( - (1300, 'cat3'), - (1500, 'cat5'), - (1510, 'cat5e'), - (1600, 'cat6'), - (1610, 'cat6a'), - (1700, 'cat7'), - (1800, 'dac-active'), - (1810, 'dac-passive'), - (1900, 'coaxial'), - (3000, 'mmf'), - (3010, 'mmf-om1'), - (3020, 'mmf-om2'), - (3030, 'mmf-om3'), - (3040, 'mmf-om4'), - (3500, 'smf'), - (3510, 'smf-os1'), - (3520, 'smf-os2'), - (3800, 'aoc'), - (5000, 'power'), -) - -CABLE_STATUS_CHOICES = ( - ('true', 'connected'), - ('false', 'planned'), -) - -CABLE_LENGTH_UNIT_CHOICES = ( - (1200, 'm'), - (1100, 'cm'), - (2100, 'ft'), - (2000, 'in'), -) - -POWERFEED_STATUS_CHOICES = ( - (0, 'offline'), - (1, 'active'), - (2, 'planned'), - (4, 'failed'), -) - -POWERFEED_TYPE_CHOICES = ( - (1, 'primary'), - (2, 'redundant'), -) - -POWERFEED_SUPPLY_CHOICES = ( - (1, 'ac'), - (2, 'dc'), -) - -POWERFEED_PHASE_CHOICES = ( - (1, 'single-phase'), - (3, 'three-phase'), -) - -POWEROUTLET_FEED_LEG_CHOICES_CHOICES = ( - (1, 'A'), - (2, 'B'), - (3, 'C'), -) - - -def cache_cable_devices(apps, schema_editor): - Cable = apps.get_model('dcim', 'Cable') - - if 'test' not in sys.argv: - print("\nUpdating cable device terminations...") - cable_count = Cable.objects.count() - - # Cache A/B termination devices on all existing Cables. Note that the custom save() method on Cable is not - # available during a migration, so we replicate its logic here. - for i, cable in enumerate(Cable.objects.all(), start=1): - - if not i % 1000 and 'test' not in sys.argv: - print("[{}/{}]".format(i, cable_count)) - - termination_a_model = apps.get_model(cable.termination_a_type.app_label, cable.termination_a_type.model) - termination_a_device = None - if hasattr(termination_a_model, 'device'): - termination_a = termination_a_model.objects.get(pk=cable.termination_a_id) - termination_a_device = termination_a.device - - termination_b_model = apps.get_model(cable.termination_b_type.app_label, cable.termination_b_type.model) - termination_b_device = None - if hasattr(termination_b_model, 'device'): - termination_b = termination_b_model.objects.get(pk=cable.termination_b_id) - termination_b_device = termination_b.device - - Cable.objects.filter(pk=cable.pk).update( - _termination_a_device=termination_a_device, - _termination_b_device=termination_b_device - ) - - -def site_status_to_slug(apps, schema_editor): - Site = apps.get_model('dcim', 'Site') - for id, slug in SITE_STATUS_CHOICES: - Site.objects.filter(status=str(id)).update(status=slug) - - -def rack_type_to_slug(apps, schema_editor): - Rack = apps.get_model('dcim', 'Rack') - for id, slug in RACK_TYPE_CHOICES: - Rack.objects.filter(type=str(id)).update(type=slug) - - -def rack_status_to_slug(apps, schema_editor): - Rack = apps.get_model('dcim', 'Rack') - for id, slug in RACK_STATUS_CHOICES: - Rack.objects.filter(status=str(id)).update(status=slug) - - -def rack_outer_unit_to_slug(apps, schema_editor): - Rack = apps.get_model('dcim', 'Rack') - for id, slug in RACK_DIMENSION_CHOICES: - Rack.objects.filter(status=str(id)).update(status=slug) - - -def devicetype_subdevicerole_to_slug(apps, schema_editor): - DeviceType = apps.get_model('dcim', 'DeviceType') - for boolean, slug in SUBDEVICE_ROLE_CHOICES: - DeviceType.objects.filter(subdevice_role=boolean).update(subdevice_role=slug) - - -def device_face_to_slug(apps, schema_editor): - Device = apps.get_model('dcim', 'Device') - for id, slug in DEVICE_FACE_CHOICES: - Device.objects.filter(face=str(id)).update(face=slug) - - -def device_status_to_slug(apps, schema_editor): - Device = apps.get_model('dcim', 'Device') - for id, slug in DEVICE_STATUS_CHOICES: - Device.objects.filter(status=str(id)).update(status=slug) - - -def interfacetemplate_type_to_slug(apps, schema_editor): - InterfaceTemplate = apps.get_model('dcim', 'InterfaceTemplate') - for id, slug in INTERFACE_TYPE_CHOICES: - InterfaceTemplate.objects.filter(type=id).update(type=slug) - - -def interface_type_to_slug(apps, schema_editor): - Interface = apps.get_model('dcim', 'Interface') - for id, slug in INTERFACE_TYPE_CHOICES: - Interface.objects.filter(type=id).update(type=slug) - - -def interface_mode_to_slug(apps, schema_editor): - Interface = apps.get_model('dcim', 'Interface') - for id, slug in INTERFACE_MODE_CHOICES: - Interface.objects.filter(mode=id).update(mode=slug) - - -def frontporttemplate_type_to_slug(apps, schema_editor): - FrontPortTemplate = apps.get_model('dcim', 'FrontPortTemplate') - for id, slug in PORT_TYPE_CHOICES: - FrontPortTemplate.objects.filter(type=id).update(type=slug) - - -def rearporttemplate_type_to_slug(apps, schema_editor): - RearPortTemplate = apps.get_model('dcim', 'RearPortTemplate') - for id, slug in PORT_TYPE_CHOICES: - RearPortTemplate.objects.filter(type=id).update(type=slug) - - -def frontport_type_to_slug(apps, schema_editor): - FrontPort = apps.get_model('dcim', 'FrontPort') - for id, slug in PORT_TYPE_CHOICES: - FrontPort.objects.filter(type=id).update(type=slug) - - -def rearport_type_to_slug(apps, schema_editor): - RearPort = apps.get_model('dcim', 'RearPort') - for id, slug in PORT_TYPE_CHOICES: - RearPort.objects.filter(type=id).update(type=slug) - - -def cable_type_to_slug(apps, schema_editor): - Cable = apps.get_model('dcim', 'Cable') - for id, slug in CABLE_TYPE_CHOICES: - Cable.objects.filter(type=id).update(type=slug) - - -def cable_status_to_slug(apps, schema_editor): - Cable = apps.get_model('dcim', 'Cable') - for bool_str, slug in CABLE_STATUS_CHOICES: - Cable.objects.filter(status=bool_str).update(status=slug) - - -def cable_length_unit_to_slug(apps, schema_editor): - Cable = apps.get_model('dcim', 'Cable') - for id, slug in CABLE_LENGTH_UNIT_CHOICES: - Cable.objects.filter(length_unit=id).update(length_unit=slug) - - -def powerfeed_status_to_slug(apps, schema_editor): - PowerFeed = apps.get_model('dcim', 'PowerFeed') - for id, slug in POWERFEED_STATUS_CHOICES: - PowerFeed.objects.filter(status=id).update(status=slug) - - -def powerfeed_type_to_slug(apps, schema_editor): - PowerFeed = apps.get_model('dcim', 'PowerFeed') - for id, slug in POWERFEED_TYPE_CHOICES: - PowerFeed.objects.filter(type=id).update(type=slug) - - -def powerfeed_supply_to_slug(apps, schema_editor): - PowerFeed = apps.get_model('dcim', 'PowerFeed') - for id, slug in POWERFEED_SUPPLY_CHOICES: - PowerFeed.objects.filter(supply=id).update(supply=slug) - - -def powerfeed_phase_to_slug(apps, schema_editor): - PowerFeed = apps.get_model('dcim', 'PowerFeed') - for id, slug in POWERFEED_PHASE_CHOICES: - PowerFeed.objects.filter(phase=id).update(phase=slug) - - -def poweroutlettemplate_feed_leg_to_slug(apps, schema_editor): - PowerOutletTemplate = apps.get_model('dcim', 'PowerOutletTemplate') - for id, slug in POWEROUTLET_FEED_LEG_CHOICES_CHOICES: - PowerOutletTemplate.objects.filter(feed_leg=id).update(feed_leg=slug) - - -def poweroutlet_feed_leg_to_slug(apps, schema_editor): - PowerOutlet = apps.get_model('dcim', 'PowerOutlet') - for id, slug in POWEROUTLET_FEED_LEG_CHOICES_CHOICES: - PowerOutlet.objects.filter(feed_leg=id).update(feed_leg=slug) - - -class Migration(migrations.Migration): - - replaces = [('dcim', '0071_device_components_add_description'), ('dcim', '0072_powerfeeds'), ('dcim', '0073_interface_form_factor_to_type'), ('dcim', '0074_increase_field_length_platform_name_slug'), ('dcim', '0075_cable_devices'), ('dcim', '0076_console_port_types'), ('dcim', '0077_power_types'), ('dcim', '0078_3569_site_fields'), ('dcim', '0079_3569_rack_fields'), ('dcim', '0080_3569_devicetype_fields'), ('dcim', '0081_3569_device_fields'), ('dcim', '0082_3569_interface_fields'), ('dcim', '0082_3569_port_fields'), ('dcim', '0083_3569_cable_fields'), ('dcim', '0084_3569_powerfeed_fields'), ('dcim', '0085_3569_poweroutlet_fields'), ('dcim', '0086_device_name_nonunique'), ('dcim', '0087_role_descriptions'), ('dcim', '0088_powerfeed_available_power')] - - dependencies = [ - ('dcim', '0070_custom_tag_models'), - ('extras', '0021_add_color_comments_changelog_to_tag'), - ('tenancy', '0006_custom_tag_models'), - ] - - operations = [ - migrations.AddField( - model_name='consoleport', - name='description', - field=models.CharField(blank=True, max_length=100), - ), - migrations.AddField( - model_name='consoleserverport', - name='description', - field=models.CharField(blank=True, max_length=100), - ), - migrations.AddField( - model_name='devicebay', - name='description', - field=models.CharField(blank=True, max_length=100), - ), - migrations.AddField( - model_name='poweroutlet', - name='description', - field=models.CharField(blank=True, max_length=100), - ), - migrations.AddField( - model_name='powerport', - name='description', - field=models.CharField(blank=True, max_length=100), - ), - migrations.CreateModel( - name='PowerPanel', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False)), - ('created', models.DateField(auto_now_add=True, null=True)), - ('last_updated', models.DateTimeField(auto_now=True, null=True)), - ('name', models.CharField(max_length=50)), - ('rack_group', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, to='dcim.RackGroup')), - ('site', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='dcim.Site')), - ], - options={ - 'ordering': ['site', 'name'], - 'unique_together': {('site', 'name')}, - }, - ), - migrations.CreateModel( - name='PowerFeed', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False)), - ('created', models.DateField(auto_now_add=True, null=True)), - ('last_updated', models.DateTimeField(auto_now=True, null=True)), - ('name', models.CharField(max_length=50)), - ('status', models.PositiveSmallIntegerField(default=1)), - ('type', models.PositiveSmallIntegerField(default=1)), - ('supply', models.PositiveSmallIntegerField(default=1)), - ('phase', models.PositiveSmallIntegerField(default=1)), - ('voltage', models.PositiveSmallIntegerField(default=120, validators=[django.core.validators.MinValueValidator(1)])), - ('amperage', models.PositiveSmallIntegerField(default=20, validators=[django.core.validators.MinValueValidator(1)])), - ('max_utilization', models.PositiveSmallIntegerField(default=80, validators=[django.core.validators.MinValueValidator(1), django.core.validators.MaxValueValidator(100)])), - ('available_power', models.PositiveSmallIntegerField(default=0, editable=False)), - ('comments', models.TextField(blank=True)), - ('cable', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='dcim.Cable')), - ('power_panel', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='powerfeeds', to='dcim.PowerPanel')), - ('rack', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, to='dcim.Rack')), - ('tags', taggit.managers.TaggableManager(help_text='A comma-separated list of tags.', through='extras.TaggedItem', to='extras.Tag', verbose_name='Tags')), - ('connected_endpoint', models.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='dcim.PowerPort')), - ('connection_status', models.NullBooleanField()), - ], - options={ - 'ordering': ['power_panel', 'name'], - 'unique_together': {('power_panel', 'name')}, - }, - ), - migrations.RenameField( - model_name='powerport', - old_name='connected_endpoint', - new_name='_connected_poweroutlet', - ), - migrations.AddField( - model_name='powerport', - name='_connected_powerfeed', - field=models.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='dcim.PowerFeed'), - ), - migrations.AddField( - model_name='powerport', - name='allocated_draw', - field=models.PositiveSmallIntegerField(blank=True, null=True, validators=[django.core.validators.MinValueValidator(1)]), - ), - migrations.AddField( - model_name='powerport', - name='maximum_draw', - field=models.PositiveSmallIntegerField(blank=True, null=True, validators=[django.core.validators.MinValueValidator(1)]), - ), - migrations.AddField( - model_name='powerporttemplate', - name='allocated_draw', - field=models.PositiveSmallIntegerField(blank=True, null=True, validators=[django.core.validators.MinValueValidator(1)]), - ), - migrations.AddField( - model_name='powerporttemplate', - name='maximum_draw', - field=models.PositiveSmallIntegerField(blank=True, null=True, validators=[django.core.validators.MinValueValidator(1)]), - ), - migrations.AddField( - model_name='poweroutlet', - name='feed_leg', - field=models.PositiveSmallIntegerField(blank=True, null=True), - ), - migrations.AddField( - model_name='poweroutlet', - name='power_port', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='poweroutlets', to='dcim.PowerPort'), - ), - migrations.AddField( - model_name='poweroutlettemplate', - name='feed_leg', - field=models.PositiveSmallIntegerField(blank=True, null=True), - ), - migrations.AddField( - model_name='poweroutlettemplate', - name='power_port', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='poweroutlet_templates', to='dcim.PowerPortTemplate'), - ), - migrations.RenameField( - model_name='interface', - old_name='form_factor', - new_name='type', - ), - migrations.RenameField( - model_name='interfacetemplate', - old_name='form_factor', - new_name='type', - ), - migrations.AlterField( - model_name='platform', - name='name', - field=models.CharField(max_length=100, unique=True), - ), - migrations.AlterField( - model_name='platform', - name='slug', - field=models.SlugField(max_length=100, unique=True), - ), - migrations.AddField( - model_name='cable', - name='_termination_a_device', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='dcim.Device'), - ), - migrations.AddField( - model_name='cable', - name='_termination_b_device', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='dcim.Device'), - ), - migrations.RunPython( - code=cache_cable_devices, - reverse_code=django.db.migrations.operations.special.RunPython.noop, - ), - migrations.AddField( - model_name='consoleport', - name='type', - field=models.CharField(blank=True, max_length=50), - ), - migrations.AddField( - model_name='consoleporttemplate', - name='type', - field=models.CharField(blank=True, max_length=50), - ), - migrations.AddField( - model_name='consoleserverport', - name='type', - field=models.CharField(blank=True, max_length=50), - ), - migrations.AddField( - model_name='consoleserverporttemplate', - name='type', - field=models.CharField(blank=True, max_length=50), - ), - migrations.AddField( - model_name='poweroutlet', - name='type', - field=models.CharField(blank=True, max_length=50), - ), - migrations.AddField( - model_name='poweroutlettemplate', - name='type', - field=models.CharField(blank=True, max_length=50), - ), - migrations.AddField( - model_name='powerport', - name='type', - field=models.CharField(blank=True, max_length=50), - ), - migrations.AddField( - model_name='powerporttemplate', - name='type', - field=models.CharField(blank=True, max_length=50), - ), - migrations.AlterField( - model_name='site', - name='status', - field=models.CharField(default='active', max_length=50), - ), - migrations.RunPython( - code=site_status_to_slug, - ), - migrations.AlterField( - model_name='rack', - name='type', - field=models.CharField(blank=True, default='', max_length=50), - ), - migrations.RunPython( - code=rack_type_to_slug, - ), - migrations.AlterField( - model_name='rack', - name='type', - field=models.CharField(blank=True, max_length=50), - ), - migrations.AlterField( - model_name='rack', - name='status', - field=models.CharField(default='active', max_length=50), - ), - migrations.RunPython( - code=rack_status_to_slug, - ), - migrations.AlterField( - model_name='rack', - name='outer_unit', - field=models.CharField(blank=True, default='', max_length=50), - ), - migrations.RunPython( - code=rack_outer_unit_to_slug, - ), - migrations.AlterField( - model_name='rack', - name='outer_unit', - field=models.CharField(blank=True, max_length=50), - ), - migrations.AlterField( - model_name='devicetype', - name='subdevice_role', - field=models.CharField(blank=True, default='', max_length=50), - ), - migrations.RunPython( - code=devicetype_subdevicerole_to_slug, - ), - migrations.AlterField( - model_name='devicetype', - name='subdevice_role', - field=models.CharField(blank=True, max_length=50), - ), - migrations.AlterField( - model_name='device', - name='face', - field=models.CharField(blank=True, default='', max_length=50), - ), - migrations.RunPython( - code=device_face_to_slug, - ), - migrations.AlterField( - model_name='device', - name='face', - field=models.CharField(blank=True, max_length=50), - ), - migrations.AlterField( - model_name='device', - name='status', - field=models.CharField(default='active', max_length=50), - ), - migrations.RunPython( - code=device_status_to_slug, - ), - migrations.AlterField( - model_name='interfacetemplate', - name='type', - field=models.CharField(max_length=50), - ), - migrations.RunPython( - code=interfacetemplate_type_to_slug, - ), - migrations.AlterField( - model_name='interface', - name='type', - field=models.CharField(max_length=50), - ), - migrations.RunPython( - code=interface_type_to_slug, - ), - migrations.AlterField( - model_name='interface', - name='mode', - field=models.CharField(blank=True, default='', max_length=50), - ), - migrations.RunPython( - code=interface_mode_to_slug, - ), - migrations.AlterField( - model_name='interface', - name='mode', - field=models.CharField(blank=True, max_length=50), - ), - migrations.AlterField( - model_name='frontporttemplate', - name='type', - field=models.CharField(max_length=50), - ), - migrations.RunPython( - code=frontporttemplate_type_to_slug, - ), - migrations.AlterField( - model_name='rearporttemplate', - name='type', - field=models.CharField(max_length=50), - ), - migrations.RunPython( - code=rearporttemplate_type_to_slug, - ), - migrations.AlterField( - model_name='frontport', - name='type', - field=models.CharField(max_length=50), - ), - migrations.RunPython( - code=frontport_type_to_slug, - ), - migrations.AlterField( - model_name='rearport', - name='type', - field=models.CharField(max_length=50), - ), - migrations.RunPython( - code=rearport_type_to_slug, - ), - migrations.AlterField( - model_name='cable', - name='type', - field=models.CharField(blank=True, default='', max_length=50), - ), - migrations.RunPython( - code=cable_type_to_slug, - ), - migrations.AlterField( - model_name='cable', - name='type', - field=models.CharField(blank=True, max_length=50), - ), - migrations.AlterField( - model_name='cable', - name='status', - field=models.CharField(default='connected', max_length=50), - ), - migrations.RunPython( - code=cable_status_to_slug, - ), - migrations.AlterField( - model_name='cable', - name='length_unit', - field=models.CharField(blank=True, default='', max_length=50), - ), - migrations.RunPython( - code=cable_length_unit_to_slug, - ), - migrations.AlterField( - model_name='cable', - name='length_unit', - field=models.CharField(blank=True, max_length=50), - ), - migrations.AlterField( - model_name='powerfeed', - name='status', - field=models.CharField(default='active', max_length=50), - ), - migrations.RunPython( - code=powerfeed_status_to_slug, - ), - migrations.AlterField( - model_name='powerfeed', - name='type', - field=models.CharField(default='primary', max_length=50), - ), - migrations.RunPython( - code=powerfeed_type_to_slug, - ), - migrations.AlterField( - model_name='powerfeed', - name='supply', - field=models.CharField(default='ac', max_length=50), - ), - migrations.RunPython( - code=powerfeed_supply_to_slug, - ), - migrations.AlterField( - model_name='powerfeed', - name='phase', - field=models.CharField(default='single-phase', max_length=50), - ), - migrations.RunPython( - code=powerfeed_phase_to_slug, - ), - migrations.AlterField( - model_name='poweroutlettemplate', - name='feed_leg', - field=models.CharField(blank=True, default='', max_length=50), - ), - migrations.RunPython( - code=poweroutlettemplate_feed_leg_to_slug, - ), - migrations.AlterField( - model_name='poweroutlettemplate', - name='feed_leg', - field=models.CharField(blank=True, max_length=50), - ), - migrations.AlterField( - model_name='poweroutlet', - name='feed_leg', - field=models.CharField(blank=True, default='', max_length=50), - ), - migrations.RunPython( - code=poweroutlet_feed_leg_to_slug, - ), - migrations.AlterField( - model_name='poweroutlet', - name='feed_leg', - field=models.CharField(blank=True, max_length=50), - ), - migrations.AlterField( - model_name='device', - name='name', - field=models.CharField(blank=True, max_length=64, null=True), - ), - migrations.AlterUniqueTogether( - name='device', - unique_together={('rack', 'position', 'face'), ('site', 'tenant', 'name'), ('virtual_chassis', 'vc_position')}, - ), - migrations.AddField( - model_name='devicerole', - name='description', - field=models.CharField(blank=True, max_length=100), - ), - migrations.AddField( - model_name='rackrole', - name='description', - field=models.CharField(blank=True, max_length=100), - ), - migrations.AlterField( - model_name='powerfeed', - name='available_power', - field=models.PositiveIntegerField(default=0, editable=False), - ), - ] From 09298dab7ad58bc3d75d5de5889bec547d885571 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Fri, 6 Mar 2020 11:17:17 -0500 Subject: [PATCH 29/30] Release v2.7.9 --- docs/release-notes/version-2.7.md | 2 +- netbox/netbox/settings.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/release-notes/version-2.7.md b/docs/release-notes/version-2.7.md index 581d395e13d..07457fba81c 100644 --- a/docs/release-notes/version-2.7.md +++ b/docs/release-notes/version-2.7.md @@ -1,4 +1,4 @@ -# v2.7.9 (FUTURE) +# v2.7.9 (2020-03-06) **Note:** This release will deploy a Python virtual environment on upgrade in the `venv/` directory. This will require modifying the paths to your Python and gunicorn executables in the systemd service files. For more detail, please see the [upgrade instructions](https://netbox.readthedocs.io/en/stable/installation/upgrading/). diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index 7f4f44b1aeb..19e124abc8d 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -12,7 +12,7 @@ # Environment setup # -VERSION = '2.7.9-dev' +VERSION = '2.7.9' # Hostname HOSTNAME = platform.node() From 5950bedfae7c06d8f38c4925a2a4fb737099b487 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Fri, 6 Mar 2020 11:26:59 -0500 Subject: [PATCH 30/30] Post-release version bump --- netbox/netbox/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index 19e124abc8d..99ef6ee5e8f 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -12,7 +12,7 @@ # Environment setup # -VERSION = '2.7.9' +VERSION = '2.7.10-dev' # Hostname HOSTNAME = platform.node()