Skip to content

Commit

Permalink
Initial work on #13381
Browse files Browse the repository at this point in the history
  • Loading branch information
jeremystretch committed Oct 20, 2023
1 parent 3f40ee5 commit 21d17d5
Show file tree
Hide file tree
Showing 22 changed files with 227 additions and 111 deletions.
24 changes: 24 additions & 0 deletions docs/plugins/development/data-backends.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
# Data Backends

[Data sources](../../models/core/datasource.md) can be defined to reference data which exists on systems of record outside NetBox, such as a git repository or Amazon S3 bucket. Plugins can register their own backend classes to introduce support for additional resource types. This is done by subclassing NetBox's `DataBackend` class.

```python
# data_backends.py
from netbox.data_backends import DataBackend

class MyDataBackend(DataBackend):
name = 'mybackend'
label = 'My Backend'
...
```

To register one or more data backends with NetBox, define a list named `backends` at the end of this file:

```python
backends = [MyDataBackend]
```

!!! tip
The path to the list of search indexes can be modified by setting `data_backends` in the PluginConfig instance.

::: core.data_backends.DataBackend
1 change: 1 addition & 0 deletions docs/plugins/development/index.md
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,7 @@ NetBox looks for the `config` variable within a plugin's `__init__.py` to load i
| `middleware` | A list of middleware classes to append after NetBox's build-in middleware |
| `queues` | A list of custom background task queues to create |
| `search_extensions` | The dotted path to the list of search index classes (default: `search.indexes`) |
| `data_backends` | The dotted path to the list of data source backend classes (default: `data_backends.backends`) |
| `template_extensions` | The dotted path to the list of template extension classes (default: `template_content.template_extensions`) |
| `menu_items` | The dotted path to the list of menu items provided by the plugin (default: `navigation.menu_items`) |
| `graphql_schema` | The dotted path to the plugin's GraphQL schema class, if any (default: `graphql.schema`) |
Expand Down
1 change: 1 addition & 0 deletions mkdocs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,7 @@ nav:
- Forms: 'plugins/development/forms.md'
- Filters & Filter Sets: 'plugins/development/filtersets.md'
- Search: 'plugins/development/search.md'
- Data Backends: 'plugins/development/data-backends.md'
- REST API: 'plugins/development/rest-api.md'
- GraphQL API: 'plugins/development/graphql-api.md'
- Background Tasks: 'plugins/development/background-tasks.md'
Expand Down
12 changes: 11 additions & 1 deletion netbox/core/api/serializers.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,10 @@
from django.core.exceptions import ValidationError
from django.utils.translation import gettext_lazy as _
from rest_framework import serializers

from core.choices import *
from core.models import *
from core.utils import get_data_backend_choices
from netbox.api.fields import ChoiceField, ContentTypeField
from netbox.api.serializers import BaseModelSerializer, NetBoxModelSerializer
from users.api.nested_serializers import NestedUserSerializer
Expand All @@ -19,7 +22,7 @@ class DataSourceSerializer(NetBoxModelSerializer):
view_name='core-api:datasource-detail'
)
type = ChoiceField(
choices=DataSourceTypeChoices
choices=get_data_backend_choices()
)
status = ChoiceField(
choices=DataSourceStatusChoices,
Expand All @@ -38,6 +41,13 @@ class Meta:
'parameters', 'ignore_rules', 'created', 'last_updated', 'file_count',
]

def clean(self):

if self.type and self.type not in get_data_backend_choices():
raise ValidationError({
'type': _("Unknown backend type: {type}".format(type=self.type))
})


class DataFileSerializer(NetBoxModelSerializer):
url = serializers.HyperlinkedIdentityField(
Expand Down
12 changes: 0 additions & 12 deletions netbox/core/choices.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,18 +7,6 @@
# Data sources
#

class DataSourceTypeChoices(ChoiceSet):
LOCAL = 'local'
GIT = 'git'
AMAZON_S3 = 'amazon-s3'

CHOICES = (
(LOCAL, _('Local'), 'gray'),
(GIT, 'Git', 'blue'),
(AMAZON_S3, 'Amazon S3', 'blue'),
)


class DataSourceStatusChoices(ChoiceSet):
NEW = 'new'
QUEUED = 'queued'
Expand Down
59 changes: 13 additions & 46 deletions netbox/core/data_backends.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,61 +10,24 @@
from django.conf import settings
from django.utils.translation import gettext as _

from netbox.registry import registry
from .choices import DataSourceTypeChoices
from core.utils import register_data_backend
from netbox.data_backends import DataBackend
from .exceptions import SyncError

__all__ = (
'LocalBackend',
'GitBackend',
'LocalBackend',
'S3Backend',
)

logger = logging.getLogger('netbox.data_backends')


def register_backend(name):
"""
Decorator for registering a DataBackend class.
"""

def _wrapper(cls):
registry['data_backends'][name] = cls
return cls

return _wrapper


class DataBackend:
parameters = {}
sensitive_parameters = []

# Prevent Django's template engine from calling the backend
# class when referenced via DataSource.backend_class
do_not_call_in_templates = True

def __init__(self, url, **kwargs):
self.url = url
self.params = kwargs
self.config = self.init_config()

def init_config(self):
"""
Hook to initialize the instance's configuration.
"""
return

@property
def url_scheme(self):
return urlparse(self.url).scheme.lower()

@contextmanager
def fetch(self):
raise NotImplemented()


@register_backend(DataSourceTypeChoices.LOCAL)
@register_data_backend()
class LocalBackend(DataBackend):
name = 'local'
label = _('Local')
is_local = True

@contextmanager
def fetch(self):
Expand All @@ -74,8 +37,10 @@ def fetch(self):
yield local_path


@register_backend(DataSourceTypeChoices.GIT)
@register_data_backend()
class GitBackend(DataBackend):
name = 'git'
label = 'Git'
parameters = {
'username': forms.CharField(
required=False,
Expand Down Expand Up @@ -144,8 +109,10 @@ def fetch(self):
local_path.cleanup()


@register_backend(DataSourceTypeChoices.AMAZON_S3)
@register_data_backend()
class S3Backend(DataBackend):
name = 'amazon-s3'
label = 'Amazon S3'
parameters = {
'aws_access_key_id': forms.CharField(
label=_('AWS access key ID'),
Expand Down
3 changes: 2 additions & 1 deletion netbox/core/filtersets.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
from netbox.filtersets import BaseFilterSet, ChangeLoggedModelFilterSet, NetBoxModelFilterSet
from .choices import *
from .models import *
from .utils import get_data_backend_choices

__all__ = (
'DataFileFilterSet',
Expand All @@ -16,7 +17,7 @@

class DataSourceFilterSet(NetBoxModelFilterSet):
type = django_filters.MultipleChoiceFilter(
choices=DataSourceTypeChoices,
choices=get_data_backend_choices,
null_value=None
)
status = django_filters.MultipleChoiceFilter(
Expand Down
6 changes: 3 additions & 3 deletions netbox/core/forms/bulk_edit.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,9 @@
from django import forms
from django.utils.translation import gettext_lazy as _

from core.choices import DataSourceTypeChoices
from core.models import *
from core.utils import get_data_backend_choices
from netbox.forms import NetBoxModelBulkEditForm
from utilities.forms import add_blank_choice
from utilities.forms.fields import CommentField
from utilities.forms.widgets import BulkEditNullBooleanSelect

Expand All @@ -16,7 +15,8 @@
class DataSourceBulkEditForm(NetBoxModelBulkEditForm):
type = forms.ChoiceField(
label=_('Type'),
choices=add_blank_choice(DataSourceTypeChoices),
# TODO: Field value should be empty on init (needs add_blank_choice())
choices=get_data_backend_choices,
required=False,
initial=''
)
Expand Down
3 changes: 2 additions & 1 deletion netbox/core/forms/filtersets.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

from core.choices import *
from core.models import *
from core.utils import get_data_backend_choices
from extras.forms.mixins import SavedFiltersMixin
from extras.utils import FeatureQuery
from netbox.forms import NetBoxModelFilterSetForm
Expand All @@ -27,7 +28,7 @@ class DataSourceFilterForm(NetBoxModelFilterSetForm):
)
type = forms.MultipleChoiceField(
label=_('Type'),
choices=DataSourceTypeChoices,
choices=get_data_backend_choices,
required=False
)
status = forms.MultipleChoiceField(
Expand Down
19 changes: 12 additions & 7 deletions netbox/core/forms/model_forms.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

from core.forms.mixins import SyncedDataMixin
from core.models import *
from core.utils import get_data_backend_choices
from netbox.forms import NetBoxModelForm
from netbox.registry import registry
from utilities.forms import get_field_value
Expand All @@ -18,6 +19,10 @@


class DataSourceForm(NetBoxModelForm):
type = forms.ChoiceField(
choices=get_data_backend_choices,
widget=HTMXSelect()
)
comments = CommentField()

class Meta:
Expand All @@ -26,7 +31,6 @@ class Meta:
'name', 'type', 'source_url', 'enabled', 'description', 'comments', 'ignore_rules', 'tags',
]
widgets = {
'type': HTMXSelect(),
'ignore_rules': forms.Textarea(
attrs={
'rows': 5,
Expand Down Expand Up @@ -56,12 +60,13 @@ def __init__(self, *args, **kwargs):

# Add backend-specific form fields
self.backend_fields = []
for name, form_field in backend.parameters.items():
field_name = f'backend_{name}'
self.backend_fields.append(field_name)
self.fields[field_name] = copy.copy(form_field)
if self.instance and self.instance.parameters:
self.fields[field_name].initial = self.instance.parameters.get(name)
if backend:
for name, form_field in backend.parameters.items():
field_name = f'backend_{name}'
self.backend_fields.append(field_name)
self.fields[field_name] = copy.copy(form_field)
if self.instance and self.instance.parameters:
self.fields[field_name].initial = self.instance.parameters.get(name)

def save(self, *args, **kwargs):

Expand Down
18 changes: 18 additions & 0 deletions netbox/core/migrations/0006_datasource_type_remove_choices.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
# Generated by Django 4.2.6 on 2023-10-20 17:47

from django.db import migrations, models


class Migration(migrations.Migration):

dependencies = [
('core', '0005_job_created_auto_now'),
]

operations = [
migrations.AlterField(
model_name='datasource',
name='type',
field=models.CharField(max_length=50),
),
]
15 changes: 5 additions & 10 deletions netbox/core/models/data.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,9 +45,7 @@ class DataSource(JobsMixin, PrimaryModel):
)
type = models.CharField(
verbose_name=_('type'),
max_length=50,
choices=DataSourceTypeChoices,
default=DataSourceTypeChoices.LOCAL
max_length=50
)
source_url = models.CharField(
max_length=200,
Expand Down Expand Up @@ -96,8 +94,9 @@ def get_absolute_url(self):
def docs_url(self):
return f'{settings.STATIC_URL}docs/models/{self._meta.app_label}/{self._meta.model_name}/'

def get_type_color(self):
return DataSourceTypeChoices.colors.get(self.type)
def get_type_display(self):
if backend := registry['data_backends'].get(self.type):
return backend.label

def get_status_color(self):
return DataSourceStatusChoices.colors.get(self.status)
Expand All @@ -110,10 +109,6 @@ def url_scheme(self):
def backend_class(self):
return registry['data_backends'].get(self.type)

@property
def is_local(self):
return self.type == DataSourceTypeChoices.LOCAL

@property
def ready_for_sync(self):
return self.enabled and self.status not in (
Expand All @@ -124,7 +119,7 @@ def ready_for_sync(self):
def clean(self):

# Ensure URL scheme matches selected type
if self.type == DataSourceTypeChoices.LOCAL and self.url_scheme not in ('file', ''):
if self.backend_class.is_local and self.url_scheme not in ('file', ''):
raise ValidationError({
'source_url': f"URLs for local sources must start with file:// (or specify no scheme)"
})
Expand Down
7 changes: 2 additions & 5 deletions netbox/core/tables/data.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,6 @@ class DataSourceTable(NetBoxTable):
verbose_name=_('Name'),
linkify=True
)
type = columns.ChoiceFieldColumn(
verbose_name=_('Type'),
)
status = columns.ChoiceFieldColumn(
verbose_name=_('Status'),
)
Expand All @@ -34,8 +31,8 @@ class DataSourceTable(NetBoxTable):
class Meta(NetBoxTable.Meta):
model = DataSource
fields = (
'pk', 'id', 'name', 'type', 'status', 'enabled', 'source_url', 'description', 'comments', 'parameters', 'created',
'last_updated', 'file_count',
'pk', 'id', 'name', 'type', 'status', 'enabled', 'source_url', 'description', 'comments', 'parameters',
'created', 'last_updated', 'file_count',
)
default_columns = ('pk', 'name', 'type', 'status', 'enabled', 'description', 'file_count')

Expand Down
Loading

0 comments on commit 21d17d5

Please sign in to comment.