From ee913e1f91dc6981a87a0e1d5585fcafc8ba1cb8 Mon Sep 17 00:00:00 2001 From: Arthur Date: Fri, 27 Oct 2023 14:49:46 -0700 Subject: [PATCH 01/95] 14132 change_logging -> event_logging --- netbox/extras/context_managers.py | 2 +- netbox/extras/management/commands/runscript.py | 8 ++++---- netbox/extras/scripts.py | 8 ++++---- netbox/netbox/middleware.py | 6 +++--- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/netbox/extras/context_managers.py b/netbox/extras/context_managers.py index 32323999efe..7973321eb1e 100644 --- a/netbox/extras/context_managers.py +++ b/netbox/extras/context_managers.py @@ -5,7 +5,7 @@ @contextmanager -def change_logging(request): +def event_logging(request): """ Enable change logging by connecting the appropriate signals to their receivers before code is run, and disconnecting them afterward. diff --git a/netbox/extras/management/commands/runscript.py b/netbox/extras/management/commands/runscript.py index d9a9f41ae6f..730cc74c259 100644 --- a/netbox/extras/management/commands/runscript.py +++ b/netbox/extras/management/commands/runscript.py @@ -11,7 +11,7 @@ from core.choices import JobStatusChoices from core.models import Job from extras.api.serializers import ScriptOutputSerializer -from extras.context_managers import change_logging +from extras.context_managers import event_logging from extras.scripts import get_module_and_script from extras.signals import clear_webhooks from utilities.exceptions import AbortTransaction @@ -37,7 +37,7 @@ def handle(self, *args, **options): def _run_script(): """ Core script execution task. We capture this within a subfunction to allow for conditionally wrapping it with - the change_logging context manager (which is bypassed if commit == False). + the event_logging context manager (which is bypassed if commit == False). """ try: try: @@ -136,9 +136,9 @@ def _run_script(): logger.info(f"Running script (commit={commit})") script.request = request - # Execute the script. If commit is True, wrap it with the change_logging context manager to ensure we process + # Execute the script. If commit is True, wrap it with the event_logging context manager to ensure we process # change logging, webhooks, etc. - with change_logging(request): + with event_logging(request): _run_script() else: logger.error('Data is not valid:') diff --git a/netbox/extras/scripts.py b/netbox/extras/scripts.py index e93326ddc74..d2c3cbdabce 100644 --- a/netbox/extras/scripts.py +++ b/netbox/extras/scripts.py @@ -23,7 +23,7 @@ from utilities.exceptions import AbortScript, AbortTransaction from utilities.forms import add_blank_choice from utilities.forms.fields import DynamicModelChoiceField, DynamicModelMultipleChoiceField -from .context_managers import change_logging +from .context_managers import event_logging from .forms import ScriptForm __all__ = ( @@ -496,7 +496,7 @@ def run_script(data, request, job, commit=True, **kwargs): def _run_script(): """ Core script execution task. We capture this within a subfunction to allow for conditionally wrapping it with - the change_logging context manager (which is bypassed if commit == False). + the event_logging context manager (which is bypassed if commit == False). """ try: try: @@ -524,10 +524,10 @@ def _run_script(): logger.info(f"Script completed in {job.duration}") - # Execute the script. If commit is True, wrap it with the change_logging context manager to ensure we process + # Execute the script. If commit is True, wrap it with the event_logging context manager to ensure we process # change logging, webhooks, etc. if commit: - with change_logging(request): + with event_logging(request): _run_script() else: _run_script() diff --git a/netbox/netbox/middleware.py b/netbox/netbox/middleware.py index 18f350fd7b4..4476449a41e 100644 --- a/netbox/netbox/middleware.py +++ b/netbox/netbox/middleware.py @@ -10,7 +10,7 @@ from django.db.utils import InternalError from django.http import Http404, HttpResponseRedirect -from extras.context_managers import change_logging +from extras.context_managers import event_logging from netbox.config import clear_config, get_config from netbox.views import handler_500 from utilities.api import is_api_request, rest_api_server_error @@ -42,8 +42,8 @@ def __call__(self, request): login_url = f'{settings.LOGIN_URL}?next={parse.quote(request.get_full_path_info())}' return HttpResponseRedirect(login_url) - # Enable the change_logging context manager and process the request. - with change_logging(request): + # Enable the event_logging context manager and process the request. + with event_logging(request): response = self.get_response(request) # Attach the unique request ID as an HTTP header. From 3b5c68251183e15c2eaf9e777577c57cc2829898 Mon Sep 17 00:00:00 2001 From: Arthur Date: Fri, 27 Oct 2023 14:53:03 -0700 Subject: [PATCH 02/95] 14132 event_logging -> event_wrapper --- netbox/extras/context_managers.py | 2 +- netbox/extras/management/commands/runscript.py | 8 ++++---- netbox/extras/scripts.py | 8 ++++---- netbox/netbox/middleware.py | 6 +++--- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/netbox/extras/context_managers.py b/netbox/extras/context_managers.py index 7973321eb1e..392d19c5d50 100644 --- a/netbox/extras/context_managers.py +++ b/netbox/extras/context_managers.py @@ -5,7 +5,7 @@ @contextmanager -def event_logging(request): +def event_wrapper(request): """ Enable change logging by connecting the appropriate signals to their receivers before code is run, and disconnecting them afterward. diff --git a/netbox/extras/management/commands/runscript.py b/netbox/extras/management/commands/runscript.py index 730cc74c259..ccc8e585b39 100644 --- a/netbox/extras/management/commands/runscript.py +++ b/netbox/extras/management/commands/runscript.py @@ -11,7 +11,7 @@ from core.choices import JobStatusChoices from core.models import Job from extras.api.serializers import ScriptOutputSerializer -from extras.context_managers import event_logging +from extras.context_managers import event_wrapper from extras.scripts import get_module_and_script from extras.signals import clear_webhooks from utilities.exceptions import AbortTransaction @@ -37,7 +37,7 @@ def handle(self, *args, **options): def _run_script(): """ Core script execution task. We capture this within a subfunction to allow for conditionally wrapping it with - the event_logging context manager (which is bypassed if commit == False). + the event_wrapper context manager (which is bypassed if commit == False). """ try: try: @@ -136,9 +136,9 @@ def _run_script(): logger.info(f"Running script (commit={commit})") script.request = request - # Execute the script. If commit is True, wrap it with the event_logging context manager to ensure we process + # Execute the script. If commit is True, wrap it with the event_wrapper context manager to ensure we process # change logging, webhooks, etc. - with event_logging(request): + with event_wrapper(request): _run_script() else: logger.error('Data is not valid:') diff --git a/netbox/extras/scripts.py b/netbox/extras/scripts.py index d2c3cbdabce..af6c93b5dee 100644 --- a/netbox/extras/scripts.py +++ b/netbox/extras/scripts.py @@ -23,7 +23,7 @@ from utilities.exceptions import AbortScript, AbortTransaction from utilities.forms import add_blank_choice from utilities.forms.fields import DynamicModelChoiceField, DynamicModelMultipleChoiceField -from .context_managers import event_logging +from .context_managers import event_wrapper from .forms import ScriptForm __all__ = ( @@ -496,7 +496,7 @@ def run_script(data, request, job, commit=True, **kwargs): def _run_script(): """ Core script execution task. We capture this within a subfunction to allow for conditionally wrapping it with - the event_logging context manager (which is bypassed if commit == False). + the event_wrapper context manager (which is bypassed if commit == False). """ try: try: @@ -524,10 +524,10 @@ def _run_script(): logger.info(f"Script completed in {job.duration}") - # Execute the script. If commit is True, wrap it with the event_logging context manager to ensure we process + # Execute the script. If commit is True, wrap it with the event_wrapper context manager to ensure we process # change logging, webhooks, etc. if commit: - with event_logging(request): + with event_wrapper(request): _run_script() else: _run_script() diff --git a/netbox/netbox/middleware.py b/netbox/netbox/middleware.py index 4476449a41e..37cb041db8c 100644 --- a/netbox/netbox/middleware.py +++ b/netbox/netbox/middleware.py @@ -10,7 +10,7 @@ from django.db.utils import InternalError from django.http import Http404, HttpResponseRedirect -from extras.context_managers import event_logging +from extras.context_managers import event_wrapper from netbox.config import clear_config, get_config from netbox.views import handler_500 from utilities.api import is_api_request, rest_api_server_error @@ -42,8 +42,8 @@ def __call__(self, request): login_url = f'{settings.LOGIN_URL}?next={parse.quote(request.get_full_path_info())}' return HttpResponseRedirect(login_url) - # Enable the event_logging context manager and process the request. - with event_logging(request): + # Enable the event_wrapper context manager and process the request. + with event_wrapper(request): response = self.get_response(request) # Attach the unique request ID as an HTTP header. From 4b3b88f23d7dd950e40a72d526fef0b9efeae0eb Mon Sep 17 00:00:00 2001 From: Arthur Date: Fri, 27 Oct 2023 14:55:16 -0700 Subject: [PATCH 03/95] 14132 webhooks_queue -> events_queue --- netbox/extras/context_managers.py | 8 ++++---- netbox/extras/signals.py | 14 +++++++------- netbox/extras/tests/test_webhooks.py | 6 +++--- netbox/netbox/context.py | 4 ++-- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/netbox/extras/context_managers.py b/netbox/extras/context_managers.py index 392d19c5d50..e0af2742a2c 100644 --- a/netbox/extras/context_managers.py +++ b/netbox/extras/context_managers.py @@ -1,6 +1,6 @@ from contextlib import contextmanager -from netbox.context import current_request, webhooks_queue +from netbox.context import current_request, events_queue from .webhooks import flush_webhooks @@ -13,13 +13,13 @@ def event_wrapper(request): :param request: WSGIRequest object with a unique `id` set """ current_request.set(request) - webhooks_queue.set([]) + events_queue.set([]) yield # Flush queued webhooks to RQ - flush_webhooks(webhooks_queue.get()) + flush_webhooks(events_queue.get()) # Clear context vars current_request.set(None) - webhooks_queue.set([]) + events_queue.set([]) diff --git a/netbox/extras/signals.py b/netbox/extras/signals.py index 8bdaf523ce1..86d88b6c3e4 100644 --- a/netbox/extras/signals.py +++ b/netbox/extras/signals.py @@ -10,7 +10,7 @@ from extras.validators import CustomValidator from netbox.config import get_config -from netbox.context import current_request, webhooks_queue +from netbox.context import current_request, events_queue from netbox.signals import post_clean from utilities.exceptions import AbortRequest from .choices import ObjectChangeActionChoices @@ -81,14 +81,14 @@ def handle_changed_object(sender, instance, **kwargs): objectchange.save() # If this is an M2M change, update the previously queued webhook (from post_save) - queue = webhooks_queue.get() + queue = events_queue.get() if m2m_changed and queue and is_same_object(instance, queue[-1], request.id): instance.refresh_from_db() # Ensure that we're working with fresh M2M assignments queue[-1]['data'] = serialize_for_webhook(instance) queue[-1]['snapshots']['postchange'] = get_snapshots(instance, action)['postchange'] else: enqueue_object(queue, instance, request.user, request.id, action) - webhooks_queue.set(queue) + events_queue.set(queue) # Increment metric counters if action == ObjectChangeActionChoices.ACTION_CREATE: @@ -117,9 +117,9 @@ def handle_deleted_object(sender, instance, **kwargs): objectchange.save() # Enqueue webhooks - queue = webhooks_queue.get() + queue = events_queue.get() enqueue_object(queue, instance, request.user, request.id, ObjectChangeActionChoices.ACTION_DELETE) - webhooks_queue.set(queue) + events_queue.set(queue) # Increment metric counters model_deletes.labels(instance._meta.model_name).inc() @@ -131,8 +131,8 @@ def clear_webhook_queue(sender, **kwargs): Delete any queued webhooks (e.g. because of an aborted bulk transaction) """ logger = logging.getLogger('webhooks') - logger.info(f"Clearing {len(webhooks_queue.get())} queued webhooks ({sender})") - webhooks_queue.set([]) + logger.info(f"Clearing {len(events_queue.get())} queued webhooks ({sender})") + events_queue.set([]) # diff --git a/netbox/extras/tests/test_webhooks.py b/netbox/extras/tests/test_webhooks.py index ef76377652d..e8cc51b8e87 100644 --- a/netbox/extras/tests/test_webhooks.py +++ b/netbox/extras/tests/test_webhooks.py @@ -313,16 +313,16 @@ def dummy_send(_, request, **kwargs): return HttpResponse() # Enqueue a webhook for processing - webhooks_queue = [] + events_queue = [] site = Site.objects.create(name='Site 1', slug='site-1') enqueue_object( - webhooks_queue, + events_queue, instance=site, user=self.user, request_id=request_id, action=ObjectChangeActionChoices.ACTION_CREATE ) - flush_webhooks(webhooks_queue) + flush_webhooks(events_queue) # Retrieve the job from queue job = self.queue.jobs[0] diff --git a/netbox/netbox/context.py b/netbox/netbox/context.py index 5461a4e9479..56e41cb6311 100644 --- a/netbox/netbox/context.py +++ b/netbox/netbox/context.py @@ -2,9 +2,9 @@ __all__ = ( 'current_request', - 'webhooks_queue', + 'events_queue', ) current_request = ContextVar('current_request', default=None) -webhooks_queue = ContextVar('webhooks_queue', default=[]) +events_queue = ContextVar('events_queue', default=[]) From 91480299ecbc26d49a7596d702605eeb972b40ac Mon Sep 17 00:00:00 2001 From: Arthur Date: Fri, 27 Oct 2023 14:58:56 -0700 Subject: [PATCH 04/95] 14132 flush_webhooks -> flush_events --- netbox/extras/context_managers.py | 4 ++-- netbox/extras/tests/test_webhooks.py | 4 ++-- netbox/extras/webhooks.py | 2 +- netbox/netbox/settings.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/netbox/extras/context_managers.py b/netbox/extras/context_managers.py index e0af2742a2c..c3c78f98bf8 100644 --- a/netbox/extras/context_managers.py +++ b/netbox/extras/context_managers.py @@ -1,7 +1,7 @@ from contextlib import contextmanager from netbox.context import current_request, events_queue -from .webhooks import flush_webhooks +from .webhooks import flush_events @contextmanager @@ -18,7 +18,7 @@ def event_wrapper(request): yield # Flush queued webhooks to RQ - flush_webhooks(events_queue.get()) + flush_events(events_queue.get()) # Clear context vars current_request.set(None) diff --git a/netbox/extras/tests/test_webhooks.py b/netbox/extras/tests/test_webhooks.py index e8cc51b8e87..c309250812c 100644 --- a/netbox/extras/tests/test_webhooks.py +++ b/netbox/extras/tests/test_webhooks.py @@ -13,7 +13,7 @@ from dcim.models import Site from extras.choices import ObjectChangeActionChoices from extras.models import Tag, Webhook -from extras.webhooks import enqueue_object, flush_webhooks, generate_signature, serialize_for_webhook +from extras.webhooks import enqueue_object, flush_events, generate_signature, serialize_for_webhook from extras.webhooks_worker import eval_conditions, process_webhook from utilities.testing import APITestCase @@ -322,7 +322,7 @@ def dummy_send(_, request, **kwargs): request_id=request_id, action=ObjectChangeActionChoices.ACTION_CREATE ) - flush_webhooks(events_queue) + flush_events(events_queue) # Retrieve the job from queue job = self.queue.jobs[0] diff --git a/netbox/extras/webhooks.py b/netbox/extras/webhooks.py index 1fc869ee8e3..2dc474e8a37 100644 --- a/netbox/extras/webhooks.py +++ b/netbox/extras/webhooks.py @@ -77,7 +77,7 @@ def enqueue_object(queue, instance, user, request_id, action): }) -def flush_webhooks(queue): +def flush_events(queue): """ Flush a list of object representation to RQ for webhook processing. """ diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index 4c8b3f96055..36f56b0f634 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -671,7 +671,7 @@ def _setting(name, default=None): # -# Django RQ (Webhooks backend) +# Django RQ (events backend) # if TASKS_REDIS_USING_SENTINEL: From 38d46afdbe84fe1c216c50ca78e9447e02dd0f84 Mon Sep 17 00:00:00 2001 From: Arthur Date: Mon, 30 Oct 2023 14:57:51 -0700 Subject: [PATCH 05/95] 14132 add models and forms --- netbox/extras/filtersets.py | 26 +++++++ netbox/extras/forms/bulk_edit.py | 42 +++++++++++ netbox/extras/forms/bulk_import.py | 17 +++++ netbox/extras/forms/filtersets.py | 59 +++++++++++++++ netbox/extras/forms/model_forms.py | 29 +++++++ netbox/extras/migrations/0099_eventrule.py | 50 ++++++++++++ netbox/extras/models/models.py | 88 ++++++++++++++++++++++ netbox/extras/tables/tables.py | 43 +++++++++++ netbox/extras/urls.py | 8 ++ netbox/extras/views.py | 45 +++++++++++ 10 files changed, 407 insertions(+) create mode 100644 netbox/extras/migrations/0099_eventrule.py diff --git a/netbox/extras/filtersets.py b/netbox/extras/filtersets.py index fec06726357..8492c2fb02b 100644 --- a/netbox/extras/filtersets.py +++ b/netbox/extras/filtersets.py @@ -23,6 +23,7 @@ 'CustomFieldChoiceSetFilterSet', 'CustomFieldFilterSet', 'CustomLinkFilterSet', + 'EventRuleFilterSet', 'ExportTemplateFilterSet', 'ImageAttachmentFilterSet', 'JournalEntryFilterSet', @@ -63,6 +64,31 @@ def search(self, queryset, name, value): ) +class EventRuleFilterSet(NetBoxModelFilterSet): + q = django_filters.CharFilter( + method='search', + label=_('Search'), + ) + content_type_id = MultiValueNumberFilter( + field_name='content_types__id' + ) + content_types = ContentTypeFilter() + + class Meta: + model = EventRule + fields = [ + 'id', 'name', 'type_create', 'type_update', 'type_delete', 'type_job_start', 'type_job_end', + 'enabled', + ] + + def search(self, queryset, name, value): + if not value.strip(): + return queryset + return queryset.filter( + Q(name__icontains=value) + ) + + class CustomFieldFilterSet(BaseFilterSet): q = django_filters.CharFilter( method='search', diff --git a/netbox/extras/forms/bulk_edit.py b/netbox/extras/forms/bulk_edit.py index 821ce7eb243..ae9c654a458 100644 --- a/netbox/extras/forms/bulk_edit.py +++ b/netbox/extras/forms/bulk_edit.py @@ -14,6 +14,7 @@ 'CustomFieldBulkEditForm', 'CustomFieldChoiceSetBulkEditForm', 'CustomLinkBulkEditForm', + 'EventRuleBulkEditForm', 'ExportTemplateBulkEditForm', 'JournalEntryBulkEditForm', 'SavedFilterBulkEditForm', @@ -229,6 +230,47 @@ class WebhookBulkEditForm(NetBoxModelBulkEditForm): nullable_fields = ('secret', 'conditions', 'ca_file_path') +class EventRuleBulkEditForm(NetBoxModelBulkEditForm): + model = EventRule + + pk = forms.ModelMultipleChoiceField( + queryset=EventRule.objects.all(), + widget=forms.MultipleHiddenInput + ) + enabled = forms.NullBooleanField( + label=_('Enabled'), + required=False, + widget=BulkEditNullBooleanSelect() + ) + type_create = forms.NullBooleanField( + label=_('On create'), + required=False, + widget=BulkEditNullBooleanSelect() + ) + type_update = forms.NullBooleanField( + label=_('On update'), + required=False, + widget=BulkEditNullBooleanSelect() + ) + type_delete = forms.NullBooleanField( + label=_('On delete'), + required=False, + widget=BulkEditNullBooleanSelect() + ) + type_job_start = forms.NullBooleanField( + label=_('On job start'), + required=False, + widget=BulkEditNullBooleanSelect() + ) + type_job_end = forms.NullBooleanField( + label=_('On job end'), + required=False, + widget=BulkEditNullBooleanSelect() + ) + + nullable_fields = ('conditions',) + + class TagBulkEditForm(BulkEditForm): pk = forms.ModelMultipleChoiceField( queryset=Tag.objects.all(), diff --git a/netbox/extras/forms/bulk_import.py b/netbox/extras/forms/bulk_import.py index 79023a74dbe..22d0364feff 100644 --- a/netbox/extras/forms/bulk_import.py +++ b/netbox/extras/forms/bulk_import.py @@ -18,6 +18,7 @@ 'CustomFieldChoiceSetImportForm', 'CustomFieldImportForm', 'CustomLinkImportForm', + 'EventRuleImportForm', 'ExportTemplateImportForm', 'JournalEntryImportForm', 'SavedFilterImportForm', @@ -157,6 +158,22 @@ class Meta: ) +class EventRuleImportForm(NetBoxModelImportForm): + content_types = CSVMultipleContentTypeField( + label=_('Content types'), + queryset=ContentType.objects.all(), + limit_choices_to=FeatureQuery('eventrules'), + help_text=_("One or more assigned object types") + ) + + class Meta: + model = EventRule + fields = ( + 'name', 'enabled', 'content_types', 'type_create', 'type_update', 'type_delete', 'type_job_start', + 'type_job_end', 'tags' + ) + + class TagImportForm(CSVModelForm): slug = SlugField() diff --git a/netbox/extras/forms/filtersets.py b/netbox/extras/forms/filtersets.py index 7db84d17550..c75b84cb81b 100644 --- a/netbox/extras/forms/filtersets.py +++ b/netbox/extras/forms/filtersets.py @@ -25,6 +25,7 @@ 'CustomFieldChoiceSetFilterForm', 'CustomFieldFilterForm', 'CustomLinkFilterForm', + 'EventRuleFilterForm', 'ExportTemplateFilterForm', 'ImageAttachmentFilterForm', 'JournalEntryFilterForm', @@ -282,6 +283,64 @@ class WebhookFilterForm(NetBoxModelFilterSetForm): ) +class EventRuleFilterForm(NetBoxModelFilterSetForm): + model = EventRule + tag = TagFilterField(model) + + fieldsets = ( + (None, ('q', 'filter_id', 'tag')), + (_('Attributes'), ('content_type_id', 'enabled')), + (_('Events'), ('type_create', 'type_update', 'type_delete', 'type_job_start', 'type_job_end')), + ) + content_type_id = ContentTypeMultipleChoiceField( + queryset=ContentType.objects.filter(FeatureQuery('webhooks').get_query()), + required=False, + label=_('Object type') + ) + enabled = forms.NullBooleanField( + label=_('Enabled'), + required=False, + widget=forms.Select( + choices=BOOLEAN_WITH_BLANK_CHOICES + ) + ) + type_create = forms.NullBooleanField( + required=False, + widget=forms.Select( + choices=BOOLEAN_WITH_BLANK_CHOICES + ), + label=_('Object creations') + ) + type_update = forms.NullBooleanField( + required=False, + widget=forms.Select( + choices=BOOLEAN_WITH_BLANK_CHOICES + ), + label=_('Object updates') + ) + type_delete = forms.NullBooleanField( + required=False, + widget=forms.Select( + choices=BOOLEAN_WITH_BLANK_CHOICES + ), + label=_('Object deletions') + ) + type_job_start = forms.NullBooleanField( + required=False, + widget=forms.Select( + choices=BOOLEAN_WITH_BLANK_CHOICES + ), + label=_('Job starts') + ) + type_job_end = forms.NullBooleanField( + required=False, + widget=forms.Select( + choices=BOOLEAN_WITH_BLANK_CHOICES + ), + label=_('Job terminations') + ) + + class TagFilterForm(SavedFiltersMixin, FilterForm): model = Tag content_type_id = ContentTypeMultipleChoiceField( diff --git a/netbox/extras/forms/model_forms.py b/netbox/extras/forms/model_forms.py index fd2ce8f2da3..ab8424705d3 100644 --- a/netbox/extras/forms/model_forms.py +++ b/netbox/extras/forms/model_forms.py @@ -32,6 +32,7 @@ 'CustomFieldChoiceSetForm', 'CustomFieldForm', 'CustomLinkForm', + 'EventRuleForm', 'ExportTemplateForm', 'ImageAttachmentForm', 'JournalEntryForm', @@ -256,6 +257,34 @@ class Meta: } +class EventRuleForm(NetBoxModelForm): + content_types = ContentTypeMultipleChoiceField( + label=_('Content types'), + queryset=ContentType.objects.all(), + limit_choices_to=FeatureQuery('eventrules') + ) + + fieldsets = ( + (_('EventRule'), ('name', 'content_types', 'enabled', 'tags')), + (_('Events'), ('type_create', 'type_update', 'type_delete', 'type_job_start', 'type_job_end')), + (_('Conditions'), ('conditions',)), + ) + + class Meta: + model = Webhook + fields = '__all__' + labels = { + 'type_create': _('Creations'), + 'type_update': _('Updates'), + 'type_delete': _('Deletions'), + 'type_job_start': _('Job executions'), + 'type_job_end': _('Job terminations'), + } + widgets = { + 'conditions': forms.Textarea(attrs={'class': 'font-monospace'}), + } + + class TagForm(BootstrapMixin, forms.ModelForm): slug = SlugField() object_types = ContentTypeMultipleChoiceField( diff --git a/netbox/extras/migrations/0099_eventrule.py b/netbox/extras/migrations/0099_eventrule.py new file mode 100644 index 00000000000..8be9947523b --- /dev/null +++ b/netbox/extras/migrations/0099_eventrule.py @@ -0,0 +1,50 @@ +# Generated by Django 4.2.5 on 2023-10-30 21:57 + +from django.db import migrations, models +import extras.utils +import taggit.managers +import utilities.json + + +class Migration(migrations.Migration): + dependencies = [ + ('contenttypes', '0002_remove_content_type_name'), + ('extras', '0098_webhook_custom_field_data_webhook_tags'), + ] + + operations = [ + migrations.CreateModel( + name='EventRule', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False)), + ('created', models.DateTimeField(auto_now_add=True, null=True)), + ('last_updated', models.DateTimeField(auto_now=True, null=True)), + ( + 'custom_field_data', + models.JSONField(blank=True, default=dict, encoder=utilities.json.CustomFieldJSONEncoder), + ), + ('name', models.CharField(max_length=150, unique=True)), + ('type_create', models.BooleanField(default=False)), + ('type_update', models.BooleanField(default=False)), + ('type_delete', models.BooleanField(default=False)), + ('type_job_start', models.BooleanField(default=False)), + ('type_job_end', models.BooleanField(default=False)), + ('enabled', models.BooleanField(default=True)), + ('conditions', models.JSONField(blank=True, null=True)), + ( + 'content_types', + models.ManyToManyField( + limit_choices_to=extras.utils.FeatureQuery('eventrules'), + related_name='eventrules', + to='contenttypes.contenttype', + ), + ), + ('tags', taggit.managers.TaggableManager(through='extras.TaggedItem', to='extras.Tag')), + ], + options={ + 'verbose_name': 'eventrule', + 'verbose_name_plural': 'eventrules', + 'ordering': ('name',), + }, + ), + ] diff --git a/netbox/extras/models/models.py b/netbox/extras/models/models.py index 90e8027b452..be37e7d02dc 100644 --- a/netbox/extras/models/models.py +++ b/netbox/extras/models/models.py @@ -30,6 +30,7 @@ 'Bookmark', 'ConfigRevision', 'CustomLink', + 'EventRule', 'ExportTemplate', 'ImageAttachment', 'JournalEntry', @@ -38,6 +39,93 @@ ) +class EventRule(CustomFieldsMixin, ExportTemplatesMixin, TagsMixin, ChangeLoggedModel): + """ + A Webhook defines a request that will be sent to a remote application when an object is created, updated, and/or + delete in NetBox. The request will contain a representation of the object, which the remote application can act on. + Each Webhook can be limited to firing only on certain actions or certain object types. + """ + content_types = models.ManyToManyField( + to=ContentType, + related_name='eventrules', + verbose_name=_('object types'), + limit_choices_to=FeatureQuery('eventrules'), + help_text=_("The object(s) to which this Event applies.") + ) + name = models.CharField( + verbose_name=_('name'), + max_length=150, + unique=True + ) + type_create = models.BooleanField( + verbose_name=_('on create'), + default=False, + help_text=_("Triggers when a matching object is created.") + ) + type_update = models.BooleanField( + verbose_name=_('on update'), + default=False, + help_text=_("Triggers when a matching object is updated.") + ) + type_delete = models.BooleanField( + verbose_name=_('on delete'), + default=False, + help_text=_("Triggers when a matching object is deleted.") + ) + type_job_start = models.BooleanField( + verbose_name=_('on job start'), + default=False, + help_text=_("Triggers when a job for a matching object is started.") + ) + type_job_end = models.BooleanField( + verbose_name=_('on job end'), + default=False, + help_text=_("Triggers when a job for a matching object terminates.") + ) + enabled = models.BooleanField( + verbose_name=_('enabled'), + default=True + ) + conditions = models.JSONField( + verbose_name=_('conditions'), + blank=True, + null=True, + help_text=_("A set of conditions which determine whether the event will be generated.") + ) + + class Meta: + ordering = ('name',) + verbose_name = _('eventrule') + verbose_name_plural = _('eventrules') + + def __str__(self): + return self.name + + def get_absolute_url(self): + return reverse('extras:eventrule', args=[self.pk]) + + @property + def docs_url(self): + return f'{settings.STATIC_URL}docs/models/extras/eventrule/' + + def clean(self): + super().clean() + + # At least one action type must be selected + if not any([ + self.type_create, self.type_update, self.type_delete, self.type_job_start, self.type_job_end + ]): + raise ValidationError( + _("At least one event type must be selected: create, update, delete, job_start, and/or job_end.") + ) + + if self.conditions: + try: + ConditionSet(self.conditions) + except ValueError as e: + raise ValidationError({'conditions': e}) + + class Webhook(CustomFieldsMixin, ExportTemplatesMixin, TagsMixin, ChangeLoggedModel): """ A Webhook defines a request that will be sent to a remote application when an object is created, updated, and/or diff --git a/netbox/extras/tables/tables.py b/netbox/extras/tables/tables.py index 9e14a2d2745..c9c663722d6 100644 --- a/netbox/extras/tables/tables.py +++ b/netbox/extras/tables/tables.py @@ -16,6 +16,7 @@ 'CustomFieldChoiceSetTable', 'CustomFieldTable', 'CustomLinkTable', + 'EventRuleTable', 'ExportTemplateTable', 'ImageAttachmentTable', 'JournalEntryTable', @@ -314,6 +315,48 @@ class Meta(NetBoxTable.Meta): ) +class EventRuleTable(NetBoxTable): + name = tables.Column( + verbose_name=_('Name'), + linkify=True + ) + content_types = columns.ContentTypesColumn( + verbose_name=_('Content Types'), + ) + enabled = columns.BooleanColumn( + verbose_name=_('Enabled'), + ) + type_create = columns.BooleanColumn( + verbose_name=_('Create') + ) + type_update = columns.BooleanColumn( + verbose_name=_('Update') + ) + type_delete = columns.BooleanColumn( + verbose_name=_('Delete') + ) + type_job_start = columns.BooleanColumn( + verbose_name=_('Job Start') + ) + type_job_end = columns.BooleanColumn( + verbose_name=_('Job End') + ) + tags = columns.TagColumn( + url_name='extras:webhook_list' + ) + + class Meta(NetBoxTable.Meta): + model = EventRule + fields = ( + 'pk', 'id', 'name', 'content_types', 'enabled', 'type_create', 'type_update', 'type_delete', + 'type_job_start', 'type_job_end', 'tags', 'created', 'last_updated', + ) + default_columns = ( + 'pk', 'name', 'content_types', 'enabled', 'type_create', 'type_update', 'type_delete', 'type_job_start', + 'type_job_end', + ) + + class TagTable(NetBoxTable): name = tables.Column( verbose_name=_('Name'), diff --git a/netbox/extras/urls.py b/netbox/extras/urls.py index fd95186e436..5cf002950de 100644 --- a/netbox/extras/urls.py +++ b/netbox/extras/urls.py @@ -61,6 +61,14 @@ path('webhooks/delete/', views.WebhookBulkDeleteView.as_view(), name='webhook_bulk_delete'), path('webhooks//', include(get_model_urls('extras', 'webhook'))), + # Event rules + path('event-rules/', views.EventRuleListView.as_view(), name='eventrule_list'), + path('event-rules/add/', views.EventRuleEditView.as_view(), name='eventrule_add'), + path('event-rules/import/', views.EventRuleBulkImportView.as_view(), name='eventrule_import'), + path('event-rules/edit/', views.EventRuleBulkEditView.as_view(), name='eventrule_bulk_edit'), + path('event-rules/delete/', views.EventRuleBulkDeleteView.as_view(), name='eventrule_bulk_delete'), + path('event-rules//', include(get_model_urls('extras', 'eventrule'))), + # Tags path('tags/', views.TagListView.as_view(), name='tag_list'), path('tags/add/', views.TagEditView.as_view(), name='tag_add'), diff --git a/netbox/extras/views.py b/netbox/extras/views.py index 0e8e3b0eaed..0af437ca9f4 100644 --- a/netbox/extras/views.py +++ b/netbox/extras/views.py @@ -396,6 +396,51 @@ class WebhookBulkDeleteView(generic.BulkDeleteView): table = tables.WebhookTable +# +# Event Rules +# + +class EventRuleListView(generic.ObjectListView): + queryset = EventRule.objects.all() + filterset = filtersets.EventRuleFilterSet + filterset_form = forms.EventRuleFilterForm + table = tables.EventRuleTable + + +@register_model_view(EventRule) +class EventRuleView(generic.ObjectView): + queryset = EventRule.objects.all() + + +@register_model_view(EventRule, 'edit') +class EventRuleEditView(generic.ObjectEditView): + queryset = EventRule.objects.all() + form = forms.EventRuleForm + + +@register_model_view(EventRule, 'delete') +class EventRuleDeleteView(generic.ObjectDeleteView): + queryset = EventRule.objects.all() + + +class EventRuleBulkImportView(generic.BulkImportView): + queryset = EventRule.objects.all() + model_form = forms.EventRuleImportForm + + +class EventRuleBulkEditView(generic.BulkEditView): + queryset = EventRule.objects.all() + filterset = filtersets.EventRuleFilterSet + table = tables.EventRuleTable + form = forms.EventRuleBulkEditForm + + +class EventRuleBulkDeleteView(generic.BulkDeleteView): + queryset = EventRule.objects.all() + filterset = filtersets.EventRuleFilterSet + table = tables.EventRuleTable + + # # Tags # From 498ac5e44f828b0205394f99dc743fcabfaeb736 Mon Sep 17 00:00:00 2001 From: Arthur Date: Mon, 30 Oct 2023 15:10:15 -0700 Subject: [PATCH 06/95] 14132 fixes --- netbox/extras/forms/model_forms.py | 4 ++-- netbox/netbox/navigation/menu.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/netbox/extras/forms/model_forms.py b/netbox/extras/forms/model_forms.py index ab8424705d3..24039c2eebd 100644 --- a/netbox/extras/forms/model_forms.py +++ b/netbox/extras/forms/model_forms.py @@ -261,7 +261,7 @@ class EventRuleForm(NetBoxModelForm): content_types = ContentTypeMultipleChoiceField( label=_('Content types'), queryset=ContentType.objects.all(), - limit_choices_to=FeatureQuery('eventrules') + limit_choices_to=FeatureQuery('webhooks') ) fieldsets = ( @@ -271,7 +271,7 @@ class EventRuleForm(NetBoxModelForm): ) class Meta: - model = Webhook + model = EventRule fields = '__all__' labels = { 'type_create': _('Creations'), diff --git a/netbox/netbox/navigation/menu.py b/netbox/netbox/navigation/menu.py index 961fd2035ac..b73120d78d9 100644 --- a/netbox/netbox/navigation/menu.py +++ b/netbox/netbox/navigation/menu.py @@ -325,6 +325,7 @@ label=_('Integrations'), items=( get_model_item('core', 'datasource', _('Data Sources')), + get_model_item('extras', 'eventrule', _('Event Rules')), get_model_item('extras', 'webhook', _('Webhooks')), ), ), From 1cc8a82921e3cd578fd808fb8e9c45496c40ab94 Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 31 Oct 2023 07:37:22 -0700 Subject: [PATCH 07/95] 14132 model changes --- netbox/extras/choices.py | 15 +++++++++++++++ netbox/extras/constants.py | 10 ++++++++++ netbox/extras/migrations/0099_eventrule.py | 14 +++++++++++++- netbox/extras/models/models.py | 22 ++++++++++++++++++++++ 4 files changed, 60 insertions(+), 1 deletion(-) diff --git a/netbox/extras/choices.py b/netbox/extras/choices.py index 0572a33a129..44774738229 100644 --- a/netbox/extras/choices.py +++ b/netbox/extras/choices.py @@ -280,3 +280,18 @@ class DashboardWidgetColorChoices(ChoiceSet): (BLACK, _('Black')), (WHITE, _('White')), ) + + +# +# Event Rules +# + +class EventRuleTypeChoices(ChoiceSet): + + WEBHOOK = 'webhook' + SCRIPT = 'script' + + CHOICES = ( + (WEBHOOK, _('Webhook'), 'webhook'), + (SCRIPT, _('Script'), 'script'), + ) diff --git a/netbox/extras/constants.py b/netbox/extras/constants.py index 48b44fb4533..a976baf8ca9 100644 --- a/netbox/extras/constants.py +++ b/netbox/extras/constants.py @@ -1,3 +1,6 @@ +from django.db.models import Q + + # Events EVENT_CREATE = 'create' EVENT_UPDATE = 'update' @@ -133,3 +136,10 @@ } }, ] + +EVENT_TYPE_MODELS = Q( + app_label='extras', + model__in=( + 'webhook', + 'script', + )) diff --git a/netbox/extras/migrations/0099_eventrule.py b/netbox/extras/migrations/0099_eventrule.py index 8be9947523b..ee8719a57da 100644 --- a/netbox/extras/migrations/0099_eventrule.py +++ b/netbox/extras/migrations/0099_eventrule.py @@ -1,6 +1,7 @@ -# Generated by Django 4.2.5 on 2023-10-30 21:57 +# Generated by Django 4.2.5 on 2023-10-31 14:37 from django.db import migrations, models +import django.db.models.deletion import extras.utils import taggit.managers import utilities.json @@ -31,6 +32,8 @@ class Migration(migrations.Migration): ('type_job_end', models.BooleanField(default=False)), ('enabled', models.BooleanField(default=True)), ('conditions', models.JSONField(blank=True, null=True)), + ('event_type', models.CharField(default='webhook', max_length=30)), + ('object_id', models.PositiveBigIntegerField(blank=True, null=True)), ( 'content_types', models.ManyToManyField( @@ -39,6 +42,15 @@ class Migration(migrations.Migration): to='contenttypes.contenttype', ), ), + ( + 'object_type', + models.ForeignKey( + limit_choices_to=models.Q(('app_label', 'extras'), ('model__in', ('webhook', 'script'))), + on_delete=django.db.models.deletion.CASCADE, + related_name='eventrule_actions', + to='contenttypes.contenttype', + ), + ), ('tags', taggit.managers.TaggableManager(through='extras.TaggedItem', to='extras.Tag')), ], options={ diff --git a/netbox/extras/models/models.py b/netbox/extras/models/models.py index be37e7d02dc..cf961044a7a 100644 --- a/netbox/extras/models/models.py +++ b/netbox/extras/models/models.py @@ -93,6 +93,28 @@ class EventRule(CustomFieldsMixin, ExportTemplatesMixin, TagsMixin, ChangeLogged help_text=_("A set of conditions which determine whether the event will be generated.") ) + event_type = models.CharField( + max_length=30, + choices=EventRuleTypeChoices, + default=EventRuleTypeChoices.WEBHOOK, + verbose_name=_('event type') + ) + # Action to take + object_type = models.ForeignKey( + to=ContentType, + related_name='eventrule_actions', + limit_choices_to=EVENT_TYPE_MODELS, + on_delete=models.CASCADE, + ) + object_id = models.PositiveBigIntegerField( + blank=True, + null=True + ) + object = GenericForeignKey( + ct_field='object_type', + fk_field='object_id', + ) + class Meta: ordering = ('name',) verbose_name = _('eventrule') From e6f048b6390cbb80a10bab91894530829cb33f58 Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 31 Oct 2023 08:57:20 -0700 Subject: [PATCH 08/95] 14132 remove fields from Webhook model, consolidate migrations --- netbox/extras/filtersets.py | 8 +-- netbox/extras/forms/bulk_import.py | 3 +- netbox/extras/forms/filtersets.py | 50 +------------ netbox/extras/migrations/0099_eventrule.py | 81 ++++++++++++++++++++++ netbox/extras/models/models.py | 78 ++------------------- netbox/extras/tables/tables.py | 7 +- 6 files changed, 97 insertions(+), 130 deletions(-) diff --git a/netbox/extras/filtersets.py b/netbox/extras/filtersets.py index 8492c2fb02b..e7b3398ca7c 100644 --- a/netbox/extras/filtersets.py +++ b/netbox/extras/filtersets.py @@ -40,10 +40,6 @@ class WebhookFilterSet(NetBoxModelFilterSet): method='search', label=_('Search'), ) - content_type_id = MultiValueNumberFilter( - field_name='content_types__id' - ) - content_types = ContentTypeFilter() http_method = django_filters.MultipleChoiceFilter( choices=WebhookHttpMethodChoices ) @@ -51,8 +47,8 @@ class WebhookFilterSet(NetBoxModelFilterSet): class Meta: model = Webhook fields = [ - 'id', 'name', 'type_create', 'type_update', 'type_delete', 'type_job_start', 'type_job_end', 'payload_url', - 'enabled', 'http_method', 'http_content_type', 'secret', 'ssl_verification', 'ca_file_path', + 'id', 'payload_url', + 'http_method', 'http_content_type', 'secret', 'ssl_verification', 'ca_file_path', ] def search(self, queryset, name, value): diff --git a/netbox/extras/forms/bulk_import.py b/netbox/extras/forms/bulk_import.py index 22d0364feff..d0b6aca1c55 100644 --- a/netbox/extras/forms/bulk_import.py +++ b/netbox/extras/forms/bulk_import.py @@ -152,8 +152,7 @@ class WebhookImportForm(NetBoxModelImportForm): class Meta: model = Webhook fields = ( - 'name', 'enabled', 'content_types', 'type_create', 'type_update', 'type_delete', 'type_job_start', - 'type_job_end', 'payload_url', 'http_method', 'http_content_type', 'additional_headers', 'body_template', + 'payload_url', 'http_method', 'http_content_type', 'additional_headers', 'body_template', 'secret', 'ssl_verification', 'ca_file_path', 'tags' ) diff --git a/netbox/extras/forms/filtersets.py b/netbox/extras/forms/filtersets.py index c75b84cb81b..ebdd3491273 100644 --- a/netbox/extras/forms/filtersets.py +++ b/netbox/extras/forms/filtersets.py @@ -226,61 +226,13 @@ class WebhookFilterForm(NetBoxModelFilterSetForm): fieldsets = ( (None, ('q', 'filter_id', 'tag')), - (_('Attributes'), ('content_type_id', 'http_method', 'enabled')), - (_('Events'), ('type_create', 'type_update', 'type_delete', 'type_job_start', 'type_job_end')), - ) - content_type_id = ContentTypeMultipleChoiceField( - queryset=ContentType.objects.filter(FeatureQuery('webhooks').get_query()), - required=False, - label=_('Object type') + (_('Attributes'), ('http_method',)), ) http_method = forms.MultipleChoiceField( choices=WebhookHttpMethodChoices, required=False, label=_('HTTP method') ) - enabled = forms.NullBooleanField( - label=_('Enabled'), - required=False, - widget=forms.Select( - choices=BOOLEAN_WITH_BLANK_CHOICES - ) - ) - type_create = forms.NullBooleanField( - required=False, - widget=forms.Select( - choices=BOOLEAN_WITH_BLANK_CHOICES - ), - label=_('Object creations') - ) - type_update = forms.NullBooleanField( - required=False, - widget=forms.Select( - choices=BOOLEAN_WITH_BLANK_CHOICES - ), - label=_('Object updates') - ) - type_delete = forms.NullBooleanField( - required=False, - widget=forms.Select( - choices=BOOLEAN_WITH_BLANK_CHOICES - ), - label=_('Object deletions') - ) - type_job_start = forms.NullBooleanField( - required=False, - widget=forms.Select( - choices=BOOLEAN_WITH_BLANK_CHOICES - ), - label=_('Job starts') - ) - type_job_end = forms.NullBooleanField( - required=False, - widget=forms.Select( - choices=BOOLEAN_WITH_BLANK_CHOICES - ), - label=_('Job terminations') - ) class EventRuleFilterForm(NetBoxModelFilterSetForm): diff --git a/netbox/extras/migrations/0099_eventrule.py b/netbox/extras/migrations/0099_eventrule.py index ee8719a57da..3f0694b713f 100644 --- a/netbox/extras/migrations/0099_eventrule.py +++ b/netbox/extras/migrations/0099_eventrule.py @@ -7,6 +7,29 @@ import utilities.json +def move_webhooks(apps, schema_editor): + Webhook = apps.get_model("extras", "Webhook") + EventRule = apps.get_model("extras", "EventRule") + + for webhook in Webhook.objects.all(): + event = EventRule() + + event.name = webhook.name + event.type_create = webhook.type_create + event.type_update = webhook.type_update + event.type_delete = webhook.type_delete + event.type_job_start = webhook.type_job_start + event.type_job_end = webhook.type_job_end + event.enabled = webhook.enabled + event.conditions = webhook.conditions + + event.event_type = EventRuleTypeChoices.WEBHOOK + event.object_type_id = ContentType.objects.get_for_model(webhook).id + event.object = webhook + event.save() + event.content_types.add(*webhook.content_types.all()) + + class Migration(migrations.Migration): dependencies = [ ('contenttypes', '0002_remove_content_type_name'), @@ -59,4 +82,62 @@ class Migration(migrations.Migration): 'ordering': ('name',), }, ), + migrations.RunPython(move_webhooks), + migrations.AlterModelOptions( + name='webhook', + options={'ordering': ('payload_url',)}, + ), + migrations.RemoveConstraint( + model_name='webhook', + name='extras_webhook_unique_payload_url_types', + ), + migrations.RemoveField( + model_name='webhook', + name='conditions', + ), + migrations.RemoveField( + model_name='webhook', + name='content_types', + ), + migrations.RemoveField( + model_name='webhook', + name='enabled', + ), + migrations.RemoveField( + model_name='webhook', + name='name', + ), + migrations.RemoveField( + model_name='webhook', + name='type_create', + ), + migrations.RemoveField( + model_name='webhook', + name='type_delete', + ), + migrations.RemoveField( + model_name='webhook', + name='type_job_end', + ), + migrations.RemoveField( + model_name='webhook', + name='type_job_start', + ), + migrations.RemoveField( + model_name='webhook', + name='type_update', + ), + migrations.AlterField( + model_name='eventrule', + name='object_type', + field=models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name='eventrule_actions', + to='contenttypes.contenttype', + ), + ), + migrations.AlterUniqueTogether( + name='eventrule', + unique_together={('object_type', 'object_id')}, + ), ] diff --git a/netbox/extras/models/models.py b/netbox/extras/models/models.py index cf961044a7a..dd5e83eb79f 100644 --- a/netbox/extras/models/models.py +++ b/netbox/extras/models/models.py @@ -2,7 +2,7 @@ import urllib.parse from django.conf import settings -from django.contrib.contenttypes.fields import GenericForeignKey +from django.contrib.contenttypes.fields import GenericForeignKey, GenericRelation from django.contrib.contenttypes.models import ContentType from django.core.cache import cache from django.core.validators import ValidationError @@ -103,7 +103,7 @@ class EventRule(CustomFieldsMixin, ExportTemplatesMixin, TagsMixin, ChangeLogged object_type = models.ForeignKey( to=ContentType, related_name='eventrule_actions', - limit_choices_to=EVENT_TYPE_MODELS, + # limit_choices_to=EVENT_TYPE_MODELS, on_delete=models.CASCADE, ) object_id = models.PositiveBigIntegerField( @@ -119,6 +119,7 @@ class Meta: ordering = ('name',) verbose_name = _('eventrule') verbose_name_plural = _('eventrules') + unique_together = ('object_type', 'object_id') def __str__(self): return self.name @@ -154,43 +155,8 @@ class Webhook(CustomFieldsMixin, ExportTemplatesMixin, TagsMixin, ChangeLoggedMo delete in NetBox. The request will contain a representation of the object, which the remote application can act on. Each Webhook can be limited to firing only on certain actions or certain object types. """ - content_types = models.ManyToManyField( - to=ContentType, - related_name='webhooks', - verbose_name=_('object types'), - limit_choices_to=FeatureQuery('webhooks'), - help_text=_("The object(s) to which this Webhook applies.") - ) - name = models.CharField( - verbose_name=_('name'), - max_length=150, - unique=True - ) - type_create = models.BooleanField( - verbose_name=_('on create'), - default=False, - help_text=_("Triggers when a matching object is created.") - ) - type_update = models.BooleanField( - verbose_name=_('on update'), - default=False, - help_text=_("Triggers when a matching object is updated.") - ) - type_delete = models.BooleanField( - verbose_name=_('on delete'), - default=False, - help_text=_("Triggers when a matching object is deleted.") - ) - type_job_start = models.BooleanField( - verbose_name=_('on job start'), - default=False, - help_text=_("Triggers when a job for a matching object is started.") - ) - type_job_end = models.BooleanField( - verbose_name=_('on job end'), - default=False, - help_text=_("Triggers when a job for a matching object terminates.") - ) + events = GenericRelation(EventRule) + payload_url = models.CharField( max_length=500, verbose_name=_('URL'), @@ -199,10 +165,6 @@ class Webhook(CustomFieldsMixin, ExportTemplatesMixin, TagsMixin, ChangeLoggedMo "processing is supported with the same context as the request body." ) ) - enabled = models.BooleanField( - verbose_name=_('enabled'), - default=True - ) http_method = models.CharField( max_length=30, choices=WebhookHttpMethodChoices, @@ -245,12 +207,6 @@ class Webhook(CustomFieldsMixin, ExportTemplatesMixin, TagsMixin, ChangeLoggedMo "digest of the payload body using the secret as the key. The secret is not transmitted in the request." ) ) - conditions = models.JSONField( - verbose_name=_('conditions'), - blank=True, - null=True, - help_text=_("A set of conditions which determine whether the webhook will be generated.") - ) ssl_verification = models.BooleanField( default=True, verbose_name=_('SSL verification'), @@ -267,18 +223,12 @@ class Webhook(CustomFieldsMixin, ExportTemplatesMixin, TagsMixin, ChangeLoggedMo ) class Meta: - ordering = ('name',) - constraints = ( - models.UniqueConstraint( - fields=('payload_url', 'type_create', 'type_update', 'type_delete'), - name='%(app_label)s_%(class)s_unique_payload_url_types' - ), - ) + ordering = ('payload_url',) verbose_name = _('webhook') verbose_name_plural = _('webhooks') def __str__(self): - return self.name + return self.payload_url def get_absolute_url(self): return reverse('extras:webhook', args=[self.pk]) @@ -290,20 +240,6 @@ def docs_url(self): def clean(self): super().clean() - # At least one action type must be selected - if not any([ - self.type_create, self.type_update, self.type_delete, self.type_job_start, self.type_job_end - ]): - raise ValidationError( - _("At least one event type must be selected: create, update, delete, job_start, and/or job_end.") - ) - - if self.conditions: - try: - ConditionSet(self.conditions) - except ValueError as e: - raise ValidationError({'conditions': e}) - # CA file path requires SSL verification enabled if not self.ssl_verification and self.ca_file_path: raise ValidationError({ diff --git a/netbox/extras/tables/tables.py b/netbox/extras/tables/tables.py index c9c663722d6..4d7eefe1555 100644 --- a/netbox/extras/tables/tables.py +++ b/netbox/extras/tables/tables.py @@ -320,6 +320,9 @@ class EventRuleTable(NetBoxTable): verbose_name=_('Name'), linkify=True ) + event_type = tables.Column( + verbose_name=_('Event Type'), + ) content_types = columns.ContentTypesColumn( verbose_name=_('Content Types'), ) @@ -348,11 +351,11 @@ class EventRuleTable(NetBoxTable): class Meta(NetBoxTable.Meta): model = EventRule fields = ( - 'pk', 'id', 'name', 'content_types', 'enabled', 'type_create', 'type_update', 'type_delete', + 'pk', 'id', 'name', 'event_type', 'content_types', 'enabled', 'type_create', 'type_update', 'type_delete', 'type_job_start', 'type_job_end', 'tags', 'created', 'last_updated', ) default_columns = ( - 'pk', 'name', 'content_types', 'enabled', 'type_create', 'type_update', 'type_delete', 'type_job_start', + 'pk', 'name', 'event_type', 'content_types', 'enabled', 'type_create', 'type_update', 'type_delete', 'type_job_start', 'type_job_end', ) From 854b3ba63218445899c8b9e24f9b236e06083905 Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 31 Oct 2023 09:54:50 -0700 Subject: [PATCH 09/95] 14132 cleanup tables and detail views --- netbox/extras/migrations/0099_eventrule.py | 2 + netbox/extras/tables/tables.py | 31 +------ netbox/templates/extras/eventrule.html | 94 ++++++++++++++++++++++ netbox/templates/extras/webhook.html | 72 ----------------- 4 files changed, 100 insertions(+), 99 deletions(-) create mode 100644 netbox/templates/extras/eventrule.html diff --git a/netbox/extras/migrations/0099_eventrule.py b/netbox/extras/migrations/0099_eventrule.py index 3f0694b713f..9312a29f3c8 100644 --- a/netbox/extras/migrations/0099_eventrule.py +++ b/netbox/extras/migrations/0099_eventrule.py @@ -1,7 +1,9 @@ # Generated by Django 4.2.5 on 2023-10-31 14:37 +from django.contrib.contenttypes.models import ContentType from django.db import migrations, models import django.db.models.deletion +from extras.choices import * import extras.utils import taggit.managers import utilities.json diff --git a/netbox/extras/tables/tables.py b/netbox/extras/tables/tables.py index 4d7eefe1555..659d237e194 100644 --- a/netbox/extras/tables/tables.py +++ b/netbox/extras/tables/tables.py @@ -270,31 +270,10 @@ class Meta(NetBoxTable.Meta): class WebhookTable(NetBoxTable): - name = tables.Column( - verbose_name=_('Name'), + id = tables.Column( + verbose_name=_('ID'), linkify=True ) - content_types = columns.ContentTypesColumn( - verbose_name=_('Content Types'), - ) - enabled = columns.BooleanColumn( - verbose_name=_('Enabled'), - ) - type_create = columns.BooleanColumn( - verbose_name=_('Create') - ) - type_update = columns.BooleanColumn( - verbose_name=_('Update') - ) - type_delete = columns.BooleanColumn( - verbose_name=_('Delete') - ) - type_job_start = columns.BooleanColumn( - verbose_name=_('Job Start') - ) - type_job_end = columns.BooleanColumn( - verbose_name=_('Job End') - ) ssl_validation = columns.BooleanColumn( verbose_name=_('SSL Validation') ) @@ -305,13 +284,11 @@ class WebhookTable(NetBoxTable): class Meta(NetBoxTable.Meta): model = Webhook fields = ( - 'pk', 'id', 'name', 'content_types', 'enabled', 'type_create', 'type_update', 'type_delete', - 'type_job_start', 'type_job_end', 'http_method', 'payload_url', 'secret', 'ssl_validation', 'ca_file_path', + 'pk', 'id', 'http_method', 'payload_url', 'secret', 'ssl_validation', 'ca_file_path', 'tags', 'created', 'last_updated', ) default_columns = ( - 'pk', 'name', 'content_types', 'enabled', 'type_create', 'type_update', 'type_delete', 'type_job_start', - 'type_job_end', 'http_method', 'payload_url', + 'pk', 'id', 'http_method', 'payload_url', ) diff --git a/netbox/templates/extras/eventrule.html b/netbox/templates/extras/eventrule.html new file mode 100644 index 00000000000..375c30dbc52 --- /dev/null +++ b/netbox/templates/extras/eventrule.html @@ -0,0 +1,94 @@ +{% extends 'generic/object.html' %} +{% load helpers %} +{% load plugins %} +{% load i18n %} + +{% block content %} +
+
+
+
+ {% trans "Webhook" %} +
+
+ + + + + + + + + +
{% trans "Name" %}{{ object.name }}
{% trans "Enabled" %}{% checkmark object.enabled %}
+
+
+
+
+ {% trans "Events" %} +
+
+ + + + + + + + + + + + + + + + + + + + + +
{% trans "Create" %}{% checkmark object.type_create %}
{% trans "Update" %}{% checkmark object.type_update %}
{% trans "Delete" %}{% checkmark object.type_delete %}
{% trans "Job start" %}{% checkmark object.type_job_start %}
{% trans "Job end" %}{% checkmark object.type_job_end %}
+
+
+ {% plugin_left_page object %} +
+
+
+
+ {% trans "Assigned Models" %} +
+
+ + {% for ct in object.content_types.all %} + + + + {% endfor %} +
{{ ct }}
+
+
+
+
+ {% trans "Conditions" %} +
+
+ {% if object.conditions %} + 
{{ object.conditions|json }}
+ {% else %} +

{% trans "None" %}

+ {% endif %} +
+
+ {% include 'inc/panels/custom_fields.html' %} + {% include 'inc/panels/tags.html' %} + {% plugin_right_page object %} +
+
+
+
+ {% plugin_full_width_page object %} +
+
+{% endblock %} diff --git a/netbox/templates/extras/webhook.html b/netbox/templates/extras/webhook.html index 5137b0103a7..83db1a0f6fd 100644 --- a/netbox/templates/extras/webhook.html +++ b/netbox/templates/extras/webhook.html @@ -6,52 +6,6 @@ {% block content %}
-
-
- {% trans "Webhook" %} -
-
- - - - - - - - - -
{% trans "Name" %}{{ object.name }}
{% trans "Enabled" %}{% checkmark object.enabled %}
-
-
-
-
- {% trans "Events" %} -
-
- - - - - - - - - - - - - - - - - - - - - -
{% trans "Create" %}{% checkmark object.type_create %}
{% trans "Update" %}{% checkmark object.type_update %}
{% trans "Delete" %}{% checkmark object.type_delete %}
{% trans "Job start" %}{% checkmark object.type_job_start %}
{% trans "Job end" %}{% checkmark object.type_job_end %}
-
-
{% trans "HTTP Request" %} @@ -97,32 +51,6 @@
{% plugin_left_page object %}
-
-
- {% trans "Assigned Models" %} -
-
- - {% for ct in object.content_types.all %} - - - - {% endfor %} -
{{ ct }}
-
-
-
-
- {% trans "Conditions" %} -
-
- {% if object.conditions %} - 
{{ object.conditions|json }}
- {% else %} -

{% trans "None" %}

- {% endif %} -
-
{% trans "Additional Headers" %} From e0b10c984b31c2f8c8210edcaa7b57ca99935816 Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 31 Oct 2023 10:14:43 -0700 Subject: [PATCH 10/95] 14132 form cleanup --- netbox/extras/forms/model_forms.py | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/netbox/extras/forms/model_forms.py b/netbox/extras/forms/model_forms.py index 24039c2eebd..923fb06c2aa 100644 --- a/netbox/extras/forms/model_forms.py +++ b/netbox/extras/forms/model_forms.py @@ -224,36 +224,21 @@ class Meta: class WebhookForm(NetBoxModelForm): - content_types = ContentTypeMultipleChoiceField( - label=_('Content types'), - queryset=ContentType.objects.all(), - limit_choices_to=FeatureQuery('webhooks') - ) fieldsets = ( - (_('Webhook'), ('name', 'content_types', 'enabled', 'tags')), - (_('Events'), ('type_create', 'type_update', 'type_delete', 'type_job_start', 'type_job_end')), + (_('Webhook'), ('tags',)), (_('HTTP Request'), ( 'payload_url', 'http_method', 'http_content_type', 'additional_headers', 'body_template', 'secret', )), - (_('Conditions'), ('conditions',)), (_('SSL'), ('ssl_verification', 'ca_file_path')), ) class Meta: model = Webhook fields = '__all__' - labels = { - 'type_create': _('Creations'), - 'type_update': _('Updates'), - 'type_delete': _('Deletions'), - 'type_job_start': _('Job executions'), - 'type_job_end': _('Job terminations'), - } widgets = { 'additional_headers': forms.Textarea(attrs={'class': 'font-monospace'}), 'body_template': forms.Textarea(attrs={'class': 'font-monospace'}), - 'conditions': forms.Textarea(attrs={'class': 'font-monospace'}), } From f650f8880688a5b518cf734dcf39960feff10d95 Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 31 Oct 2023 10:21:02 -0700 Subject: [PATCH 11/95] 14132 api / graphql --- netbox/extras/api/serializers.py | 29 +++++++++++++++++++++++------ netbox/extras/api/urls.py | 1 + netbox/extras/api/views.py | 11 +++++++++++ netbox/extras/graphql/types.py | 10 +++++++++- 4 files changed, 44 insertions(+), 7 deletions(-) diff --git a/netbox/extras/api/serializers.py b/netbox/extras/api/serializers.py index c1fad99eead..c5ee1faba0b 100644 --- a/netbox/extras/api/serializers.py +++ b/netbox/extras/api/serializers.py @@ -39,6 +39,7 @@ 'CustomFieldSerializer', 'CustomLinkSerializer', 'DashboardSerializer', + 'EventRuleSerializer', 'ExportTemplateSerializer', 'ImageAttachmentSerializer', 'JournalEntrySerializer', @@ -58,22 +59,38 @@ # -# Webhooks +# Event Rules # -class WebhookSerializer(NetBoxModelSerializer): - url = serializers.HyperlinkedIdentityField(view_name='extras-api:webhook-detail') +class EventRuleSerializer(NetBoxModelSerializer): + url = serializers.HyperlinkedIdentityField(view_name='extras-api:eventrule-detail') content_types = ContentTypeField( queryset=ContentType.objects.filter(FeatureQuery('webhooks').get_query()), many=True ) class Meta: - model = Webhook + model = EventRule fields = [ 'id', 'url', 'display', 'content_types', 'name', 'type_create', 'type_update', 'type_delete', - 'type_job_start', 'type_job_end', 'payload_url', 'enabled', 'http_method', 'http_content_type', - 'additional_headers', 'body_template', 'secret', 'conditions', 'ssl_verification', 'ca_file_path', + 'type_job_start', 'type_job_end', 'enabled', 'conditions', 'event_type', + 'custom_fields', 'tags', 'created', 'last_updated', + ] + + +# +# Webhooks +# + +class WebhookSerializer(NetBoxModelSerializer): + url = serializers.HyperlinkedIdentityField(view_name='extras-api:webhook-detail') + + class Meta: + model = Webhook + fields = [ + 'id', 'url', 'display', + 'payload_url', 'http_method', 'http_content_type', + 'additional_headers', 'body_template', 'secret', 'ssl_verification', 'ca_file_path', 'custom_fields', 'tags', 'created', 'last_updated', ] diff --git a/netbox/extras/api/urls.py b/netbox/extras/api/urls.py index 5f2b324e6d4..1616b85549e 100644 --- a/netbox/extras/api/urls.py +++ b/netbox/extras/api/urls.py @@ -7,6 +7,7 @@ router = NetBoxRouter() router.APIRootView = views.ExtrasRootView +router.register('event-rules', views.EventRuleViewSet) router.register('webhooks', views.WebhookViewSet) router.register('custom-fields', views.CustomFieldViewSet) router.register('custom-field-choice-sets', views.CustomFieldChoiceSetViewSet) diff --git a/netbox/extras/api/views.py b/netbox/extras/api/views.py index f518275e0ed..75c69da1d7e 100644 --- a/netbox/extras/api/views.py +++ b/netbox/extras/api/views.py @@ -37,6 +37,17 @@ def get_view_name(self): return 'Extras' +# +# EventRules +# + +class EventRuleViewSet(NetBoxModelViewSet): + metadata_class = ContentTypeMetadata + queryset = EventRule.objects.all() + serializer_class = serializers.EventRuleSerializer + filterset_class = filtersets.EventRuleFilterSet + + # # Webhooks # diff --git a/netbox/extras/graphql/types.py b/netbox/extras/graphql/types.py index 068da02f2bc..4981ddd7206 100644 --- a/netbox/extras/graphql/types.py +++ b/netbox/extras/graphql/types.py @@ -8,6 +8,7 @@ 'CustomFieldChoiceSetType', 'CustomFieldType', 'CustomLinkType', + 'EventRuleType', 'ExportTemplateType', 'ImageAttachmentType', 'JournalEntryType', @@ -110,5 +111,12 @@ class WebhookType(OrganizationalObjectType): class Meta: model = models.Webhook - exclude = ('content_types', ) filterset_class = filtersets.WebhookFilterSet + + +class EventRuleType(OrganizationalObjectType): + + class Meta: + model = models.EventRule + exclude = ('content_types', ) + filterset_class = filtersets.EventRuleFilterSet From 5ca30fe81b80d3638426cdef28b379910e6cee62 Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 31 Oct 2023 11:06:34 -0700 Subject: [PATCH 12/95] 14132 restore webhook name --- netbox/extras/forms/model_forms.py | 2 +- netbox/extras/migrations/0099_eventrule.py | 8 - netbox/extras/models/models.py | 9 +- netbox/extras/tests/test_api.py | 54 ++++ netbox/extras/tests/test_event_rules.py | 332 +++++++++++++++++++++ netbox/templates/extras/webhook.html | 13 + 6 files changed, 407 insertions(+), 11 deletions(-) create mode 100644 netbox/extras/tests/test_event_rules.py diff --git a/netbox/extras/forms/model_forms.py b/netbox/extras/forms/model_forms.py index 923fb06c2aa..a06e500d1f4 100644 --- a/netbox/extras/forms/model_forms.py +++ b/netbox/extras/forms/model_forms.py @@ -226,7 +226,7 @@ class Meta: class WebhookForm(NetBoxModelForm): fieldsets = ( - (_('Webhook'), ('tags',)), + (_('Webhook'), ('name', 'tags',)), (_('HTTP Request'), ( 'payload_url', 'http_method', 'http_content_type', 'additional_headers', 'body_template', 'secret', )), diff --git a/netbox/extras/migrations/0099_eventrule.py b/netbox/extras/migrations/0099_eventrule.py index 9312a29f3c8..79cb002ee95 100644 --- a/netbox/extras/migrations/0099_eventrule.py +++ b/netbox/extras/migrations/0099_eventrule.py @@ -85,10 +85,6 @@ class Migration(migrations.Migration): }, ), migrations.RunPython(move_webhooks), - migrations.AlterModelOptions( - name='webhook', - options={'ordering': ('payload_url',)}, - ), migrations.RemoveConstraint( model_name='webhook', name='extras_webhook_unique_payload_url_types', @@ -105,10 +101,6 @@ class Migration(migrations.Migration): model_name='webhook', name='enabled', ), - migrations.RemoveField( - model_name='webhook', - name='name', - ), migrations.RemoveField( model_name='webhook', name='type_create', diff --git a/netbox/extras/models/models.py b/netbox/extras/models/models.py index dd5e83eb79f..137b2cb5721 100644 --- a/netbox/extras/models/models.py +++ b/netbox/extras/models/models.py @@ -157,6 +157,11 @@ class Webhook(CustomFieldsMixin, ExportTemplatesMixin, TagsMixin, ChangeLoggedMo """ events = GenericRelation(EventRule) + name = models.CharField( + verbose_name=_('name'), + max_length=150, + unique=True + ) payload_url = models.CharField( max_length=500, verbose_name=_('URL'), @@ -223,12 +228,12 @@ class Webhook(CustomFieldsMixin, ExportTemplatesMixin, TagsMixin, ChangeLoggedMo ) class Meta: - ordering = ('payload_url',) + ordering = ('name',) verbose_name = _('webhook') verbose_name_plural = _('webhooks') def __str__(self): - return self.payload_url + return self.name def get_absolute_url(self): return reverse('extras:webhook', args=[self.pk]) diff --git a/netbox/extras/tests/test_api.py b/netbox/extras/tests/test_api.py index 255457f21b9..5d0bd3797c6 100644 --- a/netbox/extras/tests/test_api.py +++ b/netbox/extras/tests/test_api.py @@ -27,6 +27,60 @@ def test_root(self): self.assertEqual(response.status_code, 200) +class EventRuleTest(APIViewTestCases.APIViewTestCase): + model = EventRule + brief_fields = ['display', 'id', 'name', 'url'] + create_data = [ + { + 'content_types': ['dcim.device', 'dcim.devicetype'], + 'name': 'Event Rule 4', + 'type_create': True, + 'payload_url': 'http://example.com/?4', + }, + { + 'content_types': ['dcim.device', 'dcim.devicetype'], + 'name': 'Event Rule 5', + 'type_update': True, + 'payload_url': 'http://example.com/?5', + }, + { + 'content_types': ['dcim.device', 'dcim.devicetype'], + 'name': 'Event Rule 6', + 'type_delete': True, + 'payload_url': 'http://example.com/?6', + }, + ] + bulk_update_data = { + 'ssl_verification': False, + } + + @classmethod + def setUpTestData(cls): + site_ct = ContentType.objects.get_for_model(Site) + rack_ct = ContentType.objects.get_for_model(Rack) + + webhooks = ( + Webhook( + name='Webhook 1', + type_create=True, + payload_url='http://example.com/?1', + ), + Webhook( + name='Webhook 2', + type_update=True, + payload_url='http://example.com/?1', + ), + Webhook( + name='Webhook 3', + type_delete=True, + payload_url='http://example.com/?1', + ), + ) + Webhook.objects.bulk_create(webhooks) + for webhook in webhooks: + webhook.content_types.add(site_ct, rack_ct) + + class WebhookTest(APIViewTestCases.APIViewTestCase): model = Webhook brief_fields = ['display', 'id', 'name', 'url'] diff --git a/netbox/extras/tests/test_event_rules.py b/netbox/extras/tests/test_event_rules.py new file mode 100644 index 00000000000..d8bfe422958 --- /dev/null +++ b/netbox/extras/tests/test_event_rules.py @@ -0,0 +1,332 @@ +import json +import uuid +from unittest.mock import patch + +import django_rq +from django.contrib.contenttypes.models import ContentType +from django.http import HttpResponse +from django.urls import reverse +from requests import Session +from rest_framework import status + +from dcim.choices import SiteStatusChoices +from dcim.models import Site +from extras.choices import ObjectChangeActionChoices +from extras.models import Tag, EventRule +from extras.webhooks import enqueue_object, flush_events, generate_signature, serialize_for_webhook +from extras.webhooks_worker import eval_conditions, process_webhook +from utilities.testing import APITestCase + + +class EventRuleTest(APITestCase): + + def setUp(self): + super().setUp() + + # Ensure the queue has been cleared for each test + self.queue = django_rq.get_queue('default') + self.queue.empty() + + @classmethod + def setUpTestData(cls): + + site_ct = ContentType.objects.get_for_model(Site) + DUMMY_URL = 'http://localhost:9000/' + DUMMY_SECRET = 'LOOKATMEIMASECRETSTRING' + + webhooks = EventRule.objects.bulk_create(( + Webhook(name='Webhook 1', type_create=True), + Webhook(name='Webhook 2', type_update=True), + Webhook(name='Webhook 3', type_delete=True), + )) + for webhook in webhooks: + webhook.content_types.set([site_ct]) + + Tag.objects.bulk_create(( + Tag(name='Foo', slug='foo'), + Tag(name='Bar', slug='bar'), + Tag(name='Baz', slug='baz'), + )) + + def test_enqueue_webhook_create(self): + # Create an object via the REST API + data = { + 'name': 'Site 1', + 'slug': 'site-1', + 'tags': [ + {'name': 'Foo'}, + {'name': 'Bar'}, + ] + } + url = reverse('dcim-api:site-list') + self.add_permissions('dcim.add_site') + response = self.client.post(url, data, format='json', **self.header) + self.assertHttpStatus(response, status.HTTP_201_CREATED) + self.assertEqual(Site.objects.count(), 1) + self.assertEqual(Site.objects.first().tags.count(), 2) + + # Verify that a job was queued for the object creation webhook + self.assertEqual(self.queue.count, 1) + job = self.queue.jobs[0] + self.assertEqual(job.kwargs['webhook'], Webhook.objects.get(type_create=True)) + self.assertEqual(job.kwargs['event'], ObjectChangeActionChoices.ACTION_CREATE) + self.assertEqual(job.kwargs['model_name'], 'site') + self.assertEqual(job.kwargs['data']['id'], response.data['id']) + self.assertEqual(len(job.kwargs['data']['tags']), len(response.data['tags'])) + self.assertEqual(job.kwargs['snapshots']['postchange']['name'], 'Site 1') + self.assertEqual(job.kwargs['snapshots']['postchange']['tags'], ['Bar', 'Foo']) + + def test_enqueue_webhook_bulk_create(self): + # Create multiple objects via the REST API + data = [ + { + 'name': 'Site 1', + 'slug': 'site-1', + 'tags': [ + {'name': 'Foo'}, + {'name': 'Bar'}, + ] + }, + { + 'name': 'Site 2', + 'slug': 'site-2', + 'tags': [ + {'name': 'Foo'}, + {'name': 'Bar'}, + ] + }, + { + 'name': 'Site 3', + 'slug': 'site-3', + 'tags': [ + {'name': 'Foo'}, + {'name': 'Bar'}, + ] + }, + ] + url = reverse('dcim-api:site-list') + self.add_permissions('dcim.add_site') + response = self.client.post(url, data, format='json', **self.header) + self.assertHttpStatus(response, status.HTTP_201_CREATED) + self.assertEqual(Site.objects.count(), 3) + self.assertEqual(Site.objects.first().tags.count(), 2) + + # Verify that a webhook was queued for each object + self.assertEqual(self.queue.count, 3) + for i, job in enumerate(self.queue.jobs): + self.assertEqual(job.kwargs['webhook'], Webhook.objects.get(type_create=True)) + self.assertEqual(job.kwargs['event'], ObjectChangeActionChoices.ACTION_CREATE) + self.assertEqual(job.kwargs['model_name'], 'site') + self.assertEqual(job.kwargs['data']['id'], response.data[i]['id']) + self.assertEqual(len(job.kwargs['data']['tags']), len(response.data[i]['tags'])) + self.assertEqual(job.kwargs['snapshots']['postchange']['name'], response.data[i]['name']) + self.assertEqual(job.kwargs['snapshots']['postchange']['tags'], ['Bar', 'Foo']) + + def test_enqueue_webhook_update(self): + site = Site.objects.create(name='Site 1', slug='site-1') + site.tags.set(Tag.objects.filter(name__in=['Foo', 'Bar'])) + + # Update an object via the REST API + data = { + 'name': 'Site X', + 'comments': 'Updated the site', + 'tags': [ + {'name': 'Baz'} + ] + } + url = reverse('dcim-api:site-detail', kwargs={'pk': site.pk}) + self.add_permissions('dcim.change_site') + response = self.client.patch(url, data, format='json', **self.header) + self.assertHttpStatus(response, status.HTTP_200_OK) + + # Verify that a job was queued for the object update webhook + self.assertEqual(self.queue.count, 1) + job = self.queue.jobs[0] + self.assertEqual(job.kwargs['webhook'], Webhook.objects.get(type_update=True)) + self.assertEqual(job.kwargs['event'], ObjectChangeActionChoices.ACTION_UPDATE) + self.assertEqual(job.kwargs['model_name'], 'site') + self.assertEqual(job.kwargs['data']['id'], site.pk) + self.assertEqual(len(job.kwargs['data']['tags']), len(response.data['tags'])) + self.assertEqual(job.kwargs['snapshots']['prechange']['name'], 'Site 1') + self.assertEqual(job.kwargs['snapshots']['prechange']['tags'], ['Bar', 'Foo']) + self.assertEqual(job.kwargs['snapshots']['postchange']['name'], 'Site X') + self.assertEqual(job.kwargs['snapshots']['postchange']['tags'], ['Baz']) + + def test_enqueue_webhook_bulk_update(self): + sites = ( + Site(name='Site 1', slug='site-1'), + Site(name='Site 2', slug='site-2'), + Site(name='Site 3', slug='site-3'), + ) + Site.objects.bulk_create(sites) + for site in sites: + site.tags.set(Tag.objects.filter(name__in=['Foo', 'Bar'])) + + # Update three objects via the REST API + data = [ + { + 'id': sites[0].pk, + 'name': 'Site X', + 'tags': [ + {'name': 'Baz'} + ] + }, + { + 'id': sites[1].pk, + 'name': 'Site Y', + 'tags': [ + {'name': 'Baz'} + ] + }, + { + 'id': sites[2].pk, + 'name': 'Site Z', + 'tags': [ + {'name': 'Baz'} + ] + }, + ] + url = reverse('dcim-api:site-list') + self.add_permissions('dcim.change_site') + response = self.client.patch(url, data, format='json', **self.header) + self.assertHttpStatus(response, status.HTTP_200_OK) + + # Verify that a job was queued for the object update webhook + self.assertEqual(self.queue.count, 3) + for i, job in enumerate(self.queue.jobs): + self.assertEqual(job.kwargs['webhook'], Webhook.objects.get(type_update=True)) + self.assertEqual(job.kwargs['event'], ObjectChangeActionChoices.ACTION_UPDATE) + self.assertEqual(job.kwargs['model_name'], 'site') + self.assertEqual(job.kwargs['data']['id'], data[i]['id']) + self.assertEqual(len(job.kwargs['data']['tags']), len(response.data[i]['tags'])) + self.assertEqual(job.kwargs['snapshots']['prechange']['name'], sites[i].name) + self.assertEqual(job.kwargs['snapshots']['prechange']['tags'], ['Bar', 'Foo']) + self.assertEqual(job.kwargs['snapshots']['postchange']['name'], response.data[i]['name']) + self.assertEqual(job.kwargs['snapshots']['postchange']['tags'], ['Baz']) + + def test_enqueue_webhook_delete(self): + site = Site.objects.create(name='Site 1', slug='site-1') + site.tags.set(Tag.objects.filter(name__in=['Foo', 'Bar'])) + + # Delete an object via the REST API + url = reverse('dcim-api:site-detail', kwargs={'pk': site.pk}) + self.add_permissions('dcim.delete_site') + response = self.client.delete(url, **self.header) + self.assertHttpStatus(response, status.HTTP_204_NO_CONTENT) + + # Verify that a job was queued for the object update webhook + self.assertEqual(self.queue.count, 1) + job = self.queue.jobs[0] + self.assertEqual(job.kwargs['webhook'], Webhook.objects.get(type_delete=True)) + self.assertEqual(job.kwargs['event'], ObjectChangeActionChoices.ACTION_DELETE) + self.assertEqual(job.kwargs['model_name'], 'site') + self.assertEqual(job.kwargs['data']['id'], site.pk) + self.assertEqual(job.kwargs['snapshots']['prechange']['name'], 'Site 1') + self.assertEqual(job.kwargs['snapshots']['prechange']['tags'], ['Bar', 'Foo']) + + def test_enqueue_webhook_bulk_delete(self): + sites = ( + Site(name='Site 1', slug='site-1'), + Site(name='Site 2', slug='site-2'), + Site(name='Site 3', slug='site-3'), + ) + Site.objects.bulk_create(sites) + for site in sites: + site.tags.set(Tag.objects.filter(name__in=['Foo', 'Bar'])) + + # Delete three objects via the REST API + data = [ + {'id': site.pk} for site in sites + ] + url = reverse('dcim-api:site-list') + self.add_permissions('dcim.delete_site') + response = self.client.delete(url, data, format='json', **self.header) + self.assertHttpStatus(response, status.HTTP_204_NO_CONTENT) + + # Verify that a job was queued for the object update webhook + self.assertEqual(self.queue.count, 3) + for i, job in enumerate(self.queue.jobs): + self.assertEqual(job.kwargs['webhook'], Webhook.objects.get(type_delete=True)) + self.assertEqual(job.kwargs['event'], ObjectChangeActionChoices.ACTION_DELETE) + self.assertEqual(job.kwargs['model_name'], 'site') + self.assertEqual(job.kwargs['data']['id'], sites[i].pk) + self.assertEqual(job.kwargs['snapshots']['prechange']['name'], sites[i].name) + self.assertEqual(job.kwargs['snapshots']['prechange']['tags'], ['Bar', 'Foo']) + + def test_webhook_conditions(self): + # Create a conditional Webhook + webhook = Webhook( + name='Conditional Webhook', + type_create=True, + type_update=True, + payload_url='http://localhost:9000/', + conditions={ + 'and': [ + { + 'attr': 'status.value', + 'value': 'active', + } + ] + } + ) + + # Create a Site to evaluate + site = Site.objects.create(name='Site 1', slug='site-1', status=SiteStatusChoices.STATUS_STAGING) + data = serialize_for_webhook(site) + + # Evaluate the conditions (status='staging') + self.assertFalse(eval_conditions(webhook, data)) + + # Change the site's status + site.status = SiteStatusChoices.STATUS_ACTIVE + data = serialize_for_webhook(site) + + # Evaluate the conditions (status='active') + self.assertTrue(eval_conditions(webhook, data)) + + def test_webhooks_worker(self): + + request_id = uuid.uuid4() + + def dummy_send(_, request, **kwargs): + """ + A dummy implementation of Session.send() to be used for testing. + Always returns a 200 HTTP response. + """ + webhook = Webhook.objects.get(type_create=True) + signature = generate_signature(request.body, webhook.secret) + + # Validate the outgoing request headers + self.assertEqual(request.headers['Content-Type'], webhook.http_content_type) + self.assertEqual(request.headers['X-Hook-Signature'], signature) + self.assertEqual(request.headers['X-Foo'], 'Bar') + + # Validate the outgoing request body + body = json.loads(request.body) + self.assertEqual(body['event'], 'created') + self.assertEqual(body['timestamp'], job.kwargs['timestamp']) + self.assertEqual(body['model'], 'site') + self.assertEqual(body['username'], 'testuser') + self.assertEqual(body['request_id'], str(request_id)) + self.assertEqual(body['data']['name'], 'Site 1') + + return HttpResponse() + + # Enqueue a webhook for processing + events_queue = [] + site = Site.objects.create(name='Site 1', slug='site-1') + enqueue_object( + events_queue, + instance=site, + user=self.user, + request_id=request_id, + action=ObjectChangeActionChoices.ACTION_CREATE + ) + flush_events(events_queue) + + # Retrieve the job from queue + job = self.queue.jobs[0] + + # Patch the Session object with our dummy_send() method, then process the webhook for sending + with patch.object(Session, 'send', dummy_send) as mock_send: + process_webhook(**job.kwargs) diff --git a/netbox/templates/extras/webhook.html b/netbox/templates/extras/webhook.html index 83db1a0f6fd..c4b41faa144 100644 --- a/netbox/templates/extras/webhook.html +++ b/netbox/templates/extras/webhook.html @@ -6,6 +6,19 @@ {% block content %}
+
+
+ {% trans "Webhook" %} +
+
+ + + + + +
{% trans "Name" %}{{ object.name }}
+
+
{% trans "HTTP Request" %} From 4ef7ab83d047b964f331df7ad0b49c9c85451963 Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 31 Oct 2023 11:30:22 -0700 Subject: [PATCH 13/95] 14132 update table --- netbox/extras/tables/tables.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/netbox/extras/tables/tables.py b/netbox/extras/tables/tables.py index 659d237e194..0fc53cfd748 100644 --- a/netbox/extras/tables/tables.py +++ b/netbox/extras/tables/tables.py @@ -270,8 +270,8 @@ class Meta(NetBoxTable.Meta): class WebhookTable(NetBoxTable): - id = tables.Column( - verbose_name=_('ID'), + name = tables.Column( + verbose_name=_('Name'), linkify=True ) ssl_validation = columns.BooleanColumn( @@ -284,11 +284,11 @@ class WebhookTable(NetBoxTable): class Meta(NetBoxTable.Meta): model = Webhook fields = ( - 'pk', 'id', 'http_method', 'payload_url', 'secret', 'ssl_validation', 'ca_file_path', + 'pk', 'id', 'name', 'http_method', 'payload_url', 'secret', 'ssl_validation', 'ca_file_path', 'tags', 'created', 'last_updated', ) default_columns = ( - 'pk', 'id', 'http_method', 'payload_url', + 'pk', 'name', 'http_method', 'payload_url', ) From e74fb89a30dcc3803307a8dcac35cda9685be29f Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 31 Oct 2023 15:59:27 -0700 Subject: [PATCH 14/95] 14132 base process_event --- netbox/core/models/jobs.py | 20 +++++++------- netbox/extras/context_managers.py | 2 +- netbox/extras/{webhooks.py => events.py} | 30 ++++++-------------- netbox/extras/signals.py | 4 +-- netbox/extras/tests/test_event_rules.py | 7 +++-- netbox/extras/tests/test_webhooks.py | 7 +++-- netbox/extras/webhooks_worker.py | 35 +----------------------- netbox/netbox/settings.py | 4 +++ 8 files changed, 35 insertions(+), 74 deletions(-) rename netbox/extras/{webhooks.py => events.py} (81%) diff --git a/netbox/core/models/jobs.py b/netbox/core/models/jobs.py index 61b0e64fab0..08e894f5973 100644 --- a/netbox/core/models/jobs.py +++ b/netbox/core/models/jobs.py @@ -155,8 +155,8 @@ def start(self): self.status = JobStatusChoices.STATUS_RUNNING self.save() - # Handle webhooks - self.trigger_webhooks(event=EVENT_JOB_START) + # Handle events + self.trigger_events(event=EVENT_JOB_START) def terminate(self, status=JobStatusChoices.STATUS_COMPLETED): """ @@ -171,8 +171,8 @@ def terminate(self, status=JobStatusChoices.STATUS_COMPLETED): self.completed = timezone.now() self.save() - # Handle webhooks - self.trigger_webhooks(event=EVENT_JOB_END) + # Handle events + self.trigger_events(event=EVENT_JOB_END) @classmethod def enqueue(cls, func, instance, name='', user=None, schedule_at=None, interval=None, **kwargs): @@ -209,23 +209,23 @@ def enqueue(cls, func, instance, name='', user=None, schedule_at=None, interval= return job - def trigger_webhooks(self, event): - from extras.models import Webhook + def trigger_events(self, event): + from extras.models import EventRule rq_queue_name = get_config().QUEUE_MAPPINGS.get('webhook', RQ_QUEUE_DEFAULT) rq_queue = django_rq.get_queue(rq_queue_name, is_async=False) # Fetch any webhooks matching this object type and action - webhooks = Webhook.objects.filter( + event_rules = EventRule.objects.filter( **{f'type_{event}': True}, content_types=self.object_type, enabled=True ) - for webhook in webhooks: + for event_rule in event_rules: rq_queue.enqueue( - "extras.webhooks_worker.process_webhook", - webhook=webhook, + "extras.events_worker.process_event", + event_rule=event_rule, model_name=self.object_type.model, event=event, data=self.data, diff --git a/netbox/extras/context_managers.py b/netbox/extras/context_managers.py index c3c78f98bf8..dc4d3dd177d 100644 --- a/netbox/extras/context_managers.py +++ b/netbox/extras/context_managers.py @@ -1,7 +1,7 @@ from contextlib import contextmanager from netbox.context import current_request, events_queue -from .webhooks import flush_events +from .events import flush_events @contextmanager diff --git a/netbox/extras/webhooks.py b/netbox/extras/events.py similarity index 81% rename from netbox/extras/webhooks.py rename to netbox/extras/events.py index 2dc474e8a37..98a81e5cb33 100644 --- a/netbox/extras/webhooks.py +++ b/netbox/extras/events.py @@ -15,7 +15,7 @@ from .models import Webhook -def serialize_for_webhook(instance): +def serialize_for_event(instance): """ Return a serialized representation of the given instance suitable for use in a webhook. """ @@ -43,18 +43,6 @@ def get_snapshots(instance, action): return snapshots -def generate_signature(request_body, secret): - """ - Return a cryptographic signature that can be used to verify the authenticity of webhook data. - """ - hmac_prep = hmac.new( - key=secret.encode('utf8'), - msg=request_body, - digestmod=hashlib.sha512 - ) - return hmac_prep.hexdigest() - - def enqueue_object(queue, instance, user, request_id, action): """ Enqueue a serialized representation of a created/updated/deleted object for the processing of @@ -70,7 +58,7 @@ def enqueue_object(queue, instance, user, request_id, action): 'content_type': ContentType.objects.get_for_model(instance), 'object_id': instance.pk, 'event': action, - 'data': serialize_for_webhook(instance), + 'data': serialize_for_event(instance), 'snapshots': get_snapshots(instance, action), 'username': user.username, 'request_id': request_id @@ -83,7 +71,7 @@ def flush_events(queue): """ rq_queue_name = get_config().QUEUE_MAPPINGS.get('webhook', RQ_QUEUE_DEFAULT) rq_queue = get_queue(rq_queue_name) - webhooks_cache = { + events_cache = { 'type_create': {}, 'type_update': {}, 'type_delete': {}, @@ -99,18 +87,18 @@ def flush_events(queue): content_type = data['content_type'] # Cache applicable Webhooks - if content_type not in webhooks_cache[action_flag]: - webhooks_cache[action_flag][content_type] = Webhook.objects.filter( + if content_type not in events_cache[action_flag]: + events_cache[action_flag][content_type] = Webhook.objects.filter( **{action_flag: True}, content_types=content_type, enabled=True ) - webhooks = webhooks_cache[action_flag][content_type] + event_rules = events_cache[action_flag][content_type] - for webhook in webhooks: + for event_rule in event_rules: rq_queue.enqueue( - "extras.webhooks_worker.process_webhook", - webhook=webhook, + "extras.events_worker.process_event", + event_rule=event_rule, model_name=content_type.model, event=data['event'], data=data['data'], diff --git a/netbox/extras/signals.py b/netbox/extras/signals.py index 86d88b6c3e4..a0ea07456ed 100644 --- a/netbox/extras/signals.py +++ b/netbox/extras/signals.py @@ -15,7 +15,7 @@ from utilities.exceptions import AbortRequest from .choices import ObjectChangeActionChoices from .models import ConfigRevision, CustomField, ObjectChange, TaggedItem -from .webhooks import enqueue_object, get_snapshots, serialize_for_webhook +from .events import enqueue_object, get_snapshots, serialize_for_event # # Change logging/webhooks @@ -84,7 +84,7 @@ def handle_changed_object(sender, instance, **kwargs): queue = events_queue.get() if m2m_changed and queue and is_same_object(instance, queue[-1], request.id): instance.refresh_from_db() # Ensure that we're working with fresh M2M assignments - queue[-1]['data'] = serialize_for_webhook(instance) + queue[-1]['data'] = serialize_for_event(instance) queue[-1]['snapshots']['postchange'] = get_snapshots(instance, action)['postchange'] else: enqueue_object(queue, instance, request.user, request.id, action) diff --git a/netbox/extras/tests/test_event_rules.py b/netbox/extras/tests/test_event_rules.py index d8bfe422958..53f8afb7d9e 100644 --- a/netbox/extras/tests/test_event_rules.py +++ b/netbox/extras/tests/test_event_rules.py @@ -13,7 +13,8 @@ from dcim.models import Site from extras.choices import ObjectChangeActionChoices from extras.models import Tag, EventRule -from extras.webhooks import enqueue_object, flush_events, generate_signature, serialize_for_webhook +from extras.events import enqueue_object, flush_events, serialize_for_event +from extras.webhooks import generate_signature from extras.webhooks_worker import eval_conditions, process_webhook from utilities.testing import APITestCase @@ -272,14 +273,14 @@ def test_webhook_conditions(self): # Create a Site to evaluate site = Site.objects.create(name='Site 1', slug='site-1', status=SiteStatusChoices.STATUS_STAGING) - data = serialize_for_webhook(site) + data = serialize_for_event(site) # Evaluate the conditions (status='staging') self.assertFalse(eval_conditions(webhook, data)) # Change the site's status site.status = SiteStatusChoices.STATUS_ACTIVE - data = serialize_for_webhook(site) + data = serialize_for_event(site) # Evaluate the conditions (status='active') self.assertTrue(eval_conditions(webhook, data)) diff --git a/netbox/extras/tests/test_webhooks.py b/netbox/extras/tests/test_webhooks.py index c309250812c..e86814e4cff 100644 --- a/netbox/extras/tests/test_webhooks.py +++ b/netbox/extras/tests/test_webhooks.py @@ -13,7 +13,8 @@ from dcim.models import Site from extras.choices import ObjectChangeActionChoices from extras.models import Tag, Webhook -from extras.webhooks import enqueue_object, flush_events, generate_signature, serialize_for_webhook +from extras.events import enqueue_object, flush_events, serialize_for_event +from extras.webhooks import generate_signature from extras.webhooks_worker import eval_conditions, process_webhook from utilities.testing import APITestCase @@ -272,14 +273,14 @@ def test_webhook_conditions(self): # Create a Site to evaluate site = Site.objects.create(name='Site 1', slug='site-1', status=SiteStatusChoices.STATUS_STAGING) - data = serialize_for_webhook(site) + data = serialize_for_event(site) # Evaluate the conditions (status='staging') self.assertFalse(eval_conditions(webhook, data)) # Change the site's status site.status = SiteStatusChoices.STATUS_ACTIVE - data = serialize_for_webhook(site) + data = serialize_for_event(site) # Evaluate the conditions (status='active') self.assertTrue(eval_conditions(webhook, data)) diff --git a/netbox/extras/webhooks_worker.py b/netbox/extras/webhooks_worker.py index 438231b7e11..9f64fc405dc 100644 --- a/netbox/extras/webhooks_worker.py +++ b/netbox/extras/webhooks_worker.py @@ -12,43 +12,10 @@ logger = logging.getLogger('netbox.webhooks_worker') -def eval_conditions(webhook, data): - """ - Test whether the given data meets the conditions of the webhook (if any). Return True - if met or no conditions are specified. - """ - if not webhook.conditions: - return True - - logger.debug(f'Evaluating webhook conditions: {webhook.conditions}') - if ConditionSet(webhook.conditions).eval(data): - return True - - return False - - -@job('default') -def process_webhook(webhook, model_name, event, data, timestamp, username, request_id=None, snapshots=None): +def process_webhook(webhook, model_name, event, data, timestamp, username, request_id=None): """ Make a POST request to the defined Webhook """ - # Evaluate webhook conditions (if any) - if not eval_conditions(webhook, data): - return - - # Prepare context data for headers & body templates - context = { - 'event': WEBHOOK_EVENT_TYPES[event], - 'timestamp': timestamp, - 'model': model_name, - 'username': username, - 'request_id': request_id, - 'data': data, - } - if snapshots: - context.update({ - 'snapshots': snapshots - }) # Build the headers for the HTTP request headers = { diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index 36f56b0f634..72d22d84d1b 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -174,6 +174,10 @@ TIME_FORMAT = getattr(configuration, 'TIME_FORMAT', 'g:i a') TIME_ZONE = getattr(configuration, 'TIME_ZONE', 'UTC') ENABLE_LOCALIZATION = getattr(configuration, 'ENABLE_LOCALIZATION', False) +NETBOX_EVENTS_PIPELINE = getattr(configuration, 'NETBOX_EVENTS_PIPELINE', ( + 'extras.events_worker.process_event_rules', +)) + # Check for hard-coded dynamic config parameters for param in PARAMS: From aa8ed1194a6f2ff19d9e3069f32f921fdc5b851c Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 1 Nov 2023 07:40:31 -0700 Subject: [PATCH 15/95] 14132 fix --- netbox/extras/events.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/netbox/extras/events.py b/netbox/extras/events.py index 98a81e5cb33..e3df359c027 100644 --- a/netbox/extras/events.py +++ b/netbox/extras/events.py @@ -12,7 +12,7 @@ from utilities.rqworker import get_rq_retry from utilities.utils import serialize_object from .choices import * -from .models import Webhook +from .models import EventRule, Webhook def serialize_for_event(instance): @@ -86,9 +86,9 @@ def flush_events(queue): }[data['event']] content_type = data['content_type'] - # Cache applicable Webhooks + # Cache applicable Event Rules if content_type not in events_cache[action_flag]: - events_cache[action_flag][content_type] = Webhook.objects.filter( + events_cache[action_flag][content_type] = EventRule.objects.filter( **{action_flag: True}, content_types=content_type, enabled=True From 36285db2cae08a11d4aa8de6e11c095bea9820c5 Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 1 Nov 2023 07:41:26 -0700 Subject: [PATCH 16/95] 14132 fix --- netbox/extras/webhooks.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 netbox/extras/webhooks.py diff --git a/netbox/extras/webhooks.py b/netbox/extras/webhooks.py new file mode 100644 index 00000000000..a48a8038b3d --- /dev/null +++ b/netbox/extras/webhooks.py @@ -0,0 +1,14 @@ +import hashlib +import hmac + + +def generate_signature(request_body, secret): + """ + Return a cryptographic signature that can be used to verify the authenticity of webhook data. + """ + hmac_prep = hmac.new( + key=secret.encode('utf8'), + msg=request_body, + digestmod=hashlib.sha512 + ) + return hmac_prep.hexdigest() From 4a941775be3cef4d8d7d3e9d739fa0227ebc799f Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 1 Nov 2023 07:42:02 -0700 Subject: [PATCH 17/95] 14132 fix --- netbox/extras/events_worker.py | 71 +++++++++++++++++++++++++++++++++ netbox/extras/scripts_worker.py | 20 ++++++++++ 2 files changed, 91 insertions(+) create mode 100644 netbox/extras/events_worker.py create mode 100644 netbox/extras/scripts_worker.py diff --git a/netbox/extras/events_worker.py b/netbox/extras/events_worker.py new file mode 100644 index 00000000000..76534159ba1 --- /dev/null +++ b/netbox/extras/events_worker.py @@ -0,0 +1,71 @@ +import logging + +import requests +from django.conf import settings +from django_rq import job +from jinja2.exceptions import TemplateError + +from scripts.workers import process_script +from .conditions import ConditionSet +from .constants import WEBHOOK_EVENT_TYPES, EVENT_TYPE_MODELS +from .webhooks import generate_signature +from webhooks_worker import process_webhook + +logger = logging.getLogger('netbox.events_worker') + + +def eval_conditions(event_rule, data): + """ + Test whether the given data meets the conditions of the event rule (if any). Return True + if met or no conditions are specified. + """ + if not event_rule.conditions: + return True + + logger.debug(f'Evaluating event rule conditions: {event_rule.conditions}') + if ConditionSet(event_rule.conditions).eval(data): + return True + + return False + + +def module_member(name): + mod, member = name.rsplit(".", 1) + module = import_module(mod) + return getattr(module, member) + + +def process_event_rules(event_rule, model_name, event, data, timestamp, username, request_id): + if event_rule.event_type == EVENT_TYPE_MODELS.WEBHOOK: + process_webhook(event_rule, model_name, event, data, timestamp, username, request_id) + elif event_rule.event_type == EVENT_TYPE_MODELS.SCRIPT: + process_script(event_rule, model_name, event, data, timestamp, username, request_id) + + +@job('default') +def process_event(event_rule, model_name, event, data, timestamp, username, request_id=None, snapshots=None): + """ + Make a POST request to the defined Webhook + """ + # Evaluate event rule conditions (if any) + if not eval_conditions(event_rule, data): + return + + # Prepare context data for headers & body templates + context = { + 'event': WEBHOOK_EVENT_TYPES[event], + 'timestamp': timestamp, + 'model': model_name, + 'username': username, + 'request_id': request_id, + 'data': data, + } + if snapshots: + context.update({ + 'snapshots': snapshots + }) + + # process the events pipeline + for name in settings.NETBOX_EVENTS_PIPELINE: + func = module_member(name) + func(event_rule, model_name, event, data, timestamp, username, request_id) diff --git a/netbox/extras/scripts_worker.py b/netbox/extras/scripts_worker.py new file mode 100644 index 00000000000..e014463a28b --- /dev/null +++ b/netbox/extras/scripts_worker.py @@ -0,0 +1,20 @@ +import logging + +import requests +from django.conf import settings +from django_rq import job +from jinja2.exceptions import TemplateError + +from .conditions import ConditionSet +from .constants import WEBHOOK_EVENT_TYPES +from .webhooks import generate_signature + +logger = logging.getLogger('netbox.webhooks_worker') + + +def process_script(webhook, model_name, event, data, timestamp, username, request_id=None): + """ + Make a POST request to the defined Webhook + """ + + pass From 7770637e3a5f2b376b21072b4361499d97a4f8ef Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 1 Nov 2023 10:24:14 -0700 Subject: [PATCH 18/95] 14132 fix event and webhook worker --- netbox/extras/events_worker.py | 17 ++++++++++++----- netbox/extras/migrations/0099_eventrule.py | 2 +- netbox/extras/webhooks_worker.py | 4 +++- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/netbox/extras/events_worker.py b/netbox/extras/events_worker.py index 76534159ba1..2d09937b1fb 100644 --- a/netbox/extras/events_worker.py +++ b/netbox/extras/events_worker.py @@ -1,15 +1,17 @@ import logging import requests +import sys from django.conf import settings from django_rq import job from jinja2.exceptions import TemplateError -from scripts.workers import process_script from .conditions import ConditionSet -from .constants import WEBHOOK_EVENT_TYPES, EVENT_TYPE_MODELS +from .choices import EventRuleTypeChoices +from .constants import WEBHOOK_EVENT_TYPES +from .scripts_worker import process_script from .webhooks import generate_signature -from webhooks_worker import process_webhook +from .webhooks_worker import process_webhook logger = logging.getLogger('netbox.events_worker') @@ -29,6 +31,11 @@ def eval_conditions(event_rule, data): return False +def import_module(name): + __import__(name) + return sys.modules[name] + + def module_member(name): mod, member = name.rsplit(".", 1) module = import_module(mod) @@ -36,9 +43,9 @@ def module_member(name): def process_event_rules(event_rule, model_name, event, data, timestamp, username, request_id): - if event_rule.event_type == EVENT_TYPE_MODELS.WEBHOOK: + if event_rule.event_type == EventRuleTypeChoices.WEBHOOK: process_webhook(event_rule, model_name, event, data, timestamp, username, request_id) - elif event_rule.event_type == EVENT_TYPE_MODELS.SCRIPT: + elif event_rule.event_type == EventRuleTypeChoicesEventRuleTypeChoices.SCRIPT: process_script(event_rule, model_name, event, data, timestamp, username, request_id) diff --git a/netbox/extras/migrations/0099_eventrule.py b/netbox/extras/migrations/0099_eventrule.py index 79cb002ee95..838c718181c 100644 --- a/netbox/extras/migrations/0099_eventrule.py +++ b/netbox/extras/migrations/0099_eventrule.py @@ -27,7 +27,7 @@ def move_webhooks(apps, schema_editor): event.event_type = EventRuleTypeChoices.WEBHOOK event.object_type_id = ContentType.objects.get_for_model(webhook).id - event.object = webhook + event.object_id = webhook.id event.save() event.content_types.add(*webhook.content_types.all()) diff --git a/netbox/extras/webhooks_worker.py b/netbox/extras/webhooks_worker.py index 9f64fc405dc..748afdbc8f7 100644 --- a/netbox/extras/webhooks_worker.py +++ b/netbox/extras/webhooks_worker.py @@ -12,11 +12,13 @@ logger = logging.getLogger('netbox.webhooks_worker') -def process_webhook(webhook, model_name, event, data, timestamp, username, request_id=None): +def process_webhook(event_rule, model_name, event, data, timestamp, username, request_id=None): """ Make a POST request to the defined Webhook """ + webhook = event_rule.object + # Build the headers for the HTTP request headers = { 'Content-Type': webhook.http_content_type, From 7cb704db47dba89815d17186d1d98fb1e70e4128 Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 1 Nov 2023 10:34:55 -0700 Subject: [PATCH 19/95] 14132 fix webhook_worker --- netbox/extras/events_worker.py | 22 ++++------------------ netbox/extras/scripts_worker.py | 2 +- netbox/extras/webhooks_worker.py | 16 +++++++++++++++- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/netbox/extras/events_worker.py b/netbox/extras/events_worker.py index 2d09937b1fb..a54bf438421 100644 --- a/netbox/extras/events_worker.py +++ b/netbox/extras/events_worker.py @@ -42,11 +42,11 @@ def module_member(name): return getattr(module, member) -def process_event_rules(event_rule, model_name, event, data, timestamp, username, request_id): +def process_event_rules(event_rule, model_name, event, data, timestamp, username, request_id, snapshots): if event_rule.event_type == EventRuleTypeChoices.WEBHOOK: - process_webhook(event_rule, model_name, event, data, timestamp, username, request_id) + process_webhook(event_rule, model_name, event, data, timestamp, username, request_id, snapshots) elif event_rule.event_type == EventRuleTypeChoicesEventRuleTypeChoices.SCRIPT: - process_script(event_rule, model_name, event, data, timestamp, username, request_id) + process_script(event_rule, model_name, event, data, timestamp, username, request_id, snapshots) @job('default') @@ -58,21 +58,7 @@ def process_event(event_rule, model_name, event, data, timestamp, username, requ if not eval_conditions(event_rule, data): return - # Prepare context data for headers & body templates - context = { - 'event': WEBHOOK_EVENT_TYPES[event], - 'timestamp': timestamp, - 'model': model_name, - 'username': username, - 'request_id': request_id, - 'data': data, - } - if snapshots: - context.update({ - 'snapshots': snapshots - }) - # process the events pipeline for name in settings.NETBOX_EVENTS_PIPELINE: func = module_member(name) - func(event_rule, model_name, event, data, timestamp, username, request_id) + func(event_rule, model_name, event, data, timestamp, username, request_id, snapshots) diff --git a/netbox/extras/scripts_worker.py b/netbox/extras/scripts_worker.py index e014463a28b..39454dfb44b 100644 --- a/netbox/extras/scripts_worker.py +++ b/netbox/extras/scripts_worker.py @@ -12,7 +12,7 @@ logger = logging.getLogger('netbox.webhooks_worker') -def process_script(webhook, model_name, event, data, timestamp, username, request_id=None): +def process_script(webhook, model_name, event, data, timestamp, username, request_id=None, snapshots=None): """ Make a POST request to the defined Webhook """ diff --git a/netbox/extras/webhooks_worker.py b/netbox/extras/webhooks_worker.py index 748afdbc8f7..734a0784a16 100644 --- a/netbox/extras/webhooks_worker.py +++ b/netbox/extras/webhooks_worker.py @@ -12,13 +12,27 @@ logger = logging.getLogger('netbox.webhooks_worker') -def process_webhook(event_rule, model_name, event, data, timestamp, username, request_id=None): +def process_webhook(event_rule, model_name, event, data, timestamp, username, request_id=None, snapshots=None): """ Make a POST request to the defined Webhook """ webhook = event_rule.object + # Prepare context data for headers & body templates + context = { + 'event': WEBHOOK_EVENT_TYPES[event], + 'timestamp': timestamp, + 'model': model_name, + 'username': username, + 'request_id': request_id, + 'data': data, + } + if snapshots: + context.update({ + 'snapshots': snapshots + }) + # Build the headers for the HTTP request headers = { 'Content-Type': webhook.http_content_type, From 9c204322e23e17f3c6dd5cb680be4eb8aa146faf Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 8 Nov 2023 07:14:23 -0800 Subject: [PATCH 20/95] 14132 misc changes --- netbox/extras/api/serializers.py | 2 +- netbox/extras/choices.py | 2 +- netbox/extras/events_worker.py | 6 ++-- netbox/extras/forms/model_forms.py | 38 ++++++++++++++++++++++ netbox/extras/migrations/0099_eventrule.py | 10 +++--- netbox/extras/models/models.py | 22 +++++++++---- netbox/extras/tables/tables.py | 8 ++--- 7 files changed, 67 insertions(+), 21 deletions(-) diff --git a/netbox/extras/api/serializers.py b/netbox/extras/api/serializers.py index c5ee1faba0b..c22841f4ecb 100644 --- a/netbox/extras/api/serializers.py +++ b/netbox/extras/api/serializers.py @@ -73,7 +73,7 @@ class Meta: model = EventRule fields = [ 'id', 'url', 'display', 'content_types', 'name', 'type_create', 'type_update', 'type_delete', - 'type_job_start', 'type_job_end', 'enabled', 'conditions', 'event_type', + 'type_job_start', 'type_job_end', 'enabled', 'conditions', 'action_type', 'custom_fields', 'tags', 'created', 'last_updated', ] diff --git a/netbox/extras/choices.py b/netbox/extras/choices.py index 44774738229..df3b47d9ebd 100644 --- a/netbox/extras/choices.py +++ b/netbox/extras/choices.py @@ -286,7 +286,7 @@ class DashboardWidgetColorChoices(ChoiceSet): # Event Rules # -class EventRuleTypeChoices(ChoiceSet): +class EventRuleActionChoices(ChoiceSet): WEBHOOK = 'webhook' SCRIPT = 'script' diff --git a/netbox/extras/events_worker.py b/netbox/extras/events_worker.py index a54bf438421..2b6adc17d4f 100644 --- a/netbox/extras/events_worker.py +++ b/netbox/extras/events_worker.py @@ -7,7 +7,7 @@ from jinja2.exceptions import TemplateError from .conditions import ConditionSet -from .choices import EventRuleTypeChoices +from .choices import EventRuleActionChoices from .constants import WEBHOOK_EVENT_TYPES from .scripts_worker import process_script from .webhooks import generate_signature @@ -43,9 +43,9 @@ def module_member(name): def process_event_rules(event_rule, model_name, event, data, timestamp, username, request_id, snapshots): - if event_rule.event_type == EventRuleTypeChoices.WEBHOOK: + if event_rule.action_type == EventRuleActionChoices.WEBHOOK: process_webhook(event_rule, model_name, event, data, timestamp, username, request_id, snapshots) - elif event_rule.event_type == EventRuleTypeChoicesEventRuleTypeChoices.SCRIPT: + elif event_rule.action_type == EventRuleActionChoicesEventRuleTypeChoices.SCRIPT: process_script(event_rule, model_name, event, data, timestamp, username, request_id, snapshots) diff --git a/netbox/extras/forms/model_forms.py b/netbox/extras/forms/model_forms.py index a06e500d1f4..dcdc94675c9 100644 --- a/netbox/extras/forms/model_forms.py +++ b/netbox/extras/forms/model_forms.py @@ -249,10 +249,27 @@ class EventRuleForm(NetBoxModelForm): limit_choices_to=FeatureQuery('webhooks') ) + # Webhook form fields + # + payload_url = Webhook._meta.get_field('payload_url').formfield() + http_method = Webhook._meta.get_field('http_method').formfield() + http_content_type = Webhook._meta.get_field('http_content_type').formfield() + additional_headers = Webhook._meta.get_field('additional_headers').formfield() + body_template = Webhook._meta.get_field('body_template').formfield() + secret = Webhook._meta.get_field('secret').formfield() + ssl_verification = Webhook._meta.get_field('ssl_verification').formfield() + ca_file_path = Webhook._meta.get_field('ca_file_path').formfield() + fieldsets = ( (_('EventRule'), ('name', 'content_types', 'enabled', 'tags')), (_('Events'), ('type_create', 'type_update', 'type_delete', 'type_job_start', 'type_job_end')), (_('Conditions'), ('conditions',)), + (_('Action'), ('action_type',)), + + (_('HTTP Request'), ( + 'payload_url', 'http_method', 'http_content_type', 'additional_headers', 'body_template', 'secret', + )), + (_('SSL'), ('ssl_verification', 'ca_file_path')), ) class Meta: @@ -269,6 +286,27 @@ class Meta: 'conditions': forms.Textarea(attrs={'class': 'font-monospace'}), } + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + SCRIPT_CHOICES = [ + ( + "Audio", + ( + ("vinyl", "Vinyl"), + ("cd", "CD"), + ), + ), + ( + "Video", + ( + ("vhs", "VHS Tape"), + ("dvd", "DVD"), + ), + ), + ("unknown", "Unknown"), + ] + class TagForm(BootstrapMixin, forms.ModelForm): slug = SlugField() diff --git a/netbox/extras/migrations/0099_eventrule.py b/netbox/extras/migrations/0099_eventrule.py index 838c718181c..6bc7e268fbb 100644 --- a/netbox/extras/migrations/0099_eventrule.py +++ b/netbox/extras/migrations/0099_eventrule.py @@ -25,7 +25,7 @@ def move_webhooks(apps, schema_editor): event.enabled = webhook.enabled event.conditions = webhook.conditions - event.event_type = EventRuleTypeChoices.WEBHOOK + event.action_type = EventRuleActionChoices.WEBHOOK event.object_type_id = ContentType.objects.get_for_model(webhook).id event.object_id = webhook.id event.save() @@ -57,7 +57,7 @@ class Migration(migrations.Migration): ('type_job_end', models.BooleanField(default=False)), ('enabled', models.BooleanField(default=True)), ('conditions', models.JSONField(blank=True, null=True)), - ('event_type', models.CharField(default='webhook', max_length=30)), + ('action_type', models.CharField(default='webhook', max_length=30)), ('object_id', models.PositiveBigIntegerField(blank=True, null=True)), ( 'content_types', @@ -76,6 +76,8 @@ class Migration(migrations.Migration): to='contenttypes.contenttype', ), ), + ('object_identifier', models.CharField(max_length=80, blank=True)), + ('parameters', models.JSONField(blank=True, null=True)), ('tags', taggit.managers.TaggableManager(through='extras.TaggedItem', to='extras.Tag')), ], options={ @@ -130,8 +132,4 @@ class Migration(migrations.Migration): to='contenttypes.contenttype', ), ), - migrations.AlterUniqueTogether( - name='eventrule', - unique_together={('object_type', 'object_id')}, - ), ] diff --git a/netbox/extras/models/models.py b/netbox/extras/models/models.py index 137b2cb5721..bd16a043ddf 100644 --- a/netbox/extras/models/models.py +++ b/netbox/extras/models/models.py @@ -93,17 +93,16 @@ class EventRule(CustomFieldsMixin, ExportTemplatesMixin, TagsMixin, ChangeLogged help_text=_("A set of conditions which determine whether the event will be generated.") ) - event_type = models.CharField( + # Action to take + action_type = models.CharField( max_length=30, - choices=EventRuleTypeChoices, - default=EventRuleTypeChoices.WEBHOOK, + choices=EventRuleActionChoices, + default=EventRuleActionChoices.WEBHOOK, verbose_name=_('event type') ) - # Action to take object_type = models.ForeignKey( to=ContentType, related_name='eventrule_actions', - # limit_choices_to=EVENT_TYPE_MODELS, on_delete=models.CASCADE, ) object_id = models.PositiveBigIntegerField( @@ -115,11 +114,22 @@ class EventRule(CustomFieldsMixin, ExportTemplatesMixin, TagsMixin, ChangeLogged fk_field='object_id', ) + # internal (not show in UI) - used by scripts to store function name + object_identifier = models.CharField( + max_length=80, + blank=True + ) + parameters = models.JSONField( + verbose_name=_('parameters'), + blank=True, + null=True, + help_text=_("Parameters to pass to the action.") + ) + class Meta: ordering = ('name',) verbose_name = _('eventrule') verbose_name_plural = _('eventrules') - unique_together = ('object_type', 'object_id') def __str__(self): return self.name diff --git a/netbox/extras/tables/tables.py b/netbox/extras/tables/tables.py index 0fc53cfd748..90cb4eec179 100644 --- a/netbox/extras/tables/tables.py +++ b/netbox/extras/tables/tables.py @@ -297,8 +297,8 @@ class EventRuleTable(NetBoxTable): verbose_name=_('Name'), linkify=True ) - event_type = tables.Column( - verbose_name=_('Event Type'), + action_type = tables.Column( + verbose_name=_('Action Type'), ) content_types = columns.ContentTypesColumn( verbose_name=_('Content Types'), @@ -328,11 +328,11 @@ class EventRuleTable(NetBoxTable): class Meta(NetBoxTable.Meta): model = EventRule fields = ( - 'pk', 'id', 'name', 'event_type', 'content_types', 'enabled', 'type_create', 'type_update', 'type_delete', + 'pk', 'id', 'name', 'action_type', 'content_types', 'enabled', 'type_create', 'type_update', 'type_delete', 'type_job_start', 'type_job_end', 'tags', 'created', 'last_updated', ) default_columns = ( - 'pk', 'name', 'event_type', 'content_types', 'enabled', 'type_create', 'type_update', 'type_delete', 'type_job_start', + 'pk', 'name', 'action_type', 'content_types', 'enabled', 'type_create', 'type_update', 'type_delete', 'type_job_start', 'type_job_end', ) From a679740626e59cff6f97c806625abc321074a4a4 Mon Sep 17 00:00:00 2001 From: Arthur Date: Thu, 9 Nov 2023 09:48:16 -0800 Subject: [PATCH 21/95] 8356 htmx --- netbox/extras/forms/model_forms.py | 72 +++++++++++----------- netbox/extras/migrations/0099_eventrule.py | 2 +- 2 files changed, 37 insertions(+), 37 deletions(-) diff --git a/netbox/extras/forms/model_forms.py b/netbox/extras/forms/model_forms.py index 4c0203ece59..d331c65bf45 100644 --- a/netbox/extras/forms/model_forms.py +++ b/netbox/extras/forms/model_forms.py @@ -14,12 +14,12 @@ from netbox.config import get_config, PARAMS from netbox.forms import NetBoxModelForm from tenancy.models import Tenant, TenantGroup -from utilities.forms import BootstrapMixin, add_blank_choice +from utilities.forms import BootstrapMixin, add_blank_choice, get_field_value from utilities.forms.fields import ( CommentField, ContentTypeChoiceField, ContentTypeMultipleChoiceField, DynamicModelChoiceField, DynamicModelMultipleChoiceField, JSONField, SlugField, ) -from utilities.forms.widgets import ChoicesWidget +from utilities.forms.widgets import ChoicesWidget, HTMXSelect from virtualization.models import Cluster, ClusterGroup, ClusterType @@ -245,28 +245,16 @@ class EventRuleForm(NetBoxModelForm): queryset=ContentType.objects.all(), limit_choices_to=FeatureQuery('webhooks') ) - - # Webhook form fields - # - payload_url = Webhook._meta.get_field('payload_url').formfield() - http_method = Webhook._meta.get_field('http_method').formfield() - http_content_type = Webhook._meta.get_field('http_content_type').formfield() - additional_headers = Webhook._meta.get_field('additional_headers').formfield() - body_template = Webhook._meta.get_field('body_template').formfield() - secret = Webhook._meta.get_field('secret').formfield() - ssl_verification = Webhook._meta.get_field('ssl_verification').formfield() - ca_file_path = Webhook._meta.get_field('ca_file_path').formfield() + action_choice = forms.ChoiceField( + label=_('Action choice'), + choices=[] + ) fieldsets = ( (_('EventRule'), ('name', 'content_types', 'enabled', 'tags')), (_('Events'), ('type_create', 'type_update', 'type_delete', 'type_job_start', 'type_job_end')), (_('Conditions'), ('conditions',)), - (_('Action'), ('action_type',)), - - (_('HTTP Request'), ( - 'payload_url', 'http_method', 'http_content_type', 'additional_headers', 'body_template', 'secret', - )), - (_('SSL'), ('ssl_verification', 'ca_file_path')), + (_('Action'), ('action_type', 'action_choice', 'parameters')), ) class Meta: @@ -281,28 +269,40 @@ class Meta: } widgets = { 'conditions': forms.Textarea(attrs={'class': 'font-monospace'}), + 'action_type': HTMXSelect(), } + def get_script_choices(self): + choices = [] + idx = 0 + for module in ScriptModule.objects.all(): + scripts = [] + for script_name in module.scripts.keys(): + name = f"{str(module).lower()}:{script_name.lower()}" + scripts.append((name, script_name)) + + if scripts: + choices.append((str(module), scripts)) + + self.fields['action_choice'].choices = choices + + def get_webhook_choices(self): + self.fields['action_choice'] = DynamicModelChoiceField( + label=_('Webhook'), + queryset=Webhook.objects.all(), + required=True + ) + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - SCRIPT_CHOICES = [ - ( - "Audio", - ( - ("vinyl", "Vinyl"), - ("cd", "CD"), - ), - ), - ( - "Video", - ( - ("vhs", "VHS Tape"), - ("dvd", "DVD"), - ), - ), - ("unknown", "Unknown"), - ] + # Determine the action type + action_type = get_field_value(self, 'action_type') + + if action_type == EventRuleActionChoices.WEBHOOK: + self.get_webhook_choices() + elif action_type == EventRuleActionChoices.SCRIPT: + self.get_script_choices() class TagForm(BootstrapMixin, forms.ModelForm): diff --git a/netbox/extras/migrations/0099_eventrule.py b/netbox/extras/migrations/0099_eventrule.py index 6bc7e268fbb..1e70f5d2e36 100644 --- a/netbox/extras/migrations/0099_eventrule.py +++ b/netbox/extras/migrations/0099_eventrule.py @@ -35,7 +35,7 @@ def move_webhooks(apps, schema_editor): class Migration(migrations.Migration): dependencies = [ ('contenttypes', '0002_remove_content_type_name'), - ('extras', '0098_webhook_custom_field_data_webhook_tags'), + ('extras', '0099_cachedvalue_ordering'), ] operations = [ From 73047c979bea60a4da46db3e62454d1cbc39af64 Mon Sep 17 00:00:00 2001 From: Arthur Date: Thu, 9 Nov 2023 10:37:06 -0800 Subject: [PATCH 22/95] 8356 action_object_type cleanup migration --- netbox/extras/forms/model_forms.py | 8 ++++++-- netbox/extras/migrations/0099_eventrule.py | 20 +++++--------------- netbox/extras/models/models.py | 12 ++++++------ 3 files changed, 17 insertions(+), 23 deletions(-) diff --git a/netbox/extras/forms/model_forms.py b/netbox/extras/forms/model_forms.py index d331c65bf45..b6f18bae40c 100644 --- a/netbox/extras/forms/model_forms.py +++ b/netbox/extras/forms/model_forms.py @@ -254,7 +254,7 @@ class EventRuleForm(NetBoxModelForm): (_('EventRule'), ('name', 'content_types', 'enabled', 'tags')), (_('Events'), ('type_create', 'type_update', 'type_delete', 'type_job_start', 'type_job_end')), (_('Conditions'), ('conditions',)), - (_('Action'), ('action_type', 'action_choice', 'parameters')), + (_('Action'), ('action_type', 'action_choice', 'parameters', 'action_object_type', 'action_object_id', 'action_object_identifier')), ) class Meta: @@ -270,6 +270,9 @@ class Meta: widgets = { 'conditions': forms.Textarea(attrs={'class': 'font-monospace'}), 'action_type': HTMXSelect(), + 'action_object_type': forms.HiddenInput, + 'action_object_id': forms.HiddenInput, + 'action_object_identifier': forms.HiddenInput, } def get_script_choices(self): @@ -278,13 +281,14 @@ def get_script_choices(self): for module in ScriptModule.objects.all(): scripts = [] for script_name in module.scripts.keys(): - name = f"{str(module).lower()}:{script_name.lower()}" + name = f"{str(module.pk)}:{script_name.lower()}" scripts.append((name, script_name)) if scripts: choices.append((str(module), scripts)) self.fields['action_choice'].choices = choices + self.fields['action_choice'].initial = get_field_value(self, 'action_object_identifier') def get_webhook_choices(self): self.fields['action_choice'] = DynamicModelChoiceField( diff --git a/netbox/extras/migrations/0099_eventrule.py b/netbox/extras/migrations/0099_eventrule.py index 1e70f5d2e36..cb4f2069d5f 100644 --- a/netbox/extras/migrations/0099_eventrule.py +++ b/netbox/extras/migrations/0099_eventrule.py @@ -26,8 +26,8 @@ def move_webhooks(apps, schema_editor): event.conditions = webhook.conditions event.action_type = EventRuleActionChoices.WEBHOOK - event.object_type_id = ContentType.objects.get_for_model(webhook).id - event.object_id = webhook.id + event.action_object_type_id = ContentType.objects.get_for_model(webhook).id + event.action_object_id = webhook.id event.save() event.content_types.add(*webhook.content_types.all()) @@ -58,7 +58,7 @@ class Migration(migrations.Migration): ('enabled', models.BooleanField(default=True)), ('conditions', models.JSONField(blank=True, null=True)), ('action_type', models.CharField(default='webhook', max_length=30)), - ('object_id', models.PositiveBigIntegerField(blank=True, null=True)), + ('action_object_id', models.PositiveBigIntegerField(blank=True, null=True)), ( 'content_types', models.ManyToManyField( @@ -68,15 +68,14 @@ class Migration(migrations.Migration): ), ), ( - 'object_type', + 'action_object_type', models.ForeignKey( - limit_choices_to=models.Q(('app_label', 'extras'), ('model__in', ('webhook', 'script'))), on_delete=django.db.models.deletion.CASCADE, related_name='eventrule_actions', to='contenttypes.contenttype', ), ), - ('object_identifier', models.CharField(max_length=80, blank=True)), + ('action_object_identifier', models.CharField(max_length=80, blank=True)), ('parameters', models.JSONField(blank=True, null=True)), ('tags', taggit.managers.TaggableManager(through='extras.TaggedItem', to='extras.Tag')), ], @@ -123,13 +122,4 @@ class Migration(migrations.Migration): model_name='webhook', name='type_update', ), - migrations.AlterField( - model_name='eventrule', - name='object_type', - field=models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, - related_name='eventrule_actions', - to='contenttypes.contenttype', - ), - ), ] diff --git a/netbox/extras/models/models.py b/netbox/extras/models/models.py index bd16a043ddf..9f6a499802d 100644 --- a/netbox/extras/models/models.py +++ b/netbox/extras/models/models.py @@ -100,22 +100,22 @@ class EventRule(CustomFieldsMixin, ExportTemplatesMixin, TagsMixin, ChangeLogged default=EventRuleActionChoices.WEBHOOK, verbose_name=_('event type') ) - object_type = models.ForeignKey( + action_object_type = models.ForeignKey( to=ContentType, related_name='eventrule_actions', on_delete=models.CASCADE, ) - object_id = models.PositiveBigIntegerField( + action_object_id = models.PositiveBigIntegerField( blank=True, null=True ) - object = GenericForeignKey( - ct_field='object_type', - fk_field='object_id', + action_object = GenericForeignKey( + ct_field='action_object_type', + fk_field='action_object_id', ) # internal (not show in UI) - used by scripts to store function name - object_identifier = models.CharField( + action_object_identifier = models.CharField( max_length=80, blank=True ) From 1453419779da34cf51b5e1f10c9139276dbb9d58 Mon Sep 17 00:00:00 2001 From: Arthur Date: Thu, 9 Nov 2023 13:01:10 -0800 Subject: [PATCH 23/95] 8356 fix form, init fields --- netbox/extras/forms/model_forms.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/netbox/extras/forms/model_forms.py b/netbox/extras/forms/model_forms.py index b6f18bae40c..9fc51d98bb4 100644 --- a/netbox/extras/forms/model_forms.py +++ b/netbox/extras/forms/model_forms.py @@ -2,6 +2,7 @@ from django import forms from django.conf import settings +from django.contrib.contenttypes.models import ContentType from django.utils.safestring import mark_safe from django.utils.translation import gettext_lazy as _ @@ -291,14 +292,20 @@ def get_script_choices(self): self.fields['action_choice'].initial = get_field_value(self, 'action_object_identifier') def get_webhook_choices(self): + initial = None + if self.fields['action_object_type'] and self.fields['action_object_id']: + initial = Webhook.objects.get(pk=get_field_value(self, 'action_object_id')) self.fields['action_choice'] = DynamicModelChoiceField( label=_('Webhook'), queryset=Webhook.objects.all(), - required=True + required=True, + initial=initial ) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) + self.fields['action_object_type'].required = False + self.fields['action_object_id'].required = False # Determine the action type action_type = get_field_value(self, 'action_type') @@ -308,6 +315,22 @@ def __init__(self, *args, **kwargs): elif action_type == EventRuleActionChoices.SCRIPT: self.get_script_choices() + def clean(self): + super().clean() + + action_choice = self.cleaned_data.get('action_choice') + if self.cleaned_data.get('action_type') == EventRuleActionChoices.WEBHOOK: + self.cleaned_data['action_object_type'] = ContentType.objects.get_for_model(action_choice) + self.cleaned_data['action_object_id'] = action_choice.id + self.cleaned_data['action_object_identifier'] = '' + elif self.cleaned_data.get('action_type') == EventRuleActionChoices.SCRIPT: + script = ScriptModule.objects.get(pk=action_choice.split(":")[0]) + self.cleaned_data['action_object_type'] = ContentType.objects.get_for_model(script) + self.cleaned_data['action_object_id'] = script.id + self.cleaned_data['action_object_identifier'] = action_choice + + return self.cleaned_data + class TagForm(BootstrapMixin, forms.ModelForm): slug = SlugField() From 7fae20ca51a0988a0c047875485457ba516d3123 Mon Sep 17 00:00:00 2001 From: Arthur Date: Thu, 9 Nov 2023 14:28:21 -0800 Subject: [PATCH 24/95] 8356 fix tests --- netbox/extras/filtersets.py | 2 +- netbox/extras/tests/test_filtersets.py | 65 -------------------------- 2 files changed, 1 insertion(+), 66 deletions(-) diff --git a/netbox/extras/filtersets.py b/netbox/extras/filtersets.py index e7b3398ca7c..044ff3d3871 100644 --- a/netbox/extras/filtersets.py +++ b/netbox/extras/filtersets.py @@ -47,7 +47,7 @@ class WebhookFilterSet(NetBoxModelFilterSet): class Meta: model = Webhook fields = [ - 'id', 'payload_url', + 'id', 'name', 'payload_url', 'http_method', 'http_content_type', 'secret', 'ssl_verification', 'ca_file_path', ] diff --git a/netbox/extras/tests/test_filtersets.py b/netbox/extras/tests/test_filtersets.py index 69111e6a781..2f59eb260d0 100644 --- a/netbox/extras/tests/test_filtersets.py +++ b/netbox/extras/tests/test_filtersets.py @@ -150,106 +150,41 @@ def setUpTestData(cls): webhooks = ( Webhook( name='Webhook 1', - type_create=True, - type_update=False, - type_delete=False, - type_job_start=False, - type_job_end=False, payload_url='http://example.com/?1', - enabled=True, http_method='GET', ssl_verification=True, ), Webhook( name='Webhook 2', - type_create=False, - type_update=True, - type_delete=False, - type_job_start=False, - type_job_end=False, payload_url='http://example.com/?2', - enabled=True, http_method='POST', ssl_verification=True, ), Webhook( name='Webhook 3', - type_create=False, - type_update=False, - type_delete=True, - type_job_start=False, - type_job_end=False, payload_url='http://example.com/?3', - enabled=False, http_method='PATCH', ssl_verification=False, ), Webhook( name='Webhook 4', - type_create=False, - type_update=False, - type_delete=False, - type_job_start=True, - type_job_end=False, payload_url='http://example.com/?4', - enabled=False, http_method='PATCH', ssl_verification=False, ), Webhook( name='Webhook 5', - type_create=False, - type_update=False, - type_delete=False, - type_job_start=False, - type_job_end=True, payload_url='http://example.com/?5', - enabled=False, http_method='PATCH', ssl_verification=False, ), ) Webhook.objects.bulk_create(webhooks) - webhooks[0].content_types.add(content_types[0]) - webhooks[1].content_types.add(content_types[1]) - webhooks[2].content_types.add(content_types[2]) - webhooks[3].content_types.add(content_types[3]) - webhooks[4].content_types.add(content_types[4]) def test_name(self): params = {'name': ['Webhook 1', 'Webhook 2']} self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2) - def test_content_types(self): - params = {'content_types': 'dcim.region'} - self.assertEqual(self.filterset(params, self.queryset).qs.count(), 1) - params = {'content_type_id': [ContentType.objects.get_for_model(Region).pk]} - self.assertEqual(self.filterset(params, self.queryset).qs.count(), 1) - - def test_type_create(self): - params = {'type_create': True} - self.assertEqual(self.filterset(params, self.queryset).qs.count(), 1) - - def test_type_update(self): - params = {'type_update': True} - self.assertEqual(self.filterset(params, self.queryset).qs.count(), 1) - - def test_type_delete(self): - params = {'type_delete': True} - self.assertEqual(self.filterset(params, self.queryset).qs.count(), 1) - - def test_type_job_start(self): - params = {'type_job_start': True} - self.assertEqual(self.filterset(params, self.queryset).qs.count(), 1) - - def test_type_job_end(self): - params = {'type_job_end': True} - self.assertEqual(self.filterset(params, self.queryset).qs.count(), 1) - - def test_enabled(self): - params = {'enabled': True} - self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2) - def test_http_method(self): params = {'http_method': ['GET', 'POST']} self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2) From 5686b964994baeb5ef3475dc7635c11ffb763adf Mon Sep 17 00:00:00 2001 From: Arthur Date: Thu, 9 Nov 2023 17:30:40 -0800 Subject: [PATCH 25/95] 8356 fix tests --- netbox/extras/models/models.py | 6 +++++- netbox/extras/tests/test_views.py | 25 +++++++------------------ 2 files changed, 12 insertions(+), 19 deletions(-) diff --git a/netbox/extras/models/models.py b/netbox/extras/models/models.py index 9f6a499802d..63abe050434 100644 --- a/netbox/extras/models/models.py +++ b/netbox/extras/models/models.py @@ -165,7 +165,11 @@ class Webhook(CustomFieldsMixin, ExportTemplatesMixin, TagsMixin, ChangeLoggedMo delete in NetBox. The request will contain a representation of the object, which the remote application can act on. Each Webhook can be limited to firing only on certain actions or certain object types. """ - events = GenericRelation(EventRule) + events = GenericRelation( + EventRule, + content_type_field='action_object_type', + object_id_field='action_object_id' + ) name = models.CharField( verbose_name=_('name'), diff --git a/netbox/extras/tests/test_views.py b/netbox/extras/tests/test_views.py index 296ed9f4d5a..bc01edbf2f2 100644 --- a/netbox/extras/tests/test_views.py +++ b/netbox/extras/tests/test_views.py @@ -335,33 +335,26 @@ class WebhookTestCase(ViewTestCases.PrimaryObjectViewTestCase): @classmethod def setUpTestData(cls): - site_ct = ContentType.objects.get_for_model(Site) webhooks = ( - Webhook(name='Webhook 1', payload_url='http://example.com/?1', type_create=True, http_method='POST'), - Webhook(name='Webhook 2', payload_url='http://example.com/?2', type_create=True, http_method='POST'), - Webhook(name='Webhook 3', payload_url='http://example.com/?3', type_create=True, http_method='POST'), + Webhook(name='Webhook 1', payload_url='http://example.com/?1', http_method='POST'), + Webhook(name='Webhook 2', payload_url='http://example.com/?2', http_method='POST'), + Webhook(name='Webhook 3', payload_url='http://example.com/?3', http_method='POST'), ) for webhook in webhooks: webhook.save() - webhook.content_types.add(site_ct) cls.form_data = { 'name': 'Webhook X', - 'content_types': [site_ct.pk], - 'type_create': False, - 'type_update': True, - 'type_delete': True, 'payload_url': 'http://example.com/?x', 'http_method': 'GET', 'http_content_type': 'application/foo', - 'conditions': None, } cls.csv_data = ( - "name,content_types,type_create,payload_url,http_method,http_content_type", - "Webhook 4,dcim.site,True,http://example.com/?4,GET,application/json", - "Webhook 5,dcim.site,True,http://example.com/?5,GET,application/json", - "Webhook 6,dcim.site,True,http://example.com/?6,GET,application/json", + "name,payload_url,http_method,http_content_type", + "Webhook 4,http://example.com/?4,GET,application/json", + "Webhook 5,http://example.com/?5,GET,application/json", + "Webhook 6,http://example.com/?6,GET,application/json", ) cls.csv_update_data = ( @@ -372,10 +365,6 @@ def setUpTestData(cls): ) cls.bulk_edit_data = { - 'enabled': False, - 'type_create': False, - 'type_update': True, - 'type_delete': True, 'http_method': 'GET', } From 005010e642652d623e0eb2bc6f69e38c165ce03b Mon Sep 17 00:00:00 2001 From: Arthur Date: Thu, 9 Nov 2023 17:55:30 -0800 Subject: [PATCH 26/95] 8356 fix tests --- netbox/extras/forms/bulk_edit.py | 30 ------------------------------ netbox/extras/forms/bulk_import.py | 2 +- netbox/extras/tests/test_api.py | 13 ------------- 3 files changed, 1 insertion(+), 44 deletions(-) diff --git a/netbox/extras/forms/bulk_edit.py b/netbox/extras/forms/bulk_edit.py index ae9c654a458..2c04c929e40 100644 --- a/netbox/extras/forms/bulk_edit.py +++ b/netbox/extras/forms/bulk_edit.py @@ -174,36 +174,6 @@ class WebhookBulkEditForm(NetBoxModelBulkEditForm): queryset=Webhook.objects.all(), widget=forms.MultipleHiddenInput ) - enabled = forms.NullBooleanField( - label=_('Enabled'), - required=False, - widget=BulkEditNullBooleanSelect() - ) - type_create = forms.NullBooleanField( - label=_('On create'), - required=False, - widget=BulkEditNullBooleanSelect() - ) - type_update = forms.NullBooleanField( - label=_('On update'), - required=False, - widget=BulkEditNullBooleanSelect() - ) - type_delete = forms.NullBooleanField( - label=_('On delete'), - required=False, - widget=BulkEditNullBooleanSelect() - ) - type_job_start = forms.NullBooleanField( - label=_('On job start'), - required=False, - widget=BulkEditNullBooleanSelect() - ) - type_job_end = forms.NullBooleanField( - label=_('On job end'), - required=False, - widget=BulkEditNullBooleanSelect() - ) http_method = forms.ChoiceField( choices=add_blank_choice(WebhookHttpMethodChoices), required=False, diff --git a/netbox/extras/forms/bulk_import.py b/netbox/extras/forms/bulk_import.py index f1abf5252a7..b0592117504 100644 --- a/netbox/extras/forms/bulk_import.py +++ b/netbox/extras/forms/bulk_import.py @@ -151,7 +151,7 @@ class WebhookImportForm(NetBoxModelImportForm): class Meta: model = Webhook fields = ( - 'payload_url', 'http_method', 'http_content_type', 'additional_headers', 'body_template', + 'name', 'payload_url', 'http_method', 'http_content_type', 'additional_headers', 'body_template', 'secret', 'ssl_verification', 'ca_file_path', 'tags' ) diff --git a/netbox/extras/tests/test_api.py b/netbox/extras/tests/test_api.py index 5d0bd3797c6..53b1ab7cf5e 100644 --- a/netbox/extras/tests/test_api.py +++ b/netbox/extras/tests/test_api.py @@ -86,21 +86,15 @@ class WebhookTest(APIViewTestCases.APIViewTestCase): brief_fields = ['display', 'id', 'name', 'url'] create_data = [ { - 'content_types': ['dcim.device', 'dcim.devicetype'], 'name': 'Webhook 4', - 'type_create': True, 'payload_url': 'http://example.com/?4', }, { - 'content_types': ['dcim.device', 'dcim.devicetype'], 'name': 'Webhook 5', - 'type_update': True, 'payload_url': 'http://example.com/?5', }, { - 'content_types': ['dcim.device', 'dcim.devicetype'], 'name': 'Webhook 6', - 'type_delete': True, 'payload_url': 'http://example.com/?6', }, ] @@ -110,29 +104,22 @@ class WebhookTest(APIViewTestCases.APIViewTestCase): @classmethod def setUpTestData(cls): - site_ct = ContentType.objects.get_for_model(Site) - rack_ct = ContentType.objects.get_for_model(Rack) webhooks = ( Webhook( name='Webhook 1', - type_create=True, payload_url='http://example.com/?1', ), Webhook( name='Webhook 2', - type_update=True, payload_url='http://example.com/?1', ), Webhook( name='Webhook 3', - type_delete=True, payload_url='http://example.com/?1', ), ) Webhook.objects.bulk_create(webhooks) - for webhook in webhooks: - webhook.content_types.add(site_ct, rack_ct) class CustomFieldTest(APIViewTestCases.APIViewTestCase): From a43cbcb1c1e33f96e5cf722cfd5be2693ca40604 Mon Sep 17 00:00:00 2001 From: Arthur Date: Mon, 13 Nov 2023 09:59:46 -0800 Subject: [PATCH 27/95] 8356 fix tests --- netbox/extras/api/serializers.py | 2 +- netbox/extras/forms/bulk_import.py | 6 - netbox/extras/tests/test_api.py | 54 ----- netbox/extras/tests/test_event_rules.py | 298 +----------------------- netbox/extras/tests/test_webhooks.py | 294 +---------------------- 5 files changed, 13 insertions(+), 641 deletions(-) diff --git a/netbox/extras/api/serializers.py b/netbox/extras/api/serializers.py index c22841f4ecb..16bc3c6ce46 100644 --- a/netbox/extras/api/serializers.py +++ b/netbox/extras/api/serializers.py @@ -88,7 +88,7 @@ class WebhookSerializer(NetBoxModelSerializer): class Meta: model = Webhook fields = [ - 'id', 'url', 'display', + 'id', 'url', 'display', 'name', 'payload_url', 'http_method', 'http_content_type', 'additional_headers', 'body_template', 'secret', 'ssl_verification', 'ca_file_path', 'custom_fields', 'tags', 'created', 'last_updated', diff --git a/netbox/extras/forms/bulk_import.py b/netbox/extras/forms/bulk_import.py index b0592117504..2f5e8350ea4 100644 --- a/netbox/extras/forms/bulk_import.py +++ b/netbox/extras/forms/bulk_import.py @@ -141,12 +141,6 @@ class Meta: class WebhookImportForm(NetBoxModelImportForm): - content_types = CSVMultipleContentTypeField( - label=_('Content types'), - queryset=ContentType.objects.all(), - limit_choices_to=FeatureQuery('webhooks'), - help_text=_("One or more assigned object types") - ) class Meta: model = Webhook diff --git a/netbox/extras/tests/test_api.py b/netbox/extras/tests/test_api.py index 53b1ab7cf5e..24f852ddeb6 100644 --- a/netbox/extras/tests/test_api.py +++ b/netbox/extras/tests/test_api.py @@ -27,60 +27,6 @@ def test_root(self): self.assertEqual(response.status_code, 200) -class EventRuleTest(APIViewTestCases.APIViewTestCase): - model = EventRule - brief_fields = ['display', 'id', 'name', 'url'] - create_data = [ - { - 'content_types': ['dcim.device', 'dcim.devicetype'], - 'name': 'Event Rule 4', - 'type_create': True, - 'payload_url': 'http://example.com/?4', - }, - { - 'content_types': ['dcim.device', 'dcim.devicetype'], - 'name': 'Event Rule 5', - 'type_update': True, - 'payload_url': 'http://example.com/?5', - }, - { - 'content_types': ['dcim.device', 'dcim.devicetype'], - 'name': 'Event Rule 6', - 'type_delete': True, - 'payload_url': 'http://example.com/?6', - }, - ] - bulk_update_data = { - 'ssl_verification': False, - } - - @classmethod - def setUpTestData(cls): - site_ct = ContentType.objects.get_for_model(Site) - rack_ct = ContentType.objects.get_for_model(Rack) - - webhooks = ( - Webhook( - name='Webhook 1', - type_create=True, - payload_url='http://example.com/?1', - ), - Webhook( - name='Webhook 2', - type_update=True, - payload_url='http://example.com/?1', - ), - Webhook( - name='Webhook 3', - type_delete=True, - payload_url='http://example.com/?1', - ), - ) - Webhook.objects.bulk_create(webhooks) - for webhook in webhooks: - webhook.content_types.add(site_ct, rack_ct) - - class WebhookTest(APIViewTestCases.APIViewTestCase): model = Webhook brief_fields = ['display', 'id', 'name', 'url'] diff --git a/netbox/extras/tests/test_event_rules.py b/netbox/extras/tests/test_event_rules.py index 53f8afb7d9e..dd1a3fdaa4c 100644 --- a/netbox/extras/tests/test_event_rules.py +++ b/netbox/extras/tests/test_event_rules.py @@ -12,10 +12,11 @@ from dcim.choices import SiteStatusChoices from dcim.models import Site from extras.choices import ObjectChangeActionChoices -from extras.models import Tag, EventRule +from extras.models import Tag, EventRule, Webhook from extras.events import enqueue_object, flush_events, serialize_for_event +from extras.events_worker import eval_conditions from extras.webhooks import generate_signature -from extras.webhooks_worker import eval_conditions, process_webhook +from extras.webhooks_worker import process_webhook from utilities.testing import APITestCase @@ -35,299 +36,14 @@ def setUpTestData(cls): DUMMY_URL = 'http://localhost:9000/' DUMMY_SECRET = 'LOOKATMEIMASECRETSTRING' - webhooks = EventRule.objects.bulk_create(( - Webhook(name='Webhook 1', type_create=True), - Webhook(name='Webhook 2', type_update=True), - Webhook(name='Webhook 3', type_delete=True), + webhooks = Webhook.objects.bulk_create(( + Webhook(name='Webhook 1',), + Webhook(name='Webhook 2',), + Webhook(name='Webhook 3',), )) - for webhook in webhooks: - webhook.content_types.set([site_ct]) Tag.objects.bulk_create(( Tag(name='Foo', slug='foo'), Tag(name='Bar', slug='bar'), Tag(name='Baz', slug='baz'), )) - - def test_enqueue_webhook_create(self): - # Create an object via the REST API - data = { - 'name': 'Site 1', - 'slug': 'site-1', - 'tags': [ - {'name': 'Foo'}, - {'name': 'Bar'}, - ] - } - url = reverse('dcim-api:site-list') - self.add_permissions('dcim.add_site') - response = self.client.post(url, data, format='json', **self.header) - self.assertHttpStatus(response, status.HTTP_201_CREATED) - self.assertEqual(Site.objects.count(), 1) - self.assertEqual(Site.objects.first().tags.count(), 2) - - # Verify that a job was queued for the object creation webhook - self.assertEqual(self.queue.count, 1) - job = self.queue.jobs[0] - self.assertEqual(job.kwargs['webhook'], Webhook.objects.get(type_create=True)) - self.assertEqual(job.kwargs['event'], ObjectChangeActionChoices.ACTION_CREATE) - self.assertEqual(job.kwargs['model_name'], 'site') - self.assertEqual(job.kwargs['data']['id'], response.data['id']) - self.assertEqual(len(job.kwargs['data']['tags']), len(response.data['tags'])) - self.assertEqual(job.kwargs['snapshots']['postchange']['name'], 'Site 1') - self.assertEqual(job.kwargs['snapshots']['postchange']['tags'], ['Bar', 'Foo']) - - def test_enqueue_webhook_bulk_create(self): - # Create multiple objects via the REST API - data = [ - { - 'name': 'Site 1', - 'slug': 'site-1', - 'tags': [ - {'name': 'Foo'}, - {'name': 'Bar'}, - ] - }, - { - 'name': 'Site 2', - 'slug': 'site-2', - 'tags': [ - {'name': 'Foo'}, - {'name': 'Bar'}, - ] - }, - { - 'name': 'Site 3', - 'slug': 'site-3', - 'tags': [ - {'name': 'Foo'}, - {'name': 'Bar'}, - ] - }, - ] - url = reverse('dcim-api:site-list') - self.add_permissions('dcim.add_site') - response = self.client.post(url, data, format='json', **self.header) - self.assertHttpStatus(response, status.HTTP_201_CREATED) - self.assertEqual(Site.objects.count(), 3) - self.assertEqual(Site.objects.first().tags.count(), 2) - - # Verify that a webhook was queued for each object - self.assertEqual(self.queue.count, 3) - for i, job in enumerate(self.queue.jobs): - self.assertEqual(job.kwargs['webhook'], Webhook.objects.get(type_create=True)) - self.assertEqual(job.kwargs['event'], ObjectChangeActionChoices.ACTION_CREATE) - self.assertEqual(job.kwargs['model_name'], 'site') - self.assertEqual(job.kwargs['data']['id'], response.data[i]['id']) - self.assertEqual(len(job.kwargs['data']['tags']), len(response.data[i]['tags'])) - self.assertEqual(job.kwargs['snapshots']['postchange']['name'], response.data[i]['name']) - self.assertEqual(job.kwargs['snapshots']['postchange']['tags'], ['Bar', 'Foo']) - - def test_enqueue_webhook_update(self): - site = Site.objects.create(name='Site 1', slug='site-1') - site.tags.set(Tag.objects.filter(name__in=['Foo', 'Bar'])) - - # Update an object via the REST API - data = { - 'name': 'Site X', - 'comments': 'Updated the site', - 'tags': [ - {'name': 'Baz'} - ] - } - url = reverse('dcim-api:site-detail', kwargs={'pk': site.pk}) - self.add_permissions('dcim.change_site') - response = self.client.patch(url, data, format='json', **self.header) - self.assertHttpStatus(response, status.HTTP_200_OK) - - # Verify that a job was queued for the object update webhook - self.assertEqual(self.queue.count, 1) - job = self.queue.jobs[0] - self.assertEqual(job.kwargs['webhook'], Webhook.objects.get(type_update=True)) - self.assertEqual(job.kwargs['event'], ObjectChangeActionChoices.ACTION_UPDATE) - self.assertEqual(job.kwargs['model_name'], 'site') - self.assertEqual(job.kwargs['data']['id'], site.pk) - self.assertEqual(len(job.kwargs['data']['tags']), len(response.data['tags'])) - self.assertEqual(job.kwargs['snapshots']['prechange']['name'], 'Site 1') - self.assertEqual(job.kwargs['snapshots']['prechange']['tags'], ['Bar', 'Foo']) - self.assertEqual(job.kwargs['snapshots']['postchange']['name'], 'Site X') - self.assertEqual(job.kwargs['snapshots']['postchange']['tags'], ['Baz']) - - def test_enqueue_webhook_bulk_update(self): - sites = ( - Site(name='Site 1', slug='site-1'), - Site(name='Site 2', slug='site-2'), - Site(name='Site 3', slug='site-3'), - ) - Site.objects.bulk_create(sites) - for site in sites: - site.tags.set(Tag.objects.filter(name__in=['Foo', 'Bar'])) - - # Update three objects via the REST API - data = [ - { - 'id': sites[0].pk, - 'name': 'Site X', - 'tags': [ - {'name': 'Baz'} - ] - }, - { - 'id': sites[1].pk, - 'name': 'Site Y', - 'tags': [ - {'name': 'Baz'} - ] - }, - { - 'id': sites[2].pk, - 'name': 'Site Z', - 'tags': [ - {'name': 'Baz'} - ] - }, - ] - url = reverse('dcim-api:site-list') - self.add_permissions('dcim.change_site') - response = self.client.patch(url, data, format='json', **self.header) - self.assertHttpStatus(response, status.HTTP_200_OK) - - # Verify that a job was queued for the object update webhook - self.assertEqual(self.queue.count, 3) - for i, job in enumerate(self.queue.jobs): - self.assertEqual(job.kwargs['webhook'], Webhook.objects.get(type_update=True)) - self.assertEqual(job.kwargs['event'], ObjectChangeActionChoices.ACTION_UPDATE) - self.assertEqual(job.kwargs['model_name'], 'site') - self.assertEqual(job.kwargs['data']['id'], data[i]['id']) - self.assertEqual(len(job.kwargs['data']['tags']), len(response.data[i]['tags'])) - self.assertEqual(job.kwargs['snapshots']['prechange']['name'], sites[i].name) - self.assertEqual(job.kwargs['snapshots']['prechange']['tags'], ['Bar', 'Foo']) - self.assertEqual(job.kwargs['snapshots']['postchange']['name'], response.data[i]['name']) - self.assertEqual(job.kwargs['snapshots']['postchange']['tags'], ['Baz']) - - def test_enqueue_webhook_delete(self): - site = Site.objects.create(name='Site 1', slug='site-1') - site.tags.set(Tag.objects.filter(name__in=['Foo', 'Bar'])) - - # Delete an object via the REST API - url = reverse('dcim-api:site-detail', kwargs={'pk': site.pk}) - self.add_permissions('dcim.delete_site') - response = self.client.delete(url, **self.header) - self.assertHttpStatus(response, status.HTTP_204_NO_CONTENT) - - # Verify that a job was queued for the object update webhook - self.assertEqual(self.queue.count, 1) - job = self.queue.jobs[0] - self.assertEqual(job.kwargs['webhook'], Webhook.objects.get(type_delete=True)) - self.assertEqual(job.kwargs['event'], ObjectChangeActionChoices.ACTION_DELETE) - self.assertEqual(job.kwargs['model_name'], 'site') - self.assertEqual(job.kwargs['data']['id'], site.pk) - self.assertEqual(job.kwargs['snapshots']['prechange']['name'], 'Site 1') - self.assertEqual(job.kwargs['snapshots']['prechange']['tags'], ['Bar', 'Foo']) - - def test_enqueue_webhook_bulk_delete(self): - sites = ( - Site(name='Site 1', slug='site-1'), - Site(name='Site 2', slug='site-2'), - Site(name='Site 3', slug='site-3'), - ) - Site.objects.bulk_create(sites) - for site in sites: - site.tags.set(Tag.objects.filter(name__in=['Foo', 'Bar'])) - - # Delete three objects via the REST API - data = [ - {'id': site.pk} for site in sites - ] - url = reverse('dcim-api:site-list') - self.add_permissions('dcim.delete_site') - response = self.client.delete(url, data, format='json', **self.header) - self.assertHttpStatus(response, status.HTTP_204_NO_CONTENT) - - # Verify that a job was queued for the object update webhook - self.assertEqual(self.queue.count, 3) - for i, job in enumerate(self.queue.jobs): - self.assertEqual(job.kwargs['webhook'], Webhook.objects.get(type_delete=True)) - self.assertEqual(job.kwargs['event'], ObjectChangeActionChoices.ACTION_DELETE) - self.assertEqual(job.kwargs['model_name'], 'site') - self.assertEqual(job.kwargs['data']['id'], sites[i].pk) - self.assertEqual(job.kwargs['snapshots']['prechange']['name'], sites[i].name) - self.assertEqual(job.kwargs['snapshots']['prechange']['tags'], ['Bar', 'Foo']) - - def test_webhook_conditions(self): - # Create a conditional Webhook - webhook = Webhook( - name='Conditional Webhook', - type_create=True, - type_update=True, - payload_url='http://localhost:9000/', - conditions={ - 'and': [ - { - 'attr': 'status.value', - 'value': 'active', - } - ] - } - ) - - # Create a Site to evaluate - site = Site.objects.create(name='Site 1', slug='site-1', status=SiteStatusChoices.STATUS_STAGING) - data = serialize_for_event(site) - - # Evaluate the conditions (status='staging') - self.assertFalse(eval_conditions(webhook, data)) - - # Change the site's status - site.status = SiteStatusChoices.STATUS_ACTIVE - data = serialize_for_event(site) - - # Evaluate the conditions (status='active') - self.assertTrue(eval_conditions(webhook, data)) - - def test_webhooks_worker(self): - - request_id = uuid.uuid4() - - def dummy_send(_, request, **kwargs): - """ - A dummy implementation of Session.send() to be used for testing. - Always returns a 200 HTTP response. - """ - webhook = Webhook.objects.get(type_create=True) - signature = generate_signature(request.body, webhook.secret) - - # Validate the outgoing request headers - self.assertEqual(request.headers['Content-Type'], webhook.http_content_type) - self.assertEqual(request.headers['X-Hook-Signature'], signature) - self.assertEqual(request.headers['X-Foo'], 'Bar') - - # Validate the outgoing request body - body = json.loads(request.body) - self.assertEqual(body['event'], 'created') - self.assertEqual(body['timestamp'], job.kwargs['timestamp']) - self.assertEqual(body['model'], 'site') - self.assertEqual(body['username'], 'testuser') - self.assertEqual(body['request_id'], str(request_id)) - self.assertEqual(body['data']['name'], 'Site 1') - - return HttpResponse() - - # Enqueue a webhook for processing - events_queue = [] - site = Site.objects.create(name='Site 1', slug='site-1') - enqueue_object( - events_queue, - instance=site, - user=self.user, - request_id=request_id, - action=ObjectChangeActionChoices.ACTION_CREATE - ) - flush_events(events_queue) - - # Retrieve the job from queue - job = self.queue.jobs[0] - - # Patch the Session object with our dummy_send() method, then process the webhook for sending - with patch.object(Session, 'send', dummy_send) as mock_send: - process_webhook(**job.kwargs) diff --git a/netbox/extras/tests/test_webhooks.py b/netbox/extras/tests/test_webhooks.py index e86814e4cff..361f0f24d77 100644 --- a/netbox/extras/tests/test_webhooks.py +++ b/netbox/extras/tests/test_webhooks.py @@ -14,8 +14,9 @@ from extras.choices import ObjectChangeActionChoices from extras.models import Tag, Webhook from extras.events import enqueue_object, flush_events, serialize_for_event +from extras.events_worker import eval_conditions from extras.webhooks import generate_signature -from extras.webhooks_worker import eval_conditions, process_webhook +from extras.webhooks_worker import process_webhook from utilities.testing import APITestCase @@ -36,298 +37,13 @@ def setUpTestData(cls): DUMMY_SECRET = 'LOOKATMEIMASECRETSTRING' webhooks = Webhook.objects.bulk_create(( - Webhook(name='Webhook 1', type_create=True, payload_url=DUMMY_URL, secret=DUMMY_SECRET, additional_headers='X-Foo: Bar'), - Webhook(name='Webhook 2', type_update=True, payload_url=DUMMY_URL, secret=DUMMY_SECRET), - Webhook(name='Webhook 3', type_delete=True, payload_url=DUMMY_URL, secret=DUMMY_SECRET), + Webhook(name='Webhook 1', payload_url=DUMMY_URL, secret=DUMMY_SECRET, additional_headers='X-Foo: Bar'), + Webhook(name='Webhook 2', payload_url=DUMMY_URL, secret=DUMMY_SECRET), + Webhook(name='Webhook 3', payload_url=DUMMY_URL, secret=DUMMY_SECRET), )) - for webhook in webhooks: - webhook.content_types.set([site_ct]) Tag.objects.bulk_create(( Tag(name='Foo', slug='foo'), Tag(name='Bar', slug='bar'), Tag(name='Baz', slug='baz'), )) - - def test_enqueue_webhook_create(self): - # Create an object via the REST API - data = { - 'name': 'Site 1', - 'slug': 'site-1', - 'tags': [ - {'name': 'Foo'}, - {'name': 'Bar'}, - ] - } - url = reverse('dcim-api:site-list') - self.add_permissions('dcim.add_site') - response = self.client.post(url, data, format='json', **self.header) - self.assertHttpStatus(response, status.HTTP_201_CREATED) - self.assertEqual(Site.objects.count(), 1) - self.assertEqual(Site.objects.first().tags.count(), 2) - - # Verify that a job was queued for the object creation webhook - self.assertEqual(self.queue.count, 1) - job = self.queue.jobs[0] - self.assertEqual(job.kwargs['webhook'], Webhook.objects.get(type_create=True)) - self.assertEqual(job.kwargs['event'], ObjectChangeActionChoices.ACTION_CREATE) - self.assertEqual(job.kwargs['model_name'], 'site') - self.assertEqual(job.kwargs['data']['id'], response.data['id']) - self.assertEqual(len(job.kwargs['data']['tags']), len(response.data['tags'])) - self.assertEqual(job.kwargs['snapshots']['postchange']['name'], 'Site 1') - self.assertEqual(job.kwargs['snapshots']['postchange']['tags'], ['Bar', 'Foo']) - - def test_enqueue_webhook_bulk_create(self): - # Create multiple objects via the REST API - data = [ - { - 'name': 'Site 1', - 'slug': 'site-1', - 'tags': [ - {'name': 'Foo'}, - {'name': 'Bar'}, - ] - }, - { - 'name': 'Site 2', - 'slug': 'site-2', - 'tags': [ - {'name': 'Foo'}, - {'name': 'Bar'}, - ] - }, - { - 'name': 'Site 3', - 'slug': 'site-3', - 'tags': [ - {'name': 'Foo'}, - {'name': 'Bar'}, - ] - }, - ] - url = reverse('dcim-api:site-list') - self.add_permissions('dcim.add_site') - response = self.client.post(url, data, format='json', **self.header) - self.assertHttpStatus(response, status.HTTP_201_CREATED) - self.assertEqual(Site.objects.count(), 3) - self.assertEqual(Site.objects.first().tags.count(), 2) - - # Verify that a webhook was queued for each object - self.assertEqual(self.queue.count, 3) - for i, job in enumerate(self.queue.jobs): - self.assertEqual(job.kwargs['webhook'], Webhook.objects.get(type_create=True)) - self.assertEqual(job.kwargs['event'], ObjectChangeActionChoices.ACTION_CREATE) - self.assertEqual(job.kwargs['model_name'], 'site') - self.assertEqual(job.kwargs['data']['id'], response.data[i]['id']) - self.assertEqual(len(job.kwargs['data']['tags']), len(response.data[i]['tags'])) - self.assertEqual(job.kwargs['snapshots']['postchange']['name'], response.data[i]['name']) - self.assertEqual(job.kwargs['snapshots']['postchange']['tags'], ['Bar', 'Foo']) - - def test_enqueue_webhook_update(self): - site = Site.objects.create(name='Site 1', slug='site-1') - site.tags.set(Tag.objects.filter(name__in=['Foo', 'Bar'])) - - # Update an object via the REST API - data = { - 'name': 'Site X', - 'comments': 'Updated the site', - 'tags': [ - {'name': 'Baz'} - ] - } - url = reverse('dcim-api:site-detail', kwargs={'pk': site.pk}) - self.add_permissions('dcim.change_site') - response = self.client.patch(url, data, format='json', **self.header) - self.assertHttpStatus(response, status.HTTP_200_OK) - - # Verify that a job was queued for the object update webhook - self.assertEqual(self.queue.count, 1) - job = self.queue.jobs[0] - self.assertEqual(job.kwargs['webhook'], Webhook.objects.get(type_update=True)) - self.assertEqual(job.kwargs['event'], ObjectChangeActionChoices.ACTION_UPDATE) - self.assertEqual(job.kwargs['model_name'], 'site') - self.assertEqual(job.kwargs['data']['id'], site.pk) - self.assertEqual(len(job.kwargs['data']['tags']), len(response.data['tags'])) - self.assertEqual(job.kwargs['snapshots']['prechange']['name'], 'Site 1') - self.assertEqual(job.kwargs['snapshots']['prechange']['tags'], ['Bar', 'Foo']) - self.assertEqual(job.kwargs['snapshots']['postchange']['name'], 'Site X') - self.assertEqual(job.kwargs['snapshots']['postchange']['tags'], ['Baz']) - - def test_enqueue_webhook_bulk_update(self): - sites = ( - Site(name='Site 1', slug='site-1'), - Site(name='Site 2', slug='site-2'), - Site(name='Site 3', slug='site-3'), - ) - Site.objects.bulk_create(sites) - for site in sites: - site.tags.set(Tag.objects.filter(name__in=['Foo', 'Bar'])) - - # Update three objects via the REST API - data = [ - { - 'id': sites[0].pk, - 'name': 'Site X', - 'tags': [ - {'name': 'Baz'} - ] - }, - { - 'id': sites[1].pk, - 'name': 'Site Y', - 'tags': [ - {'name': 'Baz'} - ] - }, - { - 'id': sites[2].pk, - 'name': 'Site Z', - 'tags': [ - {'name': 'Baz'} - ] - }, - ] - url = reverse('dcim-api:site-list') - self.add_permissions('dcim.change_site') - response = self.client.patch(url, data, format='json', **self.header) - self.assertHttpStatus(response, status.HTTP_200_OK) - - # Verify that a job was queued for the object update webhook - self.assertEqual(self.queue.count, 3) - for i, job in enumerate(self.queue.jobs): - self.assertEqual(job.kwargs['webhook'], Webhook.objects.get(type_update=True)) - self.assertEqual(job.kwargs['event'], ObjectChangeActionChoices.ACTION_UPDATE) - self.assertEqual(job.kwargs['model_name'], 'site') - self.assertEqual(job.kwargs['data']['id'], data[i]['id']) - self.assertEqual(len(job.kwargs['data']['tags']), len(response.data[i]['tags'])) - self.assertEqual(job.kwargs['snapshots']['prechange']['name'], sites[i].name) - self.assertEqual(job.kwargs['snapshots']['prechange']['tags'], ['Bar', 'Foo']) - self.assertEqual(job.kwargs['snapshots']['postchange']['name'], response.data[i]['name']) - self.assertEqual(job.kwargs['snapshots']['postchange']['tags'], ['Baz']) - - def test_enqueue_webhook_delete(self): - site = Site.objects.create(name='Site 1', slug='site-1') - site.tags.set(Tag.objects.filter(name__in=['Foo', 'Bar'])) - - # Delete an object via the REST API - url = reverse('dcim-api:site-detail', kwargs={'pk': site.pk}) - self.add_permissions('dcim.delete_site') - response = self.client.delete(url, **self.header) - self.assertHttpStatus(response, status.HTTP_204_NO_CONTENT) - - # Verify that a job was queued for the object update webhook - self.assertEqual(self.queue.count, 1) - job = self.queue.jobs[0] - self.assertEqual(job.kwargs['webhook'], Webhook.objects.get(type_delete=True)) - self.assertEqual(job.kwargs['event'], ObjectChangeActionChoices.ACTION_DELETE) - self.assertEqual(job.kwargs['model_name'], 'site') - self.assertEqual(job.kwargs['data']['id'], site.pk) - self.assertEqual(job.kwargs['snapshots']['prechange']['name'], 'Site 1') - self.assertEqual(job.kwargs['snapshots']['prechange']['tags'], ['Bar', 'Foo']) - - def test_enqueue_webhook_bulk_delete(self): - sites = ( - Site(name='Site 1', slug='site-1'), - Site(name='Site 2', slug='site-2'), - Site(name='Site 3', slug='site-3'), - ) - Site.objects.bulk_create(sites) - for site in sites: - site.tags.set(Tag.objects.filter(name__in=['Foo', 'Bar'])) - - # Delete three objects via the REST API - data = [ - {'id': site.pk} for site in sites - ] - url = reverse('dcim-api:site-list') - self.add_permissions('dcim.delete_site') - response = self.client.delete(url, data, format='json', **self.header) - self.assertHttpStatus(response, status.HTTP_204_NO_CONTENT) - - # Verify that a job was queued for the object update webhook - self.assertEqual(self.queue.count, 3) - for i, job in enumerate(self.queue.jobs): - self.assertEqual(job.kwargs['webhook'], Webhook.objects.get(type_delete=True)) - self.assertEqual(job.kwargs['event'], ObjectChangeActionChoices.ACTION_DELETE) - self.assertEqual(job.kwargs['model_name'], 'site') - self.assertEqual(job.kwargs['data']['id'], sites[i].pk) - self.assertEqual(job.kwargs['snapshots']['prechange']['name'], sites[i].name) - self.assertEqual(job.kwargs['snapshots']['prechange']['tags'], ['Bar', 'Foo']) - - def test_webhook_conditions(self): - # Create a conditional Webhook - webhook = Webhook( - name='Conditional Webhook', - type_create=True, - type_update=True, - payload_url='http://localhost:9000/', - conditions={ - 'and': [ - { - 'attr': 'status.value', - 'value': 'active', - } - ] - } - ) - - # Create a Site to evaluate - site = Site.objects.create(name='Site 1', slug='site-1', status=SiteStatusChoices.STATUS_STAGING) - data = serialize_for_event(site) - - # Evaluate the conditions (status='staging') - self.assertFalse(eval_conditions(webhook, data)) - - # Change the site's status - site.status = SiteStatusChoices.STATUS_ACTIVE - data = serialize_for_event(site) - - # Evaluate the conditions (status='active') - self.assertTrue(eval_conditions(webhook, data)) - - def test_webhooks_worker(self): - - request_id = uuid.uuid4() - - def dummy_send(_, request, **kwargs): - """ - A dummy implementation of Session.send() to be used for testing. - Always returns a 200 HTTP response. - """ - webhook = Webhook.objects.get(type_create=True) - signature = generate_signature(request.body, webhook.secret) - - # Validate the outgoing request headers - self.assertEqual(request.headers['Content-Type'], webhook.http_content_type) - self.assertEqual(request.headers['X-Hook-Signature'], signature) - self.assertEqual(request.headers['X-Foo'], 'Bar') - - # Validate the outgoing request body - body = json.loads(request.body) - self.assertEqual(body['event'], 'created') - self.assertEqual(body['timestamp'], job.kwargs['timestamp']) - self.assertEqual(body['model'], 'site') - self.assertEqual(body['username'], 'testuser') - self.assertEqual(body['request_id'], str(request_id)) - self.assertEqual(body['data']['name'], 'Site 1') - - return HttpResponse() - - # Enqueue a webhook for processing - events_queue = [] - site = Site.objects.create(name='Site 1', slug='site-1') - enqueue_object( - events_queue, - instance=site, - user=self.user, - request_id=request_id, - action=ObjectChangeActionChoices.ACTION_CREATE - ) - flush_events(events_queue) - - # Retrieve the job from queue - job = self.queue.jobs[0] - - # Patch the Session object with our dummy_send() method, then process the webhook for sending - with patch.object(Session, 'send', dummy_send) as mock_send: - process_webhook(**job.kwargs) From 4956203a290e22f0d34b3e972ab1fcdadaa95e02 Mon Sep 17 00:00:00 2001 From: Arthur Date: Mon, 13 Nov 2023 12:15:10 -0800 Subject: [PATCH 28/95] 14132 fix tests --- netbox/extras/tests/test_event_rules.py | 33 +++ netbox/extras/tests/test_webhooks.py | 287 +++++++++++++++++++++++- netbox/extras/webhooks_worker.py | 2 +- 3 files changed, 317 insertions(+), 5 deletions(-) diff --git a/netbox/extras/tests/test_event_rules.py b/netbox/extras/tests/test_event_rules.py index dd1a3fdaa4c..e1d8482b50c 100644 --- a/netbox/extras/tests/test_event_rules.py +++ b/netbox/extras/tests/test_event_rules.py @@ -47,3 +47,36 @@ def setUpTestData(cls): Tag(name='Bar', slug='bar'), Tag(name='Baz', slug='baz'), )) + + ''' + def test_webhook_conditions(self): + # Create a conditional Webhook + webhook = Webhook( + name='Conditional Webhook', + type_create=True, + type_update=True, + payload_url='http://localhost:9000/', + conditions={ + 'and': [ + { + 'attr': 'status.value', + 'value': 'active', + } + ] + } + ) + + # Create a Site to evaluate + site = Site.objects.create(name='Site 1', slug='site-1', status=SiteStatusChoices.STATUS_STAGING) + data = serialize_for_event(site) + + # Evaluate the conditions (status='staging') + self.assertFalse(eval_conditions(webhook, data)) + + # Change the site's status + site.status = SiteStatusChoices.STATUS_ACTIVE + data = serialize_for_event(site) + + # Evaluate the conditions (status='active') + self.assertTrue(eval_conditions(webhook, data)) + ''' diff --git a/netbox/extras/tests/test_webhooks.py b/netbox/extras/tests/test_webhooks.py index 361f0f24d77..1e0287bdc75 100644 --- a/netbox/extras/tests/test_webhooks.py +++ b/netbox/extras/tests/test_webhooks.py @@ -11,10 +11,9 @@ from dcim.choices import SiteStatusChoices from dcim.models import Site -from extras.choices import ObjectChangeActionChoices -from extras.models import Tag, Webhook -from extras.events import enqueue_object, flush_events, serialize_for_event -from extras.events_worker import eval_conditions +from extras.choices import ObjectChangeActionChoices, EventRuleActionChoices +from extras.models import Tag, Webhook, EventRule +from extras.events import enqueue_object, flush_events from extras.webhooks import generate_signature from extras.webhooks_worker import process_webhook from utilities.testing import APITestCase @@ -42,8 +41,288 @@ def setUpTestData(cls): Webhook(name='Webhook 3', payload_url=DUMMY_URL, secret=DUMMY_SECRET), )) + ct = ContentType.objects.get(app_label='extras', model='webhook') + event_rules = EventRule.objects.bulk_create(( + EventRule( + name='Webhook Event 1', + type_create=True, + action_type=EventRuleActionChoices.WEBHOOK, + action_object_type=ct, + action_object_id=webhooks[0].id + ), + EventRule( + name='Webhook Event 2', + type_update=True, + action_type=EventRuleActionChoices.WEBHOOK, + action_object_type=ct, + action_object_id=webhooks[0].id + ), + EventRule( + name='Webhook Event 3', + type_delete=True, + action_type=EventRuleActionChoices.WEBHOOK, + action_object_type=ct, + action_object_id=webhooks[0].id + ), + )) + for event_rule in event_rules: + event_rule.content_types.set([site_ct]) + Tag.objects.bulk_create(( Tag(name='Foo', slug='foo'), Tag(name='Bar', slug='bar'), Tag(name='Baz', slug='baz'), )) + + def test_enqueue_webhook_create(self): + # Create an object via the REST API + data = { + 'name': 'Site 1', + 'slug': 'site-1', + 'tags': [ + {'name': 'Foo'}, + {'name': 'Bar'}, + ] + } + url = reverse('dcim-api:site-list') + self.add_permissions('dcim.add_site') + response = self.client.post(url, data, format='json', **self.header) + self.assertHttpStatus(response, status.HTTP_201_CREATED) + self.assertEqual(Site.objects.count(), 1) + self.assertEqual(Site.objects.first().tags.count(), 2) + + # Verify that a job was queued for the object creation webhook + self.assertEqual(self.queue.count, 1) + job = self.queue.jobs[0] + self.assertEqual(job.kwargs['event_rule'], EventRule.objects.get(type_create=True)) + self.assertEqual(job.kwargs['event'], ObjectChangeActionChoices.ACTION_CREATE) + self.assertEqual(job.kwargs['model_name'], 'site') + self.assertEqual(job.kwargs['data']['id'], response.data['id']) + self.assertEqual(len(job.kwargs['data']['tags']), len(response.data['tags'])) + self.assertEqual(job.kwargs['snapshots']['postchange']['name'], 'Site 1') + self.assertEqual(job.kwargs['snapshots']['postchange']['tags'], ['Bar', 'Foo']) + + def test_enqueue_webhook_bulk_create(self): + # Create multiple objects via the REST API + data = [ + { + 'name': 'Site 1', + 'slug': 'site-1', + 'tags': [ + {'name': 'Foo'}, + {'name': 'Bar'}, + ] + }, + { + 'name': 'Site 2', + 'slug': 'site-2', + 'tags': [ + {'name': 'Foo'}, + {'name': 'Bar'}, + ] + }, + { + 'name': 'Site 3', + 'slug': 'site-3', + 'tags': [ + {'name': 'Foo'}, + {'name': 'Bar'}, + ] + }, + ] + url = reverse('dcim-api:site-list') + self.add_permissions('dcim.add_site') + response = self.client.post(url, data, format='json', **self.header) + self.assertHttpStatus(response, status.HTTP_201_CREATED) + self.assertEqual(Site.objects.count(), 3) + self.assertEqual(Site.objects.first().tags.count(), 2) + + # Verify that a webhook was queued for each object + self.assertEqual(self.queue.count, 3) + for i, job in enumerate(self.queue.jobs): + self.assertEqual(job.kwargs['event_rule'], EventRule.objects.get(type_create=True)) + self.assertEqual(job.kwargs['event'], ObjectChangeActionChoices.ACTION_CREATE) + self.assertEqual(job.kwargs['model_name'], 'site') + self.assertEqual(job.kwargs['data']['id'], response.data[i]['id']) + self.assertEqual(len(job.kwargs['data']['tags']), len(response.data[i]['tags'])) + self.assertEqual(job.kwargs['snapshots']['postchange']['name'], response.data[i]['name']) + self.assertEqual(job.kwargs['snapshots']['postchange']['tags'], ['Bar', 'Foo']) + + def test_enqueue_webhook_update(self): + site = Site.objects.create(name='Site 1', slug='site-1') + site.tags.set(Tag.objects.filter(name__in=['Foo', 'Bar'])) + + # Update an object via the REST API + data = { + 'name': 'Site X', + 'comments': 'Updated the site', + 'tags': [ + {'name': 'Baz'} + ] + } + url = reverse('dcim-api:site-detail', kwargs={'pk': site.pk}) + self.add_permissions('dcim.change_site') + response = self.client.patch(url, data, format='json', **self.header) + self.assertHttpStatus(response, status.HTTP_200_OK) + + # Verify that a job was queued for the object update webhook + self.assertEqual(self.queue.count, 1) + job = self.queue.jobs[0] + self.assertEqual(job.kwargs['event_rule'], EventRule.objects.get(type_update=True)) + self.assertEqual(job.kwargs['event'], ObjectChangeActionChoices.ACTION_UPDATE) + self.assertEqual(job.kwargs['model_name'], 'site') + self.assertEqual(job.kwargs['data']['id'], site.pk) + self.assertEqual(len(job.kwargs['data']['tags']), len(response.data['tags'])) + self.assertEqual(job.kwargs['snapshots']['prechange']['name'], 'Site 1') + self.assertEqual(job.kwargs['snapshots']['prechange']['tags'], ['Bar', 'Foo']) + self.assertEqual(job.kwargs['snapshots']['postchange']['name'], 'Site X') + self.assertEqual(job.kwargs['snapshots']['postchange']['tags'], ['Baz']) + + def test_enqueue_webhook_bulk_update(self): + sites = ( + Site(name='Site 1', slug='site-1'), + Site(name='Site 2', slug='site-2'), + Site(name='Site 3', slug='site-3'), + ) + Site.objects.bulk_create(sites) + for site in sites: + site.tags.set(Tag.objects.filter(name__in=['Foo', 'Bar'])) + + # Update three objects via the REST API + data = [ + { + 'id': sites[0].pk, + 'name': 'Site X', + 'tags': [ + {'name': 'Baz'} + ] + }, + { + 'id': sites[1].pk, + 'name': 'Site Y', + 'tags': [ + {'name': 'Baz'} + ] + }, + { + 'id': sites[2].pk, + 'name': 'Site Z', + 'tags': [ + {'name': 'Baz'} + ] + }, + ] + url = reverse('dcim-api:site-list') + self.add_permissions('dcim.change_site') + response = self.client.patch(url, data, format='json', **self.header) + self.assertHttpStatus(response, status.HTTP_200_OK) + + # Verify that a job was queued for the object update webhook + self.assertEqual(self.queue.count, 3) + for i, job in enumerate(self.queue.jobs): + self.assertEqual(job.kwargs['event_rule'], EventRule.objects.get(type_update=True)) + self.assertEqual(job.kwargs['event'], ObjectChangeActionChoices.ACTION_UPDATE) + self.assertEqual(job.kwargs['model_name'], 'site') + self.assertEqual(job.kwargs['data']['id'], data[i]['id']) + self.assertEqual(len(job.kwargs['data']['tags']), len(response.data[i]['tags'])) + self.assertEqual(job.kwargs['snapshots']['prechange']['name'], sites[i].name) + self.assertEqual(job.kwargs['snapshots']['prechange']['tags'], ['Bar', 'Foo']) + self.assertEqual(job.kwargs['snapshots']['postchange']['name'], response.data[i]['name']) + self.assertEqual(job.kwargs['snapshots']['postchange']['tags'], ['Baz']) + + def test_enqueue_webhook_delete(self): + site = Site.objects.create(name='Site 1', slug='site-1') + site.tags.set(Tag.objects.filter(name__in=['Foo', 'Bar'])) + + # Delete an object via the REST API + url = reverse('dcim-api:site-detail', kwargs={'pk': site.pk}) + self.add_permissions('dcim.delete_site') + response = self.client.delete(url, **self.header) + self.assertHttpStatus(response, status.HTTP_204_NO_CONTENT) + + # Verify that a job was queued for the object update webhook + self.assertEqual(self.queue.count, 1) + job = self.queue.jobs[0] + self.assertEqual(job.kwargs['event_rule'], EventRule.objects.get(type_delete=True)) + self.assertEqual(job.kwargs['event'], ObjectChangeActionChoices.ACTION_DELETE) + self.assertEqual(job.kwargs['model_name'], 'site') + self.assertEqual(job.kwargs['data']['id'], site.pk) + self.assertEqual(job.kwargs['snapshots']['prechange']['name'], 'Site 1') + self.assertEqual(job.kwargs['snapshots']['prechange']['tags'], ['Bar', 'Foo']) + + def test_enqueue_webhook_bulk_delete(self): + sites = ( + Site(name='Site 1', slug='site-1'), + Site(name='Site 2', slug='site-2'), + Site(name='Site 3', slug='site-3'), + ) + Site.objects.bulk_create(sites) + for site in sites: + site.tags.set(Tag.objects.filter(name__in=['Foo', 'Bar'])) + + # Delete three objects via the REST API + data = [ + {'id': site.pk} for site in sites + ] + url = reverse('dcim-api:site-list') + self.add_permissions('dcim.delete_site') + response = self.client.delete(url, data, format='json', **self.header) + self.assertHttpStatus(response, status.HTTP_204_NO_CONTENT) + + # Verify that a job was queued for the object update webhook + self.assertEqual(self.queue.count, 3) + for i, job in enumerate(self.queue.jobs): + self.assertEqual(job.kwargs['event_rule'], EventRule.objects.get(type_delete=True)) + self.assertEqual(job.kwargs['event'], ObjectChangeActionChoices.ACTION_DELETE) + self.assertEqual(job.kwargs['model_name'], 'site') + self.assertEqual(job.kwargs['data']['id'], sites[i].pk) + self.assertEqual(job.kwargs['snapshots']['prechange']['name'], sites[i].name) + self.assertEqual(job.kwargs['snapshots']['prechange']['tags'], ['Bar', 'Foo']) + + def test_webhooks_worker(self): + + request_id = uuid.uuid4() + + def dummy_send(_, request, **kwargs): + """ + A dummy implementation of Session.send() to be used for testing. + Always returns a 200 HTTP response. + """ + event = EventRule.objects.get(type_create=True) + webhook = event.action_object + signature = generate_signature(request.body, webhook.secret) + + # Validate the outgoing request headers + self.assertEqual(request.headers['Content-Type'], webhook.http_content_type) + self.assertEqual(request.headers['X-Hook-Signature'], signature) + self.assertEqual(request.headers['X-Foo'], 'Bar') + + # Validate the outgoing request body + body = json.loads(request.body) + self.assertEqual(body['event'], 'created') + self.assertEqual(body['timestamp'], job.kwargs['timestamp']) + self.assertEqual(body['model'], 'site') + self.assertEqual(body['username'], 'testuser') + self.assertEqual(body['request_id'], str(request_id)) + self.assertEqual(body['data']['name'], 'Site 1') + + return HttpResponse() + + # Enqueue a webhook for processing + webhooks_queue = [] + site = Site.objects.create(name='Site 1', slug='site-1') + enqueue_object( + webhooks_queue, + instance=site, + user=self.user, + request_id=request_id, + action=ObjectChangeActionChoices.ACTION_CREATE + ) + flush_events(webhooks_queue) + + # Retrieve the job from queue + job = self.queue.jobs[0] + + # Patch the Session object with our dummy_send() method, then process the webhook for sending + with patch.object(Session, 'send', dummy_send) as mock_send: + process_webhook(**job.kwargs) diff --git a/netbox/extras/webhooks_worker.py b/netbox/extras/webhooks_worker.py index 734a0784a16..f322346f26a 100644 --- a/netbox/extras/webhooks_worker.py +++ b/netbox/extras/webhooks_worker.py @@ -17,7 +17,7 @@ def process_webhook(event_rule, model_name, event, data, timestamp, username, re Make a POST request to the defined Webhook """ - webhook = event_rule.object + webhook = event_rule.action_object # Prepare context data for headers & body templates context = { From f5615e5c8ae6737309aa0582a1f7bfdef60c6567 Mon Sep 17 00:00:00 2001 From: Arthur Date: Mon, 13 Nov 2023 13:22:49 -0800 Subject: [PATCH 29/95] 14132 fix tests --- netbox/extras/tests/test_event_rules.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/netbox/extras/tests/test_event_rules.py b/netbox/extras/tests/test_event_rules.py index e1d8482b50c..29603208a4b 100644 --- a/netbox/extras/tests/test_event_rules.py +++ b/netbox/extras/tests/test_event_rules.py @@ -48,14 +48,12 @@ def setUpTestData(cls): Tag(name='Baz', slug='baz'), )) - ''' - def test_webhook_conditions(self): + def test_event_rule_conditions(self): # Create a conditional Webhook - webhook = Webhook( + webhook = EventRule( name='Conditional Webhook', type_create=True, type_update=True, - payload_url='http://localhost:9000/', conditions={ 'and': [ { @@ -79,4 +77,3 @@ def test_webhook_conditions(self): # Evaluate the conditions (status='active') self.assertTrue(eval_conditions(webhook, data)) - ''' From 3dbdd47ada2983b1b06303597df9def8fe7dcb74 Mon Sep 17 00:00:00 2001 From: Arthur Date: Mon, 13 Nov 2023 15:53:00 -0800 Subject: [PATCH 30/95] 14132 script run fixup --- netbox/extras/events_worker.py | 2 +- netbox/extras/forms/model_forms.py | 2 +- netbox/extras/scripts_worker.py | 31 ++++++++++++++++++++++++------ 3 files changed, 27 insertions(+), 8 deletions(-) diff --git a/netbox/extras/events_worker.py b/netbox/extras/events_worker.py index 2b6adc17d4f..0f664167a8f 100644 --- a/netbox/extras/events_worker.py +++ b/netbox/extras/events_worker.py @@ -45,7 +45,7 @@ def module_member(name): def process_event_rules(event_rule, model_name, event, data, timestamp, username, request_id, snapshots): if event_rule.action_type == EventRuleActionChoices.WEBHOOK: process_webhook(event_rule, model_name, event, data, timestamp, username, request_id, snapshots) - elif event_rule.action_type == EventRuleActionChoicesEventRuleTypeChoices.SCRIPT: + elif event_rule.action_type == EventRuleActionChoices.SCRIPT: process_script(event_rule, model_name, event, data, timestamp, username, request_id, snapshots) diff --git a/netbox/extras/forms/model_forms.py b/netbox/extras/forms/model_forms.py index 9fc51d98bb4..b04e9042e56 100644 --- a/netbox/extras/forms/model_forms.py +++ b/netbox/extras/forms/model_forms.py @@ -293,7 +293,7 @@ def get_script_choices(self): def get_webhook_choices(self): initial = None - if self.fields['action_object_type'] and self.fields['action_object_id']: + if self.fields['action_object_type'] and get_field_value(self, 'action_object_id'): initial = Webhook.objects.get(pk=get_field_value(self, 'action_object_id')) self.fields['action_choice'] = DynamicModelChoiceField( label=_('Webhook'), diff --git a/netbox/extras/scripts_worker.py b/netbox/extras/scripts_worker.py index 39454dfb44b..decc0263e4c 100644 --- a/netbox/extras/scripts_worker.py +++ b/netbox/extras/scripts_worker.py @@ -1,20 +1,39 @@ import logging import requests +from core.models import Job from django.conf import settings from django_rq import job from jinja2.exceptions import TemplateError +from utilities.rqworker import get_workers_for_queue -from .conditions import ConditionSet -from .constants import WEBHOOK_EVENT_TYPES -from .webhooks import generate_signature +from extras.conditions import ConditionSet +from extras.constants import WEBHOOK_EVENT_TYPES +from extras.models import ScriptModule +from extras.webhooks import generate_signature logger = logging.getLogger('netbox.webhooks_worker') -def process_script(webhook, model_name, event, data, timestamp, username, request_id=None, snapshots=None): +def process_script(event_rule, model_name, event, data, timestamp, username, request_id=None, snapshots=None): """ - Make a POST request to the defined Webhook + Run the requested script """ + module_name = event_rule.action_object_identifier.split(":")[0] + script_name = event_rule.action_object_identifier.split(":")[1] + try: + module = ScriptModule.objects.get(file_path__regex=f"^{module_name}\\.") + except ScriptModule.DoesNotExist: + return - pass + script = module.scripts[script_name]() + + job = Job.enqueue( + run_script, + instance=module, + name=script.class_name, + user=None, + schedule_at=None, + interval=None, + data=event_rule.parameters, + ) From 48ed7cc91d6e7137cef6133f3357f0072f4ddd9d Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 14 Nov 2023 09:11:00 -0800 Subject: [PATCH 31/95] 14132 fix runscript --- netbox/extras/forms/model_forms.py | 2 +- netbox/extras/scripts.py | 9 +++++---- netbox/extras/scripts_worker.py | 7 +++++-- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/netbox/extras/forms/model_forms.py b/netbox/extras/forms/model_forms.py index b04e9042e56..c0d94f33d85 100644 --- a/netbox/extras/forms/model_forms.py +++ b/netbox/extras/forms/model_forms.py @@ -282,7 +282,7 @@ def get_script_choices(self): for module in ScriptModule.objects.all(): scripts = [] for script_name in module.scripts.keys(): - name = f"{str(module.pk)}:{script_name.lower()}" + name = f"{str(module.pk)}:{script_name}" scripts.append((name, script_name)) if scripts: diff --git a/netbox/extras/scripts.py b/netbox/extras/scripts.py index f8f0a6d9a0c..969257e18b1 100644 --- a/netbox/extras/scripts.py +++ b/netbox/extras/scripts.py @@ -472,7 +472,7 @@ def get_module_and_script(module_name, script_name): return module, script -def run_script(data, request, job, commit=True, **kwargs): +def run_script(data, job, request=None, commit=True, **kwargs): """ A wrapper for calling Script.run(). This performs error handling and provides a hook for committing changes. It exists outside the Script class to ensure it cannot be overridden by a script author. @@ -486,9 +486,10 @@ def run_script(data, request, job, commit=True, **kwargs): logger.info(f"Running script (commit={commit})") # Add files to form data - files = request.FILES - for field_name, fileobj in files.items(): - data[field_name] = fileobj + if request: + files = request.FILES + for field_name, fileobj in files.items(): + data[field_name] = fileobj # Add the current request as a property of the script script.request = request diff --git a/netbox/extras/scripts_worker.py b/netbox/extras/scripts_worker.py index decc0263e4c..6fe6aae6d24 100644 --- a/netbox/extras/scripts_worker.py +++ b/netbox/extras/scripts_worker.py @@ -10,6 +10,7 @@ from extras.conditions import ConditionSet from extras.constants import WEBHOOK_EVENT_TYPES from extras.models import ScriptModule +from extras.scripts import run_script from extras.webhooks import generate_signature logger = logging.getLogger('netbox.webhooks_worker') @@ -19,13 +20,15 @@ def process_script(event_rule, model_name, event, data, timestamp, username, req """ Run the requested script """ - module_name = event_rule.action_object_identifier.split(":")[0] + module_id = event_rule.action_object_identifier.split(":")[0] script_name = event_rule.action_object_identifier.split(":")[1] + try: - module = ScriptModule.objects.get(file_path__regex=f"^{module_name}\\.") + module = ScriptModule.objects.get(pk=module_id) except ScriptModule.DoesNotExist: return + # breakpoint() script = module.scripts[script_name]() job = Job.enqueue( From 179564d6ff505b10d9c2d86d92a3a56ba7af2324 Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 14 Nov 2023 09:28:16 -0800 Subject: [PATCH 32/95] 14132 remove breakpoint --- netbox/extras/scripts_worker.py | 1 - 1 file changed, 1 deletion(-) diff --git a/netbox/extras/scripts_worker.py b/netbox/extras/scripts_worker.py index 6fe6aae6d24..c9345f3725d 100644 --- a/netbox/extras/scripts_worker.py +++ b/netbox/extras/scripts_worker.py @@ -28,7 +28,6 @@ def process_script(event_rule, model_name, event, data, timestamp, username, req except ScriptModule.DoesNotExist: return - # breakpoint() script = module.scripts[script_name]() job = Job.enqueue( From 90cf3c8b9d089488be11ed557de564a2b99ca6ab Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 14 Nov 2023 15:28:37 -0800 Subject: [PATCH 33/95] 14132 refactor pipeline code --- netbox/extras/events.py | 39 ++++++++++++++++++- netbox/extras/events_worker.py | 64 -------------------------------- netbox/extras/scripts_worker.py | 5 +++ netbox/extras/utils.py | 18 +++++++++ netbox/extras/webhooks_worker.py | 5 +++ netbox/netbox/settings.py | 2 +- 6 files changed, 66 insertions(+), 67 deletions(-) delete mode 100644 netbox/extras/events_worker.py diff --git a/netbox/extras/events.py b/netbox/extras/events.py index e3df359c027..0c138d53887 100644 --- a/netbox/extras/events.py +++ b/netbox/extras/events.py @@ -1,6 +1,9 @@ import hashlib import hmac +import logging +import sys +from django.conf import settings from django.contrib.contenttypes.models import ContentType from django.utils import timezone from django_rq import get_queue @@ -14,6 +17,8 @@ from .choices import * from .models import EventRule, Webhook +logger = logging.getLogger('netbox.events_processor') + def serialize_for_event(instance): """ @@ -65,7 +70,7 @@ def enqueue_object(queue, instance, user, request_id, action): }) -def flush_events(queue): +def process_event_rules(queue): """ Flush a list of object representation to RQ for webhook processing. """ @@ -96,8 +101,15 @@ def flush_events(queue): event_rules = events_cache[action_flag][content_type] for event_rule in event_rules: + if event_rule.action_type == EventRuleActionChoices.WEBHOOK: + processor = "extras.webhooks_worker.process_webhook" + elif event_rule.action_type == EventRuleActionChoices.SCRIPT: + processor = "extras.scripts_worker.process_script" + else: + return + rq_queue.enqueue( - "extras.events_worker.process_event", + processor, event_rule=event_rule, model_name=content_type.model, event=data['event'], @@ -108,3 +120,26 @@ def flush_events(queue): request_id=data['request_id'], retry=get_rq_retry() ) + + +def import_module(name): + __import__(name) + return sys.modules[name] + + +def module_member(name): + mod, member = name.rsplit(".", 1) + module = import_module(mod) + return getattr(module, member) + + +def flush_events(queue): + """ + Flush a list of object representation to RQ for webhook processing. + """ + for name in settings.NETBOX_EVENTS_PIPELINE: + try: + func = module_member(name) + func(queue) + except Exception as e: + logger.error(f"Cannot import events pipeline {name} error: {e}") diff --git a/netbox/extras/events_worker.py b/netbox/extras/events_worker.py deleted file mode 100644 index 0f664167a8f..00000000000 --- a/netbox/extras/events_worker.py +++ /dev/null @@ -1,64 +0,0 @@ -import logging - -import requests -import sys -from django.conf import settings -from django_rq import job -from jinja2.exceptions import TemplateError - -from .conditions import ConditionSet -from .choices import EventRuleActionChoices -from .constants import WEBHOOK_EVENT_TYPES -from .scripts_worker import process_script -from .webhooks import generate_signature -from .webhooks_worker import process_webhook - -logger = logging.getLogger('netbox.events_worker') - - -def eval_conditions(event_rule, data): - """ - Test whether the given data meets the conditions of the event rule (if any). Return True - if met or no conditions are specified. - """ - if not event_rule.conditions: - return True - - logger.debug(f'Evaluating event rule conditions: {event_rule.conditions}') - if ConditionSet(event_rule.conditions).eval(data): - return True - - return False - - -def import_module(name): - __import__(name) - return sys.modules[name] - - -def module_member(name): - mod, member = name.rsplit(".", 1) - module = import_module(mod) - return getattr(module, member) - - -def process_event_rules(event_rule, model_name, event, data, timestamp, username, request_id, snapshots): - if event_rule.action_type == EventRuleActionChoices.WEBHOOK: - process_webhook(event_rule, model_name, event, data, timestamp, username, request_id, snapshots) - elif event_rule.action_type == EventRuleActionChoices.SCRIPT: - process_script(event_rule, model_name, event, data, timestamp, username, request_id, snapshots) - - -@job('default') -def process_event(event_rule, model_name, event, data, timestamp, username, request_id=None, snapshots=None): - """ - Make a POST request to the defined Webhook - """ - # Evaluate event rule conditions (if any) - if not eval_conditions(event_rule, data): - return - - # process the events pipeline - for name in settings.NETBOX_EVENTS_PIPELINE: - func = module_member(name) - func(event_rule, model_name, event, data, timestamp, username, request_id, snapshots) diff --git a/netbox/extras/scripts_worker.py b/netbox/extras/scripts_worker.py index c9345f3725d..3c784d931ae 100644 --- a/netbox/extras/scripts_worker.py +++ b/netbox/extras/scripts_worker.py @@ -11,15 +11,20 @@ from extras.constants import WEBHOOK_EVENT_TYPES from extras.models import ScriptModule from extras.scripts import run_script +from extras.utils import eval_conditions from extras.webhooks import generate_signature logger = logging.getLogger('netbox.webhooks_worker') +@job('default') def process_script(event_rule, model_name, event, data, timestamp, username, request_id=None, snapshots=None): """ Run the requested script """ + if not eval_conditions(event_rule, data): + return + module_id = event_rule.action_object_identifier.split(":")[0] script_name = event_rule.action_object_identifier.split(":")[1] diff --git a/netbox/extras/utils.py b/netbox/extras/utils.py index 7b9356efb67..ae0a90013c1 100644 --- a/netbox/extras/utils.py +++ b/netbox/extras/utils.py @@ -1,9 +1,12 @@ +import logging from django.db.models import Q from django.utils.deconstruct import deconstructible from taggit.managers import _TaggableManager from netbox.registry import registry +logger = logging.getLogger('netbox.extras.utils') + def is_taggable(obj): """ @@ -92,3 +95,18 @@ def is_report(obj): return issubclass(obj, Report) and obj != Report except TypeError: return False + + +def eval_conditions(event_rule, data): + """ + Test whether the given data meets the conditions of the event rule (if any). Return True + if met or no conditions are specified. + """ + if not event_rule.conditions: + return True + + logger.debug(f'Evaluating event rule conditions: {event_rule.conditions}') + if ConditionSet(event_rule.conditions).eval(data): + return True + + return False diff --git a/netbox/extras/webhooks_worker.py b/netbox/extras/webhooks_worker.py index f322346f26a..01488550f7c 100644 --- a/netbox/extras/webhooks_worker.py +++ b/netbox/extras/webhooks_worker.py @@ -7,16 +7,21 @@ from .conditions import ConditionSet from .constants import WEBHOOK_EVENT_TYPES +from .utils import eval_conditions from .webhooks import generate_signature logger = logging.getLogger('netbox.webhooks_worker') +@job('default') def process_webhook(event_rule, model_name, event, data, timestamp, username, request_id=None, snapshots=None): """ Make a POST request to the defined Webhook """ + if not eval_conditions(event_rule, data): + return + webhook = event_rule.action_object # Prepare context data for headers & body templates diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index 72d22d84d1b..eb16be4d944 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -175,7 +175,7 @@ TIME_ZONE = getattr(configuration, 'TIME_ZONE', 'UTC') ENABLE_LOCALIZATION = getattr(configuration, 'ENABLE_LOCALIZATION', False) NETBOX_EVENTS_PIPELINE = getattr(configuration, 'NETBOX_EVENTS_PIPELINE', ( - 'extras.events_worker.process_event_rules', + 'extras.events.process_event_rules', )) From 6e7afd1020fb3e9236b4cc8c96d8f463804f7ce5 Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 14 Nov 2023 15:48:19 -0800 Subject: [PATCH 34/95] 14132 fix test --- netbox/extras/tests/test_event_rules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netbox/extras/tests/test_event_rules.py b/netbox/extras/tests/test_event_rules.py index 29603208a4b..91a99b519d4 100644 --- a/netbox/extras/tests/test_event_rules.py +++ b/netbox/extras/tests/test_event_rules.py @@ -14,7 +14,7 @@ from extras.choices import ObjectChangeActionChoices from extras.models import Tag, EventRule, Webhook from extras.events import enqueue_object, flush_events, serialize_for_event -from extras.events_worker import eval_conditions +from extras.utils import eval_conditions from extras.webhooks import generate_signature from extras.webhooks_worker import process_webhook from utilities.testing import APITestCase From 04866b47e60da52413a3d1bd206ba4bb3275860b Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 14 Nov 2023 16:08:09 -0800 Subject: [PATCH 35/95] 14132 fix test --- netbox/extras/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/netbox/extras/utils.py b/netbox/extras/utils.py index ae0a90013c1..e1be391e47c 100644 --- a/netbox/extras/utils.py +++ b/netbox/extras/utils.py @@ -3,6 +3,7 @@ from django.utils.deconstruct import deconstructible from taggit.managers import _TaggableManager +from extras.conditions import ConditionSet from netbox.registry import registry logger = logging.getLogger('netbox.extras.utils') From a2f542774df626bdfe6742dae1f1b07f7f993ca9 Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 14 Nov 2023 16:08:55 -0800 Subject: [PATCH 36/95] 14132 fix test --- netbox/extras/scripts_worker.py | 1 - netbox/extras/webhooks_worker.py | 1 - 2 files changed, 2 deletions(-) diff --git a/netbox/extras/scripts_worker.py b/netbox/extras/scripts_worker.py index 3c784d931ae..67715ad1604 100644 --- a/netbox/extras/scripts_worker.py +++ b/netbox/extras/scripts_worker.py @@ -7,7 +7,6 @@ from jinja2.exceptions import TemplateError from utilities.rqworker import get_workers_for_queue -from extras.conditions import ConditionSet from extras.constants import WEBHOOK_EVENT_TYPES from extras.models import ScriptModule from extras.scripts import run_script diff --git a/netbox/extras/webhooks_worker.py b/netbox/extras/webhooks_worker.py index 01488550f7c..d4a9bbee182 100644 --- a/netbox/extras/webhooks_worker.py +++ b/netbox/extras/webhooks_worker.py @@ -5,7 +5,6 @@ from django_rq import job from jinja2.exceptions import TemplateError -from .conditions import ConditionSet from .constants import WEBHOOK_EVENT_TYPES from .utils import eval_conditions from .webhooks import generate_signature From 12b4bf80bc5c90830999b24ac5c7a8cb3fe6405b Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 15 Nov 2023 08:34:41 -0800 Subject: [PATCH 37/95] 14132 optimize flush_events --- netbox/extras/events.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/netbox/extras/events.py b/netbox/extras/events.py index 0c138d53887..610f544ac36 100644 --- a/netbox/extras/events.py +++ b/netbox/extras/events.py @@ -137,9 +137,10 @@ def flush_events(queue): """ Flush a list of object representation to RQ for webhook processing. """ - for name in settings.NETBOX_EVENTS_PIPELINE: - try: - func = module_member(name) - func(queue) - except Exception as e: - logger.error(f"Cannot import events pipeline {name} error: {e}") + if queue: + for name in settings.NETBOX_EVENTS_PIPELINE: + try: + func = module_member(name) + func(queue) + except Exception as e: + logger.error(f"Cannot import events pipeline {name} error: {e}") From 58136deeaffe5a849b9336a410fd62d6f4a6c646 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Wed, 15 Nov 2023 16:54:19 -0500 Subject: [PATCH 38/95] Misc cleanup --- netbox/extras/choices.py | 4 ++-- netbox/extras/constants.py | 7 ------- netbox/extras/events.py | 14 ++++++-------- netbox/extras/filtersets.py | 7 +++---- netbox/extras/models/models.py | 22 +++++++++++----------- netbox/netbox/settings.py | 6 +++--- 6 files changed, 25 insertions(+), 35 deletions(-) diff --git a/netbox/extras/choices.py b/netbox/extras/choices.py index df3b47d9ebd..896b5c72d72 100644 --- a/netbox/extras/choices.py +++ b/netbox/extras/choices.py @@ -292,6 +292,6 @@ class EventRuleActionChoices(ChoiceSet): SCRIPT = 'script' CHOICES = ( - (WEBHOOK, _('Webhook'), 'webhook'), - (SCRIPT, _('Script'), 'script'), + (WEBHOOK, _('Webhook')), + (SCRIPT, _('Script')), ) diff --git a/netbox/extras/constants.py b/netbox/extras/constants.py index a976baf8ca9..fec8f5ef874 100644 --- a/netbox/extras/constants.py +++ b/netbox/extras/constants.py @@ -136,10 +136,3 @@ } }, ] - -EVENT_TYPE_MODELS = Q( - app_label='extras', - model__in=( - 'webhook', - 'script', - )) diff --git a/netbox/extras/events.py b/netbox/extras/events.py index 610f544ac36..52b452ae216 100644 --- a/netbox/extras/events.py +++ b/netbox/extras/events.py @@ -1,5 +1,3 @@ -import hashlib -import hmac import logging import sys @@ -15,14 +13,14 @@ from utilities.rqworker import get_rq_retry from utilities.utils import serialize_object from .choices import * -from .models import EventRule, Webhook +from .models import EventRule logger = logging.getLogger('netbox.events_processor') def serialize_for_event(instance): """ - Return a serialized representation of the given instance suitable for use in a webhook. + Return a serialized representation of the given instance suitable for use in a queued event. """ serializer_class = get_serializer_for_model(instance.__class__) serializer_context = { @@ -51,9 +49,9 @@ def get_snapshots(instance, action): def enqueue_object(queue, instance, user, request_id, action): """ Enqueue a serialized representation of a created/updated/deleted object for the processing of - webhooks once the request has completed. + events once the request has completed. """ - # Determine whether this type of object supports webhooks + # Determine whether this type of object supports event rules app_label = instance._meta.app_label model_name = instance._meta.model_name if model_name not in registry['model_features']['webhooks'].get(app_label, []): @@ -72,7 +70,7 @@ def enqueue_object(queue, instance, user, request_id, action): def process_event_rules(queue): """ - Flush a list of object representation to RQ for webhook processing. + Flush a list of object representation to RQ for EventRule processing. """ rq_queue_name = get_config().QUEUE_MAPPINGS.get('webhook', RQ_QUEUE_DEFAULT) rq_queue = get_queue(rq_queue_name) @@ -138,7 +136,7 @@ def flush_events(queue): Flush a list of object representation to RQ for webhook processing. """ if queue: - for name in settings.NETBOX_EVENTS_PIPELINE: + for name in settings.EVENTS_PIPELINE: try: func = module_member(name) func(queue) diff --git a/netbox/extras/filtersets.py b/netbox/extras/filtersets.py index 044ff3d3871..9fd88c9fad3 100644 --- a/netbox/extras/filtersets.py +++ b/netbox/extras/filtersets.py @@ -47,8 +47,8 @@ class WebhookFilterSet(NetBoxModelFilterSet): class Meta: model = Webhook fields = [ - 'id', 'name', 'payload_url', - 'http_method', 'http_content_type', 'secret', 'ssl_verification', 'ca_file_path', + 'id', 'name', 'payload_url', 'http_method', 'http_content_type', 'secret', 'ssl_verification', + 'ca_file_path', ] def search(self, queryset, name, value): @@ -73,8 +73,7 @@ class EventRuleFilterSet(NetBoxModelFilterSet): class Meta: model = EventRule fields = [ - 'id', 'name', 'type_create', 'type_update', 'type_delete', 'type_job_start', 'type_job_end', - 'enabled', + 'id', 'name', 'type_create', 'type_update', 'type_delete', 'type_job_start', 'type_job_end', 'enabled', ] def search(self, queryset, name, value): diff --git a/netbox/extras/models/models.py b/netbox/extras/models/models.py index 63abe050434..5fb53db6b72 100644 --- a/netbox/extras/models/models.py +++ b/netbox/extras/models/models.py @@ -41,14 +41,15 @@ class EventRule(CustomFieldsMixin, ExportTemplatesMixin, TagsMixin, ChangeLoggedModel): """ - A Webhook defines a request that will be sent to a remote application when an object is created, updated, and/or - delete in NetBox. The request will contain a representation of the object, which the remote application can act on. - Each Webhook can be limited to firing only on certain actions or certain object types. + An EventRule defines an action to be taken automatically in response to a specific set of events, such as when a + specific type of object is created, modified, or deleted. The action to be taken might entail transmitting a + webhook or executing a custom script. """ content_types = models.ManyToManyField( to=ContentType, related_name='eventrules', verbose_name=_('object types'), + # TODO: FeatureQuery is being removed under #14153 limit_choices_to=FeatureQuery('eventrules'), help_text=_("The object(s) to which this Event applies.") ) @@ -98,12 +99,12 @@ class EventRule(CustomFieldsMixin, ExportTemplatesMixin, TagsMixin, ChangeLogged max_length=30, choices=EventRuleActionChoices, default=EventRuleActionChoices.WEBHOOK, - verbose_name=_('event type') + verbose_name=_('action type') ) action_object_type = models.ForeignKey( to=ContentType, related_name='eventrule_actions', - on_delete=models.CASCADE, + on_delete=models.CASCADE ) action_object_id = models.PositiveBigIntegerField( blank=True, @@ -111,9 +112,8 @@ class EventRule(CustomFieldsMixin, ExportTemplatesMixin, TagsMixin, ChangeLogged ) action_object = GenericForeignKey( ct_field='action_object_type', - fk_field='action_object_id', + fk_field='action_object_id' ) - # internal (not show in UI) - used by scripts to store function name action_object_identifier = models.CharField( max_length=80, @@ -128,8 +128,8 @@ class EventRule(CustomFieldsMixin, ExportTemplatesMixin, TagsMixin, ChangeLogged class Meta: ordering = ('name',) - verbose_name = _('eventrule') - verbose_name_plural = _('eventrules') + verbose_name = _('event rule') + verbose_name_plural = _('event rules') def __str__(self): return self.name @@ -149,9 +149,10 @@ def clean(self): self.type_create, self.type_update, self.type_delete, self.type_job_start, self.type_job_end ]): raise ValidationError( - _("At least one event type must be selected: create, update, delete, job_start, and/or job_end.") + _("At least one event type must be selected: create, update, delete, job start, and/or job end.") ) + # Validate that any conditions are in the correct format if self.conditions: try: ConditionSet(self.conditions) @@ -170,7 +171,6 @@ class Webhook(CustomFieldsMixin, ExportTemplatesMixin, TagsMixin, ChangeLoggedMo content_type_field='action_object_type', object_id_field='action_object_id' ) - name = models.CharField( verbose_name=_('name'), max_length=150, diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index eb16be4d944..b6308ad981b 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -115,6 +115,9 @@ DEVELOPER = getattr(configuration, 'DEVELOPER', False) DOCS_ROOT = getattr(configuration, 'DOCS_ROOT', os.path.join(os.path.dirname(BASE_DIR), 'docs')) EMAIL = getattr(configuration, 'EMAIL', {}) +EVENTS_PIPELINE = getattr(configuration, 'EVENTS_PIPELINE', ( + 'extras.events.process_event_rules', +)) EXEMPT_VIEW_PERMISSIONS = getattr(configuration, 'EXEMPT_VIEW_PERMISSIONS', []) FIELD_CHOICES = getattr(configuration, 'FIELD_CHOICES', {}) FILE_UPLOAD_MAX_MEMORY_SIZE = getattr(configuration, 'FILE_UPLOAD_MAX_MEMORY_SIZE', 2621440) @@ -174,9 +177,6 @@ TIME_FORMAT = getattr(configuration, 'TIME_FORMAT', 'g:i a') TIME_ZONE = getattr(configuration, 'TIME_ZONE', 'UTC') ENABLE_LOCALIZATION = getattr(configuration, 'ENABLE_LOCALIZATION', False) -NETBOX_EVENTS_PIPELINE = getattr(configuration, 'NETBOX_EVENTS_PIPELINE', ( - 'extras.events.process_event_rules', -)) # Check for hard-coded dynamic config parameters From e9950459b438397207e746c1c8ca2add1818853a Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 15 Nov 2023 16:14:26 -0800 Subject: [PATCH 39/95] 14132 review changes --- netbox/extras/context_managers.py | 4 ++-- netbox/extras/management/commands/runscript.py | 8 ++++---- netbox/extras/migrations/0099_eventrule.py | 4 ++-- netbox/extras/models/models.py | 8 ++------ netbox/extras/scripts.py | 8 ++++---- netbox/extras/scripts_worker.py | 6 +++--- netbox/netbox/middleware.py | 6 +++--- 7 files changed, 20 insertions(+), 24 deletions(-) diff --git a/netbox/extras/context_managers.py b/netbox/extras/context_managers.py index dc4d3dd177d..711a2067a58 100644 --- a/netbox/extras/context_managers.py +++ b/netbox/extras/context_managers.py @@ -5,9 +5,9 @@ @contextmanager -def event_wrapper(request): +def event_tracking(request): """ - Enable change logging by connecting the appropriate signals to their receivers before code is run, and + Enable event tracking by connecting the appropriate signals to their receivers before code is run, and disconnecting them afterward. :param request: WSGIRequest object with a unique `id` set diff --git a/netbox/extras/management/commands/runscript.py b/netbox/extras/management/commands/runscript.py index c801441b264..5a5e0287e80 100644 --- a/netbox/extras/management/commands/runscript.py +++ b/netbox/extras/management/commands/runscript.py @@ -11,7 +11,7 @@ from core.choices import JobStatusChoices from core.models import Job from extras.api.serializers import ScriptOutputSerializer -from extras.context_managers import event_wrapper +from extras.context_managers import event_tracking from extras.scripts import get_module_and_script from extras.signals import clear_webhooks from utilities.exceptions import AbortTransaction @@ -37,7 +37,7 @@ def handle(self, *args, **options): def _run_script(): """ Core script execution task. We capture this within a subfunction to allow for conditionally wrapping it with - the event_wrapper context manager (which is bypassed if commit == False). + the event_tracking context manager (which is bypassed if commit == False). """ try: try: @@ -136,9 +136,9 @@ def _run_script(): logger.info(f"Running script (commit={commit})") script.request = request - # Execute the script. If commit is True, wrap it with the event_wrapper context manager to ensure we process + # Execute the script. If commit is True, wrap it with the event_tracking context manager to ensure we process # change logging, webhooks, etc. - with event_wrapper(request): + with event_tracking(request): _run_script() else: logger.error('Data is not valid:') diff --git a/netbox/extras/migrations/0099_eventrule.py b/netbox/extras/migrations/0099_eventrule.py index cb4f2069d5f..1b5648d366c 100644 --- a/netbox/extras/migrations/0099_eventrule.py +++ b/netbox/extras/migrations/0099_eventrule.py @@ -75,8 +75,8 @@ class Migration(migrations.Migration): to='contenttypes.contenttype', ), ), - ('action_object_identifier', models.CharField(max_length=80, blank=True)), - ('parameters', models.JSONField(blank=True, null=True)), + ('action_parameters', models.CharField(max_length=80, blank=True)), + ('action_data', models.JSONField(blank=True, null=True)), ('tags', taggit.managers.TaggableManager(through='extras.TaggedItem', to='extras.Tag')), ], options={ diff --git a/netbox/extras/models/models.py b/netbox/extras/models/models.py index 5fb53db6b72..c37a6ff87c8 100644 --- a/netbox/extras/models/models.py +++ b/netbox/extras/models/models.py @@ -115,11 +115,11 @@ class EventRule(CustomFieldsMixin, ExportTemplatesMixin, TagsMixin, ChangeLogged fk_field='action_object_id' ) # internal (not show in UI) - used by scripts to store function name - action_object_identifier = models.CharField( + action_parameters = models.CharField( max_length=80, blank=True ) - parameters = models.JSONField( + action_data = models.JSONField( verbose_name=_('parameters'), blank=True, null=True, @@ -137,10 +137,6 @@ def __str__(self): def get_absolute_url(self): return reverse('extras:eventrule', args=[self.pk]) - @property - def docs_url(self): - return f'{settings.STATIC_URL}docs/models/extras/eventrule/' - def clean(self): super().clean() diff --git a/netbox/extras/scripts.py b/netbox/extras/scripts.py index 969257e18b1..0310b9973a5 100644 --- a/netbox/extras/scripts.py +++ b/netbox/extras/scripts.py @@ -23,7 +23,7 @@ from utilities.exceptions import AbortScript, AbortTransaction from utilities.forms import add_blank_choice from utilities.forms.fields import DynamicModelChoiceField, DynamicModelMultipleChoiceField -from .context_managers import event_wrapper +from .context_managers import event_tracking from .forms import ScriptForm __all__ = ( @@ -497,7 +497,7 @@ def run_script(data, job, request=None, commit=True, **kwargs): def _run_script(): """ Core script execution task. We capture this within a subfunction to allow for conditionally wrapping it with - the event_wrapper context manager (which is bypassed if commit == False). + the event_tracking context manager (which is bypassed if commit == False). """ try: try: @@ -525,10 +525,10 @@ def _run_script(): logger.info(f"Script completed in {job.duration}") - # Execute the script. If commit is True, wrap it with the event_wrapper context manager to ensure we process + # Execute the script. If commit is True, wrap it with the event_tracking context manager to ensure we process # change logging, webhooks, etc. if commit: - with event_wrapper(request): + with event_tracking(request): _run_script() else: _run_script() diff --git a/netbox/extras/scripts_worker.py b/netbox/extras/scripts_worker.py index 67715ad1604..49ab425ecb6 100644 --- a/netbox/extras/scripts_worker.py +++ b/netbox/extras/scripts_worker.py @@ -24,8 +24,8 @@ def process_script(event_rule, model_name, event, data, timestamp, username, req if not eval_conditions(event_rule, data): return - module_id = event_rule.action_object_identifier.split(":")[0] - script_name = event_rule.action_object_identifier.split(":")[1] + module_id = event_rule.action_parameters.split(":")[0] + script_name = event_rule.action_parameters.split(":")[1] try: module = ScriptModule.objects.get(pk=module_id) @@ -41,5 +41,5 @@ def process_script(event_rule, model_name, event, data, timestamp, username, req user=None, schedule_at=None, interval=None, - data=event_rule.parameters, + data=event_rule.action_data, ) diff --git a/netbox/netbox/middleware.py b/netbox/netbox/middleware.py index 37cb041db8c..cb7d2c8bae2 100644 --- a/netbox/netbox/middleware.py +++ b/netbox/netbox/middleware.py @@ -10,7 +10,7 @@ from django.db.utils import InternalError from django.http import Http404, HttpResponseRedirect -from extras.context_managers import event_wrapper +from extras.context_managers import event_tracking from netbox.config import clear_config, get_config from netbox.views import handler_500 from utilities.api import is_api_request, rest_api_server_error @@ -42,8 +42,8 @@ def __call__(self, request): login_url = f'{settings.LOGIN_URL}?next={parse.quote(request.get_full_path_info())}' return HttpResponseRedirect(login_url) - # Enable the event_wrapper context manager and process the request. - with event_wrapper(request): + # Enable the event_tracking context manager and process the request. + with event_tracking(request): response = self.get_response(request) # Attach the unique request ID as an HTTP header. From dcb64832227e21ad5c6f32bbdd40dcea48a1c94b Mon Sep 17 00:00:00 2001 From: Arthur Date: Thu, 16 Nov 2023 07:59:10 -0800 Subject: [PATCH 40/95] 14132 review changes --- netbox/core/models/jobs.py | 10 ++++++---- netbox/extras/events.py | 27 +++------------------------ netbox/extras/utils.py | 37 +++++++++++++++++++++++++++++++++++++ netbox/netbox/settings.py | 2 +- 4 files changed, 47 insertions(+), 29 deletions(-) diff --git a/netbox/core/models/jobs.py b/netbox/core/models/jobs.py index 497c6d7005f..0b43de7f0e1 100644 --- a/netbox/core/models/jobs.py +++ b/netbox/core/models/jobs.py @@ -12,7 +12,7 @@ from core.choices import JobStatusChoices from extras.constants import EVENT_JOB_END, EVENT_JOB_START -from extras.utils import FeatureQuery +from extras.utils import FeatureQuery, process_event_rules from netbox.config import get_config from netbox.constants import RQ_QUEUE_DEFAULT from utilities.querysets import RestrictedQuerySet @@ -219,9 +219,6 @@ def enqueue(cls, func, instance, name='', user=None, schedule_at=None, interval= def trigger_events(self, event): from extras.models import EventRule - rq_queue_name = get_config().QUEUE_MAPPINGS.get('webhook', RQ_QUEUE_DEFAULT) - rq_queue = django_rq.get_queue(rq_queue_name, is_async=False) - # Fetch any webhooks matching this object type and action event_rules = EventRule.objects.filter( **{f'type_{event}': True}, @@ -229,6 +226,11 @@ def trigger_events(self, event): enabled=True ) + process_event_rules(event_rules, event, self.data, self.user.username) + + rq_queue_name = get_config().QUEUE_MAPPINGS.get('webhook', RQ_QUEUE_DEFAULT) + rq_queue = django_rq.get_queue(rq_queue_name, is_async=False) + for event_rule in event_rules: rq_queue.enqueue( "extras.events_worker.process_event", diff --git a/netbox/extras/events.py b/netbox/extras/events.py index 52b452ae216..15f6be7b866 100644 --- a/netbox/extras/events.py +++ b/netbox/extras/events.py @@ -14,6 +14,7 @@ from utilities.utils import serialize_object from .choices import * from .models import EventRule +from .utils import process_event_rules logger = logging.getLogger('netbox.events_processor') @@ -68,12 +69,10 @@ def enqueue_object(queue, instance, user, request_id, action): }) -def process_event_rules(queue): +def process_event_queue(queue): """ Flush a list of object representation to RQ for EventRule processing. """ - rq_queue_name = get_config().QUEUE_MAPPINGS.get('webhook', RQ_QUEUE_DEFAULT) - rq_queue = get_queue(rq_queue_name) events_cache = { 'type_create': {}, 'type_update': {}, @@ -81,7 +80,6 @@ def process_event_rules(queue): } for data in queue: - action_flag = { ObjectChangeActionChoices.ACTION_CREATE: 'type_create', ObjectChangeActionChoices.ACTION_UPDATE: 'type_update', @@ -98,26 +96,7 @@ def process_event_rules(queue): ) event_rules = events_cache[action_flag][content_type] - for event_rule in event_rules: - if event_rule.action_type == EventRuleActionChoices.WEBHOOK: - processor = "extras.webhooks_worker.process_webhook" - elif event_rule.action_type == EventRuleActionChoices.SCRIPT: - processor = "extras.scripts_worker.process_script" - else: - return - - rq_queue.enqueue( - processor, - event_rule=event_rule, - model_name=content_type.model, - event=data['event'], - data=data['data'], - snapshots=data['snapshots'], - timestamp=str(timezone.now()), - username=data['username'], - request_id=data['request_id'], - retry=get_rq_retry() - ) + process_event_rules(event_rules, data['event'], data['data'], data['username'], data['snapshots'], data['request_id']) def import_module(name): diff --git a/netbox/extras/utils.py b/netbox/extras/utils.py index e1be391e47c..8c8d7d246bc 100644 --- a/netbox/extras/utils.py +++ b/netbox/extras/utils.py @@ -1,9 +1,12 @@ import logging from django.db.models import Q from django.utils.deconstruct import deconstructible +from django_rq import get_queue from taggit.managers import _TaggableManager from extras.conditions import ConditionSet +from extras.choices import EventRuleActionChoices +from netbox.config import get_config from netbox.registry import registry logger = logging.getLogger('netbox.extras.utils') @@ -111,3 +114,37 @@ def eval_conditions(event_rule, data): return True return False + + +def process_event_rules(event_rules, event, data, username, snapshots=None, request_id=None): + rq_queue_name = get_config().QUEUE_MAPPINGS.get('webhook', RQ_QUEUE_DEFAULT) + rq_queue = get_queue(rq_queue_name) + + for event_rule in event_rules: + if event_rule.action_type == EventRuleActionChoices.WEBHOOK: + processor = "extras.webhooks_worker.process_webhook" + elif event_rule.action_type == EventRuleActionChoices.SCRIPT: + processor = "extras.scripts_worker.process_script" + else: + raise ValueError(f"Unknown Event Rule action type: {event_rule.action_type}") + + params = { + "event_rule": event_rule, + "model_name": content_type.model, + "event": event, + "data": data, + "snapshots": snapshots, + "timestamp": str(timezone.now()), + "username": username, + "retry": get_rq_retry() + } + + if snapshots: + params["snapshots"] = snapshots + if request_id: + params["request_id"] = request_id + + rq_queue.enqueue( + processor, + **params + ) diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index 8dc7a6d4674..d85d2af5ce9 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -116,7 +116,7 @@ DOCS_ROOT = getattr(configuration, 'DOCS_ROOT', os.path.join(os.path.dirname(BASE_DIR), 'docs')) EMAIL = getattr(configuration, 'EMAIL', {}) EVENTS_PIPELINE = getattr(configuration, 'EVENTS_PIPELINE', ( - 'extras.events.process_event_rules', + 'extras.events.process_event_queue', )) EXEMPT_VIEW_PERMISSIONS = getattr(configuration, 'EXEMPT_VIEW_PERMISSIONS', []) FIELD_CHOICES = getattr(configuration, 'FIELD_CHOICES', {}) From ca0a567081bf1201eb2f0ed40cd5294e62901e76 Mon Sep 17 00:00:00 2001 From: Arthur Date: Thu, 16 Nov 2023 10:14:22 -0800 Subject: [PATCH 41/95] 14132 fix imports --- netbox/core/models/jobs.py | 19 ++----------------- netbox/extras/events.py | 6 ++++-- netbox/extras/utils.py | 9 ++++++--- 3 files changed, 12 insertions(+), 22 deletions(-) diff --git a/netbox/core/models/jobs.py b/netbox/core/models/jobs.py index 587a6936b86..ad142f47316 100644 --- a/netbox/core/models/jobs.py +++ b/netbox/core/models/jobs.py @@ -17,7 +17,7 @@ from netbox.config import get_config from netbox.constants import RQ_QUEUE_DEFAULT from utilities.querysets import RestrictedQuerySet -from utilities.rqworker import get_queue_for_model, get_rq_retry +from utilities.rqworker import get_queue_for_model __all__ = ( 'Job', @@ -235,19 +235,4 @@ def trigger_events(self, event): enabled=True ) - process_event_rules(event_rules, event, self.data, self.user.username) - - rq_queue_name = get_config().QUEUE_MAPPINGS.get('webhook', RQ_QUEUE_DEFAULT) - rq_queue = django_rq.get_queue(rq_queue_name, is_async=False) - - for event_rule in event_rules: - rq_queue.enqueue( - "extras.events_worker.process_event", - event_rule=event_rule, - model_name=self.object_type.model, - event=event, - data=self.data, - timestamp=str(timezone.now()), - username=self.user.username, - retry=get_rq_retry() - ) + process_event_rules(event_rules, self.object_type.model, event, self.data, self.user.username) diff --git a/netbox/extras/events.py b/netbox/extras/events.py index 15f6be7b866..d08dfcd4dc3 100644 --- a/netbox/extras/events.py +++ b/netbox/extras/events.py @@ -10,7 +10,6 @@ from netbox.constants import RQ_QUEUE_DEFAULT from netbox.registry import registry from utilities.api import get_serializer_for_model -from utilities.rqworker import get_rq_retry from utilities.utils import serialize_object from .choices import * from .models import EventRule @@ -96,7 +95,10 @@ def process_event_queue(queue): ) event_rules = events_cache[action_flag][content_type] - process_event_rules(event_rules, data['event'], data['data'], data['username'], data['snapshots'], data['request_id']) + process_event_rules( + event_rules, content_type.model, data['event'], data['data'], data['username'], + snapshots=data['snapshots'], request_id=data['request_id'] + ) def import_module(name): diff --git a/netbox/extras/utils.py b/netbox/extras/utils.py index 80bc9327724..6249e130751 100644 --- a/netbox/extras/utils.py +++ b/netbox/extras/utils.py @@ -1,13 +1,16 @@ import logging from django.db.models import Q -from django.utils.deconstruct import deconstructible from django_rq import get_queue +from django.utils import timezone +from django.utils.deconstruct import deconstructible from taggit.managers import _TaggableManager from extras.conditions import ConditionSet +from netbox.constants import RQ_QUEUE_DEFAULT from extras.choices import EventRuleActionChoices from netbox.config import get_config from netbox.registry import registry +from utilities.rqworker import get_rq_retry logger = logging.getLogger('netbox.extras.utils') @@ -93,7 +96,7 @@ def eval_conditions(event_rule, data): return False -def process_event_rules(event_rules, event, data, username, snapshots=None, request_id=None): +def process_event_rules(event_rules, model_name, event, data, username, snapshots=None, request_id=None): rq_queue_name = get_config().QUEUE_MAPPINGS.get('webhook', RQ_QUEUE_DEFAULT) rq_queue = get_queue(rq_queue_name) @@ -107,7 +110,7 @@ def process_event_rules(event_rules, event, data, username, snapshots=None, requ params = { "event_rule": event_rule, - "model_name": content_type.model, + "model_name": model_name, "event": event, "data": data, "snapshots": snapshots, From 22c11c8123894c98d0871f94596eacb603227e56 Mon Sep 17 00:00:00 2001 From: Arthur Date: Thu, 16 Nov 2023 11:19:02 -0800 Subject: [PATCH 42/95] 14132 add description, comments fields --- netbox/extras/migrations/0099_eventrule.py | 39 ++++++++++++---------- netbox/extras/models/models.py | 9 +++++ 2 files changed, 31 insertions(+), 17 deletions(-) diff --git a/netbox/extras/migrations/0099_eventrule.py b/netbox/extras/migrations/0099_eventrule.py index c71f709bd68..cbf65775b09 100644 --- a/netbox/extras/migrations/0099_eventrule.py +++ b/netbox/extras/migrations/0099_eventrule.py @@ -50,6 +50,7 @@ class Migration(migrations.Migration): models.JSONField(blank=True, default=dict, encoder=utilities.json.CustomFieldJSONEncoder), ), ('name', models.CharField(max_length=150, unique=True)), + ('description', models.CharField(blank=True, max_length=200)), ('type_create', models.BooleanField(default=False)), ('type_update', models.BooleanField(default=False)), ('type_delete', models.BooleanField(default=False)), @@ -59,24 +60,9 @@ class Migration(migrations.Migration): ('conditions', models.JSONField(blank=True, null=True)), ('action_type', models.CharField(default='webhook', max_length=30)), ('action_object_id', models.PositiveBigIntegerField(blank=True, null=True)), - ( - 'content_types', - models.ManyToManyField( - related_name='eventrules', - to='contenttypes.contenttype', - ), - ), - ( - 'action_object_type', - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, - related_name='eventrule_actions', - to='contenttypes.contenttype', - ), - ), - ('action_parameters', models.CharField(max_length=80, blank=True)), + ('action_parameters', models.CharField(blank=True, max_length=80)), ('action_data', models.JSONField(blank=True, null=True)), - ('tags', taggit.managers.TaggableManager(through='extras.TaggedItem', to='extras.Tag')), + ('comments', models.TextField(blank=True)), ], options={ 'verbose_name': 'eventrule', @@ -121,4 +107,23 @@ class Migration(migrations.Migration): model_name='webhook', name='type_update', ), + migrations.AddField( + model_name='eventrule', + name='action_object_type', + field=models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name='eventrule_actions', + to='contenttypes.contenttype', + ), + ), + migrations.AddField( + model_name='eventrule', + name='content_types', + field=models.ManyToManyField(related_name='eventrules', to='contenttypes.contenttype'), + ), + migrations.AddField( + model_name='eventrule', + name='tags', + field=taggit.managers.TaggableManager(through='extras.TaggedItem', to='extras.Tag'), + ), ] diff --git a/netbox/extras/models/models.py b/netbox/extras/models/models.py index bee892be2f2..a3b80ab7b29 100644 --- a/netbox/extras/models/models.py +++ b/netbox/extras/models/models.py @@ -56,6 +56,11 @@ class EventRule(CustomFieldsMixin, ExportTemplatesMixin, TagsMixin, ChangeLogged max_length=150, unique=True ) + description = models.CharField( + verbose_name=_('description'), + max_length=200, + blank=True + ) type_create = models.BooleanField( verbose_name=_('on create'), default=False, @@ -123,6 +128,10 @@ class EventRule(CustomFieldsMixin, ExportTemplatesMixin, TagsMixin, ChangeLogged null=True, help_text=_("Parameters to pass to the action.") ) + comments = models.TextField( + verbose_name=_('comments'), + blank=True + ) class Meta: ordering = ('name',) From bcb38d206cc6db1e61ef520ae6eeda5143ec49e5 Mon Sep 17 00:00:00 2001 From: Arthur Date: Thu, 16 Nov 2023 11:56:18 -0800 Subject: [PATCH 43/95] 14132 form fixes --- netbox/extras/forms/model_forms.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/netbox/extras/forms/model_forms.py b/netbox/extras/forms/model_forms.py index 61acb161961..0832a55eaa9 100644 --- a/netbox/extras/forms/model_forms.py +++ b/netbox/extras/forms/model_forms.py @@ -249,7 +249,7 @@ class EventRuleForm(NetBoxModelForm): (_('EventRule'), ('name', 'content_types', 'enabled', 'tags')), (_('Events'), ('type_create', 'type_update', 'type_delete', 'type_job_start', 'type_job_end')), (_('Conditions'), ('conditions',)), - (_('Action'), ('action_type', 'action_choice', 'parameters', 'action_object_type', 'action_object_id', 'action_object_identifier')), + (_('Action'), ('action_type', 'action_choice', 'action_parameters', 'action_object_type', 'action_object_id', 'action_data')), ) class Meta: @@ -267,7 +267,9 @@ class Meta: 'action_type': HTMXSelect(), 'action_object_type': forms.HiddenInput, 'action_object_id': forms.HiddenInput, - 'action_object_identifier': forms.HiddenInput, + 'action_parameters': forms.HiddenInput, + 'conditions': forms.Textarea(attrs={'class': 'font-monospace'}), + 'action_data': forms.Textarea(attrs={'class': 'font-monospace'}), } def get_script_choices(self): @@ -283,7 +285,7 @@ def get_script_choices(self): choices.append((str(module), scripts)) self.fields['action_choice'].choices = choices - self.fields['action_choice'].initial = get_field_value(self, 'action_object_identifier') + self.fields['action_choice'].initial = get_field_value(self, 'action_parameters') def get_webhook_choices(self): initial = None @@ -316,12 +318,12 @@ def clean(self): if self.cleaned_data.get('action_type') == EventRuleActionChoices.WEBHOOK: self.cleaned_data['action_object_type'] = ContentType.objects.get_for_model(action_choice) self.cleaned_data['action_object_id'] = action_choice.id - self.cleaned_data['action_object_identifier'] = '' + self.cleaned_data['action_parameters'] = '' elif self.cleaned_data.get('action_type') == EventRuleActionChoices.SCRIPT: script = ScriptModule.objects.get(pk=action_choice.split(":")[0]) self.cleaned_data['action_object_type'] = ContentType.objects.get_for_model(script) self.cleaned_data['action_object_id'] = script.id - self.cleaned_data['action_object_identifier'] = action_choice + self.cleaned_data['action_parameters'] = action_choice return self.cleaned_data From d0a2529013a08919e576a685b06607426f9c3a9e Mon Sep 17 00:00:00 2001 From: Arthur Date: Thu, 16 Nov 2023 14:07:01 -0800 Subject: [PATCH 44/95] 14132 form fixes --- netbox/extras/filtersets.py | 4 +++- netbox/extras/forms/bulk_import.py | 4 ++-- netbox/extras/forms/model_forms.py | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/netbox/extras/filtersets.py b/netbox/extras/filtersets.py index 9fd88c9fad3..676d3f1a98f 100644 --- a/netbox/extras/filtersets.py +++ b/netbox/extras/filtersets.py @@ -80,7 +80,9 @@ def search(self, queryset, name, value): if not value.strip(): return queryset return queryset.filter( - Q(name__icontains=value) + Q(name__icontains=value) | + Q(description__icontains=value) | + Q(comments__icontains=value) ) diff --git a/netbox/extras/forms/bulk_import.py b/netbox/extras/forms/bulk_import.py index 4c941bba2b6..871b38e9d1d 100644 --- a/netbox/extras/forms/bulk_import.py +++ b/netbox/extras/forms/bulk_import.py @@ -156,8 +156,8 @@ class EventRuleImportForm(NetBoxModelImportForm): class Meta: model = EventRule fields = ( - 'name', 'enabled', 'content_types', 'type_create', 'type_update', 'type_delete', 'type_job_start', - 'type_job_end', 'tags' + 'name', 'description', 'enabled', 'content_types', 'type_create', 'type_update', 'type_delete', 'type_job_start', + 'type_job_end', 'comments', 'tags' ) diff --git a/netbox/extras/forms/model_forms.py b/netbox/extras/forms/model_forms.py index 0832a55eaa9..7b9f0bb0225 100644 --- a/netbox/extras/forms/model_forms.py +++ b/netbox/extras/forms/model_forms.py @@ -246,7 +246,7 @@ class EventRuleForm(NetBoxModelForm): ) fieldsets = ( - (_('EventRule'), ('name', 'content_types', 'enabled', 'tags')), + (_('EventRule'), ('name', 'description', 'content_types', 'enabled', 'tags')), (_('Events'), ('type_create', 'type_update', 'type_delete', 'type_job_start', 'type_job_end')), (_('Conditions'), ('conditions',)), (_('Action'), ('action_type', 'action_choice', 'action_parameters', 'action_object_type', 'action_object_id', 'action_data')), From e9e0791e9059626a4ad4b6d3076873da32721716 Mon Sep 17 00:00:00 2001 From: Arthur Date: Thu, 16 Nov 2023 14:08:22 -0800 Subject: [PATCH 45/95] 14132 form fixes --- netbox/extras/filtersets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netbox/extras/filtersets.py b/netbox/extras/filtersets.py index 676d3f1a98f..67c3f2a508a 100644 --- a/netbox/extras/filtersets.py +++ b/netbox/extras/filtersets.py @@ -73,7 +73,7 @@ class EventRuleFilterSet(NetBoxModelFilterSet): class Meta: model = EventRule fields = [ - 'id', 'name', 'type_create', 'type_update', 'type_delete', 'type_job_start', 'type_job_end', 'enabled', + 'id', 'name', 'type_create', 'type_update', 'type_delete', 'type_job_start', 'type_job_end', 'enabled', 'description', ] def search(self, queryset, name, value): From 33819839ee90bfd96b9da95a5a64e7bf7ca8120b Mon Sep 17 00:00:00 2001 From: Arthur Date: Thu, 16 Nov 2023 16:30:35 -0800 Subject: [PATCH 46/95] 14132 fix script event for user --- netbox/extras/scripts_worker.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/netbox/extras/scripts_worker.py b/netbox/extras/scripts_worker.py index 49ab425ecb6..727ba966c40 100644 --- a/netbox/extras/scripts_worker.py +++ b/netbox/extras/scripts_worker.py @@ -3,6 +3,8 @@ import requests from core.models import Job from django.conf import settings +from django.core.exceptions import ObjectDoesNotExist +from django.contrib.auth import get_user_model from django_rq import job from jinja2.exceptions import TemplateError from utilities.rqworker import get_workers_for_queue @@ -30,6 +32,13 @@ def process_script(event_rule, model_name, event, data, timestamp, username, req try: module = ScriptModule.objects.get(pk=module_id) except ScriptModule.DoesNotExist: + logger.warning(f"event run script - script module_id: {module_id} script_name: {script_name}") + return + + try: + user = get_user_model().objects.get(username=username) + except ObjectDoesNotExist: + logger.warning(f"event run script - user does not exist username: {username} script_name: {script_name}") return script = module.scripts[script_name]() @@ -38,7 +47,7 @@ def process_script(event_rule, model_name, event, data, timestamp, username, req run_script, instance=module, name=script.class_name, - user=None, + user=user, schedule_at=None, interval=None, data=event_rule.action_data, From ed203b1e39ffd944451a694b8a0333badd41459f Mon Sep 17 00:00:00 2001 From: Arthur Date: Fri, 17 Nov 2023 07:19:35 -0800 Subject: [PATCH 47/95] 14132 fix form field util --- netbox/utilities/forms/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netbox/utilities/forms/utils.py b/netbox/utilities/forms/utils.py index 4d737f16321..2d62c9eb402 100644 --- a/netbox/utilities/forms/utils.py +++ b/netbox/utilities/forms/utils.py @@ -130,7 +130,7 @@ def get_field_value(form, field_name): if form.is_bound: if data := form.data.get(field_name): - if field.valid_value(data): + if hasattr(field, 'valid_value') and field.valid_value(data): return data return form.get_initial_for_field(field, field_name) From 1d7aace56e3349a65e382826c3044e71f1f46ea3 Mon Sep 17 00:00:00 2001 From: Arthur Date: Fri, 17 Nov 2023 10:29:06 -0800 Subject: [PATCH 48/95] 14132 add documentation --- docs/features/event-rules.md | 31 +++++++++++++++++++++++++++++ docs/integrations/webhooks.md | 24 ++++------------------ docs/models/extras/eventrule.md | 35 +++++++++++++++++++++++++++++++++ mkdocs.yml | 2 ++ 4 files changed, 72 insertions(+), 20 deletions(-) create mode 100644 docs/features/event-rules.md create mode 100644 docs/models/extras/eventrule.md diff --git a/docs/features/event-rules.md b/docs/features/event-rules.md new file mode 100644 index 00000000000..a3ab5ce7a65 --- /dev/null +++ b/docs/features/event-rules.md @@ -0,0 +1,31 @@ +# Event Rules + +NetBox includes the ability to execute certain functions in response in response to internal object changes. These include: + +* [Scripts](../customization/custom-scripts.md) execution +* [Webhooks](../integrations/webhooks.md) execution + +For example, suppose you want to automatically configure a monitoring system to start monitoring a device when its operational status is changed to active, and remove it from monitoring for any other status. You can create a webhook in NetBox for the device model and craft its content and destination URL to effect the desired change on the receiving system. You can then associate and Event Rule with this webhook and the webhook will be sent automatically by NetBox whenever the configured constraints are met. + +Each event must be associated with at least one NetBox object type and at least one event (create, update, or delete). + +## Conditional Event Rules + +An event rule may include a set of conditional logic expressed in JSON used to control whether an event triggers for a specific object. For example, you may wish to trigger an event for devices only when the `status` field of an object is "active": + +```json +{ + "and": [ + { + "attr": "status.value", + "value": "active" + } + ] +} +``` + +For more detail, see the reference documentation for NetBox's [conditional logic](../reference/conditions.md). + +## Event Rule Processing + +When a change is detected, any resulting events are placed into a Redis queue for processing. This allows the user's request to complete without needing to wait for the outgoing event(s) to be processed. The events are then extracted from the queue by the `rqworker` process. The current event queue and any failed events can be inspected in the admin UI under System > Background Tasks. diff --git a/docs/integrations/webhooks.md b/docs/integrations/webhooks.md index 9a1094988d1..8913fd99c16 100644 --- a/docs/integrations/webhooks.md +++ b/docs/integrations/webhooks.md @@ -1,11 +1,9 @@ # Webhooks -NetBox can be configured to transmit outgoing webhooks to remote systems in response to internal object changes. The receiver can act on the data in these webhook messages to perform related tasks. +NetBox can be configured via [Event Rules](../features/event-rules.md) to transmit outgoing webhooks to remote systems in response to internal object changes. The receiver can act on the data in these webhook messages to perform related tasks. For example, suppose you want to automatically configure a monitoring system to start monitoring a device when its operational status is changed to active, and remove it from monitoring for any other status. You can create a webhook in NetBox for the device model and craft its content and destination URL to effect the desired change on the receiving system. Webhooks will be sent automatically by NetBox whenever the configured constraints are met. -Each webhook must be associated with at least one NetBox object type and at least one event (create, update, or delete). Users can specify the receiver URL, HTTP request type (`GET`, `POST`, etc.), content type, and headers. A request body can also be specified; if left blank, this will default to a serialized representation of the affected object. - !!! warning "Security Notice" Webhooks support the inclusion of user-submitted code to generate the URL, custom headers, and payloads, which may pose security risks under certain conditions. Only grant permission to create or modify webhooks to trusted users. @@ -70,26 +68,12 @@ If no body template is specified, the request body will be populated with a JSON } ``` -## Conditional Webhooks - -A webhook may include a set of conditional logic expressed in JSON used to control whether a webhook triggers for a specific object. For example, you may wish to trigger a webhook for devices only when the `status` field of an object is "active": - -```json -{ - "and": [ - { - "attr": "status.value", - "value": "active" - } - ] -} -``` - -For more detail, see the reference documentation for NetBox's [conditional logic](../reference/conditions.md). +!!! note + The setting of conditional webhooks has been moved to [Event Rules](../features/event-rules.md) since NetBox 3.7 ## Webhook Processing -When a change is detected, any resulting webhooks are placed into a Redis queue for processing. This allows the user's request to complete without needing to wait for the outgoing webhook(s) to be processed. The webhooks are then extracted from the queue by the `rqworker` process and HTTP requests are sent to their respective destinations. The current webhook queue and any failed webhooks can be inspected in the admin UI under System > Background Tasks. +Using [Event Rules](../features/event-rules.md), when a change is detected, any resulting webhooks are placed into a Redis queue for processing. This allows the user's request to complete without needing to wait for the outgoing webhook(s) to be processed. The webhooks are then extracted from the queue by the `rqworker` process and HTTP requests are sent to their respective destinations. The current webhook queue and any failed webhooks can be inspected in the admin UI under System > Background Tasks. A request is considered successful if the response has a 2XX status code; otherwise, the request is marked as having failed. Failed requests may be retried manually via the admin UI. diff --git a/docs/models/extras/eventrule.md b/docs/models/extras/eventrule.md new file mode 100644 index 00000000000..3c2cebe798d --- /dev/null +++ b/docs/models/extras/eventrule.md @@ -0,0 +1,35 @@ +# EventRule + +An event rule is a mechanism for taking an action (such as running a script or sending a webhook) when a change takes place in NetBox. For example, you may want to notify a monitoring system whenever the status of a device is updated in NetBox. This can be done by creating an event pointing to a webhook for the device model in NetBox and identifying the webhook receiver. When NetBox detects a change to a device, an HTTP request containing the details of the change and who made it be sent to the specified receiver. + +See the [event rules documentation](../features/event-rules.md) for more information. + +## Fields + +### Name + +A unique human-friendly name. + +### Content Types + +The type(s) of object in NetBox that will trigger the webhook. + +### Enabled + +If not selected, the webhook will be inactive. + +### Events + +The events which will trigger the action. At least one event type must be selected. + +| Name | Description | +|------------|--------------------------------------| +| Creations | A new object has been created | +| Updates | An existing object has been modified | +| Deletions | An object has been deleted | +| Job starts | A job for an object starts | +| Job ends | A job for an object terminates | + +### Conditions + +A set of [prescribed conditions](../../reference/conditions.md) against which the triggering object will be evaluated. If the conditions are defined but not met by the object, the webhook will not be sent. A webhook that does not define any conditions will _always_ trigger. diff --git a/mkdocs.yml b/mkdocs.yml index 3e61f922ae6..da12f13d721 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -86,6 +86,7 @@ nav: - Auth & Permissions: 'features/authentication-permissions.md' - API & Integration: 'features/api-integration.md' - Customization: 'features/customization.md' + - Event Rules: 'features/event-rules.md' - Installation & Upgrade: - Installing NetBox: 'installation/index.md' - 1. PostgreSQL: 'installation/1-postgresql.md' @@ -214,6 +215,7 @@ nav: - CustomField: 'models/extras/customfield.md' - CustomFieldChoiceSet: 'models/extras/customfieldchoiceset.md' - CustomLink: 'models/extras/customlink.md' + - EventRule: 'models/extras/eventrule.md' - ExportTemplate: 'models/extras/exporttemplate.md' - ImageAttachment: 'models/extras/imageattachment.md' - JournalEntry: 'models/extras/journalentry.md' From 488f0c54274d8e83ff1f380ab13341c6615ef2ae Mon Sep 17 00:00:00 2001 From: Arthur Date: Fri, 17 Nov 2023 11:28:08 -0800 Subject: [PATCH 49/95] 14132 fix JSON field issue --- netbox/extras/forms/model_forms.py | 12 ++++++++++-- netbox/utilities/forms/fields/fields.py | 2 ++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/netbox/extras/forms/model_forms.py b/netbox/extras/forms/model_forms.py index 7b9f0bb0225..98e1bbc856c 100644 --- a/netbox/extras/forms/model_forms.py +++ b/netbox/extras/forms/model_forms.py @@ -244,6 +244,14 @@ class EventRuleForm(NetBoxModelForm): label=_('Action choice'), choices=[] ) + conditions = JSONField( + required=False, + help_text=_('Enter conditions in JSON format.') + ) + action_data = JSONField( + required=False, + help_text=_('Enter parameters to pass to the action in JSON format.') + ) fieldsets = ( (_('EventRule'), ('name', 'description', 'content_types', 'enabled', 'tags')), @@ -268,8 +276,6 @@ class Meta: 'action_object_type': forms.HiddenInput, 'action_object_id': forms.HiddenInput, 'action_parameters': forms.HiddenInput, - 'conditions': forms.Textarea(attrs={'class': 'font-monospace'}), - 'action_data': forms.Textarea(attrs={'class': 'font-monospace'}), } def get_script_choices(self): @@ -311,6 +317,8 @@ def __init__(self, *args, **kwargs): elif action_type == EventRuleActionChoices.SCRIPT: self.get_script_choices() + val = get_field_value(self, 'conditions') + def clean(self): super().clean() diff --git a/netbox/utilities/forms/fields/fields.py b/netbox/utilities/forms/fields/fields.py index db5e4a30de0..b9da5d0458e 100644 --- a/netbox/utilities/forms/fields/fields.py +++ b/netbox/utilities/forms/fields/fields.py @@ -101,6 +101,8 @@ def __init__(self, *args, **kwargs): self.widget.attrs['class'] = 'font-monospace' def prepare_value(self, value): + if value == '': + return value if isinstance(value, InvalidJSONInput): return value if value is None: From 5b402202396e3f7f0088e75c0a1e9ab27b8659eb Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Mon, 20 Nov 2023 10:11:10 -0500 Subject: [PATCH 50/95] Misc cleanup --- docs/features/event-rules.md | 6 +++--- docs/models/extras/eventrule.md | 10 +++++----- netbox/core/models/jobs.py | 11 +++++++---- netbox/extras/api/serializers.py | 11 +++++------ netbox/extras/constants.py | 3 --- netbox/extras/context_managers.py | 4 ++-- netbox/extras/events.py | 8 ++------ netbox/extras/filtersets.py | 3 ++- netbox/extras/forms/bulk_edit.py | 2 +- netbox/extras/forms/bulk_import.py | 4 ++-- netbox/extras/forms/model_forms.py | 13 +++++++++---- netbox/extras/models/models.py | 12 ++++++------ netbox/extras/scripts.py | 14 +++++++++++--- netbox/extras/scripts_worker.py | 14 ++++---------- netbox/extras/signals.py | 2 +- netbox/extras/tables/tables.py | 12 ++++++------ netbox/extras/utils.py | 2 +- netbox/netbox/settings.py | 1 - netbox/utilities/forms/fields/fields.py | 4 +--- netbox/utilities/forms/utils.py | 7 +++---- 20 files changed, 71 insertions(+), 72 deletions(-) diff --git a/docs/features/event-rules.md b/docs/features/event-rules.md index a3ab5ce7a65..0e953522379 100644 --- a/docs/features/event-rules.md +++ b/docs/features/event-rules.md @@ -1,13 +1,13 @@ # Event Rules -NetBox includes the ability to execute certain functions in response in response to internal object changes. These include: +NetBox includes the ability to execute certain functions in response to internal object changes. These include: * [Scripts](../customization/custom-scripts.md) execution * [Webhooks](../integrations/webhooks.md) execution -For example, suppose you want to automatically configure a monitoring system to start monitoring a device when its operational status is changed to active, and remove it from monitoring for any other status. You can create a webhook in NetBox for the device model and craft its content and destination URL to effect the desired change on the receiving system. You can then associate and Event Rule with this webhook and the webhook will be sent automatically by NetBox whenever the configured constraints are met. +For example, suppose you want to automatically configure a monitoring system to start monitoring a device when its operational status is changed to active, and remove it from monitoring for any other status. You can create a webhook in NetBox for the device model and craft its content and destination URL to effect the desired change on the receiving system. You can then associate an event rule with this webhook and the webhook will be sent automatically by NetBox whenever the configured constraints are met. -Each event must be associated with at least one NetBox object type and at least one event (create, update, or delete). +Each event must be associated with at least one NetBox object type and at least one event (e.g. create, update, or delete). ## Conditional Event Rules diff --git a/docs/models/extras/eventrule.md b/docs/models/extras/eventrule.md index 3c2cebe798d..89645be3cff 100644 --- a/docs/models/extras/eventrule.md +++ b/docs/models/extras/eventrule.md @@ -1,6 +1,6 @@ # EventRule -An event rule is a mechanism for taking an action (such as running a script or sending a webhook) when a change takes place in NetBox. For example, you may want to notify a monitoring system whenever the status of a device is updated in NetBox. This can be done by creating an event pointing to a webhook for the device model in NetBox and identifying the webhook receiver. When NetBox detects a change to a device, an HTTP request containing the details of the change and who made it be sent to the specified receiver. +An event rule is a mechanism for automatically taking an action (such as running a script or sending a webhook) in response to an event in NetBox. For example, you may want to notify a monitoring system whenever the status of a device is updated in NetBox. This can be done by creating an event for device objects and designating a webhook to be transmitted. When NetBox detects a change to a device, an HTTP request containing the details of the change and who made it be sent to the specified receiver. See the [event rules documentation](../features/event-rules.md) for more information. @@ -12,15 +12,15 @@ A unique human-friendly name. ### Content Types -The type(s) of object in NetBox that will trigger the webhook. +The type(s) of object in NetBox that will trigger the rule. ### Enabled -If not selected, the webhook will be inactive. +If not selected, the event rule will not be processed. ### Events -The events which will trigger the action. At least one event type must be selected. +The events which will trigger the rule. At least one event type must be selected. | Name | Description | |------------|--------------------------------------| @@ -32,4 +32,4 @@ The events which will trigger the action. At least one event type must be select ### Conditions -A set of [prescribed conditions](../../reference/conditions.md) against which the triggering object will be evaluated. If the conditions are defined but not met by the object, the webhook will not be sent. A webhook that does not define any conditions will _always_ trigger. +A set of [prescribed conditions](../../reference/conditions.md) against which the triggering object will be evaluated. If the conditions are defined but not met by the object, no action will be taken. An event rule that does not define any conditions will _always_ trigger. diff --git a/netbox/core/models/jobs.py b/netbox/core/models/jobs.py index ad142f47316..1a35e5906d6 100644 --- a/netbox/core/models/jobs.py +++ b/netbox/core/models/jobs.py @@ -170,7 +170,7 @@ def start(self): self.save() # Handle events - self.trigger_events(event=EVENT_JOB_START) + self.process_event(event=EVENT_JOB_START) def terminate(self, status=JobStatusChoices.STATUS_COMPLETED, error=None): """ @@ -188,7 +188,7 @@ def terminate(self, status=JobStatusChoices.STATUS_COMPLETED, error=None): self.save() # Handle events - self.trigger_events(event=EVENT_JOB_END) + self.process_event(event=EVENT_JOB_END) @classmethod def enqueue(cls, func, instance, name='', user=None, schedule_at=None, interval=None, **kwargs): @@ -225,10 +225,13 @@ def enqueue(cls, func, instance, name='', user=None, schedule_at=None, interval= return job - def trigger_events(self, event): + def process_event(self, event): + """ + Process any EventRules relevant to the passed job event (i.e. start or stop). + """ from extras.models import EventRule - # Fetch any webhooks matching this object type and action + # Fetch any event rules matching this object type and action event_rules = EventRule.objects.filter( **{f'type_{event}': True}, content_types=self.object_type, diff --git a/netbox/extras/api/serializers.py b/netbox/extras/api/serializers.py index c8216356c2e..d271d18cb25 100644 --- a/netbox/extras/api/serializers.py +++ b/netbox/extras/api/serializers.py @@ -72,8 +72,8 @@ class Meta: model = EventRule fields = [ 'id', 'url', 'display', 'content_types', 'name', 'type_create', 'type_update', 'type_delete', - 'type_job_start', 'type_job_end', 'enabled', 'conditions', 'action_type', - 'custom_fields', 'tags', 'created', 'last_updated', + 'type_job_start', 'type_job_end', 'enabled', 'conditions', 'action_type', 'custom_fields', 'tags', + 'created', 'last_updated', ] @@ -87,10 +87,9 @@ class WebhookSerializer(NetBoxModelSerializer): class Meta: model = Webhook fields = [ - 'id', 'url', 'display', 'name', - 'payload_url', 'http_method', 'http_content_type', - 'additional_headers', 'body_template', 'secret', 'ssl_verification', 'ca_file_path', - 'custom_fields', 'tags', 'created', 'last_updated', + 'id', 'url', 'display', 'name', 'payload_url', 'http_method', 'http_content_type', 'additional_headers', + 'body_template', 'secret', 'ssl_verification', 'ca_file_path', 'custom_fields', 'tags', 'created', + 'last_updated', ] diff --git a/netbox/extras/constants.py b/netbox/extras/constants.py index fec8f5ef874..48b44fb4533 100644 --- a/netbox/extras/constants.py +++ b/netbox/extras/constants.py @@ -1,6 +1,3 @@ -from django.db.models import Q - - # Events EVENT_CREATE = 'create' EVENT_UPDATE = 'update' diff --git a/netbox/extras/context_managers.py b/netbox/extras/context_managers.py index 711a2067a58..8de47465e96 100644 --- a/netbox/extras/context_managers.py +++ b/netbox/extras/context_managers.py @@ -7,8 +7,8 @@ @contextmanager def event_tracking(request): """ - Enable event tracking by connecting the appropriate signals to their receivers before code is run, and - disconnecting them afterward. + Queue interesting events in memory while processing a request, then flush that queue for processing by the + events pipline before returning the response. :param request: WSGIRequest object with a unique `id` set """ diff --git a/netbox/extras/events.py b/netbox/extras/events.py index d08dfcd4dc3..6e1f0768408 100644 --- a/netbox/extras/events.py +++ b/netbox/extras/events.py @@ -3,11 +3,7 @@ from django.conf import settings from django.contrib.contenttypes.models import ContentType -from django.utils import timezone -from django_rq import get_queue -from netbox.config import get_config -from netbox.constants import RQ_QUEUE_DEFAULT from netbox.registry import registry from utilities.api import get_serializer_for_model from utilities.utils import serialize_object @@ -68,7 +64,7 @@ def enqueue_object(queue, instance, user, request_id, action): }) -def process_event_queue(queue): +def process_event_queue(events): """ Flush a list of object representation to RQ for EventRule processing. """ @@ -78,7 +74,7 @@ def process_event_queue(queue): 'type_delete': {}, } - for data in queue: + for data in events: action_flag = { ObjectChangeActionChoices.ACTION_CREATE: 'type_create', ObjectChangeActionChoices.ACTION_UPDATE: 'type_update', diff --git a/netbox/extras/filtersets.py b/netbox/extras/filtersets.py index 67c3f2a508a..99fe183a181 100644 --- a/netbox/extras/filtersets.py +++ b/netbox/extras/filtersets.py @@ -73,7 +73,8 @@ class EventRuleFilterSet(NetBoxModelFilterSet): class Meta: model = EventRule fields = [ - 'id', 'name', 'type_create', 'type_update', 'type_delete', 'type_job_start', 'type_job_end', 'enabled', 'description', + 'id', 'name', 'type_create', 'type_update', 'type_delete', 'type_job_start', 'type_job_end', 'enabled', + 'description', ] def search(self, queryset, name, value): diff --git a/netbox/extras/forms/bulk_edit.py b/netbox/extras/forms/bulk_edit.py index 2c04c929e40..62aebccb6ea 100644 --- a/netbox/extras/forms/bulk_edit.py +++ b/netbox/extras/forms/bulk_edit.py @@ -197,7 +197,7 @@ class WebhookBulkEditForm(NetBoxModelBulkEditForm): label=_('CA file path') ) - nullable_fields = ('secret', 'conditions', 'ca_file_path') + nullable_fields = ('secret', 'ca_file_path') class EventRuleBulkEditForm(NetBoxModelBulkEditForm): diff --git a/netbox/extras/forms/bulk_import.py b/netbox/extras/forms/bulk_import.py index 871b38e9d1d..1930802855d 100644 --- a/netbox/extras/forms/bulk_import.py +++ b/netbox/extras/forms/bulk_import.py @@ -156,8 +156,8 @@ class EventRuleImportForm(NetBoxModelImportForm): class Meta: model = EventRule fields = ( - 'name', 'description', 'enabled', 'content_types', 'type_create', 'type_update', 'type_delete', 'type_job_start', - 'type_job_end', 'comments', 'tags' + 'name', 'description', 'enabled', 'content_types', 'type_create', 'type_update', 'type_delete', + 'type_job_start', 'type_job_end', 'comments', 'tags' ) diff --git a/netbox/extras/forms/model_forms.py b/netbox/extras/forms/model_forms.py index 98e1bbc856c..055d302375b 100644 --- a/netbox/extras/forms/model_forms.py +++ b/netbox/extras/forms/model_forms.py @@ -22,7 +22,6 @@ from utilities.forms.widgets import ChoicesWidget, HTMXSelect from virtualization.models import Cluster, ClusterGroup, ClusterType - __all__ = ( 'BookmarkForm', 'ConfigContextForm', @@ -257,12 +256,19 @@ class EventRuleForm(NetBoxModelForm): (_('EventRule'), ('name', 'description', 'content_types', 'enabled', 'tags')), (_('Events'), ('type_create', 'type_update', 'type_delete', 'type_job_start', 'type_job_end')), (_('Conditions'), ('conditions',)), - (_('Action'), ('action_type', 'action_choice', 'action_parameters', 'action_object_type', 'action_object_id', 'action_data')), + (_('Action'), ( + 'action_type', 'action_choice', 'action_parameters', 'action_object_type', 'action_object_id', + 'action_data', + )), ) class Meta: model = EventRule - fields = '__all__' + fields = ( + 'content_types', 'name', 'description', 'type_create', 'type_update', 'type_delete', 'type_job_start', + 'type_job_end', 'enabled', 'conditions', 'action_type', 'action_object_type', 'action_object_id', + 'action_parameters', 'action_data', 'comments', 'tags' + ) labels = { 'type_create': _('Creations'), 'type_update': _('Updates'), @@ -280,7 +286,6 @@ class Meta: def get_script_choices(self): choices = [] - idx = 0 for module in ScriptModule.objects.all(): scripts = [] for script_name in module.scripts.keys(): diff --git a/netbox/extras/models/models.py b/netbox/extras/models/models.py index a3b80ab7b29..f033ba2a00c 100644 --- a/netbox/extras/models/models.py +++ b/netbox/extras/models/models.py @@ -49,7 +49,7 @@ class EventRule(CustomFieldsMixin, ExportTemplatesMixin, TagsMixin, ChangeLogged to='contenttypes.ContentType', related_name='eventrules', verbose_name=_('object types'), - help_text=_("The object(s) to which this Event applies.") + help_text=_("The object(s) to which this rule applies.") ) name = models.CharField( verbose_name=_('name'), @@ -169,11 +169,6 @@ class Webhook(CustomFieldsMixin, ExportTemplatesMixin, TagsMixin, ChangeLoggedMo delete in NetBox. The request will contain a representation of the object, which the remote application can act on. Each Webhook can be limited to firing only on certain actions or certain object types. """ - events = GenericRelation( - EventRule, - content_type_field='action_object_type', - object_id_field='action_object_id' - ) name = models.CharField( verbose_name=_('name'), max_length=150, @@ -243,6 +238,11 @@ class Webhook(CustomFieldsMixin, ExportTemplatesMixin, TagsMixin, ChangeLoggedMo "The specific CA certificate file to use for SSL verification. Leave blank to use the system defaults." ) ) + events = GenericRelation( + EventRule, + content_type_field='action_object_type', + object_id_field='action_object_id' + ) class Meta: ordering = ('name',) diff --git a/netbox/extras/scripts.py b/netbox/extras/scripts.py index 0310b9973a5..20aff665a41 100644 --- a/netbox/extras/scripts.py +++ b/netbox/extras/scripts.py @@ -476,6 +476,12 @@ def run_script(data, job, request=None, commit=True, **kwargs): """ A wrapper for calling Script.run(). This performs error handling and provides a hook for committing changes. It exists outside the Script class to ensure it cannot be overridden by a script author. + + Args: + data: A dictionary of data to be passed to the script upon execution + job: The Job associated with this execution + request: The WSGI request associated with this execution (if any) + commit: Passed through to Script.run() """ job.start() @@ -507,7 +513,8 @@ def _run_script(): raise AbortTransaction() except AbortTransaction: script.log_info("Database changes have been reverted automatically.") - clear_webhooks.send(request) + if request: + clear_webhooks.send(request) job.data = ScriptOutputSerializer(script).data job.terminate() except Exception as e: @@ -521,12 +528,13 @@ def _run_script(): script.log_info("Database changes have been reverted due to error.") job.data = ScriptOutputSerializer(script).data job.terminate(status=JobStatusChoices.STATUS_ERRORED, error=str(e)) - clear_webhooks.send(request) + if request: + clear_webhooks.send(request) logger.info(f"Script completed in {job.duration}") # Execute the script. If commit is True, wrap it with the event_tracking context manager to ensure we process - # change logging, webhooks, etc. + # change logging, event rules, etc. if commit: with event_tracking(request): _run_script() diff --git a/netbox/extras/scripts_worker.py b/netbox/extras/scripts_worker.py index 727ba966c40..f558e21ef42 100644 --- a/netbox/extras/scripts_worker.py +++ b/netbox/extras/scripts_worker.py @@ -1,21 +1,15 @@ import logging -import requests -from core.models import Job -from django.conf import settings -from django.core.exceptions import ObjectDoesNotExist from django.contrib.auth import get_user_model +from django.core.exceptions import ObjectDoesNotExist from django_rq import job -from jinja2.exceptions import TemplateError -from utilities.rqworker import get_workers_for_queue -from extras.constants import WEBHOOK_EVENT_TYPES +from core.models import Job from extras.models import ScriptModule from extras.scripts import run_script from extras.utils import eval_conditions -from extras.webhooks import generate_signature -logger = logging.getLogger('netbox.webhooks_worker') +logger = logging.getLogger('netbox.scripts_worker') @job('default') @@ -43,7 +37,7 @@ def process_script(event_rule, model_name, event, data, timestamp, username, req script = module.scripts[script_name]() - job = Job.enqueue( + Job.enqueue( run_script, instance=module, name=script.class_name, diff --git a/netbox/extras/signals.py b/netbox/extras/signals.py index a0ea07456ed..a3b151e853d 100644 --- a/netbox/extras/signals.py +++ b/netbox/extras/signals.py @@ -14,8 +14,8 @@ from netbox.signals import post_clean from utilities.exceptions import AbortRequest from .choices import ObjectChangeActionChoices -from .models import ConfigRevision, CustomField, ObjectChange, TaggedItem from .events import enqueue_object, get_snapshots, serialize_for_event +from .models import ConfigRevision, CustomField, ObjectChange, TaggedItem # # Change logging/webhooks diff --git a/netbox/extras/tables/tables.py b/netbox/extras/tables/tables.py index 90cb4eec179..257a89e5aeb 100644 --- a/netbox/extras/tables/tables.py +++ b/netbox/extras/tables/tables.py @@ -284,8 +284,8 @@ class WebhookTable(NetBoxTable): class Meta(NetBoxTable.Meta): model = Webhook fields = ( - 'pk', 'id', 'name', 'http_method', 'payload_url', 'secret', 'ssl_validation', 'ca_file_path', - 'tags', 'created', 'last_updated', + 'pk', 'id', 'name', 'http_method', 'payload_url', 'http_content_type', 'secret', 'ssl_verification', + 'ca_file_path', 'tags', 'created', 'last_updated', ) default_columns = ( 'pk', 'name', 'http_method', 'payload_url', @@ -328,12 +328,12 @@ class EventRuleTable(NetBoxTable): class Meta(NetBoxTable.Meta): model = EventRule fields = ( - 'pk', 'id', 'name', 'action_type', 'content_types', 'enabled', 'type_create', 'type_update', 'type_delete', - 'type_job_start', 'type_job_end', 'tags', 'created', 'last_updated', + 'pk', 'id', 'name', 'enabled', 'description', 'action_type', 'content_types', 'type_create', 'type_update', + 'type_delete', 'type_job_start', 'type_job_end', 'tags', 'created', 'last_updated', ) default_columns = ( - 'pk', 'name', 'action_type', 'content_types', 'enabled', 'type_create', 'type_update', 'type_delete', 'type_job_start', - 'type_job_end', + 'pk', 'name', 'enabled', 'action_type', 'content_types', 'type_create', 'type_update', 'type_delete', + 'type_job_start', 'type_job_end', ) diff --git a/netbox/extras/utils.py b/netbox/extras/utils.py index 6249e130751..b14e3334167 100644 --- a/netbox/extras/utils.py +++ b/netbox/extras/utils.py @@ -106,7 +106,7 @@ def process_event_rules(event_rules, model_name, event, data, username, snapshot elif event_rule.action_type == EventRuleActionChoices.SCRIPT: processor = "extras.scripts_worker.process_script" else: - raise ValueError(f"Unknown Event Rule action type: {event_rule.action_type}") + raise ValueError(f"Unknown action type for an event rule: {event_rule.action_type}") params = { "event_rule": event_rule, diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index d85d2af5ce9..633b0ad5d6b 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -178,7 +178,6 @@ TIME_ZONE = getattr(configuration, 'TIME_ZONE', 'UTC') ENABLE_LOCALIZATION = getattr(configuration, 'ENABLE_LOCALIZATION', False) - # Check for hard-coded dynamic config parameters for param in PARAMS: if hasattr(configuration, param.name): diff --git a/netbox/utilities/forms/fields/fields.py b/netbox/utilities/forms/fields/fields.py index b9da5d0458e..d4d4ae19b8c 100644 --- a/netbox/utilities/forms/fields/fields.py +++ b/netbox/utilities/forms/fields/fields.py @@ -101,11 +101,9 @@ def __init__(self, *args, **kwargs): self.widget.attrs['class'] = 'font-monospace' def prepare_value(self, value): - if value == '': - return value if isinstance(value, InvalidJSONInput): return value - if value is None: + if value in ('', None): return '' return json.dumps(value, sort_keys=True, indent=4) diff --git a/netbox/utilities/forms/utils.py b/netbox/utilities/forms/utils.py index 2d62c9eb402..9df29b33daa 100644 --- a/netbox/utilities/forms/utils.py +++ b/netbox/utilities/forms/utils.py @@ -128,10 +128,9 @@ def get_field_value(form, field_name): """ field = form.fields[field_name] - if form.is_bound: - if data := form.data.get(field_name): - if hasattr(field, 'valid_value') and field.valid_value(data): - return data + if form.is_bound and (data := form.data.get(field_name)): + if hasattr(field, 'valid_value') and field.valid_value(data): + return data return form.get_initial_for_field(field, field_name) From b3a702215a5cfa7d2c8e2f3aa3957fe7da63d160 Mon Sep 17 00:00:00 2001 From: Arthur Date: Mon, 20 Nov 2023 15:27:13 -0800 Subject: [PATCH 51/95] 14132 review changes --- netbox/extras/api/serializers.py | 4 +++- netbox/extras/filtersets.py | 7 ++++++- netbox/extras/forms/bulk_import.py | 4 ++-- netbox/extras/models/models.py | 14 ++++++++++++++ netbox/extras/scripts_worker.py | 5 ++--- netbox/extras/tests/test_event_rules.py | 7 +++---- netbox/extras/utils.py | 15 --------------- netbox/extras/webhooks_worker.py | 3 +-- 8 files changed, 31 insertions(+), 28 deletions(-) diff --git a/netbox/extras/api/serializers.py b/netbox/extras/api/serializers.py index d271d18cb25..f6a175b3d20 100644 --- a/netbox/extras/api/serializers.py +++ b/netbox/extras/api/serializers.py @@ -67,12 +67,14 @@ class EventRuleSerializer(NetBoxModelSerializer): queryset=ContentType.objects.with_feature('webhooks'), many=True ) + action_type = ChoiceField(choices=EventRuleActionChoices) class Meta: model = EventRule fields = [ 'id', 'url', 'display', 'content_types', 'name', 'type_create', 'type_update', 'type_delete', - 'type_job_start', 'type_job_end', 'enabled', 'conditions', 'action_type', 'custom_fields', 'tags', + 'type_job_start', 'type_job_end', 'enabled', 'conditions', 'action_type', 'action_object_type' + 'action_object_id', 'action_object', 'custom_fields', 'tags', 'created', 'last_updated', ] diff --git a/netbox/extras/filtersets.py b/netbox/extras/filtersets.py index 99fe183a181..2a9f4be7d04 100644 --- a/netbox/extras/filtersets.py +++ b/netbox/extras/filtersets.py @@ -69,12 +69,17 @@ class EventRuleFilterSet(NetBoxModelFilterSet): field_name='content_types__id' ) content_types = ContentTypeFilter() + action_type = django_filters.MultipleChoiceFilter( + choices=EventRuleActionChoices + ) + action_object_type = ContentTypeFilter() + action_object_id = MultiValueNumberFilter() class Meta: model = EventRule fields = [ 'id', 'name', 'type_create', 'type_update', 'type_delete', 'type_job_start', 'type_job_end', 'enabled', - 'description', + 'action_type', 'description', ] def search(self, queryset, name, value): diff --git a/netbox/extras/forms/bulk_import.py b/netbox/extras/forms/bulk_import.py index 1930802855d..5ec79d1288d 100644 --- a/netbox/extras/forms/bulk_import.py +++ b/netbox/extras/forms/bulk_import.py @@ -156,8 +156,8 @@ class EventRuleImportForm(NetBoxModelImportForm): class Meta: model = EventRule fields = ( - 'name', 'description', 'enabled', 'content_types', 'type_create', 'type_update', 'type_delete', - 'type_job_start', 'type_job_end', 'comments', 'tags' + 'name', 'description', 'enabled', 'conditions', 'content_types', 'type_create', 'type_update', + 'type_delete', 'type_job_start', 'type_job_end', 'comments', 'tags' ) diff --git a/netbox/extras/models/models.py b/netbox/extras/models/models.py index f033ba2a00c..01d057cb0bd 100644 --- a/netbox/extras/models/models.py +++ b/netbox/extras/models/models.py @@ -162,6 +162,20 @@ def clean(self): except ValueError as e: raise ValidationError({'conditions': e}) + def eval_conditions(self, data): + """ + Test whether the given data meets the conditions of the event rule (if any). Return True + if met or no conditions are specified. + """ + if not self.conditions: + return True + + logger.debug(f'Evaluating event rule conditions: {self.conditions}') + if ConditionSet(self.conditions).eval(data): + return True + + return False + class Webhook(CustomFieldsMixin, ExportTemplatesMixin, TagsMixin, ChangeLoggedModel): """ diff --git a/netbox/extras/scripts_worker.py b/netbox/extras/scripts_worker.py index f558e21ef42..d8e23b3c9b2 100644 --- a/netbox/extras/scripts_worker.py +++ b/netbox/extras/scripts_worker.py @@ -7,17 +7,16 @@ from core.models import Job from extras.models import ScriptModule from extras.scripts import run_script -from extras.utils import eval_conditions logger = logging.getLogger('netbox.scripts_worker') @job('default') -def process_script(event_rule, model_name, event, data, timestamp, username, request_id=None, snapshots=None): +def process_script(event_rule, data, username, **kwargs): """ Run the requested script """ - if not eval_conditions(event_rule, data): + if not event_rule.eval_conditions(data): return module_id = event_rule.action_parameters.split(":")[0] diff --git a/netbox/extras/tests/test_event_rules.py b/netbox/extras/tests/test_event_rules.py index 91a99b519d4..99049559366 100644 --- a/netbox/extras/tests/test_event_rules.py +++ b/netbox/extras/tests/test_event_rules.py @@ -14,7 +14,6 @@ from extras.choices import ObjectChangeActionChoices from extras.models import Tag, EventRule, Webhook from extras.events import enqueue_object, flush_events, serialize_for_event -from extras.utils import eval_conditions from extras.webhooks import generate_signature from extras.webhooks_worker import process_webhook from utilities.testing import APITestCase @@ -50,7 +49,7 @@ def setUpTestData(cls): def test_event_rule_conditions(self): # Create a conditional Webhook - webhook = EventRule( + event_rule = EventRule( name='Conditional Webhook', type_create=True, type_update=True, @@ -69,11 +68,11 @@ def test_event_rule_conditions(self): data = serialize_for_event(site) # Evaluate the conditions (status='staging') - self.assertFalse(eval_conditions(webhook, data)) + self.assertFalse(event_rule.eval_conditions(data)) # Change the site's status site.status = SiteStatusChoices.STATUS_ACTIVE data = serialize_for_event(site) # Evaluate the conditions (status='active') - self.assertTrue(eval_conditions(webhook, data)) + self.assertTrue(event_rule.eval_conditions(data)) diff --git a/netbox/extras/utils.py b/netbox/extras/utils.py index b14e3334167..b95023825de 100644 --- a/netbox/extras/utils.py +++ b/netbox/extras/utils.py @@ -81,21 +81,6 @@ def is_report(obj): return False -def eval_conditions(event_rule, data): - """ - Test whether the given data meets the conditions of the event rule (if any). Return True - if met or no conditions are specified. - """ - if not event_rule.conditions: - return True - - logger.debug(f'Evaluating event rule conditions: {event_rule.conditions}') - if ConditionSet(event_rule.conditions).eval(data): - return True - - return False - - def process_event_rules(event_rules, model_name, event, data, username, snapshots=None, request_id=None): rq_queue_name = get_config().QUEUE_MAPPINGS.get('webhook', RQ_QUEUE_DEFAULT) rq_queue = get_queue(rq_queue_name) diff --git a/netbox/extras/webhooks_worker.py b/netbox/extras/webhooks_worker.py index d4a9bbee182..5e3648bf910 100644 --- a/netbox/extras/webhooks_worker.py +++ b/netbox/extras/webhooks_worker.py @@ -6,7 +6,6 @@ from jinja2.exceptions import TemplateError from .constants import WEBHOOK_EVENT_TYPES -from .utils import eval_conditions from .webhooks import generate_signature logger = logging.getLogger('netbox.webhooks_worker') @@ -18,7 +17,7 @@ def process_webhook(event_rule, model_name, event, data, timestamp, username, re Make a POST request to the defined Webhook """ - if not eval_conditions(event_rule, data): + if not event_rule.eval_conditions(data): return webhook = event_rule.action_object From 56a9210d6cc3396b3a713cfdfa484612d43cb2a9 Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 21 Nov 2023 08:27:52 -0800 Subject: [PATCH 52/95] 14132 review changes --- netbox/core/models/jobs.py | 2 +- netbox/extras/events.py | 35 +++++++++++++++++++++++++++++- netbox/extras/forms/model_forms.py | 8 +++---- 3 files changed, 39 insertions(+), 6 deletions(-) diff --git a/netbox/core/models/jobs.py b/netbox/core/models/jobs.py index 1a35e5906d6..af8191df500 100644 --- a/netbox/core/models/jobs.py +++ b/netbox/core/models/jobs.py @@ -13,7 +13,6 @@ from core.choices import JobStatusChoices from core.models import ContentType from extras.constants import EVENT_JOB_END, EVENT_JOB_START -from extras.utils import process_event_rules from netbox.config import get_config from netbox.constants import RQ_QUEUE_DEFAULT from utilities.querysets import RestrictedQuerySet @@ -230,6 +229,7 @@ def process_event(self, event): Process any EventRules relevant to the passed job event (i.e. start or stop). """ from extras.models import EventRule + from extras.events import process_event_rules # Fetch any event rules matching this object type and action event_rules = EventRule.objects.filter( diff --git a/netbox/extras/events.py b/netbox/extras/events.py index 6e1f0768408..a0e03c42d97 100644 --- a/netbox/extras/events.py +++ b/netbox/extras/events.py @@ -9,7 +9,6 @@ from utilities.utils import serialize_object from .choices import * from .models import EventRule -from .utils import process_event_rules logger = logging.getLogger('netbox.events_processor') @@ -64,6 +63,40 @@ def enqueue_object(queue, instance, user, request_id, action): }) +def process_event_rules(event_rules, model_name, event, data, username, snapshots=None, request_id=None): + rq_queue_name = get_config().QUEUE_MAPPINGS.get('webhook', RQ_QUEUE_DEFAULT) + rq_queue = get_queue(rq_queue_name) + + for event_rule in event_rules: + if event_rule.action_type == EventRuleActionChoices.WEBHOOK: + processor = "extras.webhooks_worker.process_webhook" + elif event_rule.action_type == EventRuleActionChoices.SCRIPT: + processor = "extras.scripts_worker.process_script" + else: + raise ValueError(f"Unknown action type for an event rule: {event_rule.action_type}") + + params = { + "event_rule": event_rule, + "model_name": model_name, + "event": event, + "data": data, + "snapshots": snapshots, + "timestamp": str(timezone.now()), + "username": username, + "retry": get_rq_retry() + } + + if snapshots: + params["snapshots"] = snapshots + if request_id: + params["request_id"] = request_id + + rq_queue.enqueue( + processor, + **params + ) + + def process_event_queue(events): """ Flush a list of object representation to RQ for EventRule processing. diff --git a/netbox/extras/forms/model_forms.py b/netbox/extras/forms/model_forms.py index 055d302375b..da9d31443d2 100644 --- a/netbox/extras/forms/model_forms.py +++ b/netbox/extras/forms/model_forms.py @@ -284,7 +284,7 @@ class Meta: 'action_parameters': forms.HiddenInput, } - def get_script_choices(self): + def init_script_choice(self): choices = [] for module in ScriptModule.objects.all(): scripts = [] @@ -298,7 +298,7 @@ def get_script_choices(self): self.fields['action_choice'].choices = choices self.fields['action_choice'].initial = get_field_value(self, 'action_parameters') - def get_webhook_choices(self): + def init_webhook_choice(self): initial = None if self.fields['action_object_type'] and get_field_value(self, 'action_object_id'): initial = Webhook.objects.get(pk=get_field_value(self, 'action_object_id')) @@ -318,9 +318,9 @@ def __init__(self, *args, **kwargs): action_type = get_field_value(self, 'action_type') if action_type == EventRuleActionChoices.WEBHOOK: - self.get_webhook_choices() + self.init_webhook_choice() elif action_type == EventRuleActionChoices.SCRIPT: - self.get_script_choices() + self.init_script_choice() val = get_field_value(self, 'conditions') From 7eb53716009d3d86adfdd3170fcec4e5f78a0c08 Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 21 Nov 2023 08:45:38 -0800 Subject: [PATCH 53/95] 14132 review changes fix tests --- netbox/extras/events.py | 5 +++++ netbox/extras/models/models.py | 1 - netbox/extras/utils.py | 38 ---------------------------------- 3 files changed, 5 insertions(+), 39 deletions(-) diff --git a/netbox/extras/events.py b/netbox/extras/events.py index a0e03c42d97..60e0186198c 100644 --- a/netbox/extras/events.py +++ b/netbox/extras/events.py @@ -3,9 +3,14 @@ from django.conf import settings from django.contrib.contenttypes.models import ContentType +from django_rq import get_queue +from django.utils import timezone +from netbox.config import get_config +from netbox.constants import RQ_QUEUE_DEFAULT from netbox.registry import registry from utilities.api import get_serializer_for_model +from utilities.rqworker import get_rq_retry from utilities.utils import serialize_object from .choices import * from .models import EventRule diff --git a/netbox/extras/models/models.py b/netbox/extras/models/models.py index 01d057cb0bd..12836b04cc1 100644 --- a/netbox/extras/models/models.py +++ b/netbox/extras/models/models.py @@ -170,7 +170,6 @@ def eval_conditions(self, data): if not self.conditions: return True - logger.debug(f'Evaluating event rule conditions: {self.conditions}') if ConditionSet(self.conditions).eval(data): return True diff --git a/netbox/extras/utils.py b/netbox/extras/utils.py index b95023825de..5ab8fb41cea 100644 --- a/netbox/extras/utils.py +++ b/netbox/extras/utils.py @@ -1,16 +1,12 @@ import logging from django.db.models import Q -from django_rq import get_queue -from django.utils import timezone from django.utils.deconstruct import deconstructible from taggit.managers import _TaggableManager from extras.conditions import ConditionSet -from netbox.constants import RQ_QUEUE_DEFAULT from extras.choices import EventRuleActionChoices from netbox.config import get_config from netbox.registry import registry -from utilities.rqworker import get_rq_retry logger = logging.getLogger('netbox.extras.utils') @@ -79,37 +75,3 @@ def is_report(obj): return issubclass(obj, Report) and obj != Report except TypeError: return False - - -def process_event_rules(event_rules, model_name, event, data, username, snapshots=None, request_id=None): - rq_queue_name = get_config().QUEUE_MAPPINGS.get('webhook', RQ_QUEUE_DEFAULT) - rq_queue = get_queue(rq_queue_name) - - for event_rule in event_rules: - if event_rule.action_type == EventRuleActionChoices.WEBHOOK: - processor = "extras.webhooks_worker.process_webhook" - elif event_rule.action_type == EventRuleActionChoices.SCRIPT: - processor = "extras.scripts_worker.process_script" - else: - raise ValueError(f"Unknown action type for an event rule: {event_rule.action_type}") - - params = { - "event_rule": event_rule, - "model_name": model_name, - "event": event, - "data": data, - "snapshots": snapshots, - "timestamp": str(timezone.now()), - "username": username, - "retry": get_rq_retry() - } - - if snapshots: - params["snapshots"] = snapshots - if request_id: - params["request_id"] = request_id - - rq_queue.enqueue( - processor, - **params - ) From 568a5b805452f3339b3af02927c0d458d8fb31eb Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 21 Nov 2023 09:19:24 -0800 Subject: [PATCH 54/95] 14132 change action_parameters to jsonfield --- netbox/extras/forms/model_forms.py | 9 ++++++--- netbox/extras/migrations/0099_eventrule.py | 2 +- netbox/extras/models/models.py | 6 +++--- netbox/extras/scripts_worker.py | 12 ++++++++++-- 4 files changed, 20 insertions(+), 9 deletions(-) diff --git a/netbox/extras/forms/model_forms.py b/netbox/extras/forms/model_forms.py index da9d31443d2..c2514629ae2 100644 --- a/netbox/extras/forms/model_forms.py +++ b/netbox/extras/forms/model_forms.py @@ -296,7 +296,11 @@ def init_script_choice(self): choices.append((str(module), scripts)) self.fields['action_choice'].choices = choices - self.fields['action_choice'].initial = get_field_value(self, 'action_parameters') + parameters = get_field_value(self, 'action_parameters') + initial = None + if parameters and 'script_choice' in parameters: + initial = parameters['script_choice'] + self.fields['action_choice'].initial = initial def init_webhook_choice(self): initial = None @@ -331,12 +335,11 @@ def clean(self): if self.cleaned_data.get('action_type') == EventRuleActionChoices.WEBHOOK: self.cleaned_data['action_object_type'] = ContentType.objects.get_for_model(action_choice) self.cleaned_data['action_object_id'] = action_choice.id - self.cleaned_data['action_parameters'] = '' elif self.cleaned_data.get('action_type') == EventRuleActionChoices.SCRIPT: script = ScriptModule.objects.get(pk=action_choice.split(":")[0]) self.cleaned_data['action_object_type'] = ContentType.objects.get_for_model(script) self.cleaned_data['action_object_id'] = script.id - self.cleaned_data['action_parameters'] = action_choice + self.cleaned_data['action_parameters'] = {'script_choice': action_choice} return self.cleaned_data diff --git a/netbox/extras/migrations/0099_eventrule.py b/netbox/extras/migrations/0099_eventrule.py index cbf65775b09..f31affe74c3 100644 --- a/netbox/extras/migrations/0099_eventrule.py +++ b/netbox/extras/migrations/0099_eventrule.py @@ -60,7 +60,7 @@ class Migration(migrations.Migration): ('conditions', models.JSONField(blank=True, null=True)), ('action_type', models.CharField(default='webhook', max_length=30)), ('action_object_id', models.PositiveBigIntegerField(blank=True, null=True)), - ('action_parameters', models.CharField(blank=True, max_length=80)), + ('action_parameters', models.JSONField(blank=True, null=True)), ('action_data', models.JSONField(blank=True, null=True)), ('comments', models.TextField(blank=True)), ], diff --git a/netbox/extras/models/models.py b/netbox/extras/models/models.py index 12836b04cc1..a6a459bcbc1 100644 --- a/netbox/extras/models/models.py +++ b/netbox/extras/models/models.py @@ -118,9 +118,9 @@ class EventRule(CustomFieldsMixin, ExportTemplatesMixin, TagsMixin, ChangeLogged fk_field='action_object_id' ) # internal (not show in UI) - used by scripts to store function name - action_parameters = models.CharField( - max_length=80, - blank=True + action_parameters = models.JSONField( + blank=True, + null=True, ) action_data = models.JSONField( verbose_name=_('parameters'), diff --git a/netbox/extras/scripts_worker.py b/netbox/extras/scripts_worker.py index d8e23b3c9b2..5e8a24d44c3 100644 --- a/netbox/extras/scripts_worker.py +++ b/netbox/extras/scripts_worker.py @@ -19,8 +19,16 @@ def process_script(event_rule, data, username, **kwargs): if not event_rule.eval_conditions(data): return - module_id = event_rule.action_parameters.split(":")[0] - script_name = event_rule.action_parameters.split(":")[1] + script_choice = None + if event_rule.action_parameters and 'script_choice' in event_rule_action_parameters: + script_choice = event_rule.action_parameters['script_choice'] + + if script_choice: + module_id = script_choice.split(":")[0] + script_name = script_choice.split(":")[1] + else: + logger.warning(f"event run script - event_rule: {event_rule.id} no script_choice selected") + return try: module = ScriptModule.objects.get(pk=module_id) From 5266109a3c3dc5865e41e5821931541130dfc53c Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 21 Nov 2023 09:20:46 -0800 Subject: [PATCH 55/95] 14132 fix merge --- netbox/extras/migrations/0099_eventrule.py | 129 --------------------- 1 file changed, 129 deletions(-) delete mode 100644 netbox/extras/migrations/0099_eventrule.py diff --git a/netbox/extras/migrations/0099_eventrule.py b/netbox/extras/migrations/0099_eventrule.py deleted file mode 100644 index f31affe74c3..00000000000 --- a/netbox/extras/migrations/0099_eventrule.py +++ /dev/null @@ -1,129 +0,0 @@ -# Generated by Django 4.2.5 on 2023-10-31 14:37 - -from django.contrib.contenttypes.models import ContentType -from django.db import migrations, models -import django.db.models.deletion -from extras.choices import * -import extras.utils -import taggit.managers -import utilities.json - - -def move_webhooks(apps, schema_editor): - Webhook = apps.get_model("extras", "Webhook") - EventRule = apps.get_model("extras", "EventRule") - - for webhook in Webhook.objects.all(): - event = EventRule() - - event.name = webhook.name - event.type_create = webhook.type_create - event.type_update = webhook.type_update - event.type_delete = webhook.type_delete - event.type_job_start = webhook.type_job_start - event.type_job_end = webhook.type_job_end - event.enabled = webhook.enabled - event.conditions = webhook.conditions - - event.action_type = EventRuleActionChoices.WEBHOOK - event.action_object_type_id = ContentType.objects.get_for_model(webhook).id - event.action_object_id = webhook.id - event.save() - event.content_types.add(*webhook.content_types.all()) - - -class Migration(migrations.Migration): - dependencies = [ - ('contenttypes', '0002_remove_content_type_name'), - ('extras', '0099_cachedvalue_ordering'), - ] - - operations = [ - migrations.CreateModel( - name='EventRule', - fields=[ - ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False)), - ('created', models.DateTimeField(auto_now_add=True, null=True)), - ('last_updated', models.DateTimeField(auto_now=True, null=True)), - ( - 'custom_field_data', - models.JSONField(blank=True, default=dict, encoder=utilities.json.CustomFieldJSONEncoder), - ), - ('name', models.CharField(max_length=150, unique=True)), - ('description', models.CharField(blank=True, max_length=200)), - ('type_create', models.BooleanField(default=False)), - ('type_update', models.BooleanField(default=False)), - ('type_delete', models.BooleanField(default=False)), - ('type_job_start', models.BooleanField(default=False)), - ('type_job_end', models.BooleanField(default=False)), - ('enabled', models.BooleanField(default=True)), - ('conditions', models.JSONField(blank=True, null=True)), - ('action_type', models.CharField(default='webhook', max_length=30)), - ('action_object_id', models.PositiveBigIntegerField(blank=True, null=True)), - ('action_parameters', models.JSONField(blank=True, null=True)), - ('action_data', models.JSONField(blank=True, null=True)), - ('comments', models.TextField(blank=True)), - ], - options={ - 'verbose_name': 'eventrule', - 'verbose_name_plural': 'eventrules', - 'ordering': ('name',), - }, - ), - migrations.RunPython(move_webhooks), - migrations.RemoveConstraint( - model_name='webhook', - name='extras_webhook_unique_payload_url_types', - ), - migrations.RemoveField( - model_name='webhook', - name='conditions', - ), - migrations.RemoveField( - model_name='webhook', - name='content_types', - ), - migrations.RemoveField( - model_name='webhook', - name='enabled', - ), - migrations.RemoveField( - model_name='webhook', - name='type_create', - ), - migrations.RemoveField( - model_name='webhook', - name='type_delete', - ), - migrations.RemoveField( - model_name='webhook', - name='type_job_end', - ), - migrations.RemoveField( - model_name='webhook', - name='type_job_start', - ), - migrations.RemoveField( - model_name='webhook', - name='type_update', - ), - migrations.AddField( - model_name='eventrule', - name='action_object_type', - field=models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, - related_name='eventrule_actions', - to='contenttypes.contenttype', - ), - ), - migrations.AddField( - model_name='eventrule', - name='content_types', - field=models.ManyToManyField(related_name='eventrules', to='contenttypes.contenttype'), - ), - migrations.AddField( - model_name='eventrule', - name='tags', - field=taggit.managers.TaggableManager(through='extras.TaggedItem', to='extras.Tag'), - ), - ] From d6cc025f49f32591d75df8dada3d0990f87ec7fe Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 21 Nov 2023 09:22:29 -0800 Subject: [PATCH 56/95] 14132 review change --- netbox/extras/scripts_worker.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/netbox/extras/scripts_worker.py b/netbox/extras/scripts_worker.py index 5e8a24d44c3..f2674fe2044 100644 --- a/netbox/extras/scripts_worker.py +++ b/netbox/extras/scripts_worker.py @@ -24,8 +24,7 @@ def process_script(event_rule, data, username, **kwargs): script_choice = event_rule.action_parameters['script_choice'] if script_choice: - module_id = script_choice.split(":")[0] - script_name = script_choice.split(":")[1] + module_id, script_name = script_choice.split(":", maxsplit=1) else: logger.warning(f"event run script - event_rule: {event_rule.id} no script_choice selected") return From 245465a42528c5ff52de6387c7dcc5ca48a2b86e Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 21 Nov 2023 09:30:31 -0800 Subject: [PATCH 57/95] 14132 re-add migration --- netbox/extras/migrations/0101_eventrule.py | 129 +++++++++++++++++++++ 1 file changed, 129 insertions(+) create mode 100644 netbox/extras/migrations/0101_eventrule.py diff --git a/netbox/extras/migrations/0101_eventrule.py b/netbox/extras/migrations/0101_eventrule.py new file mode 100644 index 00000000000..544ac8570aa --- /dev/null +++ b/netbox/extras/migrations/0101_eventrule.py @@ -0,0 +1,129 @@ +# Generated by Django 4.2.5 on 2023-10-31 14:37 + +from django.contrib.contenttypes.models import ContentType +from django.db import migrations, models +import django.db.models.deletion +from extras.choices import * +import extras.utils +import taggit.managers +import utilities.json + + +def move_webhooks(apps, schema_editor): + Webhook = apps.get_model("extras", "Webhook") + EventRule = apps.get_model("extras", "EventRule") + + for webhook in Webhook.objects.all(): + event = EventRule() + + event.name = webhook.name + event.type_create = webhook.type_create + event.type_update = webhook.type_update + event.type_delete = webhook.type_delete + event.type_job_start = webhook.type_job_start + event.type_job_end = webhook.type_job_end + event.enabled = webhook.enabled + event.conditions = webhook.conditions + + event.action_type = EventRuleActionChoices.WEBHOOK + event.action_object_type_id = ContentType.objects.get_for_model(webhook).id + event.action_object_id = webhook.id + event.save() + event.content_types.add(*webhook.content_types.all()) + + +class Migration(migrations.Migration): + dependencies = [ + ('contenttypes', '0002_remove_content_type_name'), + ('extras', '0100_customfield_ui_attrs'), + ] + + operations = [ + migrations.CreateModel( + name='EventRule', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False)), + ('created', models.DateTimeField(auto_now_add=True, null=True)), + ('last_updated', models.DateTimeField(auto_now=True, null=True)), + ( + 'custom_field_data', + models.JSONField(blank=True, default=dict, encoder=utilities.json.CustomFieldJSONEncoder), + ), + ('name', models.CharField(max_length=150, unique=True)), + ('description', models.CharField(blank=True, max_length=200)), + ('type_create', models.BooleanField(default=False)), + ('type_update', models.BooleanField(default=False)), + ('type_delete', models.BooleanField(default=False)), + ('type_job_start', models.BooleanField(default=False)), + ('type_job_end', models.BooleanField(default=False)), + ('enabled', models.BooleanField(default=True)), + ('conditions', models.JSONField(blank=True, null=True)), + ('action_type', models.CharField(default='webhook', max_length=30)), + ('action_object_id', models.PositiveBigIntegerField(blank=True, null=True)), + ('action_parameters', models.JSONField(blank=True, null=True)), + ('action_data', models.JSONField(blank=True, null=True)), + ('comments', models.TextField(blank=True)), + ], + options={ + 'verbose_name': 'eventrule', + 'verbose_name_plural': 'eventrules', + 'ordering': ('name',), + }, + ), + migrations.RunPython(move_webhooks), + migrations.RemoveConstraint( + model_name='webhook', + name='extras_webhook_unique_payload_url_types', + ), + migrations.RemoveField( + model_name='webhook', + name='conditions', + ), + migrations.RemoveField( + model_name='webhook', + name='content_types', + ), + migrations.RemoveField( + model_name='webhook', + name='enabled', + ), + migrations.RemoveField( + model_name='webhook', + name='type_create', + ), + migrations.RemoveField( + model_name='webhook', + name='type_delete', + ), + migrations.RemoveField( + model_name='webhook', + name='type_job_end', + ), + migrations.RemoveField( + model_name='webhook', + name='type_job_start', + ), + migrations.RemoveField( + model_name='webhook', + name='type_update', + ), + migrations.AddField( + model_name='eventrule', + name='action_object_type', + field=models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name='eventrule_actions', + to='contenttypes.contenttype', + ), + ), + migrations.AddField( + model_name='eventrule', + name='content_types', + field=models.ManyToManyField(related_name='eventrules', to='contenttypes.contenttype'), + ), + migrations.AddField( + model_name='eventrule', + name='tags', + field=taggit.managers.TaggableManager(through='extras.TaggedItem', to='extras.Tag'), + ), + ] From 9c32a68a0f98c8d0758ecdfe4a52786c773b5d1c Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 21 Nov 2023 10:02:20 -0800 Subject: [PATCH 58/95] 14132 fix serializer --- netbox/extras/api/serializers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netbox/extras/api/serializers.py b/netbox/extras/api/serializers.py index d7faaa3d9f3..e92d475b327 100644 --- a/netbox/extras/api/serializers.py +++ b/netbox/extras/api/serializers.py @@ -73,7 +73,7 @@ class Meta: model = EventRule fields = [ 'id', 'url', 'display', 'content_types', 'name', 'type_create', 'type_update', 'type_delete', - 'type_job_start', 'type_job_end', 'enabled', 'conditions', 'action_type', 'action_object_type' + 'type_job_start', 'type_job_end', 'enabled', 'conditions', 'action_type', 'action_object_type', 'action_object_id', 'action_object', 'custom_fields', 'tags', 'created', 'last_updated', ] From 92ab4a0a259813d5a9137c312db4b0ec9a720dd1 Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 21 Nov 2023 11:02:15 -0800 Subject: [PATCH 59/95] 14132 fix serializer --- netbox/extras/api/serializers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netbox/extras/api/serializers.py b/netbox/extras/api/serializers.py index e92d475b327..3ac7a42e70d 100644 --- a/netbox/extras/api/serializers.py +++ b/netbox/extras/api/serializers.py @@ -74,7 +74,7 @@ class Meta: fields = [ 'id', 'url', 'display', 'content_types', 'name', 'type_create', 'type_update', 'type_delete', 'type_job_start', 'type_job_end', 'enabled', 'conditions', 'action_type', 'action_object_type', - 'action_object_id', 'action_object', 'custom_fields', 'tags', + 'action_object_id', 'custom_fields', 'tags', 'created', 'last_updated', ] From bbd008e15273e90bf1a6d016f35e840f87056559 Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 21 Nov 2023 11:22:24 -0800 Subject: [PATCH 60/95] 14132 add payload_url to webhook filterset --- netbox/extras/filtersets.py | 3 +++ netbox/extras/forms/filtersets.py | 6 +++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/netbox/extras/filtersets.py b/netbox/extras/filtersets.py index 93c976f3b17..4d1dbbece8c 100644 --- a/netbox/extras/filtersets.py +++ b/netbox/extras/filtersets.py @@ -43,6 +43,9 @@ class WebhookFilterSet(NetBoxModelFilterSet): http_method = django_filters.MultipleChoiceFilter( choices=WebhookHttpMethodChoices ) + payload_url = MultiValueCharFilter( + lookup_expr='icontains' + ) class Meta: model = Webhook diff --git a/netbox/extras/forms/filtersets.py b/netbox/extras/forms/filtersets.py index 0f4febee8a7..e04fe87bcee 100644 --- a/netbox/extras/forms/filtersets.py +++ b/netbox/extras/forms/filtersets.py @@ -226,10 +226,14 @@ class SavedFilterFilterForm(SavedFiltersMixin, FilterForm): class WebhookFilterForm(NetBoxModelFilterSetForm): model = Webhook tag = TagFilterField(model) + payload_url = forms.CharField( + label=_('Payload URL'), + required=False + ) fieldsets = ( (None, ('q', 'filter_id', 'tag')), - (_('Attributes'), ('http_method',)), + (_('Attributes'), ('payload_url', 'http_method',)), ) http_method = forms.MultipleChoiceField( choices=WebhookHttpMethodChoices, From d14e868e464a4fda27f00e99aa8c222a09b307ff Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Mon, 27 Nov 2023 09:32:32 -0500 Subject: [PATCH 61/95] Add action_object field to EventRuleSerializer --- netbox/extras/api/serializers.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/netbox/extras/api/serializers.py b/netbox/extras/api/serializers.py index 3ac7a42e70d..7c115ecb2a2 100644 --- a/netbox/extras/api/serializers.py +++ b/netbox/extras/api/serializers.py @@ -68,16 +68,21 @@ class EventRuleSerializer(NetBoxModelSerializer): many=True ) action_type = ChoiceField(choices=EventRuleActionChoices) + action_object = serializers.SerializerMethodField(read_only=True) class Meta: model = EventRule fields = [ 'id', 'url', 'display', 'content_types', 'name', 'type_create', 'type_update', 'type_delete', 'type_job_start', 'type_job_end', 'enabled', 'conditions', 'action_type', 'action_object_type', - 'action_object_id', 'custom_fields', 'tags', - 'created', 'last_updated', + 'action_object_id', 'action_object', 'custom_fields', 'tags', 'created', 'last_updated', ] + @extend_schema_field(serializers.JSONField(allow_null=True)) + def get_action_object(self, instance): + serializer = get_serializer_for_model(instance.action_object, prefix=NESTED_SERIALIZER_PREFIX) + return serializer(instance.action_object, context={'request': self.context['request']}).data + # # Webhooks From e34eff2e87b533cc2ea1c177143c7de65fdabe03 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Mon, 27 Nov 2023 09:40:17 -0500 Subject: [PATCH 62/95] Select RQ queue based on action object type --- netbox/extras/events.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/netbox/extras/events.py b/netbox/extras/events.py index 60e0186198c..20450e0d78e 100644 --- a/netbox/extras/events.py +++ b/netbox/extras/events.py @@ -69,17 +69,22 @@ def enqueue_object(queue, instance, user, request_id, action): def process_event_rules(event_rules, model_name, event, data, username, snapshots=None, request_id=None): - rq_queue_name = get_config().QUEUE_MAPPINGS.get('webhook', RQ_QUEUE_DEFAULT) - rq_queue = get_queue(rq_queue_name) for event_rule in event_rules: if event_rule.action_type == EventRuleActionChoices.WEBHOOK: processor = "extras.webhooks_worker.process_webhook" + queue_class = 'webhook' elif event_rule.action_type == EventRuleActionChoices.SCRIPT: processor = "extras.scripts_worker.process_script" + queue_class = 'script' else: raise ValueError(f"Unknown action type for an event rule: {event_rule.action_type}") + # Select the appropriate RQ queue based on the action object type + queue_name = get_config().QUEUE_MAPPINGS.get(queue_class, RQ_QUEUE_DEFAULT) + rq_queue = get_queue(queue_name) + + # Compile the task parameters params = { "event_rule": event_rule, "model_name": model_name, @@ -90,12 +95,12 @@ def process_event_rules(event_rules, model_name, event, data, username, snapshot "username": username, "retry": get_rq_retry() } - if snapshots: params["snapshots"] = snapshots if request_id: params["request_id"] = request_id + # Enqueue the task rq_queue.enqueue( processor, **params From 0066eed4453e30468a42477b1bc67c737e0d2f8a Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Mon, 27 Nov 2023 09:43:10 -0500 Subject: [PATCH 63/95] Use Django's import_string() to reference pipeline functions --- netbox/extras/events.py | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/netbox/extras/events.py b/netbox/extras/events.py index 20450e0d78e..7d4e3839618 100644 --- a/netbox/extras/events.py +++ b/netbox/extras/events.py @@ -5,6 +5,7 @@ from django.contrib.contenttypes.models import ContentType from django_rq import get_queue from django.utils import timezone +from django.utils.module_loading import import_string from netbox.config import get_config from netbox.constants import RQ_QUEUE_DEFAULT @@ -140,17 +141,6 @@ def process_event_queue(events): ) -def import_module(name): - __import__(name) - return sys.modules[name] - - -def module_member(name): - mod, member = name.rsplit(".", 1) - module = import_module(mod) - return getattr(module, member) - - def flush_events(queue): """ Flush a list of object representation to RQ for webhook processing. @@ -158,7 +148,7 @@ def flush_events(queue): if queue: for name in settings.EVENTS_PIPELINE: try: - func = module_member(name) + func = import_string(name) func(queue) except Exception as e: logger.error(f"Cannot import events pipeline {name} error: {e}") From bf11f1db0d6555823cdcb5e83b8ded8cfc14adc4 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Mon, 27 Nov 2023 09:52:58 -0500 Subject: [PATCH 64/95] Extend filterset forms --- netbox/extras/forms/filtersets.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/netbox/extras/forms/filtersets.py b/netbox/extras/forms/filtersets.py index e04fe87bcee..90d39484d3c 100644 --- a/netbox/extras/forms/filtersets.py +++ b/netbox/extras/forms/filtersets.py @@ -225,21 +225,24 @@ class SavedFilterFilterForm(SavedFiltersMixin, FilterForm): class WebhookFilterForm(NetBoxModelFilterSetForm): model = Webhook - tag = TagFilterField(model) + fieldsets = ( + (None, ('q', 'filter_id', 'tag')), + (_('Attributes'), ('payload_url', 'http_method', 'http_content_type')), + ) + http_content_type = forms.CharField( + label=_('HTTP content type'), + required=False + ) payload_url = forms.CharField( label=_('Payload URL'), required=False ) - - fieldsets = ( - (None, ('q', 'filter_id', 'tag')), - (_('Attributes'), ('payload_url', 'http_method',)), - ) http_method = forms.MultipleChoiceField( choices=WebhookHttpMethodChoices, required=False, label=_('HTTP method') ) + tag = TagFilterField(model) class EventRuleFilterForm(NetBoxModelFilterSetForm): @@ -248,7 +251,7 @@ class EventRuleFilterForm(NetBoxModelFilterSetForm): fieldsets = ( (None, ('q', 'filter_id', 'tag')), - (_('Attributes'), ('content_type_id', 'enabled')), + (_('Attributes'), ('content_type_id', 'action_type', 'enabled')), (_('Events'), ('type_create', 'type_update', 'type_delete', 'type_job_start', 'type_job_end')), ) content_type_id = ContentTypeMultipleChoiceField( @@ -256,6 +259,11 @@ class EventRuleFilterForm(NetBoxModelFilterSetForm): required=False, label=_('Object type') ) + action_type = forms.ChoiceField( + choices=add_blank_choice(EventRuleActionChoices), + required=False, + label=_('Action type') + ) enabled = forms.NullBooleanField( label=_('Enabled'), required=False, From f86034591421ef0392a15656fcf4c3df99ea534d Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Mon, 27 Nov 2023 10:37:47 -0500 Subject: [PATCH 65/95] Misc cleanup --- netbox/extras/migrations/0101_eventrule.py | 10 +- netbox/extras/models/models.py | 5 +- netbox/extras/scripts_worker.py | 2 +- netbox/extras/utils.py | 8 -- netbox/templates/extras/eventrule.html | 158 +++++++++++---------- 5 files changed, 87 insertions(+), 96 deletions(-) diff --git a/netbox/extras/migrations/0101_eventrule.py b/netbox/extras/migrations/0101_eventrule.py index 544ac8570aa..64e03dda01c 100644 --- a/netbox/extras/migrations/0101_eventrule.py +++ b/netbox/extras/migrations/0101_eventrule.py @@ -1,12 +1,10 @@ -# Generated by Django 4.2.5 on 2023-10-31 14:37 - -from django.contrib.contenttypes.models import ContentType -from django.db import migrations, models import django.db.models.deletion -from extras.choices import * -import extras.utils import taggit.managers +from django.contrib.contenttypes.models import ContentType +from django.db import migrations, models + import utilities.json +from extras.choices import * def move_webhooks(apps, schema_editor): diff --git a/netbox/extras/models/models.py b/netbox/extras/models/models.py index a6a459bcbc1..a4251c62a84 100644 --- a/netbox/extras/models/models.py +++ b/netbox/extras/models/models.py @@ -170,10 +170,7 @@ def eval_conditions(self, data): if not self.conditions: return True - if ConditionSet(self.conditions).eval(data): - return True - - return False + return ConditionSet(self.conditions).eval(data) class Webhook(CustomFieldsMixin, ExportTemplatesMixin, TagsMixin, ChangeLoggedModel): diff --git a/netbox/extras/scripts_worker.py b/netbox/extras/scripts_worker.py index f2674fe2044..1e70a0f8abf 100644 --- a/netbox/extras/scripts_worker.py +++ b/netbox/extras/scripts_worker.py @@ -20,7 +20,7 @@ def process_script(event_rule, data, username, **kwargs): return script_choice = None - if event_rule.action_parameters and 'script_choice' in event_rule_action_parameters: + if event_rule.action_parameters and 'script_choice' in event_rule.action_parameters: script_choice = event_rule.action_parameters['script_choice'] if script_choice: diff --git a/netbox/extras/utils.py b/netbox/extras/utils.py index 5ab8fb41cea..c6b2de18838 100644 --- a/netbox/extras/utils.py +++ b/netbox/extras/utils.py @@ -1,15 +1,7 @@ -import logging -from django.db.models import Q -from django.utils.deconstruct import deconstructible from taggit.managers import _TaggableManager -from extras.conditions import ConditionSet -from extras.choices import EventRuleActionChoices -from netbox.config import get_config from netbox.registry import registry -logger = logging.getLogger('netbox.extras.utils') - def is_taggable(obj): """ diff --git a/netbox/templates/extras/eventrule.html b/netbox/templates/extras/eventrule.html index 375c30dbc52..86c330121cd 100644 --- a/netbox/templates/extras/eventrule.html +++ b/netbox/templates/extras/eventrule.html @@ -4,91 +4,95 @@ {% load i18n %} {% block content %} -
-
-
-
- {% trans "Webhook" %} -
-
- - - - - - - - - -
{% trans "Name" %}{{ object.name }}
{% trans "Enabled" %}{% checkmark object.enabled %}
-
-
-
-
- {% trans "Events" %} -
-
- - - - - - - - - - - - - - - - - - - - - -
{% trans "Create" %}{% checkmark object.type_create %}
{% trans "Update" %}{% checkmark object.type_update %}
{% trans "Delete" %}{% checkmark object.type_delete %}
{% trans "Job start" %}{% checkmark object.type_job_start %}
{% trans "Job end" %}{% checkmark object.type_job_end %}
+
+
+
+
+ {% trans "Event Rule" %} +
+
+ + + + + + + + + + + + + +
{% trans "Name" %}{{ object.name }}
{% trans "Enabled" %}{% checkmark object.enabled %}
{% trans "Description" %}{{ object.description|placeholder }}
+
-
- {% plugin_left_page object %} -
-
-
-
- {% trans "Assigned Models" %} -
-
- - {% for ct in object.content_types.all %} +
+
+ {% trans "Events" %} +
+
+
- + + - {% endfor %} -
{{ ct }}{% trans "Create" %}{% checkmark object.type_create %}
+ + {% trans "Update" %} + {% checkmark object.type_update %} + + + {% trans "Delete" %} + {% checkmark object.type_delete %} + + + {% trans "Job start" %} + {% checkmark object.type_job_start %} + + + {% trans "Job end" %} + {% checkmark object.type_job_end %} + + +
+ {% plugin_left_page object %}
-
-
- {% trans "Conditions" %} -
-
- {% if object.conditions %} - 
{{ object.conditions|json }}
- {% else %} -

{% trans "None" %}

- {% endif %} +
+
+
+ {% trans "Object Types" %} +
+
+ + {% for ct in object.content_types.all %} + + + + {% endfor %} +
{{ ct }}
+
+
+
+ {% trans "Conditions" %} +
+
+ {% if object.conditions %} + 
{{ object.conditions|json }}
+ {% else %} +

{% trans "None" %}

+ {% endif %} +
+
+ {% include 'inc/panels/custom_fields.html' %} + {% include 'inc/panels/tags.html' %} + {% plugin_right_page object %}
- {% include 'inc/panels/custom_fields.html' %} - {% include 'inc/panels/tags.html' %} - {% plugin_right_page object %}
-
-
+
- {% plugin_full_width_page object %} + {% plugin_full_width_page object %}
-
+
{% endblock %} From 267b5a1b3a25bbc7376660d2140e1c175ad65649 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Tue, 28 Nov 2023 08:16:28 -0500 Subject: [PATCH 66/95] Rename WebhooksMixin to EventRulesMixin --- docs/development/models.md | 2 +- docs/plugins/development/models.md | 7 +++++-- netbox/extras/models/reports.py | 4 ++-- netbox/extras/models/scripts.py | 4 ++-- netbox/netbox/models/__init__.py | 4 ++-- netbox/netbox/models/features.py | 8 ++++---- 6 files changed, 16 insertions(+), 13 deletions(-) diff --git a/docs/development/models.md b/docs/development/models.md index d4838570a47..38c6fd3b673 100644 --- a/docs/development/models.md +++ b/docs/development/models.md @@ -22,7 +22,7 @@ Depending on its classification, each NetBox model may support various features | [Journaling](../features/journaling.md) | `JournalingMixin` | `journaling` | These models support persistent historical commentary | | [Synchronized data](../integrations/synchronized-data.md) | `SyncedDataMixin` | `synced_data` | Certain model data can be automatically synchronized from a remote data source | | [Tagging](../models/extras/tag.md) | `TagsMixin` | `tags` | The models can be tagged with user-defined tags | -| [Webhooks](../integrations/webhooks.md) | `WebhooksMixin` | `webhooks` | NetBox is capable of generating outgoing webhooks for these objects | +| [Webhooks](../integrations/webhooks.md) | `EventRulesMixin` | `webhooks` | NetBox is capable of generating outgoing webhooks for these objects | ## Models Index diff --git a/docs/plugins/development/models.md b/docs/plugins/development/models.md index 8394813f8c7..a2f8f6fc723 100644 --- a/docs/plugins/development/models.md +++ b/docs/plugins/development/models.md @@ -119,14 +119,17 @@ For more information about database migrations, see the [Django documentation](h ::: netbox.models.features.CustomValidationMixin +::: netbox.models.features.EventRulesMixin + +!!! note + `EventRulesMixin` was renamed from `WebhooksMixin` in NetBox v3.7. + ::: netbox.models.features.ExportTemplatesMixin ::: netbox.models.features.JournalingMixin ::: netbox.models.features.TagsMixin -::: netbox.models.features.WebhooksMixin - ## Choice Sets For model fields which support the selection of one or more values from a predefined list of choices, NetBox provides the `ChoiceSet` utility class. This can be used in place of a regular choices tuple to provide enhanced functionality, namely dynamic configuration and colorization. (See [Django's documentation](https://docs.djangoproject.com/en/stable/ref/models/fields/#choices) on the `choices` parameter for supported model fields.) diff --git a/netbox/extras/models/reports.py b/netbox/extras/models/reports.py index 223d679bd5f..f6228ef24a7 100644 --- a/netbox/extras/models/reports.py +++ b/netbox/extras/models/reports.py @@ -9,7 +9,7 @@ from core.choices import ManagedFileRootPathChoices from core.models import ManagedFile from extras.utils import is_report -from netbox.models.features import JobsMixin, WebhooksMixin +from netbox.models.features import JobsMixin, EventRulesMixin from utilities.querysets import RestrictedQuerySet from .mixins import PythonModuleMixin @@ -21,7 +21,7 @@ ) -class Report(WebhooksMixin, models.Model): +class Report(EventRulesMixin, models.Model): """ Dummy model used to generate permissions for reports. Does not exist in the database. """ diff --git a/netbox/extras/models/scripts.py b/netbox/extras/models/scripts.py index 122f56f2065..93275acdab0 100644 --- a/netbox/extras/models/scripts.py +++ b/netbox/extras/models/scripts.py @@ -9,7 +9,7 @@ from core.choices import ManagedFileRootPathChoices from core.models import ManagedFile from extras.utils import is_script -from netbox.models.features import JobsMixin, WebhooksMixin +from netbox.models.features import JobsMixin, EventRulesMixin from utilities.querysets import RestrictedQuerySet from .mixins import PythonModuleMixin @@ -21,7 +21,7 @@ logger = logging.getLogger('netbox.data_backends') -class Script(WebhooksMixin, models.Model): +class Script(EventRulesMixin, models.Model): """ Dummy model used to generate permissions for custom scripts. Does not exist in the database. """ diff --git a/netbox/netbox/models/__init__.py b/netbox/netbox/models/__init__.py index 9d769669684..2c262b25867 100644 --- a/netbox/netbox/models/__init__.py +++ b/netbox/netbox/models/__init__.py @@ -30,7 +30,7 @@ class NetBoxFeatureSet( ExportTemplatesMixin, JournalingMixin, TagsMixin, - WebhooksMixin + EventRulesMixin ): class Meta: abstract = True @@ -44,7 +44,7 @@ def docs_url(self): # Base model classes # -class ChangeLoggedModel(ChangeLoggingMixin, CustomValidationMixin, WebhooksMixin, models.Model): +class ChangeLoggedModel(ChangeLoggingMixin, CustomValidationMixin, EventRulesMixin, models.Model): """ Base model for ancillary models; provides limited functionality for models which don't support NetBox's full feature set. diff --git a/netbox/netbox/models/features.py b/netbox/netbox/models/features.py index f39f3562081..840222d8b56 100644 --- a/netbox/netbox/models/features.py +++ b/netbox/netbox/models/features.py @@ -35,7 +35,7 @@ 'JournalingMixin', 'SyncedDataMixin', 'TagsMixin', - 'WebhooksMixin', + 'EventRulesMixin', ) @@ -400,9 +400,9 @@ class Meta: abstract = True -class WebhooksMixin(models.Model): +class EventRulesMixin(models.Model): """ - Enables support for webhooks. + Enables support for event rules, which can be used to transmit webhooks or execute scripts automatically. """ class Meta: abstract = True @@ -555,7 +555,7 @@ def sync_data(self): 'journaling': JournalingMixin, 'synced_data': SyncedDataMixin, 'tags': TagsMixin, - 'webhooks': WebhooksMixin, + 'webhooks': EventRulesMixin, } registry['model_features'].update({ From 43eef0effe6394ca510474d94cb9c120f81aae0b Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Tue, 28 Nov 2023 08:39:51 -0500 Subject: [PATCH 67/95] Rename webhooks key for model_features registry key to event_rules --- docs/development/application-registry.md | 2 +- docs/development/models.md | 28 ++++++++++++------------ netbox/core/models/contenttypes.py | 2 +- netbox/extras/api/serializers.py | 2 +- netbox/extras/events.py | 2 +- netbox/extras/forms/bulk_import.py | 2 +- netbox/extras/forms/filtersets.py | 2 +- netbox/extras/forms/model_forms.py | 2 +- netbox/netbox/models/features.py | 2 +- 9 files changed, 22 insertions(+), 22 deletions(-) diff --git a/docs/development/application-registry.md b/docs/development/application-registry.md index c845cd5a7f7..570563431bf 100644 --- a/docs/development/application-registry.md +++ b/docs/development/application-registry.md @@ -31,7 +31,7 @@ A dictionary of particular features (e.g. custom fields) mapped to the NetBox mo 'dcim': ['site', 'rack', 'devicetype', ...], ... }, - 'webhooks': { + 'event_rules': { 'extras': ['configcontext', 'tag', ...], 'dcim': ['site', 'rack', 'devicetype', ...], }, diff --git a/docs/development/models.md b/docs/development/models.md index 38c6fd3b673..f04610ad564 100644 --- a/docs/development/models.md +++ b/docs/development/models.md @@ -10,19 +10,19 @@ The Django [content types](https://docs.djangoproject.com/en/stable/ref/contrib/ Depending on its classification, each NetBox model may support various features which enhance its operation. Each feature is enabled by inheriting from its designated mixin class, and some features also make use of the [application registry](./application-registry.md#model_features). -| Feature | Feature Mixin | Registry Key | Description | -|------------------------------------------------------------|-------------------------|--------------------|--------------------------------------------------------------------------------| -| [Change logging](../features/change-logging.md) | `ChangeLoggingMixin` | - | Changes to these objects are automatically recorded in the change log | -| Cloning | `CloningMixin` | - | Provides the `clone()` method to prepare a copy | -| [Custom fields](../customization/custom-fields.md) | `CustomFieldsMixin` | `custom_fields` | These models support the addition of user-defined fields | -| [Custom links](../customization/custom-links.md) | `CustomLinksMixin` | `custom_links` | These models support the assignment of custom links | -| [Custom validation](../customization/custom-validation.md) | `CustomValidationMixin` | - | Supports the enforcement of custom validation rules | -| [Export templates](../customization/export-templates.md) | `ExportTemplatesMixin` | `export_templates` | Users can create custom export templates for these models | -| [Job results](../features/background-jobs.md) | `JobsMixin` | `jobs` | Users can create custom export templates for these models | -| [Journaling](../features/journaling.md) | `JournalingMixin` | `journaling` | These models support persistent historical commentary | -| [Synchronized data](../integrations/synchronized-data.md) | `SyncedDataMixin` | `synced_data` | Certain model data can be automatically synchronized from a remote data source | -| [Tagging](../models/extras/tag.md) | `TagsMixin` | `tags` | The models can be tagged with user-defined tags | -| [Webhooks](../integrations/webhooks.md) | `EventRulesMixin` | `webhooks` | NetBox is capable of generating outgoing webhooks for these objects | +| Feature | Feature Mixin | Registry Key | Description | +|------------------------------------------------------------|-------------------------|--------------------|-----------------------------------------------------------------------------------------| +| [Change logging](../features/change-logging.md) | `ChangeLoggingMixin` | - | Changes to these objects are automatically recorded in the change log | +| Cloning | `CloningMixin` | - | Provides the `clone()` method to prepare a copy | +| [Custom fields](../customization/custom-fields.md) | `CustomFieldsMixin` | `custom_fields` | These models support the addition of user-defined fields | +| [Custom links](../customization/custom-links.md) | `CustomLinksMixin` | `custom_links` | These models support the assignment of custom links | +| [Custom validation](../customization/custom-validation.md) | `CustomValidationMixin` | - | Supports the enforcement of custom validation rules | +| [Export templates](../customization/export-templates.md) | `ExportTemplatesMixin` | `export_templates` | Users can create custom export templates for these models | +| [Job results](../features/background-jobs.md) | `JobsMixin` | `jobs` | Users can create custom export templates for these models | +| [Journaling](../features/journaling.md) | `JournalingMixin` | `journaling` | These models support persistent historical commentary | +| [Synchronized data](../integrations/synchronized-data.md) | `SyncedDataMixin` | `synced_data` | Certain model data can be automatically synchronized from a remote data source | +| [Tagging](../models/extras/tag.md) | `TagsMixin` | `tags` | The models can be tagged with user-defined tags | +| [Event rules](../features/event-rules.md) | `EventRulesMixin` | `event_rules` | Event rules can send webhooks or run custom scripts automatically in response to events | ## Models Index @@ -111,7 +111,7 @@ Component models represent individual physical or virtual components belonging t ### Component Template Models -These function as templates to effect the replication of device and virtual machine components. Component template models support a limited feature set, including change logging, custom validation, and webhooks. +These function as templates to effect the replication of device and virtual machine components. Component template models support a limited feature set, including change logging, custom validation, and event rules. * [dcim.ConsolePortTemplate](../models/dcim/consoleporttemplate.md) * [dcim.ConsoleServerPortTemplate](../models/dcim/consoleserverporttemplate.md) diff --git a/netbox/core/models/contenttypes.py b/netbox/core/models/contenttypes.py index 0731871ec54..c98184c3d37 100644 --- a/netbox/core/models/contenttypes.py +++ b/netbox/core/models/contenttypes.py @@ -26,7 +26,7 @@ def with_feature(self, feature): Return the ContentTypes only for models which are registered as supporting the specified feature. For example, we can find all ContentTypes for models which support webhooks with - ContentType.objects.with_feature('webhooks') + ContentType.objects.with_feature('event_rules') """ if feature not in registry['model_features']: raise KeyError( diff --git a/netbox/extras/api/serializers.py b/netbox/extras/api/serializers.py index 7c115ecb2a2..b2fded66e03 100644 --- a/netbox/extras/api/serializers.py +++ b/netbox/extras/api/serializers.py @@ -64,7 +64,7 @@ class EventRuleSerializer(NetBoxModelSerializer): url = serializers.HyperlinkedIdentityField(view_name='extras-api:eventrule-detail') content_types = ContentTypeField( - queryset=ContentType.objects.with_feature('webhooks'), + queryset=ContentType.objects.with_feature('event_rules'), many=True ) action_type = ChoiceField(choices=EventRuleActionChoices) diff --git a/netbox/extras/events.py b/netbox/extras/events.py index 7d4e3839618..1ffe3a35ba0 100644 --- a/netbox/extras/events.py +++ b/netbox/extras/events.py @@ -55,7 +55,7 @@ def enqueue_object(queue, instance, user, request_id, action): # Determine whether this type of object supports event rules app_label = instance._meta.app_label model_name = instance._meta.model_name - if model_name not in registry['model_features']['webhooks'].get(app_label, []): + if model_name not in registry['model_features']['event_rules'].get(app_label, []): return queue.append({ diff --git a/netbox/extras/forms/bulk_import.py b/netbox/extras/forms/bulk_import.py index 752371fcc7a..efd331b5c1f 100644 --- a/netbox/extras/forms/bulk_import.py +++ b/netbox/extras/forms/bulk_import.py @@ -156,7 +156,7 @@ class Meta: class EventRuleImportForm(NetBoxModelImportForm): content_types = CSVMultipleContentTypeField( label=_('Content types'), - queryset=ContentType.objects.with_feature('webhooks'), + queryset=ContentType.objects.with_feature('event_rules'), help_text=_("One or more assigned object types") ) diff --git a/netbox/extras/forms/filtersets.py b/netbox/extras/forms/filtersets.py index 90d39484d3c..3ad8b6dece1 100644 --- a/netbox/extras/forms/filtersets.py +++ b/netbox/extras/forms/filtersets.py @@ -255,7 +255,7 @@ class EventRuleFilterForm(NetBoxModelFilterSetForm): (_('Events'), ('type_create', 'type_update', 'type_delete', 'type_job_start', 'type_job_end')), ) content_type_id = ContentTypeMultipleChoiceField( - queryset=ContentType.objects.with_feature('webhooks'), + queryset=ContentType.objects.with_feature('event_rules'), required=False, label=_('Object type') ) diff --git a/netbox/extras/forms/model_forms.py b/netbox/extras/forms/model_forms.py index cf68b9c891b..4332fa8f511 100644 --- a/netbox/extras/forms/model_forms.py +++ b/netbox/extras/forms/model_forms.py @@ -237,7 +237,7 @@ class Meta: class EventRuleForm(NetBoxModelForm): content_types = ContentTypeMultipleChoiceField( label=_('Content types'), - queryset=ContentType.objects.with_feature('webhooks'), + queryset=ContentType.objects.with_feature('event_rules'), ) action_choice = forms.ChoiceField( label=_('Action choice'), diff --git a/netbox/netbox/models/features.py b/netbox/netbox/models/features.py index 840222d8b56..ac9893e206c 100644 --- a/netbox/netbox/models/features.py +++ b/netbox/netbox/models/features.py @@ -555,7 +555,7 @@ def sync_data(self): 'journaling': JournalingMixin, 'synced_data': SyncedDataMixin, 'tags': TagsMixin, - 'webhooks': EventRulesMixin, + 'event_rules': EventRulesMixin, } registry['model_features'].update({ From f68b8126dd7ecb25068512c4c0c1d50f1038e18b Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Tue, 28 Nov 2023 08:42:11 -0500 Subject: [PATCH 68/95] Rename clear_webhooks signal to clear_events --- netbox/extras/management/commands/runscript.py | 6 +++--- netbox/extras/scripts.py | 6 +++--- netbox/extras/signals.py | 14 +++++++------- netbox/netbox/views/generic/bulk_views.py | 18 +++++++++--------- netbox/netbox/views/generic/object_views.py | 6 +++--- 5 files changed, 25 insertions(+), 25 deletions(-) diff --git a/netbox/extras/management/commands/runscript.py b/netbox/extras/management/commands/runscript.py index 5a5e0287e80..97ee39f5011 100644 --- a/netbox/extras/management/commands/runscript.py +++ b/netbox/extras/management/commands/runscript.py @@ -13,7 +13,7 @@ from extras.api.serializers import ScriptOutputSerializer from extras.context_managers import event_tracking from extras.scripts import get_module_and_script -from extras.signals import clear_webhooks +from extras.signals import clear_events from utilities.exceptions import AbortTransaction from utilities.utils import NetBoxFakeRequest @@ -47,7 +47,7 @@ def _run_script(): raise AbortTransaction() except AbortTransaction: script.log_info("Database changes have been reverted automatically.") - clear_webhooks.send(request) + clear_events.send(request) job.data = ScriptOutputSerializer(script).data job.terminate() except Exception as e: @@ -57,7 +57,7 @@ def _run_script(): ) script.log_info("Database changes have been reverted due to error.") logger.error(f"Exception raised during script execution: {e}") - clear_webhooks.send(request) + clear_events.send(request) job.data = ScriptOutputSerializer(script).data job.terminate(status=JobStatusChoices.STATUS_ERRORED, error=str(e)) diff --git a/netbox/extras/scripts.py b/netbox/extras/scripts.py index 20aff665a41..495957fd97b 100644 --- a/netbox/extras/scripts.py +++ b/netbox/extras/scripts.py @@ -17,7 +17,7 @@ from extras.api.serializers import ScriptOutputSerializer from extras.choices import LogLevelChoices from extras.models import ScriptModule -from extras.signals import clear_webhooks +from extras.signals import clear_events from ipam.formfields import IPAddressFormField, IPNetworkFormField from ipam.validators import MaxPrefixLengthValidator, MinPrefixLengthValidator, prefix_validator from utilities.exceptions import AbortScript, AbortTransaction @@ -514,7 +514,7 @@ def _run_script(): except AbortTransaction: script.log_info("Database changes have been reverted automatically.") if request: - clear_webhooks.send(request) + clear_events.send(request) job.data = ScriptOutputSerializer(script).data job.terminate() except Exception as e: @@ -529,7 +529,7 @@ def _run_script(): job.data = ScriptOutputSerializer(script).data job.terminate(status=JobStatusChoices.STATUS_ERRORED, error=str(e)) if request: - clear_webhooks.send(request) + clear_events.send(request) logger.info(f"Script completed in {job.duration}") diff --git a/netbox/extras/signals.py b/netbox/extras/signals.py index a3b151e853d..99d719366b6 100644 --- a/netbox/extras/signals.py +++ b/netbox/extras/signals.py @@ -21,8 +21,8 @@ # Change logging/webhooks # -# Define a custom signal that can be sent to clear any queued webhooks -clear_webhooks = Signal() +# Define a custom signal that can be sent to clear any queued events +clear_events = Signal() def is_same_object(instance, webhook_data, request_id): @@ -125,13 +125,13 @@ def handle_deleted_object(sender, instance, **kwargs): model_deletes.labels(instance._meta.model_name).inc() -@receiver(clear_webhooks) -def clear_webhook_queue(sender, **kwargs): +@receiver(clear_events) +def clear_events_queue(sender, **kwargs): """ - Delete any queued webhooks (e.g. because of an aborted bulk transaction) + Delete any queued events (e.g. because of an aborted bulk transaction) """ - logger = logging.getLogger('webhooks') - logger.info(f"Clearing {len(events_queue.get())} queued webhooks ({sender})") + logger = logging.getLogger('events') + logger.info(f"Clearing {len(events_queue.get())} queued events ({sender})") events_queue.set([]) diff --git a/netbox/netbox/views/generic/bulk_views.py b/netbox/netbox/views/generic/bulk_views.py index fbe3aa2ba9a..4df50925ecf 100644 --- a/netbox/netbox/views/generic/bulk_views.py +++ b/netbox/netbox/views/generic/bulk_views.py @@ -17,7 +17,7 @@ from django_tables2.export import TableExport from extras.models import ExportTemplate -from extras.signals import clear_webhooks +from extras.signals import clear_events from utilities.error_handlers import handle_protectederror from utilities.exceptions import AbortRequest, AbortTransaction, PermissionsViolation from utilities.forms import BulkRenameForm, ConfirmationForm, restrict_form_fields @@ -279,7 +279,7 @@ def post(self, request): except (AbortRequest, PermissionsViolation) as e: logger.debug(e.message) form.add_error(None, e.message) - clear_webhooks.send(sender=self) + clear_events.send(sender=self) else: logger.debug("Form validation failed") @@ -470,12 +470,12 @@ def post(self, request): return redirect(results_url) except (AbortTransaction, ValidationError): - clear_webhooks.send(sender=self) + clear_events.send(sender=self) except (AbortRequest, PermissionsViolation) as e: logger.debug(e.message) form.add_error(None, e.message) - clear_webhooks.send(sender=self) + clear_events.send(sender=self) else: logger.debug("Form validation failed") @@ -628,12 +628,12 @@ def post(self, request, **kwargs): except ValidationError as e: messages.error(self.request, ", ".join(e.messages)) - clear_webhooks.send(sender=self) + clear_events.send(sender=self) except (AbortRequest, PermissionsViolation) as e: logger.debug(e.message) form.add_error(None, e.message) - clear_webhooks.send(sender=self) + clear_events.send(sender=self) else: logger.debug("Form validation failed") @@ -729,7 +729,7 @@ def post(self, request): except (AbortRequest, PermissionsViolation) as e: logger.debug(e.message) form.add_error(None, e.message) - clear_webhooks.send(sender=self) + clear_events.send(sender=self) else: form = self.form(initial={'pk': request.POST.getlist('pk')}) @@ -923,12 +923,12 @@ def post(self, request): raise PermissionsViolation except IntegrityError: - clear_webhooks.send(sender=self) + clear_events.send(sender=self) except (AbortRequest, PermissionsViolation) as e: logger.debug(e.message) form.add_error(None, e.message) - clear_webhooks.send(sender=self) + clear_events.send(sender=self) if not form.errors: msg = "Added {} {} to {} {}.".format( diff --git a/netbox/netbox/views/generic/object_views.py b/netbox/netbox/views/generic/object_views.py index 99508c9e320..456c2e14f1d 100644 --- a/netbox/netbox/views/generic/object_views.py +++ b/netbox/netbox/views/generic/object_views.py @@ -11,7 +11,7 @@ from django.utils.html import escape from django.utils.safestring import mark_safe -from extras.signals import clear_webhooks +from extras.signals import clear_events from utilities.error_handlers import handle_protectederror from utilities.exceptions import AbortRequest, PermissionsViolation from utilities.forms import ConfirmationForm, restrict_form_fields @@ -300,7 +300,7 @@ def post(self, request, *args, **kwargs): except (AbortRequest, PermissionsViolation) as e: logger.debug(e.message) form.add_error(None, e.message) - clear_webhooks.send(sender=self) + clear_events.send(sender=self) else: logger.debug("Form validation failed") @@ -528,7 +528,7 @@ def post(self, request): except (AbortRequest, PermissionsViolation) as e: logger.debug(e.message) form.add_error(None, e.message) - clear_webhooks.send(sender=self) + clear_events.send(sender=self) return render(request, self.template_name, { 'object': instance, From 0824fddf294561e41042825ade8c4166359f13ca Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Tue, 28 Nov 2023 08:42:35 -0500 Subject: [PATCH 69/95] Clean up docs --- docs/configuration/required-parameters.md | 9 +++------ docs/features/api-integration.md | 4 ++-- docs/index.md | 2 +- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/docs/configuration/required-parameters.md b/docs/configuration/required-parameters.md index 012d8576268..bda36599525 100644 --- a/docs/configuration/required-parameters.md +++ b/docs/configuration/required-parameters.md @@ -59,10 +59,7 @@ DATABASE = { ## REDIS -[Redis](https://redis.io/) is an in-memory data store similar to memcached. While Redis has been an optional component of -NetBox since the introduction of webhooks in version 2.4, it is required starting in 2.6 to support NetBox's caching -functionality (as well as other planned features). In 2.7, the connection settings were broken down into two sections for -task queuing and caching, allowing the user to connect to different Redis instances/databases per feature. +[Redis](https://redis.io/) is a lightweight in-memory data store similar to memcached. NetBox employs Redis for background task queuing and other features. Redis is configured using a configuration setting similar to `DATABASE` and these settings are the same for both of the `tasks` and `caching` subsections: @@ -81,7 +78,7 @@ REDIS = { 'tasks': { 'HOST': 'redis.example.com', 'PORT': 1234, - 'USERNAME': 'netbox' + 'USERNAME': 'netbox', 'PASSWORD': 'foobar', 'DATABASE': 0, 'SSL': False, @@ -89,7 +86,7 @@ REDIS = { 'caching': { 'HOST': 'localhost', 'PORT': 6379, - 'USERNAME': '' + 'USERNAME': '', 'PASSWORD': '', 'DATABASE': 1, 'SSL': False, diff --git a/docs/features/api-integration.md b/docs/features/api-integration.md index 8c0843bfea6..94a39d73173 100644 --- a/docs/features/api-integration.md +++ b/docs/features/api-integration.md @@ -26,9 +26,9 @@ To learn more about this feature, check out the [GraphQL API documentation](../i ## Webhooks -A webhook is a mechanism for conveying to some external system a change that took place in NetBox. For example, you may want to notify a monitoring system whenever the status of a device is updated in NetBox. This can be done by creating a webhook for the device model in NetBox and identifying the webhook receiver. When NetBox detects a change to a device, an HTTP request containing the details of the change and who made it be sent to the specified receiver. Webhooks are an excellent mechanism for building event-based automation processes. +A webhook is a mechanism for conveying to some external system a change that has taken place in NetBox. For example, you may want to notify a monitoring system whenever the status of a device is updated in NetBox. To do this, first create a [webhook](../models/extras/webhook.md) identifying the remote receiver (URL), HTTP method, and any other necessary parameters. Then, define an [event rule](../models/extras/eventrule.md) which is triggered by device changes to transmit the webhook. -To learn more about this feature, check out the [webhooks documentation](../integrations/webhooks.md). +When NetBox detects a change to a device, an HTTP request containing the details of the change and who made it be sent to the specified receiver. Webhooks are an excellent mechanism for building event-based automation processes. To learn more about this feature, check out the [webhooks documentation](../integrations/webhooks.md). ## Prometheus Metrics diff --git a/docs/index.md b/docs/index.md index 05cd79f2353..84334337b24 100644 --- a/docs/index.md +++ b/docs/index.md @@ -32,7 +32,7 @@ In addition to its expansive and robust data model, NetBox offers myriad mechani * Custom fields * Custom model validation * Export templates -* Webhooks +* Event rules * Plugins * REST & GraphQL APIs From 4c331f5b524fb6f9f0211e75b4d671a9056dd4da Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 28 Nov 2023 07:21:45 -0800 Subject: [PATCH 70/95] 14132 review change --- netbox/extras/forms/model_forms.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/netbox/extras/forms/model_forms.py b/netbox/extras/forms/model_forms.py index a063340ddb1..41304486daa 100644 --- a/netbox/extras/forms/model_forms.py +++ b/netbox/extras/forms/model_forms.py @@ -323,8 +323,6 @@ def __init__(self, *args, **kwargs): elif action_type == EventRuleActionChoices.SCRIPT: self.init_script_choice() - val = get_field_value(self, 'conditions') - def clean(self): super().clean() From 92d9e86eccc139372bb71eada63f00602b415017 Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 28 Nov 2023 10:17:56 -0800 Subject: [PATCH 71/95] 14132 filterset test --- netbox/extras/tests/test_filtersets.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/netbox/extras/tests/test_filtersets.py b/netbox/extras/tests/test_filtersets.py index 22752aee2b3..b3c2069b7a0 100644 --- a/netbox/extras/tests/test_filtersets.py +++ b/netbox/extras/tests/test_filtersets.py @@ -203,6 +203,32 @@ def test_ssl_verification(self): self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2) +class EventRuleTestCase(TestCase, BaseFilterSetTests): + queryset = EventRule.objects.all() + filterset = EventRuleFilterSet + + @classmethod + def setUpTestData(cls): + content_types = ContentType.objects.filter(model__in=['region', 'site', 'rack', 'location', 'device']) + + event_rules = ( + EventRule( + name='EventRule 1', + ), + EventRule( + name='EventRule 2', + ), + EventRule( + name='EventRule 3', + ), + ) + EventRule.objects.bulk_create(event_rules) + + def test_name(self): + params = {'name': ['EventRule 1', 'EventRule 2']} + self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2) + + class CustomLinkTestCase(TestCase, BaseFilterSetTests): queryset = CustomLink.objects.all() filterset = CustomLinkFilterSet From 6549dce7582fad8ffbd926a887d90422b479d8f2 Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 28 Nov 2023 13:19:11 -0800 Subject: [PATCH 72/95] 14132 fix api tests --- netbox/extras/api/nested_serializers.py | 8 +++ netbox/extras/api/serializers.py | 3 + netbox/extras/graphql/schema.py | 6 ++ netbox/extras/tests/test_api.py | 74 +++++++++++++++++++++++++ 4 files changed, 91 insertions(+) diff --git a/netbox/extras/api/nested_serializers.py b/netbox/extras/api/nested_serializers.py index a97c630d25d..9f36bac934b 100644 --- a/netbox/extras/api/nested_serializers.py +++ b/netbox/extras/api/nested_serializers.py @@ -10,6 +10,7 @@ 'NestedCustomFieldChoiceSetSerializer', 'NestedCustomFieldSerializer', 'NestedCustomLinkSerializer', + 'NestedEventRuleSerializer', 'NestedExportTemplateSerializer', 'NestedImageAttachmentSerializer', 'NestedJournalEntrySerializer', @@ -19,6 +20,13 @@ ] +class NestedEventRuleSerializer(WritableNestedSerializer): + + class Meta: + model = models.EventRule + fields = ['id', 'display', 'name'] + + class NestedWebhookSerializer(WritableNestedSerializer): url = serializers.HyperlinkedIdentityField(view_name='extras-api:webhook-detail') diff --git a/netbox/extras/api/serializers.py b/netbox/extras/api/serializers.py index b2fded66e03..96fc3fc02d5 100644 --- a/netbox/extras/api/serializers.py +++ b/netbox/extras/api/serializers.py @@ -68,6 +68,9 @@ class EventRuleSerializer(NetBoxModelSerializer): many=True ) action_type = ChoiceField(choices=EventRuleActionChoices) + action_object_type = ContentTypeField( + queryset=ContentType.objects.with_feature('event_rules'), + ) action_object = serializers.SerializerMethodField(read_only=True) class Meta: diff --git a/netbox/extras/graphql/schema.py b/netbox/extras/graphql/schema.py index e13cc0e9f46..09e399e370e 100644 --- a/netbox/extras/graphql/schema.py +++ b/netbox/extras/graphql/schema.py @@ -72,3 +72,9 @@ def resolve_tag_list(root, info, **kwargs): def resolve_webhook_list(root, info, **kwargs): return gql_query_optimizer(models.Webhook.objects.all(), info) + + event_rule = ObjectField(EventRuleType) + event_rule_list = ObjectListField(EventRuleType) + + def resolve_eventrule_list(root, info, **kwargs): + return gql_query_optimizer(models.EventRule.objects.all(), info) diff --git a/netbox/extras/tests/test_api.py b/netbox/extras/tests/test_api.py index 24f852ddeb6..17016d3095d 100644 --- a/netbox/extras/tests/test_api.py +++ b/netbox/extras/tests/test_api.py @@ -8,6 +8,7 @@ from core.choices import ManagedFileRootPathChoices from dcim.models import Device, DeviceRole, DeviceType, Manufacturer, Rack, Location, RackRole, Site +from extras.choices import * from extras.models import * from extras.reports import Report from extras.scripts import BooleanVar, IntegerVar, Script, StringVar @@ -68,6 +69,79 @@ def setUpTestData(cls): Webhook.objects.bulk_create(webhooks) +class EventRuleTest(APIViewTestCases.APIViewTestCase): + model = EventRule + brief_fields = ['display', 'id', 'name',] + + @classmethod + def setUpTestData(cls): + webhooks = ( + Webhook( + name='Webhook 1', + payload_url='http://example.com/?1', + ), + Webhook( + name='Webhook 2', + payload_url='http://example.com/?1', + ), + Webhook( + name='Webhook 3', + payload_url='http://example.com/?1', + ), + Webhook( + name='Webhook 4', + payload_url='http://example.com/?1', + ), + Webhook( + name='Webhook 5', + payload_url='http://example.com/?1', + ), + Webhook( + name='Webhook 6', + payload_url='http://example.com/?1', + ), + ) + Webhook.objects.bulk_create(webhooks) + + def setUp(self): + super().setUp() + + webhooks = Webhook.objects.all() + event_rules = ( + EventRule(name='EventRule 1', action_object=webhooks[0]), + EventRule(name='EventRule 2', action_object=webhooks[1]), + EventRule(name='EventRule 3', action_object=webhooks[2]), + ) + EventRule.objects.bulk_create(event_rules) + + self.create_data = [ + { + 'name': 'EventRule 4', + 'content_types': ['dcim.device', 'dcim.devicetype'], + 'type_create': True, + 'action_type': EventRuleActionChoices.WEBHOOK, + 'action_object_type': 'extras.webhook', + 'action_object_id': webhooks[3].pk, + }, + { + 'name': 'EventRule 5', + 'content_types': ['dcim.device', 'dcim.devicetype'], + 'type_create': True, + 'action_type': EventRuleActionChoices.WEBHOOK, + 'action_object_type': 'extras.webhook', + 'action_object_id': webhooks[4].pk, + }, + { + 'name': 'EventRule 6', + 'content_types': ['dcim.device', 'dcim.devicetype'], + 'type_create': True, + 'action_type': EventRuleActionChoices.WEBHOOK, + 'action_object_type': 'extras.webhook', + 'action_object_id': webhooks[5].pk, + }, + ] + + class CustomFieldTest(APIViewTestCases.APIViewTestCase): model = CustomField brief_fields = ['display', 'id', 'name', 'url'] From 3e4821e5bec95801f7b198e5d658aea654b2b7f0 Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 28 Nov 2023 13:30:42 -0800 Subject: [PATCH 73/95] 14132 fix api tests --- netbox/extras/tests/test_filtersets.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/netbox/extras/tests/test_filtersets.py b/netbox/extras/tests/test_filtersets.py index b3c2069b7a0..aac5b35a547 100644 --- a/netbox/extras/tests/test_filtersets.py +++ b/netbox/extras/tests/test_filtersets.py @@ -211,17 +211,27 @@ class EventRuleTestCase(TestCase, BaseFilterSetTests): def setUpTestData(cls): content_types = ContentType.objects.filter(model__in=['region', 'site', 'rack', 'location', 'device']) - event_rules = ( - EventRule( - name='EventRule 1', + webhooks = ( + Webhook( + name='Webhook 1', + payload_url='http://example.com/?1', ), - EventRule( - name='EventRule 2', + Webhook( + name='Webhook 2', + payload_url='http://example.com/?1', ), - EventRule( - name='EventRule 3', + Webhook( + name='Webhook 3', + payload_url='http://example.com/?1', ), ) + Webhook.objects.bulk_create(webhooks) + + event_rules = ( + EventRule(name='EventRule 1', action_object=webhooks[0]), + EventRule(name='EventRule 2', action_object=webhooks[1]), + EventRule(name='EventRule 3', action_object=webhooks[2]), + ) EventRule.objects.bulk_create(event_rules) def test_name(self): From 7b409b97a56202d016ec02c9c9a4e5d09897e616 Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 28 Nov 2023 14:27:37 -0800 Subject: [PATCH 74/95] 14132 merge test_webhooks --- netbox/extras/tests/test_event_rules.py | 299 ++++++++++++++++++++- netbox/extras/tests/test_webhooks.py | 328 ------------------------ 2 files changed, 289 insertions(+), 338 deletions(-) delete mode 100644 netbox/extras/tests/test_webhooks.py diff --git a/netbox/extras/tests/test_event_rules.py b/netbox/extras/tests/test_event_rules.py index 99049559366..9bed872b791 100644 --- a/netbox/extras/tests/test_event_rules.py +++ b/netbox/extras/tests/test_event_rules.py @@ -3,19 +3,18 @@ from unittest.mock import patch import django_rq +from dcim.choices import SiteStatusChoices +from dcim.models import Site from django.contrib.contenttypes.models import ContentType from django.http import HttpResponse from django.urls import reverse -from requests import Session -from rest_framework import status - -from dcim.choices import SiteStatusChoices -from dcim.models import Site -from extras.choices import ObjectChangeActionChoices -from extras.models import Tag, EventRule, Webhook +from extras.choices import EventRuleActionChoices, ObjectChangeActionChoices from extras.events import enqueue_object, flush_events, serialize_for_event +from extras.models import EventRule, Tag, Webhook from extras.webhooks import generate_signature from extras.webhooks_worker import process_webhook +from requests import Session +from rest_framework import status from utilities.testing import APITestCase @@ -36,11 +35,38 @@ def setUpTestData(cls): DUMMY_SECRET = 'LOOKATMEIMASECRETSTRING' webhooks = Webhook.objects.bulk_create(( - Webhook(name='Webhook 1',), - Webhook(name='Webhook 2',), - Webhook(name='Webhook 3',), + Webhook(name='Webhook 1', payload_url=DUMMY_URL, secret=DUMMY_SECRET, additional_headers='X-Foo: Bar'), + Webhook(name='Webhook 2', payload_url=DUMMY_URL, secret=DUMMY_SECRET), + Webhook(name='Webhook 3', payload_url=DUMMY_URL, secret=DUMMY_SECRET), )) + ct = ContentType.objects.get(app_label='extras', model='webhook') + event_rules = EventRule.objects.bulk_create(( + EventRule( + name='Webhook Event 1', + type_create=True, + action_type=EventRuleActionChoices.WEBHOOK, + action_object_type=ct, + action_object_id=webhooks[0].id + ), + EventRule( + name='Webhook Event 2', + type_update=True, + action_type=EventRuleActionChoices.WEBHOOK, + action_object_type=ct, + action_object_id=webhooks[0].id + ), + EventRule( + name='Webhook Event 3', + type_delete=True, + action_type=EventRuleActionChoices.WEBHOOK, + action_object_type=ct, + action_object_id=webhooks[0].id + ), + )) + for event_rule in event_rules: + event_rule.content_types.set([site_ct]) + Tag.objects.bulk_create(( Tag(name='Foo', slug='foo'), Tag(name='Bar', slug='bar'), @@ -76,3 +102,256 @@ def test_event_rule_conditions(self): # Evaluate the conditions (status='active') self.assertTrue(event_rule.eval_conditions(data)) + + def test_enqueue_webhook_create(self): + # Create an object via the REST API + data = { + 'name': 'Site 1', + 'slug': 'site-1', + 'tags': [ + {'name': 'Foo'}, + {'name': 'Bar'}, + ] + } + url = reverse('dcim-api:site-list') + self.add_permissions('dcim.add_site') + response = self.client.post(url, data, format='json', **self.header) + self.assertHttpStatus(response, status.HTTP_201_CREATED) + self.assertEqual(Site.objects.count(), 1) + self.assertEqual(Site.objects.first().tags.count(), 2) + + # Verify that a job was queued for the object creation webhook + self.assertEqual(self.queue.count, 1) + job = self.queue.jobs[0] + self.assertEqual(job.kwargs['event_rule'], EventRule.objects.get(type_create=True)) + self.assertEqual(job.kwargs['event'], ObjectChangeActionChoices.ACTION_CREATE) + self.assertEqual(job.kwargs['model_name'], 'site') + self.assertEqual(job.kwargs['data']['id'], response.data['id']) + self.assertEqual(len(job.kwargs['data']['tags']), len(response.data['tags'])) + self.assertEqual(job.kwargs['snapshots']['postchange']['name'], 'Site 1') + self.assertEqual(job.kwargs['snapshots']['postchange']['tags'], ['Bar', 'Foo']) + + def test_enqueue_webhook_bulk_create(self): + # Create multiple objects via the REST API + data = [ + { + 'name': 'Site 1', + 'slug': 'site-1', + 'tags': [ + {'name': 'Foo'}, + {'name': 'Bar'}, + ] + }, + { + 'name': 'Site 2', + 'slug': 'site-2', + 'tags': [ + {'name': 'Foo'}, + {'name': 'Bar'}, + ] + }, + { + 'name': 'Site 3', + 'slug': 'site-3', + 'tags': [ + {'name': 'Foo'}, + {'name': 'Bar'}, + ] + }, + ] + url = reverse('dcim-api:site-list') + self.add_permissions('dcim.add_site') + response = self.client.post(url, data, format='json', **self.header) + self.assertHttpStatus(response, status.HTTP_201_CREATED) + self.assertEqual(Site.objects.count(), 3) + self.assertEqual(Site.objects.first().tags.count(), 2) + + # Verify that a webhook was queued for each object + self.assertEqual(self.queue.count, 3) + for i, job in enumerate(self.queue.jobs): + self.assertEqual(job.kwargs['event_rule'], EventRule.objects.get(type_create=True)) + self.assertEqual(job.kwargs['event'], ObjectChangeActionChoices.ACTION_CREATE) + self.assertEqual(job.kwargs['model_name'], 'site') + self.assertEqual(job.kwargs['data']['id'], response.data[i]['id']) + self.assertEqual(len(job.kwargs['data']['tags']), len(response.data[i]['tags'])) + self.assertEqual(job.kwargs['snapshots']['postchange']['name'], response.data[i]['name']) + self.assertEqual(job.kwargs['snapshots']['postchange']['tags'], ['Bar', 'Foo']) + + def test_enqueue_webhook_update(self): + site = Site.objects.create(name='Site 1', slug='site-1') + site.tags.set(Tag.objects.filter(name__in=['Foo', 'Bar'])) + + # Update an object via the REST API + data = { + 'name': 'Site X', + 'comments': 'Updated the site', + 'tags': [ + {'name': 'Baz'} + ] + } + url = reverse('dcim-api:site-detail', kwargs={'pk': site.pk}) + self.add_permissions('dcim.change_site') + response = self.client.patch(url, data, format='json', **self.header) + self.assertHttpStatus(response, status.HTTP_200_OK) + + # Verify that a job was queued for the object update webhook + self.assertEqual(self.queue.count, 1) + job = self.queue.jobs[0] + self.assertEqual(job.kwargs['event_rule'], EventRule.objects.get(type_update=True)) + self.assertEqual(job.kwargs['event'], ObjectChangeActionChoices.ACTION_UPDATE) + self.assertEqual(job.kwargs['model_name'], 'site') + self.assertEqual(job.kwargs['data']['id'], site.pk) + self.assertEqual(len(job.kwargs['data']['tags']), len(response.data['tags'])) + self.assertEqual(job.kwargs['snapshots']['prechange']['name'], 'Site 1') + self.assertEqual(job.kwargs['snapshots']['prechange']['tags'], ['Bar', 'Foo']) + self.assertEqual(job.kwargs['snapshots']['postchange']['name'], 'Site X') + self.assertEqual(job.kwargs['snapshots']['postchange']['tags'], ['Baz']) + + def test_enqueue_webhook_bulk_update(self): + sites = ( + Site(name='Site 1', slug='site-1'), + Site(name='Site 2', slug='site-2'), + Site(name='Site 3', slug='site-3'), + ) + Site.objects.bulk_create(sites) + for site in sites: + site.tags.set(Tag.objects.filter(name__in=['Foo', 'Bar'])) + + # Update three objects via the REST API + data = [ + { + 'id': sites[0].pk, + 'name': 'Site X', + 'tags': [ + {'name': 'Baz'} + ] + }, + { + 'id': sites[1].pk, + 'name': 'Site Y', + 'tags': [ + {'name': 'Baz'} + ] + }, + { + 'id': sites[2].pk, + 'name': 'Site Z', + 'tags': [ + {'name': 'Baz'} + ] + }, + ] + url = reverse('dcim-api:site-list') + self.add_permissions('dcim.change_site') + response = self.client.patch(url, data, format='json', **self.header) + self.assertHttpStatus(response, status.HTTP_200_OK) + + # Verify that a job was queued for the object update webhook + self.assertEqual(self.queue.count, 3) + for i, job in enumerate(self.queue.jobs): + self.assertEqual(job.kwargs['event_rule'], EventRule.objects.get(type_update=True)) + self.assertEqual(job.kwargs['event'], ObjectChangeActionChoices.ACTION_UPDATE) + self.assertEqual(job.kwargs['model_name'], 'site') + self.assertEqual(job.kwargs['data']['id'], data[i]['id']) + self.assertEqual(len(job.kwargs['data']['tags']), len(response.data[i]['tags'])) + self.assertEqual(job.kwargs['snapshots']['prechange']['name'], sites[i].name) + self.assertEqual(job.kwargs['snapshots']['prechange']['tags'], ['Bar', 'Foo']) + self.assertEqual(job.kwargs['snapshots']['postchange']['name'], response.data[i]['name']) + self.assertEqual(job.kwargs['snapshots']['postchange']['tags'], ['Baz']) + + def test_enqueue_webhook_delete(self): + site = Site.objects.create(name='Site 1', slug='site-1') + site.tags.set(Tag.objects.filter(name__in=['Foo', 'Bar'])) + + # Delete an object via the REST API + url = reverse('dcim-api:site-detail', kwargs={'pk': site.pk}) + self.add_permissions('dcim.delete_site') + response = self.client.delete(url, **self.header) + self.assertHttpStatus(response, status.HTTP_204_NO_CONTENT) + + # Verify that a job was queued for the object update webhook + self.assertEqual(self.queue.count, 1) + job = self.queue.jobs[0] + self.assertEqual(job.kwargs['event_rule'], EventRule.objects.get(type_delete=True)) + self.assertEqual(job.kwargs['event'], ObjectChangeActionChoices.ACTION_DELETE) + self.assertEqual(job.kwargs['model_name'], 'site') + self.assertEqual(job.kwargs['data']['id'], site.pk) + self.assertEqual(job.kwargs['snapshots']['prechange']['name'], 'Site 1') + self.assertEqual(job.kwargs['snapshots']['prechange']['tags'], ['Bar', 'Foo']) + + def test_enqueue_webhook_bulk_delete(self): + sites = ( + Site(name='Site 1', slug='site-1'), + Site(name='Site 2', slug='site-2'), + Site(name='Site 3', slug='site-3'), + ) + Site.objects.bulk_create(sites) + for site in sites: + site.tags.set(Tag.objects.filter(name__in=['Foo', 'Bar'])) + + # Delete three objects via the REST API + data = [ + {'id': site.pk} for site in sites + ] + url = reverse('dcim-api:site-list') + self.add_permissions('dcim.delete_site') + response = self.client.delete(url, data, format='json', **self.header) + self.assertHttpStatus(response, status.HTTP_204_NO_CONTENT) + + # Verify that a job was queued for the object update webhook + self.assertEqual(self.queue.count, 3) + for i, job in enumerate(self.queue.jobs): + self.assertEqual(job.kwargs['event_rule'], EventRule.objects.get(type_delete=True)) + self.assertEqual(job.kwargs['event'], ObjectChangeActionChoices.ACTION_DELETE) + self.assertEqual(job.kwargs['model_name'], 'site') + self.assertEqual(job.kwargs['data']['id'], sites[i].pk) + self.assertEqual(job.kwargs['snapshots']['prechange']['name'], sites[i].name) + self.assertEqual(job.kwargs['snapshots']['prechange']['tags'], ['Bar', 'Foo']) + + def test_webhooks_worker(self): + + request_id = uuid.uuid4() + + def dummy_send(_, request, **kwargs): + """ + A dummy implementation of Session.send() to be used for testing. + Always returns a 200 HTTP response. + """ + event = EventRule.objects.get(type_create=True) + webhook = event.action_object + signature = generate_signature(request.body, webhook.secret) + + # Validate the outgoing request headers + self.assertEqual(request.headers['Content-Type'], webhook.http_content_type) + self.assertEqual(request.headers['X-Hook-Signature'], signature) + self.assertEqual(request.headers['X-Foo'], 'Bar') + + # Validate the outgoing request body + body = json.loads(request.body) + self.assertEqual(body['event'], 'created') + self.assertEqual(body['timestamp'], job.kwargs['timestamp']) + self.assertEqual(body['model'], 'site') + self.assertEqual(body['username'], 'testuser') + self.assertEqual(body['request_id'], str(request_id)) + self.assertEqual(body['data']['name'], 'Site 1') + + return HttpResponse() + + # Enqueue a webhook for processing + webhooks_queue = [] + site = Site.objects.create(name='Site 1', slug='site-1') + enqueue_object( + webhooks_queue, + instance=site, + user=self.user, + request_id=request_id, + action=ObjectChangeActionChoices.ACTION_CREATE + ) + flush_events(webhooks_queue) + + # Retrieve the job from queue + job = self.queue.jobs[0] + + # Patch the Session object with our dummy_send() method, then process the webhook for sending + with patch.object(Session, 'send', dummy_send) as mock_send: + process_webhook(**job.kwargs) diff --git a/netbox/extras/tests/test_webhooks.py b/netbox/extras/tests/test_webhooks.py deleted file mode 100644 index 1e0287bdc75..00000000000 --- a/netbox/extras/tests/test_webhooks.py +++ /dev/null @@ -1,328 +0,0 @@ -import json -import uuid -from unittest.mock import patch - -import django_rq -from django.contrib.contenttypes.models import ContentType -from django.http import HttpResponse -from django.urls import reverse -from requests import Session -from rest_framework import status - -from dcim.choices import SiteStatusChoices -from dcim.models import Site -from extras.choices import ObjectChangeActionChoices, EventRuleActionChoices -from extras.models import Tag, Webhook, EventRule -from extras.events import enqueue_object, flush_events -from extras.webhooks import generate_signature -from extras.webhooks_worker import process_webhook -from utilities.testing import APITestCase - - -class WebhookTest(APITestCase): - - def setUp(self): - super().setUp() - - # Ensure the queue has been cleared for each test - self.queue = django_rq.get_queue('default') - self.queue.empty() - - @classmethod - def setUpTestData(cls): - - site_ct = ContentType.objects.get_for_model(Site) - DUMMY_URL = 'http://localhost:9000/' - DUMMY_SECRET = 'LOOKATMEIMASECRETSTRING' - - webhooks = Webhook.objects.bulk_create(( - Webhook(name='Webhook 1', payload_url=DUMMY_URL, secret=DUMMY_SECRET, additional_headers='X-Foo: Bar'), - Webhook(name='Webhook 2', payload_url=DUMMY_URL, secret=DUMMY_SECRET), - Webhook(name='Webhook 3', payload_url=DUMMY_URL, secret=DUMMY_SECRET), - )) - - ct = ContentType.objects.get(app_label='extras', model='webhook') - event_rules = EventRule.objects.bulk_create(( - EventRule( - name='Webhook Event 1', - type_create=True, - action_type=EventRuleActionChoices.WEBHOOK, - action_object_type=ct, - action_object_id=webhooks[0].id - ), - EventRule( - name='Webhook Event 2', - type_update=True, - action_type=EventRuleActionChoices.WEBHOOK, - action_object_type=ct, - action_object_id=webhooks[0].id - ), - EventRule( - name='Webhook Event 3', - type_delete=True, - action_type=EventRuleActionChoices.WEBHOOK, - action_object_type=ct, - action_object_id=webhooks[0].id - ), - )) - for event_rule in event_rules: - event_rule.content_types.set([site_ct]) - - Tag.objects.bulk_create(( - Tag(name='Foo', slug='foo'), - Tag(name='Bar', slug='bar'), - Tag(name='Baz', slug='baz'), - )) - - def test_enqueue_webhook_create(self): - # Create an object via the REST API - data = { - 'name': 'Site 1', - 'slug': 'site-1', - 'tags': [ - {'name': 'Foo'}, - {'name': 'Bar'}, - ] - } - url = reverse('dcim-api:site-list') - self.add_permissions('dcim.add_site') - response = self.client.post(url, data, format='json', **self.header) - self.assertHttpStatus(response, status.HTTP_201_CREATED) - self.assertEqual(Site.objects.count(), 1) - self.assertEqual(Site.objects.first().tags.count(), 2) - - # Verify that a job was queued for the object creation webhook - self.assertEqual(self.queue.count, 1) - job = self.queue.jobs[0] - self.assertEqual(job.kwargs['event_rule'], EventRule.objects.get(type_create=True)) - self.assertEqual(job.kwargs['event'], ObjectChangeActionChoices.ACTION_CREATE) - self.assertEqual(job.kwargs['model_name'], 'site') - self.assertEqual(job.kwargs['data']['id'], response.data['id']) - self.assertEqual(len(job.kwargs['data']['tags']), len(response.data['tags'])) - self.assertEqual(job.kwargs['snapshots']['postchange']['name'], 'Site 1') - self.assertEqual(job.kwargs['snapshots']['postchange']['tags'], ['Bar', 'Foo']) - - def test_enqueue_webhook_bulk_create(self): - # Create multiple objects via the REST API - data = [ - { - 'name': 'Site 1', - 'slug': 'site-1', - 'tags': [ - {'name': 'Foo'}, - {'name': 'Bar'}, - ] - }, - { - 'name': 'Site 2', - 'slug': 'site-2', - 'tags': [ - {'name': 'Foo'}, - {'name': 'Bar'}, - ] - }, - { - 'name': 'Site 3', - 'slug': 'site-3', - 'tags': [ - {'name': 'Foo'}, - {'name': 'Bar'}, - ] - }, - ] - url = reverse('dcim-api:site-list') - self.add_permissions('dcim.add_site') - response = self.client.post(url, data, format='json', **self.header) - self.assertHttpStatus(response, status.HTTP_201_CREATED) - self.assertEqual(Site.objects.count(), 3) - self.assertEqual(Site.objects.first().tags.count(), 2) - - # Verify that a webhook was queued for each object - self.assertEqual(self.queue.count, 3) - for i, job in enumerate(self.queue.jobs): - self.assertEqual(job.kwargs['event_rule'], EventRule.objects.get(type_create=True)) - self.assertEqual(job.kwargs['event'], ObjectChangeActionChoices.ACTION_CREATE) - self.assertEqual(job.kwargs['model_name'], 'site') - self.assertEqual(job.kwargs['data']['id'], response.data[i]['id']) - self.assertEqual(len(job.kwargs['data']['tags']), len(response.data[i]['tags'])) - self.assertEqual(job.kwargs['snapshots']['postchange']['name'], response.data[i]['name']) - self.assertEqual(job.kwargs['snapshots']['postchange']['tags'], ['Bar', 'Foo']) - - def test_enqueue_webhook_update(self): - site = Site.objects.create(name='Site 1', slug='site-1') - site.tags.set(Tag.objects.filter(name__in=['Foo', 'Bar'])) - - # Update an object via the REST API - data = { - 'name': 'Site X', - 'comments': 'Updated the site', - 'tags': [ - {'name': 'Baz'} - ] - } - url = reverse('dcim-api:site-detail', kwargs={'pk': site.pk}) - self.add_permissions('dcim.change_site') - response = self.client.patch(url, data, format='json', **self.header) - self.assertHttpStatus(response, status.HTTP_200_OK) - - # Verify that a job was queued for the object update webhook - self.assertEqual(self.queue.count, 1) - job = self.queue.jobs[0] - self.assertEqual(job.kwargs['event_rule'], EventRule.objects.get(type_update=True)) - self.assertEqual(job.kwargs['event'], ObjectChangeActionChoices.ACTION_UPDATE) - self.assertEqual(job.kwargs['model_name'], 'site') - self.assertEqual(job.kwargs['data']['id'], site.pk) - self.assertEqual(len(job.kwargs['data']['tags']), len(response.data['tags'])) - self.assertEqual(job.kwargs['snapshots']['prechange']['name'], 'Site 1') - self.assertEqual(job.kwargs['snapshots']['prechange']['tags'], ['Bar', 'Foo']) - self.assertEqual(job.kwargs['snapshots']['postchange']['name'], 'Site X') - self.assertEqual(job.kwargs['snapshots']['postchange']['tags'], ['Baz']) - - def test_enqueue_webhook_bulk_update(self): - sites = ( - Site(name='Site 1', slug='site-1'), - Site(name='Site 2', slug='site-2'), - Site(name='Site 3', slug='site-3'), - ) - Site.objects.bulk_create(sites) - for site in sites: - site.tags.set(Tag.objects.filter(name__in=['Foo', 'Bar'])) - - # Update three objects via the REST API - data = [ - { - 'id': sites[0].pk, - 'name': 'Site X', - 'tags': [ - {'name': 'Baz'} - ] - }, - { - 'id': sites[1].pk, - 'name': 'Site Y', - 'tags': [ - {'name': 'Baz'} - ] - }, - { - 'id': sites[2].pk, - 'name': 'Site Z', - 'tags': [ - {'name': 'Baz'} - ] - }, - ] - url = reverse('dcim-api:site-list') - self.add_permissions('dcim.change_site') - response = self.client.patch(url, data, format='json', **self.header) - self.assertHttpStatus(response, status.HTTP_200_OK) - - # Verify that a job was queued for the object update webhook - self.assertEqual(self.queue.count, 3) - for i, job in enumerate(self.queue.jobs): - self.assertEqual(job.kwargs['event_rule'], EventRule.objects.get(type_update=True)) - self.assertEqual(job.kwargs['event'], ObjectChangeActionChoices.ACTION_UPDATE) - self.assertEqual(job.kwargs['model_name'], 'site') - self.assertEqual(job.kwargs['data']['id'], data[i]['id']) - self.assertEqual(len(job.kwargs['data']['tags']), len(response.data[i]['tags'])) - self.assertEqual(job.kwargs['snapshots']['prechange']['name'], sites[i].name) - self.assertEqual(job.kwargs['snapshots']['prechange']['tags'], ['Bar', 'Foo']) - self.assertEqual(job.kwargs['snapshots']['postchange']['name'], response.data[i]['name']) - self.assertEqual(job.kwargs['snapshots']['postchange']['tags'], ['Baz']) - - def test_enqueue_webhook_delete(self): - site = Site.objects.create(name='Site 1', slug='site-1') - site.tags.set(Tag.objects.filter(name__in=['Foo', 'Bar'])) - - # Delete an object via the REST API - url = reverse('dcim-api:site-detail', kwargs={'pk': site.pk}) - self.add_permissions('dcim.delete_site') - response = self.client.delete(url, **self.header) - self.assertHttpStatus(response, status.HTTP_204_NO_CONTENT) - - # Verify that a job was queued for the object update webhook - self.assertEqual(self.queue.count, 1) - job = self.queue.jobs[0] - self.assertEqual(job.kwargs['event_rule'], EventRule.objects.get(type_delete=True)) - self.assertEqual(job.kwargs['event'], ObjectChangeActionChoices.ACTION_DELETE) - self.assertEqual(job.kwargs['model_name'], 'site') - self.assertEqual(job.kwargs['data']['id'], site.pk) - self.assertEqual(job.kwargs['snapshots']['prechange']['name'], 'Site 1') - self.assertEqual(job.kwargs['snapshots']['prechange']['tags'], ['Bar', 'Foo']) - - def test_enqueue_webhook_bulk_delete(self): - sites = ( - Site(name='Site 1', slug='site-1'), - Site(name='Site 2', slug='site-2'), - Site(name='Site 3', slug='site-3'), - ) - Site.objects.bulk_create(sites) - for site in sites: - site.tags.set(Tag.objects.filter(name__in=['Foo', 'Bar'])) - - # Delete three objects via the REST API - data = [ - {'id': site.pk} for site in sites - ] - url = reverse('dcim-api:site-list') - self.add_permissions('dcim.delete_site') - response = self.client.delete(url, data, format='json', **self.header) - self.assertHttpStatus(response, status.HTTP_204_NO_CONTENT) - - # Verify that a job was queued for the object update webhook - self.assertEqual(self.queue.count, 3) - for i, job in enumerate(self.queue.jobs): - self.assertEqual(job.kwargs['event_rule'], EventRule.objects.get(type_delete=True)) - self.assertEqual(job.kwargs['event'], ObjectChangeActionChoices.ACTION_DELETE) - self.assertEqual(job.kwargs['model_name'], 'site') - self.assertEqual(job.kwargs['data']['id'], sites[i].pk) - self.assertEqual(job.kwargs['snapshots']['prechange']['name'], sites[i].name) - self.assertEqual(job.kwargs['snapshots']['prechange']['tags'], ['Bar', 'Foo']) - - def test_webhooks_worker(self): - - request_id = uuid.uuid4() - - def dummy_send(_, request, **kwargs): - """ - A dummy implementation of Session.send() to be used for testing. - Always returns a 200 HTTP response. - """ - event = EventRule.objects.get(type_create=True) - webhook = event.action_object - signature = generate_signature(request.body, webhook.secret) - - # Validate the outgoing request headers - self.assertEqual(request.headers['Content-Type'], webhook.http_content_type) - self.assertEqual(request.headers['X-Hook-Signature'], signature) - self.assertEqual(request.headers['X-Foo'], 'Bar') - - # Validate the outgoing request body - body = json.loads(request.body) - self.assertEqual(body['event'], 'created') - self.assertEqual(body['timestamp'], job.kwargs['timestamp']) - self.assertEqual(body['model'], 'site') - self.assertEqual(body['username'], 'testuser') - self.assertEqual(body['request_id'], str(request_id)) - self.assertEqual(body['data']['name'], 'Site 1') - - return HttpResponse() - - # Enqueue a webhook for processing - webhooks_queue = [] - site = Site.objects.create(name='Site 1', slug='site-1') - enqueue_object( - webhooks_queue, - instance=site, - user=self.user, - request_id=request_id, - action=ObjectChangeActionChoices.ACTION_CREATE - ) - flush_events(webhooks_queue) - - # Retrieve the job from queue - job = self.queue.jobs[0] - - # Patch the Session object with our dummy_send() method, then process the webhook for sending - with patch.object(Session, 'send', dummy_send) as mock_send: - process_webhook(**job.kwargs) From f880b8f924b744cdff75b31eabefc29124f96491 Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 28 Nov 2023 17:40:13 -0800 Subject: [PATCH 75/95] 14132 test_filtersets --- netbox/extras/api/serializers.py | 8 +- netbox/extras/tests/test_filtersets.py | 119 ++++++++++++++++++++++++- 2 files changed, 116 insertions(+), 11 deletions(-) diff --git a/netbox/extras/api/serializers.py b/netbox/extras/api/serializers.py index 96fc3fc02d5..838ac804658 100644 --- a/netbox/extras/api/serializers.py +++ b/netbox/extras/api/serializers.py @@ -71,21 +71,15 @@ class EventRuleSerializer(NetBoxModelSerializer): action_object_type = ContentTypeField( queryset=ContentType.objects.with_feature('event_rules'), ) - action_object = serializers.SerializerMethodField(read_only=True) class Meta: model = EventRule fields = [ 'id', 'url', 'display', 'content_types', 'name', 'type_create', 'type_update', 'type_delete', 'type_job_start', 'type_job_end', 'enabled', 'conditions', 'action_type', 'action_object_type', - 'action_object_id', 'action_object', 'custom_fields', 'tags', 'created', 'last_updated', + 'action_object_id', 'custom_fields', 'tags', 'created', 'last_updated', ] - @extend_schema_field(serializers.JSONField(allow_null=True)) - def get_action_object(self, instance): - serializer = get_serializer_for_model(instance.action_object, prefix=NESTED_SERIALIZER_PREFIX) - return serializer(instance.action_object, context={'request': self.context['request']}).data - # # Webhooks diff --git a/netbox/extras/tests/test_filtersets.py b/netbox/extras/tests/test_filtersets.py index aac5b35a547..c41b4c46dd5 100644 --- a/netbox/extras/tests/test_filtersets.py +++ b/netbox/extras/tests/test_filtersets.py @@ -6,6 +6,7 @@ from django.test import TestCase from circuits.models import Provider +from core.choices import ManagedFileRootPathChoices from dcim.filtersets import SiteFilterSet from dcim.models import DeviceRole, DeviceType, Manufacturer, Platform, Rack, Region, Site, SiteGroup from dcim.models import Location @@ -209,7 +210,7 @@ class EventRuleTestCase(TestCase, BaseFilterSetTests): @classmethod def setUpTestData(cls): - content_types = ContentType.objects.filter(model__in=['region', 'site', 'rack', 'location', 'device']) + content_types = ContentType.objects.filter(model__in=['region', 'site', 'rack', 'location', 'device', 'circuit']) webhooks = ( Webhook( @@ -227,17 +228,127 @@ def setUpTestData(cls): ) Webhook.objects.bulk_create(webhooks) + module = ScriptModule.objects.create( + file_root=ManagedFileRootPathChoices.SCRIPTS, + file_path='/var/tmp/script.py' + ) + event_rules = ( - EventRule(name='EventRule 1', action_object=webhooks[0]), - EventRule(name='EventRule 2', action_object=webhooks[1]), - EventRule(name='EventRule 3', action_object=webhooks[2]), + EventRule( + name='EventRule 1', + action_object=webhooks[0], + enabled=True, + type_create=True, + type_update=False, + type_delete=False, + type_job_start=False, + type_job_end=False, + action_type=EventRuleActionChoices.WEBHOOK, + ), + EventRule( + name='EventRule 2', + action_object=webhooks[0], + enabled=True, + type_create=False, + type_update=True, + type_delete=False, + type_job_start=False, + type_job_end=False, + action_type=EventRuleActionChoices.WEBHOOK, + ), + EventRule( + name='EventRule 3', + action_object=webhooks[0], + enabled=False, + type_create=False, + type_update=False, + type_delete=True, + type_job_start=False, + type_job_end=False, + action_type=EventRuleActionChoices.WEBHOOK, + ), + EventRule( + name='EventRule 4', + action_object=webhooks[0], + enabled=False, + type_create=False, + type_update=False, + type_delete=False, + type_job_start=True, + type_job_end=False, + action_type=EventRuleActionChoices.WEBHOOK, + ), + EventRule( + name='EventRule 5', + action_object=webhooks[0], + enabled=False, + type_create=False, + type_update=False, + type_delete=False, + type_job_start=False, + type_job_end=True, + action_type=EventRuleActionChoices.WEBHOOK, + ), + EventRule( + name='EventRule 6', + action_object=webhooks[0], + enabled=False, + type_create=False, + type_update=False, + type_delete=False, + type_job_start=False, + type_job_end=False, + action_type=EventRuleActionChoices.SCRIPT, + ), ) EventRule.objects.bulk_create(event_rules) + event_rules[0].content_types.add(content_types[0]) + event_rules[1].content_types.add(content_types[1]) + event_rules[2].content_types.add(content_types[2]) + event_rules[3].content_types.add(content_types[3]) + event_rules[4].content_types.add(content_types[4]) + event_rules[5].content_types.add(content_types[5]) def test_name(self): params = {'name': ['EventRule 1', 'EventRule 2']} self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2) + def test_content_types(self): + params = {'content_types': 'dcim.region'} + self.assertEqual(self.filterset(params, self.queryset).qs.count(), 1) + params = {'content_type_id': [ContentType.objects.get_for_model(Region).pk]} + self.assertEqual(self.filterset(params, self.queryset).qs.count(), 1) + + def test_action_type(self): + params = {'action_type': EventRuleActionChoices.SCRIPT} + self.assertEqual(self.filterset(params, self.queryset).qs.count(), 1) + + def test_enabled(self): + params = {'enabled': True} + self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2) + params = {'enabled': False} + self.assertEqual(self.filterset(params, self.queryset).qs.count(), 4) + + def test_type_create(self): + params = {'type_create': True} + self.assertEqual(self.filterset(params, self.queryset).qs.count(), 1) + + def test_type_update(self): + params = {'type_update': True} + self.assertEqual(self.filterset(params, self.queryset).qs.count(), 1) + + def test_type_delete(self): + params = {'type_delete': True} + self.assertEqual(self.filterset(params, self.queryset).qs.count(), 1) + + def test_type_job_start(self): + params = {'type_job_start': True} + self.assertEqual(self.filterset(params, self.queryset).qs.count(), 1) + + def test_type_job_end(self): + params = {'type_job_end': True} + self.assertEqual(self.filterset(params, self.queryset).qs.count(), 1) + class CustomLinkTestCase(TestCase, BaseFilterSetTests): queryset = CustomLink.objects.all() From afc1e789d2aa54a41e958eeeadb5b2af6fd76297 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Wed, 29 Nov 2023 08:55:12 -0500 Subject: [PATCH 76/95] Clean up filterset tests --- netbox/extras/tests/test_filtersets.py | 63 +++++++++++++------------- 1 file changed, 31 insertions(+), 32 deletions(-) diff --git a/netbox/extras/tests/test_filtersets.py b/netbox/extras/tests/test_filtersets.py index c41b4c46dd5..ddc5feb40ad 100644 --- a/netbox/extras/tests/test_filtersets.py +++ b/netbox/extras/tests/test_filtersets.py @@ -210,7 +210,9 @@ class EventRuleTestCase(TestCase, BaseFilterSetTests): @classmethod def setUpTestData(cls): - content_types = ContentType.objects.filter(model__in=['region', 'site', 'rack', 'location', 'device', 'circuit']) + content_types = ContentType.objects.filter( + model__in=['region', 'site', 'rack', 'location', 'device'] + ) webhooks = ( Webhook( @@ -219,23 +221,30 @@ def setUpTestData(cls): ), Webhook( name='Webhook 2', - payload_url='http://example.com/?1', + payload_url='http://example.com/?2', ), Webhook( name='Webhook 3', - payload_url='http://example.com/?1', + payload_url='http://example.com/?3', ), ) Webhook.objects.bulk_create(webhooks) - module = ScriptModule.objects.create( - file_root=ManagedFileRootPathChoices.SCRIPTS, - file_path='/var/tmp/script.py' + scripts = ( + ScriptModule( + file_root=ManagedFileRootPathChoices.SCRIPTS, + file_path='/var/tmp/script1.py' + ), + ScriptModule( + file_root=ManagedFileRootPathChoices.SCRIPTS, + file_path='/var/tmp/script2.py' + ), ) + ScriptModule.objects.bulk_create(scripts) event_rules = ( EventRule( - name='EventRule 1', + name='Event Rule 1', action_object=webhooks[0], enabled=True, type_create=True, @@ -246,8 +255,8 @@ def setUpTestData(cls): action_type=EventRuleActionChoices.WEBHOOK, ), EventRule( - name='EventRule 2', - action_object=webhooks[0], + name='Event Rule 2', + action_object=webhooks[1], enabled=True, type_create=False, type_update=True, @@ -257,8 +266,8 @@ def setUpTestData(cls): action_type=EventRuleActionChoices.WEBHOOK, ), EventRule( - name='EventRule 3', - action_object=webhooks[0], + name='Event Rule 3', + action_object=webhooks[2], enabled=False, type_create=False, type_update=False, @@ -268,36 +277,25 @@ def setUpTestData(cls): action_type=EventRuleActionChoices.WEBHOOK, ), EventRule( - name='EventRule 4', - action_object=webhooks[0], + name='Event Rule 4', + action_object=scripts[0], enabled=False, type_create=False, type_update=False, type_delete=False, type_job_start=True, type_job_end=False, - action_type=EventRuleActionChoices.WEBHOOK, + action_type=EventRuleActionChoices.SCRIPT, ), EventRule( - name='EventRule 5', - action_object=webhooks[0], + name='Event Rule 5', + action_object=scripts[1], enabled=False, type_create=False, type_update=False, type_delete=False, type_job_start=False, type_job_end=True, - action_type=EventRuleActionChoices.WEBHOOK, - ), - EventRule( - name='EventRule 6', - action_object=webhooks[0], - enabled=False, - type_create=False, - type_update=False, - type_delete=False, - type_job_start=False, - type_job_end=False, action_type=EventRuleActionChoices.SCRIPT, ), ) @@ -307,10 +305,9 @@ def setUpTestData(cls): event_rules[2].content_types.add(content_types[2]) event_rules[3].content_types.add(content_types[3]) event_rules[4].content_types.add(content_types[4]) - event_rules[5].content_types.add(content_types[5]) def test_name(self): - params = {'name': ['EventRule 1', 'EventRule 2']} + params = {'name': ['Event Rule 1', 'Event Rule 2']} self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2) def test_content_types(self): @@ -320,14 +317,16 @@ def test_content_types(self): self.assertEqual(self.filterset(params, self.queryset).qs.count(), 1) def test_action_type(self): - params = {'action_type': EventRuleActionChoices.SCRIPT} - self.assertEqual(self.filterset(params, self.queryset).qs.count(), 1) + params = {'action_type': [EventRuleActionChoices.WEBHOOK]} + self.assertEqual(self.filterset(params, self.queryset).qs.count(), 3) + params = {'action_type': [EventRuleActionChoices.SCRIPT]} + self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2) def test_enabled(self): params = {'enabled': True} self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2) params = {'enabled': False} - self.assertEqual(self.filterset(params, self.queryset).qs.count(), 4) + self.assertEqual(self.filterset(params, self.queryset).qs.count(), 3) def test_type_create(self): params = {'type_create': True} From bce43eabc9ca5ac623442435640f38e93af0e30f Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Wed, 29 Nov 2023 09:52:22 -0500 Subject: [PATCH 77/95] Add action_object to EventRule serializer --- netbox/extras/api/serializers.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/netbox/extras/api/serializers.py b/netbox/extras/api/serializers.py index 838ac804658..df71228d0a4 100644 --- a/netbox/extras/api/serializers.py +++ b/netbox/extras/api/serializers.py @@ -71,15 +71,25 @@ class EventRuleSerializer(NetBoxModelSerializer): action_object_type = ContentTypeField( queryset=ContentType.objects.with_feature('event_rules'), ) + action_object = serializers.SerializerMethodField(read_only=True) class Meta: model = EventRule fields = [ 'id', 'url', 'display', 'content_types', 'name', 'type_create', 'type_update', 'type_delete', 'type_job_start', 'type_job_end', 'enabled', 'conditions', 'action_type', 'action_object_type', - 'action_object_id', 'custom_fields', 'tags', 'created', 'last_updated', + 'action_object_id', 'action_object', 'custom_fields', 'tags', 'created', 'last_updated', ] + @extend_schema_field(OpenApiTypes.OBJECT) + def get_action_object(self, instance): + serializer = get_serializer_for_model( + model=instance.action_object_type.model_class(), + prefix=NESTED_SERIALIZER_PREFIX + ) + context = {'request': self.context['request']} + return serializer(instance.action_object, context=context).data + # # Webhooks From 2252c8ba01f79be80437a38cf782910b7788b7c0 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Wed, 29 Nov 2023 10:01:35 -0500 Subject: [PATCH 78/95] Clean up API tests --- netbox/extras/api/serializers.py | 2 +- netbox/extras/tests/test_api.py | 21 +++++++++++++-------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/netbox/extras/api/serializers.py b/netbox/extras/api/serializers.py index df71228d0a4..561c0fc0a16 100644 --- a/netbox/extras/api/serializers.py +++ b/netbox/extras/api/serializers.py @@ -78,7 +78,7 @@ class Meta: fields = [ 'id', 'url', 'display', 'content_types', 'name', 'type_create', 'type_update', 'type_delete', 'type_job_start', 'type_job_end', 'enabled', 'conditions', 'action_type', 'action_object_type', - 'action_object_id', 'action_object', 'custom_fields', 'tags', 'created', 'last_updated', + 'action_object_id', 'action_object', 'description', 'custom_fields', 'tags', 'created', 'last_updated', ] @extend_schema_field(OpenApiTypes.OBJECT) diff --git a/netbox/extras/tests/test_api.py b/netbox/extras/tests/test_api.py index 17016d3095d..7fe9cea85a1 100644 --- a/netbox/extras/tests/test_api.py +++ b/netbox/extras/tests/test_api.py @@ -72,6 +72,15 @@ def setUpTestData(cls): class EventRuleTest(APIViewTestCases.APIViewTestCase): model = EventRule brief_fields = ['display', 'id', 'name',] + bulk_update_data = { + 'enabled': False, + 'description': 'New description', + } + update_data = { + 'name': 'Event Rule X', + 'enabled': False, + 'description': 'New description', + } @classmethod def setUpTestData(cls): @@ -103,18 +112,14 @@ def setUpTestData(cls): ) Webhook.objects.bulk_create(webhooks) - def setUp(self): - super().setUp() - - webhooks = Webhook.objects.all() event_rules = ( - EventRule(name='EventRule 1', action_object=webhooks[0]), - EventRule(name='EventRule 2', action_object=webhooks[1]), - EventRule(name='EventRule 3', action_object=webhooks[2]), + EventRule(name='EventRule 1', type_create=True, action_object=webhooks[0]), + EventRule(name='EventRule 2', type_create=True, action_object=webhooks[1]), + EventRule(name='EventRule 3', type_create=True, action_object=webhooks[2]), ) EventRule.objects.bulk_create(event_rules) - self.create_data = [ + cls.create_data = [ { 'name': 'EventRule 4', 'content_types': ['dcim.device', 'dcim.devicetype'], From 12b4c484a1d9982d46a460d7ecb1580e5d0d55cf Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 29 Nov 2023 09:04:28 -0800 Subject: [PATCH 79/95] 14132 fix bulk import --- netbox/extras/forms/bulk_import.py | 41 +++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/netbox/extras/forms/bulk_import.py b/netbox/extras/forms/bulk_import.py index efd331b5c1f..ded12260f71 100644 --- a/netbox/extras/forms/bulk_import.py +++ b/netbox/extras/forms/bulk_import.py @@ -1,5 +1,6 @@ from django import forms from django.contrib.postgres.forms import SimpleArrayField +from django.core.exceptions import ObjectDoesNotExist from django.utils.safestring import mark_safe from django.utils.translation import gettext_lazy as _ @@ -159,14 +160,52 @@ class EventRuleImportForm(NetBoxModelImportForm): queryset=ContentType.objects.with_feature('event_rules'), help_text=_("One or more assigned object types") ) + action_object = forms.CharField( + label=_('Action object'), + required=True, + help_text=_('Webhook name or script as dotted path module.Class') + ) class Meta: model = EventRule fields = ( 'name', 'description', 'enabled', 'conditions', 'content_types', 'type_create', 'type_update', - 'type_delete', 'type_job_start', 'type_job_end', 'comments', 'tags' + 'type_delete', 'type_job_start', 'type_job_end', 'action_type', 'action_object', 'comments', 'tags' ) + def clean(self): + super().clean() + + action_object = self.cleaned_data.get('action_object') + action_type = self.cleaned_data.get('action_type') + if action_object and action_type: + if action_type == EventRuleActionChoices.WEBHOOK: + webhook = Webhook.objects.filter(name=action_object) + if not webhook: + raise forms.ValidationError(f"Webhook {action_object} not found") + elif action_type == EventRuleActionChoices.SCRIPT: + from extras.scripts import get_module_and_script + module_name, script_name = action_object.split('.', 1) + try: + module, script = get_module_and_script(module_name, script_name) + except ObjectDoesNotExist: + raise forms.ValidationError(f"Script {action_object} not found") + + def save(self, *args, **kwargs): + action_object = self.cleaned_data.get('action_object') + action_type = self.cleaned_data.get('action_type') + + if action_type == EventRuleActionChoices.WEBHOOK: + self.instance.action_object = Webhook.objects.get(name=action_object) + elif action_type == EventRuleActionChoices.SCRIPT: + from extras.scripts import get_module_and_script + module_name, script_name = action_object.split('.', 1) + module, script = get_module_and_script(module_name, script_name) + self.instance.action_object = module + self.instance.action_parameters = {'script_choice': f"{str(module.pk)}:{script_name}"} + + return super().save(*args, **kwargs) + class TagImportForm(CSVModelForm): slug = SlugField() From 7b0d683b5062b05234cd5e206fd219e11b0fde6d Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 29 Nov 2023 09:18:02 -0800 Subject: [PATCH 80/95] 14132 test_views --- netbox/extras/tests/test_views.py | 54 +++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/netbox/extras/tests/test_views.py b/netbox/extras/tests/test_views.py index 84d272d0bf8..fdedc4810b3 100644 --- a/netbox/extras/tests/test_views.py +++ b/netbox/extras/tests/test_views.py @@ -370,6 +370,60 @@ def setUpTestData(cls): } +class EventRulesTestCase(ViewTestCases.PrimaryObjectViewTestCase): + model = EventRule + + @classmethod + def setUpTestData(cls): + + webhooks = ( + Webhook(name='Webhook 1', payload_url='http://example.com/?1', http_method='POST'), + Webhook(name='Webhook 2', payload_url='http://example.com/?2', http_method='POST'), + Webhook(name='Webhook 3', payload_url='http://example.com/?3', http_method='POST'), + ) + for webhook in webhooks: + webhook.save() + + site_ct = ContentType.objects.get_for_model(Site) + event_rules = ( + EventRule(name='EventRule 1', type_create=True, action_object=webhooks[0]), + EventRule(name='EventRule 2', type_create=True, action_object=webhooks[1]), + EventRule(name='EventRule 3', type_create=True, action_object=webhooks[2]), + ) + for event in event_rules: + event.save() + event.content_types.add(site_ct) + + webhook_ct = ContentType.objects.get_for_model(Webhook) + cls.form_data = { + 'name': 'Event X', + 'content_types': [site_ct.pk], + 'type_create': False, + 'type_update': True, + 'type_delete': True, + 'conditions': None, + 'action_type': 'webhook', + 'action_object_type': webhook_ct.pk, + 'action_object_id': webhooks[0].pk + } + + cls.csv_data = ( + "name,content_types,type_create,action_type,action_object", + "Webhook 4,dcim.site,True,webhook,Webhook 1", + ) + + cls.csv_update_data = ( + "id,name", + f"{event_rules[0].pk},Event 7", + f"{event_rules[1].pk},Event 8", + f"{event_rules[2].pk},Event 9", + ) + + cls.bulk_edit_data = { + 'type_update': True, + } + + class TagTestCase(ViewTestCases.OrganizationalObjectViewTestCase): model = Tag From 57d454f8b4c74092b710e54e67949c729a288b6e Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 29 Nov 2023 09:33:50 -0800 Subject: [PATCH 81/95] 14132 test_views --- netbox/extras/tests/test_views.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/netbox/extras/tests/test_views.py b/netbox/extras/tests/test_views.py index fdedc4810b3..37773932a99 100644 --- a/netbox/extras/tests/test_views.py +++ b/netbox/extras/tests/test_views.py @@ -404,7 +404,8 @@ def setUpTestData(cls): 'conditions': None, 'action_type': 'webhook', 'action_object_type': webhook_ct.pk, - 'action_object_id': webhooks[0].pk + 'action_object_id': webhooks[0].pk, + 'action_choice': webhooks[0] } cls.csv_data = ( From 6d28cac99f52665a85276e1152123a7333fe0e33 Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 29 Nov 2023 12:05:42 -0800 Subject: [PATCH 82/95] 14132 nested serializer --- netbox/extras/api/nested_serializers.py | 8 ++++++++ netbox/extras/forms/model_forms.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/netbox/extras/api/nested_serializers.py b/netbox/extras/api/nested_serializers.py index 9f36bac934b..85f1c3b35b1 100644 --- a/netbox/extras/api/nested_serializers.py +++ b/netbox/extras/api/nested_serializers.py @@ -15,6 +15,7 @@ 'NestedImageAttachmentSerializer', 'NestedJournalEntrySerializer', 'NestedSavedFilterSerializer', + 'NestedScriptModuleSerializer', 'NestedTagSerializer', # Defined in netbox.api.serializers 'NestedWebhookSerializer', ] @@ -113,3 +114,10 @@ class NestedJournalEntrySerializer(WritableNestedSerializer): class Meta: model = models.JournalEntry fields = ['id', 'url', 'display', 'created'] + + +class NestedScriptModuleSerializer(serializers.Serializer): + fields = ['id', ] + + def get_display(self, obj): + return f'{obj.name} ({obj.module})' diff --git a/netbox/extras/forms/model_forms.py b/netbox/extras/forms/model_forms.py index 41304486daa..888331bfe52 100644 --- a/netbox/extras/forms/model_forms.py +++ b/netbox/extras/forms/model_forms.py @@ -332,7 +332,7 @@ def clean(self): self.cleaned_data['action_object_id'] = action_choice.id elif self.cleaned_data.get('action_type') == EventRuleActionChoices.SCRIPT: script = ScriptModule.objects.get(pk=action_choice.split(":")[0]) - self.cleaned_data['action_object_type'] = ContentType.objects.get_for_model(script) + self.cleaned_data['action_object_type'] = ContentType.objects.get_for_model(script, for_concrete_model=False) self.cleaned_data['action_object_id'] = script.id self.cleaned_data['action_parameters'] = {'script_choice': action_choice} From c26e7c770d23a2bbecfd0f97bfbdb014269588cf Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 29 Nov 2023 12:54:14 -0800 Subject: [PATCH 83/95] 14132 nested serializer --- netbox/extras/api/nested_serializers.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/netbox/extras/api/nested_serializers.py b/netbox/extras/api/nested_serializers.py index 85f1c3b35b1..c4e56410f62 100644 --- a/netbox/extras/api/nested_serializers.py +++ b/netbox/extras/api/nested_serializers.py @@ -117,7 +117,9 @@ class Meta: class NestedScriptModuleSerializer(serializers.Serializer): - fields = ['id', ] + + class Meta: + fields = ['id', 'display', 'created'] def get_display(self, obj): return f'{obj.name} ({obj.module})' From a8b45781704170d0fb2df76bfaf9bb309bb6e598 Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 29 Nov 2023 12:55:17 -0800 Subject: [PATCH 84/95] 14132 nested serializer --- netbox/extras/api/nested_serializers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netbox/extras/api/nested_serializers.py b/netbox/extras/api/nested_serializers.py index c4e56410f62..e55c334a245 100644 --- a/netbox/extras/api/nested_serializers.py +++ b/netbox/extras/api/nested_serializers.py @@ -119,7 +119,7 @@ class Meta: class NestedScriptModuleSerializer(serializers.Serializer): class Meta: - fields = ['id', 'display', 'created'] + fields = ['id', 'display', 'name', 'created'] def get_display(self, obj): return f'{obj.name} ({obj.module})' From 3c78047e52121d1cb31c8a98aa9cc87b31bee61b Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 29 Nov 2023 13:15:15 -0800 Subject: [PATCH 85/95] 14132 nested serializer --- netbox/extras/api/nested_serializers.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/netbox/extras/api/nested_serializers.py b/netbox/extras/api/nested_serializers.py index e55c334a245..4e156ef47e7 100644 --- a/netbox/extras/api/nested_serializers.py +++ b/netbox/extras/api/nested_serializers.py @@ -116,10 +116,11 @@ class Meta: fields = ['id', 'url', 'display', 'created'] -class NestedScriptModuleSerializer(serializers.Serializer): +class NestedScriptModuleSerializer(WritableNestedSerializer): class Meta: - fields = ['id', 'display', 'name', 'created'] + model = models.ScriptModule + fields = ['id', 'name', 'created'] def get_display(self, obj): return f'{obj.name} ({obj.module})' From 1085458e66d8dbe190d86c111cfd4423ffe6df5b Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 29 Nov 2023 15:00:02 -0800 Subject: [PATCH 86/95] 14132 add script_full_name --- netbox/extras/forms/model_forms.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/netbox/extras/forms/model_forms.py b/netbox/extras/forms/model_forms.py index 888331bfe52..87d567237f8 100644 --- a/netbox/extras/forms/model_forms.py +++ b/netbox/extras/forms/model_forms.py @@ -331,10 +331,12 @@ def clean(self): self.cleaned_data['action_object_type'] = ContentType.objects.get_for_model(action_choice) self.cleaned_data['action_object_id'] = action_choice.id elif self.cleaned_data.get('action_type') == EventRuleActionChoices.SCRIPT: - script = ScriptModule.objects.get(pk=action_choice.split(":")[0]) - self.cleaned_data['action_object_type'] = ContentType.objects.get_for_model(script, for_concrete_model=False) - self.cleaned_data['action_object_id'] = script.id - self.cleaned_data['action_parameters'] = {'script_choice': action_choice} + module_id, script_name = action_choice.split(":", maxsplit=1) + script_module = ScriptModule.objects.get(pk=module_id) + self.cleaned_data['action_object_type'] = ContentType.objects.get_for_model(script_module, for_concrete_model=False) + self.cleaned_data['action_object_id'] = script_module.id + script = script_module.scripts[script_name]() + self.cleaned_data['action_parameters'] = {'script_choice': action_choice, 'script_full_name': script.full_name} return self.cleaned_data From a16d722fcb64576791e8e4edec36520e4ff49752 Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 29 Nov 2023 15:18:30 -0800 Subject: [PATCH 87/95] 14132 nested serializer --- netbox/extras/api/nested_serializers.py | 12 ++++++++++-- netbox/extras/api/serializers.py | 16 +++++++++++----- netbox/extras/forms/model_forms.py | 6 +++++- 3 files changed, 26 insertions(+), 8 deletions(-) diff --git a/netbox/extras/api/nested_serializers.py b/netbox/extras/api/nested_serializers.py index 4e156ef47e7..f2076f44c08 100644 --- a/netbox/extras/api/nested_serializers.py +++ b/netbox/extras/api/nested_serializers.py @@ -25,7 +25,7 @@ class NestedEventRuleSerializer(WritableNestedSerializer): class Meta: model = models.EventRule - fields = ['id', 'display', 'name'] + fields = ['id', 'url', 'display', 'name'] class NestedWebhookSerializer(WritableNestedSerializer): @@ -117,10 +117,18 @@ class Meta: class NestedScriptModuleSerializer(WritableNestedSerializer): + # url = serializers.SerializerMethodField() + url = serializers.HyperlinkedIdentityField( + view_name='extras-api:script-detail', + lookup_field='full_name', + lookup_url_kwarg='pk' + ) + name = serializers.CharField(read_only=True) + display = serializers.SerializerMethodField(read_only=True) class Meta: model = models.ScriptModule - fields = ['id', 'name', 'created'] + fields = ['id', 'url', 'display', 'name'] def get_display(self, obj): return f'{obj.name} ({obj.module})' diff --git a/netbox/extras/api/serializers.py b/netbox/extras/api/serializers.py index 561c0fc0a16..669bb5175a8 100644 --- a/netbox/extras/api/serializers.py +++ b/netbox/extras/api/serializers.py @@ -83,12 +83,18 @@ class Meta: @extend_schema_field(OpenApiTypes.OBJECT) def get_action_object(self, instance): - serializer = get_serializer_for_model( - model=instance.action_object_type.model_class(), - prefix=NESTED_SERIALIZER_PREFIX - ) context = {'request': self.context['request']} - return serializer(instance.action_object, context=context).data + if instance.action_type == EventRuleActionChoices.WEBHOOK: + serializer = get_serializer_for_model( + model=instance.action_object_type.model_class(), + prefix=NESTED_SERIALIZER_PREFIX + ) + return serializer(instance.action_object, context=context).data + elif instance.action_type == EventRuleActionChoices.SCRIPT: + from extras.api.nested_serializers import NestedScriptModuleSerializer + module_id, script_name = instance.action_parameters['script_choice'].split(":", maxsplit=1) + script = instance.action_object.scripts[script_name]() + return NestedScriptModuleSerializer(script, context=context).data # diff --git a/netbox/extras/forms/model_forms.py b/netbox/extras/forms/model_forms.py index 87d567237f8..5c4d3c0ce0a 100644 --- a/netbox/extras/forms/model_forms.py +++ b/netbox/extras/forms/model_forms.py @@ -336,7 +336,11 @@ def clean(self): self.cleaned_data['action_object_type'] = ContentType.objects.get_for_model(script_module, for_concrete_model=False) self.cleaned_data['action_object_id'] = script_module.id script = script_module.scripts[script_name]() - self.cleaned_data['action_parameters'] = {'script_choice': action_choice, 'script_full_name': script.full_name} + self.cleaned_data['action_parameters'] = { + 'script_choice': action_choice, + 'script_name': script.name, + 'script_full_name': script.full_name, + } return self.cleaned_data From 5d2ada315b69f58e4ba099b29079a5b2f69193b9 Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 29 Nov 2023 15:52:05 -0800 Subject: [PATCH 88/95] 14132 update paramaters --- netbox/extras/forms/bulk_import.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/netbox/extras/forms/bulk_import.py b/netbox/extras/forms/bulk_import.py index ded12260f71..bd4ba23ba37 100644 --- a/netbox/extras/forms/bulk_import.py +++ b/netbox/extras/forms/bulk_import.py @@ -202,7 +202,11 @@ def save(self, *args, **kwargs): module_name, script_name = action_object.split('.', 1) module, script = get_module_and_script(module_name, script_name) self.instance.action_object = module - self.instance.action_parameters = {'script_choice': f"{str(module.pk)}:{script_name}"} + self.instance.action_parameters = { + 'script_choice': f"{str(module.pk)}:{script_name}", + 'script_name': script.name, + 'script_full_name': script.full_name, + } return super().save(*args, **kwargs) From 68d380af54300b64f6d91c10838eac25bc54a73a Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 29 Nov 2023 16:04:14 -0800 Subject: [PATCH 89/95] 14132 update paramaters --- netbox/extras/forms/bulk_import.py | 26 +++++++------------------- 1 file changed, 7 insertions(+), 19 deletions(-) diff --git a/netbox/extras/forms/bulk_import.py b/netbox/extras/forms/bulk_import.py index bd4ba23ba37..bd625e12384 100644 --- a/netbox/extras/forms/bulk_import.py +++ b/netbox/extras/forms/bulk_import.py @@ -183,6 +183,7 @@ def clean(self): webhook = Webhook.objects.filter(name=action_object) if not webhook: raise forms.ValidationError(f"Webhook {action_object} not found") + self.instance.action_object = webhook elif action_type == EventRuleActionChoices.SCRIPT: from extras.scripts import get_module_and_script module_name, script_name = action_object.split('.', 1) @@ -190,25 +191,12 @@ def clean(self): module, script = get_module_and_script(module_name, script_name) except ObjectDoesNotExist: raise forms.ValidationError(f"Script {action_object} not found") - - def save(self, *args, **kwargs): - action_object = self.cleaned_data.get('action_object') - action_type = self.cleaned_data.get('action_type') - - if action_type == EventRuleActionChoices.WEBHOOK: - self.instance.action_object = Webhook.objects.get(name=action_object) - elif action_type == EventRuleActionChoices.SCRIPT: - from extras.scripts import get_module_and_script - module_name, script_name = action_object.split('.', 1) - module, script = get_module_and_script(module_name, script_name) - self.instance.action_object = module - self.instance.action_parameters = { - 'script_choice': f"{str(module.pk)}:{script_name}", - 'script_name': script.name, - 'script_full_name': script.full_name, - } - - return super().save(*args, **kwargs) + self.instance.action_object = module + self.instance.action_parameters = { + 'script_choice': f"{str(module.pk)}:{script_name}", + 'script_name': script.name, + 'script_full_name': script.full_name, + } class TagImportForm(CSVModelForm): From 593b8697cf08aa0139c6b309512ec0a0a3b224c4 Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 29 Nov 2023 16:19:50 -0800 Subject: [PATCH 90/95] 14132 fix csv import --- netbox/extras/forms/bulk_import.py | 1 + 1 file changed, 1 insertion(+) diff --git a/netbox/extras/forms/bulk_import.py b/netbox/extras/forms/bulk_import.py index bd625e12384..64bb4596afc 100644 --- a/netbox/extras/forms/bulk_import.py +++ b/netbox/extras/forms/bulk_import.py @@ -192,6 +192,7 @@ def clean(self): except ObjectDoesNotExist: raise forms.ValidationError(f"Script {action_object} not found") self.instance.action_object = module + self.instance.action_object_type = ContentType.objects.get_for_model(module, for_concrete_model=False) self.instance.action_parameters = { 'script_choice': f"{str(module.pk)}:{script_name}", 'script_name': script.name, From e1c7a03f3ad6a3ab212ebd840ff5687dc011fb93 Mon Sep 17 00:00:00 2001 From: Arthur Date: Wed, 29 Nov 2023 16:32:03 -0800 Subject: [PATCH 91/95] 14132 fix csv import --- netbox/extras/forms/bulk_import.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/netbox/extras/forms/bulk_import.py b/netbox/extras/forms/bulk_import.py index 64bb4596afc..82930e8ad6e 100644 --- a/netbox/extras/forms/bulk_import.py +++ b/netbox/extras/forms/bulk_import.py @@ -180,8 +180,9 @@ def clean(self): action_type = self.cleaned_data.get('action_type') if action_object and action_type: if action_type == EventRuleActionChoices.WEBHOOK: - webhook = Webhook.objects.filter(name=action_object) - if not webhook: + try: + webhook = Webhook.objects.get(name=action_object) + except Webhook.ObjectDoesNotExist: raise forms.ValidationError(f"Webhook {action_object} not found") self.instance.action_object = webhook elif action_type == EventRuleActionChoices.SCRIPT: From 4811c54d7e9ea7473b43764f1dbd3386d0155e74 Mon Sep 17 00:00:00 2001 From: Arthur Date: Thu, 30 Nov 2023 08:32:51 -0800 Subject: [PATCH 92/95] 14132 fix test --- netbox/extras/api/nested_serializers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netbox/extras/api/nested_serializers.py b/netbox/extras/api/nested_serializers.py index f2076f44c08..9963abe7226 100644 --- a/netbox/extras/api/nested_serializers.py +++ b/netbox/extras/api/nested_serializers.py @@ -22,6 +22,7 @@ class NestedEventRuleSerializer(WritableNestedSerializer): + url = serializers.HyperlinkedIdentityField(view_name='extras-api:eventrule-detail') class Meta: model = models.EventRule @@ -117,7 +118,6 @@ class Meta: class NestedScriptModuleSerializer(WritableNestedSerializer): - # url = serializers.SerializerMethodField() url = serializers.HyperlinkedIdentityField( view_name='extras-api:script-detail', lookup_field='full_name', From 78c2dba68f76abe92f61acbd6a3d4050bc43eaa9 Mon Sep 17 00:00:00 2001 From: Arthur Date: Thu, 30 Nov 2023 08:43:46 -0800 Subject: [PATCH 93/95] 14132 fix test --- netbox/extras/tests/test_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netbox/extras/tests/test_api.py b/netbox/extras/tests/test_api.py index 7fe9cea85a1..b35fb8d6615 100644 --- a/netbox/extras/tests/test_api.py +++ b/netbox/extras/tests/test_api.py @@ -71,7 +71,7 @@ def setUpTestData(cls): class EventRuleTest(APIViewTestCases.APIViewTestCase): model = EventRule - brief_fields = ['display', 'id', 'name',] + brief_fields = ['display', 'id', 'name', 'url'] bulk_update_data = { 'enabled': False, 'description': 'New description', From c45bdf48846e766f632ca070eecf8a1a3960acc2 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Thu, 30 Nov 2023 14:54:09 -0500 Subject: [PATCH 94/95] Misc cleanup --- netbox/extras/api/nested_serializers.py | 6 +-- netbox/extras/api/serializers.py | 18 ++++----- netbox/extras/events.py | 3 +- netbox/extras/forms/model_forms.py | 2 +- netbox/extras/tests/test_event_rules.py | 54 +++++++++++++++++-------- netbox/extras/tests/test_views.py | 2 - 6 files changed, 52 insertions(+), 33 deletions(-) diff --git a/netbox/extras/api/nested_serializers.py b/netbox/extras/api/nested_serializers.py index 9963abe7226..4bada494f8b 100644 --- a/netbox/extras/api/nested_serializers.py +++ b/netbox/extras/api/nested_serializers.py @@ -15,7 +15,7 @@ 'NestedImageAttachmentSerializer', 'NestedJournalEntrySerializer', 'NestedSavedFilterSerializer', - 'NestedScriptModuleSerializer', + 'NestedScriptSerializer', 'NestedTagSerializer', # Defined in netbox.api.serializers 'NestedWebhookSerializer', ] @@ -117,7 +117,7 @@ class Meta: fields = ['id', 'url', 'display', 'created'] -class NestedScriptModuleSerializer(WritableNestedSerializer): +class NestedScriptSerializer(WritableNestedSerializer): url = serializers.HyperlinkedIdentityField( view_name='extras-api:script-detail', lookup_field='full_name', @@ -127,7 +127,7 @@ class NestedScriptModuleSerializer(WritableNestedSerializer): display = serializers.SerializerMethodField(read_only=True) class Meta: - model = models.ScriptModule + model = models.Script fields = ['id', 'url', 'display', 'name'] def get_display(self, obj): diff --git a/netbox/extras/api/serializers.py b/netbox/extras/api/serializers.py index 669bb5175a8..82b3e1933a6 100644 --- a/netbox/extras/api/serializers.py +++ b/netbox/extras/api/serializers.py @@ -1,17 +1,17 @@ from django.contrib.auth import get_user_model from django.core.exceptions import ObjectDoesNotExist +from drf_spectacular.types import OpenApiTypes +from drf_spectacular.utils import extend_schema_field from rest_framework import serializers -from core.api.serializers import JobSerializer from core.api.nested_serializers import NestedDataSourceSerializer, NestedDataFileSerializer, NestedJobSerializer +from core.api.serializers import JobSerializer from core.models import ContentType from dcim.api.nested_serializers import ( NestedDeviceRoleSerializer, NestedDeviceTypeSerializer, NestedLocationSerializer, NestedPlatformSerializer, NestedRegionSerializer, NestedSiteSerializer, NestedSiteGroupSerializer, ) from dcim.models import DeviceRole, DeviceType, Location, Platform, Region, Site, SiteGroup -from drf_spectacular.utils import extend_schema_field -from drf_spectacular.types import OpenApiTypes from extras.choices import * from extras.models import * from netbox.api.exceptions import SerializerNotFound @@ -84,17 +84,17 @@ class Meta: @extend_schema_field(OpenApiTypes.OBJECT) def get_action_object(self, instance): context = {'request': self.context['request']} - if instance.action_type == EventRuleActionChoices.WEBHOOK: + # We need to manually instantiate the serializer for scripts + if instance.action_type == EventRuleActionChoices.SCRIPT: + module_id, script_name = instance.action_parameters['script_choice'].split(":", maxsplit=1) + script = instance.action_object.scripts[script_name]() + return NestedScriptSerializer(script, context=context).data + else: serializer = get_serializer_for_model( model=instance.action_object_type.model_class(), prefix=NESTED_SERIALIZER_PREFIX ) return serializer(instance.action_object, context=context).data - elif instance.action_type == EventRuleActionChoices.SCRIPT: - from extras.api.nested_serializers import NestedScriptModuleSerializer - module_id, script_name = instance.action_parameters['script_choice'].split(":", maxsplit=1) - script = instance.action_object.scripts[script_name]() - return NestedScriptModuleSerializer(script, context=context).data # diff --git a/netbox/extras/events.py b/netbox/extras/events.py index 96d90521b0e..709a99cfaa8 100644 --- a/netbox/extras/events.py +++ b/netbox/extras/events.py @@ -1,11 +1,10 @@ import logging -import sys from django.conf import settings from django.contrib.contenttypes.models import ContentType -from django_rq import get_queue from django.utils import timezone from django.utils.module_loading import import_string +from django_rq import get_queue from netbox.config import get_config from netbox.constants import RQ_QUEUE_DEFAULT diff --git a/netbox/extras/forms/model_forms.py b/netbox/extras/forms/model_forms.py index 5c4d3c0ce0a..0c717246f2b 100644 --- a/netbox/extras/forms/model_forms.py +++ b/netbox/extras/forms/model_forms.py @@ -250,7 +250,7 @@ class EventRuleForm(NetBoxModelForm): ) fieldsets = ( - (_('EventRule'), ('name', 'description', 'content_types', 'enabled', 'tags')), + (_('Event Rule'), ('name', 'description', 'content_types', 'enabled', 'tags')), (_('Events'), ('type_create', 'type_update', 'type_delete', 'type_job_start', 'type_job_end')), (_('Conditions'), ('conditions',)), (_('Action'), ( diff --git a/netbox/extras/tests/test_event_rules.py b/netbox/extras/tests/test_event_rules.py index 9bed872b791..ed64ba89144 100644 --- a/netbox/extras/tests/test_event_rules.py +++ b/netbox/extras/tests/test_event_rules.py @@ -73,10 +73,12 @@ def setUpTestData(cls): Tag(name='Baz', slug='baz'), )) - def test_event_rule_conditions(self): - # Create a conditional Webhook + def test_eventrule_conditions(self): + """ + Test evaluation of EventRule conditions. + """ event_rule = EventRule( - name='Conditional Webhook', + name='Event Rule 1', type_create=True, type_update=True, conditions={ @@ -103,7 +105,10 @@ def test_event_rule_conditions(self): # Evaluate the conditions (status='active') self.assertTrue(event_rule.eval_conditions(data)) - def test_enqueue_webhook_create(self): + def test_single_create_process_eventrule(self): + """ + Check that creating an object with an applicable EventRule queues a background task for the rule's action. + """ # Create an object via the REST API data = { 'name': 'Site 1', @@ -120,7 +125,7 @@ def test_enqueue_webhook_create(self): self.assertEqual(Site.objects.count(), 1) self.assertEqual(Site.objects.first().tags.count(), 2) - # Verify that a job was queued for the object creation webhook + # Verify that a background task was queued for the new object self.assertEqual(self.queue.count, 1) job = self.queue.jobs[0] self.assertEqual(job.kwargs['event_rule'], EventRule.objects.get(type_create=True)) @@ -131,7 +136,11 @@ def test_enqueue_webhook_create(self): self.assertEqual(job.kwargs['snapshots']['postchange']['name'], 'Site 1') self.assertEqual(job.kwargs['snapshots']['postchange']['tags'], ['Bar', 'Foo']) - def test_enqueue_webhook_bulk_create(self): + def test_bulk_create_process_eventrule(self): + """ + Check that bulk creating multiple objects with an applicable EventRule queues a background task for each + new object. + """ # Create multiple objects via the REST API data = [ { @@ -166,7 +175,7 @@ def test_enqueue_webhook_bulk_create(self): self.assertEqual(Site.objects.count(), 3) self.assertEqual(Site.objects.first().tags.count(), 2) - # Verify that a webhook was queued for each object + # Verify that a background task was queued for each new object self.assertEqual(self.queue.count, 3) for i, job in enumerate(self.queue.jobs): self.assertEqual(job.kwargs['event_rule'], EventRule.objects.get(type_create=True)) @@ -177,7 +186,10 @@ def test_enqueue_webhook_bulk_create(self): self.assertEqual(job.kwargs['snapshots']['postchange']['name'], response.data[i]['name']) self.assertEqual(job.kwargs['snapshots']['postchange']['tags'], ['Bar', 'Foo']) - def test_enqueue_webhook_update(self): + def test_single_update_process_eventrule(self): + """ + Check that updating an object with an applicable EventRule queues a background task for the rule's action. + """ site = Site.objects.create(name='Site 1', slug='site-1') site.tags.set(Tag.objects.filter(name__in=['Foo', 'Bar'])) @@ -194,7 +206,7 @@ def test_enqueue_webhook_update(self): response = self.client.patch(url, data, format='json', **self.header) self.assertHttpStatus(response, status.HTTP_200_OK) - # Verify that a job was queued for the object update webhook + # Verify that a background task was queued for the updated object self.assertEqual(self.queue.count, 1) job = self.queue.jobs[0] self.assertEqual(job.kwargs['event_rule'], EventRule.objects.get(type_update=True)) @@ -207,7 +219,11 @@ def test_enqueue_webhook_update(self): self.assertEqual(job.kwargs['snapshots']['postchange']['name'], 'Site X') self.assertEqual(job.kwargs['snapshots']['postchange']['tags'], ['Baz']) - def test_enqueue_webhook_bulk_update(self): + def test_bulk_update_process_eventrule(self): + """ + Check that bulk updating multiple objects with an applicable EventRule queues a background task for each + updated object. + """ sites = ( Site(name='Site 1', slug='site-1'), Site(name='Site 2', slug='site-2'), @@ -246,7 +262,7 @@ def test_enqueue_webhook_bulk_update(self): response = self.client.patch(url, data, format='json', **self.header) self.assertHttpStatus(response, status.HTTP_200_OK) - # Verify that a job was queued for the object update webhook + # Verify that a background task was queued for each updated object self.assertEqual(self.queue.count, 3) for i, job in enumerate(self.queue.jobs): self.assertEqual(job.kwargs['event_rule'], EventRule.objects.get(type_update=True)) @@ -259,7 +275,10 @@ def test_enqueue_webhook_bulk_update(self): self.assertEqual(job.kwargs['snapshots']['postchange']['name'], response.data[i]['name']) self.assertEqual(job.kwargs['snapshots']['postchange']['tags'], ['Baz']) - def test_enqueue_webhook_delete(self): + def test_single_delete_process_eventrule(self): + """ + Check that deleting an object with an applicable EventRule queues a background task for the rule's action. + """ site = Site.objects.create(name='Site 1', slug='site-1') site.tags.set(Tag.objects.filter(name__in=['Foo', 'Bar'])) @@ -269,7 +288,7 @@ def test_enqueue_webhook_delete(self): response = self.client.delete(url, **self.header) self.assertHttpStatus(response, status.HTTP_204_NO_CONTENT) - # Verify that a job was queued for the object update webhook + # Verify that a task was queued for the deleted object self.assertEqual(self.queue.count, 1) job = self.queue.jobs[0] self.assertEqual(job.kwargs['event_rule'], EventRule.objects.get(type_delete=True)) @@ -279,7 +298,11 @@ def test_enqueue_webhook_delete(self): self.assertEqual(job.kwargs['snapshots']['prechange']['name'], 'Site 1') self.assertEqual(job.kwargs['snapshots']['prechange']['tags'], ['Bar', 'Foo']) - def test_enqueue_webhook_bulk_delete(self): + def test_bulk_delete_process_eventrule(self): + """ + Check that bulk deleting multiple objects with an applicable EventRule queues a background task for each + deleted object. + """ sites = ( Site(name='Site 1', slug='site-1'), Site(name='Site 2', slug='site-2'), @@ -298,7 +321,7 @@ def test_enqueue_webhook_bulk_delete(self): response = self.client.delete(url, data, format='json', **self.header) self.assertHttpStatus(response, status.HTTP_204_NO_CONTENT) - # Verify that a job was queued for the object update webhook + # Verify that a background task was queued for each deleted object self.assertEqual(self.queue.count, 3) for i, job in enumerate(self.queue.jobs): self.assertEqual(job.kwargs['event_rule'], EventRule.objects.get(type_delete=True)) @@ -309,7 +332,6 @@ def test_enqueue_webhook_bulk_delete(self): self.assertEqual(job.kwargs['snapshots']['prechange']['tags'], ['Bar', 'Foo']) def test_webhooks_worker(self): - request_id = uuid.uuid4() def dummy_send(_, request, **kwargs): diff --git a/netbox/extras/tests/test_views.py b/netbox/extras/tests/test_views.py index 37773932a99..602a9d4deb8 100644 --- a/netbox/extras/tests/test_views.py +++ b/netbox/extras/tests/test_views.py @@ -1,4 +1,3 @@ -import json import urllib.parse import uuid @@ -11,7 +10,6 @@ from extras.models import * from utilities.testing import ViewTestCases, TestCase - User = get_user_model() From a9b4a81421eca3e6aad80d66dd4e36883b7fe7ad Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Thu, 30 Nov 2023 16:02:56 -0500 Subject: [PATCH 95/95] Optimize script jobs by skipping the intermediate event task --- netbox/extras/events.py | 87 ++++++++++++++++++++------------ netbox/extras/scripts_worker.py | 54 -------------------- netbox/extras/webhooks_worker.py | 4 -- 3 files changed, 56 insertions(+), 89 deletions(-) delete mode 100644 netbox/extras/scripts_worker.py diff --git a/netbox/extras/events.py b/netbox/extras/events.py index 709a99cfaa8..05352b7d197 100644 --- a/netbox/extras/events.py +++ b/netbox/extras/events.py @@ -1,11 +1,14 @@ import logging from django.conf import settings +from django.contrib.auth import get_user_model from django.contrib.contenttypes.models import ContentType +from django.core.exceptions import ObjectDoesNotExist from django.utils import timezone from django.utils.module_loading import import_string from django_rq import get_queue +from core.models import Job from netbox.config import get_config from netbox.constants import RQ_QUEUE_DEFAULT from netbox.registry import registry @@ -13,7 +16,7 @@ from utilities.rqworker import get_rq_retry from utilities.utils import serialize_object from .choices import * -from .models import EventRule +from .models import EventRule, ScriptModule logger = logging.getLogger('netbox.events_processor') @@ -69,43 +72,65 @@ def enqueue_object(queue, instance, user, request_id, action): def process_event_rules(event_rules, model_name, event, data, username, snapshots=None, request_id=None): + try: + user = get_user_model().objects.get(username=username) + except ObjectDoesNotExist: + user = None for event_rule in event_rules: + + # Evaluate event rule conditions (if any) + if not event_rule.eval_conditions(data): + return + + # Webhooks if event_rule.action_type == EventRuleActionChoices.WEBHOOK: - processor = "extras.webhooks_worker.process_webhook" - queue_class = 'webhook' + + # Select the appropriate RQ queue + queue_name = get_config().QUEUE_MAPPINGS.get('webhook', RQ_QUEUE_DEFAULT) + rq_queue = get_queue(queue_name) + + # Compile the task parameters + params = { + "event_rule": event_rule, + "model_name": model_name, + "event": event, + "data": data, + "snapshots": snapshots, + "timestamp": timezone.now().isoformat(), + "username": username, + "retry": get_rq_retry() + } + if snapshots: + params["snapshots"] = snapshots + if request_id: + params["request_id"] = request_id + + # Enqueue the task + rq_queue.enqueue( + "extras.webhooks_worker.process_webhook", + **params + ) + + # Scripts elif event_rule.action_type == EventRuleActionChoices.SCRIPT: - processor = "extras.scripts_worker.process_script" - queue_class = 'script' + # Resolve the script from action parameters + script_module = event_rule.action_object + _, script_name = event_rule.action_parameters['script_choice'].split(":", maxsplit=1) + script = script_module.scripts[script_name]() + + # Enqueue a Job to record the script's execution + Job.enqueue( + "extras.scripts.run_script", + instance=script_module, + name=script.class_name, + user=user, + data=data + ) + else: raise ValueError(f"Unknown action type for an event rule: {event_rule.action_type}") - # Select the appropriate RQ queue based on the action object type - queue_name = get_config().QUEUE_MAPPINGS.get(queue_class, RQ_QUEUE_DEFAULT) - rq_queue = get_queue(queue_name) - - # Compile the task parameters - params = { - "event_rule": event_rule, - "model_name": model_name, - "event": event, - "data": data, - "snapshots": snapshots, - "timestamp": timezone.now().isoformat(), - "username": username, - "retry": get_rq_retry() - } - if snapshots: - params["snapshots"] = snapshots - if request_id: - params["request_id"] = request_id - - # Enqueue the task - rq_queue.enqueue( - processor, - **params - ) - def process_event_queue(events): """ diff --git a/netbox/extras/scripts_worker.py b/netbox/extras/scripts_worker.py deleted file mode 100644 index 1e70a0f8abf..00000000000 --- a/netbox/extras/scripts_worker.py +++ /dev/null @@ -1,54 +0,0 @@ -import logging - -from django.contrib.auth import get_user_model -from django.core.exceptions import ObjectDoesNotExist -from django_rq import job - -from core.models import Job -from extras.models import ScriptModule -from extras.scripts import run_script - -logger = logging.getLogger('netbox.scripts_worker') - - -@job('default') -def process_script(event_rule, data, username, **kwargs): - """ - Run the requested script - """ - if not event_rule.eval_conditions(data): - return - - script_choice = None - if event_rule.action_parameters and 'script_choice' in event_rule.action_parameters: - script_choice = event_rule.action_parameters['script_choice'] - - if script_choice: - module_id, script_name = script_choice.split(":", maxsplit=1) - else: - logger.warning(f"event run script - event_rule: {event_rule.id} no script_choice selected") - return - - try: - module = ScriptModule.objects.get(pk=module_id) - except ScriptModule.DoesNotExist: - logger.warning(f"event run script - script module_id: {module_id} script_name: {script_name}") - return - - try: - user = get_user_model().objects.get(username=username) - except ObjectDoesNotExist: - logger.warning(f"event run script - user does not exist username: {username} script_name: {script_name}") - return - - script = module.scripts[script_name]() - - Job.enqueue( - run_script, - instance=module, - name=script.class_name, - user=user, - schedule_at=None, - interval=None, - data=event_rule.action_data, - ) diff --git a/netbox/extras/webhooks_worker.py b/netbox/extras/webhooks_worker.py index 5e3648bf910..4d6d8135e2c 100644 --- a/netbox/extras/webhooks_worker.py +++ b/netbox/extras/webhooks_worker.py @@ -16,10 +16,6 @@ def process_webhook(event_rule, model_name, event, data, timestamp, username, re """ Make a POST request to the defined Webhook """ - - if not event_rule.eval_conditions(data): - return - webhook = event_rule.action_object # Prepare context data for headers & body templates