From 40572b543fdcdefaaa3da742141771a4d55aeea0 Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Mon, 27 Mar 2023 11:43:12 -0400 Subject: [PATCH] Rename JobResult to Job and move to core --- docs/development/models.md | 2 +- docs/features/background-jobs.md | 2 +- .../{extras/jobresult.md => core/job.md} | 4 +- mkdocs.yml | 2 +- netbox/core/api/nested_serializers.py | 20 +- netbox/core/api/serializers.py | 25 +- netbox/core/api/urls.py | 3 + netbox/core/api/views.py | 14 +- netbox/core/choices.py | 29 +++ netbox/core/filtersets.py | 62 ++++- netbox/core/forms/filtersets.py | 70 +++++- netbox/core/jobs.py | 4 +- .../migrations/0003_move_jobresult_to_core.py | 40 ++++ netbox/core/models/__init__.py | 1 + netbox/core/models/data.py | 5 +- netbox/core/models/jobs.py | 219 ++++++++++++++++++ netbox/core/tables/__init__.py | 1 + netbox/core/tables/jobs.py | 34 +++ netbox/core/urls.py | 5 + netbox/core/views.py | 22 ++ netbox/extras/admin.py | 2 +- netbox/extras/api/nested_serializers.py | 17 +- netbox/extras/api/serializers.py | 34 +-- netbox/extras/api/urls.py | 1 - netbox/extras/api/views.py | 60 ++--- netbox/extras/filtersets.py | 64 ----- netbox/extras/forms/filtersets.py | 66 +----- .../management/commands/housekeeping.py | 10 +- .../extras/management/commands/runreport.py | 21 +- .../extras/management/commands/runscript.py | 12 +- netbox/extras/migrations/0001_squashed.py | 2 +- netbox/extras/models/models.py | 2 +- netbox/extras/models/reports.py | 4 +- netbox/extras/models/scripts.py | 4 +- netbox/extras/reports.py | 18 +- netbox/extras/scripts.py | 10 +- netbox/extras/tables/tables.py | 31 --- netbox/extras/urls.py | 5 - netbox/extras/views.py | 74 ++---- netbox/netbox/models/features.py | 6 +- netbox/netbox/navigation/menu.py | 4 +- 41 files changed, 650 insertions(+), 361 deletions(-) rename docs/models/{extras/jobresult.md => core/job.md} (89%) create mode 100644 netbox/core/migrations/0003_move_jobresult_to_core.py create mode 100644 netbox/core/models/jobs.py create mode 100644 netbox/core/tables/jobs.py diff --git a/docs/development/models.md b/docs/development/models.md index 6f3998977a6..1131738912c 100644 --- a/docs/development/models.md +++ b/docs/development/models.md @@ -18,7 +18,7 @@ Depending on its classification, each NetBox model may support various features | [Custom links](../customization/custom-links.md) | `CustomLinksMixin` | `custom_links` | These models support the assignment of custom links | | [Custom validation](../customization/custom-validation.md) | `CustomValidationMixin` | - | Supports the enforcement of custom validation rules | | [Export templates](../customization/export-templates.md) | `ExportTemplatesMixin` | `export_templates` | Users can create custom export templates for these models | -| [Job results](../features/background-jobs.md) | `JobResultsMixin` | `job_results` | Users can create custom export templates for these models | +| [Job results](../features/background-jobs.md) | `JobsMixin` | `jobs` | Users can create custom export templates for these models | | [Journaling](../features/journaling.md) | `JournalingMixin` | `journaling` | These models support persistent historical commentary | | [Synchronized data](../integrations/synchronized-data.md) | `SyncedDataMixin` | `synced_data` | Certain model data can be automatically synchronized from a remote data source | | [Tagging](../models/extras/tag.md) | `TagsMixin` | `tags` | The models can be tagged with user-defined tags | diff --git a/docs/features/background-jobs.md b/docs/features/background-jobs.md index a36192ab3a6..204951ba74b 100644 --- a/docs/features/background-jobs.md +++ b/docs/features/background-jobs.md @@ -6,7 +6,7 @@ NetBox includes the ability to execute certain functions as background tasks. Th * [Custom script](../customization/custom-scripts.md) execution * Synchronization of [remote data sources](../integrations/synchronized-data.md) -Additionally, NetBox plugins can enqueue their own background tasks. This is accomplished using the [JobResult model](../models/extras/jobresult.md). Background tasks are executed by the `rqworker` process(es). +Additionally, NetBox plugins can enqueue their own background tasks. This is accomplished using the [Job model](../models/core/job.md). Background tasks are executed by the `rqworker` process(es). ## Scheduled Jobs diff --git a/docs/models/extras/jobresult.md b/docs/models/core/job.md similarity index 89% rename from docs/models/extras/jobresult.md rename to docs/models/core/job.md index 81ab75745a4..05ed53024df 100644 --- a/docs/models/extras/jobresult.md +++ b/docs/models/core/job.md @@ -1,6 +1,6 @@ -# Job Results +# Jobs -The JobResult model is used to schedule and record the execution of [background tasks](../../features/background-jobs.md). +The Job model is used to schedule and record the execution of [background tasks](../../features/background-jobs.md). ## Fields diff --git a/mkdocs.yml b/mkdocs.yml index b5ce9bf00f2..2b594f92450 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -159,6 +159,7 @@ nav: - Core: - DataFile: 'models/core/datafile.md' - DataSource: 'models/core/datasource.md' + - Job: 'models/core/job.md' - DCIM: - Cable: 'models/dcim/cable.md' - ConsolePort: 'models/dcim/consoleport.md' @@ -208,7 +209,6 @@ nav: - CustomLink: 'models/extras/customlink.md' - ExportTemplate: 'models/extras/exporttemplate.md' - ImageAttachment: 'models/extras/imageattachment.md' - - JobResult: 'models/extras/jobresult.md' - JournalEntry: 'models/extras/journalentry.md' - SavedFilter: 'models/extras/savedfilter.md' - StagedChange: 'models/extras/stagedchange.md' diff --git a/netbox/core/api/nested_serializers.py b/netbox/core/api/nested_serializers.py index 0a8351fec79..d99738cbe2a 100644 --- a/netbox/core/api/nested_serializers.py +++ b/netbox/core/api/nested_serializers.py @@ -1,12 +1,16 @@ from rest_framework import serializers +from core.choices import JobStatusChoices from core.models import * +from netbox.api.fields import ChoiceField from netbox.api.serializers import WritableNestedSerializer +from users.api.nested_serializers import NestedUserSerializer -__all__ = [ +__all__ = ( 'NestedDataFileSerializer', 'NestedDataSourceSerializer', -] + 'NestedJobSerializer', +) class NestedDataSourceSerializer(WritableNestedSerializer): @@ -23,3 +27,15 @@ class NestedDataFileSerializer(WritableNestedSerializer): class Meta: model = DataFile fields = ['id', 'url', 'display', 'path'] + + +class NestedJobSerializer(serializers.ModelSerializer): + url = serializers.HyperlinkedIdentityField(view_name='core-api:job-detail') + status = ChoiceField(choices=JobStatusChoices) + user = NestedUserSerializer( + read_only=True + ) + + class Meta: + model = Job + fields = ['url', 'created', 'completed', 'user', 'status'] diff --git a/netbox/core/api/serializers.py b/netbox/core/api/serializers.py index 4c29fd69ea1..aa0cc705097 100644 --- a/netbox/core/api/serializers.py +++ b/netbox/core/api/serializers.py @@ -2,12 +2,15 @@ from core.choices import * from core.models import * -from netbox.api.fields import ChoiceField -from netbox.api.serializers import NetBoxModelSerializer +from netbox.api.fields import ChoiceField, ContentTypeField +from netbox.api.serializers import BaseModelSerializer, NetBoxModelSerializer +from users.api.nested_serializers import NestedUserSerializer from .nested_serializers import * __all__ = ( + 'DataFileSerializer', 'DataSourceSerializer', + 'JobSerializer', ) @@ -49,3 +52,21 @@ class Meta: fields = [ 'id', 'url', 'display', 'source', 'path', 'last_updated', 'size', 'hash', ] + + +class JobSerializer(BaseModelSerializer): + url = serializers.HyperlinkedIdentityField(view_name='core-api:job-detail') + user = NestedUserSerializer( + read_only=True + ) + status = ChoiceField(choices=JobStatusChoices, read_only=True) + object_type = ContentTypeField( + read_only=True + ) + + class Meta: + model = Job + fields = [ + 'id', 'url', 'display', 'status', 'created', 'scheduled', 'interval', 'started', 'completed', 'name', + 'object_type', 'user', 'data', 'job_id', + ] diff --git a/netbox/core/api/urls.py b/netbox/core/api/urls.py index 364e5db5559..4546b67d44b 100644 --- a/netbox/core/api/urls.py +++ b/netbox/core/api/urls.py @@ -9,5 +9,8 @@ router.register('data-sources', views.DataSourceViewSet) router.register('data-files', views.DataFileViewSet) +# Jobs +router.register('job-results', views.JobViewSet) + app_name = 'core-api' urlpatterns = router.urls diff --git a/netbox/core/api/views.py b/netbox/core/api/views.py index b2d8c0ed461..fc4ef2927c8 100644 --- a/netbox/core/api/views.py +++ b/netbox/core/api/views.py @@ -4,6 +4,7 @@ from rest_framework.exceptions import PermissionDenied from rest_framework.response import Response from rest_framework.routers import APIRootView +from rest_framework.viewsets import ReadOnlyModelViewSet from core import filtersets from core.models import * @@ -20,10 +21,6 @@ def get_view_name(self): return 'Core' -# -# Data sources -# - class DataSourceViewSet(NetBoxModelViewSet): queryset = DataSource.objects.annotate( file_count=count_related(DataFile, 'source') @@ -50,3 +47,12 @@ class DataFileViewSet(NetBoxReadOnlyModelViewSet): queryset = DataFile.objects.defer('data').prefetch_related('source') serializer_class = serializers.DataFileSerializer filterset_class = filtersets.DataFileFilterSet + + +class JobViewSet(ReadOnlyModelViewSet): + """ + Retrieve a list of job results + """ + queryset = Job.objects.prefetch_related('user') + serializer_class = serializers.JobSerializer + filterset_class = filtersets.JobFilterSet diff --git a/netbox/core/choices.py b/netbox/core/choices.py index 2b3d90a4284..8a676a98e7d 100644 --- a/netbox/core/choices.py +++ b/netbox/core/choices.py @@ -47,3 +47,32 @@ class ManagedFileRootPathChoices(ChoiceSet): (SCRIPTS, _('Scripts')), (REPORTS, _('Reports')), ) + + +# +# Jobs +# + +class JobStatusChoices(ChoiceSet): + + STATUS_PENDING = 'pending' + STATUS_SCHEDULED = 'scheduled' + STATUS_RUNNING = 'running' + STATUS_COMPLETED = 'completed' + STATUS_ERRORED = 'errored' + STATUS_FAILED = 'failed' + + CHOICES = ( + (STATUS_PENDING, 'Pending', 'cyan'), + (STATUS_SCHEDULED, 'Scheduled', 'gray'), + (STATUS_RUNNING, 'Running', 'blue'), + (STATUS_COMPLETED, 'Completed', 'green'), + (STATUS_ERRORED, 'Errored', 'red'), + (STATUS_FAILED, 'Failed', 'red'), + ) + + TERMINAL_STATE_CHOICES = ( + STATUS_COMPLETED, + STATUS_ERRORED, + STATUS_FAILED, + ) diff --git a/netbox/core/filtersets.py b/netbox/core/filtersets.py index 3bff3415887..6d3e82e15dd 100644 --- a/netbox/core/filtersets.py +++ b/netbox/core/filtersets.py @@ -3,13 +3,14 @@ import django_filters -from netbox.filtersets import ChangeLoggedModelFilterSet, NetBoxModelFilterSet +from netbox.filtersets import BaseFilterSet, ChangeLoggedModelFilterSet, NetBoxModelFilterSet from .choices import * from .models import * __all__ = ( 'DataFileFilterSet', 'DataSourceFilterSet', + 'JobFilterSet', ) @@ -62,3 +63,62 @@ def search(self, queryset, name, value): return queryset.filter( Q(path__icontains=value) ) + + +class JobFilterSet(BaseFilterSet): + q = django_filters.CharFilter( + method='search', + label=_('Search'), + ) + created = django_filters.DateTimeFilter() + created__before = django_filters.DateTimeFilter( + field_name='created', + lookup_expr='lte' + ) + created__after = django_filters.DateTimeFilter( + field_name='created', + lookup_expr='gte' + ) + scheduled = django_filters.DateTimeFilter() + scheduled__before = django_filters.DateTimeFilter( + field_name='scheduled', + lookup_expr='lte' + ) + scheduled__after = django_filters.DateTimeFilter( + field_name='scheduled', + lookup_expr='gte' + ) + started = django_filters.DateTimeFilter() + started__before = django_filters.DateTimeFilter( + field_name='started', + lookup_expr='lte' + ) + started__after = django_filters.DateTimeFilter( + field_name='started', + lookup_expr='gte' + ) + completed = django_filters.DateTimeFilter() + completed__before = django_filters.DateTimeFilter( + field_name='completed', + lookup_expr='lte' + ) + completed__after = django_filters.DateTimeFilter( + field_name='completed', + lookup_expr='gte' + ) + status = django_filters.MultipleChoiceFilter( + choices=JobStatusChoices, + null_value=None + ) + + class Meta: + model = Job + fields = ('id', 'interval', 'status', 'user', 'object_type', 'name') + + def search(self, queryset, name, value): + if not value.strip(): + return queryset + return queryset.filter( + Q(user__username__icontains=value) | + Q(name__icontains=value) + ) diff --git a/netbox/core/forms/filtersets.py b/netbox/core/forms/filtersets.py index a549415370d..ee8faa125d3 100644 --- a/netbox/core/forms/filtersets.py +++ b/netbox/core/forms/filtersets.py @@ -1,14 +1,22 @@ from django import forms +from django.contrib.auth.models import User +from django.contrib.contenttypes.models import ContentType from django.utils.translation import gettext as _ from core.choices import * from core.models import * +from extras.forms.mixins import SavedFiltersMixin +from extras.utils import FeatureQuery from netbox.forms import NetBoxModelFilterSetForm -from utilities.forms import BOOLEAN_WITH_BLANK_CHOICES, DynamicModelMultipleChoiceField +from utilities.forms import ( + APISelectMultiple, BOOLEAN_WITH_BLANK_CHOICES, ContentTypeChoiceField, DateTimePicker, + DynamicModelMultipleChoiceField, FilterForm, +) __all__ = ( 'DataFileFilterForm', 'DataSourceFilterForm', + 'JobFilterForm', ) @@ -45,3 +53,63 @@ class DataFileFilterForm(NetBoxModelFilterSetForm): required=False, label=_('Data source') ) + + +class JobFilterForm(SavedFiltersMixin, FilterForm): + fieldsets = ( + (None, ('q', 'filter_id')), + ('Attributes', ('object_type', 'status')), + ('Creation', ( + 'created__before', 'created__after', 'scheduled__before', 'scheduled__after', 'started__before', + 'started__after', 'completed__before', 'completed__after', 'user', + )), + ) + object_type = ContentTypeChoiceField( + label=_('Object Type'), + queryset=ContentType.objects.filter(FeatureQuery('jobs').get_query()), + required=False, + ) + status = forms.MultipleChoiceField( + choices=JobStatusChoices, + required=False + ) + created__after = forms.DateTimeField( + required=False, + widget=DateTimePicker() + ) + created__before = forms.DateTimeField( + required=False, + widget=DateTimePicker() + ) + scheduled__after = forms.DateTimeField( + required=False, + widget=DateTimePicker() + ) + scheduled__before = forms.DateTimeField( + required=False, + widget=DateTimePicker() + ) + started__after = forms.DateTimeField( + required=False, + widget=DateTimePicker() + ) + started__before = forms.DateTimeField( + required=False, + widget=DateTimePicker() + ) + completed__after = forms.DateTimeField( + required=False, + widget=DateTimePicker() + ) + completed__before = forms.DateTimeField( + required=False, + widget=DateTimePicker() + ) + user = DynamicModelMultipleChoiceField( + queryset=User.objects.all(), + required=False, + label=_('User'), + widget=APISelectMultiple( + api_url='/api/users/users/', + ) + ) diff --git a/netbox/core/jobs.py b/netbox/core/jobs.py index 8ef8c4e727f..fd31347e303 100644 --- a/netbox/core/jobs.py +++ b/netbox/core/jobs.py @@ -1,6 +1,6 @@ import logging -from extras.choices import JobResultStatusChoices +from .choices import JobStatusChoices from netbox.search.backends import search_backend from .choices import * from .exceptions import SyncError @@ -25,6 +25,6 @@ def sync_datasource(job_result, *args, **kwargs): job_result.terminate() except SyncError as e: - job_result.terminate(status=JobResultStatusChoices.STATUS_ERRORED) + job_result.terminate(status=JobStatusChoices.STATUS_ERRORED) DataSource.objects.filter(pk=datasource.pk).update(status=DataSourceStatusChoices.FAILED) logging.error(e) diff --git a/netbox/core/migrations/0003_move_jobresult_to_core.py b/netbox/core/migrations/0003_move_jobresult_to_core.py new file mode 100644 index 00000000000..cf1b9e74386 --- /dev/null +++ b/netbox/core/migrations/0003_move_jobresult_to_core.py @@ -0,0 +1,40 @@ +# Generated by Django 4.1.7 on 2023-03-27 15:02 + +from django.conf import settings +import django.core.validators +from django.db import migrations, models +import django.db.models.deletion +import extras.utils + + +class Migration(migrations.Migration): + + dependencies = [ + ('contenttypes', '0002_remove_content_type_name'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('core', '0002_managedfile'), + ] + + operations = [ + migrations.CreateModel( + name='Job', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False)), + ('object_id', models.PositiveBigIntegerField(blank=True, null=True)), + ('name', models.CharField(max_length=200)), + ('created', models.DateTimeField(auto_now_add=True)), + ('scheduled', models.DateTimeField(blank=True, null=True)), + ('interval', models.PositiveIntegerField(blank=True, null=True, validators=[django.core.validators.MinValueValidator(1)])), + ('started', models.DateTimeField(blank=True, null=True)), + ('completed', models.DateTimeField(blank=True, null=True)), + ('status', models.CharField(default='pending', max_length=30)), + ('data', models.JSONField(blank=True, null=True)), + ('job_id', models.UUIDField(unique=True)), + ('object_type', models.ForeignKey(limit_choices_to=extras.utils.FeatureQuery('jobs'), on_delete=django.db.models.deletion.CASCADE, related_name='jobs', to='contenttypes.contenttype')), + ('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to=settings.AUTH_USER_MODEL)), + ], + options={ + 'ordering': ['-created'], + }, + ), + ] diff --git a/netbox/core/models/__init__.py b/netbox/core/models/__init__.py index e21b9f8cf43..185622f5f28 100644 --- a/netbox/core/models/__init__.py +++ b/netbox/core/models/__init__.py @@ -1,2 +1,3 @@ from .data import * from .files import * +from .jobs import * diff --git a/netbox/core/models/data.py b/netbox/core/models/data.py index cbf29ae3833..a4422ac7900 100644 --- a/netbox/core/models/data.py +++ b/netbox/core/models/data.py @@ -21,6 +21,7 @@ from ..choices import * from ..exceptions import SyncError from ..signals import post_sync, pre_sync +from .jobs import Job __all__ = ( 'DataFile', @@ -112,14 +113,12 @@ def enqueue_sync_job(self, request): """ Enqueue a background job to synchronize the DataSource by calling sync(). """ - from extras.models import JobResult - # Set the status to "syncing" self.status = DataSourceStatusChoices.QUEUED DataSource.objects.filter(pk=self.pk).update(status=self.status) # Enqueue a sync job - job_result = JobResult.enqueue_job( + job_result = Job.enqueue_job( import_string('core.jobs.sync_datasource'), name=self.name, obj_type=ContentType.objects.get_for_model(DataSource), diff --git a/netbox/core/models/jobs.py b/netbox/core/models/jobs.py new file mode 100644 index 00000000000..0ae626a7417 --- /dev/null +++ b/netbox/core/models/jobs.py @@ -0,0 +1,219 @@ +import uuid + +import django_rq +from django.contrib.auth.models import User +from django.contrib.contenttypes.fields import GenericForeignKey +from django.contrib.contenttypes.models import ContentType +from django.core.validators import MinValueValidator +from django.db import models +from django.urls import reverse +from django.urls.exceptions import NoReverseMatch +from django.utils import timezone +from django.utils.translation import gettext as _ + +from core.choices import JobStatusChoices +from extras.constants import EVENT_JOB_END, EVENT_JOB_START +from extras.utils import FeatureQuery +from netbox.config import get_config +from netbox.constants import RQ_QUEUE_DEFAULT +from utilities.querysets import RestrictedQuerySet +from utilities.rqworker import get_queue_for_model + +__all__ = ( + 'Job', +) + + +class Job(models.Model): + """ + Tracks the lifecycle of a job which represents a background task (e.g. the execution of a custom script). + """ + object_type = models.ForeignKey( + to=ContentType, + related_name='jobs', + limit_choices_to=FeatureQuery('jobs'), + on_delete=models.CASCADE, + ) + object_id = models.PositiveBigIntegerField( + blank=True, + null=True + ) + object = GenericForeignKey( + ct_field='object_type', + fk_field='object_id' + ) + name = models.CharField( + max_length=200 + ) + created = models.DateTimeField( + auto_now_add=True + ) + scheduled = models.DateTimeField( + null=True, + blank=True + ) + interval = models.PositiveIntegerField( + blank=True, + null=True, + validators=( + MinValueValidator(1), + ), + help_text=_("Recurrence interval (in minutes)") + ) + started = models.DateTimeField( + null=True, + blank=True + ) + completed = models.DateTimeField( + null=True, + blank=True + ) + user = models.ForeignKey( + to=User, + on_delete=models.SET_NULL, + related_name='+', + blank=True, + null=True + ) + status = models.CharField( + max_length=30, + choices=JobStatusChoices, + default=JobStatusChoices.STATUS_PENDING + ) + data = models.JSONField( + null=True, + blank=True + ) + job_id = models.UUIDField( + unique=True + ) + + objects = RestrictedQuerySet.as_manager() + + class Meta: + ordering = ['-created'] + + def __str__(self): + return str(self.job_id) + + def delete(self, *args, **kwargs): + super().delete(*args, **kwargs) + + rq_queue_name = get_config().QUEUE_MAPPINGS.get(self.object_type.model, RQ_QUEUE_DEFAULT) + queue = django_rq.get_queue(rq_queue_name) + job = queue.fetch_job(str(self.job_id)) + + if job: + job.cancel() + + def get_absolute_url(self): + try: + return reverse(f'extras:{self.object_type.model}_result', args=[self.pk]) + except NoReverseMatch: + return None + + def get_status_color(self): + return JobStatusChoices.colors.get(self.status) + + @property + def duration(self): + if not self.completed: + return None + + start_time = self.started or self.created + + if not start_time: + return None + + duration = self.completed - start_time + minutes, seconds = divmod(duration.total_seconds(), 60) + + return f"{int(minutes)} minutes, {seconds:.2f} seconds" + + def start(self): + """ + Record the job's start time and update its status to "running." + """ + if self.started is not None: + return + + # Start the job + self.started = timezone.now() + self.status = JobStatusChoices.STATUS_RUNNING + Job.objects.filter(pk=self.pk).update(started=self.started, status=self.status) + + # Handle webhooks + self.trigger_webhooks(event=EVENT_JOB_START) + + def terminate(self, status=JobStatusChoices.STATUS_COMPLETED): + """ + Mark the job as completed, optionally specifying a particular termination status. + """ + valid_statuses = JobStatusChoices.TERMINAL_STATE_CHOICES + if status not in valid_statuses: + raise ValueError(f"Invalid status for job termination. Choices are: {', '.join(valid_statuses)}") + + # Mark the job as completed + self.status = status + self.completed = timezone.now() + Job.objects.filter(pk=self.pk).update(status=self.status, completed=self.completed) + + # Handle webhooks + self.trigger_webhooks(event=EVENT_JOB_END) + + @classmethod + def enqueue_job(cls, func, name, obj_type, user, schedule_at=None, interval=None, *args, **kwargs): + """ + Create a Job instance and enqueue a job using the given callable + + Args: + func: The callable object to be enqueued for execution + name: Name for the job (optional) + obj_type: ContentType to link to the Job instance object_type + user: User object to link to the Job instance + schedule_at: Schedule the job to be executed at the passed date and time + interval: Recurrence interval (in minutes) + """ + rq_queue_name = get_queue_for_model(obj_type.model) + queue = django_rq.get_queue(rq_queue_name) + status = JobStatusChoices.STATUS_SCHEDULED if schedule_at else JobStatusChoices.STATUS_PENDING + job = Job.objects.create( + name=name, + status=status, + object_type=obj_type, + scheduled=schedule_at, + interval=interval, + user=user, + job_id=uuid.uuid4() + ) + + if schedule_at: + queue.enqueue_at(schedule_at, func, job_id=str(job.job_id), job_result=job, **kwargs) + else: + queue.enqueue(func, job_id=str(job.job_id), job_result=job, **kwargs) + + return job + + def trigger_webhooks(self, event): + from extras.models import Webhook + + rq_queue_name = get_config().QUEUE_MAPPINGS.get('webhook', RQ_QUEUE_DEFAULT) + rq_queue = django_rq.get_queue(rq_queue_name, is_async=False) + + # Fetch any webhooks matching this object type and action + webhooks = Webhook.objects.filter( + **{f'type_{event}': True}, + content_types=self.object_type, + enabled=True + ) + + for webhook in webhooks: + rq_queue.enqueue( + "extras.webhooks_worker.process_webhook", + webhook=webhook, + model_name=self.object_type.model, + event=event, + data=self.data, + timestamp=str(timezone.now()), + username=self.user.username + ) diff --git a/netbox/core/tables/__init__.py b/netbox/core/tables/__init__.py index df22d8bbb44..052f68b6875 100644 --- a/netbox/core/tables/__init__.py +++ b/netbox/core/tables/__init__.py @@ -1 +1,2 @@ from .data import * +from .jobs import * diff --git a/netbox/core/tables/jobs.py b/netbox/core/tables/jobs.py new file mode 100644 index 00000000000..662119dbcb4 --- /dev/null +++ b/netbox/core/tables/jobs.py @@ -0,0 +1,34 @@ +import django_tables2 as tables +from django.utils.translation import gettext as _ + +from netbox.tables import NetBoxTable, columns +from ..models import Job + + +class JobTable(NetBoxTable): + name = tables.Column( + linkify=True + ) + object_type = columns.ContentTypeColumn( + verbose_name=_('Type') + ) + status = columns.ChoiceFieldColumn() + created = columns.DateTimeColumn() + scheduled = columns.DateTimeColumn() + interval = columns.DurationColumn() + started = columns.DateTimeColumn() + completed = columns.DateTimeColumn() + actions = columns.ActionsColumn( + actions=('delete',) + ) + + class Meta(NetBoxTable.Meta): + model = Job + fields = ( + 'pk', 'id', 'object_type', 'name', 'status', 'created', 'scheduled', 'interval', 'started', 'completed', + 'user', 'job_id', + ) + default_columns = ( + 'pk', 'id', 'object_type', 'name', 'status', 'created', 'scheduled', 'interval', 'started', 'completed', + 'user', + ) diff --git a/netbox/core/urls.py b/netbox/core/urls.py index 128020890b5..b7d0e9915ba 100644 --- a/netbox/core/urls.py +++ b/netbox/core/urls.py @@ -19,4 +19,9 @@ path('data-files/delete/', views.DataFileBulkDeleteView.as_view(), name='datafile_bulk_delete'), path('data-files//', include(get_model_urls('core', 'datafile'))), + # Job results + path('jobs/', views.JobListView.as_view(), name='job_list'), + path('jobs/delete/', views.JobBulkDeleteView.as_view(), name='job_bulk_delete'), + path('jobs//delete/', views.JobDeleteView.as_view(), name='job_delete'), + ) diff --git a/netbox/core/views.py b/netbox/core/views.py index 7a603ba1aa4..d49ac00236f 100644 --- a/netbox/core/views.py +++ b/netbox/core/views.py @@ -120,3 +120,25 @@ class DataFileBulkDeleteView(generic.BulkDeleteView): queryset = DataFile.objects.defer('data') filterset = filtersets.DataFileFilterSet table = tables.DataFileTable + + +# +# Jobs +# + +class JobListView(generic.ObjectListView): + queryset = Job.objects.all() + filterset = filtersets.JobFilterSet + filterset_form = forms.JobFilterForm + table = tables.JobTable + actions = ('export', 'delete', 'bulk_delete', ) + + +class JobDeleteView(generic.ObjectDeleteView): + queryset = Job.objects.all() + + +class JobBulkDeleteView(generic.BulkDeleteView): + queryset = Job.objects.all() + filterset = filtersets.JobFilterSet + table = tables.JobTable diff --git a/netbox/extras/admin.py b/netbox/extras/admin.py index 18cc860b128..a6e8007fbc2 100644 --- a/netbox/extras/admin.py +++ b/netbox/extras/admin.py @@ -6,7 +6,7 @@ from netbox.config import get_config, PARAMS from .forms import ConfigRevisionForm -from .models import ConfigRevision, JobResult +from .models import ConfigRevision @admin.register(ConfigRevision) diff --git a/netbox/extras/api/nested_serializers.py b/netbox/extras/api/nested_serializers.py index dab0798fee3..29ef679437e 100644 --- a/netbox/extras/api/nested_serializers.py +++ b/netbox/extras/api/nested_serializers.py @@ -1,9 +1,7 @@ from rest_framework import serializers -from extras import choices, models -from netbox.api.fields import ChoiceField +from extras import models from netbox.api.serializers import NestedTagSerializer, WritableNestedSerializer -from users.api.nested_serializers import NestedUserSerializer __all__ = [ 'NestedConfigContextSerializer', @@ -12,7 +10,6 @@ 'NestedCustomLinkSerializer', 'NestedExportTemplateSerializer', 'NestedImageAttachmentSerializer', - 'NestedJobResultSerializer', 'NestedJournalEntrySerializer', 'NestedSavedFilterSerializer', 'NestedTagSerializer', # Defined in netbox.api.serializers @@ -90,15 +87,3 @@ class NestedJournalEntrySerializer(WritableNestedSerializer): class Meta: model = models.JournalEntry fields = ['id', 'url', 'display', 'created'] - - -class NestedJobResultSerializer(serializers.ModelSerializer): - url = serializers.HyperlinkedIdentityField(view_name='extras-api:jobresult-detail') - status = ChoiceField(choices=choices.JobResultStatusChoices) - user = NestedUserSerializer( - read_only=True - ) - - class Meta: - model = models.JobResult - fields = ['url', 'created', 'completed', 'user', 'status'] diff --git a/netbox/extras/api/serializers.py b/netbox/extras/api/serializers.py index 59e264ccf79..cc78698f8d9 100644 --- a/netbox/extras/api/serializers.py +++ b/netbox/extras/api/serializers.py @@ -4,7 +4,8 @@ from drf_yasg.utils import swagger_serializer_method from rest_framework import serializers -from core.api.nested_serializers import NestedDataSourceSerializer, NestedDataFileSerializer +from core.api.serializers import JobSerializer +from core.api.nested_serializers import NestedDataSourceSerializer, NestedDataFileSerializer, NestedJobSerializer from dcim.api.nested_serializers import ( NestedDeviceRoleSerializer, NestedDeviceTypeSerializer, NestedLocationSerializer, NestedPlatformSerializer, NestedRegionSerializer, NestedSiteSerializer, NestedSiteGroupSerializer, @@ -37,7 +38,6 @@ 'DashboardSerializer', 'ExportTemplateSerializer', 'ImageAttachmentSerializer', - 'JobResultSerializer', 'JournalEntrySerializer', 'ObjectChangeSerializer', 'ReportDetailSerializer', @@ -409,28 +409,6 @@ class Meta: ] -# -# Job Results -# - -class JobResultSerializer(BaseModelSerializer): - url = serializers.HyperlinkedIdentityField(view_name='extras-api:jobresult-detail') - user = NestedUserSerializer( - read_only=True - ) - status = ChoiceField(choices=JobResultStatusChoices, read_only=True) - obj_type = ContentTypeField( - read_only=True - ) - - class Meta: - model = JobResult - fields = [ - 'id', 'url', 'display', 'status', 'created', 'scheduled', 'interval', 'started', 'completed', 'name', - 'obj_type', 'user', 'data', 'job_id', - ] - - # # Reports # @@ -446,11 +424,11 @@ class ReportSerializer(serializers.Serializer): name = serializers.CharField(max_length=255) description = serializers.CharField(max_length=255, required=False) test_methods = serializers.ListField(child=serializers.CharField(max_length=255)) - result = NestedJobResultSerializer() + result = NestedJobSerializer() class ReportDetailSerializer(ReportSerializer): - result = JobResultSerializer() + result = JobSerializer() class ReportInputSerializer(serializers.Serializer): @@ -473,7 +451,7 @@ class ScriptSerializer(serializers.Serializer): name = serializers.CharField(read_only=True) description = serializers.CharField(read_only=True) vars = serializers.SerializerMethodField(read_only=True) - result = NestedJobResultSerializer() + result = NestedJobSerializer() @swagger_serializer_method(serializer_or_field=serializers.JSONField) def get_vars(self, instance): @@ -483,7 +461,7 @@ def get_vars(self, instance): class ScriptDetailSerializer(ScriptSerializer): - result = JobResultSerializer() + result = JobSerializer() class ScriptInputSerializer(serializers.Serializer): diff --git a/netbox/extras/api/urls.py b/netbox/extras/api/urls.py index e796f0fdbfa..80dc56ae11d 100644 --- a/netbox/extras/api/urls.py +++ b/netbox/extras/api/urls.py @@ -20,7 +20,6 @@ router.register('reports', views.ReportViewSet, basename='report') router.register('scripts', views.ScriptViewSet, basename='script') router.register('object-changes', views.ObjectChangeViewSet) -router.register('job-results', views.JobResultViewSet) router.register('content-types', views.ContentTypeViewSet) app_name = 'extras-api' diff --git a/netbox/extras/api/views.py b/netbox/extras/api/views.py index d09c88abe3d..bd775c668aa 100644 --- a/netbox/extras/api/views.py +++ b/netbox/extras/api/views.py @@ -12,10 +12,10 @@ from rest_framework.viewsets import ReadOnlyModelViewSet, ViewSet from rq import Worker +from core.choices import JobStatusChoices +from core.models import Job from extras import filtersets -from extras.choices import JobResultStatusChoices from extras.models import * -from extras.models import CustomField from extras.reports import get_report, run_report from extras.scripts import get_script, run_script from netbox.api.authentication import IsAuthenticatedOrLoginNotRequired @@ -191,9 +191,9 @@ def list(self, request): report_content_type = ContentType.objects.get(app_label='extras', model='report') results = { r.name: r - for r in JobResult.objects.filter( - obj_type=report_content_type, - status__in=JobResultStatusChoices.TERMINAL_STATE_CHOICES + for r in Job.objects.filter( + object_type=report_content_type, + status__in=JobStatusChoices.TERMINAL_STATE_CHOICES ).order_by('name', '-created').distinct('name').defer('data') } @@ -201,7 +201,7 @@ def list(self, request): for report_module in ReportModule.objects.restrict(request.user): report_list.extend([report() for report in report_module.reports.values()]) - # Attach JobResult objects to each report (if any) + # Attach Job objects to each report (if any) for report in report_list: report.result = results.get(report.full_name, None) @@ -216,13 +216,13 @@ def retrieve(self, request, pk): Retrieve a single Report identified as ".". """ - # Retrieve the Report and JobResult, if any. + # Retrieve the Report and Job, if any. report = self._retrieve_report(pk) report_content_type = ContentType.objects.get(app_label='extras', model='report') - report.result = JobResult.objects.filter( - obj_type=report_content_type, + report.result = Job.objects.filter( + object_type=report_content_type, name=report.full_name, - status__in=JobResultStatusChoices.TERMINAL_STATE_CHOICES + status__in=JobStatusChoices.TERMINAL_STATE_CHOICES ).first() serializer = serializers.ReportDetailSerializer(report, context={ @@ -234,7 +234,7 @@ def retrieve(self, request, pk): @action(detail=True, methods=['post']) def run(self, request, pk): """ - Run a Report identified as ".