From f7a2eb8aef53d3ec2e4bf55c2199fd59cce76ea2 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Fri, 24 Mar 2023 21:00:36 -0400 Subject: [PATCH] Closes #11890: Sync/upload reports & scripts (#12059) * Initial work on #11890 * Consolidate get_scripts() and get_reports() functions * Introduce proxy models for script & report modules * Add add/delete views for reports & scripts * Add deletion links for modules * Enable resolving scripts/reports from module class * Remove get_modules() utility function * Show results in report/script lists * Misc cleanup * Fix file uploads * Support automatic migration for submodules * Fix module child ordering * Template cleanup * Remove ManagedFile views * Move is_script(), is_report() into extras.utils * Fix URLs for nested reports & scripts * Misc cleanup --- netbox/core/choices.py | 15 ++- netbox/core/forms/model_forms.py | 36 ++++++ netbox/core/migrations/0002_managedfile.py | 39 ++++++ netbox/core/models/__init__.py | 1 + netbox/core/models/data.py | 14 ++- netbox/core/models/files.py | 88 +++++++++++++ netbox/extras/api/views.py | 28 ++--- .../extras/management/commands/runreport.py | 14 +-- .../migrations/0091_create_managedfiles.py | 86 +++++++++++++ netbox/extras/models/__init__.py | 2 + netbox/extras/models/models.py | 119 +++++++++++++++++- netbox/extras/reports.py | 84 +++---------- netbox/extras/scripts.py | 93 +++----------- netbox/extras/urls.py | 16 ++- netbox/extras/utils.py | 22 ++++ netbox/extras/views.py | 108 +++++++++------- .../templates/extras/htmx/report_result.html | 2 +- .../templates/extras/htmx/script_result.html | 2 +- netbox/templates/extras/inc/job_label.html | 15 --- netbox/templates/extras/report.html | 2 +- netbox/templates/extras/report_list.html | 99 +++++++++------ netbox/templates/extras/script.html | 2 +- netbox/templates/extras/script_list.html | 86 ++++++++----- 23 files changed, 658 insertions(+), 315 deletions(-) create mode 100644 netbox/core/migrations/0002_managedfile.py create mode 100644 netbox/core/models/files.py create mode 100644 netbox/extras/migrations/0091_create_managedfiles.py delete mode 100644 netbox/templates/extras/inc/job_label.html diff --git a/netbox/core/choices.py b/netbox/core/choices.py index be33afabae2..2b3d90a4284 100644 --- a/netbox/core/choices.py +++ b/netbox/core/choices.py @@ -20,7 +20,6 @@ class DataSourceTypeChoices(ChoiceSet): class DataSourceStatusChoices(ChoiceSet): - NEW = 'new' QUEUED = 'queued' SYNCING = 'syncing' @@ -34,3 +33,17 @@ class DataSourceStatusChoices(ChoiceSet): (COMPLETED, _('Completed'), 'green'), (FAILED, _('Failed'), 'red'), ) + + +# +# Managed files +# + +class ManagedFileRootPathChoices(ChoiceSet): + SCRIPTS = 'scripts' # settings.SCRIPTS_ROOT + REPORTS = 'reports' # settings.REPORTS_ROOT + + CHOICES = ( + (SCRIPTS, _('Scripts')), + (REPORTS, _('Reports')), + ) diff --git a/netbox/core/forms/model_forms.py b/netbox/core/forms/model_forms.py index a3a478be5ef..99e2786efcd 100644 --- a/netbox/core/forms/model_forms.py +++ b/netbox/core/forms/model_forms.py @@ -3,12 +3,14 @@ from django import forms from core.models import * +from extras.forms.mixins import SyncedDataMixin from netbox.forms import NetBoxModelForm from netbox.registry import registry from utilities.forms import CommentField, get_field_value __all__ = ( 'DataSourceForm', + 'ManagedFileForm', ) @@ -73,3 +75,37 @@ def save(self, *args, **kwargs): self.instance.parameters = parameters return super().save(*args, **kwargs) + + +class ManagedFileForm(SyncedDataMixin, NetBoxModelForm): + upload_file = forms.FileField( + required=False + ) + + fieldsets = ( + ('File Upload', ('upload_file',)), + ('Data Source', ('data_source', 'data_file')), + ) + + class Meta: + model = ManagedFile + fields = ('data_source', 'data_file') + + def clean(self): + super().clean() + + if self.cleaned_data.get('upload_file') and self.cleaned_data.get('data_file'): + raise forms.ValidationError("Cannot upload a file and sync from an existing file") + if not self.cleaned_data.get('upload_file') and not self.cleaned_data.get('data_file'): + raise forms.ValidationError("Must upload a file or select a data file to sync") + + return self.cleaned_data + + def save(self, *args, **kwargs): + # If a file was uploaded, save it to disk + if self.cleaned_data['upload_file']: + self.instance.file_path = self.cleaned_data['upload_file'].name + with open(self.instance.full_path, 'wb+') as new_file: + new_file.write(self.cleaned_data['upload_file'].read()) + + return super().save(*args, **kwargs) diff --git a/netbox/core/migrations/0002_managedfile.py b/netbox/core/migrations/0002_managedfile.py new file mode 100644 index 00000000000..da6b1e3be98 --- /dev/null +++ b/netbox/core/migrations/0002_managedfile.py @@ -0,0 +1,39 @@ +# Generated by Django 4.1.7 on 2023-03-23 17:35 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0001_initial'), + ] + + operations = [ + migrations.CreateModel( + name='ManagedFile', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False)), + ('data_path', models.CharField(blank=True, editable=False, max_length=1000)), + ('data_synced', models.DateTimeField(blank=True, editable=False, null=True)), + ('created', models.DateTimeField(auto_now_add=True)), + ('last_updated', models.DateTimeField(blank=True, editable=False, null=True)), + ('file_root', models.CharField(max_length=1000)), + ('file_path', models.FilePathField(editable=False)), + ('data_file', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='core.datafile')), + ('data_source', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='+', to='core.datasource')), + ], + options={ + 'ordering': ('file_root', 'file_path'), + }, + ), + migrations.AddIndex( + model_name='managedfile', + index=models.Index(fields=['file_root', 'file_path'], name='core_managedfile_root_path'), + ), + migrations.AddConstraint( + model_name='managedfile', + constraint=models.UniqueConstraint(fields=('file_root', 'file_path'), name='core_managedfile_unique_root_path'), + ), + ] diff --git a/netbox/core/models/__init__.py b/netbox/core/models/__init__.py index df22d8bbb44..e21b9f8cf43 100644 --- a/netbox/core/models/__init__.py +++ b/netbox/core/models/__init__.py @@ -1 +1,2 @@ from .data import * +from .files import * diff --git a/netbox/core/models/data.py b/netbox/core/models/data.py index 8d716404706..cbf29ae3833 100644 --- a/netbox/core/models/data.py +++ b/netbox/core/models/data.py @@ -14,7 +14,6 @@ from django.utils.module_loading import import_string from django.utils.translation import gettext as _ -from extras.models import JobResult from netbox.models import PrimaryModel from netbox.registry import registry from utilities.files import sha256_hash @@ -113,6 +112,8 @@ def enqueue_sync_job(self, request): """ Enqueue a background job to synchronize the DataSource by calling sync(). """ + from extras.models import JobResult + # Set the status to "syncing" self.status = DataSourceStatusChoices.QUEUED DataSource.objects.filter(pk=self.pk).update(status=self.status) @@ -314,3 +315,14 @@ def refresh_from_disk(self, source_root): self.data = f.read() return is_modified + + def write_to_disk(self, path, overwrite=False): + """ + Write the object's data to disk at the specified path + """ + # Check whether file already exists + if os.path.isfile(path) and not overwrite: + raise FileExistsError() + + with open(path, 'wb+') as new_file: + new_file.write(self.data) diff --git a/netbox/core/models/files.py b/netbox/core/models/files.py new file mode 100644 index 00000000000..a725ea0ac28 --- /dev/null +++ b/netbox/core/models/files.py @@ -0,0 +1,88 @@ +import logging +import os + +from django.conf import settings +from django.db import models +from django.urls import reverse +from django.utils.translation import gettext as _ + +from ..choices import ManagedFileRootPathChoices +from netbox.models.features import SyncedDataMixin +from utilities.querysets import RestrictedQuerySet + +__all__ = ( + 'ManagedFile', +) + +logger = logging.getLogger('netbox.core.files') + + +class ManagedFile(SyncedDataMixin, models.Model): + """ + Database representation for a file on disk. This class is typically wrapped by a proxy class (e.g. ScriptModule) + to provide additional functionality. + """ + created = models.DateTimeField( + auto_now_add=True + ) + last_updated = models.DateTimeField( + editable=False, + blank=True, + null=True + ) + file_root = models.CharField( + max_length=1000, + choices=ManagedFileRootPathChoices + ) + file_path = models.FilePathField( + editable=False, + help_text=_("File path relative to the designated root path") + ) + + objects = RestrictedQuerySet.as_manager() + + class Meta: + ordering = ('file_root', 'file_path') + constraints = ( + models.UniqueConstraint( + fields=('file_root', 'file_path'), + name='%(app_label)s_%(class)s_unique_root_path' + ), + ) + indexes = [ + models.Index(fields=('file_root', 'file_path'), name='core_managedfile_root_path'), + ] + + def __str__(self): + return self.name + + def get_absolute_url(self): + return reverse('core:managedfile', args=[self.pk]) + + @property + def name(self): + return self.file_path + + @property + def full_path(self): + return os.path.join(self._resolve_root_path(), self.file_path) + + def _resolve_root_path(self): + return { + 'scripts': settings.SCRIPTS_ROOT, + 'reports': settings.REPORTS_ROOT, + }[self.file_root] + + def sync_data(self): + if self.data_file: + self.file_path = os.path.basename(self.data_path) + self.data_file.write_to_disk(self.full_path, overwrite=True) + + def delete(self, *args, **kwargs): + # Delete file from disk + try: + os.remove(self.full_path) + except FileNotFoundError: + pass + + return super().delete(*args, **kwargs) diff --git a/netbox/extras/api/views.py b/netbox/extras/api/views.py index dfee44a51a8..d09c88abe3d 100644 --- a/netbox/extras/api/views.py +++ b/netbox/extras/api/views.py @@ -16,8 +16,8 @@ from extras.choices import JobResultStatusChoices from extras.models import * from extras.models import CustomField -from extras.reports import get_report, get_reports, run_report -from extras.scripts import get_script, get_scripts, run_script +from extras.reports import get_report, run_report +from extras.scripts import get_script, run_script from netbox.api.authentication import IsAuthenticatedOrLoginNotRequired from netbox.api.features import SyncedDataMixin from netbox.api.metadata import ContentTypeMetadata @@ -27,7 +27,6 @@ from utilities.utils import copy_safe_request, count_related from . import serializers from .mixins import ConfigTemplateRenderMixin -from .nested_serializers import NestedConfigTemplateSerializer class ExtrasRootView(APIRootView): @@ -189,7 +188,6 @@ def list(self, request): """ Compile all reports and their related results (if any). Result data is deferred in the list view. """ - report_list = [] report_content_type = ContentType.objects.get(app_label='extras', model='report') results = { r.name: r @@ -199,13 +197,13 @@ def list(self, request): ).order_by('name', '-created').distinct('name').defer('data') } - # Iterate through all available Reports. - for module_name, reports in get_reports().items(): - for report in reports.values(): + report_list = [] + for report_module in ReportModule.objects.restrict(request.user): + report_list.extend([report() for report in report_module.reports.values()]) - # Attach the relevant JobResult (if any) to each Report. - report.result = results.get(report.full_name, None) - report_list.append(report) + # Attach JobResult objects to each report (if any) + for report in report_list: + report.result = results.get(report.full_name, None) serializer = serializers.ReportSerializer(report_list, many=True, context={ 'request': request, @@ -296,15 +294,15 @@ def list(self, request): ).order_by('name', '-created').distinct('name').defer('data') } - flat_list = [] - for script_list in get_scripts().values(): - flat_list.extend(script_list.values()) + script_list = [] + for script_module in ScriptModule.objects.restrict(request.user): + script_list.extend(script_module.scripts.values()) # Attach JobResult objects to each script (if any) - for script in flat_list: + for script in script_list: script.result = results.get(script.full_name, None) - serializer = serializers.ScriptSerializer(flat_list, many=True, context={'request': request}) + serializer = serializers.ScriptSerializer(script_list, many=True, context={'request': request}) return Response(serializer.data) diff --git a/netbox/extras/management/commands/runreport.py b/netbox/extras/management/commands/runreport.py index 38d43561337..9a61e780524 100644 --- a/netbox/extras/management/commands/runreport.py +++ b/netbox/extras/management/commands/runreport.py @@ -5,8 +5,8 @@ from django.utils import timezone from extras.choices import JobResultStatusChoices -from extras.models import JobResult -from extras.reports import get_reports, run_report +from extras.models import JobResult, ReportModule +from extras.reports import run_report class Command(BaseCommand): @@ -17,13 +17,9 @@ def add_arguments(self, parser): def handle(self, *args, **options): - # Gather all available reports - reports = get_reports() - - # Run reports - for module_name, report_list in reports.items(): - for report in report_list.values(): - if module_name in options['reports'] or report.full_name in options['reports']: + for module in ReportModule.objects.all(): + for report in module.reports.values(): + if module.name in options['reports'] or report.full_name in options['reports']: # Run the report and create a new JobResult self.stdout.write( diff --git a/netbox/extras/migrations/0091_create_managedfiles.py b/netbox/extras/migrations/0091_create_managedfiles.py new file mode 100644 index 00000000000..71581614ae9 --- /dev/null +++ b/netbox/extras/migrations/0091_create_managedfiles.py @@ -0,0 +1,86 @@ +import os +import pkgutil + +from django.conf import settings +from django.db import migrations, models +import extras.models.models + + +def create_files(cls, root_name, root_path): + + path_tree = [ + path for path, _, _ in os.walk(root_path) + if os.path.basename(path)[0] not in ('_', '.') + ] + + modules = list(pkgutil.iter_modules(path_tree)) + filenames = [] + for importer, module_name, is_pkg in modules: + if is_pkg: + continue + try: + module = importer.find_module(module_name).load_module(module_name) + rel_path = os.path.relpath(module.__file__, root_path) + filenames.append(rel_path) + except ImportError: + pass + + managed_files = [ + cls(file_root=root_name, file_path=filename) + for filename in filenames + ] + cls.objects.bulk_create(managed_files) + + +def replicate_scripts(apps, schema_editor): + ScriptModule = apps.get_model('extras', 'ScriptModule') + create_files(ScriptModule, 'scripts', settings.SCRIPTS_ROOT) + + +def replicate_reports(apps, schema_editor): + ReportModule = apps.get_model('extras', 'ReportModule') + create_files(ReportModule, 'reports', settings.REPORTS_ROOT) + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0002_managedfile'), + ('extras', '0090_objectchange_index_request_id'), + ] + + operations = [ + # Create proxy models + migrations.CreateModel( + name='ReportModule', + fields=[ + ], + options={ + 'proxy': True, + 'indexes': [], + 'constraints': [], + }, + bases=(extras.models.models.PythonModuleMixin, 'core.managedfile', models.Model), + ), + migrations.CreateModel( + name='ScriptModule', + fields=[ + ], + options={ + 'proxy': True, + 'indexes': [], + 'constraints': [], + }, + bases=(extras.models.models.PythonModuleMixin, 'core.managedfile', models.Model), + ), + + # Instantiate ManagedFiles to represent scripts & reports + migrations.RunPython( + code=replicate_scripts, + reverse_code=migrations.RunPython.noop + ), + migrations.RunPython( + code=replicate_reports, + reverse_code=migrations.RunPython.noop + ), + ] diff --git a/netbox/extras/models/__init__.py b/netbox/extras/models/__init__.py index 14e23366fdc..b89b958f5e4 100644 --- a/netbox/extras/models/__init__.py +++ b/netbox/extras/models/__init__.py @@ -23,8 +23,10 @@ 'JournalEntry', 'ObjectChange', 'Report', + 'ReportModule', 'SavedFilter', 'Script', + 'ScriptModule', 'StagedChange', 'Tag', 'TaggedItem', diff --git a/netbox/extras/models/models.py b/netbox/extras/models/models.py index b0c6d7fe367..f1e70190b4f 100644 --- a/netbox/extras/models/models.py +++ b/netbox/extras/models/models.py @@ -1,6 +1,11 @@ +import inspect import json +import os import uuid +from functools import cached_property +from pkgutil import ModuleInfo, get_importer +import django_rq from django.conf import settings from django.contrib import admin from django.contrib.auth.models import User @@ -16,12 +21,13 @@ from django.utils.formats import date_format from django.utils.translation import gettext as _ from rest_framework.utils.encoders import JSONEncoder -import django_rq +from core.choices import ManagedFileRootPathChoices +from core.models import ManagedFile from extras.choices import * -from extras.constants import * from extras.conditions import ConditionSet -from extras.utils import FeatureQuery, image_upload +from extras.constants import * +from extras.utils import FeatureQuery, image_upload, is_report, is_script from netbox.config import get_config from netbox.constants import RQ_QUEUE_DEFAULT from netbox.models import ChangeLoggedModel @@ -41,8 +47,10 @@ 'JobResult', 'JournalEntry', 'Report', + 'ReportModule', 'SavedFilter', 'Script', + 'ScriptModule', 'Webhook', ) @@ -814,6 +822,27 @@ def is_active(self): # Custom scripts & reports # +class PythonModuleMixin: + + @property + def path(self): + return os.path.splitext(self.file_path)[0] + + def get_module_info(self): + path = os.path.dirname(self.full_path) + module_name = os.path.basename(self.path) + return ModuleInfo( + module_finder=get_importer(path), + name=module_name, + ispkg=False + ) + + def get_module(self): + importer, module_name, _ = self.get_module_info() + module = importer.find_module(module_name).load_module(module_name) + return module + + class Script(JobResultsMixin, WebhooksMixin, models.Model): """ Dummy model used to generate permissions for custom scripts. Does not exist in the database. @@ -822,6 +851,48 @@ class Meta: managed = False +class ScriptModuleManager(models.Manager.from_queryset(RestrictedQuerySet)): + + def get_queryset(self): + return super().get_queryset().filter(file_root=ManagedFileRootPathChoices.SCRIPTS) + + +class ScriptModule(PythonModuleMixin, ManagedFile): + """ + Proxy model for script module files. + """ + objects = ScriptModuleManager() + + class Meta: + proxy = True + + def get_absolute_url(self): + return reverse('extras:script_list') + + @cached_property + def scripts(self): + + def _get_name(cls): + # For child objects in submodules use the full import path w/o the root module as the name + return cls.full_name.split(".", maxsplit=1)[1] + + module = self.get_module() + scripts = {} + ordered = getattr(module, 'script_order', []) + + for cls in ordered: + scripts[_get_name(cls)] = cls + for name, cls in inspect.getmembers(module, is_script): + if cls not in ordered: + scripts[_get_name(cls)] = cls + + return scripts + + def save(self, *args, **kwargs): + self.file_root = ManagedFileRootPathChoices.SCRIPTS + return super().save(*args, **kwargs) + + # # Reports # @@ -832,3 +903,45 @@ class Report(JobResultsMixin, WebhooksMixin, models.Model): """ class Meta: managed = False + + +class ReportModuleManager(models.Manager.from_queryset(RestrictedQuerySet)): + + def get_queryset(self): + return super().get_queryset().filter(file_root=ManagedFileRootPathChoices.REPORTS) + + +class ReportModule(PythonModuleMixin, ManagedFile): + """ + Proxy model for report module files. + """ + objects = ReportModuleManager() + + class Meta: + proxy = True + + def get_absolute_url(self): + return reverse('extras:report_list') + + @cached_property + def reports(self): + + def _get_name(cls): + # For child objects in submodules use the full import path w/o the root module as the name + return cls.full_name.split(".", maxsplit=1)[1] + + module = self.get_module() + reports = {} + ordered = getattr(module, 'report_order', []) + + for cls in ordered: + reports[_get_name(cls)] = cls + for name, cls in inspect.getmembers(module, is_report): + if cls not in ordered: + reports[_get_name(cls)] = cls + + return reports + + def save(self, *args, **kwargs): + self.file_root = ManagedFileRootPathChoices.REPORTS + return super().save(*args, **kwargs) diff --git a/netbox/extras/reports.py b/netbox/extras/reports.py index 0a944a0d26c..cedc1491fdd 100644 --- a/netbox/extras/reports.py +++ b/netbox/extras/reports.py @@ -1,75 +1,23 @@ -import inspect import logging -import pkgutil import traceback from datetime import timedelta -from django.conf import settings from django.utils import timezone +from django.utils.functional import classproperty from django_rq import job from .choices import JobResultStatusChoices, LogLevelChoices -from .models import JobResult +from .models import JobResult, ReportModule logger = logging.getLogger(__name__) -def is_report(obj): - """ - Returns True if the given object is a Report. - """ - return obj in Report.__subclasses__() - - def get_report(module_name, report_name): """ Return a specific report from within a module. """ - reports = get_reports() - module = reports.get(module_name) - - if module is None: - return None - - report = module.get(report_name) - - if report is None: - return None - - return report - - -def get_reports(): - """ - Compile a list of all reports available across all modules in the reports path. Returns a list of tuples: - - [ - (module_name, (report, report, report, ...)), - (module_name, (report, report, report, ...)), - ... - ] - """ - module_list = {} - - # Iterate through all modules within the reports path. These are the user-created files in which reports are - # defined. - for importer, module_name, _ in pkgutil.iter_modules([settings.REPORTS_ROOT]): - module = importer.find_module(module_name).load_module(module_name) - report_order = getattr(module, "report_order", ()) - ordered_reports = [cls() for cls in report_order if is_report(cls)] - unordered_reports = [cls() for _, cls in inspect.getmembers(module, is_report) if cls not in report_order] - - module_reports = {} - - for cls in [*ordered_reports, *unordered_reports]: - # For reports in submodules use the full import path w/o the root module as the name - report_name = cls.full_name.split(".", maxsplit=1)[1] - module_reports[report_name] = cls - - if module_reports: - module_list[module_name] = module_reports - - return module_list + module = ReportModule.objects.get(file_path=f'{module_name}.py') + return module.reports.get(report_name) @job('default') @@ -79,7 +27,7 @@ def run_report(job_result, *args, **kwargs): method for queueing into the background processor. """ module_name, report_name = job_result.name.split('.', 1) - report = get_report(module_name, report_name) + report = get_report(module_name, report_name)() try: job_result.start() @@ -136,7 +84,7 @@ def __init__(self): self.active_test = None self.failed = False - self.logger = logging.getLogger(f"netbox.reports.{self.full_name}") + self.logger = logging.getLogger(f"netbox.reports.{self.__module__}.{self.__class__.__name__}") # Compile test methods and initialize results skeleton test_methods = [] @@ -154,13 +102,17 @@ def __init__(self): raise Exception("A report must contain at least one test method.") self.test_methods = test_methods - @property + @classproperty def module(self): return self.__module__ - @property + @classproperty def class_name(self): - return self.__class__.__name__ + return self.__name__ + + @classproperty + def full_name(self): + return f'{self.module}.{self.class_name}' @property def name(self): @@ -169,9 +121,9 @@ def name(self): """ return self.class_name - @property - def full_name(self): - return f'{self.module}.{self.class_name}' + # + # Logging methods + # def _log(self, obj, message, level=LogLevelChoices.LOG_DEFAULT): """ @@ -228,6 +180,10 @@ def log_failure(self, obj, message): self.logger.info(f"Failure | {obj}: {message}") self.failed = True + # + # Run methods + # + def run(self, job_result): """ Run the report and save its results. Each test method will be executed in order. diff --git a/netbox/extras/scripts.py b/netbox/extras/scripts.py index 89dfa32686f..23a9f9f73ac 100644 --- a/netbox/extras/scripts.py +++ b/netbox/extras/scripts.py @@ -2,9 +2,6 @@ import json import logging import os -import pkgutil -import sys -import threading import traceback from datetime import timedelta @@ -17,7 +14,7 @@ from extras.api.serializers import ScriptOutputSerializer from extras.choices import JobResultStatusChoices, LogLevelChoices -from extras.models import JobResult +from extras.models import JobResult, ScriptModule from extras.signals import clear_webhooks from ipam.formfields import IPAddressFormField, IPNetworkFormField from ipam.validators import MaxPrefixLengthValidator, MinPrefixLengthValidator, prefix_validator @@ -43,8 +40,6 @@ 'TextVar', ] -lock = threading.Lock() - # # Script variables @@ -272,7 +267,7 @@ class Meta: def __init__(self): # Initiate the log - self.logger = logging.getLogger(f"netbox.scripts.{self.module()}.{self.__class__.__name__}") + self.logger = logging.getLogger(f"netbox.scripts.{self.__module__}.{self.__class__.__name__}") self.log = [] # Declare the placeholder for the current request @@ -286,21 +281,25 @@ def __str__(self): return self.name @classproperty - def name(self): - return getattr(self.Meta, 'name', self.__name__) + def module(self): + return self.__module__ + + @classproperty + def class_name(self): + return self.__name__ @classproperty def full_name(self): - return '.'.join([self.__module__, self.__name__]) + return f'{self.module}.{self.class_name}' + + @classproperty + def name(self): + return getattr(self.Meta, 'name', self.__name__) @classproperty def description(self): return getattr(self.Meta, 'description', '') - @classmethod - def module(cls): - return cls.__module__ - @classmethod def root_module(cls): return cls.__module__.split(".")[0] @@ -427,15 +426,6 @@ class Script(BaseScript): # Functions # -def is_script(obj): - """ - Returns True if the object is a Script. - """ - try: - return issubclass(obj, Script) and obj != Script - except TypeError: - return False - def is_variable(obj): """ @@ -452,10 +442,10 @@ def run_script(data, request, commit=True, *args, **kwargs): job_result = kwargs.pop('job_result') job_result.start() - module, script_name = job_result.name.split('.', 1) - script = get_script(module, script_name)() + module_name, script_name = job_result.name.split('.', 1) + script = get_script(module_name, script_name)() - logger = logging.getLogger(f"netbox.scripts.{module}.{script_name}") + logger = logging.getLogger(f"netbox.scripts.{module_name}.{script_name}") logger.info(f"Running script (commit={commit})") # Add files to form data @@ -522,56 +512,9 @@ def _run_script(): ) -def get_scripts(use_names=False): - """ - Return a dict of dicts mapping all scripts to their modules. Set use_names to True to use each module's human- - defined name in place of the actual module name. - """ - scripts = {} - - # Get all modules within the scripts path. These are the user-created files in which scripts are - # defined. - modules = list(pkgutil.iter_modules([settings.SCRIPTS_ROOT])) - modules_bases = set([name.split(".")[0] for _, name, _ in modules]) - - # Deleting from sys.modules needs to done behind a lock to prevent race conditions where a module is - # removed from sys.modules while another thread is importing - with lock: - for module_name in list(sys.modules.keys()): - # Everything sharing a base module path with a module in the script folder is removed. - # We also remove all modules with a base module called "scripts". This allows modifying imported - # non-script modules without having to reload the RQ worker. - module_base = module_name.split(".")[0] - if module_base == "scripts" or module_base in modules_bases: - del sys.modules[module_name] - - for importer, module_name, _ in modules: - module = importer.find_module(module_name).load_module(module_name) - - if use_names and hasattr(module, 'name'): - module_name = module.name - - module_scripts = {} - script_order = getattr(module, "script_order", ()) - ordered_scripts = [cls for cls in script_order if is_script(cls)] - unordered_scripts = [cls for _, cls in inspect.getmembers(module, is_script) if cls not in script_order] - - for cls in [*ordered_scripts, *unordered_scripts]: - # For scripts in submodules use the full import path w/o the root module as the name - script_name = cls.full_name.split(".", maxsplit=1)[1] - module_scripts[script_name] = cls - - if module_scripts: - scripts[module_name] = module_scripts - - return scripts - - def get_script(module_name, script_name): """ Retrieve a script class by module and name. Returns None if the script does not exist. """ - scripts = get_scripts() - module = scripts.get(module_name) - if module: - return module.get(script_name) + module = ScriptModule.objects.get(file_path=f'{module_name}.py') + return module.scripts.get(script_name) diff --git a/netbox/extras/urls.py b/netbox/extras/urls.py index e9bea09aaf6..f346120689e 100644 --- a/netbox/extras/urls.py +++ b/netbox/extras/urls.py @@ -94,19 +94,23 @@ # Reports path('reports/', views.ReportListView.as_view(), name='report_list'), + path('reports/add/', views.ReportModuleCreateView.as_view(), name='reportmodule_add'), path('reports/results//', views.ReportResultView.as_view(), name='report_result'), - re_path(r'^reports/(?P.([^.]+)).(?P.(.+))/', views.ReportView.as_view(), name='report'), + path('reports//', include(get_model_urls('extras', 'reportmodule'))), + path('reports/./', views.ReportView.as_view(), name='report'), + + # Scripts + path('scripts/', views.ScriptListView.as_view(), name='script_list'), + path('scripts/add/', views.ScriptModuleCreateView.as_view(), name='scriptmodule_add'), + path('scripts/results//', views.ScriptResultView.as_view(), name='script_result'), + path('scripts//', include(get_model_urls('extras', 'scriptmodule'))), + path('scripts/./', views.ScriptView.as_view(), name='script'), # Job results path('job-results/', views.JobResultListView.as_view(), name='jobresult_list'), path('job-results/delete/', views.JobResultBulkDeleteView.as_view(), name='jobresult_bulk_delete'), path('job-results//delete/', views.JobResultDeleteView.as_view(), name='jobresult_delete'), - # Scripts - path('scripts/', views.ScriptListView.as_view(), name='script_list'), - path('scripts/results//', views.ScriptResultView.as_view(), name='script_result'), - re_path(r'^scripts/(?P.([^.]+)).(?P.(.+))/', views.ScriptView.as_view(), name='script'), - # Markdown path('render/markdown/', views.RenderMarkdownView.as_view(), name="render_markdown") ] diff --git a/netbox/extras/utils.py b/netbox/extras/utils.py index f90858bcfd5..23892e09819 100644 --- a/netbox/extras/utils.py +++ b/netbox/extras/utils.py @@ -66,3 +66,25 @@ def register_features(model, features): raise KeyError( f"{feature} is not a valid model feature! Valid keys are: {registry['model_features'].keys()}" ) + + +def is_script(obj): + """ + Returns True if the object is a Script. + """ + from .scripts import Script + try: + return issubclass(obj, Script) and obj != Script + except TypeError: + return False + + +def is_report(obj): + """ + Returns True if the given object is a Report. + """ + from .reports import Report + try: + return issubclass(obj, Report) and obj != Report + except TypeError: + return False diff --git a/netbox/extras/views.py b/netbox/extras/views.py index 8677c505e23..2ab354bab4c 100644 --- a/netbox/extras/views.py +++ b/netbox/extras/views.py @@ -7,6 +7,8 @@ from django.urls import reverse from django.views.generic import View +from core.choices import ManagedFileRootPathChoices +from core.forms import ManagedFileForm from extras.dashboard.forms import DashboardWidgetAddForm, DashboardWidgetForm from extras.dashboard.utils import get_widget_class from netbox.views import generic @@ -20,8 +22,8 @@ from .choices import JobResultStatusChoices from .forms.reports import ReportForm from .models import * -from .reports import get_report, get_reports, run_report -from .scripts import get_scripts, run_script +from .reports import get_report, run_report +from .scripts import run_script # @@ -790,18 +792,34 @@ def post(self, request, id): # Reports # +@register_model_view(ReportModule, 'edit') +class ReportModuleCreateView(generic.ObjectEditView): + queryset = ReportModule.objects.all() + form = ManagedFileForm + + def alter_object(self, obj, *args, **kwargs): + obj.file_root = ManagedFileRootPathChoices.REPORTS + return obj + + +@register_model_view(ReportModule, 'delete') +class ReportModuleDeleteView(generic.ObjectDeleteView): + queryset = ReportModule.objects.all() + default_return_url = 'extras:report_list' + + class ReportListView(ContentTypePermissionRequiredMixin, View): """ - Retrieve all of the available reports from disk and the recorded JobResult (if any) for each. + Retrieve all the available reports from disk and the recorded JobResult (if any) for each. """ def get_required_permission(self): return 'extras.view_report' def get(self, request): + report_modules = ReportModule.objects.restrict(request.user) - reports = get_reports() report_content_type = ContentType.objects.get(app_label='extras', model='report') - results = { + job_results = { r.name: r for r in JobResult.objects.filter( obj_type=report_content_type, @@ -809,17 +827,10 @@ def get(self, request): ).order_by('name', '-created').distinct('name').defer('data') } - ret = [] - - for module, report_list in reports.items(): - module_reports = [] - for report in report_list.values(): - report.result = results.get(report.full_name, None) - module_reports.append(report) - ret.append((module, module_reports)) - return render(request, 'extras/report_list.html', { - 'reports': ret, + 'model': ReportModule, + 'report_modules': report_modules, + 'job_results': job_results, }) @@ -831,10 +842,8 @@ def get_required_permission(self): return 'extras.view_report' def get(self, request, module, name): - - report = get_report(module, name) - if report is None: - raise Http404 + module = get_object_or_404(ReportModule.objects.restrict(request.user), file_path=f'{module}.py') + report = module.reports[name]() report_content_type = ContentType.objects.get(app_label='extras', model='report') report.result = JobResult.objects.filter( @@ -844,20 +853,17 @@ def get(self, request, module, name): ).first() return render(request, 'extras/report.html', { + 'module': module, 'report': report, 'form': ReportForm(), }) def post(self, request, module, name): - - # Permissions check if not request.user.has_perm('extras.run_report'): return HttpResponseForbidden() - report = get_report(module, name) - if report is None: - raise Http404 - + module = get_object_or_404(ReportModule.objects.restrict(request.user), file_path=f'{module}.py') + report = module.reports[name]() form = ReportForm(request.POST) if form.is_valid(): @@ -883,6 +889,7 @@ def post(self, request, module, name): return redirect('extras:report_result', job_result_pk=job_result.pk) return render(request, 'extras/report.html', { + 'module': module, 'report': report, 'form': form, }) @@ -924,15 +931,20 @@ def get(self, request, job_result_pk): # Scripts # -class GetScriptMixin: - def _get_script(self, name, module=None): - if module is None: - module, name = name.split('.', 1) - scripts = get_scripts() - try: - return scripts[module][name]() - except KeyError: - raise Http404 +@register_model_view(ScriptModule, 'edit') +class ScriptModuleCreateView(generic.ObjectEditView): + queryset = ScriptModule.objects.all() + form = ManagedFileForm + + def alter_object(self, obj, *args, **kwargs): + obj.file_root = ManagedFileRootPathChoices.SCRIPTS + return obj + + +@register_model_view(ScriptModule, 'delete') +class ScriptModuleDeleteView(generic.ObjectDeleteView): + queryset = ScriptModule.objects.all() + default_return_url = 'extras:script_list' class ScriptListView(ContentTypePermissionRequiredMixin, View): @@ -941,10 +953,10 @@ def get_required_permission(self): return 'extras.view_script' def get(self, request): + script_modules = ScriptModule.objects.restrict(request.user) - scripts = get_scripts(use_names=True) script_content_type = ContentType.objects.get(app_label='extras', model='script') - results = { + job_results = { r.name: r for r in JobResult.objects.filter( obj_type=script_content_type, @@ -952,22 +964,21 @@ def get(self, request): ).order_by('name', '-created').distinct('name').defer('data') } - for _scripts in scripts.values(): - for script in _scripts.values(): - script.result = results.get(script.full_name) - return render(request, 'extras/script_list.html', { - 'scripts': scripts, + 'model': ScriptModule, + 'script_modules': script_modules, + 'job_results': job_results, }) -class ScriptView(ContentTypePermissionRequiredMixin, GetScriptMixin, View): +class ScriptView(ContentTypePermissionRequiredMixin, View): def get_required_permission(self): return 'extras.view_script' def get(self, request, module, name): - script = self._get_script(name, module) + module = get_object_or_404(ScriptModule.objects.restrict(request.user), file_path=f'{module}.py') + script = module.scripts[name]() form = script.as_form(initial=normalize_querydict(request.GET)) # Look for a pending JobResult (use the latest one by creation timestamp) @@ -985,12 +996,11 @@ def get(self, request, module, name): }) def post(self, request, module, name): - - # Permissions check if not request.user.has_perm('extras.run_script'): return HttpResponseForbidden() - script = self._get_script(name, module) + module = get_object_or_404(ScriptModule.objects.restrict(request.user), file_path=f'{module}.py') + script = module.scripts[name]() form = script.as_form(request.POST, request.FILES) # Allow execution only if RQ worker process is running @@ -1020,7 +1030,7 @@ def post(self, request, module, name): }) -class ScriptResultView(ContentTypePermissionRequiredMixin, GetScriptMixin, View): +class ScriptResultView(ContentTypePermissionRequiredMixin, View): def get_required_permission(self): return 'extras.view_script' @@ -1031,7 +1041,9 @@ def get(self, request, job_result_pk): if result.obj_type != script_content_type: raise Http404 - script = self._get_script(result.name) + module_name, script_name = result.name.split('.', 1) + module = get_object_or_404(ScriptModule.objects.restrict(request.user), file_path=f'{module_name}.py') + script = module.scripts[script_name]() # If this is an HTMX request, return only the result HTML if is_htmx(request): diff --git a/netbox/templates/extras/htmx/report_result.html b/netbox/templates/extras/htmx/report_result.html index ddf2c94f4b0..fcf8cae68c4 100644 --- a/netbox/templates/extras/htmx/report_result.html +++ b/netbox/templates/extras/htmx/report_result.html @@ -12,7 +12,7 @@ {% if result.completed %} Duration: {{ result.duration }} {% endif %} - {% include 'extras/inc/job_label.html' %} + {% badge result.get_status_display result.get_status_color %}

{% if result.completed %}
diff --git a/netbox/templates/extras/htmx/script_result.html b/netbox/templates/extras/htmx/script_result.html index fe06b830986..6037c305230 100644 --- a/netbox/templates/extras/htmx/script_result.html +++ b/netbox/templates/extras/htmx/script_result.html @@ -13,7 +13,7 @@ {% if result.completed %} Duration: {{ result.duration }} {% endif %} - {% include 'extras/inc/job_label.html' %} + {% badge result.get_status_display result.get_status_color %}

{% if result.completed %}
diff --git a/netbox/templates/extras/inc/job_label.html b/netbox/templates/extras/inc/job_label.html deleted file mode 100644 index 7ff788edef5..00000000000 --- a/netbox/templates/extras/inc/job_label.html +++ /dev/null @@ -1,15 +0,0 @@ -{% if result.status == 'failed' %} - Failed -{% elif result.status == 'errored' %} - Errored -{% elif result.status == 'pending' %} - Pending -{% elif result.status == 'scheduled' %} - Scheduled -{% elif result.status == 'running' %} - Running -{% elif result.status == 'completed' %} - Completed -{% else %} - N/A -{% endif %} diff --git a/netbox/templates/extras/report.html b/netbox/templates/extras/report.html index a3876d31927..382c0669f7b 100644 --- a/netbox/templates/extras/report.html +++ b/netbox/templates/extras/report.html @@ -10,7 +10,7 @@ {% block breadcrumbs %}
  • Reports
    • -
  • {{ report.module|bettertitle }}
    • +
  • {{ report.module|bettertitle }}
    • {% endblock breadcrumbs %} {% block subtitle %} diff --git a/netbox/templates/extras/report_list.html b/netbox/templates/extras/report_list.html index 56b92c96dd9..f2c52701319 100644 --- a/netbox/templates/extras/report_list.html +++ b/netbox/templates/extras/report_list.html @@ -1,5 +1,7 @@ {% extends 'base/layout.html' %} +{% load buttons %} {% load helpers %} +{% load perms %} {% block title %}Reports{% endblock %} @@ -11,50 +13,67 @@ {% endblock tabs %} +{% block controls %} +
    +
    + {% block extra_controls %}{% endblock %} + {% add_button model %} +
    +
    +{% endblock controls %} + {% block content-wrapper %}
    - {% if reports %} - {% for module, module_reports in reports %} -
    -
    - - {{ module|bettertitle }} -
    -
    - - - - - - - - - - - - {% for report in module_reports %} + {% for module in report_modules %} +
    +
    + {% if perms.extras.delete_reportmodule %} +
    + + Delete + +
    + {% endif %} + {{ module.name|bettertitle }} +
    +
    + {% include 'inc/sync_warning.html' with object=module %} +
    NameStatusDescriptionLast Run
    + + + + + + + + + + + {% for report_name, report in module.reports.items %} + {% with last_result=job_results|get_key:report.full_name %} - - + {% if last_result %} + + + {% else %} + + + {% endif %} - {% for method, stats in report.result.data.items %} + {% for method, stats in last_result.data.items %} {% endfor %} - {% endfor %} - -
    NameDescriptionLast RunStatus
    - {{ report.name }} - - {% include 'extras/inc/job_label.html' with result=report.result %} + {{ report.name }} {{ report.description|markdown|placeholder }} - {% if report.result %} - {{ report.result.created|annotated_date }} - {% else %} - Never - {% endif %} - + {{ last_result.created|annotated_date }} + + {% badge last_result.get_status_display last_result.get_status_color %} + Never{{ ''|placeholder }} {% if perms.extras.run_report %}
    {% csrf_token %}
    {{ method }} @@ -78,19 +97,19 @@
    -
    + {% endwith %} + {% endfor %} + +
    - {% endfor %} - {% else %} +
    + {% empty %} - {% endif %} + {% endfor %}
    {% endblock content-wrapper %} diff --git a/netbox/templates/extras/script.html b/netbox/templates/extras/script.html index a7093ec151a..cc12d5a7c19 100644 --- a/netbox/templates/extras/script.html +++ b/netbox/templates/extras/script.html @@ -11,7 +11,7 @@ {% block breadcrumbs %}
  • Scripts
    • -
  • {{ module|bettertitle }}
    • +
  • {{ module|bettertitle }}
    • {% endblock breadcrumbs %} {% block subtitle %} diff --git a/netbox/templates/extras/script_list.html b/netbox/templates/extras/script_list.html index 1f34f4d5eb4..7377d5e8a94 100644 --- a/netbox/templates/extras/script_list.html +++ b/netbox/templates/extras/script_list.html @@ -1,8 +1,18 @@ {% extends 'base/layout.html' %} +{% load buttons %} {% load helpers %} {% block title %}Scripts{% endblock %} +{% block controls %} +
    +
    + {% block extra_controls %}{% endblock %} + {% add_button model %} +
    +
    +{% endblock controls %} + {% block tabs %}
    {% endblock content-wrapper %}