From dd0e0bdf0c7089be1977b3a498cafe912159518b Mon Sep 17 00:00:00 2001 From: tdruez <489057+tdruez@users.noreply.github.com> Date: Fri, 17 Jan 2025 16:59:34 +0400 Subject: [PATCH 01/18] Add support for download URL as --input-list in batch-create #1524 (#1544) Signed-off-by: tdruez --- docs/command-line-interface.rst | 28 ++++++++++++++++---- scanpipe/management/commands/batch-create.py | 17 ++++++++++-- 2 files changed, 38 insertions(+), 7 deletions(-) diff --git a/docs/command-line-interface.rst b/docs/command-line-interface.rst index ef16047a8..9f0101f73 100644 --- a/docs/command-line-interface.rst +++ b/docs/command-line-interface.rst @@ -174,6 +174,10 @@ Required arguments (one of): | project-2 | pkg:deb/debian/curl@7.50.3 | +----------------+---------------------------------+ +.. tip:: + In place of a local path, a download URL to the CSV file is supported for the + ``--input-list`` argument. + Optional arguments: - ``--project-name-suffix`` Optional custom suffix to append to project names. @@ -194,14 +198,15 @@ Optional arguments: Example: Processing Multiple Docker Images ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Assume multiple Docker images are available in a directory named ``local-data/`` on +Suppose you have multiple Docker images stored in a directory named ``local-data/`` on the host machine. -To process these images with the ``analyze_docker_image`` pipeline using asynchronous -execution:: +To process these images using the ``analyze_docker_image`` pipeline with asynchronous +execution, you can use this command:: $ docker compose run --rm \ - --volume local-data/:/input-data:ro \ - web scanpipe batch-create input-data/ \ + --volume local-data/:/input-data/:ro \ + web scanpipe batch-create + --input-directory /input-data/ \ --pipeline analyze_docker_image \ --label "Docker" \ --execute --async @@ -224,6 +229,19 @@ Each Docker image in the ``local-data/`` directory will result in the creation o project with the specified pipeline (``analyze_docker_image``) executed by worker services. +Example: Processing Multiple Develop to Deploy Mapping +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +To process an input list CSV file with the ``map_deploy_to_develop`` pipeline using +asynchronous execution:: + + $ docker compose run --rm \ + web scanpipe batch-create \ + --input-list https://url/input_list.csv \ + --pipeline map_deploy_to_develop \ + --label "d2d_mapping" \ + --execute --async + `$ scanpipe list-pipeline [--verbosity {0,1,2,3}]` -------------------------------------------------- diff --git a/scanpipe/management/commands/batch-create.py b/scanpipe/management/commands/batch-create.py index 47b5c36c5..cf56d5aa4 100644 --- a/scanpipe/management/commands/batch-create.py +++ b/scanpipe/management/commands/batch-create.py @@ -27,8 +27,11 @@ from django.core.management import CommandError from django.core.management.base import BaseCommand +import requests + from scanpipe.management.commands import CreateProjectCommandMixin from scanpipe.management.commands import PipelineCommandMixin +from scanpipe.pipes import fetch class Command(CreateProjectCommandMixin, PipelineCommandMixin, BaseCommand): @@ -54,7 +57,8 @@ def add_arguments(self, parser): "Path to a CSV file with project names and input URLs. " "The first column must contain project names, and the second column " "should list comma-separated input URLs (e.g., Download URL, PURL, or " - "Docker reference)." + "Docker reference). " + "In place of a local path, a download URL to the CSV file is supported." ), ) parser.add_argument( @@ -110,7 +114,16 @@ def handle_input_directory(self, **options): self.created_project_count += 1 def handle_input_list(self, **options): - input_file = Path(options["input_list"]) + input_file = options["input_list"] + + if input_file.startswith("http"): + try: + download = fetch.fetch_http(input_file) + except requests.exceptions.RequestException as e: + raise CommandError(e) + input_file = download.path + + input_file = Path(input_file) if not input_file.exists(): raise CommandError(f"The {input_file} file does not exist.") From ce227cd622e6284728850a5102cc00628db937d7 Mon Sep 17 00:00:00 2001 From: tdruez <489057+tdruez@users.noreply.github.com> Date: Fri, 17 Jan 2025 17:42:45 +0400 Subject: [PATCH 02/18] Add a report management command to generate XLSX reports #1524 (#1545) * Add a report management command to generate XLSX reports #1524 Signed-off-by: tdruez * Rename "todos" in "todo" in mapping for consistency #1524 Signed-off-by: tdruez * Set the proper exclude for the report XLSX outputs #1524 Signed-off-by: tdruez --------- Signed-off-by: tdruez --- CHANGELOG.rst | 4 + docs/command-line-interface.rst | 41 +++++++++ scanpipe/forms.py | 2 +- scanpipe/management/commands/report.py | 121 +++++++++++++++++++++++++ scanpipe/pipes/output.py | 28 ++++-- scanpipe/tests/test_commands.py | 48 ++++++++++ scanpipe/tests/test_views.py | 2 +- scanpipe/views.py | 3 +- 8 files changed, 236 insertions(+), 13 deletions(-) create mode 100644 scanpipe/management/commands/report.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 54d7dd356..0e930bb49 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -47,6 +47,10 @@ v34.9.4 (unreleased) sheets with a dedicated VULNERABILITIES sheet. https://github.com/aboutcode-org/scancode.io/issues/1519 +- Add a ``report`` management command that allows to generate XLSX reports for + multiple projects at once using labels and searching by project name. + https://github.com/aboutcode-org/scancode.io/issues/1524 + v34.9.3 (2024-12-31) -------------------- diff --git a/docs/command-line-interface.rst b/docs/command-line-interface.rst index 9f0101f73..28875183b 100644 --- a/docs/command-line-interface.rst +++ b/docs/command-line-interface.rst @@ -68,6 +68,7 @@ ScanPipe's own commands are listed under the ``[scanpipe]`` section:: list-project output purldb-scan-worker + report reset-project run show-pipeline @@ -393,6 +394,46 @@ your outputs on the host machine when running with Docker. .. tip:: To specify a CycloneDX spec version (default to latest), use the syntax ``cyclonedx:VERSION`` as format value. For example: ``--format cyclonedx:1.5``. +.. _cli_report: + +`$ scanpipe report --sheet SHEET` +--------------------------------- + +Generates an XLSX report of selected projects based on the provided criteria. + +Required arguments: + +- ``--sheet {package,dependency,resource,relation,message,todo}`` + Specifies the sheet to include in the XLSX report. Available choices are based on + predefined object types. + +Optional arguments: + +- ``--output-directory OUTPUT_DIRECTORY`` + The path to the directory where the report file will be created. If not provided, + the report file will be created in the current working directory. + +- ``--search SEARCH`` + Filter projects by searching for the provided string in their name. + +- ``--label LABELS`` + Filter projects by the provided label(s). Multiple labels can be provided by using + this argument multiple times. + +.. note:: + Either ``--label`` or ``--search`` must be provided to select projects. + +Example usage: + +1. Generate a report for all projects tagged with "d2d" and include the **TODOS** +worksheet:: + + $ scanpipe report --sheet todo --label d2d + +2. Generate a report for projects whose names contain the word "audit" and include the +**PACKAGES** worksheet:: + + $ scanpipe report --sheet package --search audit .. _cli_check_compliance: diff --git a/scanpipe/forms.py b/scanpipe/forms.py index d8c539258..0d7c543ba 100644 --- a/scanpipe/forms.py +++ b/scanpipe/forms.py @@ -281,7 +281,7 @@ class ProjectReportForm(forms.Form): ("codebaseresource", "Resources"), ("codebaserelation", "Relations"), ("projectmessage", "Messages"), - ("todos", "TODOs"), + ("todo", "TODOs"), ], required=True, initial="discoveredpackage", diff --git a/scanpipe/management/commands/report.py b/scanpipe/management/commands/report.py new file mode 100644 index 000000000..f2912ae71 --- /dev/null +++ b/scanpipe/management/commands/report.py @@ -0,0 +1,121 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# http://nexb.com and https://github.com/aboutcode-org/scancode.io +# The ScanCode.io software is licensed under the Apache License version 2.0. +# Data generated with ScanCode.io is provided as-is without warranties. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode.io should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# +# ScanCode.io is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/aboutcode-org/scancode.io for support and download. + +from pathlib import Path +from timeit import default_timer as timer + +from django.core.management import CommandError +from django.core.management.base import BaseCommand + +import xlsxwriter + +from aboutcode.pipeline import humanize_time +from scanpipe.models import Project +from scanpipe.pipes import filename_now +from scanpipe.pipes import output + + +class Command(BaseCommand): + help = "Report of selected projects." + + def add_arguments(self, parser): + super().add_arguments(parser) + parser.add_argument( + "--output-directory", + help=( + "The path to the directory where the report file will be created. " + "If not provided, the report file will be created in the current " + "working directory." + ), + ) + parser.add_argument( + "--sheet", + required=True, + choices=list(output.object_type_to_model_name.keys()), + help="Specifies the sheet to include in the XLSX report.", + ) + parser.add_argument( + "--search", + help="Select projects searching for the provided string in their name.", + ) + parser.add_argument( + "--label", + action="append", + dest="labels", + default=list(), + help=( + "Filter projects by the provided label(s). Multiple labels can be " + "provided by using this argument multiple times." + ), + ) + + def handle(self, *args, **options): + start_time = timer() + self.verbosity = options["verbosity"] + + output_directory = options["output_directory"] + labels = options["labels"] + search = options["search"] + sheet = options["sheet"] + model_name = output.object_type_to_model_name.get(sheet) + + if not (labels or search): + raise CommandError( + "You must provide either --label or --search to select projects." + ) + + project_qs = Project.objects.all() + if labels: + project_qs = project_qs.filter(labels__name__in=labels) + if search: + project_qs = project_qs.filter(name__icontains=search) + project_count = project_qs.count() + + if not project_count: + raise CommandError("No projects found for the provided criteria.") + + if self.verbosity > 0: + msg = f"{project_count} project(s) will be included in the report." + self.stdout.write(msg, self.style.SUCCESS) + + worksheet_queryset = output.get_queryset(project=None, model_name=model_name) + worksheet_queryset = worksheet_queryset.filter(project__in=project_qs) + + filename = f"scancodeio-report-{filename_now()}.xlsx" + if output_directory: + output_file = Path(f"{output_directory}/{filename}") + else: + output_file = Path(filename) + + with xlsxwriter.Workbook(output_file) as workbook: + output.queryset_to_xlsx_worksheet( + worksheet_queryset, + workbook, + exclude_fields=output.XLSX_EXCLUDE_FIELDS, + prepend_fields=["project"], + worksheet_name="TODOS", + ) + + run_time = timer() - start_time + if self.verbosity > 0: + msg = f"Report generated at {output_file} in {humanize_time(run_time)}." + self.stdout.write(msg, self.style.SUCCESS) diff --git a/scanpipe/pipes/output.py b/scanpipe/pipes/output.py index 786fad734..84a795b09 100644 --- a/scanpipe/pipes/output.py +++ b/scanpipe/pipes/output.py @@ -96,7 +96,7 @@ def get_queryset(project, model_name): CodebaseRelation.objects.select_related("from_resource", "to_resource") ), "projectmessage": ProjectMessage.objects.all(), - "todos": CodebaseResource.objects.files().status(flag.REQUIRES_REVIEW), + "todo": CodebaseResource.objects.files().status(flag.REQUIRES_REVIEW), } queryset = querysets.get(model_name) @@ -309,6 +309,11 @@ def to_json(project): "codebaseresource": "resource", "codebaserelation": "relation", "projectmessage": "message", + "todo": "todo", +} + +object_type_to_model_name = { + value: key for key, value in model_name_to_object_type.items() } @@ -469,6 +474,16 @@ def _adapt_value_for_xlsx(fieldname, value, maximum_length=32767, _adapt=True): return value, error +XLSX_EXCLUDE_FIELDS = [ + "extra_data", + "package_data", + "license_detections", + "other_license_detections", + "license_clues", + "affected_by_vulnerabilities", +] + + def to_xlsx(project): """ Generate output for the provided ``project`` in XLSX format. @@ -479,15 +494,8 @@ def to_xlsx(project): with possible error messages for a row when converting the data to XLSX exceed the limits of what can be stored in a cell. """ + exclude_fields = XLSX_EXCLUDE_FIELDS.copy() output_file = project.get_output_file_path("results", "xlsx") - exclude_fields = [ - "extra_data", - "package_data", - "license_detections", - "other_license_detections", - "license_clues", - "affected_by_vulnerabilities", - ] if not project.policies_enabled: exclude_fields.append("compliance_alert") @@ -572,7 +580,7 @@ def add_vulnerabilities_sheet(workbook, project): def add_todos_sheet(workbook, project, exclude_fields): - todos_queryset = get_queryset(project, "todos") + todos_queryset = get_queryset(project, "todo") if todos_queryset: queryset_to_xlsx_worksheet( todos_queryset, workbook, exclude_fields, worksheet_name="TODOS" diff --git a/scanpipe/tests/test_commands.py b/scanpipe/tests/test_commands.py index 69c2152ee..3f3156dcd 100644 --- a/scanpipe/tests/test_commands.py +++ b/scanpipe/tests/test_commands.py @@ -22,6 +22,7 @@ import datetime import json +import tempfile import uuid from contextlib import redirect_stdout from io import StringIO @@ -37,14 +38,18 @@ from django.test import override_settings from django.utils import timezone +import openpyxl + from scanpipe.management import commands from scanpipe.models import CodebaseResource from scanpipe.models import DiscoveredPackage from scanpipe.models import Project from scanpipe.models import Run from scanpipe.models import WebhookSubscription +from scanpipe.pipes import flag from scanpipe.pipes import purldb from scanpipe.tests import make_package +from scanpipe.tests import make_project from scanpipe.tests import make_resource_file scanpipe_app = apps.get_app_config("scanpipe") @@ -1092,6 +1097,49 @@ def test_scanpipe_management_command_check_compliance(self): ) self.assertEqual(expected, out_value) + def test_scanpipe_management_command_report(self): + project1 = make_project("project1") + label1 = "label1" + project1.labels.add(label1) + make_resource_file(project1, path="file.ext", status=flag.REQUIRES_REVIEW) + make_project("project2") + + expected = "Error: the following arguments are required: --sheet" + with self.assertRaisesMessage(CommandError, expected): + call_command("report") + + options = ["--sheet", "UNKNOWN"] + expected = "Error: argument --sheet: invalid choice: 'UNKNOWN'" + with self.assertRaisesMessage(CommandError, expected): + call_command("report", *options) + + options = ["--sheet", "todo"] + expected = "You must provide either --label or --search to select projects." + with self.assertRaisesMessage(CommandError, expected): + call_command("report", *options) + + expected = "No projects found for the provided criteria." + with self.assertRaisesMessage(CommandError, expected): + call_command("report", *options, *["--label", "UNKNOWN"]) + + output_directory = Path(tempfile.mkdtemp()) + options.extend(["--output-directory", str(output_directory), "--label", label1]) + out = StringIO() + call_command("report", *options, stdout=out) + self.assertIn("1 project(s) will be included in the report.", out.getvalue()) + output_file = list(output_directory.glob("*.xlsx"))[0] + self.assertIn(f"Report generated at {output_file}", out.getvalue()) + + workbook = openpyxl.load_workbook(output_file, read_only=True, data_only=True) + self.assertEqual(["TODOS"], workbook.get_sheet_names()) + todos_sheet = workbook.get_sheet_by_name("TODOS") + header = list(todos_sheet.values)[0] + + self.assertNotIn("extra_data", header) + row1 = list(todos_sheet.values)[1] + expected = ("project1", "file.ext", "file", "file.ext", "requires-review") + self.assertEqual(expected, row1[0:5]) + class ScanPipeManagementCommandMixinTest(TestCase): class CreateProjectCommand( diff --git a/scanpipe/tests/test_views.py b/scanpipe/tests/test_views.py index 79758f03e..a6f47a24b 100644 --- a/scanpipe/tests/test_views.py +++ b/scanpipe/tests/test_views.py @@ -197,7 +197,7 @@ def test_scanpipe_views_project_action_report_view(self): data = { "action": "report", "selected_ids": f"{self.project1.uuid}", - "model_name": "todos", + "model_name": "todo", } response = self.client.post(url, data=data, follow=True) self.assertEqual("report.xlsx", response.filename) diff --git a/scanpipe/views.py b/scanpipe/views.py index 9257daba5..9aafa2981 100644 --- a/scanpipe/views.py +++ b/scanpipe/views.py @@ -466,6 +466,7 @@ def export_xlsx_file_response(self): output.queryset_to_xlsx_worksheet( queryset, workbook, + exclude_fields=output.XLSX_EXCLUDE_FIELDS, prepend_fields=prepend_fields, worksheet_name=worksheet_name, ) @@ -1245,7 +1246,7 @@ def get_export_xlsx_prepend_fields(self): return ["project"] def get_export_xlsx_worksheet_name(self): - if self.report_form.cleaned_data.get("model_name") == "todos": + if self.report_form.cleaned_data.get("model_name") == "todo": return "TODOS" def get_export_xlsx_filename(self): From 54e63b61101c2593f7771408cb631a48dafe475a Mon Sep 17 00:00:00 2001 From: tdruez <489057+tdruez@users.noreply.github.com> Date: Mon, 20 Jan 2025 15:54:59 +0400 Subject: [PATCH 03/18] Keep the InputSource objects when using reset on Projects #1536 (#1549) * Keep the InputSource objects when using ``reset`` on Projects #1536 Signed-off-by: tdruez * Remove connectivity requirement for a unit test Signed-off-by: tdruez --------- Signed-off-by: tdruez --- CHANGELOG.rst | 3 ++ scanpipe/models.py | 10 +++-- scanpipe/tests/__init__.py | 34 +++++++++++----- scanpipe/tests/pipes/test_output.py | 10 +---- scanpipe/tests/test_api.py | 10 +---- scanpipe/tests/test_models.py | 63 +++++++++++++++++------------ scanpipe/tests/test_pipelines.py | 10 +++-- 7 files changed, 81 insertions(+), 59 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 0e930bb49..7a0c2b5f1 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -47,6 +47,9 @@ v34.9.4 (unreleased) sheets with a dedicated VULNERABILITIES sheet. https://github.com/aboutcode-org/scancode.io/issues/1519 +- Keep the InputSource objects when using ``reset`` on Projects. + https://github.com/aboutcode-org/scancode.io/issues/1536 + - Add a ``report`` management command that allows to generate XLSX reports for multiple projects at once using labels and searching by project name. https://github.com/aboutcode-org/scancode.io/issues/1524 diff --git a/scanpipe/models.py b/scanpipe/models.py index e419a79e2..02d8a8e6d 100644 --- a/scanpipe/models.py +++ b/scanpipe/models.py @@ -640,14 +640,14 @@ def archive(self, remove_input=False, remove_codebase=False, remove_output=False self.is_archived = True self.save(update_fields=["is_archived"]) - def delete_related_objects(self): + def delete_related_objects(self, keep_input=False): """ Delete all related object instances using the private `_raw_delete` model API. This bypass the objects collection, cascade deletions, and signals. It results in a much faster objects deletion, but it needs to be applied in the correct models order as the cascading event will not be triggered. Note that this approach is used in Django's `fast_deletes` but the scanpipe - models are cannot be fast-deleted as they have cascades and relations. + models cannot be fast-deleted as they have cascades and relations. """ # Use default `delete()` on the DiscoveredPackage model, as the # `codebase_resources (ManyToManyField)` records need to collected and @@ -667,9 +667,11 @@ def delete_related_objects(self): self.discovereddependencies, self.codebaseresources, self.runs, - self.inputsources, ] + if not keep_input: + relationships.append(self.inputsources) + for qs in relationships: count = qs.all()._raw_delete(qs.db) deleted_counter[qs.model._meta.label] = count @@ -695,7 +697,7 @@ def reset(self, keep_input=True): """ self._raise_if_run_in_progress() - self.delete_related_objects() + self.delete_related_objects(keep_input=keep_input) work_directories = [ self.codebase_path, diff --git a/scanpipe/tests/__init__.py b/scanpipe/tests/__init__.py index fe2451cee..502412750 100644 --- a/scanpipe/tests/__init__.py +++ b/scanpipe/tests/__init__.py @@ -31,6 +31,7 @@ from scanpipe.models import DiscoveredDependency from scanpipe.models import DiscoveredPackage from scanpipe.models import Project +from scanpipe.models import ProjectMessage from scanpipe.tests.pipelines.do_nothing import DoNothing from scanpipe.tests.pipelines.download_inputs import DownloadInput from scanpipe.tests.pipelines.profile_step import ProfileStep @@ -47,12 +48,12 @@ mocked_now = mock.Mock(now=lambda: datetime(2010, 10, 10, 10, 10, 10)) -def make_project(name=None, **extra): +def make_project(name=None, **data): name = name or str(uuid.uuid4())[:8] - return Project.objects.create(name=name, **extra) + return Project.objects.create(name=name, **data) -def make_resource_file(project, path, **extra): +def make_resource_file(project, path, **data): return CodebaseResource.objects.create( project=project, path=path, @@ -61,30 +62,43 @@ def make_resource_file(project, path, **extra): type=CodebaseResource.Type.FILE, is_text=True, tag=path.split("/")[0], - **extra, + **data, ) -def make_resource_directory(project, path, **extra): +def make_resource_directory(project, path, **data): return CodebaseResource.objects.create( project=project, path=path, name=path.split("/")[-1], type=CodebaseResource.Type.DIRECTORY, tag=path.split("/")[0], - **extra, + **data, ) -def make_package(project, package_url, **extra): - package = DiscoveredPackage(project=project, **extra) +def make_package(project, package_url, **data): + package = DiscoveredPackage(project=project, **data) package.set_package_url(package_url) package.save() return package -def make_dependency(project, **extra): - return DiscoveredDependency.objects.create(project=project, **extra) +def make_dependency(project, **data): + return DiscoveredDependency.objects.create(project=project, **data) + + +def make_message(project, **data): + if "model" not in data: + data["model"] = str(uuid.uuid4())[:8] + + if "severity" not in data: + data["severity"] = ProjectMessage.Severity.ERROR + + return ProjectMessage.objects.create( + project=project, + **data, + ) resource_data1 = { diff --git a/scanpipe/tests/pipes/test_output.py b/scanpipe/tests/pipes/test_output.py index 88d20982b..25104710f 100644 --- a/scanpipe/tests/pipes/test_output.py +++ b/scanpipe/tests/pipes/test_output.py @@ -42,11 +42,11 @@ from scanpipe import pipes from scanpipe.models import CodebaseResource from scanpipe.models import Project -from scanpipe.models import ProjectMessage from scanpipe.pipes import flag from scanpipe.pipes import output from scanpipe.tests import FIXTURES_REGEN from scanpipe.tests import make_dependency +from scanpipe.tests import make_message from scanpipe.tests import make_package from scanpipe.tests import make_resource_file from scanpipe.tests import mocked_now @@ -206,13 +206,7 @@ def test_scanpipe_pipes_outputs_to_xlsx(self): call_command("loaddata", fixtures, **{"verbosity": 0}) project = Project.objects.get(name="asgiref") - ProjectMessage.objects.create( - project=project, - severity=ProjectMessage.Severity.ERROR, - description="Error", - model="Model", - details={}, - ) + make_message(project, description="Error") make_resource_file( project=project, path="path/file1.ext", status=flag.REQUIRES_REVIEW ) diff --git a/scanpipe/tests/test_api.py b/scanpipe/tests/test_api.py index d91087946..5969b09da 100644 --- a/scanpipe/tests/test_api.py +++ b/scanpipe/tests/test_api.py @@ -54,6 +54,7 @@ from scanpipe.pipes.input import copy_input from scanpipe.pipes.output import JSONResultsGenerator from scanpipe.tests import dependency_data1 +from scanpipe.tests import make_message from scanpipe.tests import make_package from scanpipe.tests import make_project from scanpipe.tests import make_resource_file @@ -795,13 +796,7 @@ def test_scanpipe_api_project_action_relations_filterset(self): def test_scanpipe_api_project_action_messages(self): url = reverse("project-messages", args=[self.project1.uuid]) - ProjectMessage.objects.create( - project=self.project1, - severity=ProjectMessage.Severity.ERROR, - description="Error", - model="ModelName", - details={}, - ) + make_message(self.project1, description="Error") response = self.csrf_client.get(url) self.assertEqual(1, response.data["count"]) @@ -812,7 +807,6 @@ def test_scanpipe_api_project_action_messages(self): message = response.data["results"][0] self.assertEqual("error", message["severity"]) self.assertEqual("Error", message["description"]) - self.assertEqual("ModelName", message["model"]) self.assertEqual({}, message["details"]) def test_scanpipe_api_project_action_file_content(self): diff --git a/scanpipe/tests/test_models.py b/scanpipe/tests/test_models.py index 71064fcdf..267f8e98f 100644 --- a/scanpipe/tests/test_models.py +++ b/scanpipe/tests/test_models.py @@ -76,7 +76,9 @@ from scanpipe.tests import dependency_data2 from scanpipe.tests import license_policies_index from scanpipe.tests import make_dependency +from scanpipe.tests import make_message from scanpipe.tests import make_package +from scanpipe.tests import make_project from scanpipe.tests import make_resource_directory from scanpipe.tests import make_resource_file from scanpipe.tests import mocked_now @@ -93,7 +95,7 @@ class ScanPipeModelsTest(TestCase): fixtures = [data / "asgiref" / "asgiref-3.3.0_fixtures.json"] def setUp(self): - self.project1 = Project.objects.create(name="Analysis") + self.project1 = make_project("Analysis") self.project_asgiref = Project.objects.get(name="asgiref") def create_run(self, pipeline="pipeline", **kwargs): @@ -118,7 +120,7 @@ def test_scanpipe_project_model_work_directories(self): self.assertTrue(self.project1.tmp_path.exists()) def test_scanpipe_get_project_work_directory(self): - project = Project.objects.create(name="Name with spaces and @£$éæ") + project = make_project("Name with spaces and @£$éæ") expected = f"/projects/name-with-spaces-and-e-{project.short_uuid}" self.assertTrue(get_project_work_directory(project).endswith(expected)) self.assertTrue(project.work_directory.endswith(expected)) @@ -191,7 +193,7 @@ def test_scanpipe_project_model_delete(self): self.assertTrue(work_path.exists()) uploaded_file = SimpleUploadedFile("file.ext", content=b"content") - self.project1.write_input_file(uploaded_file) + self.project1.add_upload(uploaded_file=uploaded_file, tag="tag1") self.project1.add_pipeline("analyze_docker_image") resource = CodebaseResource.objects.create(project=self.project1, path="path") package = DiscoveredPackage.objects.create(project=self.project1) @@ -209,11 +211,18 @@ def test_scanpipe_project_model_reset(self): self.assertTrue(work_path.exists()) uploaded_file = SimpleUploadedFile("file.ext", content=b"content") - self.project1.write_input_file(uploaded_file) + self.project1.add_upload(uploaded_file=uploaded_file, tag="tag1") self.project1.add_pipeline("analyze_docker_image") resource = CodebaseResource.objects.create(project=self.project1, path="path") package = DiscoveredPackage.objects.create(project=self.project1) resource.discovered_packages.add(package) + make_message(self.project1, description="Error") + + self.assertEqual(1, self.project1.projectmessages.count()) + self.assertEqual(1, self.project1.runs.count()) + self.assertEqual(1, self.project1.discoveredpackages.count()) + self.assertEqual(1, self.project1.codebaseresources.count()) + self.assertEqual(1, self.project1.inputsources.count()) self.project1.reset() @@ -223,6 +232,8 @@ def test_scanpipe_project_model_reset(self): self.assertEqual(0, self.project1.discoveredpackages.count()) self.assertEqual(0, self.project1.codebaseresources.count()) + # The InputSource objects are kept + self.assertEqual(1, self.project1.inputsources.count()) self.assertTrue(work_path.exists()) self.assertTrue(self.project1.input_path.exists()) self.assertEqual(["file.ext"], self.project1.input_root) @@ -604,7 +615,7 @@ def test_scanpipe_project_related_model_clone(self): target_url="http://domain.url" ) - new_project = Project.objects.create(name="New Project") + new_project = make_project("New Project") subscription1.clone(to_project=new_project) cloned_subscription = new_project.webhooksubscriptions.get() @@ -1884,7 +1895,7 @@ def test_scanpipe_discovered_package_queryset_vulnerable(self): self.assertIn(p2, DiscoveredPackage.objects.vulnerable()) def test_scanpipe_discovered_package_queryset_dependency_methods(self): - project = Project.objects.create(name="project") + project = make_project("project") a = make_package(project, "pkg:type/a") b = make_package(project, "pkg:type/b") c = make_package(project, "pkg:type/c") @@ -1978,7 +1989,7 @@ def test_scanpipe_codebase_resource_model_walk_method(self): self.assertEqual(sorted(expected_siblings), sorted(asgiref_resource_siblings)) def test_scanpipe_codebase_resource_model_walk_method_problematic_filenames(self): - project = Project.objects.create(name="walk_test_problematic_filenames") + project = make_project("walk_test_problematic_filenames") resource1 = CodebaseResource.objects.create( project=project, path="qt-everywhere-opensource-src-5.3.2/gnuwin32/bin" ) @@ -2331,7 +2342,7 @@ def test_scanpipe_discovered_dependency_model_update_from_data(self): self.assertEqual(new_data["scope"], dependency.scope) def test_scanpipe_discovered_dependency_model_many_to_many(self): - project = Project.objects.create(name="project") + project = make_project("project") a = make_package(project, "pkg:type/a") b = make_package(project, "pkg:type/b") @@ -2432,7 +2443,7 @@ def test_scanpipe_codebase_resource_queryset_has_directory_content_fingerprint( self.assertQuerySetEqual(expected, results, ordered=False) def test_scanpipe_codebase_resource_queryset_elfs(self): - project = Project.objects.create(name="Test") + project = make_project("Test") resource_starting_with_elf_and_executable_in_file_type = CodebaseResource( file_type="""ELF 32-bit LSB executable, ARM, version 1 (ARM), statically linked, with debug_info, not stripped""", @@ -2510,7 +2521,7 @@ class ScanPipeModelsTransactionTest(TransactionTestCase): @mock.patch("scanpipe.models.Run.execute_task_async") def test_scanpipe_project_model_add_pipeline(self, mock_execute_task): - project1 = Project.objects.create(name="Analysis") + project1 = make_project("Analysis") self.assertEqual(0, project1.runs.count()) @@ -2529,13 +2540,13 @@ def test_scanpipe_project_model_add_pipeline(self, mock_execute_task): self.assertEqual(pipeline_class.get_summary(), run.description) mock_execute_task.assert_not_called() - project2 = Project.objects.create(name="Analysis 2") + project2 = make_project("Analysis 2") project2.add_pipeline(pipeline_name, execute_now=True) mock_execute_task.assert_called_once() @mock.patch("scanpipe.models.Run.execute_task_async") def test_scanpipe_project_model_add_pipeline_run_can_start(self, mock_execute_task): - project1 = Project.objects.create(name="Analysis") + project1 = make_project("Analysis") pipeline_name = "inspect_packages" run1 = project1.add_pipeline(pipeline_name, execute_now=False) run2 = project1.add_pipeline(pipeline_name, execute_now=True) @@ -2547,7 +2558,7 @@ def test_scanpipe_project_model_add_pipeline_run_can_start(self, mock_execute_ta @mock.patch("scanpipe.models.Run.execute_task_async") def test_scanpipe_project_model_add_pipeline_start_method(self, mock_execute_task): - project1 = Project.objects.create(name="Analysis") + project1 = make_project("Analysis") pipeline_name = "inspect_packages" run1 = project1.add_pipeline(pipeline_name, execute_now=False) run2 = project1.add_pipeline(pipeline_name, execute_now=False) @@ -2564,7 +2575,7 @@ def test_scanpipe_project_model_add_pipeline_start_method(self, mock_execute_tas mock_execute_task.assert_called_once() def test_scanpipe_project_model_add_pipeline_selected_groups(self): - project1 = Project.objects.create(name="Analysis") + project1 = make_project("Analysis") pipeline_name = "scan_codebase" run1 = project1.add_pipeline(pipeline_name, selected_groups=[]) @@ -2580,7 +2591,7 @@ def test_scanpipe_project_model_add_pipeline_selected_groups(self): project1.add_pipeline(pipeline_name, selected_groups={}) def test_scanpipe_project_model_add_info(self): - project1 = Project.objects.create(name="Analysis") + project1 = make_project("Analysis") message = project1.add_info(description="This is an info") self.assertEqual(message, ProjectMessage.objects.get()) self.assertEqual("", message.model) @@ -2590,7 +2601,7 @@ def test_scanpipe_project_model_add_info(self): self.assertEqual("", message.traceback) def test_scanpipe_project_model_add_warning(self): - project1 = Project.objects.create(name="Analysis") + project1 = make_project("Analysis") message = project1.add_warning(description="This is a warning") self.assertEqual(message, ProjectMessage.objects.get()) self.assertEqual("", message.model) @@ -2600,7 +2611,7 @@ def test_scanpipe_project_model_add_warning(self): self.assertEqual("", message.traceback) def test_scanpipe_project_model_add_error(self): - project1 = Project.objects.create(name="Analysis") + project1 = make_project("Analysis") details = { "name": "value", "release_date": datetime.fromisoformat("2008-02-01"), @@ -2618,7 +2629,7 @@ def test_scanpipe_project_model_add_error(self): self.assertEqual("", message.traceback) def test_scanpipe_project_model_update_extra_data(self): - project1 = Project.objects.create(name="Analysis") + project1 = make_project("Analysis") self.assertEqual({}, project1.extra_data) with self.assertRaises(ValueError): @@ -2647,7 +2658,7 @@ def test_scanpipe_project_model_update_extra_data(self): self.assertEqual(expected, project1.extra_data) def test_scanpipe_codebase_resource_model_add_error(self): - project1 = Project.objects.create(name="Analysis") + project1 = make_project("Analysis") codebase_resource = CodebaseResource.objects.create(project=project1, path="a") error = codebase_resource.add_error(Exception("Error message")) @@ -2659,7 +2670,7 @@ def test_scanpipe_codebase_resource_model_add_error(self): self.assertEqual(codebase_resource.path, error.details["resource_path"]) def test_scanpipe_codebase_resource_model_add_errors(self): - project1 = Project.objects.create(name="Analysis") + project1 = make_project("Analysis") codebase_resource = CodebaseResource.objects.create(project=project1) codebase_resource.add_error(Exception("Error1")) codebase_resource.add_error(Exception("Error2")) @@ -2667,7 +2678,7 @@ def test_scanpipe_codebase_resource_model_add_errors(self): @skipIf(connection.vendor == "sqlite", "No max_length constraints on SQLite.") def test_scanpipe_project_error_model_save_non_valid_related_object(self): - project1 = Project.objects.create(name="Analysis") + project1 = make_project("Analysis") long_value = "value" * 1000 package = DiscoveredPackage.objects.create( @@ -2695,7 +2706,7 @@ def test_scanpipe_project_error_model_save_non_valid_related_object(self): @skipIf(connection.vendor == "sqlite", "No max_length constraints on SQLite.") def test_scanpipe_discovered_package_model_create_from_data(self): - project1 = Project.objects.create(name="Analysis") + project1 = make_project("Analysis") package = DiscoveredPackage.create_from_data(project1, package_data1) self.assertEqual(project1, package.project) @@ -2737,7 +2748,7 @@ def test_scanpipe_discovered_package_model_create_from_data(self): self.assertEqual(project_message_count, ProjectMessage.objects.count()) def test_scanpipe_discovered_package_model_create_from_data_missing_type(self): - project1 = Project.objects.create(name="Analysis") + project1 = make_project("Analysis") incomplete_data = dict(package_data1) incomplete_data["type"] = "" @@ -2749,7 +2760,7 @@ def test_scanpipe_discovered_package_model_create_from_data_missing_type(self): @skipIf(connection.vendor == "sqlite", "No max_length constraints on SQLite.") def test_scanpipe_discovered_dependency_model_create_from_data(self): - project1 = Project.objects.create(name="Analysis") + project1 = make_project("Analysis") DiscoveredPackage.create_from_data(project1, package_data1) CodebaseResource.objects.create( @@ -2797,7 +2808,7 @@ def test_scanpipe_discovered_dependency_model_create_from_data(self): self.assertEqual("", message.traceback) def test_scanpipe_discovered_package_model_unique_package_uid_in_project(self): - project1 = Project.objects.create(name="Analysis") + project1 = make_project("Analysis") self.assertTrue(package_data1["package_uid"]) package = DiscoveredPackage.create_from_data(project1, package_data1) @@ -2817,7 +2828,7 @@ def test_scanpipe_discovered_package_model_unique_package_uid_in_project(self): @skipIf(connection.vendor == "sqlite", "No max_length constraints on SQLite.") def test_scanpipe_codebase_resource_create_and_add_package_warnings(self): - project1 = Project.objects.create(name="Analysis") + project1 = make_project("Analysis") resource = CodebaseResource.objects.create(project=project1, path="p") package_count = DiscoveredPackage.objects.count() diff --git a/scanpipe/tests/test_pipelines.py b/scanpipe/tests/test_pipelines.py index bca1ec48f..5c864a8f7 100644 --- a/scanpipe/tests/test_pipelines.py +++ b/scanpipe/tests/test_pipelines.py @@ -1271,7 +1271,10 @@ def test_scanpipe_resolve_dependencies_pipeline_integration_empty_manifest(self) expected = "No packages could be resolved" self.assertIn(expected, message.description) - def test_scanpipe_resolve_dependencies_pipeline_integration_misc(self): + @mock.patch("scanpipe.pipes.resolve.resolve_dependencies") + def test_scanpipe_resolve_dependencies_pipeline_integration_misc( + self, mock_resolve_dependencies + ): pipeline_name = "resolve_dependencies" project1 = Project.objects.create(name="Analysis") selected_groups = ["DynamicResolver"] @@ -1284,13 +1287,14 @@ def test_scanpipe_resolve_dependencies_pipeline_integration_misc(self): ) pipeline = run.make_pipeline_instance() + mock_resolve_dependencies.return_value = mock.Mock(packages=[package_data1]) exitcode, out = pipeline.execute() self.assertEqual(0, exitcode, msg=out) self.assertEqual(1, project1.discoveredpackages.count()) @mock.patch("scanpipe.pipes.resolve.resolve_dependencies") def test_scanpipe_resolve_dependencies_pipeline_pypi_integration( - self, resolve_dependencies + self, mock_resolve_dependencies ): pipeline_name = "resolve_dependencies" project1 = Project.objects.create(name="Analysis") @@ -1302,7 +1306,7 @@ def test_scanpipe_resolve_dependencies_pipeline_pypi_integration( pipeline = run.make_pipeline_instance() project1.move_input_from(tempfile.mkstemp(suffix="requirements.txt")[1]) - resolve_dependencies.return_value = mock.Mock(packages=[package_data1]) + mock_resolve_dependencies.return_value = mock.Mock(packages=[package_data1]) exitcode, out = pipeline.execute() self.assertEqual(0, exitcode, msg=out) From a6f7c860d4fbd07457dd7b37ea998fd0e54d31f6 Mon Sep 17 00:00:00 2001 From: tdruez <489057+tdruez@users.noreply.github.com> Date: Tue, 21 Jan 2025 17:18:33 +0900 Subject: [PATCH 04/18] Add a select_across checkbox on the ProjectReportForm #1524 (#1534) Signed-off-by: tdruez --- CHANGELOG.rst | 4 + scancodeio/static/main.js | 3 +- scanpipe/forms.py | 27 +++- scanpipe/pipes/__init__.py | 2 +- scanpipe/templates/registration/login.html | 2 - .../modals/project_archive_modal.html | 21 ++- .../modals/projects_archive_modal.html | 40 ++++- .../modals/projects_delete_modal.html | 2 + .../modals/projects_download_modal.html | 23 ++- .../modals/projects_report_modal.html | 23 ++- .../scanpipe/modals/projects_reset_modal.html | 14 ++ .../scanpipe/panels/project_inputs.html | 8 +- scanpipe/templates/scanpipe/project_list.html | 18 +++ scanpipe/tests/test_views.py | 69 ++++++++- scanpipe/views.py | 139 +++++++++++------- 15 files changed, 309 insertions(+), 86 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 7a0c2b5f1..d813ead9a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -54,6 +54,10 @@ v34.9.4 (unreleased) multiple projects at once using labels and searching by project name. https://github.com/aboutcode-org/scancode.io/issues/1524 +- Add the ability to "select across" in Projects list when using the "select all" + checkbox on paginated list. + https://github.com/aboutcode-org/scancode.io/issues/1524 + v34.9.3 (2024-12-31) -------------------- diff --git a/scancodeio/static/main.js b/scancodeio/static/main.js index 04b74db51..0be80c02d 100644 --- a/scancodeio/static/main.js +++ b/scancodeio/static/main.js @@ -232,7 +232,8 @@ function setupSelectCheckbox() { updateButtonAndDropdownState(); // Check if all row checkboxes are checked and update the "Select All" checkbox accordingly - selectAllCheckbox.checked = Array.from(rowCheckboxes).every((cb) => cb.checked); + const allRowCheckboxesChecked = Array.from(rowCheckboxes).every((cb) => cb.checked); + selectAllCheckbox.checked = allRowCheckboxesChecked; }); }); diff --git a/scanpipe/forms.py b/scanpipe/forms.py index 0d7c543ba..94ce46428 100644 --- a/scanpipe/forms.py +++ b/scanpipe/forms.py @@ -238,7 +238,21 @@ def save(self, project): return input_source -class ArchiveProjectForm(forms.Form): +class BaseProjectActionForm(forms.Form): + select_across = forms.BooleanField( + label="", + required=False, + initial=0, + help_text="All project matching current search and filters will be included.", + ) + url_query = forms.CharField( + widget=forms.HiddenInput, + required=False, + help_text="Stores the current URL filters.", + ) + + +class ArchiveProjectForm(BaseProjectActionForm): remove_input = forms.BooleanField( label="Remove inputs", initial=True, @@ -255,8 +269,15 @@ class ArchiveProjectForm(forms.Form): required=False, ) + def get_action_kwargs(self): + return { + "remove_input": self.cleaned_data["remove_input"], + "remove_codebase": self.cleaned_data["remove_codebase"], + "remove_output": self.cleaned_data["remove_output"], + } + -class ProjectOutputDownloadForm(forms.Form): +class ProjectOutputDownloadForm(BaseProjectActionForm): output_format = forms.ChoiceField( label="Choose the output format to include in the ZIP file", choices=[ @@ -272,7 +293,7 @@ class ProjectOutputDownloadForm(forms.Form): ) -class ProjectReportForm(forms.Form): +class ProjectReportForm(BaseProjectActionForm): model_name = forms.ChoiceField( label="Choose the object type to include in the XLSX file", choices=[ diff --git a/scanpipe/pipes/__init__.py b/scanpipe/pipes/__init__.py index df1b5ecc3..610b703a7 100644 --- a/scanpipe/pipes/__init__.py +++ b/scanpipe/pipes/__init__.py @@ -113,7 +113,7 @@ def collect_and_create_codebase_resources(project, batch_size=5000): Collect and create codebase resources including the "to/" and "from/" context using the resource tag field. - The default ``batch_size`` can be overriden, although the benefits of a value + The default ``batch_size`` can be overridden, although the benefits of a value greater than 5000 objects are usually not significant. """ model_class = CodebaseResource diff --git a/scanpipe/templates/registration/login.html b/scanpipe/templates/registration/login.html index ab91dd273..232b02479 100644 --- a/scanpipe/templates/registration/login.html +++ b/scanpipe/templates/registration/login.html @@ -21,7 +21,6 @@
- @@ -32,7 +31,6 @@
- diff --git a/scanpipe/templates/scanpipe/modals/project_archive_modal.html b/scanpipe/templates/scanpipe/modals/project_archive_modal.html index 09bb2f706..5fe738843 100644 --- a/scanpipe/templates/scanpipe/modals/project_archive_modal.html +++ b/scanpipe/templates/scanpipe/modals/project_archive_modal.html @@ -14,9 +14,24 @@

The selected directories will be removed:

-
    - {{ archive_form.as_ul }} -
+
+ +
+
+ +
+
+ +

Are you sure you want to do this?

diff --git a/scanpipe/templates/scanpipe/modals/projects_archive_modal.html b/scanpipe/templates/scanpipe/modals/projects_archive_modal.html index 8a90d1a6e..1fee867cc 100644 --- a/scanpipe/templates/scanpipe/modals/projects_archive_modal.html +++ b/scanpipe/templates/scanpipe/modals/projects_archive_modal.html @@ -1,3 +1,4 @@ +{% load humanize %}