From 608f19a9c7d8d5fd71258bc99a78e4a740394c17 Mon Sep 17 00:00:00 2001 From: Laura Barcziova Date: Wed, 31 Jul 2024 12:33:18 +0200 Subject: [PATCH 1/2] Integrate support for non-git upstreams for pull-from-upstream --- packit_service/worker/checker/distgit.py | 11 +- packit_service/worker/events/new_hotness.py | 16 ++ packit_service/worker/handlers/distgit.py | 18 +- packit_service/worker/mixin.py | 10 +- tests/integration/test_new_hotness_update.py | 165 ++++++++++++++++++ tests/integration/test_pr_comment.py | 167 +++++++++++++++++++ tests/unit/test_distgit.py | 4 +- 7 files changed, 375 insertions(+), 16 deletions(-) diff --git a/packit_service/worker/checker/distgit.py b/packit_service/worker/checker/distgit.py index 1d13bd4e6..0553280c9 100644 --- a/packit_service/worker/checker/distgit.py +++ b/packit_service/worker/checker/distgit.py @@ -197,14 +197,7 @@ def pre_check(self) -> bool: f"{self.data.event_dict.get('version')}" ) - if not self.package_config.upstream_project_url: - msg_to_report = ( - "`upstream_project_url` is not set in " - "the dist-git package configuration." - ) - valid = False - - if not ( + if self.package_config.upstream_project_url and not ( self.data.event_dict.get("repo_name") and self.data.event_dict.get("repo_namespace") ): @@ -215,7 +208,7 @@ def pre_check(self) -> bool: ) valid = False - if ( + if self.package_config.upstream_project_url and ( self.data.event_type in (NewHotnessUpdateEvent.__name__,) and not self.data.tag_name ): diff --git a/packit_service/worker/events/new_hotness.py b/packit_service/worker/events/new_hotness.py index 12c7efaef..dd13c61de 100644 --- a/packit_service/worker/events/new_hotness.py +++ b/packit_service/worker/events/new_hotness.py @@ -60,6 +60,22 @@ def base_project(self): def _add_release_and_event(self): if not self._db_project_object or not self._db_project_event: + if not self.project_url: + # we do not know what is the upstream project + # and it doesn't necessarily have tag, + # let's use dist-git project and version + ( + self._db_project_object, + self._db_project_event, + ) = ProjectEventModel.add_release_event( + tag_name=self.version, + namespace="rpms", + repo_name=self.project.repo, + project_url=self.distgit_project_url, + commit_hash=None, + ) + return + if not ( self.tag_name and self.repo_name diff --git a/packit_service/worker/handlers/distgit.py b/packit_service/worker/handlers/distgit.py index 8fb15c51e..c45271df1 100644 --- a/packit_service/worker/handlers/distgit.py +++ b/packit_service/worker/handlers/distgit.py @@ -202,6 +202,7 @@ class AbstractSyncReleaseHandler( sync_release_job_type: SyncReleaseJobType job_name_for_reporting: str get_dashboard_url: ClassVar # static method from Callable[[int], str] + check_for_non_git_upstreams: bool def __init__( self, @@ -243,9 +244,8 @@ def sync_branch( is_pull_from_upstream_job = ( self.sync_release_job_type == SyncReleaseJobType.pull_from_upstream ) - downstream_pr = self.packit_api.sync_release( + kwargs = dict( dist_git_branch=branch, - tag=self.tag, create_pr=True, local_pr_branch_suffix=branch_suffix, use_downstream_specfile=is_pull_from_upstream_job, @@ -260,6 +260,11 @@ def sync_branch( # [TODO] Remove for CentOS support once it gets refined add_new_sources=self.package_config.pkg_tool in (None, "fedpkg"), ) + if not self.packit_api.non_git_upstream: + kwargs["tag"] = self.tag + elif version := self.data.event_dict.get("version"): + kwargs["versions"] = [version] + downstream_pr = self.packit_api.sync_release(**kwargs) except PackitDownloadFailedException as ex: # the archive has not been uploaded to PyPI yet # retry for the archive to become available @@ -291,9 +296,10 @@ def sync_branch( raise AbortSyncRelease() raise ex finally: - self.packit_api.up.local_project.git_repo.head.reset( - "HEAD", index=True, working_tree=True - ) + if self.packit_api.up.local_project: + self.packit_api.up.local_project.git_repo.head.reset( + "HEAD", index=True, working_tree=True + ) return downstream_pr @@ -533,6 +539,7 @@ class ProposeDownstreamHandler(AbstractSyncReleaseHandler): sync_release_job_type = SyncReleaseJobType.propose_downstream job_name_for_reporting = "propose downstream" get_dashboard_url = staticmethod(get_propose_downstream_info_url) + check_for_non_git_upstreams = False def __init__( self, @@ -594,6 +601,7 @@ class PullFromUpstreamHandler(AbstractSyncReleaseHandler): sync_release_job_type = SyncReleaseJobType.pull_from_upstream job_name_for_reporting = "Pull from upstream" get_dashboard_url = staticmethod(get_pull_from_upstream_info_url) + check_for_non_git_upstreams = True def __init__( self, diff --git a/packit_service/worker/mixin.py b/packit_service/worker/mixin.py index 9425f28d8..03d4849b2 100644 --- a/packit_service/worker/mixin.py +++ b/packit_service/worker/mixin.py @@ -188,9 +188,17 @@ def packit_api(self): upstream_local_project=self.local_project, dist_git_clone_path=Path(self.service_config.command_handler_work_dir) / SANDCASTLE_DG_REPO_DIR, + non_git_upstream=self.non_git_upstream, ) return self._packit_api + @property + def non_git_upstream(self): + return ( + self.check_for_non_git_upstreams + and self.job_config.upstream_project_url is None + ) + def clean_api(self) -> None: if self._packit_api: self._packit_api.clean() @@ -202,7 +210,7 @@ class GetSyncReleaseTagMixin(PackitAPIWithUpstreamMixin): @property def tag(self) -> Optional[str]: self._tag = self.data.tag_name - if not self._tag: + if not self._tag and not self.non_git_upstream: # there is no tag information when retriggering pull-from-upstream # from dist-git PR self._tag = self.packit_api.up.get_last_tag() diff --git a/tests/integration/test_new_hotness_update.py b/tests/integration/test_new_hotness_update.py index 6fbd71a35..91709d6be 100644 --- a/tests/integration/test_new_hotness_update.py +++ b/tests/integration/test_new_hotness_update.py @@ -77,6 +77,42 @@ def sync_release_model(): yield sync_release_model +@pytest.fixture +def sync_release_model_non_git(): + db_project_object = flexmock( + id=12, + project_event_model_type=ProjectEventModelType.release, + job_config_trigger_type=JobConfigTriggerType.release, + project=flexmock(project_url="https://src.fedoraproject.org/rpms/redis"), + ) + project_event = ( + flexmock() + .should_receive("get_project_event_object") + .and_return(db_project_object) + .mock() + ) + run_model = flexmock(PipelineModel) + flexmock(ProjectEventModel).should_receive("get_or_create").with_args( + type=ProjectEventModelType.release, event_id=12, commit_sha=None + ).and_return(project_event) + flexmock(ProjectReleaseModel).should_receive("get_or_create").with_args( + tag_name="7.0.3", + namespace="rpms", + repo_name="redis", + project_url="https://src.fedoraproject.org/rpms/redis", + commit_hash=None, + ).and_return(db_project_object) + sync_release_model = flexmock(id=123, sync_release_targets=[]) + flexmock(SyncReleaseModel).should_receive("create_with_new_run").with_args( + status=SyncReleaseStatus.running, + project_event_model=project_event, + job_type=SyncReleaseJobType.pull_from_upstream, + package_name="redis", + ).and_return(sync_release_model, run_model).once() + + yield sync_release_model + + @pytest.fixture def sync_release_target_models(fedora_branches): models = [] @@ -271,3 +307,132 @@ def test_new_hotness_update_pre_check_fail(new_hotness_update): ) SteveJobs().process_message(new_hotness_update) + + +def test_new_hotness_update_non_git(new_hotness_update, sync_release_model_non_git): + model = flexmock(status="queued", id=1234, branch="main") + flexmock(SyncReleaseTargetModel).should_receive("create").with_args( + status=SyncReleaseTargetStatus.queued, branch="main" + ).and_return(model) + flexmock(SyncReleasePullRequestModel).should_receive("get_or_create").with_args( + pr_id=21, + namespace="downstream-namespace", + repo_name="downstream-repo", + project_url="https://src.fedoraproject.org/rpms/downstream-repo", + ).and_return(flexmock(sync_release_targets=[flexmock()])) + + packit_yaml = ( + "{'specfile_path': 'hello-world.spec', " + "jobs: [{trigger: release, job: pull_from_upstream, metadata: {targets:[]}}]}" + ) + flexmock(Github, get_repo=lambda full_name_or_id: None) + distgit_project = flexmock( + get_files=lambda ref, recursive: [".packit.yaml"], + get_file_content=lambda path, ref: packit_yaml, + full_repo_name="rpms/redis", + repo="redis", + namespace="rpms", + is_private=lambda: False, + default_branch="main", + ) + project = flexmock( + full_repo_name="packit-service/hello-world", + repo="hello-world", + namespace="packit-service", + get_files=lambda ref, filter_regex: [], + get_sha_from_tag=lambda tag_name: "123456", + get_web_url=lambda: "https://github.com/packit/hello-world", + is_private=lambda: False, + default_branch="main", + ) + lp = flexmock(LocalProject, refresh_the_arguments=lambda: None) + flexmock(LocalProjectBuilder, _refresh_the_state=lambda *args: lp) + lp.working_dir = "" + lp.git_project = project + flexmock(DistGit).should_receive("local_project").and_return(lp) + + flexmock(Allowlist, check_and_report=True) + + service_config = ServiceConfig().get_service_config() + flexmock(service_config).should_receive("get_project").with_args( + "https://src.fedoraproject.org/rpms/redis", required=False + ).and_return(distgit_project) + flexmock(service_config).should_receive("get_project").with_args( + "https://src.fedoraproject.org/rpms/redis" + ).and_return(distgit_project) + + target_project = ( + flexmock(namespace="downstream-namespace", repo="downstream-repo") + .should_receive("get_web_url") + .and_return("https://src.fedoraproject.org/rpms/downstream-repo") + .mock() + ) + pr = ( + flexmock( + id=21, + url="some_url", + target_project=target_project, + description="some-title", + ) + .should_receive("comment") + .mock() + ) + flexmock(PackitAPI).should_receive("sync_release").with_args( + dist_git_branch="main", + versions=["7.0.3"], + create_pr=True, + local_pr_branch_suffix="update-pull_from_upstream", + use_downstream_specfile=True, + sync_default_files=False, + add_pr_instructions=True, + resolved_bugs=["rhbz#2106196"], + release_monitoring_project_id=4181, + sync_acls=True, + pr_description_footer=DistgitAnnouncement.get_announcement(), + add_new_sources=True, + ).and_return(pr).once() + flexmock(PackitAPI).should_receive("clean") + + flexmock(model).should_receive("set_status").with_args( + status=SyncReleaseTargetStatus.running + ).once() + flexmock(model).should_receive("set_downstream_pr_url").with_args( + downstream_pr_url="some_url" + ) + flexmock(model).should_receive("set_downstream_pr").with_args( + downstream_pr=object + ).once() + flexmock(model).should_receive("set_status").with_args( + status=SyncReleaseTargetStatus.submitted + ).once() + flexmock(model).should_receive("set_start_time").once() + flexmock(model).should_receive("set_finished_time").once() + flexmock(model).should_receive("set_logs").once() + flexmock(sync_release_model_non_git).should_receive("set_status").with_args( + status=SyncReleaseStatus.finished + ).once() + sync_release_model_non_git.should_receive("get_package_name").and_return(None) + + flexmock(AddReleaseEventToDb).should_receive("db_project_object").and_return( + flexmock( + job_config_trigger_type=JobConfigTriggerType.release, + id=123, + project_event_model_type=ProjectEventModelType.release, + ) + ) + flexmock(group).should_receive("apply_async").once() + flexmock(Pushgateway).should_receive("push").times(2).and_return() + flexmock(shutil).should_receive("rmtree").with_args("") + + processing_results = SteveJobs().process_message(new_hotness_update) + event_dict, job, job_config, package_config = get_parameters_from_results( + processing_results + ) + assert json.dumps(event_dict) + + results = run_pull_from_upstream_handler( + package_config=package_config, + event=event_dict, + job_config=job_config, + ) + assert first_dict_value(results["job"])["success"] diff --git a/tests/integration/test_pr_comment.py b/tests/integration/test_pr_comment.py index 7becbdefc..b61c9dc52 100644 --- a/tests/integration/test_pr_comment.py +++ b/tests/integration/test_pr_comment.py @@ -2778,3 +2778,170 @@ def _get_project(url, *_, **__): job_config=job_config, ) assert first_dict_value(results["job"])["success"] + + +def test_pull_from_upstream_retrigger_via_dist_git_pr_comment_non_git( + pagure_pr_comment_added, +): + pagure_pr_comment_added["pullrequest"]["comments"][0][ + "comment" + ] = "/packit pull-from-upstream --with-pr-config --resolved-bugs rhbz#123,rhbz#124" + sync_release_pr_model = flexmock(sync_release_targets=[flexmock(), flexmock()]) + model = flexmock(status="queued", id=1234, branch="main") + flexmock(SyncReleaseTargetModel).should_receive("create").with_args( + status=SyncReleaseTargetStatus.queued, branch="main" + ).and_return(model) + flexmock(SyncReleasePullRequestModel).should_receive("get_or_create").with_args( + pr_id=21, + namespace="downstream-namespace", + repo_name="downstream-repo", + project_url="https://src.fedoraproject.org/rpms/downstream-repo", + ).and_return(sync_release_pr_model) + + packit_yaml = ( + "{'specfile_path': 'hello-world.spec', " + "jobs: [{trigger: release, job: pull_from_upstream, metadata: {targets:[]}}]}" + ) + pr_mock = ( + flexmock() + .should_receive("comment") + .with_args( + "The task was accepted. You can check the recent runs of pull from upstream jobs in " + "[Packit dashboard](/jobs/pull-from-upstreams)" + f"{DistgitAnnouncement.get_comment_footer_with_announcement_if_present()}" + ) + .mock() + ) + distgit_project = flexmock( + get_files=lambda ref, recursive: [".packit.yaml"], + get_file_content=lambda path, ref: packit_yaml, + full_repo_name=pagure_pr_comment_added["pullrequest"]["project"]["fullname"], + repo=pagure_pr_comment_added["pullrequest"]["project"]["name"], + namespace=pagure_pr_comment_added["pullrequest"]["project"]["namespace"], + is_private=lambda: False, + default_branch="main", + service=flexmock(get_project=lambda **_: None), + get_pr=lambda pr_id: pr_mock, + ) + lp = flexmock(LocalProject, refresh_the_arguments=lambda: None) + flexmock(LocalProjectBuilder, _refresh_the_state=lambda *args: lp) + lp.working_dir = "" + flexmock(DistGit).should_receive("local_project").and_return(lp) + + flexmock(GithubService).should_receive("set_auth_method").with_args( + AuthMethod.token + ).once() + flexmock(Allowlist, check_and_report=True) + flexmock(PackitAPIWithDownstreamMixin).should_receive("is_packager").and_return( + True + ) + + def _get_project(url, *_, **__): + if url == pagure_pr_comment_added["pullrequest"]["project"]["full_url"]: + return distgit_project + return None + + service_config = ServiceConfig().get_service_config() + flexmock(service_config).should_receive("get_project").replace_with(_get_project) + target_project = ( + flexmock(namespace="downstream-namespace", repo="downstream-repo") + .should_receive("get_web_url") + .and_return("https://src.fedoraproject.org/rpms/downstream-repo") + .mock() + ) + pr = ( + flexmock(id=21, url="some_url", target_project=target_project) + .should_receive("comment") + .mock() + ) + flexmock(PackitAPI).should_receive("sync_release").with_args( + dist_git_branch="main", + create_pr=True, + local_pr_branch_suffix="update-pull_from_upstream", + use_downstream_specfile=True, + sync_default_files=False, + add_pr_instructions=True, + resolved_bugs=["rhbz#123", "rhbz#124"], + release_monitoring_project_id=None, + sync_acls=True, + pr_description_footer=DistgitAnnouncement.get_announcement(), + add_new_sources=True, + ).and_return(pr).once() + flexmock(PackitAPI).should_receive("clean") + + flexmock(model).should_receive("set_status").with_args( + status=SyncReleaseTargetStatus.running + ).once() + flexmock(model).should_receive("set_downstream_pr_url").with_args( + downstream_pr_url="some_url" + ).once() + flexmock(model).should_receive("set_downstream_pr").with_args( + downstream_pr=sync_release_pr_model + ).once() + flexmock(model).should_receive("set_status").with_args( + status=SyncReleaseTargetStatus.submitted + ).once() + flexmock(model).should_receive("set_start_time").once() + flexmock(model).should_receive("set_finished_time").once() + flexmock(model).should_receive("set_logs").once() + + db_project_object = flexmock( + id=12, + project_event_model_type=ProjectEventModelType.pull_request, + job_config_trigger_type=JobConfigTriggerType.pull_request, + ) + db_project_event = ( + flexmock() + .should_receive("get_project_event_object") + .and_return(db_project_object) + .mock() + ) + run_model = flexmock(PipelineModel) + flexmock(ProjectEventModel).should_receive("get_or_create").with_args( + type=ProjectEventModelType.pull_request, + event_id=12, + commit_sha="beaf90bcecc51968a46663f8d6f092bfdc92e682", + ).and_return(db_project_event) + flexmock(PullRequestModel).should_receive("get_or_create").with_args( + pr_id=pagure_pr_comment_added["pullrequest"]["id"], + namespace=pagure_pr_comment_added["pullrequest"]["project"]["namespace"], + repo_name=pagure_pr_comment_added["pullrequest"]["project"]["name"], + project_url=pagure_pr_comment_added["pullrequest"]["project"]["full_url"], + ).and_return(db_project_object) + sync_release_model = flexmock(id=123, sync_release_targets=[]) + flexmock(SyncReleaseModel).should_receive("create_with_new_run").with_args( + status=SyncReleaseStatus.running, + project_event_model=db_project_event, + job_type=SyncReleaseJobType.pull_from_upstream, + package_name="python-teamcity-messages", + ).and_return(sync_release_model, run_model).once() + flexmock(sync_release_model).should_receive("set_status").with_args( + status=SyncReleaseStatus.finished + ).once() + + flexmock(AddPullRequestEventToDb).should_receive("db_project_object").and_return( + flexmock( + job_config_trigger_type=JobConfigTriggerType.pull_request, + id=123, + project_event_model_type=ProjectEventModelType.pull_request, + ) + ) + flexmock(celery_group).should_receive("apply_async").once() + flexmock(Pushgateway).should_receive("push").times(2).and_return() + flexmock(shutil).should_receive("rmtree").with_args("") + flexmock(PullRequestCommentPagureEvent).should_receive( + "get_base_project" + ).once().and_return(distgit_project) + + processing_results = SteveJobs().process_message(pagure_pr_comment_added) + event_dict, job, job_config, package_config = get_parameters_from_results( + processing_results + ) + assert json.dumps(event_dict) + + results = run_pull_from_upstream_handler( + package_config=package_config, + event=event_dict, + job_config=job_config, + ) + assert first_dict_value(results["job"])["success"] diff --git a/tests/unit/test_distgit.py b/tests/unit/test_distgit.py index 2e9ec25f9..40959e14e 100644 --- a/tests/unit/test_distgit.py +++ b/tests/unit/test_distgit.py @@ -125,7 +125,9 @@ def test_pull_from_upstream_auth_method(): class Test(PullFromUpstreamHandler): pass - handler = Test(None, None, {"event_type": "unknown"}, None) + handler = Test( + None, flexmock(upstream_project_url="url"), {"event_type": "unknown"}, None + ) flexmock(GithubService).should_receive("set_auth_method").once() flexmock(AbstractSyncReleaseHandler).should_receive("run").once() flexmock(GithubService).should_receive("reset_auth_method").once() From 1996b57d5f67abf46b7655ce6ac8fed02140eb8d Mon Sep 17 00:00:00 2001 From: Laura Barcziova Date: Thu, 1 Aug 2024 14:05:57 +0200 Subject: [PATCH 2/2] Introduce Anitya related models - rename the attributes related to Anitya in the AnityaUpdateEvent - introduce model for project and new version in Anitya - utilise those for pull-from-upstream for non-git upstreams - change the affected API --- .../b43ff6137622_add_anitya_related_models.py | 64 ++++++++++ packit_service/models.py | 118 +++++++++++++++++- packit_service/service/api/utils.py | 43 +++++-- packit_service/worker/events/event.py | 12 ++ packit_service/worker/events/new_hotness.py | 32 ++--- packit_service/worker/handlers/distgit.py | 5 +- packit_service/worker/parser.py | 14 ++- tests/integration/test_new_hotness_update.py | 14 +-- tests/unit/events/test_anitya.py | 4 +- tests_openshift/conftest.py | 50 ++++++++ tests_openshift/database/test_events.py | 4 +- tests_openshift/service/test_api.py | 45 ++++++- 12 files changed, 356 insertions(+), 49 deletions(-) create mode 100644 alembic/versions/b43ff6137622_add_anitya_related_models.py diff --git a/alembic/versions/b43ff6137622_add_anitya_related_models.py b/alembic/versions/b43ff6137622_add_anitya_related_models.py new file mode 100644 index 000000000..fc8fa18b5 --- /dev/null +++ b/alembic/versions/b43ff6137622_add_anitya_related_models.py @@ -0,0 +1,64 @@ +"""Add Anitya related models + +Revision ID: b43ff6137622 +Revises: 64a51b961c28 +Create Date: 2024-08-01 10:37:55.286362 + +""" + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "b43ff6137622" +down_revision = "64a51b961c28" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "anitya_projects", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("project_id", sa.Integer(), nullable=True), + sa.Column("project_name", sa.String(), nullable=True), + sa.Column("package", sa.String(), nullable=True), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index( + op.f("ix_anitya_projects_project_id"), + "anitya_projects", + ["project_id"], + unique=False, + ) + op.create_table( + "anitya_versions", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("version", sa.String(), nullable=True), + sa.Column("project_id", sa.Integer(), nullable=True), + sa.ForeignKeyConstraint( + ["project_id"], + ["anitya_projects.id"], + ), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index( + op.f("ix_anitya_versions_project_id"), + "anitya_versions", + ["project_id"], + unique=False, + ) + # add the event type + op.execute("ALTER TYPE projecteventtype ADD VALUE 'anitya_version'") + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index(op.f("ix_anitya_versions_project_id"), table_name="anitya_versions") + op.drop_table("anitya_versions") + op.drop_index(op.f("ix_anitya_projects_project_id"), table_name="anitya_projects") + op.drop_table("anitya_projects") + # ### end Alembic commands ### diff --git a/packit_service/models.py b/packit_service/models.py index 6c189014b..7f56c21d8 100644 --- a/packit_service/models.py +++ b/packit_service/models.py @@ -278,6 +278,7 @@ class ProjectEventModelType(str, enum.Enum): release = "release" issue = "issue" koji_build_tag = "koji_build_tag" + anitya_version = "anitya_version" class BuildsAndTestsConnector: @@ -357,7 +358,7 @@ def get_project_event_object(self) -> Optional["AbstractProjectObjectDbType"]: project_event = self.get_project_event_model() return project_event.get_project_event_object() if project_event else None - def get_project(self) -> Optional["GitProjectModel"]: + def get_project(self) -> Optional[Union["AnityaProjectModel", "GitProjectModel"]]: project_event_object = self.get_project_event_object() return project_event_object.project if project_event_object else None @@ -398,6 +399,12 @@ def get_release_tag(self) -> Optional[str]: return project_event_object.tag_name return None + def get_anitya_version(self) -> Optional[str]: + project_event_object = self.get_project_event_object() + if isinstance(project_event_object, AnityaVersionModel): + return project_event_object.version + return None + class GroupAndTargetModelConnector: """ @@ -413,7 +420,7 @@ def get_project_event_model(self) -> Optional["ProjectEventModel"]: def get_project_event_object(self) -> Optional["AbstractProjectObjectDbType"]: return self.group_of_targets.get_project_event_object() - def get_project(self) -> Optional["GitProjectModel"]: + def get_project(self) -> Optional[Union["AnityaProjectModel", "GitProjectModel"]]: return self.group_of_targets.get_project() def get_pr_id(self) -> Optional[int]: @@ -428,6 +435,9 @@ def get_branch_name(self) -> Optional[str]: def get_release_tag(self) -> Optional[str]: return self.group_of_targets.get_release_tag() + def get_anitya_version(self) -> Optional[str]: + return self.group_of_targets.get_anitya_version() + def get_package_name(self) -> Optional[str]: return self.group_of_targets.get_package_name() @@ -449,6 +459,87 @@ def grouped_targets(self): raise NotImplementedError +class AnityaProjectModel(Base): + __tablename__ = "anitya_projects" + id = Column(Integer, primary_key=True) + project_id = Column(Integer, index=True) + project_name = Column(String) + package = Column(String) + versions = relationship("AnityaVersionModel", back_populates="project") + + @classmethod + def get_by_id(cls, id_: int) -> Optional["AnityaProjectModel"]: + with sa_session_transaction() as session: + return session.query(AnityaProjectModel).filter_by(id=id_).first() + + @classmethod + def get_or_create( + cls, project_name: str, project_id: int, package: str + ) -> "AnityaProjectModel": + with sa_session_transaction(commit=True) as session: + project = ( + session.query(AnityaProjectModel) + .filter_by( + project_name=project_name, project_id=project_id, package=package + ) + .first() + ) + if not project: + project = AnityaProjectModel() + project.project_id = project_id + project.project_name = project_name + project.package = package + session.add(project) + return project + + +class AnityaVersionModel(BuildsAndTestsConnector, Base): + __tablename__ = "anitya_versions" + id = Column(Integer, primary_key=True) # our database PK + version = Column(String) + project_id = Column(Integer, ForeignKey("anitya_projects.id"), index=True) + project = relationship("AnityaProjectModel", back_populates="versions") + + job_config_trigger_type = JobConfigTriggerType.release + project_event_model_type = ProjectEventModelType.anitya_version + + @classmethod + def get_or_create( + cls, + version: str, + project_id: int, + project_name: str, + package: str, + ) -> "AnityaVersionModel": + with sa_session_transaction(commit=True) as session: + project = AnityaProjectModel.get_or_create( + project_id=project_id, project_name=project_name, package=package + ) + project_version = ( + session.query(AnityaVersionModel) + .filter_by(version=version, project_id=project.id) + .first() + ) + if not project_version: + project_version = AnityaVersionModel() + project_version.version = version + project_version.project = project + session.add(project_version) + return project_version + + @classmethod + def get_by_id(cls, id_: int) -> Optional["AnityaVersionModel"]: + with sa_session_transaction() as session: + return session.query(AnityaVersionModel).filter_by(id=id_).first() + + def __repr__(self): + return ( + f"AnityaVersionModel(" + f"version={self.version}, " + f"project={self.project})" + ) + + class GitProjectModel(Base): __tablename__ = "git_projects" id = Column(Integer, primary_key=True) @@ -1316,6 +1407,7 @@ def __repr__(self): GitBranchModel, IssueModel, KojiBuildTagModel, + AnityaVersionModel, ] MODEL_FOR_PROJECT_EVENT: Dict[ @@ -1326,6 +1418,7 @@ def __repr__(self): ProjectEventModelType.release: ProjectReleaseModel, ProjectEventModelType.issue: IssueModel, ProjectEventModelType.koji_build_tag: KojiBuildTagModel, + ProjectEventModelType.anitya_version: AnityaVersionModel, } @@ -1422,6 +1515,27 @@ def add_release_event( ) return (release, event) + @classmethod + def add_anitya_version_event( + cls, + version: str, + project_name: str, + project_id: int, + package: str, + ) -> Tuple[AnityaVersionModel, "ProjectEventModel"]: + project_version = AnityaVersionModel.get_or_create( + version=version, + project_name=project_name, + project_id=project_id, + package=package, + ) + event = ProjectEventModel.get_or_create( + type=project_version.project_event_model_type, + event_id=project_version.id, + commit_sha=None, + ) + return (project_version, event) + @classmethod def add_issue_event( cls, diff --git a/packit_service/service/api/utils.py b/packit_service/service/api/utils.py index 8c99b48b3..a896c02a4 100644 --- a/packit_service/service/api/utils.py +++ b/packit_service/service/api/utils.py @@ -9,6 +9,7 @@ from packit_service.models import ( CoprBuildTargetModel, CoprBuildGroupModel, + GitProjectModel, KojiBuildTargetModel, KojiBuildGroupModel, SRPMBuildModel, @@ -19,6 +20,7 @@ optional_timestamp, BodhiUpdateTargetModel, VMImageBuildTargetModel, + AnityaProjectModel, ) @@ -46,15 +48,15 @@ def get_project_info_from_build( if not (project := build.get_project()): return {} - return { - "repo_namespace": project.namespace, - "repo_name": project.repo_name, - "git_repo": project.project_url, + result_dict = { "pr_id": build.get_pr_id(), "issue_id": build.get_issue_id(), "branch_name": build.get_branch_name(), "release": build.get_release_tag(), + "anitya_version": build.get_anitya_version(), } + result_dict.update(get_project_info(project)) + return result_dict def get_sync_release_target_info(sync_release_model: SyncReleaseTargetModel): @@ -89,10 +91,37 @@ def get_sync_release_info(sync_release_model: SyncReleaseModel): "pr_id": sync_release_model.get_pr_id(), "issue_id": sync_release_model.get_issue_id(), "release": sync_release_model.get_release_tag(), + "anitya_version": sync_release_model.get_anitya_version(), } project = sync_release_model.get_project() - result_dict["repo_namespace"] = project.namespace if project else "" - result_dict["repo_name"] = project.repo_name if project else "" - result_dict["project_url"] = project.project_url if project else "" + result_dict.update(get_project_info(project)) + + return result_dict + + +def get_project_info(project: Union[AnityaProjectModel, GitProjectModel]): + result_dict = {} + + anitya_project_id = anitya_project_name = anitya_package = None + project_url = repo_name = repo_namespace = None + + if isinstance(project, AnityaProjectModel): + anitya_project_id = project.project_id if project else "" + anitya_project_name = project.project_name if project else "" + anitya_package = project.package if project else "" + project_url = f"https://release-monitoring.org/project/{anitya_project_id}" + elif isinstance(project, GitProjectModel): + repo_namespace = project.namespace if project else "" + repo_name = project.repo_name if project else "" + project_url = project.project_url if project else "" + + result_dict["repo_namespace"] = repo_namespace + result_dict["repo_name"] = repo_name + result_dict["project_url"] = project_url + result_dict["anitya_project_id"] = anitya_project_id + result_dict["anitya_project_name"] = anitya_project_name + result_dict["anitya_package"] = anitya_package + result_dict["non_git_upstream"] = isinstance(project, AnityaProjectModel) + return result_dict diff --git a/packit_service/worker/events/event.py b/packit_service/worker/events/event.py index 71ee8669a..51cf06b48 100644 --- a/packit_service/worker/events/event.py +++ b/packit_service/worker/events/event.py @@ -204,6 +204,18 @@ def _add_project_object_and_event(self): elif self.event_type in { "NewHotnessUpdateEvent", }: + if not self.project_url: + ( + self._db_project_object, + self._db_project_event, + ) = ProjectEventModel.add_anitya_version_event( + version=self.event_dict.get("version"), + project_name=self.event_dict.get("anitya_project_name"), + project_id=self.event_dict.get("anitya_project_id"), + package=self.event_dict.get("package_name"), + ) + return + if self.project: namespace = self.project.namespace repo_name = self.project.repo diff --git a/packit_service/worker/events/new_hotness.py b/packit_service/worker/events/new_hotness.py index dd13c61de..81d2e4514 100644 --- a/packit_service/worker/events/new_hotness.py +++ b/packit_service/worker/events/new_hotness.py @@ -23,13 +23,15 @@ def __init__( self, package_name: str, distgit_project_url: str, - release_monitoring_project_id: int, + anitya_project_id: int, + anitya_project_name: str, ): super().__init__() self.package_name = package_name self.distgit_project_url = distgit_project_url - self.release_monitoring_project_id = release_monitoring_project_id + self.anitya_project_id = anitya_project_id + self.anitya_project_name = anitya_project_name self._repo_url: Optional[RepoUrl] = None self._db_project_object: Optional[ProjectReleaseModel] = None @@ -61,18 +63,14 @@ def base_project(self): def _add_release_and_event(self): if not self._db_project_object or not self._db_project_event: if not self.project_url: - # we do not know what is the upstream project - # and it doesn't necessarily have tag, - # let's use dist-git project and version ( self._db_project_object, self._db_project_event, - ) = ProjectEventModel.add_release_event( - tag_name=self.version, - namespace="rpms", - repo_name=self.project.repo, - project_url=self.distgit_project_url, - commit_hash=None, + ) = ProjectEventModel.add_anitya_version_event( + version=self.version, + project_name=self.anitya_project_name, + project_id=self.anitya_project_id, + package=self.package_name, ) return @@ -178,12 +176,14 @@ def __init__( version: str, distgit_project_url: str, bug_id: int, - release_monitoring_project_id: int, + anitya_project_id: int, + anitya_project_name: str, ): super().__init__( package_name=package_name, distgit_project_url=distgit_project_url, - release_monitoring_project_id=release_monitoring_project_id, + anitya_project_id=anitya_project_id, + anitya_project_name=anitya_project_name, ) self._version = version self.bug_id = bug_id @@ -202,12 +202,14 @@ def __init__( package_name: str, versions: list[str], distgit_project_url: str, - release_monitoring_project_id: int, + anitya_project_id: int, + anitya_project_name: str, ): super().__init__( package_name=package_name, distgit_project_url=distgit_project_url, - release_monitoring_project_id=release_monitoring_project_id, + anitya_project_id=anitya_project_id, + anitya_project_name=anitya_project_name, ) self._versions = versions diff --git a/packit_service/worker/handlers/distgit.py b/packit_service/worker/handlers/distgit.py index c45271df1..1383d30c8 100644 --- a/packit_service/worker/handlers/distgit.py +++ b/packit_service/worker/handlers/distgit.py @@ -156,6 +156,7 @@ class SyncFromDownstream( """Sync new specfile changes to upstream after a new git push in the dist-git.""" task_name = TaskName.sync_from_downstream + non_git_upstream = False def __init__( self, @@ -202,7 +203,7 @@ class AbstractSyncReleaseHandler( sync_release_job_type: SyncReleaseJobType job_name_for_reporting: str get_dashboard_url: ClassVar # static method from Callable[[int], str] - check_for_non_git_upstreams: bool + check_for_non_git_upstreams: bool = False def __init__( self, @@ -253,7 +254,7 @@ def sync_branch( add_pr_instructions=True, resolved_bugs=self.get_resolved_bugs(), release_monitoring_project_id=self.data.event_dict.get( - "release_monitoring_project_id" + "anitya_project_id" ), sync_acls=True, pr_description_footer=DistgitAnnouncement.get_announcement(), diff --git a/packit_service/worker/parser.py b/packit_service/worker/parser.py index ce2c50a89..4bee4568f 100644 --- a/packit_service/worker/parser.py +++ b/packit_service/worker/parser.py @@ -1597,9 +1597,8 @@ def parse_new_hotness_update_event(event) -> Optional[NewHotnessUpdateEvent]: version = nested_get(event, "trigger", "msg", "project", "version") bug_id = nested_get(event, "bug", "bug_id") - release_monitoring_project_id = nested_get( - event, "trigger", "msg", "project", "id" - ) + anitya_project_id = nested_get(event, "trigger", "msg", "project", "id") + anitya_project_name = nested_get(event, "trigger", "msg", "project", "name") logger.info( f"New hotness update event for package: {package_name}, version: {version}," @@ -1611,7 +1610,8 @@ def parse_new_hotness_update_event(event) -> Optional[NewHotnessUpdateEvent]: version=version, distgit_project_url=distgit_project_url, bug_id=bug_id, - release_monitoring_project_id=release_monitoring_project_id, + anitya_project_id=anitya_project_id, + anitya_project_name=anitya_project_name, ) @staticmethod @@ -1633,7 +1633,8 @@ def parse_anitya_version_update_event(event) -> Optional[AnityaVersionUpdateEven # ‹upstream_versions› contain the new releases versions = nested_get(event, "message", "upstream_versions") - release_monitoring_project_id = nested_get(event, "message", "project", "id") + anitya_project_id = nested_get(event, "message", "project", "id") + anitya_project_name = nested_get(event, "message", "project", "name") logger.info( f"Anitya version update event for package: {package_name}, versions: {versions}" @@ -1642,7 +1643,8 @@ def parse_anitya_version_update_event(event) -> Optional[AnityaVersionUpdateEven package_name=package_name, versions=versions, distgit_project_url=distgit_project_url, - release_monitoring_project_id=release_monitoring_project_id, + anitya_project_id=anitya_project_id, + anitya_project_name=anitya_project_name, ) # The .__func__ are needed for Python < 3.10 diff --git a/tests/integration/test_new_hotness_update.py b/tests/integration/test_new_hotness_update.py index 91709d6be..dd43f9010 100644 --- a/tests/integration/test_new_hotness_update.py +++ b/tests/integration/test_new_hotness_update.py @@ -27,6 +27,7 @@ SyncReleaseTargetStatus, SyncReleaseJobType, SyncReleasePullRequestModel, + AnityaVersionModel, ) from packit_service.service.db_project_events import AddReleaseEventToDb from packit_service.worker.allowlist import Allowlist @@ -83,7 +84,7 @@ def sync_release_model_non_git(): id=12, project_event_model_type=ProjectEventModelType.release, job_config_trigger_type=JobConfigTriggerType.release, - project=flexmock(project_url="https://src.fedoraproject.org/rpms/redis"), + project=flexmock(project_url=None), ) project_event = ( flexmock() @@ -95,12 +96,11 @@ def sync_release_model_non_git(): flexmock(ProjectEventModel).should_receive("get_or_create").with_args( type=ProjectEventModelType.release, event_id=12, commit_sha=None ).and_return(project_event) - flexmock(ProjectReleaseModel).should_receive("get_or_create").with_args( - tag_name="7.0.3", - namespace="rpms", - repo_name="redis", - project_url="https://src.fedoraproject.org/rpms/redis", - commit_hash=None, + flexmock(AnityaVersionModel).should_receive("get_or_create").with_args( + version="7.0.3", + project_name="redis", + project_id=4181, + package="redis", ).and_return(db_project_object) sync_release_model = flexmock(id=123, sync_release_targets=[]) flexmock(SyncReleaseModel).should_receive("create_with_new_run").with_args( diff --git a/tests/unit/events/test_anitya.py b/tests/unit/events/test_anitya.py index bef6c610a..58bbe40b8 100644 --- a/tests/unit/events/test_anitya.py +++ b/tests/unit/events/test_anitya.py @@ -122,7 +122,7 @@ def test_parse_new_hotness_update( assert isinstance(event_object, NewHotnessUpdateEvent) assert isinstance(event_object.project, PagureProject) assert event_object.package_name == "redis" - assert event_object.release_monitoring_project_id == 4181 + assert event_object.anitya_project_id == 4181 assert event_object.repo_namespace == repo_namespace assert event_object.repo_name == repo_name assert ( @@ -163,7 +163,7 @@ def test_parse_anitya_version_update(anitya_version_update): assert isinstance(event, AnityaVersionUpdateEvent) assert isinstance(event.project, GitlabProject) assert event.package_name == "python3-mypy-boto3" - assert event.release_monitoring_project_id == 40221 + assert event.anitya_project_id == 40221 assert event.repo_namespace == "vemel" assert event.repo_name == "mypy_boto3" assert ( diff --git a/tests_openshift/conftest.py b/tests_openshift/conftest.py index 9c356a6ac..19fb42a7b 100644 --- a/tests_openshift/conftest.py +++ b/tests_openshift/conftest.py @@ -153,6 +153,10 @@ class SampleValues: alias = "FEDORA-123" bodhi_url = "https://bodhi.fedoraproject.org/FEDORA-123" + # anitya + anitya_project_id = 12345 + anitya_project_name = "packit-anitya" + @pytest.fixture(scope="session", autouse=True) def global_service_config(): @@ -275,6 +279,17 @@ def release_model(): yield release +@pytest.fixture() +def anitya_version_model(): + release, _ = ProjectEventModel.add_anitya_version_event( + version=SampleValues.tag_name, + project_name=SampleValues.anitya_project_name, + project_id=SampleValues.anitya_project_id, + package=SampleValues.package_name, + ) + yield release + + @pytest.fixture() def different_release_model(): release, _ = ProjectEventModel.add_release_event( @@ -359,6 +374,32 @@ def pull_from_upstream_target_model(release_project_event_model): yield target_model +@pytest.fixture() +def pull_from_upstream_target_model_non_git(anitya_version_project_event_model): + pull_from_upstream_model, _ = SyncReleaseModel.create_with_new_run( + status=SyncReleaseStatus.running, + project_event_model=anitya_version_project_event_model, + job_type=SyncReleaseJobType.pull_from_upstream, + ) + + target_model = SyncReleaseTargetModel.create( + status=SyncReleaseTargetStatus.submitted, branch=SampleValues.branch + ) + sync_release_pull_request_model = SyncReleasePullRequestModel.get_or_create( + SampleValues.downstream_pr_id, + SampleValues.downstream_namespace, + SampleValues.downstream_repo, + SampleValues.downstream_project_url, + ) + target_model.set_downstream_pr_url(downstream_pr_url=SampleValues.downstream_pr_url) + target_model.set_downstream_pr(sync_release_pull_request_model) + target_model.set_finished_time(finished_time=datetime.datetime.utcnow()) + target_model.set_logs(logs="random logs") + + pull_from_upstream_model.sync_release_targets.append(target_model) + yield target_model + + @pytest.fixture() def pull_from_upstream_target_model_without_pr_model(release_project_event_model): pull_from_upstream_model, _ = SyncReleaseModel.create_with_new_run( @@ -448,6 +489,15 @@ def release_project_event_model(release_model): ) +@pytest.fixture() +def anitya_version_project_event_model(anitya_version_model): + yield ProjectEventModel.get_or_create( + type=ProjectEventModelType.anitya_version, + event_id=anitya_version_model.id, + commit_sha=None, + ) + + @pytest.fixture() def different_release_project_event_model(different_release_model): yield ProjectEventModel.get_or_create( diff --git a/tests_openshift/database/test_events.py b/tests_openshift/database/test_events.py index 338d3a97a..4e1ba71bf 100644 --- a/tests_openshift/database/test_events.py +++ b/tests_openshift/database/test_events.py @@ -220,7 +220,7 @@ def test_pr_comment_event_existing_pr( assert event_object.pr_id == 342 assert event_object.project_url == "https://github.com/the-namespace/the-repo-name" - flexmock(GithubProject).should_receive("get_pr").with_args(pr_id=342).and_return( + flexmock(GithubProject).should_receive("get_pr").with_args(342).and_return( flexmock(head_commit="12345") ) assert event_object.commit_sha == "12345" @@ -244,7 +244,7 @@ def test_pr_comment_event_non_existing_pr( assert event_object.git_ref is None assert event_object.pr_id == 342 - flexmock(GithubProject).should_receive("get_pr").with_args(pr_id=342).and_return( + flexmock(GithubProject).should_receive("get_pr").with_args(342).and_return( flexmock(head_commit="12345") ) assert event_object.commit_sha == "12345" diff --git a/tests_openshift/service/test_api.py b/tests_openshift/service/test_api.py index 533746445..784a57239 100644 --- a/tests_openshift/service/test_api.py +++ b/tests_openshift/service/test_api.py @@ -100,7 +100,7 @@ def test_detailed_copr_build_info(client, clean_before_and_after, a_copr_build_f # Project info: assert response_dict["repo_namespace"] == SampleValues.repo_namespace assert response_dict["repo_name"] == SampleValues.repo_name - assert response_dict["git_repo"] == SampleValues.project_url + assert response_dict["project_url"] == SampleValues.project_url assert response_dict["pr_id"] == SampleValues.pr_id assert response_dict["built_packages"] == SampleValues.built_packages assert "branch_name" in response_dict @@ -161,7 +161,7 @@ def test_detailed_koji_build_info(client, clean_before_and_after, a_koji_build_f # Project info: assert response_dict["repo_namespace"] == SampleValues.repo_namespace assert response_dict["repo_name"] == SampleValues.repo_name - assert response_dict["git_repo"] == SampleValues.project_url + assert response_dict["project_url"] == SampleValues.project_url assert response_dict["pr_id"] == SampleValues.pr_id assert "branch_name" in response_dict assert "release" in response_dict @@ -252,7 +252,7 @@ def test_srpm_build_info( # Project info: assert response_dict["repo_namespace"] == SampleValues.repo_namespace assert response_dict["repo_name"] == SampleValues.repo_name - assert response_dict["git_repo"] == SampleValues.project_url + assert response_dict["project_url"] == SampleValues.project_url assert response_dict["pr_id"] == SampleValues.pr_id assert "branch_name" in response_dict assert "release" in response_dict @@ -341,7 +341,7 @@ def test_get_testing_farm_result(client, clean_before_and_after, a_new_test_run_ # Project info: assert response_dict["repo_namespace"] == SampleValues.repo_namespace assert response_dict["repo_name"] == SampleValues.repo_name - assert response_dict["git_repo"] == SampleValues.project_url + assert response_dict["project_url"] == SampleValues.project_url assert response_dict["pr_id"] == SampleValues.pr_id assert "branch_name" in response_dict assert "release" in response_dict @@ -628,7 +628,7 @@ def test_detailed_propose_info_release( # Project info: assert response_dict["repo_namespace"] == SampleValues.repo_namespace assert response_dict["repo_name"] == SampleValues.repo_name - assert response_dict["git_repo"] == SampleValues.project_url + assert response_dict["project_url"] == SampleValues.project_url assert response_dict["release"] == SampleValues.tag_name @@ -655,10 +655,43 @@ def test_detailed_pull_from_upstream_info( # Project info: assert response_dict["repo_namespace"] == SampleValues.repo_namespace assert response_dict["repo_name"] == SampleValues.repo_name - assert response_dict["git_repo"] == SampleValues.project_url + assert response_dict["project_url"] == SampleValues.project_url assert response_dict["release"] == SampleValues.tag_name +def test_detailed_pull_from_upstream_info_non_git( + client, clean_before_and_after, pull_from_upstream_target_model_non_git +): + response = client.get( + url_for( + "api.pull-from-upstream_pull_result", + id=pull_from_upstream_target_model_non_git.id, + ) + ) + response_dict = response.json + + assert response_dict["status"] == SyncReleaseTargetStatus.submitted + assert response_dict["branch"] == SampleValues.branch + assert response_dict["downstream_pr_url"] == SampleValues.downstream_pr_url + assert response_dict["downstream_pr_id"] == SampleValues.downstream_pr_id + assert response_dict["downstream_pr_project"] == SampleValues.downstream_project_url + assert response_dict["submitted_time"] is not None + assert response_dict["finished_time"] is not None + assert response_dict["logs"] == "random logs" + + # Project info: + assert response_dict["repo_namespace"] is None + assert response_dict["repo_name"] is None + assert ( + response_dict["project_url"] + == f"https://release-monitoring.org/project/{SampleValues.anitya_project_id}" + ) + assert response_dict["release"] is None + assert response_dict["anitya_version"] == SampleValues.tag_name + assert response_dict["anitya_project_id"] == SampleValues.anitya_project_id + assert response_dict["anitya_project_name"] == SampleValues.anitya_project_name + + def test_detailed_propose_info_issue( client, clean_before_and_after, propose_model_submitted_issue ):