Support pull-from-upstream for non-git upstreams (#2498)

Support pull-from-upstream for non-git upstreams

Fixes #2203
TODO:

 dashboard followup

RELEASE NOTES BEGIN
pull-from-upstream is now supported also without specifying upstream_project_url in the configuration. For such cases, the upstream repository will not be detected and cloned during release syncing.
RELEASE NOTES END

Reviewed-by: Maja Massarini
Reviewed-by: Nikola Forró

alembic/versions/b43ff6137622_add_anitya_related_models.py

@@ -0,0 +1,64 @@
+"""Add Anitya related models
+
+Revision ID: b43ff6137622
+Revises: 64a51b961c28
+Create Date: 2024-08-01 10:37:55.286362
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "b43ff6137622"
+down_revision = "64a51b961c28"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "anitya_projects",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("project_id", sa.Integer(), nullable=True),
+        sa.Column("project_name", sa.String(), nullable=True),
+        sa.Column("package", sa.String(), nullable=True),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index(
+        op.f("ix_anitya_projects_project_id"),
+        "anitya_projects",
+        ["project_id"],
+        unique=False,
+    )
+    op.create_table(
+        "anitya_versions",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("version", sa.String(), nullable=True),
+        sa.Column("project_id", sa.Integer(), nullable=True),
+        sa.ForeignKeyConstraint(
+            ["project_id"],
+            ["anitya_projects.id"],
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index(
+        op.f("ix_anitya_versions_project_id"),
+        "anitya_versions",
+        ["project_id"],
+        unique=False,
+    )
+    # add the event type
+    op.execute("ALTER TYPE projecteventtype ADD VALUE 'anitya_version'")
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(op.f("ix_anitya_versions_project_id"), table_name="anitya_versions")
+    op.drop_table("anitya_versions")
+    op.drop_index(op.f("ix_anitya_projects_project_id"), table_name="anitya_projects")
+    op.drop_table("anitya_projects")
+    # ### end Alembic commands ###

packit_service/models.py

@@ -278,6 +278,7 @@ class ProjectEventModelType(str, enum.Enum):
     release = "release"
     issue = "issue"
     koji_build_tag = "koji_build_tag"
+    anitya_version = "anitya_version"
 
 
 class BuildsAndTestsConnector:
@@ -357,7 +358,7 @@ def get_project_event_object(self) -> Optional["AbstractProjectObjectDbType"]:
         project_event = self.get_project_event_model()
         return project_event.get_project_event_object() if project_event else None
 
-    def get_project(self) -> Optional["GitProjectModel"]:
+    def get_project(self) -> Optional[Union["AnityaProjectModel", "GitProjectModel"]]:
         project_event_object = self.get_project_event_object()
         return project_event_object.project if project_event_object else None
 
@@ -398,6 +399,12 @@ def get_release_tag(self) -> Optional[str]:
             return project_event_object.tag_name
         return None
 
+    def get_anitya_version(self) -> Optional[str]:
+        project_event_object = self.get_project_event_object()
+        if isinstance(project_event_object, AnityaVersionModel):
+            return project_event_object.version
+        return None
+
 
 class GroupAndTargetModelConnector:
     """
@@ -413,7 +420,7 @@ def get_project_event_model(self) -> Optional["ProjectEventModel"]:
     def get_project_event_object(self) -> Optional["AbstractProjectObjectDbType"]:
         return self.group_of_targets.get_project_event_object()
 
-    def get_project(self) -> Optional["GitProjectModel"]:
+    def get_project(self) -> Optional[Union["AnityaProjectModel", "GitProjectModel"]]:
         return self.group_of_targets.get_project()
 
     def get_pr_id(self) -> Optional[int]:
@@ -428,6 +435,9 @@ def get_branch_name(self) -> Optional[str]:
     def get_release_tag(self) -> Optional[str]:
         return self.group_of_targets.get_release_tag()
 
+    def get_anitya_version(self) -> Optional[str]:
+        return self.group_of_targets.get_anitya_version()
+
     def get_package_name(self) -> Optional[str]:
         return self.group_of_targets.get_package_name()
 
@@ -449,6 +459,87 @@ def grouped_targets(self):
         raise NotImplementedError
 
 
+class AnityaProjectModel(Base):
+    __tablename__ = "anitya_projects"
+    id = Column(Integer, primary_key=True)
+    project_id = Column(Integer, index=True)
+    project_name = Column(String)
+    package = Column(String)
+    versions = relationship("AnityaVersionModel", back_populates="project")
+
+    @classmethod
+    def get_by_id(cls, id_: int) -> Optional["AnityaProjectModel"]:
+        with sa_session_transaction() as session:
+            return session.query(AnityaProjectModel).filter_by(id=id_).first()
+
+    @classmethod
+    def get_or_create(
+        cls, project_name: str, project_id: int, package: str
+    ) -> "AnityaProjectModel":
+        with sa_session_transaction(commit=True) as session:
+            project = (
+                session.query(AnityaProjectModel)
+                .filter_by(
+                    project_name=project_name, project_id=project_id, package=package
+                )
+                .first()
+            )
+            if not project:
+                project = AnityaProjectModel()
+                project.project_id = project_id
+                project.project_name = project_name
+                project.package = package
+                session.add(project)
+            return project
+
+
+class AnityaVersionModel(BuildsAndTestsConnector, Base):
+    __tablename__ = "anitya_versions"
+    id = Column(Integer, primary_key=True)  # our database PK
+    version = Column(String)
+    project_id = Column(Integer, ForeignKey("anitya_projects.id"), index=True)
+    project = relationship("AnityaProjectModel", back_populates="versions")
+
+    job_config_trigger_type = JobConfigTriggerType.release
+    project_event_model_type = ProjectEventModelType.anitya_version
+
+    @classmethod
+    def get_or_create(
+        cls,
+        version: str,
+        project_id: int,
+        project_name: str,
+        package: str,
+    ) -> "AnityaVersionModel":
+        with sa_session_transaction(commit=True) as session:
+            project = AnityaProjectModel.get_or_create(
+                project_id=project_id, project_name=project_name, package=package
+            )
+            project_version = (
+                session.query(AnityaVersionModel)
+                .filter_by(version=version, project_id=project.id)
+                .first()
+            )
+            if not project_version:
+                project_version = AnityaVersionModel()
+                project_version.version = version
+                project_version.project = project
+                session.add(project_version)
+            return project_version
+
+    @classmethod
+    def get_by_id(cls, id_: int) -> Optional["AnityaVersionModel"]:
+        with sa_session_transaction() as session:
+            return session.query(AnityaVersionModel).filter_by(id=id_).first()
+
+    def __repr__(self):
+        return (
+            f"AnityaVersionModel("
+            f"version={self.version}, "
+            f"project={self.project})"
+        )
+
+
 class GitProjectModel(Base):
     __tablename__ = "git_projects"
     id = Column(Integer, primary_key=True)
@@ -1316,6 +1407,7 @@ def __repr__(self):
     GitBranchModel,
     IssueModel,
     KojiBuildTagModel,
+    AnityaVersionModel,
 ]
 
 MODEL_FOR_PROJECT_EVENT: Dict[
@@ -1326,6 +1418,7 @@ def __repr__(self):
     ProjectEventModelType.release: ProjectReleaseModel,
     ProjectEventModelType.issue: IssueModel,
     ProjectEventModelType.koji_build_tag: KojiBuildTagModel,
+    ProjectEventModelType.anitya_version: AnityaVersionModel,
 }
 
 
@@ -1422,6 +1515,27 @@ def add_release_event(
         )
         return (release, event)
 
+    @classmethod
+    def add_anitya_version_event(
+        cls,
+        version: str,
+        project_name: str,
+        project_id: int,
+        package: str,
+    ) -> Tuple[AnityaVersionModel, "ProjectEventModel"]:
+        project_version = AnityaVersionModel.get_or_create(
+            version=version,
+            project_name=project_name,
+            project_id=project_id,
+            package=package,
+        )
+        event = ProjectEventModel.get_or_create(
+            type=project_version.project_event_model_type,
+            event_id=project_version.id,
+            commit_sha=None,
+        )
+        return (project_version, event)
+
     @classmethod
     def add_issue_event(
         cls,

packit_service/service/api/utils.py

@@ -9,6 +9,7 @@
 from packit_service.models import (
     CoprBuildTargetModel,
     CoprBuildGroupModel,
+    GitProjectModel,
     KojiBuildTargetModel,
     KojiBuildGroupModel,
     SRPMBuildModel,
@@ -19,6 +20,7 @@
     optional_timestamp,
     BodhiUpdateTargetModel,
     VMImageBuildTargetModel,
+    AnityaProjectModel,
 )
 
 
@@ -46,15 +48,15 @@ def get_project_info_from_build(
     if not (project := build.get_project()):
         return {}
 
-    return {
-        "repo_namespace": project.namespace,
-        "repo_name": project.repo_name,
-        "git_repo": project.project_url,
+    result_dict = {
         "pr_id": build.get_pr_id(),
         "issue_id": build.get_issue_id(),
         "branch_name": build.get_branch_name(),
         "release": build.get_release_tag(),
+        "anitya_version": build.get_anitya_version(),
     }
+    result_dict.update(get_project_info(project))
+    return result_dict
 
 
 def get_sync_release_target_info(sync_release_model: SyncReleaseTargetModel):
@@ -89,10 +91,37 @@ def get_sync_release_info(sync_release_model: SyncReleaseModel):
         "pr_id": sync_release_model.get_pr_id(),
         "issue_id": sync_release_model.get_issue_id(),
         "release": sync_release_model.get_release_tag(),
+        "anitya_version": sync_release_model.get_anitya_version(),
     }
 
     project = sync_release_model.get_project()
-    result_dict["repo_namespace"] = project.namespace if project else ""
-    result_dict["repo_name"] = project.repo_name if project else ""
-    result_dict["project_url"] = project.project_url if project else ""
+    result_dict.update(get_project_info(project))
+
+    return result_dict
+
+
+def get_project_info(project: Union[AnityaProjectModel, GitProjectModel]):
+    result_dict = {}
+
+    anitya_project_id = anitya_project_name = anitya_package = None
+    project_url = repo_name = repo_namespace = None
+
+    if isinstance(project, AnityaProjectModel):
+        anitya_project_id = project.project_id if project else ""
+        anitya_project_name = project.project_name if project else ""
+        anitya_package = project.package if project else ""
+        project_url = f"https://release-monitoring.org/project/{anitya_project_id}"
+    elif isinstance(project, GitProjectModel):
+        repo_namespace = project.namespace if project else ""
+        repo_name = project.repo_name if project else ""
+        project_url = project.project_url if project else ""
+
+    result_dict["repo_namespace"] = repo_namespace
+    result_dict["repo_name"] = repo_name
+    result_dict["project_url"] = project_url
+    result_dict["anitya_project_id"] = anitya_project_id
+    result_dict["anitya_project_name"] = anitya_project_name
+    result_dict["anitya_package"] = anitya_package
+    result_dict["non_git_upstream"] = isinstance(project, AnityaProjectModel)
+
     return result_dict

packit_service/worker/checker/distgit.py

@@ -197,14 +197,7 @@ def pre_check(self) -> bool:
             f"{self.data.event_dict.get('version')}"
         )
 
-        if not self.package_config.upstream_project_url:
-            msg_to_report = (
-                "`upstream_project_url` is not set in "
-                "the dist-git package configuration."
-            )
-            valid = False
-
-        if not (
+        if self.package_config.upstream_project_url and not (
             self.data.event_dict.get("repo_name")
             and self.data.event_dict.get("repo_namespace")
         ):
@@ -215,7 +208,7 @@ def pre_check(self) -> bool:
             )
             valid = False
 
-        if (
+        if self.package_config.upstream_project_url and (
             self.data.event_type in (NewHotnessUpdateEvent.__name__,)
             and not self.data.tag_name
         ):

packit_service/worker/events/event.py

@@ -204,6 +204,18 @@ def _add_project_object_and_event(self):
         elif self.event_type in {
             "NewHotnessUpdateEvent",
         }:
+            if not self.project_url:
+                (
+                    self._db_project_object,
+                    self._db_project_event,
+                ) = ProjectEventModel.add_anitya_version_event(
+                    version=self.event_dict.get("version"),
+                    project_name=self.event_dict.get("anitya_project_name"),
+                    project_id=self.event_dict.get("anitya_project_id"),
+                    package=self.event_dict.get("package_name"),
+                )
+                return
+
             if self.project:
                 namespace = self.project.namespace
                 repo_name = self.project.repo