Tm/improve GitHub incremental entity provider

packages/backend/package.json

@@ -29,6 +29,7 @@
     "@backstage/plugin-auth-node": "^0.2.19",
     "@backstage/plugin-catalog-backend": "^1.12.4",
     "@backstage/plugin-catalog-backend-module-github": "^0.3.7",
+    "@backstage/plugin-catalog-backend-module-incremental-ingestion": "^0.4.5",
     "@backstage/plugin-catalog-graph": "^0.2.35",
     "@backstage/plugin-permission-common": "^0.7.7",
     "@backstage/plugin-permission-node": "^0.7.13",
@@ -43,7 +44,7 @@
     "@frontside/backstage-plugin-batch-loader": "0.3.5",
     "@frontside/backstage-plugin-humanitec-backend": "^0.3.9",
     "@frontside/backstage-plugin-graphql": "^0.7.4",
-    "@frontside/backstage-plugin-incremental-ingestion-backend": "*",
+    "@frontside/backstage-plugin-incremental-ingestion-github": "*",
     "@frontside/scaffolder-yaml-actions": "^0.2.2",
     "@gitbeaker/node": "^34.6.0",
     "@internal/plugin-healthcheck": "0.1.8",

packages/backend/src/plugins/catalog.ts

@@ -2,8 +2,10 @@ import {
   CatalogBuilder
 } from '@backstage/plugin-catalog-backend';
 import { ScaffolderEntitiesProcessor } from '@backstage/plugin-scaffolder-backend';
-import { IncrementalCatalogBuilder } from '@frontside/backstage-plugin-incremental-ingestion-backend';
+import { IncrementalCatalogBuilder } from '@backstage/plugin-catalog-backend-module-incremental-ingestion';
+import { GithubRepositoryEntityProvider } from '@frontside/backstage-plugin-incremental-ingestion-github';
 import { Router } from 'express';
+import { Duration } from 'luxon';
 import { PluginEnvironment } from '../types';
 
 export default async function createPlugin(
@@ -15,6 +17,20 @@ export default async function createPlugin(
   // incremental entity providers with the builder 
   const incrementalBuilder = await IncrementalCatalogBuilder.create(env, builder);
 
+  const githubRepositoryProvider = GithubRepositoryEntityProvider.create({ 
+    host: 'github.com', 
+    config: env.config 
+  })
+
+  incrementalBuilder.addIncrementalEntityProvider(
+    githubRepositoryProvider,
+    {
+      burstInterval: { seconds: 3 },
+      burstLength: { seconds: 3 },
+      restLength: { days: 1 }
+    }
+  )
+
   builder.addProcessor(new ScaffolderEntitiesProcessor());
 
   const { processingEngine, router } = await builder.build();

plugins/incremental-ingestion-github/package.json

@@ -24,12 +24,14 @@
   },
   "dependencies": {
     "@backstage/backend-common": "^0.19.4",
+    "@backstage/catalog-model": "^1.4.1",
     "@backstage/config": "^1.0.8",
     "@backstage/integration": "^1.6.2",
-    "@frontside/backstage-plugin-incremental-ingestion-backend": "*",
+    "@backstage/plugin-catalog-backend-module-incremental-ingestion": "^0.4.5",
+    "@backstage/plugin-catalog-node": "^1.4.3",
     "@graphql-codegen/near-operation-file-preset": "^2.4.1",
     "@graphql-codegen/typescript-operations": "^2.5.3",
-    "@octokit/graphql": "^4.8.0",
+    "@octokit/graphql": "^7.0.1",
     "@types/express": "*",
     "assert-ts": "^0.3.4",
     "express": "^4.17.1",

plugins/incremental-ingestion-github/src/providers/mappers.ts

@@ -0,0 +1,38 @@
+import { OrganizationMapper, RepositoryMapper } from "../types";
+import { ANNOTATION_LOCATION, ANNOTATION_ORIGIN_LOCATION } from '@backstage/catalog-model';
+
+export const defaultRepositoryMapper: RepositoryMapper = (repository) => [{
+  entity: {
+    kind: 'Resource',
+    apiVersion: 'backstage.io/v1beta1',
+    metadata: {
+      annotations: {
+        [ANNOTATION_LOCATION]: `url:${repository.url}`,
+        [ANNOTATION_ORIGIN_LOCATION]: `url:${repository.url}`,
+      },
+      name: repository.nameWithOwner.replace('/', '__'),
+    },
+    spec: {
+      type: 'github-repository'
+    }
+  },
+  locationKey: `url:${repository.url}`
+}]
+
+export const defaultOrganizationMapper: OrganizationMapper = (organization) => [{
+  entity: {
+    kind: 'Resource',
+    apiVersion: 'backstage.io/v1beta1',
+    metadata: {
+      annotations: {
+        [ANNOTATION_LOCATION]: `url:${organization.url}`,
+        [ANNOTATION_ORIGIN_LOCATION]: `url:${organization.url}`,
+      },
+      name: organization.login,
+    },
+    spec: {
+      type: 'github-organization'
+    }
+  },
+  locationKey: `url:${organization.url}`
+}]

plugins/incremental-ingestion-github/src/providers/query.ts

@@ -0,0 +1,68 @@
+export const ORGANIZATION_REPOSITORIES_QUERY = /* GraphQL */ `
+  fragment Organization on Organization {
+    login
+    id
+    url
+    repositories(first: 100, after: $repoCursor) {
+      pageInfo {
+        startCursor
+        endCursor
+        hasNextPage
+      }
+      nodes {
+        ...Repository
+      }
+    }
+  }
+  fragment Repository on Repository {
+    __typename
+    id
+    isArchived
+    name
+    nameWithOwner
+    owner {
+      __typename
+      login
+      url
+    }
+    url
+    description
+    visibility
+    languages(first: 10) {
+      nodes {
+        name
+      }
+    }
+    repositoryTopics(first: 10) {
+      nodes {
+        topic {
+          name
+        }
+      }
+    }
+    owner {
+      ... on Organization {
+        __typename
+        login
+      }
+      ... on User {
+        __typename
+        login
+      }
+    }
+  }
+  query OrganizationRepositories($orgCursor: String, $repoCursor: String) {
+    viewer {
+      organizations(first: 1, after: $orgCursor) {
+        pageInfo {
+          startCursor
+          endCursor
+          hasNextPage
+        }
+        nodes {
+          ...Organization
+        }
+      }
+    }
+  }
+`;