From 4d01358ab4109d3861dd58e4938c73946f4fab10 Mon Sep 17 00:00:00 2001 From: Taras Date: Thu, 31 Aug 2023 15:58:51 -0700 Subject: [PATCH 1/3] Revert "Remove github incremental entity provider" This reverts commit a42edbbc5823a7747b8a20cb7d00bf8b8e214c93. --- packages/backend/src/plugins/catalog.ts | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/packages/backend/src/plugins/catalog.ts b/packages/backend/src/plugins/catalog.ts index fafcb4644a..3ad56221ea 100644 --- a/packages/backend/src/plugins/catalog.ts +++ b/packages/backend/src/plugins/catalog.ts @@ -3,7 +3,9 @@ import { } from '@backstage/plugin-catalog-backend'; import { ScaffolderEntitiesProcessor } from '@backstage/plugin-scaffolder-backend'; import { IncrementalCatalogBuilder } from '@frontside/backstage-plugin-incremental-ingestion-backend'; +import { GithubRepositoryEntityProvider } from '@frontside/backstage-plugin-incremental-ingestion-github'; import { Router } from 'express'; +import { Duration } from 'luxon'; import { PluginEnvironment } from '../types'; export default async function createPlugin( @@ -15,6 +17,21 @@ export default async function createPlugin( // incremental entity providers with the builder const incrementalBuilder = await IncrementalCatalogBuilder.create(env, builder); + const githubRepositoryProvider = GithubRepositoryEntityProvider.create({ + host: 'github.com', + searchQuery: "created:>1970-01-01 user:thefrontside", + config: env.config + }) + + incrementalBuilder.addIncrementalEntityProvider( + githubRepositoryProvider, + { + burstInterval: Duration.fromObject({ seconds: 3 }), + burstLength: Duration.fromObject({ seconds: 3 }), + restLength: Duration.fromObject({ day: 1 }) + } + ) + builder.addProcessor(new ScaffolderEntitiesProcessor()); const { processingEngine, router } = await builder.build(); From bdd3d6ef54d38e938b5003b14055974ad1ea4bc8 Mon Sep 17 00:00:00 2001 From: Taras Date: Thu, 31 Aug 2023 16:08:27 -0700 Subject: [PATCH 2/3] Replaced with backstage's incremental ingestion --- packages/backend/package.json | 3 +- packages/backend/src/plugins/catalog.ts | 2 +- .../incremental-ingestion-github/package.json | 2 +- .../providers/repository-entity-provider.ts | 109 +++++++++++------- yarn.lock | 25 ++++ 5 files changed, 99 insertions(+), 42 deletions(-) diff --git a/packages/backend/package.json b/packages/backend/package.json index acaee253cb..48526a00f2 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -29,6 +29,7 @@ "@backstage/plugin-auth-node": "^0.2.19", "@backstage/plugin-catalog-backend": "^1.12.4", "@backstage/plugin-catalog-backend-module-github": "^0.3.7", + "@backstage/plugin-catalog-backend-module-incremental-ingestion": "^0.4.5", "@backstage/plugin-catalog-graph": "^0.2.35", "@backstage/plugin-permission-common": "^0.7.7", "@backstage/plugin-permission-node": "^0.7.13", @@ -43,7 +44,7 @@ "@frontside/backstage-plugin-batch-loader": "0.3.5", "@frontside/backstage-plugin-humanitec-backend": "^0.3.9", "@frontside/backstage-plugin-graphql": "^0.7.4", - "@frontside/backstage-plugin-incremental-ingestion-backend": "*", + "@frontside/backstage-plugin-incremental-ingestion-github": "*", "@frontside/scaffolder-yaml-actions": "^0.2.2", "@gitbeaker/node": "^34.6.0", "@internal/plugin-healthcheck": "0.1.8", diff --git a/packages/backend/src/plugins/catalog.ts b/packages/backend/src/plugins/catalog.ts index 3ad56221ea..766e2cdf2c 100644 --- a/packages/backend/src/plugins/catalog.ts +++ b/packages/backend/src/plugins/catalog.ts @@ -2,7 +2,7 @@ import { CatalogBuilder } from '@backstage/plugin-catalog-backend'; import { ScaffolderEntitiesProcessor } from '@backstage/plugin-scaffolder-backend'; -import { IncrementalCatalogBuilder } from '@frontside/backstage-plugin-incremental-ingestion-backend'; +import { IncrementalCatalogBuilder } from '@backstage/plugin-catalog-backend-module-incremental-ingestion'; import { GithubRepositoryEntityProvider } from '@frontside/backstage-plugin-incremental-ingestion-github'; import { Router } from 'express'; import { Duration } from 'luxon'; diff --git a/plugins/incremental-ingestion-github/package.json b/plugins/incremental-ingestion-github/package.json index 7e9030a122..9c896e9928 100644 --- a/plugins/incremental-ingestion-github/package.json +++ b/plugins/incremental-ingestion-github/package.json @@ -26,7 +26,7 @@ "@backstage/backend-common": "^0.19.4", "@backstage/config": "^1.0.8", "@backstage/integration": "^1.6.2", - "@frontside/backstage-plugin-incremental-ingestion-backend": "*", + "@backstage/plugin-catalog-backend-module-incremental-ingestion": "^0.4.5", "@graphql-codegen/near-operation-file-preset": "^2.4.1", "@graphql-codegen/typescript-operations": "^2.5.3", "@octokit/graphql": "^4.8.0", diff --git a/plugins/incremental-ingestion-github/src/providers/repository-entity-provider.ts b/plugins/incremental-ingestion-github/src/providers/repository-entity-provider.ts index 7166243bb7..fd934f25ed 100644 --- a/plugins/incremental-ingestion-github/src/providers/repository-entity-provider.ts +++ b/plugins/incremental-ingestion-github/src/providers/repository-entity-provider.ts @@ -1,20 +1,27 @@ -import { ANNOTATION_LOCATION, ANNOTATION_ORIGIN_LOCATION, DEFAULT_NAMESPACE, stringifyEntityRef } from "@backstage/catalog-model"; -import { Config } from "@backstage/config"; -import { DefaultGithubCredentialsProvider, GitHubIntegration, ScmIntegrations } from '@backstage/integration'; -import type { EntityIteratorResult, IncrementalEntityProvider } from "@frontside/backstage-plugin-incremental-ingestion-backend"; +import { + ANNOTATION_LOCATION, + ANNOTATION_ORIGIN_LOCATION, + DEFAULT_NAMESPACE, + stringifyEntityRef, +} from '@backstage/catalog-model'; +import { Config } from '@backstage/config'; +import { + DefaultGithubCredentialsProvider, + GitHubIntegration, + ScmIntegrations, +} from '@backstage/integration'; +import type { + EntityIteratorResult, + IncrementalEntityProvider, +} from '@backstage/plugin-catalog-backend-module-incremental-ingestion'; import { graphql } from '@octokit/graphql'; import assert from 'assert-ts'; import slugify from 'slugify'; -import type { RepositorySearchQuery } from "./repository-entity-provider.__generated__"; +import type { RepositorySearchQuery } from './repository-entity-provider.__generated__'; -const REPOSITORY_SEARCH_QUERY = /* GraphQL */` +const REPOSITORY_SEARCH_QUERY = /* GraphQL */ ` query RepositorySearch($searchQuery: String!, $cursor: String) { - search( - query: $searchQuery - type: REPOSITORY - first: 100 - after: $cursor - ) { + search(query: $searchQuery, type: REPOSITORY, first: 100, after: $cursor) { pageInfo { hasNextPage endCursor @@ -78,30 +85,44 @@ interface Cursor { cursor: string | null; } -interface GithubRepositoryEntityProviderConstructorOptions { - credentialsProvider: DefaultGithubCredentialsProvider; - host: string; - integration: GitHubIntegration; - searchQuery: string; +interface GithubRepositoryEntityProviderConstructorOptions { + credentialsProvider: DefaultGithubCredentialsProvider; + host: string; + integration: GitHubIntegration; + searchQuery: string; } -export class GithubRepositoryEntityProvider implements IncrementalEntityProvider { +export class GithubRepositoryEntityProvider + implements IncrementalEntityProvider +{ private host: string; private credentialsProvider: DefaultGithubCredentialsProvider; private integration: GitHubIntegration; private searchQuery: string; - static create({ host, config, searchQuery = "created:>1970-01-01" }: GithubRepositoryEntityProviderOptions) { + static create({ + host, + config, + searchQuery = 'created:>1970-01-01', + }: GithubRepositoryEntityProviderOptions) { const integrations = ScmIntegrations.fromConfig(config); - const credentialsProvider = DefaultGithubCredentialsProvider.fromIntegrations(integrations); + const credentialsProvider = + DefaultGithubCredentialsProvider.fromIntegrations(integrations); const integration = integrations.github.byHost(host); assert(integration !== undefined, `Missing Github integration for ${host}`); - return new GithubRepositoryEntityProvider({ credentialsProvider, host, integration, searchQuery }) + return new GithubRepositoryEntityProvider({ + credentialsProvider, + host, + integration, + searchQuery, + }); } - private constructor(options: GithubRepositoryEntityProviderConstructorOptions) { + private constructor( + options: GithubRepositoryEntityProviderConstructorOptions, + ) { this.credentialsProvider = options.credentialsProvider; this.host = options.host; this.integration = options.integration; @@ -113,7 +134,6 @@ export class GithubRepositoryEntityProvider implements IncrementalEntityProvider } async around(burst: (context: Context) => Promise) { - const url = `https://${this.host}`; const { headers } = await this.credentialsProvider.getCredentials({ @@ -125,21 +145,22 @@ export class GithubRepositoryEntityProvider implements IncrementalEntityProvider headers, }); - await burst({ client, url }) + await burst({ client, url }); } - async next({ client, url }: Context, { cursor }: Cursor = { cursor: null }): Promise> { - - const data = await client(REPOSITORY_SEARCH_QUERY, - { - cursor, - searchQuery: this.searchQuery, - } - ); + async next( + { client, url }: Context, + { cursor }: Cursor = { cursor: null }, + ): Promise> { + const data = await client(REPOSITORY_SEARCH_QUERY, { + cursor, + searchQuery: this.searchQuery, + }); const location = `url:${url}`; - const entities = data.search.nodes?.flatMap(node => node?.__typename === 'Repository' ? [node] : []) + const entities = data.search.nodes + ?.flatMap(node => (node?.__typename === 'Repository' ? [node] : [])) .map(node => ({ entity: { apiVersion: 'backstage.io/v1beta1', @@ -158,11 +179,21 @@ export class GithubRepositoryEntityProvider implements IncrementalEntityProvider owner: stringifyEntityRef({ kind: `Github${node.owner.__typename}`, namespace: DEFAULT_NAMESPACE, - name: node.owner.login + name: node.owner.login, }), nameWithOwner: node.nameWithOwner, - languages: node.languages?.nodes?.flatMap(_node => _node?.__typename === 'Language' ? [_node] : []).map(_node => _node.name) ?? [], - topics: node.repositoryTopics?.nodes?.flatMap(_node => _node?.__typename === 'RepositoryTopic' ? [_node] : []).map(_node => _node.topic.name) ?? [], + languages: + node.languages?.nodes + ?.flatMap(_node => + _node?.__typename === 'Language' ? [_node] : [], + ) + .map(_node => _node.name) ?? [], + topics: + node.repositoryTopics?.nodes + ?.flatMap(_node => + _node?.__typename === 'RepositoryTopic' ? [_node] : [], + ) + .map(_node => _node.topic.name) ?? [], visibility: node.visibility, }, }, @@ -172,11 +203,11 @@ export class GithubRepositoryEntityProvider implements IncrementalEntityProvider return { done: !data.search.pageInfo.hasNextPage, cursor: { cursor: data.search.pageInfo.endCursor ?? null }, - entities: entities ?? [] + entities: entities ?? [], }; } } function normalizeEntityName(name: string = '') { - return slugify(name.replace('/', '__').replace('.', '__dot__')) -} \ No newline at end of file + return slugify(name.replace('/', '__').replace('.', '__dot__')); +} diff --git a/yarn.lock b/yarn.lock index 1c81bfe7a2..2ba36a018d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3197,6 +3197,31 @@ uuid "^8.0.0" winston "^3.2.1" +"@backstage/plugin-catalog-backend-module-incremental-ingestion@^0.4.5": + version "0.4.5" + resolved "https://registry.yarnpkg.com/@backstage/plugin-catalog-backend-module-incremental-ingestion/-/plugin-catalog-backend-module-incremental-ingestion-0.4.5.tgz#2fb16533977d41124b8ec96c8c0dc5947d471c6e" + integrity sha512-8HXDbORkMPv5Xq9c18MaFgHui0Ehv9u9lZRLeBRrudoyXJXK8kCCgkW/IuJLIydjl/StDXeY5+in0Spuix0h4w== + dependencies: + "@backstage/backend-common" "^0.19.4" + "@backstage/backend-plugin-api" "^0.6.2" + "@backstage/backend-tasks" "^0.5.7" + "@backstage/catalog-model" "^1.4.1" + "@backstage/config" "^1.0.8" + "@backstage/errors" "^1.2.1" + "@backstage/plugin-catalog-backend" "^1.12.4" + "@backstage/plugin-catalog-node" "^1.4.3" + "@backstage/plugin-events-node" "^0.2.11" + "@backstage/plugin-permission-common" "^0.7.7" + "@types/express" "^4.17.6" + "@types/luxon" "^3.0.0" + express "^4.17.1" + express-promise-router "^4.1.0" + knex "^2.0.0" + lodash "^4.17.21" + luxon "^3.0.0" + uuid "^8.3.2" + winston "^3.2.1" + "@backstage/plugin-catalog-backend@^1.12.4": version "1.12.4" resolved "https://registry.yarnpkg.com/@backstage/plugin-catalog-backend/-/plugin-catalog-backend-1.12.4.tgz#d1d932777777eda83ffd81dcd60a5b878eb2da4f" From 7a3fe4df5267d061d6c3d69d067733b102e08aa0 Mon Sep 17 00:00:00 2001 From: Taras Date: Thu, 31 Aug 2023 20:32:36 -0700 Subject: [PATCH 3/3] Refactored to import all orgs and repositories --- packages/backend/src/plugins/catalog.ts | 7 +- .../incremental-ingestion-github/package.json | 4 +- .../src/providers/mappers.ts | 38 ++++ .../src/providers/query.ts | 68 ++++++ .../providers/repository-entity-provider.ts | 206 ++++++------------ .../incremental-ingestion-github/src/types.ts | 36 +++ yarn.lock | 85 +++++++- 7 files changed, 295 insertions(+), 149 deletions(-) create mode 100644 plugins/incremental-ingestion-github/src/providers/mappers.ts create mode 100644 plugins/incremental-ingestion-github/src/providers/query.ts create mode 100644 plugins/incremental-ingestion-github/src/types.ts diff --git a/packages/backend/src/plugins/catalog.ts b/packages/backend/src/plugins/catalog.ts index 766e2cdf2c..ceeca8be28 100644 --- a/packages/backend/src/plugins/catalog.ts +++ b/packages/backend/src/plugins/catalog.ts @@ -19,16 +19,15 @@ export default async function createPlugin( const githubRepositoryProvider = GithubRepositoryEntityProvider.create({ host: 'github.com', - searchQuery: "created:>1970-01-01 user:thefrontside", config: env.config }) incrementalBuilder.addIncrementalEntityProvider( githubRepositoryProvider, { - burstInterval: Duration.fromObject({ seconds: 3 }), - burstLength: Duration.fromObject({ seconds: 3 }), - restLength: Duration.fromObject({ day: 1 }) + burstInterval: { seconds: 3 }, + burstLength: { seconds: 3 }, + restLength: { days: 1 } } ) diff --git a/plugins/incremental-ingestion-github/package.json b/plugins/incremental-ingestion-github/package.json index 9c896e9928..dc60b3f989 100644 --- a/plugins/incremental-ingestion-github/package.json +++ b/plugins/incremental-ingestion-github/package.json @@ -24,12 +24,14 @@ }, "dependencies": { "@backstage/backend-common": "^0.19.4", + "@backstage/catalog-model": "^1.4.1", "@backstage/config": "^1.0.8", "@backstage/integration": "^1.6.2", "@backstage/plugin-catalog-backend-module-incremental-ingestion": "^0.4.5", + "@backstage/plugin-catalog-node": "^1.4.3", "@graphql-codegen/near-operation-file-preset": "^2.4.1", "@graphql-codegen/typescript-operations": "^2.5.3", - "@octokit/graphql": "^4.8.0", + "@octokit/graphql": "^7.0.1", "@types/express": "*", "assert-ts": "^0.3.4", "express": "^4.17.1", diff --git a/plugins/incremental-ingestion-github/src/providers/mappers.ts b/plugins/incremental-ingestion-github/src/providers/mappers.ts new file mode 100644 index 0000000000..3aee1beb3f --- /dev/null +++ b/plugins/incremental-ingestion-github/src/providers/mappers.ts @@ -0,0 +1,38 @@ +import { OrganizationMapper, RepositoryMapper } from "../types"; +import { ANNOTATION_LOCATION, ANNOTATION_ORIGIN_LOCATION } from '@backstage/catalog-model'; + +export const defaultRepositoryMapper: RepositoryMapper = (repository) => [{ + entity: { + kind: 'Resource', + apiVersion: 'backstage.io/v1beta1', + metadata: { + annotations: { + [ANNOTATION_LOCATION]: `url:${repository.url}`, + [ANNOTATION_ORIGIN_LOCATION]: `url:${repository.url}`, + }, + name: repository.nameWithOwner.replace('/', '__'), + }, + spec: { + type: 'github-repository' + } + }, + locationKey: `url:${repository.url}` +}] + +export const defaultOrganizationMapper: OrganizationMapper = (organization) => [{ + entity: { + kind: 'Resource', + apiVersion: 'backstage.io/v1beta1', + metadata: { + annotations: { + [ANNOTATION_LOCATION]: `url:${organization.url}`, + [ANNOTATION_ORIGIN_LOCATION]: `url:${organization.url}`, + }, + name: organization.login, + }, + spec: { + type: 'github-organization' + } + }, + locationKey: `url:${organization.url}` +}] \ No newline at end of file diff --git a/plugins/incremental-ingestion-github/src/providers/query.ts b/plugins/incremental-ingestion-github/src/providers/query.ts new file mode 100644 index 0000000000..9bc1e9b315 --- /dev/null +++ b/plugins/incremental-ingestion-github/src/providers/query.ts @@ -0,0 +1,68 @@ +export const ORGANIZATION_REPOSITORIES_QUERY = /* GraphQL */ ` + fragment Organization on Organization { + login + id + url + repositories(first: 100, after: $repoCursor) { + pageInfo { + startCursor + endCursor + hasNextPage + } + nodes { + ...Repository + } + } + } + fragment Repository on Repository { + __typename + id + isArchived + name + nameWithOwner + owner { + __typename + login + url + } + url + description + visibility + languages(first: 10) { + nodes { + name + } + } + repositoryTopics(first: 10) { + nodes { + topic { + name + } + } + } + owner { + ... on Organization { + __typename + login + } + ... on User { + __typename + login + } + } + } + query OrganizationRepositories($orgCursor: String, $repoCursor: String) { + viewer { + organizations(first: 1, after: $orgCursor) { + pageInfo { + startCursor + endCursor + hasNextPage + } + nodes { + ...Organization + } + } + } + } +`; \ No newline at end of file diff --git a/plugins/incremental-ingestion-github/src/providers/repository-entity-provider.ts b/plugins/incremental-ingestion-github/src/providers/repository-entity-provider.ts index fd934f25ed..e0e9ed98c2 100644 --- a/plugins/incremental-ingestion-github/src/providers/repository-entity-provider.ts +++ b/plugins/incremental-ingestion-github/src/providers/repository-entity-provider.ts @@ -2,12 +2,12 @@ import { ANNOTATION_LOCATION, ANNOTATION_ORIGIN_LOCATION, DEFAULT_NAMESPACE, + Entity, stringifyEntityRef, } from '@backstage/catalog-model'; -import { Config } from '@backstage/config'; import { DefaultGithubCredentialsProvider, - GitHubIntegration, + GithubIntegration, ScmIntegrations, } from '@backstage/integration'; import type { @@ -17,94 +17,29 @@ import type { import { graphql } from '@octokit/graphql'; import assert from 'assert-ts'; import slugify from 'slugify'; -import type { RepositorySearchQuery } from './repository-entity-provider.__generated__'; - -const REPOSITORY_SEARCH_QUERY = /* GraphQL */ ` - query RepositorySearch($searchQuery: String!, $cursor: String) { - search(query: $searchQuery, type: REPOSITORY, first: 100, after: $cursor) { - pageInfo { - hasNextPage - endCursor - } - nodes { - ... on Repository { - __typename - id - isArchived - name - nameWithOwner - url - description - visibility - languages(first: 10) { - nodes { - name - } - } - repositoryTopics(first: 10) { - nodes { - topic { - name - } - } - } - owner { - ... on Organization { - __typename - login - } - ... on User { - __typename - login - } - } - } - } - } - rateLimit { - cost - remaining - used - limit - } - } -`; - -interface GithubRepositoryEntityProviderOptions { - host: string; - config: Config; - searchQuery: string; -} - -interface Context { - client: typeof graphql; - url: string; -} - -interface Cursor { - cursor: string | null; -} - -interface GithubRepositoryEntityProviderConstructorOptions { - credentialsProvider: DefaultGithubCredentialsProvider; - host: string; - integration: GitHubIntegration; - searchQuery: string; -} +import type { OrganizationRepositoriesQuery } from './query.__generated__'; +import { + type GithubRepositoryEntityProviderConstructorOptions, + type Context, + type Cursor, + type GithubRepositoryEntityProviderOptions, + type RepositoryMapper, + type OrganizationMapper, +} from '../types'; +import { ORGANIZATION_REPOSITORIES_QUERY } from './query'; +import { defaultOrganizationMapper, defaultRepositoryMapper } from './mappers'; +import { DeferredEntity } from '@backstage/plugin-catalog-node'; export class GithubRepositoryEntityProvider implements IncrementalEntityProvider { - private host: string; - private credentialsProvider: DefaultGithubCredentialsProvider; - private integration: GitHubIntegration; - private searchQuery: string; - - static create({ - host, - config, - searchQuery = 'created:>1970-01-01', - }: GithubRepositoryEntityProviderOptions) { + private readonly host: string; + private readonly credentialsProvider: DefaultGithubCredentialsProvider; + private readonly integration: GithubIntegration; + private readonly repositoryMapper: RepositoryMapper; + private readonly organizationMapper: OrganizationMapper; + + static create({ host, config }: GithubRepositoryEntityProviderOptions) { const integrations = ScmIntegrations.fromConfig(config); const credentialsProvider = DefaultGithubCredentialsProvider.fromIntegrations(integrations); @@ -116,17 +51,17 @@ export class GithubRepositoryEntityProvider credentialsProvider, host, integration, - searchQuery, }); } private constructor( options: GithubRepositoryEntityProviderConstructorOptions, ) { - this.credentialsProvider = options.credentialsProvider; this.host = options.host; + this.credentialsProvider = options.credentialsProvider; this.integration = options.integration; - this.searchQuery = options.searchQuery; + this.organizationMapper = options.organizationMapper ?? defaultOrganizationMapper; + this.repositoryMapper = options.repositoryMapper ?? defaultRepositoryMapper; } getProviderName() { @@ -149,61 +84,52 @@ export class GithubRepositoryEntityProvider } async next( - { client, url }: Context, - { cursor }: Cursor = { cursor: null }, + { client }: Context, + cursor: Cursor, ): Promise> { - const data = await client(REPOSITORY_SEARCH_QUERY, { - cursor, - searchQuery: this.searchQuery, - }); + let orgCursor = cursor?.orgCursor || null; + let repoCursor = cursor?.repoCursor || null; + + const data = await client( + ORGANIZATION_REPOSITORIES_QUERY, + { + orgCursor, + repoCursor, + }, + ); + + const deferred: DeferredEntity[] = []; + + const [ organization ] = data.viewer.organizations.nodes ?? []; - const location = `url:${url}`; - - const entities = data.search.nodes - ?.flatMap(node => (node?.__typename === 'Repository' ? [node] : [])) - .map(node => ({ - entity: { - apiVersion: 'backstage.io/v1beta1', - kind: 'GithubRepository', - metadata: { - namespace: DEFAULT_NAMESPACE, - name: normalizeEntityName(node.nameWithOwner), - description: node.description ?? '', - annotations: { - [ANNOTATION_LOCATION]: location, - [ANNOTATION_ORIGIN_LOCATION]: location, - }, - }, - spec: { - url: node.url, - owner: stringifyEntityRef({ - kind: `Github${node.owner.__typename}`, - namespace: DEFAULT_NAMESPACE, - name: node.owner.login, - }), - nameWithOwner: node.nameWithOwner, - languages: - node.languages?.nodes - ?.flatMap(_node => - _node?.__typename === 'Language' ? [_node] : [], - ) - .map(_node => _node.name) ?? [], - topics: - node.repositoryTopics?.nodes - ?.flatMap(_node => - _node?.__typename === 'RepositoryTopic' ? [_node] : [], - ) - .map(_node => _node.topic.name) ?? [], - visibility: node.visibility, - }, - }, - locationKey: this.getProviderName(), - })); + // only call org mapper for first page of repositories + if (repoCursor === null && organization) { + deferred.push(...this.organizationMapper(organization)); + } + + for (const repository of organization?.repositories.nodes || []) { + if (repository) { + deferred.push(...this.repositoryMapper(repository)); + } + } + + let done = false; + if (!organization?.repositories.pageInfo.hasNextPage && !data.viewer.organizations.pageInfo.hasNextPage) { + // last page of repositories and no more organizations + done = true; + } else if (organization?.repositories.pageInfo.hasNextPage) { + // current organization still has repositories + repoCursor = organization?.repositories.pageInfo.endCursor ?? null; + } else if (data.viewer.organizations.pageInfo.hasNextPage) { + // start next org on the next cycle + repoCursor = null; + orgCursor = data.viewer.organizations.pageInfo.endCursor ?? null; + } return { - done: !data.search.pageInfo.hasNextPage, - cursor: { cursor: data.search.pageInfo.endCursor ?? null }, - entities: entities ?? [], + done, + cursor: { repoCursor, orgCursor }, + entities: deferred, }; } } diff --git a/plugins/incremental-ingestion-github/src/types.ts b/plugins/incremental-ingestion-github/src/types.ts new file mode 100644 index 0000000000..73bac8bdb0 --- /dev/null +++ b/plugins/incremental-ingestion-github/src/types.ts @@ -0,0 +1,36 @@ +import { Config } from '@backstage/config'; +import { graphql } from '@octokit/graphql'; +import { + DefaultGithubCredentialsProvider, + GithubIntegration, +} from '@backstage/integration'; +import { OrganizationFragment, RepositoryFragment } from './providers/query.__generated__'; +import { type DeferredEntity } from '@backstage/plugin-catalog-node'; + +export interface GithubRepositoryEntityProviderOptions { + host: string; + config: Config; + organizationMapper?: OrganizationMapper; + repositoryMapper?: RepositoryMapper +} + +export interface Context { + client: typeof graphql; + url: string; +} + +export interface Cursor { + orgCursor: string | null; + repoCursor: string | null; +} + +export interface GithubRepositoryEntityProviderConstructorOptions { + credentialsProvider: DefaultGithubCredentialsProvider; + host: string; + integration: GithubIntegration; + organizationMapper?: OrganizationMapper; + repositoryMapper?: RepositoryMapper +} + +export type OrganizationMapper = (org: OrganizationFragment) => DeferredEntity[] +export type RepositoryMapper = (repo: RepositoryFragment) => DeferredEntity[] \ No newline at end of file diff --git a/yarn.lock b/yarn.lock index 2ba36a018d..4858489342 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7959,6 +7959,15 @@ is-plain-object "^5.0.0" universal-user-agent "^6.0.0" +"@octokit/endpoint@^9.0.0": + version "9.0.0" + resolved "https://registry.yarnpkg.com/@octokit/endpoint/-/endpoint-9.0.0.tgz#c5ce19c74b999b85af9a8a189275c80faa3e90fd" + integrity sha512-szrQhiqJ88gghWY2Htt8MqUDO6++E/EIXqJ2ZEp5ma3uGS46o7LZAzSLt49myB7rT+Hfw5Y6gO3LmOxGzHijAQ== + dependencies: + "@octokit/types" "^11.0.0" + is-plain-object "^5.0.0" + universal-user-agent "^6.0.0" + "@octokit/graphql-schema@^11.1.0": version "11.1.0" resolved "https://registry.yarnpkg.com/@octokit/graphql-schema/-/graphql-schema-11.1.0.tgz#671305d0a7643fa118b152e024452f344cc5ccab" @@ -7975,7 +7984,7 @@ graphql "^16.0.0" graphql-tag "^2.10.3" -"@octokit/graphql@^4.5.8", "@octokit/graphql@^4.8.0": +"@octokit/graphql@^4.5.8": version "4.8.0" resolved "https://registry.yarnpkg.com/@octokit/graphql/-/graphql-4.8.0.tgz#664d9b11c0e12112cbf78e10f49a05959aa22cc3" integrity sha512-0gv+qLSBLKF0z8TKaSKTsS39scVKF9dbMxJpj3U0vC7wjNWFuIpL/z76Qe2fiuCbDRcJSavkXsVtMS6/dtQQsg== @@ -7993,6 +8002,15 @@ "@octokit/types" "^7.0.0" universal-user-agent "^6.0.0" +"@octokit/graphql@^7.0.1": + version "7.0.1" + resolved "https://registry.yarnpkg.com/@octokit/graphql/-/graphql-7.0.1.tgz#f2291620e17cdaa8115f8d0cdfc0644789ec2db2" + integrity sha512-T5S3oZ1JOE58gom6MIcrgwZXzTaxRnxBso58xhozxHpOqSTgDS6YNeEUvZ/kRvXgPrRz/KHnZhtb7jUMRi9E6w== + dependencies: + "@octokit/request" "^8.0.1" + "@octokit/types" "^11.0.0" + universal-user-agent "^6.0.0" + "@octokit/oauth-app@^4.0.6", "@octokit/oauth-app@^4.0.7": version "4.1.0" resolved "https://registry.yarnpkg.com/@octokit/oauth-app/-/oauth-app-4.1.0.tgz#23782e77141015a4a3483820cd339ea62e85bb85" @@ -8059,6 +8077,11 @@ resolved "https://registry.yarnpkg.com/@octokit/openapi-types/-/openapi-types-16.0.0.tgz#d92838a6cd9fb4639ca875ddb3437f1045cc625e" integrity sha512-JbFWOqTJVLHZSUUoF4FzAZKYtqdxWu9Z5m2QQnOyEa04fOFljvyh7D3GYKbfuaSWisqehImiVIMG4eyJeP5VEA== +"@octokit/openapi-types@^18.0.0": + version "18.0.0" + resolved "https://registry.yarnpkg.com/@octokit/openapi-types/-/openapi-types-18.0.0.tgz#f43d765b3c7533fd6fb88f3f25df079c24fccf69" + integrity sha512-V8GImKs3TeQRxRtXFpG2wl19V7444NIOTDF24AWuIbmNaNYOQMWRbjcGDXV5B+0n887fgDcuMNOmlul+k+oJtw== + "@octokit/plugin-enterprise-rest@^6.0.1": version "6.0.1" resolved "https://registry.yarnpkg.com/@octokit/plugin-enterprise-rest/-/plugin-enterprise-rest-6.0.1.tgz#e07896739618dab8da7d4077c658003775f95437" @@ -8179,6 +8202,15 @@ deprecation "^2.0.0" once "^1.4.0" +"@octokit/request-error@^5.0.0": + version "5.0.0" + resolved "https://registry.yarnpkg.com/@octokit/request-error/-/request-error-5.0.0.tgz#060c5770833f9d563ad9a49fec6650c41584bc40" + integrity sha512-1ue0DH0Lif5iEqT52+Rf/hf0RmGO9NWFjrzmrkArpG9trFfDM/efx00BJHdLGuro4BR/gECxCU2Twf5OKrRFsQ== + dependencies: + "@octokit/types" "^11.0.0" + deprecation "^2.0.0" + once "^1.4.0" + "@octokit/request@^5.6.0", "@octokit/request@^5.6.3": version "5.6.3" resolved "https://registry.yarnpkg.com/@octokit/request/-/request-5.6.3.tgz#19a022515a5bba965ac06c9d1334514eb50c48b0" @@ -8203,6 +8235,17 @@ node-fetch "^2.6.7" universal-user-agent "^6.0.0" +"@octokit/request@^8.0.1": + version "8.1.1" + resolved "https://registry.yarnpkg.com/@octokit/request/-/request-8.1.1.tgz#23b4d3f164e973f4c1a0f24f68256f1646c00620" + integrity sha512-8N+tdUz4aCqQmXl8FpHYfKG9GelDFd7XGVzyN8rc6WxVlYcfpHECnuRkgquzz+WzvHTK62co5di8gSXnzASZPQ== + dependencies: + "@octokit/endpoint" "^9.0.0" + "@octokit/request-error" "^5.0.0" + "@octokit/types" "^11.1.0" + is-plain-object "^5.0.0" + universal-user-agent "^6.0.0" + "@octokit/rest@^18.1.0": version "18.12.0" resolved "https://registry.yarnpkg.com/@octokit/rest/-/rest-18.12.0.tgz#f06bc4952fc87130308d810ca9d00e79f6988881" @@ -8233,6 +8276,13 @@ "@octokit/plugin-request-log" "^1.0.4" "@octokit/plugin-rest-endpoint-methods" "^6.7.0" +"@octokit/types@^11.0.0", "@octokit/types@^11.1.0": + version "11.1.0" + resolved "https://registry.yarnpkg.com/@octokit/types/-/types-11.1.0.tgz#9e5db741d582b05718a4d91bac8cc987def235ea" + integrity sha512-Fz0+7GyLm/bHt8fwEqgvRBWwIV1S6wRRyq+V6exRKLVWaKGsuy6H9QFYeBVDV7rK6fO3XwHgQOPxv+cLj2zpXQ== + dependencies: + "@octokit/openapi-types" "^18.0.0" + "@octokit/types@^6.0.3", "@octokit/types@^6.10.0", "@octokit/types@^6.12.2", "@octokit/types@^6.16.1", "@octokit/types@^6.34.0", "@octokit/types@^6.39.0", "@octokit/types@^6.8.2": version "6.40.0" resolved "https://registry.yarnpkg.com/@octokit/types/-/types-6.40.0.tgz#f2e665196d419e19bb4265603cf904a820505d0e" @@ -10489,13 +10539,20 @@ resolved "https://registry.yarnpkg.com/@types/range-parser/-/range-parser-1.2.4.tgz#cd667bcfdd025213aafb7ca5915a932590acdcdc" integrity sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw== -"@types/react-dom@*", "@types/react-dom@<18.0.0", "@types/react-dom@^17", "@types/react-dom@^18.0.0": +"@types/react-dom@*", "@types/react-dom@<18.0.0": version "17.0.20" resolved "https://registry.yarnpkg.com/@types/react-dom/-/react-dom-17.0.20.tgz#e0c8901469d732b36d8473b40b679ad899da1b53" integrity sha512-4pzIjSxDueZZ90F52mU3aPoogkHIoSIDG+oQ+wQK7Cy2B9S+MvOqY0uEA/qawKz381qrEDkvpwyt8Bm31I8sbA== dependencies: "@types/react" "^17" +"@types/react-dom@^18.0.0": + version "18.2.7" + resolved "https://registry.yarnpkg.com/@types/react-dom/-/react-dom-18.2.7.tgz#67222a08c0a6ae0a0da33c3532348277c70abb63" + integrity sha512-GRaAEriuT4zp9N4p1i8BDBYmEyfo+xQ3yHjJU4eiK5NDa1RmUZG+unZABUTK4/Ox/M+GaHwb6Ow8rUITrtjszA== + dependencies: + "@types/react" "*" + "@types/react-is@^18.2.1": version "18.2.1" resolved "https://registry.yarnpkg.com/@types/react-is/-/react-is-18.2.1.tgz#61d01c2a6fc089a53520c0b66996d458fdc46863" @@ -16863,11 +16920,21 @@ graphql-yoga@^3.3.0: lru-cache "^7.14.1" tslib "^2.3.1" -graphql@*, graphql@16.5.0, "graphql@^15.0.0 || ^16.0.0", graphql@^15.5.0, graphql@^15.5.1, graphql@^16.0.0, graphql@^16.3.0, graphql@^16.5.0, graphql@^16.6.0: +graphql@*, "graphql@^15.0.0 || ^16.0.0", graphql@^16.0.0, graphql@^16.3.0, graphql@^16.5.0, graphql@^16.6.0: version "16.6.0" resolved "https://registry.yarnpkg.com/graphql/-/graphql-16.6.0.tgz#c2dcffa4649db149f6282af726c8c83f1c7c5fdb" integrity sha512-KPIBPDlW7NxrbT/eh4qPXz5FiFdL5UbaA0XUNz2Rp3Z3hqBSkbj0GVjwFDztsWVauZUWsbKHgMg++sk8UX0bkw== +graphql@16.5.0: + version "16.5.0" + resolved "https://registry.yarnpkg.com/graphql/-/graphql-16.5.0.tgz#41b5c1182eaac7f3d47164fb247f61e4dfb69c85" + integrity sha512-qbHgh8Ix+j/qY+a/ZcJnFQ+j8ezakqPiHwPiZhV/3PgGlgf96QMBB5/f2rkiC9sgLoy/xvT6TSiaf2nTHJh5iA== + +graphql@^15.5.0, graphql@^15.5.1: + version "15.8.0" + resolved "https://registry.yarnpkg.com/graphql/-/graphql-15.8.0.tgz#33410e96b012fa3bdb1091cc99a94769db212b38" + integrity sha512-5gghUc24tP9HRznNpV2+FIoq3xKkj5dTQqf4v0CpdPbFVwFkWoxOM+o+2OC9ZSvjEMTjfmG9QT+gcvggTwW1zw== + gtoken@^6.0.0: version "6.1.0" resolved "https://registry.yarnpkg.com/gtoken/-/gtoken-6.1.0.tgz#62938c679b364662ce21077858e0db3cfe025363" @@ -27967,11 +28034,21 @@ yaml-ast-parser@0.0.43, yaml-ast-parser@^0.0.43: resolved "https://registry.yarnpkg.com/yaml-ast-parser/-/yaml-ast-parser-0.0.43.tgz#e8a23e6fb4c38076ab92995c5dca33f3d3d7c9bb" integrity sha512-2PTINUwsRqSd+s8XxKaJWQlUuEMHJQyEuh2edBbW8KNJz0SJPwUSD2zRWqezFEdN7IzAgeuYHFUCF7o8zRdZ0A== -yaml@^1.10.0, yaml@^1.10.2, yaml@^1.7.2, yaml@^2.0.0, yaml@^2.1.3, yaml@^2.2.1, yaml@^2.2.2, yaml@^2.3.1: +yaml@^1.10.0, yaml@^1.10.2, yaml@^1.7.2: + version "1.10.2" + resolved "https://registry.yarnpkg.com/yaml/-/yaml-1.10.2.tgz#2301c5ffbf12b467de8da2333a459e29e7920e4b" + integrity sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg== + +yaml@^2.0.0, yaml@^2.1.3, yaml@^2.2.1, yaml@^2.2.2: version "2.2.2" resolved "https://registry.yarnpkg.com/yaml/-/yaml-2.2.2.tgz#ec551ef37326e6d42872dad1970300f8eb83a073" integrity sha512-CBKFWExMn46Foo4cldiChEzn7S7SRV+wqiluAb6xmueD/fGyRHIhX8m14vVGgeFWjN540nKCNVj6P21eQjgTuA== +yaml@^2.3.1: + version "2.3.2" + resolved "https://registry.yarnpkg.com/yaml/-/yaml-2.3.2.tgz#f522db4313c671a0ca963a75670f1c12ea909144" + integrity sha512-N/lyzTPaJasoDmfV7YTrYCI0G/3ivm/9wdG0aHuheKowWQwGTsK0Eoiw6utmzAnI6pkJa0DUVygvp3spqqEKXg== + yargs-parser@20.2.4: version "20.2.4" resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-20.2.4.tgz#b42890f14566796f85ae8e3a25290d205f154a54"