From ab86cd0576cb87bec777ff492ce82f841a33375b Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Mon, 7 Oct 2024 15:15:05 -0300 Subject: [PATCH 1/7] refactor(sbt): Decouple `sbt-package` and `sbt-plugin` datasources --- lib/modules/datasource/sbt-package/index.ts | 2 +- lib/modules/datasource/sbt-plugin/index.ts | 138 +++++++++++++++++++- 2 files changed, 132 insertions(+), 8 deletions(-) diff --git a/lib/modules/datasource/sbt-package/index.ts b/lib/modules/datasource/sbt-package/index.ts index c0d5403b63055f..ba081d56a6fce7 100644 --- a/lib/modules/datasource/sbt-package/index.ts +++ b/lib/modules/datasource/sbt-package/index.ts @@ -16,7 +16,7 @@ import type { import { extractPageLinks, getLatestVersion } from './util'; export class SbtPackageDatasource extends MavenDatasource { - static override id = 'sbt-package'; + static override readonly id = 'sbt-package'; override readonly defaultRegistryUrls = [MAVEN_REPO]; diff --git a/lib/modules/datasource/sbt-plugin/index.ts b/lib/modules/datasource/sbt-plugin/index.ts index e053811936ec68..ab0c073d387a98 100644 --- a/lib/modules/datasource/sbt-plugin/index.ts +++ b/lib/modules/datasource/sbt-plugin/index.ts @@ -1,12 +1,13 @@ +import { XmlDocument } from 'xmldoc'; import { logger } from '../../../logger'; import { Http } from '../../../util/http'; import { regEx } from '../../../util/regex'; import { ensureTrailingSlash } from '../../../util/url'; import * as ivyVersioning from '../../versioning/ivy'; import { compare } from '../../versioning/maven/compare'; +import { MavenDatasource } from '../maven'; import { MAVEN_REPO } from '../maven/common'; import { downloadHttpProtocol } from '../maven/util'; -import { SbtPackageDatasource } from '../sbt-package'; import { extractPageLinks, getLatestVersion } from '../sbt-package/util'; import type { GetReleasesConfig, @@ -17,17 +18,15 @@ import type { export const SBT_PLUGINS_REPO = 'https://repo.scala-sbt.org/scalasbt/sbt-plugin-releases'; -export const defaultRegistryUrls = [SBT_PLUGINS_REPO, MAVEN_REPO]; - -export class SbtPluginDatasource extends SbtPackageDatasource { +export class SbtPluginDatasource extends MavenDatasource { static override readonly id = 'sbt-plugin'; - override readonly defaultRegistryUrls = defaultRegistryUrls; - - override readonly registryStrategy: RegistryStrategy = 'merge'; + override readonly defaultRegistryUrls = [SBT_PLUGINS_REPO, MAVEN_REPO]; override readonly defaultVersioning = ivyVersioning.id; + override readonly registryStrategy: RegistryStrategy = 'merge'; + override readonly sourceUrlSupport = 'package'; override readonly sourceUrlNote = 'The source URL is determined from the `scm` tags in the results.'; @@ -37,6 +36,131 @@ export class SbtPluginDatasource extends SbtPackageDatasource { this.http = new Http('sbt'); } + // istanbul ignore next: to be rewritten + async getArtifactSubdirs( + searchRoot: string, + artifact: string, + scalaVersion: string, + ): Promise { + const pkgUrl = ensureTrailingSlash(searchRoot); + const res = await downloadHttpProtocol(this.http, pkgUrl); + const indexContent = res?.body; + if (indexContent) { + const rootPath = new URL(pkgUrl).pathname; + let artifactSubdirs = extractPageLinks(indexContent, (href) => { + const path = href.replace(rootPath, ''); + if ( + path.startsWith(`${artifact}_native`) || + path.startsWith(`${artifact}_sjs`) + ) { + return null; + } + + if (path === artifact || path.startsWith(`${artifact}_`)) { + return path; + } + + return null; + }); + + if ( + scalaVersion && + artifactSubdirs.includes(`${artifact}_${scalaVersion}`) + ) { + artifactSubdirs = [`${artifact}_${scalaVersion}`]; + } + return artifactSubdirs; + } + + return null; + } + + // istanbul ignore next: to be rewritten + async getPackageReleases( + searchRoot: string, + artifactSubdirs: string[] | null, + ): Promise { + if (artifactSubdirs) { + const releases: string[] = []; + for (const searchSubdir of artifactSubdirs) { + const pkgUrl = ensureTrailingSlash(`${searchRoot}/${searchSubdir}`); + const res = await downloadHttpProtocol(this.http, pkgUrl); + const content = res?.body; + if (content) { + const rootPath = new URL(pkgUrl).pathname; + const subdirReleases = extractPageLinks(content, (href) => { + const path = href.replace(rootPath, ''); + if (path.startsWith('.')) { + return null; + } + + return path; + }); + + subdirReleases.forEach((x) => releases.push(x)); + } + } + if (releases.length) { + return [...new Set(releases)].sort(compare); + } + } + + return null; + } + + // istanbul ignore next: to be rewritten + + async getUrls( + searchRoot: string, + artifactDirs: string[] | null, + version: string | null, + ): Promise> { + const result: Partial = {}; + + if (!artifactDirs?.length) { + return result; + } + + if (!version) { + return result; + } + + for (const artifactDir of artifactDirs) { + const [artifact] = artifactDir.split('_'); + const pomFileNames = [ + `${artifactDir}-${version}.pom`, + `${artifact}-${version}.pom`, + ]; + + for (const pomFileName of pomFileNames) { + const pomUrl = `${searchRoot}/${artifactDir}/${version}/${pomFileName}`; + const res = await downloadHttpProtocol(this.http, pomUrl); + const content = res?.body; + if (content) { + const pomXml = new XmlDocument(content); + + const homepage = pomXml.valueWithPath('url'); + if (homepage) { + result.homepage = homepage; + } + + const sourceUrl = pomXml.valueWithPath('scm.url'); + if (sourceUrl) { + result.sourceUrl = sourceUrl + .replace(regEx(/^scm:/), '') + .replace(regEx(/^git:/), '') + .replace(regEx(/^git@github.com:/), 'https://github.com/') + .replace(regEx(/\.git$/), ''); + } + + return result; + } + } + } + + return result; + } + async resolvePluginReleases( rootUrl: string, artifact: string, From 7839104d0ff54117f8d87290a56471c4b95d6ab1 Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Mon, 7 Oct 2024 15:17:17 -0300 Subject: [PATCH 2/7] Fix whitespace --- lib/modules/datasource/sbt-plugin/index.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/modules/datasource/sbt-plugin/index.ts b/lib/modules/datasource/sbt-plugin/index.ts index ab0c073d387a98..0fd5347b73c4d8 100644 --- a/lib/modules/datasource/sbt-plugin/index.ts +++ b/lib/modules/datasource/sbt-plugin/index.ts @@ -109,7 +109,6 @@ export class SbtPluginDatasource extends MavenDatasource { } // istanbul ignore next: to be rewritten - async getUrls( searchRoot: string, artifactDirs: string[] | null, From 3d903dd67854a75a17a29ddbb8c45d10a3de40ce Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Mon, 7 Oct 2024 15:21:16 -0300 Subject: [PATCH 3/7] Don't inherit from `MavenDatasource` --- lib/modules/datasource/sbt-plugin/index.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/modules/datasource/sbt-plugin/index.ts b/lib/modules/datasource/sbt-plugin/index.ts index 0fd5347b73c4d8..fc84d656421be0 100644 --- a/lib/modules/datasource/sbt-plugin/index.ts +++ b/lib/modules/datasource/sbt-plugin/index.ts @@ -5,7 +5,7 @@ import { regEx } from '../../../util/regex'; import { ensureTrailingSlash } from '../../../util/url'; import * as ivyVersioning from '../../versioning/ivy'; import { compare } from '../../versioning/maven/compare'; -import { MavenDatasource } from '../maven'; +import { Datasource } from '../datasource'; import { MAVEN_REPO } from '../maven/common'; import { downloadHttpProtocol } from '../maven/util'; import { extractPageLinks, getLatestVersion } from '../sbt-package/util'; @@ -18,8 +18,8 @@ import type { export const SBT_PLUGINS_REPO = 'https://repo.scala-sbt.org/scalasbt/sbt-plugin-releases'; -export class SbtPluginDatasource extends MavenDatasource { - static override readonly id = 'sbt-plugin'; +export class SbtPluginDatasource extends Datasource { + static readonly id = 'sbt-plugin'; override readonly defaultRegistryUrls = [SBT_PLUGINS_REPO, MAVEN_REPO]; From ec01327ca7b8ce48ef8023b841577ebad83f7e67 Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Mon, 7 Oct 2024 15:29:41 -0300 Subject: [PATCH 4/7] refactor(sbt-package): Flatten fetching code --- .../datasource/sbt-package/index.spec.ts | 2 - lib/modules/datasource/sbt-package/index.ts | 210 +++++++++--------- 2 files changed, 105 insertions(+), 107 deletions(-) diff --git a/lib/modules/datasource/sbt-package/index.spec.ts b/lib/modules/datasource/sbt-package/index.spec.ts index 5613cdd4c1dbf0..72237f9d2ffd4b 100644 --- a/lib/modules/datasource/sbt-package/index.spec.ts +++ b/lib/modules/datasource/sbt-package/index.spec.ts @@ -66,8 +66,6 @@ describe('modules/datasource/sbt-package/index', () => { ) .get('/maven2/com/example/empty/') .reply(200, '') - .get('/maven2/com.example/') - .reply(404) .get('/maven2/com/example/empty/maven-metadata.xml') .reply(404) .get('/maven2/com/example/empty/index.html') diff --git a/lib/modules/datasource/sbt-package/index.ts b/lib/modules/datasource/sbt-package/index.ts index ba081d56a6fce7..4986deca6f5b3e 100644 --- a/lib/modules/datasource/sbt-package/index.ts +++ b/lib/modules/datasource/sbt-package/index.ts @@ -1,8 +1,9 @@ +import * as upath from 'upath'; import { XmlDocument } from 'xmldoc'; import { logger } from '../../../logger'; import { Http } from '../../../util/http'; import { regEx } from '../../../util/regex'; -import { ensureTrailingSlash } from '../../../util/url'; +import { ensureTrailingSlash, trimTrailingSlash } from '../../../util/url'; import * as ivyVersioning from '../../versioning/ivy'; import { compare } from '../../versioning/maven/compare'; import { MavenDatasource } from '../maven'; @@ -11,10 +12,17 @@ import { downloadHttpProtocol } from '../maven/util'; import type { GetReleasesConfig, RegistryStrategy, + Release, ReleaseResult, } from '../types'; import { extractPageLinks, getLatestVersion } from './util'; +interface ScalaDepCoordinate { + groupId: string; + artifactId: string; + scalaVersion?: string; +} + export class SbtPackageDatasource extends MavenDatasource { static override readonly id = 'sbt-package'; @@ -33,84 +41,116 @@ export class SbtPackageDatasource extends MavenDatasource { this.http = new Http('sbt'); } - async getArtifactSubdirs( - searchRoot: string, - artifact: string, - scalaVersion: string, - ): Promise { - const pkgUrl = ensureTrailingSlash(searchRoot); - const res = await downloadHttpProtocol(this.http, pkgUrl); - const indexContent = res?.body; - if (indexContent) { - const rootPath = new URL(pkgUrl).pathname; - let artifactSubdirs = extractPageLinks(indexContent, (href) => { + protected static parseDepCoordinate(packageName: string): ScalaDepCoordinate { + const [groupId, javaArtifactId] = packageName.split(':'); + const [artifactId, scalaVersion] = javaArtifactId.split('_'); + return { groupId, artifactId, scalaVersion }; + } + + async getSbtReleases( + registryUrl: string, + packageName: string, + ): Promise { + const { groupId, artifactId, scalaVersion } = + SbtPackageDatasource.parseDepCoordinate(packageName); + + const groupIdSplit = groupId.split('.'); + const repoRootUrl = ensureTrailingSlash(registryUrl); + const packageRootUrlWith = (sep: string): string => + `${repoRootUrl}${groupIdSplit.join(sep)}`; + const packageRootUrls: string[] = []; + packageRootUrls.push(ensureTrailingSlash(packageRootUrlWith('/'))); + packageRootUrls.push(ensureTrailingSlash(packageRootUrlWith('.'))); + + let dependencyUrl: string | undefined; + let packageUrls: string[] | undefined; + for (const packageRootUrl of packageRootUrls) { + const res = await downloadHttpProtocol(this.http, packageRootUrl); + if (!res) { + continue; + } + + dependencyUrl = trimTrailingSlash(packageRootUrl); + + const rootPath = new URL(packageRootUrl).pathname; + const artifactSubdirs = extractPageLinks(res.body, (href) => { const path = href.replace(rootPath, ''); + if ( - path.startsWith(`${artifact}_native`) || - path.startsWith(`${artifact}_sjs`) + path.startsWith(`${artifactId}_native`) || + path.startsWith(`${artifactId}_sjs`) ) { return null; } - if (path === artifact || path.startsWith(`${artifact}_`)) { - return path; + if (path === artifactId || path.startsWith(`${artifactId}_`)) { + return ensureTrailingSlash(`${packageRootUrl}${path}`); } return null; }); - if ( - scalaVersion && - artifactSubdirs.includes(`${artifact}_${scalaVersion}`) - ) { - artifactSubdirs = [`${artifact}_${scalaVersion}`]; + if (scalaVersion) { + const scalaSubdir = `${artifactId}_${scalaVersion}`; + if (artifactSubdirs.includes(scalaSubdir)) { + packageUrls = [scalaSubdir]; + break; + } } - return artifactSubdirs; - } - return null; - } + packageUrls = artifactSubdirs; + break; + } - async getPackageReleases( - searchRoot: string, - artifactSubdirs: string[] | null, - ): Promise { - if (artifactSubdirs) { - const releases: string[] = []; - for (const searchSubdir of artifactSubdirs) { - const pkgUrl = ensureTrailingSlash(`${searchRoot}/${searchSubdir}`); - const res = await downloadHttpProtocol(this.http, pkgUrl); - const content = res?.body; - if (content) { - const rootPath = new URL(pkgUrl).pathname; - const subdirReleases = extractPageLinks(content, (href) => { - const path = href.replace(rootPath, ''); - if (path.startsWith('.')) { - return null; - } + if (!packageUrls) { + return null; + } - return path; - }); + const validPackageUrls: string[] = []; + const allVersions = new Set(); + for (const pkgUrl of packageUrls) { + const res = await downloadHttpProtocol(this.http, pkgUrl); + if (!res) { + continue; + } + validPackageUrls.push(pkgUrl); - subdirReleases.forEach((x) => releases.push(x)); + const rootPath = new URL(pkgUrl).pathname; + const versions = extractPageLinks(res.body, (href) => { + const path = href.replace(rootPath, ''); + if (path.startsWith('.')) { + return null; } - } - if (releases.length) { - return [...new Set(releases)].sort(compare); + + return path; + }); + + for (const version of versions) { + allVersions.add(version); } } - return null; + const versions = [...allVersions]; + if (!versions.length) { + return null; + } + + const latestVersion = getLatestVersion(versions); + const pomInfo = await this.getPomInfo(packageUrls, latestVersion); + + const releases: Release[] = [...allVersions] + .sort(compare) + .map((version) => ({ version })); + return { releases, dependencyUrl, ...pomInfo }; } - async getUrls( - searchRoot: string, - artifactDirs: string[] | null, + async getPomInfo( + packageUrls: string[], version: string | null, - ): Promise> { - const result: Partial = {}; + ): Promise> { + const result: Pick = {}; - if (!artifactDirs?.length) { + if (!packageUrls?.length) { return result; } @@ -118,15 +158,13 @@ export class SbtPackageDatasource extends MavenDatasource { return result; } - for (const artifactDir of artifactDirs) { + for (const packageUrl of packageUrls) { + const artifactDir = upath.basename(packageUrl); const [artifact] = artifactDir.split('_'); - const pomFileNames = [ - `${artifactDir}-${version}.pom`, - `${artifact}-${version}.pom`, - ]; - for (const pomFileName of pomFileNames) { - const pomUrl = `${searchRoot}/${artifactDir}/${version}/${pomFileName}`; + for (const pomFilePrefix of [artifactDir, artifact]) { + const pomFileName = `${pomFilePrefix}-${version}.pom`; + const pomUrl = `${packageUrl}${version}/${pomFileName}`; const res = await downloadHttpProtocol(this.http, pomUrl); const content = res?.body; if (content) { @@ -163,58 +201,20 @@ export class SbtPackageDatasource extends MavenDatasource { return null; } - const [groupId, artifactId] = packageName.split(':'); - const groupIdSplit = groupId.split('.'); - const artifactIdSplit = artifactId.split('_'); - const [artifact, scalaVersion] = artifactIdSplit; - - const repoRoot = ensureTrailingSlash(registryUrl); - const searchRoots: string[] = []; - // Optimize lookup order - searchRoots.push(`${repoRoot}${groupIdSplit.join('/')}`); - searchRoots.push(`${repoRoot}${groupIdSplit.join('.')}`); - - for (let idx = 0; idx < searchRoots.length; idx += 1) { - const searchRoot = searchRoots[idx]; - const artifactSubdirs = await this.getArtifactSubdirs( - searchRoot, - artifact, - scalaVersion, - ); - const versions = await this.getPackageReleases( - searchRoot, - artifactSubdirs, - ); - const latestVersion = getLatestVersion(versions); - const urls = await this.getUrls( - searchRoot, - artifactSubdirs, - latestVersion, - ); - - const dependencyUrl = searchRoot; - - logger.trace({ dependency: packageName, versions }, `Package versions`); - if (versions) { - return { - ...urls, - dependencyUrl, - releases: versions.map((v) => ({ version: v })), - }; - } + const sbtReleases = await this.getSbtReleases(registryUrl, packageName); + if (sbtReleases) { + return sbtReleases; } logger.debug( - `No versions discovered for ${packageName} listing organization root package folder, fallback to maven datasource for version discovery`, + `Sbt: no versions discovered for ${packageName} listing organization root package folder, fallback to maven datasource for version discovery`, ); const mavenReleaseResult = await super.getReleases(config); if (mavenReleaseResult) { return mavenReleaseResult; } - logger.debug( - `No versions found for ${packageName} in ${searchRoots.length} repositories`, - ); + logger.debug(`Sbt: no versions found for "${packageName}"`); return null; } } From 6de7f0f2b30a129d023aa2b7115ee620970e68db Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Tue, 8 Oct 2024 17:02:15 -0300 Subject: [PATCH 5/7] Fix trivial coverage --- lib/modules/datasource/sbt-package/index.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/modules/datasource/sbt-package/index.ts b/lib/modules/datasource/sbt-package/index.ts index d03d3d4f1a125b..0d4fa34d267eec 100644 --- a/lib/modules/datasource/sbt-package/index.ts +++ b/lib/modules/datasource/sbt-package/index.ts @@ -112,6 +112,7 @@ export class SbtPackageDatasource extends MavenDatasource { const allVersions = new Set(); for (const pkgUrl of packageUrls) { const res = await downloadHttpProtocol(this.http, pkgUrl); + // istanbul ignore if if (!res) { continue; } @@ -152,10 +153,12 @@ export class SbtPackageDatasource extends MavenDatasource { ): Promise> { const result: Pick = {}; + // istanbul ignore if if (!packageUrls?.length) { return result; } + // istanbul ignore if if (!version) { return result; } From 5864e3c8edbc4bb4a98202b38c560943472cb27b Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Tue, 8 Oct 2024 17:24:31 -0300 Subject: [PATCH 6/7] Fix --- lib/modules/datasource/sbt-package/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/modules/datasource/sbt-package/index.ts b/lib/modules/datasource/sbt-package/index.ts index 0d4fa34d267eec..5d267f2dd5358c 100644 --- a/lib/modules/datasource/sbt-package/index.ts +++ b/lib/modules/datasource/sbt-package/index.ts @@ -93,8 +93,8 @@ export class SbtPackageDatasource extends MavenDatasource { }); if (scalaVersion) { - const scalaSubdir = `${artifactId}_${scalaVersion}`; - if (artifactSubdirs.includes(scalaSubdir)) { + const scalaSubdir = artifactSubdirs.find((x) => x.endsWith(`/${artifactId}_${scalaVersion}/`)) + if (scalaSubdir) { packageUrls = [scalaSubdir]; break; } From 23cdc71e71482070719cccb92a45803a4910f333 Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Tue, 8 Oct 2024 17:30:11 -0300 Subject: [PATCH 7/7] Fix lint --- lib/modules/datasource/sbt-package/index.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/modules/datasource/sbt-package/index.ts b/lib/modules/datasource/sbt-package/index.ts index 5d267f2dd5358c..2f9848a6b4a307 100644 --- a/lib/modules/datasource/sbt-package/index.ts +++ b/lib/modules/datasource/sbt-package/index.ts @@ -93,7 +93,9 @@ export class SbtPackageDatasource extends MavenDatasource { }); if (scalaVersion) { - const scalaSubdir = artifactSubdirs.find((x) => x.endsWith(`/${artifactId}_${scalaVersion}/`)) + const scalaSubdir = artifactSubdirs.find((x) => + x.endsWith(`/${artifactId}_${scalaVersion}/`), + ); if (scalaSubdir) { packageUrls = [scalaSubdir]; break;