diff --git a/lib/modules/datasource/sbt-package/index.spec.ts b/lib/modules/datasource/sbt-package/index.spec.ts index 5613cdd4c1dbf0..72237f9d2ffd4b 100644 --- a/lib/modules/datasource/sbt-package/index.spec.ts +++ b/lib/modules/datasource/sbt-package/index.spec.ts @@ -66,8 +66,6 @@ describe('modules/datasource/sbt-package/index', () => { ) .get('/maven2/com/example/empty/') .reply(200, '') - .get('/maven2/com.example/') - .reply(404) .get('/maven2/com/example/empty/maven-metadata.xml') .reply(404) .get('/maven2/com/example/empty/index.html') diff --git a/lib/modules/datasource/sbt-package/index.ts b/lib/modules/datasource/sbt-package/index.ts index 763e8dbe12de88..2f9848a6b4a307 100644 --- a/lib/modules/datasource/sbt-package/index.ts +++ b/lib/modules/datasource/sbt-package/index.ts @@ -1,8 +1,9 @@ +import * as upath from 'upath'; import { XmlDocument } from 'xmldoc'; import { logger } from '../../../logger'; import { Http } from '../../../util/http'; import { regEx } from '../../../util/regex'; -import { ensureTrailingSlash } from '../../../util/url'; +import { ensureTrailingSlash, trimTrailingSlash } from '../../../util/url'; import * as ivyVersioning from '../../versioning/ivy'; import { compare } from '../../versioning/maven/compare'; import { MavenDatasource } from '../maven'; @@ -18,6 +19,12 @@ import type { } from '../types'; import { extractPageLinks, getLatestVersion } from './util'; +interface ScalaDepCoordinate { + groupId: string; + artifactId: string; + scalaVersion?: string; +} + export class SbtPackageDatasource extends MavenDatasource { static override readonly id = 'sbt-package'; @@ -36,100 +43,135 @@ export class SbtPackageDatasource extends MavenDatasource { this.http = new Http('sbt'); } - async getArtifactSubdirs( - searchRoot: string, - artifact: string, - scalaVersion: string, - ): Promise { - const pkgUrl = ensureTrailingSlash(searchRoot); - const res = await downloadHttpProtocol(this.http, pkgUrl); - const indexContent = res?.body; - if (indexContent) { - const rootPath = new URL(pkgUrl).pathname; - let artifactSubdirs = extractPageLinks(indexContent, (href) => { + protected static parseDepCoordinate(packageName: string): ScalaDepCoordinate { + const [groupId, javaArtifactId] = packageName.split(':'); + const [artifactId, scalaVersion] = javaArtifactId.split('_'); + return { groupId, artifactId, scalaVersion }; + } + + async getSbtReleases( + registryUrl: string, + packageName: string, + ): Promise { + const { groupId, artifactId, scalaVersion } = + SbtPackageDatasource.parseDepCoordinate(packageName); + + const groupIdSplit = groupId.split('.'); + const repoRootUrl = ensureTrailingSlash(registryUrl); + const packageRootUrlWith = (sep: string): string => + `${repoRootUrl}${groupIdSplit.join(sep)}`; + const packageRootUrls: string[] = []; + packageRootUrls.push(ensureTrailingSlash(packageRootUrlWith('/'))); + packageRootUrls.push(ensureTrailingSlash(packageRootUrlWith('.'))); + + let dependencyUrl: string | undefined; + let packageUrls: string[] | undefined; + for (const packageRootUrl of packageRootUrls) { + const res = await downloadHttpProtocol(this.http, packageRootUrl); + if (!res) { + continue; + } + + dependencyUrl = trimTrailingSlash(packageRootUrl); + + const rootPath = new URL(packageRootUrl).pathname; + const artifactSubdirs = extractPageLinks(res.body, (href) => { const path = href.replace(rootPath, ''); + if ( - path.startsWith(`${artifact}_native`) || - path.startsWith(`${artifact}_sjs`) + path.startsWith(`${artifactId}_native`) || + path.startsWith(`${artifactId}_sjs`) ) { return null; } - if (path === artifact || path.startsWith(`${artifact}_`)) { - return path; + if (path === artifactId || path.startsWith(`${artifactId}_`)) { + return ensureTrailingSlash(`${packageRootUrl}${path}`); } return null; }); - if ( - scalaVersion && - artifactSubdirs.includes(`${artifact}_${scalaVersion}`) - ) { - artifactSubdirs = [`${artifact}_${scalaVersion}`]; + if (scalaVersion) { + const scalaSubdir = artifactSubdirs.find((x) => + x.endsWith(`/${artifactId}_${scalaVersion}/`), + ); + if (scalaSubdir) { + packageUrls = [scalaSubdir]; + break; + } } - return artifactSubdirs; - } - return null; - } + packageUrls = artifactSubdirs; + break; + } - async getPackageReleases( - searchRoot: string, - artifactSubdirs: string[] | null, - ): Promise { - if (artifactSubdirs) { - const releases: string[] = []; - for (const searchSubdir of artifactSubdirs) { - const pkgUrl = ensureTrailingSlash(`${searchRoot}/${searchSubdir}`); - const res = await downloadHttpProtocol(this.http, pkgUrl); - const content = res?.body; - if (content) { - const rootPath = new URL(pkgUrl).pathname; - const subdirReleases = extractPageLinks(content, (href) => { - const path = href.replace(rootPath, ''); - if (path.startsWith('.')) { - return null; - } + if (!packageUrls) { + return null; + } - return path; - }); + const validPackageUrls: string[] = []; + const allVersions = new Set(); + for (const pkgUrl of packageUrls) { + const res = await downloadHttpProtocol(this.http, pkgUrl); + // istanbul ignore if + if (!res) { + continue; + } + validPackageUrls.push(pkgUrl); - subdirReleases.forEach((x) => releases.push(x)); + const rootPath = new URL(pkgUrl).pathname; + const versions = extractPageLinks(res.body, (href) => { + const path = href.replace(rootPath, ''); + if (path.startsWith('.')) { + return null; } - } - if (releases.length) { - return [...new Set(releases)].sort(compare); + + return path; + }); + + for (const version of versions) { + allVersions.add(version); } } - return null; + const versions = [...allVersions]; + if (!versions.length) { + return null; + } + + const latestVersion = getLatestVersion(versions); + const pomInfo = await this.getPomInfo(packageUrls, latestVersion); + + const releases: Release[] = [...allVersions] + .sort(compare) + .map((version) => ({ version })); + return { releases, dependencyUrl, ...pomInfo }; } - async getUrls( - searchRoot: string, - artifactDirs: string[] | null, + async getPomInfo( + packageUrls: string[], version: string | null, - ): Promise> { - const result: Partial = {}; + ): Promise> { + const result: Pick = {}; - if (!artifactDirs?.length) { + // istanbul ignore if + if (!packageUrls?.length) { return result; } + // istanbul ignore if if (!version) { return result; } - for (const artifactDir of artifactDirs) { + for (const packageUrl of packageUrls) { + const artifactDir = upath.basename(packageUrl); const [artifact] = artifactDir.split('_'); - const pomFileNames = [ - `${artifactDir}-${version}.pom`, - `${artifact}-${version}.pom`, - ]; - for (const pomFileName of pomFileNames) { - const pomUrl = `${searchRoot}/${artifactDir}/${version}/${pomFileName}`; + for (const pomFilePrefix of [artifactDir, artifact]) { + const pomFileName = `${pomFilePrefix}-${version}.pom`; + const pomUrl = `${packageUrl}${version}/${pomFileName}`; const res = await downloadHttpProtocol(this.http, pomUrl); const content = res?.body; if (content) { @@ -166,58 +208,20 @@ export class SbtPackageDatasource extends MavenDatasource { return null; } - const [groupId, artifactId] = packageName.split(':'); - const groupIdSplit = groupId.split('.'); - const artifactIdSplit = artifactId.split('_'); - const [artifact, scalaVersion] = artifactIdSplit; - - const repoRoot = ensureTrailingSlash(registryUrl); - const searchRoots: string[] = []; - // Optimize lookup order - searchRoots.push(`${repoRoot}${groupIdSplit.join('/')}`); - searchRoots.push(`${repoRoot}${groupIdSplit.join('.')}`); - - for (let idx = 0; idx < searchRoots.length; idx += 1) { - const searchRoot = searchRoots[idx]; - const artifactSubdirs = await this.getArtifactSubdirs( - searchRoot, - artifact, - scalaVersion, - ); - const versions = await this.getPackageReleases( - searchRoot, - artifactSubdirs, - ); - const latestVersion = getLatestVersion(versions); - const urls = await this.getUrls( - searchRoot, - artifactSubdirs, - latestVersion, - ); - - const dependencyUrl = searchRoot; - - logger.trace({ dependency: packageName, versions }, `Package versions`); - if (versions) { - return { - ...urls, - dependencyUrl, - releases: versions.map((v) => ({ version: v })), - }; - } + const sbtReleases = await this.getSbtReleases(registryUrl, packageName); + if (sbtReleases) { + return sbtReleases; } logger.debug( - `No versions discovered for ${packageName} listing organization root package folder, fallback to maven datasource for version discovery`, + `Sbt: no versions discovered for ${packageName} listing organization root package folder, fallback to maven datasource for version discovery`, ); const mavenReleaseResult = await super.getReleases(config); if (mavenReleaseResult) { return mavenReleaseResult; } - logger.debug( - `No versions found for ${packageName} in ${searchRoots.length} repositories`, - ); + logger.debug(`Sbt: no versions found for "${packageName}"`); return null; }