Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor(sbt-package): Flatten fetching code #31836

Merged
merged 8 commits into from
Oct 9, 2024
Merged

refactor(sbt-package): Flatten fetching code #31836

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
ab86cd0
refactor(sbt): Decouple `sbt-package` and `sbt-plugin` datasources
zharinov Oct 7, 2024
7839104
Fix whitespace
zharinov Oct 7, 2024
3d903dd
Don't inherit from `MavenDatasource`
zharinov Oct 7, 2024
ec01327
refactor(sbt-package): Flatten fetching code
zharinov Oct 7, 2024
ce306bd
Merge branch 'main' into refactor/sbt-packages-reduce-nesting
zharinov Oct 8, 2024
6de7f0f
Fix trivial coverage
zharinov Oct 8, 2024
5864e3c
Fix
zharinov Oct 8, 2024
23cdc71
Fix lint
zharinov Oct 8, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions lib/modules/datasource/sbt-package/index.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,8 +66,6 @@ describe('modules/datasource/sbt-package/index', () => {
)
.get('/maven2/com/example/empty/')
.reply(200, '')
.get('/maven2/com.example/')
.reply(404)
.get('/maven2/com/example/empty/maven-metadata.xml')
.reply(404)
.get('/maven2/com/example/empty/index.html')
Expand Down
214 changes: 109 additions & 105 deletions lib/modules/datasource/sbt-package/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
import * as upath from 'upath';
import { XmlDocument } from 'xmldoc';
import { logger } from '../../../logger';
import { Http } from '../../../util/http';
import { regEx } from '../../../util/regex';
import { ensureTrailingSlash } from '../../../util/url';
import { ensureTrailingSlash, trimTrailingSlash } from '../../../util/url';
import * as ivyVersioning from '../../versioning/ivy';
import { compare } from '../../versioning/maven/compare';
import { MavenDatasource } from '../maven';
Expand All @@ -18,6 +19,12 @@ import type {
} from '../types';
import { extractPageLinks, getLatestVersion } from './util';

interface ScalaDepCoordinate {
groupId: string;
artifactId: string;
scalaVersion?: string;
}

export class SbtPackageDatasource extends MavenDatasource {
static override readonly id = 'sbt-package';

Expand All @@ -36,100 +43,135 @@ export class SbtPackageDatasource extends MavenDatasource {
this.http = new Http('sbt');
}

async getArtifactSubdirs(
searchRoot: string,
artifact: string,
scalaVersion: string,
): Promise<string[] | null> {
const pkgUrl = ensureTrailingSlash(searchRoot);
const res = await downloadHttpProtocol(this.http, pkgUrl);
const indexContent = res?.body;
if (indexContent) {
const rootPath = new URL(pkgUrl).pathname;
let artifactSubdirs = extractPageLinks(indexContent, (href) => {
protected static parseDepCoordinate(packageName: string): ScalaDepCoordinate {
const [groupId, javaArtifactId] = packageName.split(':');
const [artifactId, scalaVersion] = javaArtifactId.split('_');
return { groupId, artifactId, scalaVersion };
}

async getSbtReleases(
registryUrl: string,
packageName: string,
): Promise<ReleaseResult | null> {
const { groupId, artifactId, scalaVersion } =
SbtPackageDatasource.parseDepCoordinate(packageName);

const groupIdSplit = groupId.split('.');
const repoRootUrl = ensureTrailingSlash(registryUrl);
const packageRootUrlWith = (sep: string): string =>
`${repoRootUrl}${groupIdSplit.join(sep)}`;
const packageRootUrls: string[] = [];
packageRootUrls.push(ensureTrailingSlash(packageRootUrlWith('/')));
packageRootUrls.push(ensureTrailingSlash(packageRootUrlWith('.')));

let dependencyUrl: string | undefined;
let packageUrls: string[] | undefined;
for (const packageRootUrl of packageRootUrls) {
const res = await downloadHttpProtocol(this.http, packageRootUrl);
if (!res) {
continue;
}

dependencyUrl = trimTrailingSlash(packageRootUrl);

const rootPath = new URL(packageRootUrl).pathname;
const artifactSubdirs = extractPageLinks(res.body, (href) => {
const path = href.replace(rootPath, '');

if (
path.startsWith(`${artifact}_native`) ||
path.startsWith(`${artifact}_sjs`)
path.startsWith(`${artifactId}_native`) ||
path.startsWith(`${artifactId}_sjs`)
) {
return null;
}

if (path === artifact || path.startsWith(`${artifact}_`)) {
return path;
if (path === artifactId || path.startsWith(`${artifactId}_`)) {
return ensureTrailingSlash(`${packageRootUrl}${path}`);
}

return null;
});

if (
scalaVersion &&
artifactSubdirs.includes(`${artifact}_${scalaVersion}`)
) {
artifactSubdirs = [`${artifact}_${scalaVersion}`];
if (scalaVersion) {
const scalaSubdir = artifactSubdirs.find((x) =>
x.endsWith(`/${artifactId}_${scalaVersion}/`),
);
if (scalaSubdir) {
packageUrls = [scalaSubdir];
break;
}
}
return artifactSubdirs;
}

return null;
}
packageUrls = artifactSubdirs;
break;
}

async getPackageReleases(
searchRoot: string,
artifactSubdirs: string[] | null,
): Promise<string[] | null> {
if (artifactSubdirs) {
const releases: string[] = [];
for (const searchSubdir of artifactSubdirs) {
const pkgUrl = ensureTrailingSlash(`${searchRoot}/${searchSubdir}`);
const res = await downloadHttpProtocol(this.http, pkgUrl);
const content = res?.body;
if (content) {
const rootPath = new URL(pkgUrl).pathname;
const subdirReleases = extractPageLinks(content, (href) => {
const path = href.replace(rootPath, '');
if (path.startsWith('.')) {
return null;
}
if (!packageUrls) {
return null;
}

return path;
});
const validPackageUrls: string[] = [];
const allVersions = new Set<string>();
for (const pkgUrl of packageUrls) {
const res = await downloadHttpProtocol(this.http, pkgUrl);
// istanbul ignore if
if (!res) {
continue;
}
validPackageUrls.push(pkgUrl);

subdirReleases.forEach((x) => releases.push(x));
const rootPath = new URL(pkgUrl).pathname;
const versions = extractPageLinks(res.body, (href) => {
const path = href.replace(rootPath, '');
if (path.startsWith('.')) {
return null;
}
}
if (releases.length) {
return [...new Set(releases)].sort(compare);

return path;
});

for (const version of versions) {
allVersions.add(version);
}
}

return null;
const versions = [...allVersions];
if (!versions.length) {
return null;
}

const latestVersion = getLatestVersion(versions);
const pomInfo = await this.getPomInfo(packageUrls, latestVersion);

const releases: Release[] = [...allVersions]
.sort(compare)
.map((version) => ({ version }));
return { releases, dependencyUrl, ...pomInfo };
}

async getUrls(
searchRoot: string,
artifactDirs: string[] | null,
async getPomInfo(
packageUrls: string[],
version: string | null,
): Promise<Partial<ReleaseResult>> {
const result: Partial<ReleaseResult> = {};
): Promise<Pick<ReleaseResult, 'homepage' | 'sourceUrl'>> {
const result: Pick<ReleaseResult, 'homepage' | 'sourceUrl'> = {};

if (!artifactDirs?.length) {
// istanbul ignore if
if (!packageUrls?.length) {
return result;
}

// istanbul ignore if
if (!version) {
return result;
}

for (const artifactDir of artifactDirs) {
for (const packageUrl of packageUrls) {
const artifactDir = upath.basename(packageUrl);
const [artifact] = artifactDir.split('_');
const pomFileNames = [
`${artifactDir}-${version}.pom`,
`${artifact}-${version}.pom`,
];

for (const pomFileName of pomFileNames) {
const pomUrl = `${searchRoot}/${artifactDir}/${version}/${pomFileName}`;
for (const pomFilePrefix of [artifactDir, artifact]) {
const pomFileName = `${pomFilePrefix}-${version}.pom`;
const pomUrl = `${packageUrl}${version}/${pomFileName}`;
const res = await downloadHttpProtocol(this.http, pomUrl);
const content = res?.body;
if (content) {
Expand Down Expand Up @@ -166,58 +208,20 @@ export class SbtPackageDatasource extends MavenDatasource {
return null;
}

const [groupId, artifactId] = packageName.split(':');
const groupIdSplit = groupId.split('.');
const artifactIdSplit = artifactId.split('_');
const [artifact, scalaVersion] = artifactIdSplit;

const repoRoot = ensureTrailingSlash(registryUrl);
const searchRoots: string[] = [];
// Optimize lookup order
searchRoots.push(`${repoRoot}${groupIdSplit.join('/')}`);
searchRoots.push(`${repoRoot}${groupIdSplit.join('.')}`);

for (let idx = 0; idx < searchRoots.length; idx += 1) {
const searchRoot = searchRoots[idx];
const artifactSubdirs = await this.getArtifactSubdirs(
searchRoot,
artifact,
scalaVersion,
);
const versions = await this.getPackageReleases(
searchRoot,
artifactSubdirs,
);
const latestVersion = getLatestVersion(versions);
const urls = await this.getUrls(
searchRoot,
artifactSubdirs,
latestVersion,
);

const dependencyUrl = searchRoot;

logger.trace({ dependency: packageName, versions }, `Package versions`);
if (versions) {
return {
...urls,
dependencyUrl,
releases: versions.map((v) => ({ version: v })),
};
}
const sbtReleases = await this.getSbtReleases(registryUrl, packageName);
if (sbtReleases) {
return sbtReleases;
}

logger.debug(
`No versions discovered for ${packageName} listing organization root package folder, fallback to maven datasource for version discovery`,
`Sbt: no versions discovered for ${packageName} listing organization root package folder, fallback to maven datasource for version discovery`,
);
const mavenReleaseResult = await super.getReleases(config);
if (mavenReleaseResult) {
return mavenReleaseResult;
}

logger.debug(
`No versions found for ${packageName} in ${searchRoots.length} repositories`,
);
logger.debug(`Sbt: no versions found for "${packageName}"`);
return null;
}

Expand Down