diff --git a/lib/modules/datasource/maven/index.ts b/lib/modules/datasource/maven/index.ts index 62da4ee93750d8..3738414d501237 100644 --- a/lib/modules/datasource/maven/index.ts +++ b/lib/modules/datasource/maven/index.ts @@ -145,19 +145,24 @@ export class MavenDatasource extends Datasource { try { const indexUrl = getMavenUrl(dependency, repoUrl, 'index.html'); const res = await downloadHttpProtocol(this.http, indexUrl); - const { body = '' } = res; - for (const line of body.split(newlineRegex)) { - const match = line.trim().match(mavenCentralHtmlVersionRegex); - if (match) { - const { version, releaseTimestamp: timestamp } = - match?.groups ?? /* istanbul ignore next: hard to test */ {}; - if (version && timestamp) { - const date = DateTime.fromFormat(timestamp, 'yyyy-MM-dd HH:mm', { - zone: 'UTC', - }); - if (date.isValid) { - const releaseTimestamp = date.toISO(); - workingReleaseMap[version] = { version, releaseTimestamp }; + if (res) { + for (const line of res.body.split(newlineRegex)) { + const match = line.trim().match(mavenCentralHtmlVersionRegex); + if (match) { + const { version, releaseTimestamp: timestamp } = + match?.groups ?? /* istanbul ignore next: hard to test */ {}; + if (version && timestamp) { + const date = DateTime.fromFormat( + timestamp, + 'yyyy-MM-dd HH:mm', + { + zone: 'UTC', + }, + ); + if (date.isValid) { + const releaseTimestamp = date.toISO(); + workingReleaseMap[version] = { version, releaseTimestamp }; + } } } } diff --git a/lib/modules/datasource/maven/util.spec.ts b/lib/modules/datasource/maven/util.spec.ts index 0263c203762cc0..fb94fa28ea7cae 100644 --- a/lib/modules/datasource/maven/util.spec.ts +++ b/lib/modules/datasource/maven/util.spec.ts @@ -1,6 +1,7 @@ +import type Request from 'got/dist/source/core'; import { partial } from '../../../../test/util'; import { HOST_DISABLED } from '../../../constants/error-messages'; -import type { Http } from '../../../util/http'; +import { type Http, HttpError } from '../../../util/http'; import { parseUrl } from '../../../util/url'; import { checkResource, @@ -9,6 +10,38 @@ import { downloadS3Protocol, } from './util'; +function httpError({ + name, + message = 'unknown error', + code, + request = {}, + response, +}: { + name?: string; + message?: string; + code?: HttpError['code']; + request?: Partial; + response?: Partial; +}): HttpError { + type Writeable = { -readonly [P in keyof T]: T[P] }; + + const err = new HttpError( + message, + { code }, + request as never, + ) as Writeable; + + if (name) { + err.name = name; + } + + if (response) { + err.response = response as never; + } + + return err; +} + describe('modules/datasource/maven/util', () => { describe('downloadMavenXml', () => { it('returns empty object for unsupported protocols', async () => { @@ -39,51 +72,43 @@ describe('modules/datasource/maven/util', () => { describe('downloadHttpProtocol', () => { it('returns empty for HOST_DISABLED error', async () => { const http = partial({ - get: () => - Promise.reject( - Object.assign(new Error(), { message: HOST_DISABLED }), - ), + get: () => Promise.reject(httpError({ message: HOST_DISABLED })), }); const res = await downloadHttpProtocol(http, 'some://'); - expect(res).toStrictEqual({}); + expect(res).toBeNull(); }); it('returns empty for host error', async () => { const http = partial({ - get: () => - Promise.reject(Object.assign(new Error(), { code: 'ETIMEDOUT' })), + get: () => Promise.reject(httpError({ code: 'ETIMEDOUT' })), }); const res = await downloadHttpProtocol(http, 'some://'); - expect(res).toStrictEqual({}); + expect(res).toBeNull(); }); - it('returns empty for temporal error', async () => { + it('returns empty for temporary error', async () => { const http = partial({ - get: () => - Promise.reject(Object.assign(new Error(), { code: 'ECONNRESET' })), + get: () => Promise.reject(httpError({ code: 'ECONNRESET' })), }); const res = await downloadHttpProtocol(http, 'some://'); - expect(res).toStrictEqual({}); + expect(res).toBeNull(); }); it('returns empty for connection error', async () => { const http = partial({ - get: () => - Promise.reject(Object.assign(new Error(), { code: 'ECONNREFUSED' })), + get: () => Promise.reject(httpError({ code: 'ECONNREFUSED' })), }); const res = await downloadHttpProtocol(http, 'some://'); - expect(res).toStrictEqual({}); + expect(res).toBeNull(); }); it('returns empty for unsupported error', async () => { const http = partial({ get: () => - Promise.reject( - Object.assign(new Error(), { name: 'UnsupportedProtocolError' }), - ), + Promise.reject(httpError({ name: 'UnsupportedProtocolError' })), }); const res = await downloadHttpProtocol(http, 'some://'); - expect(res).toStrictEqual({}); + expect(res).toBeNull(); }); }); diff --git a/lib/modules/datasource/maven/util.ts b/lib/modules/datasource/maven/util.ts index 1ddc2eaf4abe36..b521b99d53ab1d 100644 --- a/lib/modules/datasource/maven/util.ts +++ b/lib/modules/datasource/maven/util.ts @@ -5,9 +5,10 @@ import { XmlDocument } from 'xmldoc'; import { HOST_DISABLED } from '../../../constants/error-messages'; import { logger } from '../../../logger'; import { ExternalHostError } from '../../../types/errors/external-host-error'; -import type { Http } from '../../../util/http'; +import { type Http, HttpError } from '../../../util/http'; import type { HttpOptions, HttpResponse } from '../../../util/http/types'; import { regEx } from '../../../util/regex'; +import { Result } from '../../../util/result'; import type { S3UrlParts } from '../../../util/s3'; import { getS3Client, parseS3Url } from '../../../util/s3'; import { streamToString } from '../../../util/streams'; @@ -26,32 +27,33 @@ function getHost(url: string): string | null { return parseUrl(url)?.host ?? /* istanbul ignore next: not possible */ null; } -function isMavenCentral(pkgUrl: URL | string): boolean { - const host = typeof pkgUrl === 'string' ? pkgUrl : pkgUrl.host; - return getHost(MAVEN_REPO) === host; -} +function isTemporaryError(err: HttpError): boolean { + if (err.code === 'ECONNRESET') { + return true; + } -function isTemporalError(err: { code: string; statusCode: number }): boolean { - return ( - err.code === 'ECONNRESET' || - err.statusCode === 429 || - (err.statusCode >= 500 && err.statusCode < 600) - ); + if (err.response) { + const status = err.response.statusCode; + return status === 429 || (status >= 500 && status < 600); + } + + return false; } -function isHostError(err: { code: string }): boolean { +function isHostError(err: HttpError): boolean { return err.code === 'ETIMEDOUT'; } -function isNotFoundError(err: { code: string; statusCode: number }): boolean { - return err.code === 'ENOTFOUND' || err.statusCode === 404; +function isNotFoundError(err: HttpError): boolean { + return err.code === 'ENOTFOUND' || err.response?.statusCode === 404; } -function isPermissionsIssue(err: { statusCode: number }): boolean { - return err.statusCode === 401 || err.statusCode === 403; +function isPermissionsIssue(err: HttpError): boolean { + const status = err.response?.statusCode; + return status === 401 || status === 403; } -function isConnectionError(err: { code: string }): boolean { +function isConnectionError(err: HttpError): boolean { return ( err.code === 'EAI_AGAIN' || err.code === 'ERR_TLS_CERT_ALTNAME_INVALID' || @@ -59,7 +61,7 @@ function isConnectionError(err: { code: string }): boolean { ); } -function isUnsupportedHostError(err: { name: string }): boolean { +function isUnsupportedHostError(err: HttpError): boolean { return err.name === 'UnsupportedProtocolError'; } @@ -67,37 +69,73 @@ export async function downloadHttpProtocol( http: Http, pkgUrl: URL | string, opts: HttpOptions = {}, -): Promise> { - let raw: HttpResponse; - try { - raw = await http.get(pkgUrl.toString(), opts); - return raw; - } catch (err) { - const failedUrl = pkgUrl.toString(); - if (err.message === HOST_DISABLED) { - logger.trace({ failedUrl }, 'Host disabled'); - } else if (isNotFoundError(err)) { - logger.trace({ failedUrl }, `Url not found`); - } else if (isHostError(err)) { - logger.debug(`Cannot connect to host ${failedUrl}`); - } else if (isPermissionsIssue(err)) { - logger.debug( - `Dependency lookup unauthorized. Please add authentication with a hostRule for ${failedUrl}`, - ); - } else if (isTemporalError(err)) { - logger.debug({ failedUrl, err }, 'Temporary error'); - if (isMavenCentral(pkgUrl)) { - throw new ExternalHostError(err); +): Promise { + const url = pkgUrl.toString(); + const res = await Result.wrap(http.get(url, opts)) + .onError((err) => { + // istanbul ignore next: never happens, needs for type narrowing + if (!(err instanceof HttpError)) { + return; } - } else if (isConnectionError(err)) { - logger.debug(`Connection refused to maven registry ${failedUrl}`); - } else if (isUnsupportedHostError(err)) { - logger.debug(`Unsupported host ${failedUrl} `); - } else { + + const failedUrl = url; + if (err.message === HOST_DISABLED) { + logger.trace({ failedUrl }, 'Host disabled'); + return; + } + + if (isNotFoundError(err)) { + logger.trace({ failedUrl }, `Url not found`); + return; + } + + if (isHostError(err)) { + logger.debug(`Cannot connect to host ${failedUrl}`); + return; + } + + if (isPermissionsIssue(err)) { + logger.debug( + `Dependency lookup unauthorized. Please add authentication with a hostRule for ${failedUrl}`, + ); + return; + } + + if (isTemporaryError(err)) { + logger.debug({ failedUrl, err }, 'Temporary error'); + return; + } + + if (isConnectionError(err)) { + logger.debug(`Connection refused to maven registry ${failedUrl}`); + return; + } + + if (isUnsupportedHostError(err)) { + logger.debug(`Unsupported host ${failedUrl}`); + return; + } + logger.info({ failedUrl, err }, 'Unknown HTTP download error'); - } - return {}; + }) + .catch((err): Result => { + if ( + err instanceof HttpError && + isTemporaryError(err) && + getHost(url) === getHost(MAVEN_REPO) + ) { + return Result.err(new ExternalHostError(err)); + } + + return Result.ok('silent-error'); + }) + .unwrapOrThrow(); + + if (res === 'silent-error') { + return null; } + + return res; } function isS3NotFound(err: Error): boolean { @@ -145,7 +183,7 @@ export async function downloadS3Protocol(pkgUrl: URL): Promise { export async function downloadArtifactRegistryProtocol( http: Http, pkgUrl: URL, -): Promise> { +): Promise { const opts: HttpOptions = {}; const host = pkgUrl.host; const path = pkgUrl.pathname; @@ -275,48 +313,42 @@ export async function downloadMavenXml( return {}; } - let isCacheable = false; - - let rawContent: string | undefined; - let authorization: boolean | undefined; - let statusCode: number | undefined; - switch (pkgUrl.protocol) { - case 'http:': - case 'https:': - ({ - authorization, - body: rawContent, - statusCode, - } = await downloadHttpProtocol(http, pkgUrl)); - break; - case 's3:': - rawContent = (await downloadS3Protocol(pkgUrl)) ?? undefined; - break; - case 'artifactregistry:': - ({ - authorization, - body: rawContent, - statusCode, - } = await downloadArtifactRegistryProtocol(http, pkgUrl)); - break; - default: - logger.debug(`Unsupported Maven protocol url:${pkgUrl.toString()}`); - return {}; + const protocol = pkgUrl.protocol; + + if (protocol === 'http:' || protocol === 'https:') { + const res = await downloadHttpProtocol(http, pkgUrl); + const body = res?.body; + if (body) { + return { + xml: new XmlDocument(body), + isCacheable: !res.authorization, + }; + } } - if (!rawContent) { - logger.debug( - { url: pkgUrl.toString(), statusCode }, - `Content is not found for Maven url`, - ); - return {}; + if (protocol === 'artifactregistry:') { + const res = await downloadArtifactRegistryProtocol(http, pkgUrl); + const body = res?.body; + if (body) { + return { + xml: new XmlDocument(body), + isCacheable: !res.authorization, + }; + } } - if (!authorization) { - isCacheable = true; + if (protocol === 's3:') { + const res = await downloadS3Protocol(pkgUrl); + if (res) { + return { xml: new XmlDocument(res) }; + } } - return { isCacheable, xml: new XmlDocument(rawContent) }; + logger.debug( + { url: pkgUrl.toString() }, + `Content is not found for Maven url`, + ); + return {}; } export function getDependencyParts(packageName: string): MavenDependency { diff --git a/lib/modules/datasource/sbt-package/index.ts b/lib/modules/datasource/sbt-package/index.ts index afbb629cc5df84..c0d5403b63055f 100644 --- a/lib/modules/datasource/sbt-package/index.ts +++ b/lib/modules/datasource/sbt-package/index.ts @@ -39,10 +39,8 @@ export class SbtPackageDatasource extends MavenDatasource { scalaVersion: string, ): Promise { const pkgUrl = ensureTrailingSlash(searchRoot); - const { body: indexContent } = await downloadHttpProtocol( - this.http, - pkgUrl, - ); + const res = await downloadHttpProtocol(this.http, pkgUrl); + const indexContent = res?.body; if (indexContent) { const rootPath = new URL(pkgUrl).pathname; let artifactSubdirs = extractPageLinks(indexContent, (href) => { @@ -81,7 +79,8 @@ export class SbtPackageDatasource extends MavenDatasource { const releases: string[] = []; for (const searchSubdir of artifactSubdirs) { const pkgUrl = ensureTrailingSlash(`${searchRoot}/${searchSubdir}`); - const { body: content } = await downloadHttpProtocol(this.http, pkgUrl); + const res = await downloadHttpProtocol(this.http, pkgUrl); + const content = res?.body; if (content) { const rootPath = new URL(pkgUrl).pathname; const subdirReleases = extractPageLinks(content, (href) => { @@ -128,8 +127,8 @@ export class SbtPackageDatasource extends MavenDatasource { for (const pomFileName of pomFileNames) { const pomUrl = `${searchRoot}/${artifactDir}/${version}/${pomFileName}`; - const { body: content } = await downloadHttpProtocol(this.http, pomUrl); - + const res = await downloadHttpProtocol(this.http, pomUrl); + const content = res?.body; if (content) { const pomXml = new XmlDocument(content); diff --git a/lib/modules/datasource/sbt-plugin/index.ts b/lib/modules/datasource/sbt-plugin/index.ts index e316d1e22031c3..e053811936ec68 100644 --- a/lib/modules/datasource/sbt-plugin/index.ts +++ b/lib/modules/datasource/sbt-plugin/index.ts @@ -50,13 +50,13 @@ export class SbtPluginDatasource extends SbtPackageDatasource { return href; }; - const { body: indexContent } = await downloadHttpProtocol( + const res = await downloadHttpProtocol( this.http, ensureTrailingSlash(searchRoot), ); - if (indexContent) { + if (res) { const releases: string[] = []; - const scalaVersionItems = extractPageLinks(indexContent, hrefFilterMap); + const scalaVersionItems = extractPageLinks(res.body, hrefFilterMap); const scalaVersions = scalaVersionItems.map((x) => x.replace(regEx(/^scala_/), ''), ); @@ -65,22 +65,24 @@ export class SbtPluginDatasource extends SbtPackageDatasource { : scalaVersions; for (const searchVersion of searchVersions) { const searchSubRoot = `${searchRoot}/scala_${searchVersion}`; - const { body: subRootContent } = await downloadHttpProtocol( + const subRootRes = await downloadHttpProtocol( this.http, ensureTrailingSlash(searchSubRoot), ); - if (subRootContent) { + if (subRootRes) { + const { body: subRootContent } = subRootRes; const sbtVersionItems = extractPageLinks( subRootContent, hrefFilterMap, ); for (const sbtItem of sbtVersionItems) { const releasesRoot = `${searchSubRoot}/${sbtItem}`; - const { body: releasesIndexContent } = await downloadHttpProtocol( + const releaseIndexRes = await downloadHttpProtocol( this.http, ensureTrailingSlash(releasesRoot), ); - if (releasesIndexContent) { + if (releaseIndexRes) { + const { body: releasesIndexContent } = releaseIndexRes; const releasesParsed = extractPageLinks( releasesIndexContent, hrefFilterMap,