feat(maven): Remove unnecessary HTML page fetches

lib/modules/datasource/maven/index.spec.ts

@@ -38,15 +38,13 @@ interface MockOpts {
   pom?: string | null;
   latest?: string;
   snapshots?: SnapshotOpts[] | null;
-  html?: string | null;
 }
 
 function mockGenericPackage(opts: MockOpts = {}) {
   const {
     dep = 'org.example:package',
     base = baseUrl,
     latest = '2.0.0',
-    html,
     snapshots,
   } = opts;
   const meta =
@@ -62,12 +60,6 @@ function mockGenericPackage(opts: MockOpts = {}) {
     scope.get(`/${packagePath}/maven-metadata.xml`).reply(200, meta);
   }
 
-  if (html) {
-    scope.get(`/${packagePath}/`).reply(200, html);
-  } else if (html === null) {
-    scope.get(`/${packagePath}/`).reply(404);
-  }
-
   if (pom) {
     if (latest.endsWith('-SNAPSHOT')) {
       const [major, minor, patch] = latest
@@ -129,8 +121,6 @@ describe('modules/datasource/maven/index', () => {
   it('returns null when metadata is not found', async () => {
     httpMock
       .scope(baseUrl)
-      .get('/org/example/package/')
-      .reply(404)
       .get('/org/example/package/maven-metadata.xml')
       .reply(404);
 
@@ -140,7 +130,7 @@ describe('modules/datasource/maven/index', () => {
   });
 
   it('returns releases', async () => {
-    mockGenericPackage({ html: null });
+    mockGenericPackage();
 
     const res = await get();
 
@@ -151,7 +141,6 @@ describe('modules/datasource/maven/index', () => {
     const meta = Fixtures.get('metadata-snapshot-version.xml');
     mockGenericPackage({
       meta: Fixtures.get('metadata-snapshot-only.xml'),
-      html: null,
       latest: '1.0.3-SNAPSHOT',
       snapshots: [
         {
@@ -184,7 +173,6 @@ describe('modules/datasource/maven/index', () => {
     mockGenericPackage({
       meta: Fixtures.get('metadata-snapshot-only.xml'),
       pom: null,
-      html: null,
       latest: '1.0.3-SNAPSHOT',
       snapshots: [
         {
@@ -206,32 +194,6 @@ describe('modules/datasource/maven/index', () => {
     });
   });
 
-  it('returns html-based releases', async () => {
-    mockGenericPackage({
-      latest: '2.0.0',
-      html: Fixtures.get('index.html'),
-      meta: Fixtures.get('index.xml'),
-      snapshots: null,
-    });
-
-    const res = await get();
-
-    expect(res).toEqual({
-      display: 'org.example:package',
-      group: 'org.example',
-      homepage: 'https://package.example.org/about',
-      name: 'package',
-      packageScope: 'org.example',
-      registryUrl: 'https://repo.maven.apache.org/maven2',
-      releases: [
-        { version: '1.0.0', releaseTimestamp: '2021-02-22T14:43:00.000Z' },
-        { version: '1.0.1', releaseTimestamp: '2021-04-12T15:51:00.000Z' },
-        { version: '1.0.2', releaseTimestamp: '2021-06-16T12:47:00.000Z' },
-        { version: '2.0.0', releaseTimestamp: '2021-06-18T16:24:00.000Z' },
-      ],
-    });
-  });
-
   it('returns releases from custom repository', async () => {
     mockGenericPackage({ base: baseUrlCustom });
 
@@ -241,7 +203,7 @@ describe('modules/datasource/maven/index', () => {
   });
 
   it('falls back to next registry url', async () => {
-    mockGenericPackage({ html: null });
+    mockGenericPackage();
     httpMock
       .scope('https://failed_repo')
       .get('/org/example/package/maven-metadata.xml')
@@ -294,7 +256,7 @@ describe('modules/datasource/maven/index', () => {
   });
 
   it('skips registry with invalid metadata structure', async () => {
-    mockGenericPackage({ html: null });
+    mockGenericPackage();
     httpMock
       .scope('https://invalid_metadata_repo')
       .get('/org/example/package/maven-metadata.xml')
@@ -310,7 +272,7 @@ describe('modules/datasource/maven/index', () => {
   });
 
   it('skips registry with invalid XML', async () => {
-    mockGenericPackage({ html: null });
+    mockGenericPackage();
     httpMock
       .scope('https://invalid_metadata_repo')
       .get('/org/example/package/maven-metadata.xml')
@@ -326,9 +288,9 @@ describe('modules/datasource/maven/index', () => {
   });
 
   it('handles optional slash at the end of registry url', async () => {
-    mockGenericPackage({ html: null });
+    mockGenericPackage();
     const resA = await get('org.example:package', baseUrl.replace(/\/+$/, ''));
-    mockGenericPackage({ html: null });
+    mockGenericPackage();
     const resB = await get('org.example:package', baseUrl.replace(/\/*$/, '/'));
     expect(resA).not.toBeNull();
     expect(resB).not.toBeNull();
@@ -346,7 +308,7 @@ describe('modules/datasource/maven/index', () => {
 
   it('supports scm.url values prefixed with "scm:"', async () => {
     const pom = Fixtures.get('pom.scm-prefix.xml');
-    mockGenericPackage({ pom, html: null });
+    mockGenericPackage({ pom });
 
     const res = await get();
 
@@ -504,7 +466,6 @@ describe('modules/datasource/maven/index', () => {
         meta: Fixtures.get('child-no-info/meta.xml'),
         pom: Fixtures.get('child-no-info/pom.xml'),
         latest: '2.0.0',
-        html: null,
       });
       mockGenericPackage(parentPackage);
 
@@ -521,7 +482,6 @@ describe('modules/datasource/maven/index', () => {
         meta: Fixtures.get('child-empty/meta.xml'),
         pom: Fixtures.get('child-empty/pom.xml'),
         latest: '2.0.0',
-        html: null,
       });
 
       const res = await get();
@@ -556,7 +516,6 @@ describe('modules/datasource/maven/index', () => {
       mockGenericPackage({
         ...childPomMock,
         meta: childMeta,
-        html: null,
       });
       mockGenericPackage(parentPomMock);
       mockGenericPackage(childPomMock);
@@ -576,7 +535,6 @@ describe('modules/datasource/maven/index', () => {
         meta: Fixtures.get('child-scm/meta.xml'),
         pom: Fixtures.get('child-scm/pom.xml'),
         latest: '2.0.0',
-        html: null,
       });
       mockGenericPackage(parentPackage);
 
@@ -593,7 +551,6 @@ describe('modules/datasource/maven/index', () => {
         meta: Fixtures.get('child-url/meta.xml'),
         pom: Fixtures.get('child-url/pom.xml'),
         latest: '2.0.0',
-        html: null,
       });
       mockGenericPackage(parentPackage);
 
@@ -610,7 +567,6 @@ describe('modules/datasource/maven/index', () => {
         meta: Fixtures.get('child-all-info/meta.xml'),
         pom: Fixtures.get('child-all-info/pom.xml'),
         latest: '2.0.0',
-        html: null,
       });
 
       const res = await get();
@@ -626,7 +582,6 @@ describe('modules/datasource/maven/index', () => {
         meta: Fixtures.get('child-scm-gitatcolon/meta.xml'),
         pom: Fixtures.get('child-scm-gitatcolon/pom.xml'),
         latest: '2.0.0',
-        html: null,
       });
 
       const res = await get();
@@ -641,7 +596,6 @@ describe('modules/datasource/maven/index', () => {
         meta: Fixtures.get('child-scm-gitatslash/meta.xml'),
         pom: Fixtures.get('child-scm-gitatslash/pom.xml'),
         latest: '2.0.0',
-        html: null,
       });
 
       const res = await get();
@@ -656,7 +610,6 @@ describe('modules/datasource/maven/index', () => {
         meta: Fixtures.get('child-scm-gitprotocol/meta.xml'),
         pom: Fixtures.get('child-scm-gitprotocol/pom.xml'),
         latest: '2.0.0',
-        html: null,
       });
 
       const res = await get();

lib/modules/datasource/maven/index.ts

@@ -1,11 +1,9 @@
 import is from '@sindresorhus/is';
-import { DateTime } from 'luxon';
 import type { XmlDocument } from 'xmldoc';
 import { GlobalConfig } from '../../../config/global';
 import { logger } from '../../../logger';
 import * as packageCache from '../../../util/cache/package';
 import { cache } from '../../../util/cache/package/decorator';
-import { newlineRegex, regEx } from '../../../util/regex';
 import { ensureTrailingSlash } from '../../../util/url';
 import mavenVersion from '../../versioning/maven';
 import * as mavenVersioning from '../../versioning/maven';
@@ -24,7 +22,6 @@ import type { MavenDependency, ReleaseMap } from './types';
 import {
   checkResource,
   createUrlForDependencyPom,
-  downloadHttpProtocol,
   downloadMavenXml,
   getDependencyInfo,
   getDependencyParts,
@@ -55,11 +52,6 @@ function extractVersions(metadata: XmlDocument): string[] {
   return elements.map((el) => el.val);
 }
 
-const mavenCentralHtmlVersionRegex = regEx(
-  '^<a href="(?<version>[^"]+)/" title="(?:[^"]+)/">(?:[^"]+)/</a>\\s+(?<releaseTimestamp>\\d\\d\\d\\d-\\d\\d-\\d\\d \\d\\d:\\d\\d)\\s+-$',
-  'i',
-);
-
 export const defaultRegistryUrls = [MAVEN_REPO];
 
 export class MavenDatasource extends Datasource {
@@ -124,72 +116,9 @@ export class MavenDatasource extends Datasource {
     return releaseMap;
   }
 
-  async addReleasesFromIndexPage(
-    inputReleaseMap: ReleaseMap,
-    dependency: MavenDependency,
-    repoUrl: string,
-  ): Promise<ReleaseMap> {
-    if (!repoUrl.startsWith(MAVEN_REPO)) {
-      return inputReleaseMap;
-    }
-
-    const cacheNs = 'datasource-maven:index-html-releases';
-    const cacheKey = `${repoUrl}${dependency.dependencyUrl}`;
-    let workingReleaseMap = await packageCache.get<ReleaseMap>(
-      cacheNs,
-      cacheKey,
-    );
-    if (!workingReleaseMap) {
-      workingReleaseMap = {};
-      let retryEarlier = false;
-      try {
-        const indexUrl = getMavenUrl(dependency, repoUrl, '');
-        const res = await downloadHttpProtocol(this.http, indexUrl);
-        if (res) {
-          for (const line of res.body.split(newlineRegex)) {
-            const match = line.trim().match(mavenCentralHtmlVersionRegex);
-            if (match) {
-              const { version, releaseTimestamp: timestamp } =
-                match?.groups ?? /* istanbul ignore next: hard to test */ {};
-              if (version && timestamp) {
-                const date = DateTime.fromFormat(
-                  timestamp,
-                  'yyyy-MM-dd HH:mm',
-                  {
-                    zone: 'UTC',
-                  },
-                );
-                if (date.isValid) {
-                  const releaseTimestamp = date.toISO();
-                  workingReleaseMap[version] = { version, releaseTimestamp };
-                }
-              }
-            }
-          }
-        }
-      } catch (err) /* istanbul ignore next */ {
-        retryEarlier = true;
-        logger.debug(
-          { dependency, err },
-          'Failed to get releases from package index page',
-        );
-      }
-      const cacheTTL = retryEarlier
-        ? /* istanbul ignore next: hard to test */ 60
-        : 24 * 60;
-      await packageCache.set(cacheNs, cacheKey, workingReleaseMap, cacheTTL);
-    }
-
-    const releaseMap = { ...inputReleaseMap };
-    for (const version of Object.keys(releaseMap)) {
-      releaseMap[version] ||= workingReleaseMap[version] ?? null;
-    }
-
-    return releaseMap;
-  }
-
   getReleasesFromMap(releaseMap: ReleaseMap): Release[] {
     const releases = Object.values(releaseMap).filter(is.truthy);
+    // istanbul ignore if: will be removed
     if (releases.length) {
       return releases;
     }
@@ -210,9 +139,7 @@ export class MavenDatasource extends Datasource {
 
     logger.debug(`Looking up ${dependency.display} in repository ${repoUrl}`);
 
-    let releaseMap = await this.fetchReleasesFromMetadata(dependency, repoUrl);
-    releaseMap = await this.addReleasesFromIndexPage(
-      releaseMap,
+    const releaseMap = await this.fetchReleasesFromMetadata(
       dependency,
       repoUrl,
     );

lib/modules/datasource/sbt-package/index.spec.ts

@@ -70,8 +70,6 @@ describe('modules/datasource/sbt-package/index', () => {
             <a href="empty_but_invalid/">???</a>
           `,
         )
-        .get('/maven2/com/example/empty/')
-        .reply(200, '')
         .get('/maven2/com/example/empty_but_invalid/')
         .reply(404, '')
         .get('/maven2/com/example/empty/maven-metadata.xml')

lib/util/cache/package/types.ts

@@ -78,7 +78,6 @@ export type PackageCacheNamespace =
   | 'datasource-maven'
   | 'datasource-maven:head-requests-timeout'
   | 'datasource-maven:head-requests'
-  | 'datasource-maven:index-html-releases'
   | 'datasource-maven:metadata-xml'
   | 'datasource-node-version'
   | 'datasource-npm:data'