Merge branch 'main' into implement-terraform-lock-files

docs/usage/private-modules.md

@@ -38,7 +38,7 @@ Although npm presets were the first type supported, they are now deprecated and
 However if you do still use them, private modules should work if you configure the `npmrc` file including token credentials in your bot admin config.
 Credentials stored on disk (e.g. in `~/.npmrc`) are no longer supported.
 
-The recommended way of using local presets is to configure then using "local" presets, e.g. like `"extends": ["local>myorg/renovate-config"]`, and ensure that the platform token has access to that repo.
+The recommended way of using local presets is to configure then using "local" presets, e.g. `"extends": ["local>myorg/renovate-config"]`, and ensure that the platform token has access to that repo.
 
 It's not recommended that you use a private repository to host your config while then extending it from a public repository.
 If your preset doesn't contain secrets then you should make it public, while if it does contain secrets then it's better to split your preset between a public one which all repos extend, and a private one with secrets which only other private repos extend.
@@ -52,7 +52,7 @@ In summary, the recommended approach to private presets is:
 ## Dependency Version Lookups
 
 Whenever Renovate detects that a project uses a particular dependency, it attempts to look up that dependency to see if any new versions exist.
-If such a package is private, then Renovate will need to be configured with the relevant credentials.
+If such a package is private, then Renovate must be configured with the relevant credentials.
 Renovate does not use any package managers for this step and performs all HTTP(S) lookups itself, including insertion of authentication headers.
 
 Configuring Renovate with credentials requires `hostRules`.
@@ -64,7 +64,7 @@ It can be either a "platform" name (e.g. `github`, `azure`, etc) or a "datasourc
 `baseUrl` can be used if you want to only apply the credentials for a nested path within the host, e.g. `https://registry.company.com/nested/path/`.
 If the same credentials apply to all paths on a host, then use `hostName` instead, e.g. `registry.company.com`.
 Finally, to apply credentials to all hosts within the domain, use `domainName`, e.g. `company.com`.
-You need to pick only one of these and not try to use multiple at the same time, or it will be a config error.
+You need to pick _only one_ of these and not configure more than one of these fields within the same host rule, otherwise it will error.
 
 In addition to the above options to match against a host, you need to add the credentials.
 Typically they are either `token`, or `username` + `password`.
@@ -91,7 +91,7 @@ Here is an example of some host rules:
 Renovate applies theses `hostRules` to every HTTP(s) request which is sent, so they are largely independent of any platform or datasource logic.
 With `hostRules` in place, private package lookups should all work.
 
-## Release Notes
+## Looking up Release Notes
 
 When Renovate creates Pull Requests, its default behavior is to locate and embed release notes/changelogs of packages.
 These release notes are fetched from the source repository of packages and not from the registries themselves, so if they are private then they will require different credentials.
@@ -102,7 +102,7 @@ It can be confusing for people who host their own source code privately to be as
 
 Currently the preferred way to configure `github.com` credentials for self-hosted Renovate is:
 
-- Create a read-only Personal Access Token (PAT) for a `github.com` account. It can be any account, and may even be best to be an empty account created just for this purpose.
+- Create a read-only Personal Access Token (PAT) for a `github.com` account. This can be any GitHub account, it might be better to create a "empty" account just for this purpose.
 - Add the PAT to Renovate using the environment variable `GITHUB_COM_TOKEN`
 
 ## Package Manager Credentials for Artifact Updating
@@ -155,7 +155,7 @@ If you are a user of this app, and have private modules, then the following is a
 ### Private presets with public repositories
 
 If you have a preset in a private repo but reference ("extend") it from a public repository then it won't work.
-This is because public repositories are provided with a token scoped to only that particular repository, and not for all repositories within the org.
+This is because public repositories are provided with a token scoped to only that particular repository, and not for all repositories within the organization.
 This is a security measure so that if a the token is accidentally leaked publicly, the damage is limited to the public repository it leaked to and not to every repository within the organization.
 
 The solution to this is that you should break your presets into public and private ones, and reference only the public ones from public repositories.
@@ -199,15 +199,16 @@ The WhiteSource Renovate App does not run using GitHub Actions, but such secrets
 
 ## Admin/Bot config vs User/Repository config for Self-hosted users
 
-"AdminBot config" refers to the config which the Renovate Bot administrator provides at bot startup, e.g. using environment variables, CLI parameters, or the `config.js` configuration file. User/Repository config refers to the in-repository config file which defaults to `renovate.json` but has a large number of alternative filenames supported.
+"AdminBot config" refers to the config which the Renovate Bot administrator provides at bot startup, e.g. using environment variables, CLI parameters, or the `config.js` configuration file.
+User/Repository config refers to the in-repository config file which defaults to `renovate.json` but has a large number of alternative filenames supported.
 
 If there is a need to supply custom rules for certain repository, it can still be done using the `config.js` file and the `repositories` array.
 
 If per-repository config must be done within the repository, it is still recommended against committing secrets directly (including e.g. `.npmrc` files with tokens) and instead encrypting them with a custom public key first.
 For instructions on this, see the above section on encrypting secrets for the WhiteSource Renovate App but instead:
 
-- Save a copy of the <https://app.renovatebot.com/encrypt> html file locally, or host it locally
+- Save a copy of the <https://app.renovatebot.com/encrypt> HTML file locally, or host it locally
 - Generate a public/private key pair for the app using the instructions in [privateKey](https://docs.renovatebot.com/self-hosted-configuration/#privatekey)
-- Replace the existing public key in the html with the public key you generated in the step prior
-- Use the resulting html encrypt page to encrypt secrets for your app before adding them to user/repository config
+- Replace the existing public key in the HTML with the public key you generated in the step prior
+- Use the resulting HTML encrypt page to encrypt secrets for your app before adding them to user/repository config
 - Configure the app to run with `privateKey` set to the private key you generated above

lib/datasource/maven/index.ts

@@ -1,5 +1,4 @@
 import url from 'url';
-import fs from 'fs-extra';
 import pAll from 'p-all';
 import { XmlDocument } from 'xmldoc';
 import { logger } from '../../logger';
@@ -9,7 +8,14 @@ import * as mavenVersioning from '../../versioning/maven';
 import { compare } from '../../versioning/maven/compare';
 import type { GetReleasesConfig, Release, ReleaseResult } from '../types';
 import { MAVEN_REPO } from './common';
-import { downloadHttpProtocol, isHttpResourceExists } from './util';
+import type { MavenDependency } from './types';
+import {
+  downloadMavenXml,
+  getDependencyInfo,
+  getDependencyParts,
+  getMavenUrl,
+  isHttpResourceExists,
+} from './util';
 
 export { id } from './common';
 
@@ -18,93 +24,6 @@ export const defaultRegistryUrls = [MAVEN_REPO];
 export const defaultVersioning = mavenVersioning.id;
 export const registryStrategy = 'merge';
 
-function containsPlaceholder(str: string): boolean {
-  return /\${.*?}/g.test(str);
-}
-
-async function downloadFileProtocol(pkgUrl: url.URL): Promise<string | null> {
-  const pkgPath = pkgUrl.toString().replace('file://', '');
-  if (!(await fs.exists(pkgPath))) {
-    return null;
-  }
-  return fs.readFile(pkgPath, 'utf8');
-}
-
-function getMavenUrl(
-  dependency: MavenDependency,
-  repoUrl: string,
-  path: string
-): url.URL | null {
-  return new url.URL(`${dependency.dependencyUrl}/${path}`, repoUrl);
-}
-
-interface MavenXml {
-  authorization?: boolean;
-  xml?: XmlDocument;
-}
-
-async function downloadMavenXml(
-  pkgUrl: url.URL | null
-): Promise<MavenXml | null> {
-  /* istanbul ignore if */
-  if (!pkgUrl) {
-    return {};
-  }
-  let rawContent: string;
-  let authorization: boolean;
-  switch (pkgUrl.protocol) {
-    case 'file:':
-      rawContent = await downloadFileProtocol(pkgUrl);
-      break;
-    case 'http:':
-    case 'https:':
-      ({ authorization, body: rawContent } = await downloadHttpProtocol(
-        pkgUrl
-      ));
-      break;
-    case 's3:':
-      logger.debug('Skipping s3 dependency');
-      return {};
-    default:
-      logger.debug({ url: pkgUrl.toString() }, `Unsupported Maven protocol`);
-      return {};
-  }
-
-  if (!rawContent) {
-    logger.debug(`Content is not found for Maven url: ${pkgUrl.toString()}`);
-    return {};
-  }
-
-  return { authorization, xml: new XmlDocument(rawContent) };
-}
-
-async function getDependencyInfo(
-  dependency: MavenDependency,
-  repoUrl: string,
-  version: string
-): Promise<Partial<ReleaseResult>> {
-  const result: Partial<ReleaseResult> = {};
-  const path = `${version}/${dependency.name}-${version}.pom`;
-
-  const pomUrl = getMavenUrl(dependency, repoUrl, path);
-  const { xml: pomContent } = await downloadMavenXml(pomUrl);
-  if (!pomContent) {
-    return result;
-  }
-
-  const homepage = pomContent.valueWithPath('url');
-  if (homepage && !containsPlaceholder(homepage)) {
-    result.homepage = homepage;
-  }
-
-  const sourceUrl = pomContent.valueWithPath('scm.url');
-  if (sourceUrl && !containsPlaceholder(sourceUrl)) {
-    result.sourceUrl = sourceUrl.replace(/^scm:/, '');
-  }
-
-  return result;
-}
-
 function isStableVersion(x: string): boolean {
   return mavenVersion.isStable(x);
 }
@@ -121,24 +40,6 @@ function getLatestStableVersion(releases: Release[]): string | null {
   return null;
 }
 
-interface MavenDependency {
-  display: string;
-  group?: string;
-  name?: string;
-  dependencyUrl: string;
-}
-
-function getDependencyParts(lookupName: string): MavenDependency {
-  const [group, name] = lookupName.split(':');
-  const dependencyUrl = `${group.replace(/\./g, '/')}/${name}`;
-  return {
-    display: lookupName,
-    group,
-    name,
-    dependencyUrl,
-  };
-}
-
 function extractVersions(metadata: XmlDocument): string[] {
   const versions = metadata.descendantWithPath('versioning.versions');
   const elements = versions?.childrenNamed('version');

lib/datasource/maven/types.ts

@@ -0,0 +1,13 @@
+import type { XmlDocument } from 'xmldoc';
+
+export interface MavenDependency {
+  display: string;
+  group?: string;
+  name?: string;
+  dependencyUrl: string;
+}
+
+export interface MavenXml {
+  authorization?: boolean;
+  xml?: XmlDocument;
+}

lib/datasource/maven/util.ts

@@ -1,10 +1,14 @@
 import url from 'url';
+import fs from 'fs-extra';
+import { XmlDocument } from 'xmldoc';
 import { HOST_DISABLED } from '../../constants/error-messages';
 import { logger } from '../../logger';
 import { ExternalHostError } from '../../types/errors/external-host-error';
 import { Http, HttpResponse } from '../../util/http';
 
+import type { ReleaseResult } from '../types';
 import { MAVEN_REPO, id } from './common';
+import type { MavenDependency, MavenXml } from './types';
 
 const http: Record<string, Http> = {};
 
@@ -96,6 +100,14 @@ export async function downloadHttpProtocol(
   }
 }
 
+async function downloadFileProtocol(pkgUrl: url.URL): Promise<string | null> {
+  const pkgPath = pkgUrl.toString().replace('file://', '');
+  if (!(await fs.exists(pkgPath))) {
+    return null;
+  }
+  return fs.readFile(pkgPath, 'utf8');
+}
+
 export async function isHttpResourceExists(
   pkgUrl: url.URL | string,
   hostType = id
@@ -119,3 +131,88 @@ export async function isHttpResourceExists(
     return null;
   }
 }
+
+function containsPlaceholder(str: string): boolean {
+  return /\${.*?}/g.test(str);
+}
+
+export function getMavenUrl(
+  dependency: MavenDependency,
+  repoUrl: string,
+  path: string
+): url.URL | null {
+  return new url.URL(`${dependency.dependencyUrl}/${path}`, repoUrl);
+}
+
+export async function downloadMavenXml(
+  pkgUrl: url.URL | null
+): Promise<MavenXml | null> {
+  /* istanbul ignore if */
+  if (!pkgUrl) {
+    return {};
+  }
+  let rawContent: string;
+  let authorization: boolean;
+  switch (pkgUrl.protocol) {
+    case 'file:':
+      rawContent = await downloadFileProtocol(pkgUrl);
+      break;
+    case 'http:':
+    case 'https:':
+      ({ authorization, body: rawContent } = await downloadHttpProtocol(
+        pkgUrl
+      ));
+      break;
+    case 's3:':
+      logger.debug('Skipping s3 dependency');
+      return {};
+    default:
+      logger.debug({ url: pkgUrl.toString() }, `Unsupported Maven protocol`);
+      return {};
+  }
+
+  if (!rawContent) {
+    logger.debug(`Content is not found for Maven url: ${pkgUrl.toString()}`);
+    return {};
+  }
+
+  return { authorization, xml: new XmlDocument(rawContent) };
+}
+
+export async function getDependencyInfo(
+  dependency: MavenDependency,
+  repoUrl: string,
+  version: string
+): Promise<Partial<ReleaseResult>> {
+  const result: Partial<ReleaseResult> = {};
+  const path = `${version}/${dependency.name}-${version}.pom`;
+
+  const pomUrl = getMavenUrl(dependency, repoUrl, path);
+  const { xml: pomContent } = await downloadMavenXml(pomUrl);
+  if (!pomContent) {
+    return result;
+  }
+
+  const homepage = pomContent.valueWithPath('url');
+  if (homepage && !containsPlaceholder(homepage)) {
+    result.homepage = homepage;
+  }
+
+  const sourceUrl = pomContent.valueWithPath('scm.url');
+  if (sourceUrl && !containsPlaceholder(sourceUrl)) {
+    result.sourceUrl = sourceUrl.replace(/^scm:/, '');
+  }
+
+  return result;
+}
+
+export function getDependencyParts(lookupName: string): MavenDependency {
+  const [group, name] = lookupName.split(':');
+  const dependencyUrl = `${group.replace(/\./g, '/')}/${name}`;
+  return {
+    display: lookupName,
+    group,
+    name,
+    dependencyUrl,
+  };
+}

lib/util/git/index.ts

@@ -450,13 +450,19 @@ export function getBranchList(): string[] {
 export async function isBranchStale(branchName: string): Promise<boolean> {
   await syncBranch(branchName);
   try {
+    const { currentBranchSha, currentBranch } = config;
     const branches = await git.branch([
       '--remotes',
       '--verbose',
       '--contains',
       config.currentBranchSha,
     ]);
-    return !branches.all.map(localName).includes(branchName);
+    const isStale = !branches.all.map(localName).includes(branchName);
+    logger.debug(
+      { isStale, branches, currentBranch, currentBranchSha },
+      `IsBranchStale=${isStale}`
+    );
+    return isStale;
   } catch (err) /* istanbul ignore next */ {
     checkForPlatformFailure(err);
     throw err;

lib/workers/branch/automerge.ts

@@ -11,6 +11,7 @@ export type AutomergeResult =
   | 'branch status error'
   | 'failed'
   | 'no automerge'
+  | 'stale'
   | 'not ready';
 
 export async function tryBranchAutomerge(
@@ -48,7 +49,7 @@ export async function tryBranchAutomerge(
         err.message.includes('Not possible to fast-forward')
       ) {
         logger.warn({ err }, 'Branch is not up to date - cannot automerge');
-        return 'not ready';
+        return 'stale';
       }
       if (err.message.includes('Protected branch')) {
         if (err.message.includes('status check')) {