From 6971e41d60992a17ab8a4fbd5aaadd5464381b67 Mon Sep 17 00:00:00 2001 From: Rhys Arkins Date: Sat, 24 Apr 2021 17:46:22 +0200 Subject: [PATCH 1/8] docs: app and self-hosted private modules --- docs/usage/private-modules.md | 68 ++++++++++++++++++++++++++++++++--- 1 file changed, 64 insertions(+), 4 deletions(-) diff --git a/docs/usage/private-modules.md b/docs/usage/private-modules.md index f58926e4d76769..3aba9f2421cd43 100644 --- a/docs/usage/private-modules.md +++ b/docs/usage/private-modules.md @@ -147,7 +147,67 @@ For those found, a command similar to the following is run: `dotnet nuget add so For every poetry source, a `hostRules` search is done and then any found credentials are added to env like `POETRY_HTTP_BASIC_X_USERNAME` and `POETRY_HTTP_BASIC_X_PASSWORD`. - +## WhiteSource Renovate Hosted App Encryption + +The popular [Renovate App on GitHub](https://github.com/apps/renovate) is hosted by WhiteSource. +If you are a user of this app, and have private modules, then the following is applicable. + +### Private presets with public repositories + +If you have a preset in a private repo but reference ("extend") it from a public repository then it won't work. +This is because public repositories are provided with a token scoped to only that particular repository, and not for all repositories within the org. +This is a security measure so that if a the token is accidentally leaked publicly, the damage is limited to the public repository it leaked to and not to every repository within the organization. + +The solution to this is that you should break your presets into public and private ones, and reference only the public ones from public repositories. + +### Encrypting secrets + +It is strongly recommended that you don't commit secrets to repositories, including private ones, and this includes secrets needed by Renovate to access private modules. +Therefore the preferred approach to secrets is that the bot administrator configures them as `hostRules` which are then applied to all repositories which the bot accesses. + +If you need to provide credentials to the hosted Renovate App, please do this: + +- Encrypt each secret string using . Note: this encrypts using the app's public key fully in the browser and does not send the original secret to any server. You can download this file and perform the encryption fully offline if you like. +- Wrap each secret field in an [encrypted](https://docs.renovatebot.com/configuration-options/#encrypted) object and paste in the encrypted secret value instead. An example is shown below: + +```json +{ + "hostRules": [ + { + "hostName": "registry.npmjs.org", + "encrypted": { + "token": "3f832f2983yf89hsd98ahadsjfasdfjaslf............" + } + }, + { + "baseUrl": "https://custom.registry.company.com/pypi/", + "username": "bot1", + "encrypted": { + "password": "p278djfdsi9832jnfdshufwji2r389fdskj........." + } + } + ] +} +``` + +### Access to GitHub Actions Secrets + +The WhiteSource Renovate App does not run using GitHub Actions, but such secrets would be a bad fit for the app anyway for the following reasons: + +- The app would be granted access to _all_ the repository/org secrets, not just the ones you want +- If Renovate wants access to such secrets, it would need to ask for them from every user, not just the ones who want to use this approach (GitHub does not support the concept of optional permissions for Apps, so people do not have the option to decline) + +## Admin/Bot config vs User/Repository config for Self-hosted users + +"AdminBot config" refers to the config which the Renovate Bot administrator provides at bot startup, e.g. using environment variables, CLI parameters, or the `config.js` configuration file. User/Repository config refers to the in-repository config file which defaults to `renovate.json` but has a large number of alternative filenames supported. + +If there is a need to supply custom rules for certain repository, it can still be done using the `config.js` file and the `repositories` array. + +If per-repository config must be done within the repository, it is still recommended against committing secrets directly (including e.g. `.npmrc` files with tokens) and instead encrypting them with a custom public key first. +For instructions on this, see the above section on encrypting secrets for the WhiteSource Renovate App but instead: + +- Save a copy of the html file locally, or host it locally +- Generate a public/private key pair for the app using the instructions in [privateKey](https://docs.renovatebot.com/self-hosted-configuration/#privatekey) +- Replace the existing public key in the html with the public key you generated in the step prior +- Use the resulting html encrypt page to encrypt secrets for your app before adding them to user/repository config +- Configure the app to run with `privateKey` set to the private key you generated above From b3431ffb7bf82f369bfd9471f52405ecfb6f7cd5 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sat, 24 Apr 2021 20:21:34 +0000 Subject: [PATCH 2/8] chore(deps): update dependency @types/luxon to v1.26.4 (#9706) Co-authored-by: Renovate Bot --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index ae4bd567475823..c81c12efa0d946 100644 --- a/package.json +++ b/package.json @@ -212,7 +212,7 @@ "@types/js-yaml": "3.12.6", "@types/json-dup-key-validator": "1.0.0", "@types/linkify-markdown": "1.0.0", - "@types/luxon": "1.26.3", + "@types/luxon": "1.26.4", "@types/markdown-it": "12.0.1", "@types/markdown-table": "2.0.0", "@types/moo": "0.5.4", diff --git a/yarn.lock b/yarn.lock index 193a71744540ab..48b066ac185d27 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1525,10 +1525,10 @@ resolved "https://registry.yarnpkg.com/@types/linkify-markdown/-/linkify-markdown-1.0.0.tgz#50c357a0a60bb220209f19310a79b939e1b8e2a3" integrity sha512-p3rl2HtugwjkH8qjEjUi5JByoZkVc2lz6E7D96X5/FqNyj2/jlgIrMlyrbck0U+HyjhOIy0XyTwES3RRz+1koQ== -"@types/luxon@1.26.3": - version "1.26.3" - resolved "https://registry.yarnpkg.com/@types/luxon/-/luxon-1.26.3.tgz#3d548f5f6f59723c292a8545da97631a85738b95" - integrity sha512-2TELN+Pd3Ocde87sKJMSQ9Wdj0zc/okHK3/+fOQHr3CaWv4jtVtcMzmt1Foww1+5YvPd9B5vL3XR6u5KF0daEA== +"@types/luxon@1.26.4": + version "1.26.4" + resolved "https://registry.yarnpkg.com/@types/luxon/-/luxon-1.26.4.tgz#77bbd5a1aa74ca31edff2aee4f8313ea0d93ca5c" + integrity sha512-OIvbVLZQUjyZofqSFpre2VsgvKy0V0JQdRgN0k3H1DTGRdxHiaQjT16+H2gyuhAS9r8B2PQEwrSiqP6/Zka3pQ== "@types/markdown-it@12.0.1": version "12.0.1" From a91c85317ab06704d550d34fc0bc297451fab2cd Mon Sep 17 00:00:00 2001 From: Muhammad Rehan Saeed Date: Sat, 24 Apr 2021 21:25:52 +0100 Subject: [PATCH 3/8] Add Open Telemetry Dotnet monorepo (#9710) --- lib/config/presets/internal/monorepo.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/config/presets/internal/monorepo.ts b/lib/config/presets/internal/monorepo.ts index eee396525cad08..bb13b6f8813db1 100644 --- a/lib/config/presets/internal/monorepo.ts +++ b/lib/config/presets/internal/monorepo.ts @@ -119,6 +119,7 @@ const repoGroups = { nuxtjs: 'https://github.com/nuxt/nuxt.js', openfeign: 'https://github.com/OpenFeign/feign', opentelemetry: 'https://github.com/open-telemetry/opentelemetry-js', + OpenTelemetryDotnet: 'https://github.com/open-telemetry/opentelemetry-dotnet', picasso: 'https://github.com/qlik-oss/picasso.js', pnpjs: 'https://github.com/pnp/pnpjs', playwright: 'https://github.com/Microsoft/playwright', From c542cbe7f549e384ee7c64f30285ebc2e40d7ff7 Mon Sep 17 00:00:00 2001 From: HonkingGoose <34918129+HonkingGoose@users.noreply.github.com> Date: Sun, 25 Apr 2021 15:33:11 +0200 Subject: [PATCH 4/8] docs: improve private module support documentation (#9711) --- docs/usage/private-modules.md | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/docs/usage/private-modules.md b/docs/usage/private-modules.md index 3aba9f2421cd43..b132990ba4d5a8 100644 --- a/docs/usage/private-modules.md +++ b/docs/usage/private-modules.md @@ -38,7 +38,7 @@ Although npm presets were the first type supported, they are now deprecated and However if you do still use them, private modules should work if you configure the `npmrc` file including token credentials in your bot admin config. Credentials stored on disk (e.g. in `~/.npmrc`) are no longer supported. -The recommended way of using local presets is to configure then using "local" presets, e.g. like `"extends": ["local>myorg/renovate-config"]`, and ensure that the platform token has access to that repo. +The recommended way of using local presets is to configure then using "local" presets, e.g. `"extends": ["local>myorg/renovate-config"]`, and ensure that the platform token has access to that repo. It's not recommended that you use a private repository to host your config while then extending it from a public repository. If your preset doesn't contain secrets then you should make it public, while if it does contain secrets then it's better to split your preset between a public one which all repos extend, and a private one with secrets which only other private repos extend. @@ -52,7 +52,7 @@ In summary, the recommended approach to private presets is: ## Dependency Version Lookups Whenever Renovate detects that a project uses a particular dependency, it attempts to look up that dependency to see if any new versions exist. -If such a package is private, then Renovate will need to be configured with the relevant credentials. +If such a package is private, then Renovate must be configured with the relevant credentials. Renovate does not use any package managers for this step and performs all HTTP(S) lookups itself, including insertion of authentication headers. Configuring Renovate with credentials requires `hostRules`. @@ -64,7 +64,7 @@ It can be either a "platform" name (e.g. `github`, `azure`, etc) or a "datasourc `baseUrl` can be used if you want to only apply the credentials for a nested path within the host, e.g. `https://registry.company.com/nested/path/`. If the same credentials apply to all paths on a host, then use `hostName` instead, e.g. `registry.company.com`. Finally, to apply credentials to all hosts within the domain, use `domainName`, e.g. `company.com`. -You need to pick only one of these and not try to use multiple at the same time, or it will be a config error. +You need to pick _only one_ of these and not configure more than one of these fields within the same host rule, otherwise it will error. In addition to the above options to match against a host, you need to add the credentials. Typically they are either `token`, or `username` + `password`. @@ -91,7 +91,7 @@ Here is an example of some host rules: Renovate applies theses `hostRules` to every HTTP(s) request which is sent, so they are largely independent of any platform or datasource logic. With `hostRules` in place, private package lookups should all work. -## Release Notes +## Looking up Release Notes When Renovate creates Pull Requests, its default behavior is to locate and embed release notes/changelogs of packages. These release notes are fetched from the source repository of packages and not from the registries themselves, so if they are private then they will require different credentials. @@ -102,7 +102,7 @@ It can be confusing for people who host their own source code privately to be as Currently the preferred way to configure `github.com` credentials for self-hosted Renovate is: -- Create a read-only Personal Access Token (PAT) for a `github.com` account. It can be any account, and may even be best to be an empty account created just for this purpose. +- Create a read-only Personal Access Token (PAT) for a `github.com` account. This can be any GitHub account, it might be better to create a "empty" account just for this purpose. - Add the PAT to Renovate using the environment variable `GITHUB_COM_TOKEN` ## Package Manager Credentials for Artifact Updating @@ -155,7 +155,7 @@ If you are a user of this app, and have private modules, then the following is a ### Private presets with public repositories If you have a preset in a private repo but reference ("extend") it from a public repository then it won't work. -This is because public repositories are provided with a token scoped to only that particular repository, and not for all repositories within the org. +This is because public repositories are provided with a token scoped to only that particular repository, and not for all repositories within the organization. This is a security measure so that if a the token is accidentally leaked publicly, the damage is limited to the public repository it leaked to and not to every repository within the organization. The solution to this is that you should break your presets into public and private ones, and reference only the public ones from public repositories. @@ -199,15 +199,16 @@ The WhiteSource Renovate App does not run using GitHub Actions, but such secrets ## Admin/Bot config vs User/Repository config for Self-hosted users -"AdminBot config" refers to the config which the Renovate Bot administrator provides at bot startup, e.g. using environment variables, CLI parameters, or the `config.js` configuration file. User/Repository config refers to the in-repository config file which defaults to `renovate.json` but has a large number of alternative filenames supported. +"AdminBot config" refers to the config which the Renovate Bot administrator provides at bot startup, e.g. using environment variables, CLI parameters, or the `config.js` configuration file. +User/Repository config refers to the in-repository config file which defaults to `renovate.json` but has a large number of alternative filenames supported. If there is a need to supply custom rules for certain repository, it can still be done using the `config.js` file and the `repositories` array. If per-repository config must be done within the repository, it is still recommended against committing secrets directly (including e.g. `.npmrc` files with tokens) and instead encrypting them with a custom public key first. For instructions on this, see the above section on encrypting secrets for the WhiteSource Renovate App but instead: -- Save a copy of the html file locally, or host it locally +- Save a copy of the HTML file locally, or host it locally - Generate a public/private key pair for the app using the instructions in [privateKey](https://docs.renovatebot.com/self-hosted-configuration/#privatekey) -- Replace the existing public key in the html with the public key you generated in the step prior -- Use the resulting html encrypt page to encrypt secrets for your app before adding them to user/repository config +- Replace the existing public key in the HTML with the public key you generated in the step prior +- Use the resulting HTML encrypt page to encrypt secrets for your app before adding them to user/repository config - Configure the app to run with `privateKey` set to the private key you generated above From 00cb277bb6d612f63a517644705e060abfde957f Mon Sep 17 00:00:00 2001 From: Michael Kriese Date: Sun, 25 Apr 2021 21:26:09 +0200 Subject: [PATCH 5/8] fix(git): add debug (#9683) --- lib/util/git/index.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/util/git/index.ts b/lib/util/git/index.ts index 829f18abe8cdcd..390d0f07c38531 100644 --- a/lib/util/git/index.ts +++ b/lib/util/git/index.ts @@ -450,13 +450,19 @@ export function getBranchList(): string[] { export async function isBranchStale(branchName: string): Promise { await syncBranch(branchName); try { + const { currentBranchSha, currentBranch } = config; const branches = await git.branch([ '--remotes', '--verbose', '--contains', config.currentBranchSha, ]); - return !branches.all.map(localName).includes(branchName); + const isStale = !branches.all.map(localName).includes(branchName); + logger.debug( + { isStale, branches, currentBranch, currentBranchSha }, + `IsBranchStale=${isStale}` + ); + return isStale; } catch (err) /* istanbul ignore next */ { checkForPlatformFailure(err); throw err; From 65c3992ec592e235dd1343295b683c50f36c9de7 Mon Sep 17 00:00:00 2001 From: Rhys Arkins Date: Mon, 26 Apr 2021 07:00:26 +0200 Subject: [PATCH 6/8] fix: create PR if stale branch can't automerge (#9723) --- lib/workers/branch/automerge.ts | 3 ++- lib/workers/branch/index.spec.ts | 25 +++++++++++++++++++++++++ lib/workers/branch/index.ts | 10 ++++++++++ 3 files changed, 37 insertions(+), 1 deletion(-) diff --git a/lib/workers/branch/automerge.ts b/lib/workers/branch/automerge.ts index 674953beee297f..765d0c2b018fc6 100644 --- a/lib/workers/branch/automerge.ts +++ b/lib/workers/branch/automerge.ts @@ -11,6 +11,7 @@ export type AutomergeResult = | 'branch status error' | 'failed' | 'no automerge' + | 'stale' | 'not ready'; export async function tryBranchAutomerge( @@ -48,7 +49,7 @@ export async function tryBranchAutomerge( err.message.includes('Not possible to fast-forward') ) { logger.warn({ err }, 'Branch is not up to date - cannot automerge'); - return 'not ready'; + return 'stale'; } if (err.message.includes('Protected branch')) { if (err.message.includes('status check')) { diff --git a/lib/workers/branch/index.spec.ts b/lib/workers/branch/index.spec.ts index 097e78b5490a7e..fadb85c91c6da2 100644 --- a/lib/workers/branch/index.spec.ts +++ b/lib/workers/branch/index.spec.ts @@ -448,6 +448,31 @@ describe(getName(__filename), () => { expect(platform.ensureCommentRemoval).toHaveBeenCalledTimes(0); expect(prAutomerge.checkAutoMerge).toHaveBeenCalledTimes(1); }); + it('ensures PR when impossible to automerge', async () => { + getUpdated.getUpdatedPackageFiles.mockResolvedValueOnce({ + updatedPackageFiles: [{}], + } as PackageFilesResult); + npmPostExtract.getAdditionalFiles.mockResolvedValueOnce({ + artifactErrors: [], + updatedArtifacts: [{}], + } as WriteExistingFilesResult); + git.branchExists.mockReturnValue(true); + automerge.tryBranchAutomerge.mockResolvedValueOnce('stale'); + prWorker.ensurePr.mockResolvedValueOnce({ + prResult: PrResult.Created, + pr: {}, + } as EnsurePrResult); + prAutomerge.checkAutoMerge.mockResolvedValueOnce({ automerged: false }); + commit.commitFilesToBranch.mockResolvedValueOnce(null); + await branchWorker.processBranch({ + ...config, + automerge: true, + rebaseWhen: 'conflicted', + }); + expect(prWorker.ensurePr).toHaveBeenCalledTimes(1); + expect(platform.ensureCommentRemoval).toHaveBeenCalledTimes(0); + expect(prAutomerge.checkAutoMerge).toHaveBeenCalledTimes(1); + }); it('ensures PR and adds lock file error comment if no releaseTimestamp', async () => { getUpdated.getUpdatedPackageFiles.mockResolvedValueOnce({ updatedPackageFiles: [{}], diff --git a/lib/workers/branch/index.ts b/lib/workers/branch/index.ts index d5359e2028110a..8bbc0160d87c19 100644 --- a/lib/workers/branch/index.ts +++ b/lib/workers/branch/index.ts @@ -380,6 +380,16 @@ export async function processBranch( logger.debug('Branch is automerged - returning'); return { branchExists: false, result: BranchResult.Automerged }; } + if ( + mergeStatus === 'stale' && + ['conflicted', 'never'].includes(config.rebaseWhen) + ) { + logger.warn( + 'Branch cannot automerge because it is stale and rebaseWhen setting disallows rebasing - raising a PR instead' + ); + config.forcePr = true; + config.branchAutomergeFailureMessage = mergeStatus; + } if ( mergeStatus === 'automerge aborted - PR exists' || mergeStatus === 'branch status error' || From ca9ed19b3d98560e7c475024f55bd3f19400a752 Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Mon, 26 Apr 2021 13:01:40 +0400 Subject: [PATCH 7/8] refactor(maven): Extract util functions from datasource index (#9727) --- lib/datasource/maven/index.ts | 115 +++------------------------------- lib/datasource/maven/types.ts | 13 ++++ lib/datasource/maven/util.ts | 97 ++++++++++++++++++++++++++++ 3 files changed, 118 insertions(+), 107 deletions(-) create mode 100644 lib/datasource/maven/types.ts diff --git a/lib/datasource/maven/index.ts b/lib/datasource/maven/index.ts index 5867caa41ca61c..8ee4e527b475df 100644 --- a/lib/datasource/maven/index.ts +++ b/lib/datasource/maven/index.ts @@ -1,5 +1,4 @@ import url from 'url'; -import fs from 'fs-extra'; import pAll from 'p-all'; import { XmlDocument } from 'xmldoc'; import { logger } from '../../logger'; @@ -9,7 +8,14 @@ import * as mavenVersioning from '../../versioning/maven'; import { compare } from '../../versioning/maven/compare'; import type { GetReleasesConfig, Release, ReleaseResult } from '../types'; import { MAVEN_REPO } from './common'; -import { downloadHttpProtocol, isHttpResourceExists } from './util'; +import type { MavenDependency } from './types'; +import { + downloadMavenXml, + getDependencyInfo, + getDependencyParts, + getMavenUrl, + isHttpResourceExists, +} from './util'; export { id } from './common'; @@ -18,93 +24,6 @@ export const defaultRegistryUrls = [MAVEN_REPO]; export const defaultVersioning = mavenVersioning.id; export const registryStrategy = 'merge'; -function containsPlaceholder(str: string): boolean { - return /\${.*?}/g.test(str); -} - -async function downloadFileProtocol(pkgUrl: url.URL): Promise { - const pkgPath = pkgUrl.toString().replace('file://', ''); - if (!(await fs.exists(pkgPath))) { - return null; - } - return fs.readFile(pkgPath, 'utf8'); -} - -function getMavenUrl( - dependency: MavenDependency, - repoUrl: string, - path: string -): url.URL | null { - return new url.URL(`${dependency.dependencyUrl}/${path}`, repoUrl); -} - -interface MavenXml { - authorization?: boolean; - xml?: XmlDocument; -} - -async function downloadMavenXml( - pkgUrl: url.URL | null -): Promise { - /* istanbul ignore if */ - if (!pkgUrl) { - return {}; - } - let rawContent: string; - let authorization: boolean; - switch (pkgUrl.protocol) { - case 'file:': - rawContent = await downloadFileProtocol(pkgUrl); - break; - case 'http:': - case 'https:': - ({ authorization, body: rawContent } = await downloadHttpProtocol( - pkgUrl - )); - break; - case 's3:': - logger.debug('Skipping s3 dependency'); - return {}; - default: - logger.debug({ url: pkgUrl.toString() }, `Unsupported Maven protocol`); - return {}; - } - - if (!rawContent) { - logger.debug(`Content is not found for Maven url: ${pkgUrl.toString()}`); - return {}; - } - - return { authorization, xml: new XmlDocument(rawContent) }; -} - -async function getDependencyInfo( - dependency: MavenDependency, - repoUrl: string, - version: string -): Promise> { - const result: Partial = {}; - const path = `${version}/${dependency.name}-${version}.pom`; - - const pomUrl = getMavenUrl(dependency, repoUrl, path); - const { xml: pomContent } = await downloadMavenXml(pomUrl); - if (!pomContent) { - return result; - } - - const homepage = pomContent.valueWithPath('url'); - if (homepage && !containsPlaceholder(homepage)) { - result.homepage = homepage; - } - - const sourceUrl = pomContent.valueWithPath('scm.url'); - if (sourceUrl && !containsPlaceholder(sourceUrl)) { - result.sourceUrl = sourceUrl.replace(/^scm:/, ''); - } - - return result; -} - function isStableVersion(x: string): boolean { return mavenVersion.isStable(x); } @@ -121,24 +40,6 @@ function getLatestStableVersion(releases: Release[]): string | null { return null; } -interface MavenDependency { - display: string; - group?: string; - name?: string; - dependencyUrl: string; -} - -function getDependencyParts(lookupName: string): MavenDependency { - const [group, name] = lookupName.split(':'); - const dependencyUrl = `${group.replace(/\./g, '/')}/${name}`; - return { - display: lookupName, - group, - name, - dependencyUrl, - }; -} - function extractVersions(metadata: XmlDocument): string[] { const versions = metadata.descendantWithPath('versioning.versions'); const elements = versions?.childrenNamed('version'); diff --git a/lib/datasource/maven/types.ts b/lib/datasource/maven/types.ts new file mode 100644 index 00000000000000..06d8fd4c211d26 --- /dev/null +++ b/lib/datasource/maven/types.ts @@ -0,0 +1,13 @@ +import type { XmlDocument } from 'xmldoc'; + +export interface MavenDependency { + display: string; + group?: string; + name?: string; + dependencyUrl: string; +} + +export interface MavenXml { + authorization?: boolean; + xml?: XmlDocument; +} diff --git a/lib/datasource/maven/util.ts b/lib/datasource/maven/util.ts index 335634caac0749..311c1041d4374d 100644 --- a/lib/datasource/maven/util.ts +++ b/lib/datasource/maven/util.ts @@ -1,10 +1,14 @@ import url from 'url'; +import fs from 'fs-extra'; +import { XmlDocument } from 'xmldoc'; import { HOST_DISABLED } from '../../constants/error-messages'; import { logger } from '../../logger'; import { ExternalHostError } from '../../types/errors/external-host-error'; import { Http, HttpResponse } from '../../util/http'; +import type { ReleaseResult } from '../types'; import { MAVEN_REPO, id } from './common'; +import type { MavenDependency, MavenXml } from './types'; const http: Record = {}; @@ -96,6 +100,14 @@ export async function downloadHttpProtocol( } } +async function downloadFileProtocol(pkgUrl: url.URL): Promise { + const pkgPath = pkgUrl.toString().replace('file://', ''); + if (!(await fs.exists(pkgPath))) { + return null; + } + return fs.readFile(pkgPath, 'utf8'); +} + export async function isHttpResourceExists( pkgUrl: url.URL | string, hostType = id @@ -119,3 +131,88 @@ export async function isHttpResourceExists( return null; } } + +function containsPlaceholder(str: string): boolean { + return /\${.*?}/g.test(str); +} + +export function getMavenUrl( + dependency: MavenDependency, + repoUrl: string, + path: string +): url.URL | null { + return new url.URL(`${dependency.dependencyUrl}/${path}`, repoUrl); +} + +export async function downloadMavenXml( + pkgUrl: url.URL | null +): Promise { + /* istanbul ignore if */ + if (!pkgUrl) { + return {}; + } + let rawContent: string; + let authorization: boolean; + switch (pkgUrl.protocol) { + case 'file:': + rawContent = await downloadFileProtocol(pkgUrl); + break; + case 'http:': + case 'https:': + ({ authorization, body: rawContent } = await downloadHttpProtocol( + pkgUrl + )); + break; + case 's3:': + logger.debug('Skipping s3 dependency'); + return {}; + default: + logger.debug({ url: pkgUrl.toString() }, `Unsupported Maven protocol`); + return {}; + } + + if (!rawContent) { + logger.debug(`Content is not found for Maven url: ${pkgUrl.toString()}`); + return {}; + } + + return { authorization, xml: new XmlDocument(rawContent) }; +} + +export async function getDependencyInfo( + dependency: MavenDependency, + repoUrl: string, + version: string +): Promise> { + const result: Partial = {}; + const path = `${version}/${dependency.name}-${version}.pom`; + + const pomUrl = getMavenUrl(dependency, repoUrl, path); + const { xml: pomContent } = await downloadMavenXml(pomUrl); + if (!pomContent) { + return result; + } + + const homepage = pomContent.valueWithPath('url'); + if (homepage && !containsPlaceholder(homepage)) { + result.homepage = homepage; + } + + const sourceUrl = pomContent.valueWithPath('scm.url'); + if (sourceUrl && !containsPlaceholder(sourceUrl)) { + result.sourceUrl = sourceUrl.replace(/^scm:/, ''); + } + + return result; +} + +export function getDependencyParts(lookupName: string): MavenDependency { + const [group, name] = lookupName.split(':'); + const dependencyUrl = `${group.replace(/\./g, '/')}/${name}`; + return { + display: lookupName, + group, + name, + dependencyUrl, + }; +} From 8b3f6821194910abd5b9a2f13fb105c8e04c212c Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Mon, 26 Apr 2021 12:18:47 +0300 Subject: [PATCH 8/8] Fix --- lib/datasource/sbt-package/index.ts | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/lib/datasource/sbt-package/index.ts b/lib/datasource/sbt-package/index.ts index a745d4344c66d6..a6b6beeeac50f6 100644 --- a/lib/datasource/sbt-package/index.ts +++ b/lib/datasource/sbt-package/index.ts @@ -2,11 +2,8 @@ import { logger } from '../../logger'; import * as ivyVersioning from '../../versioning/ivy'; import { compare } from '../../versioning/maven/compare'; import { MAVEN_REPO } from '../maven/common'; -import { - MavenDependency, - downloadHttpProtocol, - getDependencyInfo, -} from '../maven/util'; +import type { MavenDependency } from '../maven/types'; +import { downloadHttpProtocol, getDependencyInfo } from '../maven/util'; import { parseIndexDir } from '../sbt-plugin/util'; import type { GetReleasesConfig, ReleaseResult } from '../types';