feat(pip-compile): Provide credentials for registries in all input files

lib/modules/manager/pip-compile/artifacts.spec.ts

@@ -86,6 +86,7 @@ describe('modules/manager/pip-compile/artifacts', () => {
 
   it('returns null if all unchanged', async () => {
     fs.readLocalFile.mockResolvedValueOnce(simpleHeader);
+    fs.readLocalFile.mockResolvedValueOnce('dependency==1.2.3');
     const execSnapshots = mockExecAll();
     fs.readLocalFile.mockResolvedValueOnce('new lock');
     expect(
@@ -106,6 +107,7 @@ describe('modules/manager/pip-compile/artifacts', () => {
 
   it('returns null if no config.lockFiles', async () => {
     fs.readLocalFile.mockResolvedValueOnce(simpleHeader);
+    fs.readLocalFile.mockResolvedValueOnce('dependency==1.2.3');
     fs.readLocalFile.mockResolvedValueOnce('new lock');
     expect(
       await updateArtifacts({
@@ -125,6 +127,7 @@ describe('modules/manager/pip-compile/artifacts', () => {
 
   it('returns updated requirements.txt', async () => {
     fs.readLocalFile.mockResolvedValueOnce(simpleHeader);
+    fs.readLocalFile.mockResolvedValueOnce('dependency==1.2.3');
     const execSnapshots = mockExecAll();
     git.getRepoStatus.mockResolvedValue(
       partial<StatusResult>({
@@ -162,6 +165,7 @@ describe('modules/manager/pip-compile/artifacts', () => {
       }),
     );
     fs.readLocalFile.mockResolvedValueOnce(simpleHeader);
+    fs.readLocalFile.mockResolvedValueOnce('dependency==1.2.3');
     fs.ensureCacheDir.mockResolvedValueOnce('/tmp/renovate/cache/others/pip');
     expect(
       await updateArtifacts({
@@ -215,6 +219,7 @@ describe('modules/manager/pip-compile/artifacts', () => {
       }),
     );
     fs.readLocalFile.mockResolvedValueOnce(simpleHeader);
+    fs.readLocalFile.mockResolvedValueOnce('dependency==1.2.3');
     expect(
       await updateArtifacts({
         packageFileName: 'requirements.in',
@@ -328,6 +333,7 @@ describe('modules/manager/pip-compile/artifacts', () => {
   it('catches errors', async () => {
     const execSnapshots = mockExecAll();
     fs.readLocalFile.mockResolvedValueOnce('Current requirements.txt');
+    fs.readLocalFile.mockResolvedValueOnce('dependency==1.2.3');
     fs.writeLocalFile.mockImplementationOnce(() => {
       throw new Error('not found');
     });
@@ -348,6 +354,7 @@ describe('modules/manager/pip-compile/artifacts', () => {
 
   it('returns updated requirements.txt when doing lockfile maintenance', async () => {
     fs.readLocalFile.mockResolvedValueOnce(simpleHeader);
+    fs.readLocalFile.mockResolvedValueOnce('dependency==1.2.3');
     const execSnapshots = mockExecAll();
     git.getRepoStatus.mockResolvedValue(
       partial<StatusResult>({
@@ -370,6 +377,7 @@ describe('modules/manager/pip-compile/artifacts', () => {
 
   it('uses --upgrade-package only for isLockfileUpdate', async () => {
     fs.readLocalFile.mockResolvedValueOnce(simpleHeader);
+    fs.readLocalFile.mockResolvedValueOnce('dependency==1.2.3');
     const execSnapshots = mockExecAll();
     git.getRepoStatus.mockResolvedValue(
       partial<StatusResult>({
@@ -397,6 +405,7 @@ describe('modules/manager/pip-compile/artifacts', () => {
 
   it('uses pip-compile version from config', async () => {
     fs.readLocalFile.mockResolvedValueOnce(simpleHeader);
+    fs.readLocalFile.mockResolvedValueOnce('dependency==1.2.3');
     GlobalConfig.set(dockerAdminConfig);
     // pip-tools
     datasource.getPkgReleases.mockResolvedValueOnce({

lib/modules/manager/pip-compile/artifacts.ts

@@ -1,4 +1,5 @@
 import { quote } from 'shlex';
+import upath from 'upath';
 import { TEMPORARY_ERROR } from '../../../constants/error-messages';
 import { logger } from '../../../logger';
 import { exec } from '../../../util/exec';
@@ -8,13 +9,19 @@ import {
   writeLocalFile,
 } from '../../../util/fs';
 import { getRepoStatus } from '../../../util/git';
-import * as pipRequirements from '../pip_requirements';
-import type { UpdateArtifact, UpdateArtifactsResult, Upgrade } from '../types';
+import { extractPackageFileFlags as extractRequirementsFileFlags } from '../pip_requirements/common';
+import type {
+  PackageFileContent,
+  UpdateArtifact,
+  UpdateArtifactsResult,
+  Upgrade,
+} from '../types';
 import {
   extractHeaderCommand,
   extractPythonVersion,
   getExecOptions,
-  getRegistryCredVarsFromPackageFile,
+  getRegistryCredVarsFromPackageFiles,
+  matchManager,
 } from './common';
 import type { PipCompileArgs } from './types';
 import { inferCommandExecDir } from './utils';
@@ -113,12 +120,25 @@ export async function updateArtifacts({
       );
       const cwd = inferCommandExecDir(outputFileName, compileArgs.outputFile);
       const upgradePackages = updatedDeps.filter((dep) => dep.isLockfileUpdate);
-      const packageFile = pipRequirements.extractPackageFile(newInputContent);
+      const packageFiles: PackageFileContent[] = [];
+      for (const name of compileArgs.sourceFiles) {
+        const manager = matchManager(name);
+        if (manager === 'pip_requirements') {
+          const path = upath.join(cwd, name);
+          const content = await readLocalFile(path, 'utf8');
+          if (content) {
+            const packageFile = extractRequirementsFileFlags(content);
+            if (packageFile) {
+              packageFiles.push(packageFile);
+            }
+          }
+        }
+      }
       const cmd = constructPipCompileCmd(compileArgs, upgradePackages);
       const execOptions = await getExecOptions(
         config,
         cwd,
-        getRegistryCredVarsFromPackageFile(packageFile),
+        getRegistryCredVarsFromPackageFiles(packageFiles),
         pythonVersion,
       );
       logger.trace({ cwd, cmd }, 'pip-compile command');

lib/modules/manager/pip-compile/common.spec.ts

@@ -5,7 +5,7 @@ import {
   allowedPipOptions,
   extractHeaderCommand,
   extractPythonVersion,
-  getRegistryCredVarsFromPackageFile,
+  getRegistryCredVarsFromPackageFiles,
   matchManager,
 } from './common';
 import { inferCommandExecDir } from './utils';
@@ -187,7 +187,7 @@ describe('modules/manager/pip-compile/common', () => {
     });
   });
 
-  describe('getCredentialVarsFromPackageFile()', () => {
+  describe('getRegistryCredVarsFromPackageFiles()', () => {
     it('handles both registryUrls and additionalRegistryUrls', () => {
       hostRules.find.mockReturnValueOnce({
         username: 'user1',
@@ -198,11 +198,13 @@ describe('modules/manager/pip-compile/common', () => {
         password: 'password2',
       });
       expect(
-        getRegistryCredVarsFromPackageFile({
-          deps: [],
-          registryUrls: ['https://example.com/pypi/simple'],
-          additionalRegistryUrls: ['https://example2.com/pypi/simple'],
-        }),
+        getRegistryCredVarsFromPackageFiles([
+          {
+            deps: [],
+            registryUrls: ['https://example.com/pypi/simple'],
+            additionalRegistryUrls: ['https://example2.com/pypi/simple'],
+          },
+        ]),
       ).toEqual({
         KEYRING_SERVICE_NAME_0: 'example.com',
         KEYRING_SERVICE_USERNAME_0: 'user1',
@@ -223,13 +225,15 @@ describe('modules/manager/pip-compile/common', () => {
         password: 'password2',
       });
       expect(
-        getRegistryCredVarsFromPackageFile({
-          deps: [],
-          additionalRegistryUrls: [
-            'https://example.com/pypi/simple',
-            'https://example2.com/pypi/simple',
-          ],
-        }),
+        getRegistryCredVarsFromPackageFiles([
+          {
+            deps: [],
+            additionalRegistryUrls: [
+              'https://example.com/pypi/simple',
+              'https://example2.com/pypi/simple',
+            ],
+          },
+        ]),
       ).toEqual({
         KEYRING_SERVICE_NAME_0: 'example.com',
         KEYRING_SERVICE_USERNAME_0: 'user1',
@@ -245,10 +249,12 @@ describe('modules/manager/pip-compile/common', () => {
         username: 'user',
       });
       expect(
-        getRegistryCredVarsFromPackageFile({
-          deps: [],
-          additionalRegistryUrls: ['https://example.com/pypi/simple'],
-        }),
+        getRegistryCredVarsFromPackageFiles([
+          {
+            deps: [],
+            additionalRegistryUrls: ['https://example.com/pypi/simple'],
+          },
+        ]),
       ).toEqual({
         KEYRING_SERVICE_NAME_0: 'example.com',
         KEYRING_SERVICE_USERNAME_0: 'user',
@@ -261,10 +267,12 @@ describe('modules/manager/pip-compile/common', () => {
         password: 'password',
       });
       expect(
-        getRegistryCredVarsFromPackageFile({
-          deps: [],
-          additionalRegistryUrls: ['https://example.com/pypi/simple'],
-        }),
+        getRegistryCredVarsFromPackageFiles([
+          {
+            deps: [],
+            additionalRegistryUrls: ['https://example.com/pypi/simple'],
+          },
+        ]),
       ).toEqual({
         KEYRING_SERVICE_NAME_0: 'example.com',
         KEYRING_SERVICE_USERNAME_0: '',
@@ -277,14 +285,46 @@ describe('modules/manager/pip-compile/common', () => {
         password: 'password',
       });
       expect(
-        getRegistryCredVarsFromPackageFile({
-          deps: [],
-          additionalRegistryUrls: ['invalid-url'],
-        }),
+        getRegistryCredVarsFromPackageFiles([
+          {
+            deps: [],
+            additionalRegistryUrls: ['invalid-url'],
+          },
+        ]),
       ).toEqual({});
     });
   });
 
+  it('handles multiple package files', () => {
+    hostRules.find.mockReturnValueOnce({
+      username: 'user1',
+      password: 'password1',
+    });
+    hostRules.find.mockReturnValueOnce({
+      username: 'user2',
+      password: 'password2',
+    });
+    expect(
+      getRegistryCredVarsFromPackageFiles([
+        {
+          deps: [],
+          registryUrls: ['https://example.com/pypi/simple'],
+        },
+        {
+          deps: [],
+          additionalRegistryUrls: ['https://example2.com/pypi/simple'],
+        },
+      ]),
+    ).toEqual({
+      KEYRING_SERVICE_NAME_0: 'example.com',
+      KEYRING_SERVICE_USERNAME_0: 'user1',
+      KEYRING_SERVICE_PASSWORD_0: 'password1',
+      KEYRING_SERVICE_NAME_1: 'example2.com',
+      KEYRING_SERVICE_USERNAME_1: 'user2',
+      KEYRING_SERVICE_PASSWORD_1: 'password2',
+    });
+  });
+
   describe('matchManager()', () => {
     it('matches pip_setup setup.py', () => {
       expect(matchManager('setup.py')).toBe('pip_setup');

lib/modules/manager/pip-compile/common.ts

@@ -281,13 +281,17 @@ function cleanUrl(url: string): URL | null {
   }
 }
 
-export function getRegistryCredVarsFromPackageFile(
-  packageFile: PackageFileContent | null,
+export function getRegistryCredVarsFromPackageFiles(
+  packageFiles: PackageFileContent[],
 ): ExtraEnv<string> {
-  const urls = [
-    ...(packageFile?.registryUrls ?? []),
-    ...(packageFile?.additionalRegistryUrls ?? []),
-  ];
+  const urls: string[] = [];
+  for (const packageFile of packageFiles) {
+    urls.push(
+      ...(packageFile.registryUrls ?? []),
+      ...(packageFile.additionalRegistryUrls ?? []),
+    );
+  }
+  logger.debug(urls, 'Extracted registry URLs from package files');
 
   const uniqueHosts = new Set<URL>(
     urls.map(cleanUrl).filter(isNotNullOrUndefined),