fix(manager): optimize lockfile cache handling (#10463)

renovatebot · Jun 16, 2021 · 713e35e · 713e35e
1 parent 9184207
commit 713e35e
Show file tree

Hide file tree

Showing 6 changed files with 45 additions and 43 deletions.
diff --git a/lib/manager/terraform/lockfile/__snapshots__/index.spec.ts.snap b/lib/manager/terraform/lockfile/__snapshots__/index.spec.ts.snap
@@ -50,12 +50,10 @@ Array [
   Array [
     "hashicorp/azurerm",
     "2.56.0",
-    "/tmp/renovate/cache",
   ],
   Array [
     "hashicorp/random",
     "2.2.2",
-    "/tmp/renovate/cache",
   ],
 ]
 `;
@@ -67,12 +65,10 @@ Array [
   Array [
     "hashicorp/azurerm",
     "2.56.0",
-    "/tmp/renovate/cache",
   ],
   Array [
     "hashicorp/random",
     "2.2.2",
-    "/tmp/renovate/cache",
   ],
 ]
 `;
@@ -139,7 +135,6 @@ Array [
   Array [
     "hashicorp/aws",
     "3.36.0",
-    "/tmp/renovate/cache",
   ],
 ]
 `;
@@ -204,7 +199,6 @@ Array [
   Array [
     "hashicorp/random",
     "3.1.0",
-    "/tmp/renovate/cache",
   ],
 ]
 `;
@@ -270,7 +264,6 @@ Array [
   Array [
     "hashicorp/azurerm",
     "2.56.0",
-    "/tmp/renovate/cache",
   ],
 ]
 `;
diff --git a/lib/manager/terraform/lockfile/hash.spec.ts b/lib/manager/terraform/lockfile/hash.spec.ts
@@ -1,18 +1,27 @@
 import { createReadStream } from 'fs';
-import { join } from 'upath';
+import { DirectoryResult, dir } from 'tmp-promise';
 import * as httpMock from '../../../../test/http-mock';
 import { getFixturePath, getName, loadFixture } from '../../../../test/util';
+import { setAdminConfig } from '../../../config/admin';
 import { TerraformProviderDatasource } from '../../../datasource/terraform-provider';
-import createHashes from './hash';
+import { createHashes } from './hash';
 
 const terraformProviderDatasource = new TerraformProviderDatasource();
 const releaseBackendUrl = terraformProviderDatasource.defaultRegistryUrls[1];
 const releaseBackendAzurerm = loadFixture('releaseBackendAzurerm_2_56_0.json');
-const cacheDir = join('/tmp/renovate/cache');
 
 describe(getName(), () => {
+  let cacheDir: DirectoryResult;
+
+  beforeAll(async () => {
+    cacheDir = await dir({ unsafeCleanup: true });
+    setAdminConfig({ cacheDir: cacheDir.path });
+  });
+
+  afterAll(() => cacheDir.cleanup());
+
   it('returns null if a non hashicorp release is found ', async () => {
-    const result = await createHashes('test/gitlab', '2.56.0', cacheDir);
+    const result = await createHashes('test/gitlab', '2.56.0');
     expect(result).toBeNull();
   });
 
@@ -22,7 +31,7 @@ describe(getName(), () => {
       .get('/terraform-provider-azurerm/2.59.0/index.json')
       .reply(403, '');
 
-    const result = await createHashes('hashicorp/azurerm', '2.59.0', cacheDir);
+    const result = await createHashes('hashicorp/azurerm', '2.59.0');
     expect(result).toBeNull();
     expect(httpMock.getTrace()).toMatchSnapshot();
   });
@@ -33,7 +42,7 @@ describe(getName(), () => {
       .get('/terraform-provider-azurerm/2.56.0/index.json')
       .replyWithError('');
 
-    const result = await createHashes('hashicorp/azurerm', '2.56.0', cacheDir);
+    const result = await createHashes('hashicorp/azurerm', '2.56.0');
     expect(result).toBeNull();
     expect(httpMock.getTrace()).toMatchSnapshot();
   });
@@ -58,7 +67,7 @@ describe(getName(), () => {
       )
       .reply(200, readStreamDarwin);
 
-    const result = await createHashes('hashicorp/azurerm', '2.56.0', cacheDir);
+    const result = await createHashes('hashicorp/azurerm', '2.56.0');
     expect(result).toBeNull();
     expect(httpMock.getTrace()).toMatchSnapshot();
   });
@@ -83,7 +92,7 @@ describe(getName(), () => {
       )
       .reply(200, readStreamDarwin);
 
-    const result = await createHashes('hashicorp/azurerm', '2.56.0', cacheDir);
+    const result = await createHashes('hashicorp/azurerm', '2.56.0');
     expect(result).not.toBeNull();
     expect(result).toBeArrayOfSize(2);
     expect(result).toMatchSnapshot();

diff --git a/lib/manager/terraform/lockfile/hash.ts b/lib/manager/terraform/lockfile/hash.ts
@@ -11,7 +11,7 @@ import { logger } from '../../../logger';
 import * as packageCache from '../../../util/cache/package';
 import * as fs from '../../../util/fs';
 import { Http } from '../../../util/http';
-import { repositoryRegex } from './util';
+import { getCacheDir, repositoryRegex } from './util';
 
 const http = new Http(TerraformProviderDatasource.id);
 const hashCacheTTL = 10080; // in seconds == 1 week
@@ -74,9 +74,10 @@ async function getReleaseBackendIndex(
 }
 
 export async function calculateHashes(
-  builds: TerraformBuild[],
-  cacheDir: string
+  builds: TerraformBuild[]
 ): Promise<string[]> {
+  const cacheDir = await getCacheDir();
+
   // for each build download ZIP, extract content and generate hash for all containing files
   const hashes = await pMap(
     builds,
@@ -112,10 +113,9 @@ export async function calculateHashes(
   return hashes;
 }
 
-export default async function createHashes(
+export async function createHashes(
   repository: string,
-  version: string,
-  cacheDir: string
+  version: string
 ): Promise<string[]> {
   // check cache for hashes
   const repositoryRegexResult = repositoryRegex.exec(repository);
@@ -135,7 +135,7 @@ export default async function createHashes(
   if (cachedRelease) {
     return cachedRelease;
   }
-  let versionReleaseBackend;
+  let versionReleaseBackend: VersionDetailResponse;
   try {
     versionReleaseBackend = await getReleaseBackendIndex(
       backendLookUpName,
@@ -150,7 +150,7 @@ export default async function createHashes(
   }
 
   const builds = versionReleaseBackend.builds;
-  const hashes = await calculateHashes(builds, cacheDir);
+  const hashes = await calculateHashes(builds);
 
   // if a hash could not be produced skip caching and return null
   if (hashes.some((value) => value == null)) {

diff --git a/lib/manager/terraform/lockfile/index.spec.ts b/lib/manager/terraform/lockfile/index.spec.ts
@@ -1,9 +1,9 @@
 import { join } from 'upath';
-import { fs, getName, loadFixture } from '../../../../test/util';
+import { fs, getName, loadFixture, mocked } from '../../../../test/util';
 import { setAdminConfig } from '../../../config/admin';
 import { getPkgReleases } from '../../../datasource';
 import type { UpdateArtifactsConfig } from '../../types';
-import hash from './hash';
+import * as hash from './hash';
 import { updateArtifacts } from './index';
 
 // auto-mock fs
@@ -23,7 +23,7 @@ const adminConfig = {
 
 const validLockfile = loadFixture('validLockfile.hcl');
 
-const mockHash = hash as jest.MockedFunction<typeof hash>;
+const mockHash = mocked(hash).createHashes;
 const mockGetPkgReleases = getPkgReleases as jest.MockedFunction<
   typeof getPkgReleases
 >;

diff --git a/lib/manager/terraform/lockfile/index.ts b/lib/manager/terraform/lockfile/index.ts
@@ -1,14 +1,9 @@
 import pMap from 'p-map';
-import { getAdminConfig } from '../../../config/admin';
 import { GetPkgReleasesConfig, getPkgReleases } from '../../../datasource';
 import { logger } from '../../../logger';
 import { get as getVersioning } from '../../../versioning';
-import type {
-  UpdateArtifact,
-  UpdateArtifactsConfig,
-  UpdateArtifactsResult,
-} from '../../types';
-import hash from './hash';
+import type { UpdateArtifact, UpdateArtifactsResult } from '../../types';
+import { createHashes } from './hash';
 import type { ProviderLock, ProviderLockUpdate } from './types';
 import {
   extractLocks,
@@ -18,11 +13,8 @@ import {
 } from './util';
 
 async function updateAllLocks(
-  locks: ProviderLock[],
-  config: UpdateArtifactsConfig
+  locks: ProviderLock[]
 ): Promise<ProviderLockUpdate[]> {
-  const { cacheDir } = getAdminConfig();
-
   const updates = await pMap(
     locks,
     async (lock) => {
@@ -46,7 +38,7 @@ async function updateAllLocks(
       const update: ProviderLockUpdate = {
         newVersion,
         newConstraint: lock.constraints,
-        newHashes: await hash(lock.lookupName, newVersion, cacheDir),
+        newHashes: await createHashes(lock.lookupName, newVersion),
         ...lock,
       };
       return update;
@@ -72,8 +64,6 @@ export async function updateArtifacts({
     return null;
   }
 
-  const { cacheDir } = getAdminConfig();
-
   const lockFileContent = await readLockFile(packageFileName);
   if (!lockFileContent) {
     logger.debug('No .terraform.lock.hcl found');
@@ -88,7 +78,7 @@ export async function updateArtifacts({
   const updates: ProviderLockUpdate[] = [];
   if (config.updateType === 'lockFileMaintenance') {
     // update all locks in the file during maintenance --> only update version in constraints
-    const maintenanceUpdates = await updateAllLocks(locks, config);
+    const maintenanceUpdates = await updateAllLocks(locks);
     updates.push(...maintenanceUpdates);
   } else {
     // update only specific locks but with constrain updates
@@ -103,7 +93,7 @@ export async function updateArtifacts({
     const update: ProviderLockUpdate = {
       newVersion: config.newVersion,
       newConstraint,
-      newHashes: await hash(repository, config.newVersion, cacheDir),
+      newHashes: await createHashes(repository, config.newVersion),
       ...updateLock,
     };
     updates.push(update);

diff --git a/lib/manager/terraform/lockfile/util.ts b/lib/manager/terraform/lockfile/util.ts
@@ -1,4 +1,7 @@
-import { getSiblingFileName, readLocalFile } from '../../../util/fs';
+import { join } from 'upath';
+import { getAdminConfig } from '../../../config/admin';
+import { logger } from '../../../logger';
+import { ensureDir, getSiblingFileName, readLocalFile } from '../../../util/fs';
 import { get as getVersioning } from '../../../versioning';
 import type { UpdateArtifactsResult } from '../../types';
 import type {
@@ -207,3 +210,10 @@ export function writeLockUpdates(
     },
   };
 }
+
+export async function getCacheDir(): Promise<string> {
+  const cacheDir = join(getAdminConfig().cacheDir, './others/terraform');
+  await ensureDir(cacheDir);
+  logger.debug(`Using terraform cache: ${cacheDir}`);
+  return cacheDir;
+}