refactor(build): Pre-compile files imported from data folder

lib/manager/pip_setup/util.ts

@@ -1,5 +1,5 @@
-import { resolveFile } from '../../util';
-import { readFile, readLocalFile, writeLocalFile } from '../../util/fs';
+import dataFiles from '../../data-files.generated';
+import { readLocalFile, writeLocalFile } from '../../util/fs';
 
 // need to match filename in `data/extract.py`
 const REPORT = 'renovate-pip_setup-report.json';
@@ -9,8 +9,7 @@ let extractPy: string | undefined;
 
 export async function copyExtractFile(): Promise<string> {
   if (extractPy === undefined) {
-    const file = await resolveFile('data/extract.py');
-    extractPy = await readFile(file, 'utf8');
+    extractPy = dataFiles.get('extract.py');
   }
 
   await writeLocalFile(EXTRACT, extractPy);

lib/manager/travis/package.ts

@@ -1,9 +1,9 @@
 import is from '@sindresorhus/is';
 import equal from 'fast-deep-equal';
+import dataFiles from '../../data-files.generated';
 import { getPkgReleases } from '../../datasource';
 import * as datasourceGithubTags from '../../datasource/github-tags';
 import { logger } from '../../logger';
-import { resolveFile } from '../../util';
 import { isVersion, maxSatisfyingVersion } from '../../versioning/semver';
 import { LookupUpdate, PackageUpdateConfig } from '../common';
 
@@ -26,9 +26,10 @@ type NodeJsData = Record<string, NodeJsSchedule>;
 let policies: NodeJsPolicies;
 let refreshDate: Date;
 
-async function generatePolicies(): Promise<NodeJsData> {
-  const file = await resolveFile('data/node-js-schedule.json');
-  const nodeJsSchedule = (await import(file)) as NodeJsData;
+function generatePolicies(): NodeJsData {
+  const nodeJsSchedule = JSON.parse(
+    dataFiles.get('node-js-schedule.json')
+  ) as NodeJsData;
   policies = {
     all: [],
     lts: [],
@@ -65,11 +66,11 @@ async function generatePolicies(): Promise<NodeJsData> {
   return nodeJsSchedule;
 }
 
-async function checkPolicies(): Promise<void> {
+function checkPolicies(): void {
   if (policies && refreshDate > new Date()) {
     return;
   }
-  const nodeJsSchedule = await generatePolicies();
+  const nodeJsSchedule = generatePolicies();
   refreshDate = new Date('3000-01-01'); // y3k
   const now = new Date();
   for (const data of Object.values(nodeJsSchedule)) {
@@ -92,7 +93,7 @@ export async function getPackageUpdates(
   if (!supportPolicy?.length) {
     return [];
   }
-  await checkPolicies();
+  checkPolicies();
   for (const policy of supportPolicy) {
     if (!Object.keys(policies).includes(policy)) {
       logger.warn({ policy }, `Unknown supportPolicy`);

lib/util/index.ts

@@ -1,5 +1,3 @@
-import findUp from 'find-up';
-import { join } from 'upath';
 import { RenovateConfig } from '../config/common';
 import { setExecConfig } from './exec';
 import { setFsConfig } from './fs';
@@ -11,19 +9,6 @@ export async function setUtilConfig(
   setFsConfig(config);
 }
 
-/**
- * Resolve path for a file relative to renovate root directory (our package.json)
- * @param file a file to resolve
- */
-export async function resolveFile(file: string): Promise<string> {
-  const pkg = await findUp('package.json', { cwd: __dirname, type: 'file' });
-  // istanbul ignore if
-  if (!pkg) {
-    throw new Error('Missing package.json');
-  }
-  return join(pkg, '../', file);
-}
-
 export function sampleSize(array: string[], n: number): string[] {
   const length = array == null ? 0 : array.length;
   if (!length || n < 1) {

tools/generate-imports.ts

@@ -9,6 +9,11 @@ if (!fs.existsSync('lib')) {
   shell.exit(0);
 }
 
+if (!fs.existsSync('data')) {
+  shell.echo('> missing data folder');
+  shell.exit(0);
+}
+
 function findModules(dirname: string): string[] {
   return fs
     .readdirSync(dirname, { withFileTypes: true })
@@ -17,6 +22,7 @@ function findModules(dirname: string): string[] {
     .filter((name) => !name.startsWith('__'))
     .sort();
 }
+
 async function updateFile(file: string, code: string): Promise<void> {
   const oldCode = fs.existsSync(file) ? await fs.readFile(file, 'utf8') : null;
   if (code !== oldCode) {
@@ -44,7 +50,7 @@ async function generate({
   map?: string;
   excludes?: string[];
 }): Promise<void> {
-  shell.echo(`> ${path}`);
+  shell.echo(`> lib/${path}/`);
   let imports = '';
   let maps = '';
   for (const ds of findModules(`lib/${path}`).filter(
@@ -64,9 +70,43 @@ async function generate({
   await updateFile(`lib/${path}/api.generated.ts`, code.replace(/^\s+/gm, ''));
 }
 
+async function generateData(): Promise<void> {
+  const files = fs
+    .readdirSync('data', { withFileTypes: true })
+    .filter((dirent) => dirent.isFile())
+    .map((dirent) => dirent.name)
+    .sort();
+
+  const importDataFileType = files.map((x) => `  | '${x}'`).join('\n');
+
+  const contentMapDecl = 'const data = new Map<DataFile, string>();';
+
+  const contentMapAssignments: string[] = [];
+  for (const file of files) {
+    shell.echo(`> data/${file}`);
+    const rawFileContent = await fs.readFile(`data/${file}`, 'utf8');
+    contentMapAssignments.push(
+      `data.set('${file}', ${JSON.stringify(rawFileContent)});`
+    );
+  }
+
+  await updateFile(
+    `lib/data-files.generated.ts`,
+    [
+      `type DataFile =\n${importDataFileType};`,
+      contentMapDecl,
+      contentMapAssignments.join('\n'),
+      `export default data;\n`,
+    ].join('\n\n')
+  );
+}
+
 // eslint-disable-next-line @typescript-eslint/no-floating-promises
 (async () => {
   try {
+    // data-files
+    await generateData();
+
     // datasources
     await generate({ path: 'datasource', types: ['DatasourceApi'] });