bug:SP-2006 Fixes bug on venv indexing

src/main/modules/searchEngine/indexer/Indexer.ts

@@ -1,5 +1,6 @@
 import fs from 'fs';
 import log from 'electron-log';
+import { getHeapStatistics } from 'node:v8';
 import { IIndexer } from './IIndexer';
 import { IpcChannels } from '../../../../api/ipc-channels';
 import { getSearchConfig } from '../../../../shared/utils/search-utils';
@@ -8,7 +9,27 @@ import { broadcastManager } from '../../../broadcastManager/BroadcastManager';
 const { Index } = require('flexsearch');
 
 export class Indexer {
-  public index(files: Array<IIndexer>) {
+  private MAX_FILE_SIZE_MB = 100;
+
+  private shouldStopIndexing(): boolean {
+    const HEAP_BUFFER_MB = 200;
+    const MAX_HEAP_SIZE_MB = getHeapStatistics().heap_size_limit / (1024 * 1024);
+    const currentHeapMB = process.memoryUsage().heapUsed / (1024 * 1024);
+
+    return (MAX_HEAP_SIZE_MB - currentHeapMB) < HEAP_BUFFER_MB;
+  }
+
+  private async getFileSizeMB(path: string): Promise<number> {
+    try {
+      const stats = await fs.promises.stat(path);
+      return stats.size / (1024 * 1024);
+    } catch (e) {
+      console.error(`Error getting file size for ${path}:`, e);
+      return 0;
+    }
+  }
+
+  public async index(files: Array<IIndexer>) {
     const index = new Index(getSearchConfig());
     for (let i = 0; i < files.length; i += 1) {
       try {
@@ -17,8 +38,21 @@ export class Indexer {
             processed: i * 100 / files.length,
           });
         }
-        const fileContent = fs.readFileSync(files[i].path, 'utf-8');
-        index.add(files[i].fileId, fileContent);
+
+        // Check file size first
+        const fileSizeMB = await this.getFileSizeMB(files[i].path);
+        if (fileSizeMB > this.MAX_FILE_SIZE_MB) {
+          console.warn(`Skipping large file: ${files[i].path} (${fileSizeMB.toFixed(2)}MB)`);
+          // eslint-disable-next-line no-continue
+          continue;
+        }
+
+        if (this.shouldStopIndexing()) {
+          log.info('Skipping file indexing, maximum heap size exceeded');
+        } else {
+          const fileContent = fs.readFileSync(files[i].path, 'utf-8');
+          index.add(files[i].fileId, fileContent);
+        }
       } catch (e) {
         log.error(e);
       }