[Code] Incremental Indexing (#33485)

* [Code] Add a git api to get diff from arbitrary 2 revisions

* [Code] Apply incremental index triggering

* [Code] implement the actual incremental indexing

* [Code] apply checkpoint validation for both lsp and lsp incremental indexer

* [Code] add unit tests

* [Code] only disable index scheduler but leave update scheduler on

x-pack/plugins/code/common/git_diff.ts

@@ -6,13 +6,16 @@
 
 import { CommitInfo } from '../model/commit';
 
-export interface CommitDiff {
-  commit: CommitInfo;
+export interface Diff {
   additions: number;
   deletions: number;
   files: FileDiff[];
 }
 
+export interface CommitDiff extends Diff {
+  commit: CommitInfo;
+}
+
 export interface FileDiff {
   path: string;
   originPath?: string;
@@ -25,8 +28,8 @@ export interface FileDiff {
 }
 
 export enum DiffKind {
-  ADDED,
-  DELETED,
-  MODIFIED,
-  RENAMED,
+  ADDED = 'ADDED',
+  DELETED = 'DELETED',
+  MODIFIED = 'MODIFIED',
+  RENAMED = 'RENAMED',
 }

x-pack/plugins/code/index.ts

@@ -66,7 +66,7 @@ export const code = (kibana: any) =>
             .default(['https', 'git', 'ssh']),
         }).default(),
         maxWorkspace: Joi.number().default(5), // max workspace folder for each language server
-        disableScheduler: Joi.boolean().default(true), // Temp option to disable all schedulers.
+        disableIndexScheduler: Joi.boolean().default(true), // Temp option to disable index scheduler.
         enableGlobalReference: Joi.boolean().default(false), // Global reference as optional feature for now
         codeNode: Joi.boolean().default(false),
       }).default();

x-pack/plugins/code/model/repository.ts

@@ -22,6 +22,8 @@ export interface Repository {
   nextUpdateTimestamp?: Date;
   // The timestamp of next index for this repository.
   nextIndexTimestamp?: Date;
+  // The current indexed revision in Elasticsearch.
+  indexedRevision?: string;
 }
 
 export interface RepositoryConfig {
@@ -88,9 +90,12 @@ export interface UpdateWorkerResult extends WorkerResult {
 }
 
 export enum IndexStatsKey {
-  File = 'file-count',
-  Symbol = 'symbol-count',
-  Reference = 'reference-count',
+  File = 'file-added-count',
+  FileDeleted = 'file-deleted-count',
+  Symbol = 'symbol-added-count',
+  SymbolDeleted = 'symbol-deleted-count',
+  Reference = 'reference-added-count',
+  ReferenceDeleted = 'reference-deleted-count',
 }
 export type IndexStats = Map<IndexStatsKey, number>;
 

x-pack/plugins/code/model/search.ts

@@ -7,6 +7,7 @@
 import { DetailSymbolInformation } from '@elastic/lsp-extension';
 import { IRange } from 'monaco-editor';
 
+import { DiffKind } from '../common/git_diff';
 import { Repository, SourceHit } from '../model';
 import { RepositoryUri } from './repository';
 
@@ -31,6 +32,12 @@ export interface LspIndexRequest extends IndexRequest {
   revision: string; // The revision of the current repository
 }
 
+export interface LspIncIndexRequest extends LspIndexRequest {
+  originPath?: string;
+  kind: DiffKind;
+  originRevision: string;
+}
+
 // The request for RepositoryIndexer
 export interface RepositoryIndexRequest extends IndexRequest {
   repoUri: RepositoryUri;

x-pack/plugins/code/server/__tests__/git_operations.ts

@@ -100,4 +100,36 @@ describe('git_operations', () => {
     assert.strictEqual(count, 3, 'this repo should contains exactly 2 files');
     assert.strictEqual(totalFiles, 3, 'this repo should contains exactly 2 files');
   });
+
+  it('get diff between arbitrary 2 revisions', async () => {
+    function cloneProject(url: string, p: string) {
+      return new Promise(resolve => {
+        if (!fs.existsSync(p)) {
+          rimraf(p, error => {
+            Git.Clone.clone(url, p).then(repo => {
+              resolve(repo);
+            });
+          });
+        } else {
+          resolve();
+        }
+      });
+    }
+
+    await cloneProject(
+      'https://github.com/Microsoft/TypeScript-Node-Starter.git',
+      path.join(serverOptions.repoPath, 'github.com/Microsoft/TypeScript-Node-Starter')
+    );
+
+    const g = new GitOperations(serverOptions.repoPath);
+    const d = await g.getDiff(
+      'github.com/Microsoft/TypeScript-Node-Starter',
+      '6206f643',
+      '4779cb7e'
+    );
+    assert.equal(d.additions, 2);
+    assert.equal(d.deletions, 4);
+    assert.equal(d.files.length, 3);
+    // @ts-ignore
+  }).timeout(100000);
 });