From 4cb0ed12899bf7da308384b413ff446bf074423d Mon Sep 17 00:00:00 2001
From: Mengwei Ding <mengwei.ding@gmail.com>
Date: Tue, 19 Mar 2019 17:10:09 +0800
Subject: [PATCH 1/6] [Code] Add a git api to get diff from arbitrary 2
 revisions

---
 x-pack/plugins/code/common/git_diff.ts        |  7 +-
 .../code/server/__tests__/git_operations.ts   | 25 +++++++
 x-pack/plugins/code/server/git_operations.ts  | 62 +++++++++++++++++-
 .../server/indexer/lsp_incremental_indexer.ts | 65 +++++++++++++++++++
 .../code/server/indexer/lsp_indexer.ts        |  2 +-
 5 files changed, 157 insertions(+), 4 deletions(-)
 create mode 100644 x-pack/plugins/code/server/indexer/lsp_incremental_indexer.ts

diff --git a/x-pack/plugins/code/common/git_diff.ts b/x-pack/plugins/code/common/git_diff.ts
index 69bf01099e80..24600a7487d7 100644
--- a/x-pack/plugins/code/common/git_diff.ts
+++ b/x-pack/plugins/code/common/git_diff.ts
@@ -6,13 +6,16 @@
 
 import { CommitInfo } from '../model/commit';
 
-export interface CommitDiff {
-  commit: CommitInfo;
+export interface Diff {
   additions: number;
   deletions: number;
   files: FileDiff[];
 }
 
+export interface CommitDiff extends Diff {
+  commit: CommitInfo;
+}
+
 export interface FileDiff {
   path: string;
   originPath?: string;
diff --git a/x-pack/plugins/code/server/__tests__/git_operations.ts b/x-pack/plugins/code/server/__tests__/git_operations.ts
index 44f7676257a1..d05c94066335 100644
--- a/x-pack/plugins/code/server/__tests__/git_operations.ts
+++ b/x-pack/plugins/code/server/__tests__/git_operations.ts
@@ -100,4 +100,29 @@ describe('git_operations', () => {
     assert.strictEqual(count, 3, 'this repo should contains exactly 2 files');
     assert.strictEqual(totalFiles, 3, 'this repo should contains exactly 2 files');
   });
+
+  it('get any diff', async () => {
+    function cloneProject(url: string, p: string) {
+      return new Promise(resolve => {
+        if (!fs.existsSync(p)) {
+          rimraf(p, error => {
+            Git.Clone.clone(url, p).then(repo => {
+              resolve(repo);
+            });
+          });
+        } else {
+          resolve();
+        }
+      });
+    }
+
+    await cloneProject(
+      'https://github.com/Microsoft/TypeScript-Node-Starter.git',
+      path.join(serverOptions.repoPath, 'github.com/Microsoft/TypeScript-Node-Starter')
+    );
+
+    const g = new GitOperations(serverOptions.repoPath);
+    const d = await g.getDiff('github.com/Microsoft/TypeScript-Node-Starter', '6206f6431e75b0e98506a356fb2ded08ab0f0c89', '4779cb7e182cf41d5c62289bb80d2850e0265b71');
+  // @ts-ignore
+  }).timeout(100000);
 });
diff --git a/x-pack/plugins/code/server/git_operations.ts b/x-pack/plugins/code/server/git_operations.ts
index 683a0ca6d474..10061d851466 100644
--- a/x-pack/plugins/code/server/git_operations.ts
+++ b/x-pack/plugins/code/server/git_operations.ts
@@ -7,6 +7,7 @@
 import {
   Blame,
   Commit,
+  Diff as NodeGitDiff,
   Error,
   Object,
   Oid,
@@ -18,7 +19,7 @@ import {
 import Boom from 'boom';
 import * as Path from 'path';
 import { GitBlame } from '../common/git_blame';
-import { CommitDiff, DiffKind } from '../common/git_diff';
+import { CommitDiff, Diff, DiffKind } from '../common/git_diff';
 import { FileTree, FileTreeItemType, RepositoryUri, sortFileTree } from '../model';
 import { CommitInfo, ReferenceInfo, ReferenceType } from '../model/commit';
 import { detectLanguage } from './utils/detect_language';
@@ -336,6 +337,65 @@ export class GitOperations {
     return commitDiff;
   }
 
+  public async getDiff(uri: string, oldRevision: string, newRevision: string): Promise<Diff> {
+    const repo = await this.openRepo(uri);
+    const oldCommit = await this.getCommit(repo, oldRevision);
+    const newCommit = await this.getCommit(repo, newRevision);
+    const oldTree = await oldCommit.getTree();
+    const newTree = await newCommit.getTree();
+
+    const diff = await NodeGitDiff.treeToTree(repo, oldTree, newTree);
+
+    const res: Diff = {
+      additions: 0,
+      deletions: 0,
+      files: [],
+    };
+    const patches = await diff.patches();
+    for (const patch of patches) {
+      const { total_deletions, total_additions } = patch.lineStats();
+      res.additions += total_additions;
+      res.deletions += total_deletions;
+      if (patch.isAdded()) {
+        const path = patch.newFile().path();
+        res.files.push({
+          path,
+          additions: total_additions,
+          deletions: total_deletions,
+          kind: DiffKind.ADDED,
+        });
+      } else if (patch.isDeleted()) {
+        const path = patch.oldFile().path();
+        res.files.push({
+          path,
+          kind: DiffKind.DELETED,
+          additions: total_additions,
+          deletions: total_deletions,
+        });
+      } else if (patch.isModified()) {
+        const path = patch.newFile().path();
+        const originPath = patch.oldFile().path();
+        res.files.push({
+          path,
+          originPath,
+          kind: DiffKind.MODIFIED,
+          additions: total_additions,
+          deletions: total_deletions,
+        });
+      } else if (patch.isRenamed()) {
+        const path = patch.newFile().path();
+        res.files.push({
+          path,
+          originPath: patch.oldFile().path(),
+          kind: DiffKind.RENAMED,
+          additions: total_additions,
+          deletions: total_deletions,
+        });
+      }
+    }
+    return res;
+  }
+
   private async getOriginCode(commit: Commit, repo: Repository, path: string) {
     for (const oid of commit.parents()) {
       const parentCommit = await repo.getCommit(oid);
diff --git a/x-pack/plugins/code/server/indexer/lsp_incremental_indexer.ts b/x-pack/plugins/code/server/indexer/lsp_incremental_indexer.ts
new file mode 100644
index 000000000000..4c455eb1a93e
--- /dev/null
+++ b/x-pack/plugins/code/server/indexer/lsp_incremental_indexer.ts
@@ -0,0 +1,65 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import fs from 'fs';
+import util from 'util';
+
+import { ProgressReporter } from '.';
+import { toCanonicalUrl } from '../../common/uri_util';
+import { Document, IndexStats, IndexStatsKey, LspIndexRequest, RepositoryUri } from '../../model';
+import { GitOperations } from '../git_operations';
+import { EsClient } from '../lib/esqueue';
+import { Logger } from '../log';
+import { LspService } from '../lsp/lsp_service';
+import { ServerOptions } from '../server_options';
+import { detectLanguage, detectLanguageByFilename } from '../utils/detect_language';
+import {
+  getDocumentIndexCreationRequest,
+  getReferenceIndexCreationRequest,
+  getSymbolIndexCreationRequest,
+} from './index_creation_request';
+import { LspIndexer } from './lsp_indexer';
+import { ALL_RESERVED, DocumentIndexName, ReferenceIndexName, SymbolIndexName } from './schema';
+
+export class LspIncrementalIndexer extends LspIndexer {
+  protected type: string = 'lsp_inc';
+
+  constructor(
+    protected readonly repoUri: RepositoryUri,
+    protected readonly revision: string,
+    protected readonly lspService: LspService,
+    protected readonly options: ServerOptions,
+    protected readonly client: EsClient,
+    protected readonly log: Logger
+  ) {
+    super(repoUri, revision, lspService, options, client, log);
+  }
+
+  protected async prepareIndexCreationRequests() {
+    // We don't need to create new indices for incremental indexing.
+    return [];
+  }
+
+  protected async *getIndexRequestIterator(): AsyncIterableIterator<LspIndexRequest> {
+    // TODO: implement this
+    yield* super.getIndexRequestIterator();
+  }
+
+  protected async getIndexRequestCount(): Promise<number> {
+    try {
+      const gitOperator = new GitOperations(this.options.repoPath);
+      return await gitOperator.countRepoFiles(this.repoUri, 'head');
+    } catch (error) {
+      this.log.error(`Get lsp index requests count error.`);
+      this.log.error(error);
+      throw error;
+    }
+  }
+
+  protected async cleanIndex() {
+    this.log.info('Do not need to clean index for incremental indexing.');
+  }
+}
diff --git a/x-pack/plugins/code/server/indexer/lsp_indexer.ts b/x-pack/plugins/code/server/indexer/lsp_indexer.ts
index 3f9d252953b4..dbfadb01f3ef 100644
--- a/x-pack/plugins/code/server/indexer/lsp_indexer.ts
+++ b/x-pack/plugins/code/server/indexer/lsp_indexer.ts
@@ -27,7 +27,7 @@ import { ALL_RESERVED, DocumentIndexName, ReferenceIndexName, SymbolIndexName }
 
 export class LspIndexer extends AbstractIndexer {
   protected type: string = 'lsp';
-  private batchIndexHelper: BatchIndexHelper;
+  protected batchIndexHelper: BatchIndexHelper;
 
   constructor(
     protected readonly repoUri: RepositoryUri,

From d00107287c6efe6ce2594c1d5b90d9ac300aa71f Mon Sep 17 00:00:00 2001
From: Mengwei Ding <mengwei.ding@gmail.com>
Date: Wed, 20 Mar 2019 15:37:27 +0800
Subject: [PATCH 2/6] [Code] Apply incremental index triggering

---
 x-pack/plugins/code/index.ts                  |   6 +-
 x-pack/plugins/code/model/repository.ts       |   2 +
 x-pack/plugins/code/model/search.ts           |   7 +
 x-pack/plugins/code/server/indexer/index.ts   |   1 +
 x-pack/plugins/code/server/indexer/indexer.ts |   2 +-
 .../server/indexer/lsp_incremental_indexer.ts | 145 ++++++++++++++++--
 .../server/indexer/lsp_indexer_factory.ts     |  44 +++++-
 .../repository_index_initializer_factory.ts   |   2 +-
 .../code/server/indexer/schema/document.ts    |   3 +
 .../code/server/queue/abstract_git_worker.ts  |  21 ++-
 .../code/server/queue/abstract_worker.ts      |  12 +-
 .../plugins/code/server/queue/clone_worker.ts |   3 +-
 .../code/server/queue/delete_worker.ts        |  17 +-
 .../plugins/code/server/queue/index_worker.ts |  25 ++-
 x-pack/plugins/code/server/queue/worker.ts    |   2 +-
 .../plugins/code/server/routes/repository.ts  |   4 +-
 .../code/server/scheduler/update_scheduler.ts |  29 ++--
 x-pack/plugins/code/server/server_options.ts  |   2 +-
 18 files changed, 257 insertions(+), 70 deletions(-)

diff --git a/x-pack/plugins/code/index.ts b/x-pack/plugins/code/index.ts
index 7c17e89cc15b..1f7beaf60609 100644
--- a/x-pack/plugins/code/index.ts
+++ b/x-pack/plugins/code/index.ts
@@ -35,11 +35,11 @@ export const code = (kibana: any) =>
         // 1 hour by default.
         queueTimeout: Joi.number().default(moment.duration(1, 'hour').asMilliseconds()),
         // The frequency which update scheduler executes. 5 minutes by default.
-        updateFrequencyMs: Joi.number().default(moment.duration(5, 'minute').asMilliseconds()),
+        updateFrequencyMs: Joi.number().default(moment.duration(10, 'second').asMilliseconds()),
         // The frequency which index scheduler executes. 1 day by default.
         indexFrequencyMs: Joi.number().default(moment.duration(1, 'day').asMilliseconds()),
         // The frequency which each repo tries to update. 1 hour by default.
-        updateRepoFrequencyMs: Joi.number().default(moment.duration(1, 'hour').asMilliseconds()),
+        updateRepoFrequencyMs: Joi.number().default(moment.duration(1, 'minute').asMilliseconds()),
         // The frequency which each repo tries to index. 1 day by default.
         indexRepoFrequencyMs: Joi.number().default(moment.duration(1, 'day').asMilliseconds()),
         lsp: Joi.object({
@@ -69,7 +69,7 @@ export const code = (kibana: any) =>
             .default(['https', 'git']),
         }).default(),
         maxWorkspace: Joi.number().default(5), // max workspace folder for each language server
-        disableScheduler: Joi.boolean().default(true), // Temp option to disable all schedulers.
+        disableScheduler: Joi.boolean().default(false), // Temp option to disable all schedulers.
         enableGlobalReference: Joi.boolean().default(false), // Global reference as optional feature for now
         codeNode: Joi.boolean().default(false),
       }).default();
diff --git a/x-pack/plugins/code/model/repository.ts b/x-pack/plugins/code/model/repository.ts
index 378b93ba93f9..dc7ddecf3a52 100644
--- a/x-pack/plugins/code/model/repository.ts
+++ b/x-pack/plugins/code/model/repository.ts
@@ -21,6 +21,8 @@ export interface Repository {
   nextUpdateTimestamp?: Date;
   // The timestamp of next index for this repository.
   nextIndexTimestamp?: Date;
+  // The current indexed revision in Elasticsearch.
+  indexedRevision?: string;
 }
 
 export interface RepositoryConfig {
diff --git a/x-pack/plugins/code/model/search.ts b/x-pack/plugins/code/model/search.ts
index 706ea0b476c3..cee855b0980f 100644
--- a/x-pack/plugins/code/model/search.ts
+++ b/x-pack/plugins/code/model/search.ts
@@ -7,6 +7,7 @@
 import { DetailSymbolInformation } from '@elastic/lsp-extension';
 import { IRange } from 'monaco-editor';
 
+import { DiffKind } from '../common/git_diff';
 import { Repository, SourceHit } from '../model';
 import { RepositoryUri } from './repository';
 
@@ -31,6 +32,12 @@ export interface LspIndexRequest extends IndexRequest {
   revision: string; // The revision of the current repository
 }
 
+export interface LspIncIndexRequest extends LspIndexRequest {
+  originPath?: string;
+  kind: DiffKind;
+  originRevision: string;
+}
+
 // The request for RepositoryIndexer
 export interface RepositoryIndexRequest extends IndexRequest {
   repoUri: RepositoryUri;
diff --git a/x-pack/plugins/code/server/indexer/index.ts b/x-pack/plugins/code/server/indexer/index.ts
index a7e057733ab4..e25d60f7aa21 100644
--- a/x-pack/plugins/code/server/indexer/index.ts
+++ b/x-pack/plugins/code/server/indexer/index.ts
@@ -4,6 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
+export * from './lsp_incremental_indexer';
 export * from './lsp_indexer';
 export * from './lsp_indexer_factory';
 export * from './indexer';
diff --git a/x-pack/plugins/code/server/indexer/indexer.ts b/x-pack/plugins/code/server/indexer/indexer.ts
index fa2044925552..bdaeaba8f506 100644
--- a/x-pack/plugins/code/server/indexer/indexer.ts
+++ b/x-pack/plugins/code/server/indexer/indexer.ts
@@ -14,5 +14,5 @@ export interface Indexer {
 }
 
 export interface IndexerFactory {
-  create(repoUri: RepositoryUri, revision: string): Indexer;
+  create(repoUri: RepositoryUri, revision: string): Promise<Indexer | undefined>;
 }
diff --git a/x-pack/plugins/code/server/indexer/lsp_incremental_indexer.ts b/x-pack/plugins/code/server/indexer/lsp_incremental_indexer.ts
index 4c455eb1a93e..2afbb0754442 100644
--- a/x-pack/plugins/code/server/indexer/lsp_incremental_indexer.ts
+++ b/x-pack/plugins/code/server/indexer/lsp_incremental_indexer.ts
@@ -7,29 +7,34 @@
 import fs from 'fs';
 import util from 'util';
 
-import { ProgressReporter } from '.';
+import { Diff, DiffKind } from '../../common/git_diff';
 import { toCanonicalUrl } from '../../common/uri_util';
-import { Document, IndexStats, IndexStatsKey, LspIndexRequest, RepositoryUri } from '../../model';
+import {
+  Document,
+  IndexStats,
+  IndexStatsKey,
+  LspIncIndexRequest,
+  RepositoryUri,
+} from '../../model';
 import { GitOperations } from '../git_operations';
 import { EsClient } from '../lib/esqueue';
 import { Logger } from '../log';
 import { LspService } from '../lsp/lsp_service';
 import { ServerOptions } from '../server_options';
-import { detectLanguage, detectLanguageByFilename } from '../utils/detect_language';
-import {
-  getDocumentIndexCreationRequest,
-  getReferenceIndexCreationRequest,
-  getSymbolIndexCreationRequest,
-} from './index_creation_request';
+import { detectLanguage } from '../utils/detect_language';
 import { LspIndexer } from './lsp_indexer';
-import { ALL_RESERVED, DocumentIndexName, ReferenceIndexName, SymbolIndexName } from './schema';
+import { DocumentIndexName, ReferenceIndexName, SymbolIndexName } from './schema';
 
 export class LspIncrementalIndexer extends LspIndexer {
   protected type: string = 'lsp_inc';
+  private diff: Diff | undefined = undefined;
 
   constructor(
     protected readonly repoUri: RepositoryUri,
+    // The latest revision to be indexed
     protected readonly revision: string,
+    // The already indexed revision
+    protected readonly originRevision: string,
     protected readonly lspService: LspService,
     protected readonly options: ServerOptions,
     protected readonly client: EsClient,
@@ -43,17 +48,73 @@ export class LspIncrementalIndexer extends LspIndexer {
     return [];
   }
 
-  protected async *getIndexRequestIterator(): AsyncIterableIterator<LspIndexRequest> {
-    // TODO: implement this
-    yield* super.getIndexRequestIterator();
+  protected async processRequest(request: LspIncIndexRequest): Promise<IndexStats> {
+    const stats: IndexStats = new Map<IndexStatsKey, number>()
+      .set(IndexStatsKey.Symbol, 0)
+      .set(IndexStatsKey.Reference, 0)
+      .set(IndexStatsKey.File, 0);
+    const { kind } = request;
+
+    switch (kind) {
+      case DiffKind.ADDED: {
+        this.log.debug(`Index ADDED file`);
+        await this.handleAddedRequest(request, stats);
+      }
+      case DiffKind.DELETED: {
+        this.log.debug(`Index DELETED file`);
+        // TODO: implement delete. We need the encode document id for delete now.
+      }
+      case DiffKind.MODIFIED: {
+        this.log.debug(`Index MODIFYED file`);
+        // TODO: implement modified
+      }
+      case DiffKind.RENAMED: {
+        this.log.debug(`Index RENAMED file`);
+        // TODO: implement renamed
+      }
+      default: {
+        this.log.debug(`Unsupported diff kind ${kind} for incremental indexing.`);
+      }
+    }
+
+    return stats;
+  }
+
+  protected async *getIndexRequestIterator(): AsyncIterableIterator<LspIncIndexRequest> {
+    try {
+      const {
+        workspaceRepo,
+        workspaceRevision,
+      } = await this.lspService.workspaceHandler.openWorkspace(this.repoUri, 'head');
+      const workspaceDir = workspaceRepo.workdir();
+      if (this.diff) {
+        for (const f of this.diff.files) {
+          yield {
+            repoUri: this.repoUri,
+            localRepoPath: workspaceDir,
+            filePath: f.path,
+            originPath: f.originPath,
+            revision: workspaceRevision,
+            kind: f.kind,
+            originRevision: this.originRevision,
+          };
+        }
+      }
+    } catch (error) {
+      this.log.error(`Get lsp incremental index requests count error.`);
+      this.log.error(error);
+      throw error;
+    }
   }
 
   protected async getIndexRequestCount(): Promise<number> {
     try {
       const gitOperator = new GitOperations(this.options.repoPath);
-      return await gitOperator.countRepoFiles(this.repoUri, 'head');
+      // cache here to avoid pulling the diff twice.
+      this.diff = await gitOperator.getDiff(this.repoUri, this.originRevision, this.revision);
+      return this.diff.files.length;
     } catch (error) {
-      this.log.error(`Get lsp index requests count error.`);
+      this.log.error(`Get lsp incremental index requests count error.`);
       this.log.error(error);
       throw error;
     }
@@ -62,4 +123,60 @@ export class LspIncrementalIndexer extends LspIndexer {
   protected async cleanIndex() {
     this.log.info('Do not need to clean index for incremental indexing.');
   }
+
+  private async handleAddedRequest(request: LspIncIndexRequest, stats: IndexStats) {
+    const { repoUri, revision, filePath, localRepoPath } = request;
+
+    const lspDocUri = toCanonicalUrl({ repoUri, revision, file: filePath, schema: 'git:' });
+    const symbolNames = new Set<string>();
+
+    try {
+      const response = await this.lspService.sendRequest('textDocument/full', {
+        textDocument: {
+          uri: lspDocUri,
+        },
+        reference: this.options.enableGlobalReference,
+      });
+
+      if (response && response.result.length > 0) {
+        const { symbols, references } = response.result[0];
+        for (const symbol of symbols) {
+          await this.batchIndexHelper.index(SymbolIndexName(repoUri), symbol);
+          symbolNames.add(symbol.symbolInformation.name);
+        }
+        stats.set(IndexStatsKey.Symbol, symbols.length);
+
+        for (const ref of references) {
+          await this.batchIndexHelper.index(ReferenceIndexName(repoUri), ref);
+        }
+        stats.set(IndexStatsKey.Reference, references.length);
+      } else {
+        this.log.debug(`Empty response from lsp server. Skip symbols and references indexing.`);
+      }
+    } catch (error) {
+      this.log.error(`Index symbols or references error. Skip to file indexing.`);
+      this.log.error(error);
+    }
+
+    const localFilePath = `${localRepoPath}${filePath}`;
+    const lstat = util.promisify(fs.lstat);
+    const stat = await lstat(localFilePath);
+
+    const readLink = util.promisify(fs.readlink);
+    const readFile = util.promisify(fs.readFile);
+    const content = stat.isSymbolicLink()
+      ? await readLink(localFilePath, 'utf8')
+      : await readFile(localFilePath, 'utf8');
+
+    const language = await detectLanguage(filePath, Buffer.from(content));
+    const body: Document = {
+      repoUri,
+      path: filePath,
+      content,
+      language,
+      qnames: Array.from(symbolNames),
+    };
+    await this.batchIndexHelper.index(DocumentIndexName(repoUri), body);
+    stats.set(IndexStatsKey.File, 1);
+  }
 }
diff --git a/x-pack/plugins/code/server/indexer/lsp_indexer_factory.ts b/x-pack/plugins/code/server/indexer/lsp_indexer_factory.ts
index 2e207d6ee042..8b1b5688f9bc 100644
--- a/x-pack/plugins/code/server/indexer/lsp_indexer_factory.ts
+++ b/x-pack/plugins/code/server/indexer/lsp_indexer_factory.ts
@@ -4,22 +4,58 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { Indexer, IndexerFactory, LspIndexer } from '.';
+import { Indexer, IndexerFactory, LspIncrementalIndexer, LspIndexer } from '.';
 import { RepositoryUri } from '../../model';
 import { EsClient } from '../lib/esqueue';
 import { Logger } from '../log';
 import { LspService } from '../lsp/lsp_service';
+import { RepositoryObjectClient } from '../search';
 import { ServerOptions } from '../server_options';
 
 export class LspIndexerFactory implements IndexerFactory {
+  private objectClient: RepositoryObjectClient;
+
   constructor(
     protected readonly lspService: LspService,
     protected readonly options: ServerOptions,
     protected readonly client: EsClient,
     protected readonly log: Logger
-  ) {}
+  ) {
+    this.objectClient = new RepositoryObjectClient(this.client);
+  }
 
-  public create(repoUri: RepositoryUri, revision: string): Indexer {
-    return new LspIndexer(repoUri, revision, this.lspService, this.options, this.client, this.log);
+  public async create(repoUri: RepositoryUri, revision: string): Promise<Indexer | undefined> {
+    try {
+      const repo = await this.objectClient.getRepository(repoUri);
+      const indexedRevision = repo.indexedRevision;
+      if (indexedRevision) {
+        this.log.info(`Create indexer to index ${repoUri} from ${indexedRevision} to ${revision}`);
+        // Create the indexer to index only the diff between these 2 revisions.
+        return new LspIncrementalIndexer(
+          repoUri,
+          revision,
+          indexedRevision,
+          this.lspService,
+          this.options,
+          this.client,
+          this.log
+        );
+      } else {
+        this.log.info(`Create indexer to index ${repoUri} at ${revision}`);
+        // Create the indexer to index the entire repository.
+        return new LspIndexer(
+          repoUri,
+          revision,
+          this.lspService,
+          this.options,
+          this.client,
+          this.log
+        );
+      }
+    } catch (error) {
+      this.log.error(`Create indexer error for ${repoUri}.`);
+      this.log.error(error);
+      return undefined;
+    }
   }
 }
diff --git a/x-pack/plugins/code/server/indexer/repository_index_initializer_factory.ts b/x-pack/plugins/code/server/indexer/repository_index_initializer_factory.ts
index 2b868469b0fe..5797b06b1f79 100644
--- a/x-pack/plugins/code/server/indexer/repository_index_initializer_factory.ts
+++ b/x-pack/plugins/code/server/indexer/repository_index_initializer_factory.ts
@@ -12,7 +12,7 @@ import { Logger } from '../log';
 export class RepositoryIndexInitializerFactory implements IndexerFactory {
   constructor(protected readonly client: EsClient, protected readonly log: Logger) {}
 
-  public create(repoUri: RepositoryUri, revision: string): Indexer {
+  public async create(repoUri: RepositoryUri, revision: string): Promise<Indexer | undefined> {
     return new RepositoryIndexInitializer(repoUri, revision, this.client, this.log);
   }
 }
diff --git a/x-pack/plugins/code/server/indexer/schema/document.ts b/x-pack/plugins/code/server/indexer/schema/document.ts
index 97d706e92229..944fc879e4bd 100644
--- a/x-pack/plugins/code/server/indexer/schema/document.ts
+++ b/x-pack/plugins/code/server/indexer/schema/document.ts
@@ -61,6 +61,9 @@ export const DocumentSchema = {
       revision: {
         type: 'keyword',
       },
+      indexedRevision: {
+        type: 'keyword',
+      },
     },
   },
   repository_config: {
diff --git a/x-pack/plugins/code/server/queue/abstract_git_worker.ts b/x-pack/plugins/code/server/queue/abstract_git_worker.ts
index 475be820f817..eb740741d202 100644
--- a/x-pack/plugins/code/server/queue/abstract_git_worker.ts
+++ b/x-pack/plugins/code/server/queue/abstract_git_worker.ts
@@ -9,7 +9,6 @@ import {
   CloneProgress,
   CloneWorkerProgress,
   CloneWorkerResult,
-  RepositoryUri,
   WorkerReservedProgress,
 } from '../../model';
 import { getDefaultBranch, getHeadRevision } from '../git_operations';
@@ -35,6 +34,8 @@ export abstract class AbstractGitWorker extends AbstractWorker {
   }
 
   public async onJobCompleted(job: Job, res: CloneWorkerResult) {
+    await super.onJobCompleted(job, res);
+
     // Update the default branch.
     const repoUri = res.uri;
     const localPath = RepositoryUtils.repositoryLocalPath(this.serverOptions.repoPath, repoUri);
@@ -42,10 +43,15 @@ export abstract class AbstractGitWorker extends AbstractWorker {
     const defaultBranch = await getDefaultBranch(localPath);
 
     // Update the repository data.
-    await this.objectClient.updateRepository(repoUri, {
-      defaultBranch,
-      revision,
-    });
+    try {
+      await this.objectClient.updateRepository(repoUri, {
+        defaultBranch,
+        revision,
+      });
+    } catch (error) {
+      this.log.error(`Update repository default branch and revision error.`);
+      this.log.error(error);
+    }
 
     // Update the git operation status.
     try {
@@ -60,11 +66,10 @@ export abstract class AbstractGitWorker extends AbstractWorker {
       this.log.error(`Update revision of repo clone done error.`);
       this.log.error(error);
     }
-
-    return await super.onJobCompleted(job, res);
   }
 
-  public async updateProgress(uri: RepositoryUri, progress: number, cloneProgress?: CloneProgress) {
+  public async updateProgress(job: Job, progress: number, cloneProgress?: CloneProgress) {
+    const { uri } = job.payload;
     const p: CloneWorkerProgress = {
       uri,
       progress,
diff --git a/x-pack/plugins/code/server/queue/abstract_worker.ts b/x-pack/plugins/code/server/queue/abstract_worker.ts
index 45bce3eb7bfe..fa628007c98f 100644
--- a/x-pack/plugins/code/server/queue/abstract_worker.ts
+++ b/x-pack/plugins/code/server/queue/abstract_worker.ts
@@ -103,16 +103,16 @@ export abstract class AbstractWorker implements Worker {
 
   public async onJobEnqueued(job: Job) {
     this.log.info(`${this.id} job enqueued with details ${JSON.stringify(job)}`);
-    return await this.updateProgress(job.payload.uri, WorkerReservedProgress.INIT);
+    return await this.updateProgress(job, WorkerReservedProgress.INIT);
   }
 
-  public async onJobCompleted(job: Job, res: WorkerResult) {
+  public async onJobCompleted(job: Job, res: any) {
     this.log.info(
       `${this.id} job completed with result ${JSON.stringify(
         res
       )} in ${this.workerTaskDurationSeconds(job)} seconds.`
     );
-    return await this.updateProgress(res.uri, WorkerReservedProgress.COMPLETED);
+    return await this.updateProgress(job, WorkerReservedProgress.COMPLETED);
   }
 
   public async onJobExecutionError(res: any) {
@@ -121,7 +121,7 @@ export abstract class AbstractWorker implements Worker {
         res.job
       )} seconds.`
     );
-    return await this.updateProgress(res.job.payload.uri, WorkerReservedProgress.ERROR);
+    return await this.updateProgress(res.job, WorkerReservedProgress.ERROR);
   }
 
   public async onJobTimeOut(res: any) {
@@ -130,10 +130,10 @@ export abstract class AbstractWorker implements Worker {
         res.job
       )} seconds.`
     );
-    return await this.updateProgress(res.job.payload.uri, WorkerReservedProgress.TIMEOUT);
+    return await this.updateProgress(res.job, WorkerReservedProgress.TIMEOUT);
   }
 
-  public async updateProgress(uri: string, progress: number) {
+  public async updateProgress(job: Job, progress: number) {
     // This is an abstract class. Do nothing here. You should override this.
     return new Promise<WorkerResult>((resolve, _) => {
       resolve();
diff --git a/x-pack/plugins/code/server/queue/clone_worker.ts b/x-pack/plugins/code/server/queue/clone_worker.ts
index 210c3f9352b4..b77b7e40e9cc 100644
--- a/x-pack/plugins/code/server/queue/clone_worker.ts
+++ b/x-pack/plugins/code/server/queue/clone_worker.ts
@@ -57,7 +57,8 @@ export class CloneWorker extends AbstractGitWorker {
     const repoService = this.repoServiceFactory.newInstance(this.serverOptions.repoPath, this.log);
     const repo = RepositoryUtils.buildRepository(url);
     return await repoService.clone(repo, (progress: number, cloneProgress?: CloneProgress) => {
-      this.updateProgress(repo.uri, progress, cloneProgress);
+      job.payload.uri = repo.uri;
+      this.updateProgress(job, progress, cloneProgress);
     });
   }
 
diff --git a/x-pack/plugins/code/server/queue/delete_worker.ts b/x-pack/plugins/code/server/queue/delete_worker.ts
index a3a3871a6fc7..1a4cea4500fa 100644
--- a/x-pack/plugins/code/server/queue/delete_worker.ts
+++ b/x-pack/plugins/code/server/queue/delete_worker.ts
@@ -7,7 +7,7 @@
 import moment from 'moment';
 
 import { RepositoryUri, WorkerReservedProgress } from '../../model';
-import { WorkerProgress, WorkerResult } from '../../model/repository';
+import { WorkerProgress } from '../../model/repository';
 import { DocumentIndexName, ReferenceIndexName, SymbolIndexName } from '../indexer/schema';
 import { EsClient, Esqueue } from '../lib/esqueue';
 import { Logger } from '../log';
@@ -107,21 +107,16 @@ export class DeleteWorker extends AbstractWorker {
     return await this.objectClient.setRepositoryDeleteStatus(repoUri, progress);
   }
 
-  public async onJobCompleted(_: Job, res: WorkerResult) {
-    this.log.info(`Delete job ${this.id} completed with result ${JSON.stringify(res)}`);
-    // Don't update the delete progress anymore.
-    return new Promise<WorkerResult>((resolve, reject) => {
-      resolve();
-    });
-  }
-
-  public async updateProgress(uri: string, progress: number) {
+  public async updateProgress(job: Job, progress: number) {
+    const { uri } = job.payload;
     const p: WorkerProgress = {
       uri,
       progress,
       timestamp: new Date(),
     };
-    return await this.objectClient.updateRepositoryDeleteStatus(uri, p);
+    if (progress !== WorkerReservedProgress.COMPLETED) {
+      return await this.objectClient.updateRepositoryDeleteStatus(uri, p);
+    }
   }
 
   protected async getTimeoutMs(_: any) {
diff --git a/x-pack/plugins/code/server/queue/index_worker.ts b/x-pack/plugins/code/server/queue/index_worker.ts
index a85da3f78ba8..62680a0b966a 100644
--- a/x-pack/plugins/code/server/queue/index_worker.ts
+++ b/x-pack/plugins/code/server/queue/index_worker.ts
@@ -86,8 +86,13 @@ export class IndexWorker extends AbstractWorker {
     // Binding the index cancellation logic
     this.cancellationService.cancelIndexJob(uri);
     const indexPromises: Array<Promise<IndexStats>> = this.indexerFactories.map(
-      (indexerFactory: IndexerFactory, index: number) => {
-        const indexer = indexerFactory.create(uri, revision);
+      async (indexerFactory: IndexerFactory, index: number) => {
+        const indexer = await indexerFactory.create(uri, revision);
+        if (!indexer) {
+          this.log.info(`Failed to create indexer for ${uri}`);
+          return new Map(); // return an empty map as stats.
+        }
+
         if (cancellationToken) {
           cancellationToken.on(() => {
             indexer.cancel();
@@ -119,8 +124,20 @@ export class IndexWorker extends AbstractWorker {
     return await this.objectClient.setRepositoryLspIndexStatus(uri, progress);
   }
 
-  public async updateProgress(uri: RepositoryUri, progress: number) {
-    let p: any = {
+  public async onJobCompleted(job: Job, res: IndexWorkerResult) {
+    await super.onJobCompleted(job, res);
+    const { uri, revision } = job.payload;
+    try {
+      return await this.objectClient.updateRepository(uri, { indexedRevision: revision });
+    } catch (error) {
+      this.log.error(`Update indexed revision in repository object error.`);
+      this.log.error(error);
+    }
+  }
+
+  public async updateProgress(job: Job, progress: number) {
+    const { uri } = job.payload;
+    const p: any = {
       uri,
       progress,
       timestamp: new Date(),
diff --git a/x-pack/plugins/code/server/queue/worker.ts b/x-pack/plugins/code/server/queue/worker.ts
index 6faa4c79e6ee..a3ecff215eec 100644
--- a/x-pack/plugins/code/server/queue/worker.ts
+++ b/x-pack/plugins/code/server/queue/worker.ts
@@ -16,5 +16,5 @@ export interface Worker {
   onJobExecutionError(res: any): void;
   onJobTimeOut(res: any): void;
 
-  updateProgress(uri: string, progress: number): void;
+  updateProgress(job: Job, progress: number): void;
 }
diff --git a/x-pack/plugins/code/server/routes/repository.ts b/x-pack/plugins/code/server/routes/repository.ts
index ff92757c1741..a023e07d00c8 100644
--- a/x-pack/plugins/code/server/routes/repository.ts
+++ b/x-pack/plugins/code/server/routes/repository.ts
@@ -60,10 +60,10 @@ export function repositoryRoute(
         log.info(`Repository ${repoUrl} does not exist. Go ahead with clone.`);
         try {
           // Create the index for the repository
-          const initializer = repoIndexInitializerFactory.create(
+          const initializer = (await repoIndexInitializerFactory.create(
             repo.uri,
             ''
-          ) as RepositoryIndexInitializer;
+          )) as RepositoryIndexInitializer;
           await initializer.init();
 
           // Persist to elasticsearch
diff --git a/x-pack/plugins/code/server/scheduler/update_scheduler.ts b/x-pack/plugins/code/server/scheduler/update_scheduler.ts
index 283db5fc7fd6..858c024cdeb2 100644
--- a/x-pack/plugins/code/server/scheduler/update_scheduler.ts
+++ b/x-pack/plugins/code/server/scheduler/update_scheduler.ts
@@ -5,7 +5,6 @@
  */
 
 import { Repository, WorkerReservedProgress } from '../../model';
-import { RepositoryIndexName, RepositoryReservedField } from '../indexer/schema';
 import { EsClient } from '../lib/esqueue';
 import { Logger } from '../log';
 import { UpdateWorker } from '../queue';
@@ -43,9 +42,18 @@ export class UpdateScheduler extends AbstractScheduler {
         return;
       }
 
+      let inDelete = false;
+      try {
+        await this.objectClient.getRepositoryDeleteStatus(repo.uri);
+        inDelete = true;
+      } catch (error) {
+        inDelete = false;
+      }
+
       const cloneStatus = await this.objectClient.getRepositoryGitStatus(repo.uri);
-      // Schedule update job only when the repo has been fully cloned already
+      // Schedule update job only when the repo has been fully cloned already and not in delete
       if (
+        !inDelete &&
         cloneStatus.cloneProgress &&
         cloneStatus.cloneProgress.isCloned &&
         cloneStatus.progress === WorkerReservedProgress.COMPLETED
@@ -56,20 +64,15 @@ export class UpdateScheduler extends AbstractScheduler {
 
         // Update the next repo update timestamp.
         const nextRepoUpdateTimestamp = this.repoNextSchedulingTime();
-        this.client.update({
-          index: RepositoryIndexName(repo.uri),
-          id: repo.uri,
-          body: JSON.stringify({
-            doc: {
-              [RepositoryReservedField]: {
-                nextUpdateTimestamp: nextRepoUpdateTimestamp,
-              },
-            },
-          }),
+        await this.objectClient.updateRepository(repo.uri, {
+          nextUpdateTimestamp: nextRepoUpdateTimestamp,
         });
+
         await this.updateWorker.enqueueJob(payload, {});
       } else {
-        this.log.info(`Repo ${repo.uri} has not been fully cloned yet or in update. Skip update.`);
+        this.log.info(
+          `Repo ${repo.uri} has not been fully cloned yet or in update/delete. Skip update.`
+        );
       }
     } catch (error) {
       this.log.error(`Schedule update for ${repo.uri} error.`);
diff --git a/x-pack/plugins/code/server/server_options.ts b/x-pack/plugins/code/server/server_options.ts
index 63127489ba10..45a0b6fc5055 100644
--- a/x-pack/plugins/code/server/server_options.ts
+++ b/x-pack/plugins/code/server/server_options.ts
@@ -32,7 +32,7 @@ export class ServerOptions {
 
   public readonly jdtConfigPath = resolve(this.config.get('path.data'), 'code/jdt_config');
 
-  public readonly updateFrequencyMs: number = this.options.updateFreqencyMs;
+  public readonly updateFrequencyMs: number = this.options.updateFrequencyMs;
 
   public readonly indexFrequencyMs: number = this.options.indexFrequencyMs;
 

From 8b76beefd628e2778c908e1bf976f19071a94a79 Mon Sep 17 00:00:00 2001
From: Mengwei Ding <mengwei.ding@gmail.com>
Date: Thu, 28 Mar 2019 17:13:34 +0800
Subject: [PATCH 3/6] [Code] implement the actual incremental indexing

---
 x-pack/plugins/code/model/repository.ts       |  9 ++-
 .../code/server/indexer/abstract_indexer.ts   |  1 +
 .../server/indexer/lsp_incremental_indexer.ts | 80 +++++++++++++++++--
 .../code/server/indexer/schema/symbol.ts      |  5 +-
 .../plugins/code/server/queue/index_worker.ts |  2 +-
 5 files changed, 82 insertions(+), 15 deletions(-)

diff --git a/x-pack/plugins/code/model/repository.ts b/x-pack/plugins/code/model/repository.ts
index dc7ddecf3a52..843b3fd5fcc4 100644
--- a/x-pack/plugins/code/model/repository.ts
+++ b/x-pack/plugins/code/model/repository.ts
@@ -89,9 +89,12 @@ export interface UpdateWorkerResult extends WorkerResult {
 }
 
 export enum IndexStatsKey {
-  File = 'file-count',
-  Symbol = 'symbol-count',
-  Reference = 'reference-count',
+  File = 'file-added-count',
+  FileDeleted = 'file-deleted-count',
+  Symbol = 'symbol-added-count',
+  SymbolDeleted = 'symbol-deleted-count',
+  Reference = 'reference-added-count',
+  ReferenceDeleted = 'reference-deleted-count',
 }
 export type IndexStats = Map<IndexStatsKey, number>;
 
diff --git a/x-pack/plugins/code/server/indexer/abstract_indexer.ts b/x-pack/plugins/code/server/indexer/abstract_indexer.ts
index 6d28c43cb672..2121564c54e3 100644
--- a/x-pack/plugins/code/server/indexer/abstract_indexer.ts
+++ b/x-pack/plugins/code/server/indexer/abstract_indexer.ts
@@ -72,6 +72,7 @@ export abstract class AbstractIndexer implements Indexer {
 
       // If checkpoint is not undefined and not empty
       if (checkpointReq) {
+        // TODO(mengwei): move this checkpoint matching process to an abstract function
         // Assume for the same revision, everything we iterate the repository,
         // the order of the files is definite.
         // @ts-ignore
diff --git a/x-pack/plugins/code/server/indexer/lsp_incremental_indexer.ts b/x-pack/plugins/code/server/indexer/lsp_incremental_indexer.ts
index 2afbb0754442..24745788b935 100644
--- a/x-pack/plugins/code/server/indexer/lsp_incremental_indexer.ts
+++ b/x-pack/plugins/code/server/indexer/lsp_incremental_indexer.ts
@@ -57,23 +57,29 @@ export class LspIncrementalIndexer extends LspIndexer {
 
     switch (kind) {
       case DiffKind.ADDED: {
-        this.log.debug(`Index ADDED file`);
+        this.log.info(`Index ADDED file`);
         await this.handleAddedRequest(request, stats);
+        break;
       }
       case DiffKind.DELETED: {
-        this.log.debug(`Index DELETED file`);
-        // TODO: implement delete. We need the encode document id for delete now.
+        this.log.info(`Index DELETED file`);
+        await this.handleDeletedRequest(request, stats);
+        break;
       }
       case DiffKind.MODIFIED: {
-        this.log.debug(`Index MODIFYED file`);
-        // TODO: implement modified
+        this.log.info(`Index MODIFYED file`);
+        await this.handleModifiedRequest(request, stats);
+        break;
       }
       case DiffKind.RENAMED: {
-        this.log.debug(`Index RENAMED file`);
-        // TODO: implement renamed
+        this.log.info(`Index RENAMED file`);
+        await this.handleRenamedRequest(request, stats);
+        break;
       }
       default: {
-        this.log.debug(`Unsupported diff kind ${kind} for incremental indexing.`);
+        this.log.debug(
+          `Unsupported diff kind ${kind} for incremental indexing. Skip this request.`
+        );
       }
     }
 
@@ -179,4 +185,62 @@ export class LspIncrementalIndexer extends LspIndexer {
     await this.batchIndexHelper.index(DocumentIndexName(repoUri), body);
     stats.set(IndexStatsKey.File, 1);
   }
+
+  private async handleDeletedRequest(request: LspIncIndexRequest, stats: IndexStats) {
+    const { revision, filePath, repoUri } = request;
+
+    // Delete the document with the exact file path. TODO: add stats
+    const docRes = await this.client.deleteByQuery({
+      index: DocumentIndexName(repoUri),
+      body: {
+        query: {
+          term: {
+            'path.hierarchy': filePath,
+          },
+        },
+      },
+    });
+    stats.set(IndexStatsKey.FileDeleted, docRes.deleted);
+
+    const lspDocUri = toCanonicalUrl({ repoUri, revision, file: filePath, schema: 'git:' });
+
+    // Delete all symbols within this file
+    const symbolRes = await this.client.deleteByQuery({
+      index: SymbolIndexName(repoUri),
+      body: {
+        query: {
+          term: {
+            'symbolInformation.location.uri': lspDocUri,
+          },
+        },
+      },
+    });
+    stats.set(IndexStatsKey.SymbolDeleted, symbolRes.deleted);
+
+    // TODO: When references is enabled. Clean up the references as well.
+  }
+
+  private async handleModifiedRequest(request: LspIncIndexRequest, stats: IndexStats) {
+    const { kind, originRevision, originPath, repoUri, localRepoPath } = request;
+
+    // 1. first delete all related indexed data
+    await this.handleDeletedRequest(
+      {
+        repoUri,
+        localRepoPath,
+        revision: originRevision,
+        filePath: originPath ? originPath : '',
+        kind,
+        originRevision,
+      },
+      stats
+    );
+    // 2. index data with modified version
+    await this.handleAddedRequest(request, stats);
+  }
+
+  private async handleRenamedRequest(request: LspIncIndexRequest, stats: IndexStats) {
+    // Do the same as modified file
+    await this.handleModifiedRequest(request, stats);
+  }
 }
diff --git a/x-pack/plugins/code/server/indexer/schema/symbol.ts b/x-pack/plugins/code/server/indexer/schema/symbol.ts
index 365656e03dd3..d318730f466a 100644
--- a/x-pack/plugins/code/server/indexer/schema/symbol.ts
+++ b/x-pack/plugins/code/server/indexer/schema/symbol.ts
@@ -39,9 +39,8 @@ export const SymbolSchema = {
       location: {
         properties: {
           uri: {
-            type: 'text',
-            index: false,
-            norms: false,
+            // Indexed now for symbols batch deleting in incremental indexing
+            type: 'keyword',
           },
         },
       },
diff --git a/x-pack/plugins/code/server/queue/index_worker.ts b/x-pack/plugins/code/server/queue/index_worker.ts
index 62680a0b966a..37c049eb1b46 100644
--- a/x-pack/plugins/code/server/queue/index_worker.ts
+++ b/x-pack/plugins/code/server/queue/index_worker.ts
@@ -137,7 +137,7 @@ export class IndexWorker extends AbstractWorker {
 
   public async updateProgress(job: Job, progress: number) {
     const { uri } = job.payload;
-    const p: any = {
+    let p: any = {
       uri,
       progress,
       timestamp: new Date(),

From 5e798e1156a84b2a15975e9b66409f3ead99aeb7 Mon Sep 17 00:00:00 2001
From: Mengwei Ding <mengwei.ding@gmail.com>
Date: Fri, 29 Mar 2019 12:57:11 +0800
Subject: [PATCH 4/6] [Code] apply checkpoint validation for both lsp and lsp
 incremental indexer

---
 x-pack/plugins/code/common/git_diff.ts        |  8 ++---
 x-pack/plugins/code/index.ts                  |  6 ++--
 .../code/server/__tests__/git_operations.ts   |  3 +-
 .../code/server/indexer/abstract_indexer.ts   | 33 ++++++++++++-------
 .../server/indexer/lsp_incremental_indexer.ts | 30 ++++++++++++++---
 .../code/server/indexer/lsp_indexer.ts        | 17 ++++++++++
 6 files changed, 73 insertions(+), 24 deletions(-)

diff --git a/x-pack/plugins/code/common/git_diff.ts b/x-pack/plugins/code/common/git_diff.ts
index 24600a7487d7..89f1bd1cf2fa 100644
--- a/x-pack/plugins/code/common/git_diff.ts
+++ b/x-pack/plugins/code/common/git_diff.ts
@@ -28,8 +28,8 @@ export interface FileDiff {
 }
 
 export enum DiffKind {
-  ADDED,
-  DELETED,
-  MODIFIED,
-  RENAMED,
+  ADDED = 'ADDED',
+  DELETED = 'DELETED',
+  MODIFIED = 'MODIFIED',
+  RENAMED = 'RENAMED',
 }
diff --git a/x-pack/plugins/code/index.ts b/x-pack/plugins/code/index.ts
index 1f7beaf60609..7c17e89cc15b 100644
--- a/x-pack/plugins/code/index.ts
+++ b/x-pack/plugins/code/index.ts
@@ -35,11 +35,11 @@ export const code = (kibana: any) =>
         // 1 hour by default.
         queueTimeout: Joi.number().default(moment.duration(1, 'hour').asMilliseconds()),
         // The frequency which update scheduler executes. 5 minutes by default.
-        updateFrequencyMs: Joi.number().default(moment.duration(10, 'second').asMilliseconds()),
+        updateFrequencyMs: Joi.number().default(moment.duration(5, 'minute').asMilliseconds()),
         // The frequency which index scheduler executes. 1 day by default.
         indexFrequencyMs: Joi.number().default(moment.duration(1, 'day').asMilliseconds()),
         // The frequency which each repo tries to update. 1 hour by default.
-        updateRepoFrequencyMs: Joi.number().default(moment.duration(1, 'minute').asMilliseconds()),
+        updateRepoFrequencyMs: Joi.number().default(moment.duration(1, 'hour').asMilliseconds()),
         // The frequency which each repo tries to index. 1 day by default.
         indexRepoFrequencyMs: Joi.number().default(moment.duration(1, 'day').asMilliseconds()),
         lsp: Joi.object({
@@ -69,7 +69,7 @@ export const code = (kibana: any) =>
             .default(['https', 'git']),
         }).default(),
         maxWorkspace: Joi.number().default(5), // max workspace folder for each language server
-        disableScheduler: Joi.boolean().default(false), // Temp option to disable all schedulers.
+        disableScheduler: Joi.boolean().default(true), // Temp option to disable all schedulers.
         enableGlobalReference: Joi.boolean().default(false), // Global reference as optional feature for now
         codeNode: Joi.boolean().default(false),
       }).default();
diff --git a/x-pack/plugins/code/server/__tests__/git_operations.ts b/x-pack/plugins/code/server/__tests__/git_operations.ts
index d05c94066335..19c5d1b7ba67 100644
--- a/x-pack/plugins/code/server/__tests__/git_operations.ts
+++ b/x-pack/plugins/code/server/__tests__/git_operations.ts
@@ -122,7 +122,8 @@ describe('git_operations', () => {
     );
 
     const g = new GitOperations(serverOptions.repoPath);
-    const d = await g.getDiff('github.com/Microsoft/TypeScript-Node-Starter', '6206f6431e75b0e98506a356fb2ded08ab0f0c89', '4779cb7e182cf41d5c62289bb80d2850e0265b71');
+    // TODO: finish up this unit test
+    await g.getDiff('github.com/Microsoft/TypeScript-Node-Starter', '6206f6431e75b0e98506a356fb2ded08ab0f0c89', '4779cb7e182cf41d5c62289bb80d2850e0265b71');
   // @ts-ignore
   }).timeout(100000);
 });
diff --git a/x-pack/plugins/code/server/indexer/abstract_indexer.ts b/x-pack/plugins/code/server/indexer/abstract_indexer.ts
index 2121564c54e3..19e5f0a44cb3 100644
--- a/x-pack/plugins/code/server/indexer/abstract_indexer.ts
+++ b/x-pack/plugins/code/server/indexer/abstract_indexer.ts
@@ -35,7 +35,9 @@ export abstract class AbstractIndexer implements Indexer {
     );
     this.cancelled = false;
 
-    if (!checkpointReq) {
+    const isCheckpointValid = this.validateCheckpoint(checkpointReq);
+
+    if (this.needRefreshIndices()) {
       // Prepare the ES index
       const res = await this.prepareIndex();
       if (!res) {
@@ -70,17 +72,9 @@ export abstract class AbstractIndexer implements Indexer {
         break;
       }
 
-      // If checkpoint is not undefined and not empty
-      if (checkpointReq) {
-        // TODO(mengwei): move this checkpoint matching process to an abstract function
-        // Assume for the same revision, everything we iterate the repository,
-        // the order of the files is definite.
-        // @ts-ignore
-        if (req.filePath === checkpointReq.filePath && req.revision === checkpointReq.revision) {
-          this.log.info(`The index checkpoint has been found ${JSON.stringify(checkpointReq)}.`);
-          meetCheckpoint = true;
-        }
-
+      // If checkpoint is valid and has not been met
+      if (isCheckpointValid && !meetCheckpoint) {
+        meetCheckpoint = meetCheckpoint || this.ifCheckpointMet(req, checkpointReq!);
         if (!meetCheckpoint) {
           // If the checkpoint has not been met yet, skip current request.
           continue;
@@ -128,6 +122,21 @@ export abstract class AbstractIndexer implements Indexer {
     return this.cancelled;
   }
 
+  // If the current checkpoint is valid
+  protected validateCheckpoint(checkpointReq?: IndexRequest): boolean {
+    return checkpointReq !== undefined;
+  }
+
+  // If it's necessary to refresh (create and reset) all the related indices
+  protected needRefreshIndices(checkpointReq?: IndexRequest): boolean {
+    return false;
+  }
+
+  protected ifCheckpointMet(req: IndexRequest, checkpointReq: IndexRequest): boolean {
+    // Please override this function
+    return false;
+  }
+
   protected async cleanIndex(): Promise<void> {
     // This is the abstract implementation. You should override this.
     return new Promise<void>((resolve, reject) => {
diff --git a/x-pack/plugins/code/server/indexer/lsp_incremental_indexer.ts b/x-pack/plugins/code/server/indexer/lsp_incremental_indexer.ts
index 24745788b935..841625799355 100644
--- a/x-pack/plugins/code/server/indexer/lsp_incremental_indexer.ts
+++ b/x-pack/plugins/code/server/indexer/lsp_incremental_indexer.ts
@@ -43,6 +43,31 @@ export class LspIncrementalIndexer extends LspIndexer {
     super(repoUri, revision, lspService, options, client, log);
   }
 
+  // If the current checkpoint is valid. Otherwise, ignore the checkpoint
+  protected validateCheckpoint(checkpointReq?: LspIncIndexRequest): boolean {
+    return (
+      checkpointReq !== undefined &&
+      checkpointReq.revision === this.revision &&
+      checkpointReq.originRevision === this.originRevision
+    );
+  }
+
+  // If it's necessary to refresh (create and reset) all the related indices
+  protected needRefreshIndices(_: LspIncIndexRequest): boolean {
+    return false;
+  }
+
+  protected ifCheckpointMet(req: LspIncIndexRequest, checkpointReq: LspIncIndexRequest): boolean {
+    // Assume for the same revision pair, the order of the files we iterate the diff is definite
+    // everytime.
+    return (
+      req.filePath === checkpointReq.filePath &&
+      req.revision === checkpointReq.revision &&
+      req.originRevision === checkpointReq.originRevision &&
+      req.kind === checkpointReq.kind
+    );
+  }
+
   protected async prepareIndexCreationRequests() {
     // We don't need to create new indices for incremental indexing.
     return [];
@@ -55,24 +80,21 @@ export class LspIncrementalIndexer extends LspIndexer {
       .set(IndexStatsKey.File, 0);
     const { kind } = request;
 
+    this.log.info(`Index ${kind} request ${JSON.stringify(request, null, 2)}`);
     switch (kind) {
       case DiffKind.ADDED: {
-        this.log.info(`Index ADDED file`);
         await this.handleAddedRequest(request, stats);
         break;
       }
       case DiffKind.DELETED: {
-        this.log.info(`Index DELETED file`);
         await this.handleDeletedRequest(request, stats);
         break;
       }
       case DiffKind.MODIFIED: {
-        this.log.info(`Index MODIFYED file`);
         await this.handleModifiedRequest(request, stats);
         break;
       }
       case DiffKind.RENAMED: {
-        this.log.info(`Index RENAMED file`);
         await this.handleRenamedRequest(request, stats);
         break;
       }
diff --git a/x-pack/plugins/code/server/indexer/lsp_indexer.ts b/x-pack/plugins/code/server/indexer/lsp_indexer.ts
index dbfadb01f3ef..2bae330e953f 100644
--- a/x-pack/plugins/code/server/indexer/lsp_indexer.ts
+++ b/x-pack/plugins/code/server/indexer/lsp_indexer.ts
@@ -58,6 +58,23 @@ export class LspIndexer extends AbstractIndexer {
     super.cancel();
   }
 
+  // If the current checkpoint is valid
+  protected validateCheckpoint(checkpointReq?: LspIndexRequest): boolean {
+    return checkpointReq !== undefined && checkpointReq.revision === this.revision;
+  }
+
+  // If it's necessary to refresh (create and reset) all the related indices
+  protected needRefreshIndices(checkpointReq?: LspIndexRequest): boolean {
+    // If it's not resumed from a checkpoint, then try to refresh all the indices.
+    return !this.validateCheckpoint(checkpointReq);
+  }
+
+  protected ifCheckpointMet(req: LspIndexRequest, checkpointReq: LspIndexRequest): boolean {
+    // Assume for the same revision, the order of the files we iterate the repository is definite
+    // everytime.
+    return req.filePath === checkpointReq.filePath && req.revision === checkpointReq.revision;
+  }
+
   protected async prepareIndexCreationRequests() {
     return [
       getDocumentIndexCreationRequest(this.repoUri),

From 98e8cda2b7d3bb9a242c3b1377f4f622a1b50c8a Mon Sep 17 00:00:00 2001
From: Mengwei Ding <mengwei.ding@gmail.com>
Date: Fri, 29 Mar 2019 13:46:35 +0800
Subject: [PATCH 5/6] [Code] add unit tests

---
 .../code/server/__tests__/git_operations.ts   |  14 +-
 .../__tests__/lsp_incremental_indexer.ts      | 303 ++++++++++++++++++
 .../code/server/__tests__/lsp_indexer.ts      |   4 +-
 .../code/server/indexer/abstract_indexer.ts   |   6 +-
 .../server/indexer/lsp_incremental_indexer.ts |  35 +-
 .../code/server/queue/index_worker.test.ts    |   4 +-
 .../server/scheduler/update_scheduler.test.ts |  16 +-
 7 files changed, 357 insertions(+), 25 deletions(-)
 create mode 100644 x-pack/plugins/code/server/__tests__/lsp_incremental_indexer.ts

diff --git a/x-pack/plugins/code/server/__tests__/git_operations.ts b/x-pack/plugins/code/server/__tests__/git_operations.ts
index 19c5d1b7ba67..dcdc074eb635 100644
--- a/x-pack/plugins/code/server/__tests__/git_operations.ts
+++ b/x-pack/plugins/code/server/__tests__/git_operations.ts
@@ -101,7 +101,7 @@ describe('git_operations', () => {
     assert.strictEqual(totalFiles, 3, 'this repo should contains exactly 2 files');
   });
 
-  it('get any diff', async () => {
+  it('get diff between arbitrary 2 revisions', async () => {
     function cloneProject(url: string, p: string) {
       return new Promise(resolve => {
         if (!fs.existsSync(p)) {
@@ -122,8 +122,14 @@ describe('git_operations', () => {
     );
 
     const g = new GitOperations(serverOptions.repoPath);
-    // TODO: finish up this unit test
-    await g.getDiff('github.com/Microsoft/TypeScript-Node-Starter', '6206f6431e75b0e98506a356fb2ded08ab0f0c89', '4779cb7e182cf41d5c62289bb80d2850e0265b71');
-  // @ts-ignore
+    const d = await g.getDiff(
+      'github.com/Microsoft/TypeScript-Node-Starter',
+      '6206f643',
+      '4779cb7e'
+    );
+    assert.equal(d.additions, 2);
+    assert.equal(d.deletions, 4);
+    assert.equal(d.files.length, 3);
+    // @ts-ignore
   }).timeout(100000);
 });
diff --git a/x-pack/plugins/code/server/__tests__/lsp_incremental_indexer.ts b/x-pack/plugins/code/server/__tests__/lsp_incremental_indexer.ts
new file mode 100644
index 000000000000..2bc81619c5e0
--- /dev/null
+++ b/x-pack/plugins/code/server/__tests__/lsp_incremental_indexer.ts
@@ -0,0 +1,303 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import Git, { CloneOptions } from '@elastic/nodegit';
+import assert from 'assert';
+import fs from 'fs';
+import path from 'path';
+import rimraf from 'rimraf';
+import sinon from 'sinon';
+
+import { DiffKind } from '../../common/git_diff';
+import { WorkerReservedProgress } from '../../model';
+import { LspIncrementalIndexer } from '../indexer/lsp_incremental_indexer';
+import { RepositoryGitStatusReservedField } from '../indexer/schema';
+import { EsClient } from '../lib/esqueue';
+import { Logger } from '../log';
+import { InstallManager } from '../lsp/install_manager';
+import { LspService } from '../lsp/lsp_service';
+import { RepositoryConfigController } from '../repository_config_controller';
+import { createTestServerOption, emptyAsyncFunc } from '../test_utils';
+import { ConsoleLoggerFactory } from '../utils/console_logger_factory';
+
+const log: Logger = new ConsoleLoggerFactory().getLogger(['test']);
+
+const esClient = {
+  bulk: emptyAsyncFunc,
+  get: emptyAsyncFunc,
+  deleteByQuery: emptyAsyncFunc,
+  indices: {
+    existsAlias: emptyAsyncFunc,
+    create: emptyAsyncFunc,
+    putAlias: emptyAsyncFunc,
+  },
+};
+
+function prepareProject(url: string, p: string) {
+  const opts: CloneOptions = {
+    fetchOpts: {
+      callbacks: {
+        certificateCheck: () => 1,
+      },
+    },
+  };
+
+  return new Promise(resolve => {
+    if (!fs.existsSync(p)) {
+      rimraf(p, error => {
+        Git.Clone.clone(url, p, opts).then(repo => {
+          resolve(repo);
+        });
+      });
+    } else {
+      resolve();
+    }
+  });
+}
+
+const repoUri = 'github.com/Microsoft/TypeScript-Node-Starter';
+
+const serverOptions = createTestServerOption();
+
+function cleanWorkspace() {
+  return new Promise(resolve => {
+    rimraf(serverOptions.workspacePath, resolve);
+  });
+}
+
+function setupEsClientSpy() {
+  // Mock a git status of the repo indicating the the repo is fully cloned already.
+  const getSpy = sinon.fake.returns(
+    Promise.resolve({
+      _source: {
+        [RepositoryGitStatusReservedField]: {
+          uri: 'github.com/Microsoft/TypeScript-Node-Starter',
+          progress: WorkerReservedProgress.COMPLETED,
+          timestamp: new Date(),
+          cloneProgress: {
+            isCloned: true,
+          },
+        },
+      },
+    })
+  );
+  const existsAliasSpy = sinon.fake.returns(false);
+  const createSpy = sinon.spy();
+  const putAliasSpy = sinon.spy();
+  const deleteByQuerySpy = sinon.spy();
+  const bulkSpy = sinon.spy();
+  esClient.bulk = bulkSpy;
+  esClient.indices.existsAlias = existsAliasSpy;
+  esClient.indices.create = createSpy;
+  esClient.indices.putAlias = putAliasSpy;
+  esClient.get = getSpy;
+  esClient.deleteByQuery = deleteByQuerySpy;
+  return {
+    getSpy,
+    existsAliasSpy,
+    createSpy,
+    putAliasSpy,
+    deleteByQuerySpy,
+    bulkSpy,
+  };
+}
+
+function setupLsServiceSendRequestSpy(): sinon.SinonSpy {
+  return sinon.fake.returns(
+    Promise.resolve({
+      result: [
+        {
+          // 1 mock symbol for each file
+          symbols: [
+            {
+              symbolInformation: {
+                name: 'mocksymbolname',
+              },
+            },
+          ],
+          // 1 mock reference for each file
+          references: [{}],
+        },
+      ],
+    })
+  );
+}
+
+describe('lsp_incremental_indexer unit tests', () => {
+  // @ts-ignore
+  before(async () => {
+    return new Promise(resolve => {
+      rimraf(serverOptions.repoPath, resolve);
+    });
+  });
+
+  beforeEach(async function() {
+    // @ts-ignore
+    this.timeout(200000);
+    return await prepareProject(
+      'https://github.com/Microsoft/TypeScript-Node-Starter.git',
+      path.join(serverOptions.repoPath, repoUri)
+    );
+  });
+  // @ts-ignore
+  after(() => {
+    return cleanWorkspace();
+  });
+
+  afterEach(() => {
+    sinon.restore();
+  });
+
+  it('Normal LSP index process.', async () => {
+    // Setup the esClient spies
+    const {
+      existsAliasSpy,
+      createSpy,
+      putAliasSpy,
+      deleteByQuerySpy,
+      bulkSpy,
+    } = setupEsClientSpy();
+
+    const lspservice = new LspService(
+      '127.0.0.1',
+      serverOptions,
+      esClient as EsClient,
+      {} as InstallManager,
+      new ConsoleLoggerFactory(),
+      new RepositoryConfigController(esClient as EsClient)
+    );
+
+    lspservice.sendRequest = setupLsServiceSendRequestSpy();
+
+    const indexer = new LspIncrementalIndexer(
+      'github.com/Microsoft/TypeScript-Node-Starter',
+      '4779cb7e',
+      '6206f643',
+      lspservice,
+      serverOptions,
+      esClient as EsClient,
+      log
+    );
+    await indexer.start();
+
+    // Index and alias creation are not necessary for incremental indexing
+    assert.strictEqual(existsAliasSpy.callCount, 0);
+    assert.strictEqual(createSpy.callCount, 0);
+    assert.strictEqual(putAliasSpy.callCount, 0);
+
+    // DeletebyQuery is called 6 times (1 file + 1 symbol reuqests per diff item)
+    // for 3 MODIFIED items
+    assert.strictEqual(deleteByQuerySpy.callCount, 6);
+
+    // There are 3 MODIFIED items. 1 file + 1 symbol + 1 reference = 3 objects to
+    // index for each item. Total doc indexed should be 3 * 3 = 9, which can be
+    // fitted into a single batch index.
+    assert.ok(bulkSpy.calledOnce);
+    assert.strictEqual(bulkSpy.getCall(0).args[0].body.length, 9 * 2);
+    // @ts-ignore
+  }).timeout(20000);
+
+  it('Cancel LSP index process.', async () => {
+    // Setup the esClient spies
+    const {
+      existsAliasSpy,
+      createSpy,
+      putAliasSpy,
+      deleteByQuerySpy,
+      bulkSpy,
+    } = setupEsClientSpy();
+
+    const lspservice = new LspService(
+      '127.0.0.1',
+      serverOptions,
+      esClient as EsClient,
+      {} as InstallManager,
+      new ConsoleLoggerFactory(),
+      new RepositoryConfigController(esClient as EsClient)
+    );
+
+    lspservice.sendRequest = setupLsServiceSendRequestSpy();
+
+    const indexer = new LspIncrementalIndexer(
+      'github.com/Microsoft/TypeScript-Node-Starter',
+      '4779cb7e',
+      '6206f643',
+      lspservice,
+      serverOptions,
+      esClient as EsClient,
+      log
+    );
+    // Cancel the indexer before start.
+    indexer.cancel();
+    await indexer.start();
+
+    // Index and alias creation are not necessary for incremental indexing.
+    assert.strictEqual(existsAliasSpy.callCount, 0);
+    assert.strictEqual(createSpy.callCount, 0);
+    assert.strictEqual(putAliasSpy.callCount, 0);
+
+    // Because the indexer is cancelled already in the begining. 0 doc should be
+    // indexed and thus bulk and deleteByQuery won't be called.
+    assert.ok(bulkSpy.notCalled);
+    assert.ok(deleteByQuerySpy.notCalled);
+  });
+
+  it('Index continues from a checkpoint', async () => {
+    // Setup the esClient spies
+    const {
+      existsAliasSpy,
+      createSpy,
+      putAliasSpy,
+      deleteByQuerySpy,
+      bulkSpy,
+    } = setupEsClientSpy();
+
+    const lspservice = new LspService(
+      '127.0.0.1',
+      serverOptions,
+      esClient as EsClient,
+      {} as InstallManager,
+      new ConsoleLoggerFactory(),
+      new RepositoryConfigController(esClient as EsClient)
+    );
+
+    lspservice.sendRequest = setupLsServiceSendRequestSpy();
+
+    const indexer = new LspIncrementalIndexer(
+      'github.com/Microsoft/TypeScript-Node-Starter',
+      '46971a84',
+      '6206f643',
+      lspservice,
+      serverOptions,
+      esClient as EsClient,
+      log
+    );
+
+    // Apply a checkpoint in here.
+    await indexer.start(undefined, {
+      repoUri: '',
+      filePath: 'package.json',
+      revision: '46971a84',
+      originRevision: '6206f643',
+      localRepoPath: '',
+      kind: DiffKind.MODIFIED,
+    });
+
+    // Index and alias creation are not necessary for incremental indexing.
+    assert.strictEqual(existsAliasSpy.callCount, 0);
+    assert.strictEqual(createSpy.callCount, 0);
+    assert.strictEqual(putAliasSpy.callCount, 0);
+
+    // There are 3 MODIFIED items, but 1 item after the checkpoint. 1 file
+    // + 1 symbol + 1 ref = 3 objects to be indexed for each item. Total doc
+    // indexed should be 3 * 2 = 2, which can be fitted into a single batch index.
+    assert.ok(bulkSpy.calledOnce);
+    assert.strictEqual(bulkSpy.getCall(0).args[0].body.length, 3 * 2);
+    assert.strictEqual(deleteByQuerySpy.callCount, 2);
+    // @ts-ignore
+  }).timeout(20000);
+  // @ts-ignore
+}).timeout(20000);
diff --git a/x-pack/plugins/code/server/__tests__/lsp_indexer.ts b/x-pack/plugins/code/server/__tests__/lsp_indexer.ts
index d8d634fb9b88..4773e5ab35a5 100644
--- a/x-pack/plugins/code/server/__tests__/lsp_indexer.ts
+++ b/x-pack/plugins/code/server/__tests__/lsp_indexer.ts
@@ -124,7 +124,7 @@ function setupLsServiceSendRequestSpy(): sinon.SinonSpy {
     })
   );
 }
-describe('lsp_indexer', () => {
+describe('lsp_indexer unit tests', () => {
   // @ts-ignore
   before(async () => {
     return new Promise(resolve => {
@@ -244,7 +244,7 @@ describe('lsp_indexer', () => {
     assert.ok(bulkSpy.notCalled);
   });
 
-  it('Index continues from checkpoint', async () => {
+  it('Index continues from a checkpoint', async () => {
     // Setup the esClient spies
     const {
       existsAliasSpy,
diff --git a/x-pack/plugins/code/server/indexer/abstract_indexer.ts b/x-pack/plugins/code/server/indexer/abstract_indexer.ts
index 19e5f0a44cb3..bc7b0b8c30ab 100644
--- a/x-pack/plugins/code/server/indexer/abstract_indexer.ts
+++ b/x-pack/plugins/code/server/indexer/abstract_indexer.ts
@@ -33,11 +33,9 @@ export abstract class AbstractIndexer implements Indexer {
     this.log.info(
       `Indexer ${this.type} started for repo ${this.repoUri} with revision ${this.revision}`
     );
-    this.cancelled = false;
-
     const isCheckpointValid = this.validateCheckpoint(checkpointReq);
 
-    if (this.needRefreshIndices()) {
+    if (this.needRefreshIndices(checkpointReq)) {
       // Prepare the ES index
       const res = await this.prepareIndex();
       if (!res) {
@@ -78,6 +76,8 @@ export abstract class AbstractIndexer implements Indexer {
         if (!meetCheckpoint) {
           // If the checkpoint has not been met yet, skip current request.
           continue;
+        } else {
+          this.log.info(`Checkpoint met. Continue with indexing.`);
         }
       }
 
diff --git a/x-pack/plugins/code/server/indexer/lsp_incremental_indexer.ts b/x-pack/plugins/code/server/indexer/lsp_incremental_indexer.ts
index 841625799355..2cf88fc10c78 100644
--- a/x-pack/plugins/code/server/indexer/lsp_incremental_indexer.ts
+++ b/x-pack/plugins/code/server/indexer/lsp_incremental_indexer.ts
@@ -7,6 +7,7 @@
 import fs from 'fs';
 import util from 'util';
 
+import { ProgressReporter } from '.';
 import { Diff, DiffKind } from '../../common/git_diff';
 import { toCanonicalUrl } from '../../common/uri_util';
 import {
@@ -43,6 +44,10 @@ export class LspIncrementalIndexer extends LspIndexer {
     super(repoUri, revision, lspService, options, client, log);
   }
 
+  public async start(progressReporter?: ProgressReporter, checkpointReq?: LspIncIndexRequest) {
+    return await super.start(progressReporter, checkpointReq);
+  }
+
   // If the current checkpoint is valid. Otherwise, ignore the checkpoint
   protected validateCheckpoint(checkpointReq?: LspIncIndexRequest): boolean {
     return (
@@ -77,10 +82,18 @@ export class LspIncrementalIndexer extends LspIndexer {
     const stats: IndexStats = new Map<IndexStatsKey, number>()
       .set(IndexStatsKey.Symbol, 0)
       .set(IndexStatsKey.Reference, 0)
-      .set(IndexStatsKey.File, 0);
+      .set(IndexStatsKey.File, 0)
+      .set(IndexStatsKey.SymbolDeleted, 0)
+      .set(IndexStatsKey.ReferenceDeleted, 0)
+      .set(IndexStatsKey.FileDeleted, 0);
+    if (this.isCancelled()) {
+      this.log.debug(`Incremental indexer is cancelled. Skip.`);
+      return stats;
+    }
+
     const { kind } = request;
 
-    this.log.info(`Index ${kind} request ${JSON.stringify(request, null, 2)}`);
+    this.log.debug(`Index ${kind} request ${JSON.stringify(request, null, 2)}`);
     switch (kind) {
       case DiffKind.ADDED: {
         await this.handleAddedRequest(request, stats);
@@ -110,10 +123,10 @@ export class LspIncrementalIndexer extends LspIndexer {
 
   protected async *getIndexRequestIterator(): AsyncIterableIterator<LspIncIndexRequest> {
     try {
-      const {
-        workspaceRepo,
-        workspaceRevision,
-      } = await this.lspService.workspaceHandler.openWorkspace(this.repoUri, 'head');
+      const { workspaceRepo } = await this.lspService.workspaceHandler.openWorkspace(
+        this.repoUri,
+        'head'
+      );
       const workspaceDir = workspaceRepo.workdir();
       if (this.diff) {
         for (const f of this.diff.files) {
@@ -122,7 +135,7 @@ export class LspIncrementalIndexer extends LspIndexer {
             localRepoPath: workspaceDir,
             filePath: f.path,
             originPath: f.originPath,
-            revision: workspaceRevision,
+            revision: this.revision,
             kind: f.kind,
             originRevision: this.originRevision,
           };
@@ -222,7 +235,9 @@ export class LspIncrementalIndexer extends LspIndexer {
         },
       },
     });
-    stats.set(IndexStatsKey.FileDeleted, docRes.deleted);
+    if (docRes) {
+      stats.set(IndexStatsKey.FileDeleted, docRes.deleted);
+    }
 
     const lspDocUri = toCanonicalUrl({ repoUri, revision, file: filePath, schema: 'git:' });
 
@@ -237,7 +252,9 @@ export class LspIncrementalIndexer extends LspIndexer {
         },
       },
     });
-    stats.set(IndexStatsKey.SymbolDeleted, symbolRes.deleted);
+    if (symbolRes) {
+      stats.set(IndexStatsKey.SymbolDeleted, symbolRes.deleted);
+    }
 
     // TODO: When references is enabled. Clean up the references as well.
   }
diff --git a/x-pack/plugins/code/server/queue/index_worker.test.ts b/x-pack/plugins/code/server/queue/index_worker.test.ts
index 0c3f55f46d3b..9aab3130a46d 100644
--- a/x-pack/plugins/code/server/queue/index_worker.test.ts
+++ b/x-pack/plugins/code/server/queue/index_worker.test.ts
@@ -380,8 +380,10 @@ test('On index job completed.', async () => {
     },
     {
       uri: 'github.com/elastic/kibana',
+      revision: 'master',
+      stats: new Map(),
     }
   );
 
-  expect(updateSpy.calledOnce).toBeTruthy();
+  expect(updateSpy.calledTwice).toBeTruthy();
 });
diff --git a/x-pack/plugins/code/server/scheduler/update_scheduler.test.ts b/x-pack/plugins/code/server/scheduler/update_scheduler.test.ts
index 9ba09ec9b39b..519811040eb9 100644
--- a/x-pack/plugins/code/server/scheduler/update_scheduler.test.ts
+++ b/x-pack/plugins/code/server/scheduler/update_scheduler.test.ts
@@ -64,7 +64,7 @@ const createSearchSpy = (nextUpdateTimestamp: number): sinon.SinonSpy => {
   );
 };
 
-const createGetSpy = (progress: number): sinon.SinonSpy => {
+const createGetSpy = (progress: number): sinon.SinonStub => {
   const cloneStatus: CloneWorkerProgress = {
     uri: 'github.com/elastic/code',
     progress,
@@ -73,13 +73,16 @@ const createGetSpy = (progress: number): sinon.SinonSpy => {
       isCloned: true,
     } as CloneProgress,
   };
-  return sinon.fake.returns(
+  const stub = sinon.stub();
+  stub.onFirstCall().throwsException('Failed to get delete status');
+  stub.onSecondCall().returns(
     Promise.resolve({
       _source: {
         [RepositoryGitStatusReservedField]: cloneStatus,
       },
     })
   );
+  return stub;
 };
 
 afterEach(() => {
@@ -150,9 +153,10 @@ test('Next job should not execute when repo is still in clone.', done => {
   const onScheduleFinished = () => {
     try {
       // Expect the search stub to be called to pull all repositories and
-      // the get stub to be called to pull out git status.
+      // the get stub to be called twice to pull out git status and delete
+      // status.
       expect(searchSpy.calledOnce).toBeTruthy();
-      expect(getSpy.calledOnce).toBeTruthy();
+      expect(getSpy.calledTwice).toBeTruthy();
       // Expect no update on anything regarding the update task scheduling.
       expect(enqueueJobSpy.notCalled).toBeTruthy();
       expect(updateSpy.notCalled).toBeTruthy();
@@ -197,8 +201,8 @@ test('Next job should execute.', done => {
     try {
       // Expect the search stub to be called to pull all repositories.
       expect(searchSpy.calledOnce).toBeTruthy();
-      // Expect the get stub to be called to pull git status.
-      expect(getSpy.calledOnce).toBeTruthy();
+      // Expect the get stub to be called to pull git status and delete status.
+      expect(getSpy.calledTwice).toBeTruthy();
       // Expect the update stub to be called to update next schedule timestamp.
       expect(updateSpy.calledOnce).toBeTruthy();
       // Expect the enqueue job stub to be called to issue the update job.

From 22a22a3ef0308efe010da1db8eeb0020c8d73d3f Mon Sep 17 00:00:00 2001
From: Mengwei Ding <mengwei.ding@gmail.com>
Date: Thu, 4 Apr 2019 11:39:44 +0800
Subject: [PATCH 6/6] [Code] only disable index scheduler but leave update
 scheduler on

---
 x-pack/plugins/code/index.ts                             | 2 +-
 x-pack/plugins/code/server/init.ts                       | 4 ++--
 x-pack/plugins/code/server/scheduler/update_scheduler.ts | 3 ++-
 x-pack/plugins/code/server/server_options.ts             | 2 +-
 x-pack/plugins/code/server/test_utils.ts                 | 2 +-
 5 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/x-pack/plugins/code/index.ts b/x-pack/plugins/code/index.ts
index 7c17e89cc15b..2ccfbe5e760b 100644
--- a/x-pack/plugins/code/index.ts
+++ b/x-pack/plugins/code/index.ts
@@ -69,7 +69,7 @@ export const code = (kibana: any) =>
             .default(['https', 'git']),
         }).default(),
         maxWorkspace: Joi.number().default(5), // max workspace folder for each language server
-        disableScheduler: Joi.boolean().default(true), // Temp option to disable all schedulers.
+        disableIndexScheduler: Joi.boolean().default(true), // Temp option to disable index scheduler.
         enableGlobalReference: Joi.boolean().default(false), // Global reference as optional feature for now
         codeNode: Joi.boolean().default(false),
       }).default();
diff --git a/x-pack/plugins/code/server/init.ts b/x-pack/plugins/code/server/init.ts
index 99893259e9cc..61d0cf38ca12 100644
--- a/x-pack/plugins/code/server/init.ts
+++ b/x-pack/plugins/code/server/init.ts
@@ -212,8 +212,8 @@ async function initCodeNode(server: Server, serverOptions: ServerOptions, log: L
   // Initialize schedulers.
   const updateScheduler = new UpdateScheduler(updateWorker, serverOptions, esClient, log);
   const indexScheduler = new IndexScheduler(indexWorker, serverOptions, esClient, log);
-  if (!serverOptions.disableScheduler) {
-    updateScheduler.start();
+  updateScheduler.start();
+  if (!serverOptions.disableIndexScheduler) {
     indexScheduler.start();
   }
   // check code node repos on disk
diff --git a/x-pack/plugins/code/server/scheduler/update_scheduler.ts b/x-pack/plugins/code/server/scheduler/update_scheduler.ts
index 858c024cdeb2..49b38cf9a86a 100644
--- a/x-pack/plugins/code/server/scheduler/update_scheduler.ts
+++ b/x-pack/plugins/code/server/scheduler/update_scheduler.ts
@@ -34,13 +34,14 @@ export class UpdateScheduler extends AbstractScheduler {
   // all repositories and execute update. Later we can repeat the one we used
   // before for task throttling.
   protected async executeSchedulingJob(repo: Repository) {
-    this.log.info(`Schedule update repo request for ${repo.uri}`);
+    this.log.debug(`Try to schedule update repo request for ${repo.uri}`);
     try {
       // This repository is too soon to execute the next update job.
       if (repo.nextUpdateTimestamp && new Date() < new Date(repo.nextUpdateTimestamp)) {
         this.log.debug(`Repo ${repo.uri} is too soon to execute the next update job.`);
         return;
       }
+      this.log.info(`Start to schedule update repo request for ${repo.uri}`);
 
       let inDelete = false;
       try {
diff --git a/x-pack/plugins/code/server/server_options.ts b/x-pack/plugins/code/server/server_options.ts
index 45a0b6fc5055..b02de34f8506 100644
--- a/x-pack/plugins/code/server/server_options.ts
+++ b/x-pack/plugins/code/server/server_options.ts
@@ -42,7 +42,7 @@ export class ServerOptions {
 
   public readonly maxWorkspace: number = this.options.maxWorkspace;
 
-  public readonly disableScheduler: boolean = this.options.disableScheduler;
+  public readonly disableIndexScheduler: boolean = this.options.disableIndexScheduler;
 
   public readonly enableGlobalReference: boolean = this.options.enableGlobalReference;
 
diff --git a/x-pack/plugins/code/server/test_utils.ts b/x-pack/plugins/code/server/test_utils.ts
index ecde61d74e14..d7924a0b4fa2 100644
--- a/x-pack/plugins/code/server/test_utils.ts
+++ b/x-pack/plugins/code/server/test_utils.ts
@@ -35,7 +35,7 @@ const TEST_OPTIONS = {
   },
   repos: [],
   maxWorkspace: 5, // max workspace folder for each language server
-  disableScheduler: true, // Temp option to disable all schedulers.
+  disableIndexScheduler: true, // Temp option to disable index scheduler.
 };
 
 export function createTestServerOption() {