From 5c03c8f0863df7bc6072d8fba1f0ed3681afc901 Mon Sep 17 00:00:00 2001
From: Hailong Cui <ihailong@amazon.com>
Date: Mon, 9 Jan 2023 11:12:35 +0800
Subject: [PATCH] support reindex unique documents (#546)

Signed-off-by: Hailong Cui <ihailong@amazon.com>

Signed-off-by: Hailong Cui <ihailong@amazon.com>
---
 .../ReindexAdvancedOptions.test.tsx           |   8 ++
 .../ReindexAdvancedOptions.tsx                |  29 ++++-
 .../ReindexAdvancedOptions.test.tsx.snap      | 116 +++++++++++++++++-
 .../Reindex/container/Reindex/Reindex.tsx     |  36 ++++--
 .../__snapshots__/Reindex.test.tsx.snap       |  45 +++----
 5 files changed, 198 insertions(+), 36 deletions(-)

diff --git a/public/pages/Reindex/components/ReindexAdvancedOptions/ReindexAdvancedOptions.test.tsx b/public/pages/Reindex/components/ReindexAdvancedOptions/ReindexAdvancedOptions.test.tsx
index a89a6762a..b0e5e0361 100644
--- a/public/pages/Reindex/components/ReindexAdvancedOptions/ReindexAdvancedOptions.test.tsx
+++ b/public/pages/Reindex/components/ReindexAdvancedOptions/ReindexAdvancedOptions.test.tsx
@@ -21,6 +21,8 @@ describe("<ReindexAdvancedOptions /> spec", () => {
           ignoreConflicts={true}
           onIgnoreConflictsChange={() => {}}
           getAllPipelines={async () => [{ label: "pipeline" }]}
+          reindexUniqueDocuments={true}
+          onReindexUniqueDocumentsChange={() => {}}
         />
       </CoreServicesContext.Provider>
     );
@@ -41,6 +43,8 @@ describe("<ReindexAdvancedOptions /> spec", () => {
           ignoreConflicts={false}
           onIgnoreConflictsChange={() => {}}
           getAllPipelines={async () => [{ label: "pipeline" }]}
+          reindexUniqueDocuments={true}
+          onReindexUniqueDocumentsChange={() => {}}
         />
       </CoreServicesContext.Provider>
     );
@@ -61,6 +65,8 @@ describe("<ReindexAdvancedOptions /> spec", () => {
           ignoreConflicts={false}
           onIgnoreConflictsChange={() => {}}
           getAllPipelines={async () => Promise.reject("service not available")}
+          reindexUniqueDocuments={true}
+          onReindexUniqueDocumentsChange={() => {}}
         />
       </CoreServicesContext.Provider>
     );
@@ -82,6 +88,8 @@ describe("<ReindexAdvancedOptions /> spec", () => {
           ignoreConflicts={false}
           onIgnoreConflictsChange={() => {}}
           getAllPipelines={getPipeline}
+          reindexUniqueDocuments={true}
+          onReindexUniqueDocumentsChange={() => {}}
         />
       </CoreServicesContext.Provider>
     );
diff --git a/public/pages/Reindex/components/ReindexAdvancedOptions/ReindexAdvancedOptions.tsx b/public/pages/Reindex/components/ReindexAdvancedOptions/ReindexAdvancedOptions.tsx
index cc6ee81fe..d009b095f 100644
--- a/public/pages/Reindex/components/ReindexAdvancedOptions/ReindexAdvancedOptions.tsx
+++ b/public/pages/Reindex/components/ReindexAdvancedOptions/ReindexAdvancedOptions.tsx
@@ -18,6 +18,8 @@ interface ReindexOptionsProps {
   getAllPipelines: () => Promise<EuiComboBoxOptionOption[]>;
   ignoreConflicts: boolean;
   onIgnoreConflictsChange: (val: ChangeEvent<HTMLInputElement>) => void;
+  reindexUniqueDocuments: boolean;
+  onReindexUniqueDocumentsChange: (val: ChangeEvent<HTMLInputElement>) => void;
 }
 
 const ReindexAdvancedOptions = (props: ReindexOptionsProps) => {
@@ -34,6 +36,8 @@ const ReindexAdvancedOptions = (props: ReindexOptionsProps) => {
     getAllPipelines,
     ignoreConflicts,
     onIgnoreConflictsChange,
+    reindexUniqueDocuments,
+    onReindexUniqueDocumentsChange,
   } = props;
 
   const sliceEnabled = slices !== undefined;
@@ -52,11 +56,32 @@ const ReindexAdvancedOptions = (props: ReindexOptionsProps) => {
 
   return (
     <div style={{ padding: "10px 10px" }}>
+      <CustomFormRow
+        label="Reindex only unique documents"
+        helpText={
+          <>
+            You can choose to copy only the documents that do not exist in the destination index. By default, OpenSearch will copy all
+            documents from the source index.{" "}
+            <EuiLink href={coreServices.docLinks.links.opensearch.reindexData.unique} target="_blank">
+              Learn more.
+            </EuiLink>
+          </>
+        }
+      >
+        <EuiCheckbox
+          id="uniqueCheckbox"
+          label="Reindex only unique documents"
+          checked={reindexUniqueDocuments}
+          onChange={onReindexUniqueDocumentsChange}
+        />
+      </CustomFormRow>
+      <EuiSpacer />
+
       <CustomFormRow
         label="Version conflicts"
         helpText={
           <>
-            Instead of failing the reindexing operation, ignore any version conflicts during reindexing.
+            Instead of failing the reindexing operation, ignore any version conflicts during reindexing.{" "}
             <EuiLink href={coreServices.docLinks.links.opensearch.reindexData.unique} target="_blank">
               Learn more.
             </EuiLink>
@@ -143,7 +168,7 @@ const ReindexAdvancedOptions = (props: ReindexOptionsProps) => {
         }
         helpText={
           <>
-            Select an ingest pipeline if you need to transform documents before writing data to the destination index.
+            Select an ingest pipeline to transform documents before writing data to the destination.{" "}
             <EuiLink href={coreServices.docLinks.links.opensearch.reindexData.transform} target="_blank">
               Learn more.
             </EuiLink>
diff --git a/public/pages/Reindex/components/ReindexAdvancedOptions/__snapshots__/ReindexAdvancedOptions.test.tsx.snap b/public/pages/Reindex/components/ReindexAdvancedOptions/__snapshots__/ReindexAdvancedOptions.test.tsx.snap
index c51ea50ba..40aa69056 100644
--- a/public/pages/Reindex/components/ReindexAdvancedOptions/__snapshots__/ReindexAdvancedOptions.test.tsx.snap
+++ b/public/pages/Reindex/components/ReindexAdvancedOptions/__snapshots__/ReindexAdvancedOptions.test.tsx.snap
@@ -8,6 +8,60 @@ Object {
       <div
         style="padding: 10px 10px;"
       >
+        <div
+          class="euiFormRow"
+          id="some_html_id-row"
+        >
+          <div
+            class="euiFormRow__labelWrapper"
+          >
+            <label
+              class="euiFormLabel euiFormRow__label"
+              for="some_html_id"
+            >
+              Reindex only unique documents
+            </label>
+          </div>
+          <div
+            class="euiFormRow__fieldWrapper"
+          >
+            <div
+              class="euiFormHelpText euiFormRow__text"
+              style="padding-top: 0px; padding-bottom: 4px;"
+            >
+              You can choose to copy only the documents that do not exist in the destination index. By default, OpenSearch will copy all documents from the source index.
+               
+              <button
+                class="euiLink euiLink--primary"
+                type="button"
+              >
+                Learn more.
+              </button>
+            </div>
+            <div
+              class="euiCheckbox"
+            >
+              <input
+                checked=""
+                class="euiCheckbox__input"
+                id="uniqueCheckbox"
+                type="checkbox"
+              />
+              <div
+                class="euiCheckbox__square"
+              />
+              <label
+                class="euiCheckbox__label"
+                for="uniqueCheckbox"
+              >
+                Reindex only unique documents
+              </label>
+            </div>
+          </div>
+        </div>
+        <div
+          class="euiSpacer euiSpacer--l"
+        />
         <div
           class="euiFormRow"
           id="some_html_id-row"
@@ -30,6 +84,7 @@ Object {
               style="padding-top: 0px; padding-bottom: 4px;"
             >
               Instead of failing the reindexing operation, ignore any version conflicts during reindexing.
+               
               <button
                 class="euiLink euiLink--primary"
                 type="button"
@@ -223,7 +278,8 @@ Object {
               class="euiFormHelpText euiFormRow__text"
               style="padding-top: 0px; padding-bottom: 4px;"
             >
-              Select an ingest pipeline if you need to transform documents before writing data to the destination index.
+              Select an ingest pipeline to transform documents before writing data to the destination.
+               
               <a
                 class="euiLink euiLink--primary"
                 href="https://opensearch.org/docs/latest/opensearch/reindex-data/#transform-documents-during-reindexing"
@@ -303,6 +359,60 @@ Object {
     <div
       style="padding: 10px 10px;"
     >
+      <div
+        class="euiFormRow"
+        id="some_html_id-row"
+      >
+        <div
+          class="euiFormRow__labelWrapper"
+        >
+          <label
+            class="euiFormLabel euiFormRow__label"
+            for="some_html_id"
+          >
+            Reindex only unique documents
+          </label>
+        </div>
+        <div
+          class="euiFormRow__fieldWrapper"
+        >
+          <div
+            class="euiFormHelpText euiFormRow__text"
+            style="padding-top: 0px; padding-bottom: 4px;"
+          >
+            You can choose to copy only the documents that do not exist in the destination index. By default, OpenSearch will copy all documents from the source index.
+             
+            <button
+              class="euiLink euiLink--primary"
+              type="button"
+            >
+              Learn more.
+            </button>
+          </div>
+          <div
+            class="euiCheckbox"
+          >
+            <input
+              checked=""
+              class="euiCheckbox__input"
+              id="uniqueCheckbox"
+              type="checkbox"
+            />
+            <div
+              class="euiCheckbox__square"
+            />
+            <label
+              class="euiCheckbox__label"
+              for="uniqueCheckbox"
+            >
+              Reindex only unique documents
+            </label>
+          </div>
+        </div>
+      </div>
+      <div
+        class="euiSpacer euiSpacer--l"
+      />
       <div
         class="euiFormRow"
         id="some_html_id-row"
@@ -325,6 +435,7 @@ Object {
             style="padding-top: 0px; padding-bottom: 4px;"
           >
             Instead of failing the reindexing operation, ignore any version conflicts during reindexing.
+             
             <button
               class="euiLink euiLink--primary"
               type="button"
@@ -518,7 +629,8 @@ Object {
             class="euiFormHelpText euiFormRow__text"
             style="padding-top: 0px; padding-bottom: 4px;"
           >
-            Select an ingest pipeline if you need to transform documents before writing data to the destination index.
+            Select an ingest pipeline to transform documents before writing data to the destination.
+             
             <a
               class="euiLink euiLink--primary"
               href="https://opensearch.org/docs/latest/opensearch/reindex-data/#transform-documents-during-reindexing"
diff --git a/public/pages/Reindex/container/Reindex/Reindex.tsx b/public/pages/Reindex/container/Reindex/Reindex.tsx
index 4f5f089f2..485b2ced7 100644
--- a/public/pages/Reindex/container/Reindex/Reindex.tsx
+++ b/public/pages/Reindex/container/Reindex/Reindex.tsx
@@ -59,6 +59,7 @@ interface ReindexState {
   pipelines: EuiComboBoxOptionOption[];
   selectedPipelines?: EuiComboBoxOptionOption[];
   ignoreConflicts: boolean;
+  reindexUniqueDocuments: boolean;
   executing: boolean;
   showCreateIndexFlyout: boolean;
 }
@@ -82,6 +83,7 @@ export default class Reindex extends Component<ReindexProps, ReindexState> {
       subset: false,
       executing: false,
       showCreateIndexFlyout: false,
+      reindexUniqueDocuments: false,
     };
   }
 
@@ -198,7 +200,7 @@ export default class Reindex extends Component<ReindexProps, ReindexState> {
   };
 
   onClickAction = async () => {
-    const { sourceQuery, destination, slices, selectedPipelines, ignoreConflicts, sources, subset } = this.state;
+    const { sourceQuery, destination, slices, selectedPipelines, ignoreConflicts, sources, subset, reindexUniqueDocuments } = this.state;
 
     if (!(await this.validateSource(sources)) || !this.validateDestination(destination) || !this.validateSlices(slices)) {
       return;
@@ -222,7 +224,7 @@ export default class Reindex extends Component<ReindexProps, ReindexState> {
           },
           dest: {
             index: destination.map((item) => item.label)[0],
-            op_type: isDestAsDataStream ? "create" : "index",
+            op_type: isDestAsDataStream || reindexUniqueDocuments ? "create" : "index",
           },
         },
       };
@@ -470,13 +472,17 @@ export default class Reindex extends Component<ReindexProps, ReindexState> {
     this.setState({ ignoreConflicts: e.target.checked });
   };
 
+  onReindexUniqueDocuments = (e: ChangeEvent<HTMLInputElement>): void => {
+    this.setState({ reindexUniqueDocuments: e.target.checked });
+  };
+
   onSubsetChange = (event: EuiSwitchEvent) => {
     this.setState({ subset: event.target.checked });
   };
 
   render() {
     const { sources, destination, sourceQuery, destError, slices, sourceErr, advancedSettingsOpen, showCreateIndexFlyout } = this.state;
-    const { ignoreConflicts: ignoreConflicts, subset, executing, sourceQueryErr } = this.state;
+    const { ignoreConflicts: ignoreConflicts, reindexUniqueDocuments, subset, executing, sourceQueryErr } = this.state;
 
     const advanceTitle = (
       <EuiFlexGroup gutterSize="none" justifyContent="flexStart" alignItems="center">
@@ -500,14 +506,20 @@ export default class Reindex extends Component<ReindexProps, ReindexState> {
     );
 
     const subTitleText = (
-      <EuiText color="subdued" size="s" style={{ padding: "5px 0px" }}>
-        <p style={{ fontWeight: 200 }}>
-          Use reindex to make extensive changes to your index. Reindex will copy data of the source index into another index.{" "}
-          <EuiLink href={this.context.docLinks.links.opensearch.reindexData.base} target="_blank" rel="noopener noreferrer">
-            Learn more.
-          </EuiLink>
-        </p>
-      </EuiText>
+      <CustomFormRow
+        fullWidth
+        label=""
+        helpText={
+          <div>
+            Use reindex to make extensive changes to your index. Reindex will copy data of the source index into another index.{" "}
+            <EuiLink href={this.context.docLinks.links.opensearch.reindexData.base} target="_blank" rel="noopener noreferrer">
+              Learn more.
+            </EuiLink>
+          </div>
+        }
+      >
+        <></>
+      </CustomFormRow>
     );
 
     // expand data streams and aliases
@@ -632,6 +644,8 @@ export default class Reindex extends Component<ReindexProps, ReindexState> {
               onSelectedPipelinesChange={this.onPipelineChange}
               ignoreConflicts={ignoreConflicts}
               onIgnoreConflictsChange={this.onIgnoreConflictsChange}
+              reindexUniqueDocuments={reindexUniqueDocuments}
+              onReindexUniqueDocumentsChange={this.onReindexUniqueDocuments}
             />
           )}
         </ContentPanel>
diff --git a/public/pages/Reindex/container/Reindex/__snapshots__/Reindex.test.tsx.snap b/public/pages/Reindex/container/Reindex/__snapshots__/Reindex.test.tsx.snap
index 6c1940b57..c8e740d82 100644
--- a/public/pages/Reindex/container/Reindex/__snapshots__/Reindex.test.tsx.snap
+++ b/public/pages/Reindex/container/Reindex/__snapshots__/Reindex.test.tsx.snap
@@ -10,32 +10,35 @@ exports[`<Reindex /> spec renders the component 1`] = `
     Reindex
   </h1>
   <div
-    class="euiText euiText--small"
-    style="padding: 5px 0px;"
+    class="euiFormRow euiFormRow--fullWidth"
+    id="some_html_id-row"
   >
     <div
-      class="euiTextColor euiTextColor--subdued"
+      class="euiFormRow__fieldWrapper"
     >
-      <p
-        style="font-weight: 200;"
+      <div
+        class="euiFormHelpText euiFormRow__text"
+        style="padding-top: 0px; padding-bottom: 4px;"
       >
-        Use reindex to make extensive changes to your index. Reindex will copy data of the source index into another index.
-         
-        <a
-          class="euiLink euiLink--primary"
-          href="https://opensearch.org/docs/latest/opensearch/reindex-data/"
-          rel="noopener noreferrer"
-          target="_blank"
-        >
-          Learn more.
-          EuiIconMock
-          <span
-            class="euiScreenReaderOnly"
+        <div>
+          Use reindex to make extensive changes to your index. Reindex will copy data of the source index into another index.
+           
+          <a
+            class="euiLink euiLink--primary"
+            href="https://opensearch.org/docs/latest/opensearch/reindex-data/"
+            rel="noopener noreferrer"
+            target="_blank"
           >
-            (opens in a new tab or window)
-          </span>
-        </a>
-      </p>
+            Learn more.
+            EuiIconMock
+            <span
+              class="euiScreenReaderOnly"
+            >
+              (opens in a new tab or window)
+            </span>
+          </a>
+        </div>
+      </div>
     </div>
   </div>
   <div