Skip to content

Commit

Permalink
support reindex unique documents (#546)
Browse files Browse the repository at this point in the history 
Signed-off-by: Hailong Cui <[email protected]>

Signed-off-by: Hailong Cui <[email protected]>
  • Loading branch information
@Hailong-am
Hailong-am authored Jan 9, 2023
1 parent f1c7810 commit 5c03c8f
Show file tree
Hide file tree
Showing 5 changed files with 198 additions and 36 deletions.
8 changes: 8 additions & 0 deletions public/pages/Reindex/components/ReindexAdvancedOptions/ReindexAdvancedOptions.test.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ describe("<ReindexAdvancedOptions /> spec", () => {
ignoreConflicts={true}
onIgnoreConflictsChange={() => {}}
getAllPipelines={async () => [{ label: "pipeline" }]}
reindexUniqueDocuments={true}
onReindexUniqueDocumentsChange={() => {}}
/>
</CoreServicesContext.Provider>
);
Expand All @@ -41,6 +43,8 @@ describe("<ReindexAdvancedOptions /> spec", () => {
ignoreConflicts={false}
onIgnoreConflictsChange={() => {}}
getAllPipelines={async () => [{ label: "pipeline" }]}
reindexUniqueDocuments={true}
onReindexUniqueDocumentsChange={() => {}}
/>
</CoreServicesContext.Provider>
);
Expand All @@ -61,6 +65,8 @@ describe("<ReindexAdvancedOptions /> spec", () => {
ignoreConflicts={false}
onIgnoreConflictsChange={() => {}}
getAllPipelines={async () => Promise.reject("service not available")}
reindexUniqueDocuments={true}
onReindexUniqueDocumentsChange={() => {}}
/>
</CoreServicesContext.Provider>
);
Expand All @@ -82,6 +88,8 @@ describe("<ReindexAdvancedOptions /> spec", () => {
ignoreConflicts={false}
onIgnoreConflictsChange={() => {}}
getAllPipelines={getPipeline}
reindexUniqueDocuments={true}
onReindexUniqueDocumentsChange={() => {}}
/>
</CoreServicesContext.Provider>
);
Expand Down
29 changes: 27 additions & 2 deletions public/pages/Reindex/components/ReindexAdvancedOptions/ReindexAdvancedOptions.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ interface ReindexOptionsProps {
getAllPipelines: () => Promise<EuiComboBoxOptionOption[]>;
ignoreConflicts: boolean;
onIgnoreConflictsChange: (val: ChangeEvent<HTMLInputElement>) => void;
reindexUniqueDocuments: boolean;
onReindexUniqueDocumentsChange: (val: ChangeEvent<HTMLInputElement>) => void;
}

const ReindexAdvancedOptions = (props: ReindexOptionsProps) => {
Expand All @@ -34,6 +36,8 @@ const ReindexAdvancedOptions = (props: ReindexOptionsProps) => {
getAllPipelines,
ignoreConflicts,
onIgnoreConflictsChange,
reindexUniqueDocuments,
onReindexUniqueDocumentsChange,
} = props;

const sliceEnabled = slices !== undefined;
Expand All @@ -52,11 +56,32 @@ const ReindexAdvancedOptions = (props: ReindexOptionsProps) => {

return (
<div style={{ padding: "10px 10px" }}>
<CustomFormRow
label="Reindex only unique documents"
helpText={
<>
You can choose to copy only the documents that do not exist in the destination index. By default, OpenSearch will copy all
documents from the source index.{" "}
<EuiLink href={coreServices.docLinks.links.opensearch.reindexData.unique} target="_blank">
Learn more.
</EuiLink>
</>
}
>
<EuiCheckbox
id="uniqueCheckbox"
label="Reindex only unique documents"
checked={reindexUniqueDocuments}
onChange={onReindexUniqueDocumentsChange}
/>
</CustomFormRow>
<EuiSpacer />

<CustomFormRow
label="Version conflicts"
helpText={
<>
Instead of failing the reindexing operation, ignore any version conflicts during reindexing.
Instead of failing the reindexing operation, ignore any version conflicts during reindexing.{" "}
<EuiLink href={coreServices.docLinks.links.opensearch.reindexData.unique} target="_blank">
Learn more.
</EuiLink>
Expand Down Expand Up @@ -143,7 +168,7 @@ const ReindexAdvancedOptions = (props: ReindexOptionsProps) => {
}
helpText={
<>
Select an ingest pipeline if you need to transform documents before writing data to the destination index.
Select an ingest pipeline to transform documents before writing data to the destination.{" "}
<EuiLink href={coreServices.docLinks.links.opensearch.reindexData.transform} target="_blank">
Learn more.
</EuiLink>
Expand Down
116 changes: 114 additions & 2 deletions ...ndex/components/ReindexAdvancedOptions/__snapshots__/ReindexAdvancedOptions.test.tsx.snap
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,60 @@ Object {
<div
style="padding: 10px 10px;"
>
<div
class="euiFormRow"
id="some_html_id-row"
>
<div
class="euiFormRow__labelWrapper"
>
<label
class="euiFormLabel euiFormRow__label"
for="some_html_id"
>
Reindex only unique documents
</label>
</div>
<div
class="euiFormRow__fieldWrapper"
>
<div
class="euiFormHelpText euiFormRow__text"
style="padding-top: 0px; padding-bottom: 4px;"
>
You can choose to copy only the documents that do not exist in the destination index. By default, OpenSearch will copy all documents from the source index.
<button
class="euiLink euiLink--primary"
type="button"
>
Learn more.
</button>
</div>
<div
class="euiCheckbox"
>
<input
checked=""
class="euiCheckbox__input"
id="uniqueCheckbox"
type="checkbox"
/>
<div
class="euiCheckbox__square"
/>
<label
class="euiCheckbox__label"
for="uniqueCheckbox"
>
Reindex only unique documents
</label>
</div>
</div>
</div>
<div
class="euiSpacer euiSpacer--l"
/>
<div
class="euiFormRow"
id="some_html_id-row"
Expand All @@ -30,6 +84,7 @@ Object {
style="padding-top: 0px; padding-bottom: 4px;"
>
Instead of failing the reindexing operation, ignore any version conflicts during reindexing.
<button
class="euiLink euiLink--primary"
type="button"
Expand Down Expand Up @@ -223,7 +278,8 @@ Object {
class="euiFormHelpText euiFormRow__text"
style="padding-top: 0px; padding-bottom: 4px;"
>
Select an ingest pipeline if you need to transform documents before writing data to the destination index.
Select an ingest pipeline to transform documents before writing data to the destination.
<a
class="euiLink euiLink--primary"
href="https://opensearch.org/docs/latest/opensearch/reindex-data/#transform-documents-during-reindexing"
Expand Down Expand Up @@ -303,6 +359,60 @@ Object {
<div
style="padding: 10px 10px;"
>
<div
class="euiFormRow"
id="some_html_id-row"
>
<div
class="euiFormRow__labelWrapper"
>
<label
class="euiFormLabel euiFormRow__label"
for="some_html_id"
>
Reindex only unique documents
</label>
</div>
<div
class="euiFormRow__fieldWrapper"
>
<div
class="euiFormHelpText euiFormRow__text"
style="padding-top: 0px; padding-bottom: 4px;"
>
You can choose to copy only the documents that do not exist in the destination index. By default, OpenSearch will copy all documents from the source index.
<button
class="euiLink euiLink--primary"
type="button"
>
Learn more.
</button>
</div>
<div
class="euiCheckbox"
>
<input
checked=""
class="euiCheckbox__input"
id="uniqueCheckbox"
type="checkbox"
/>
<div
class="euiCheckbox__square"
/>
<label
class="euiCheckbox__label"
for="uniqueCheckbox"
>
Reindex only unique documents
</label>
</div>
</div>
</div>
<div
class="euiSpacer euiSpacer--l"
/>
<div
class="euiFormRow"
id="some_html_id-row"
Expand All @@ -325,6 +435,7 @@ Object {
style="padding-top: 0px; padding-bottom: 4px;"
>
Instead of failing the reindexing operation, ignore any version conflicts during reindexing.
<button
class="euiLink euiLink--primary"
type="button"
Expand Down Expand Up @@ -518,7 +629,8 @@ Object {
class="euiFormHelpText euiFormRow__text"
style="padding-top: 0px; padding-bottom: 4px;"
>
Select an ingest pipeline if you need to transform documents before writing data to the destination index.
Select an ingest pipeline to transform documents before writing data to the destination.
<a
class="euiLink euiLink--primary"
href="https://opensearch.org/docs/latest/opensearch/reindex-data/#transform-documents-during-reindexing"
Expand Down
36 changes: 25 additions & 11 deletions public/pages/Reindex/container/Reindex/Reindex.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ interface ReindexState {
pipelines: EuiComboBoxOptionOption[];
selectedPipelines?: EuiComboBoxOptionOption[];
ignoreConflicts: boolean;
reindexUniqueDocuments: boolean;
executing: boolean;
showCreateIndexFlyout: boolean;
}
Expand All @@ -82,6 +83,7 @@ export default class Reindex extends Component<ReindexProps, ReindexState> {
subset: false,
executing: false,
showCreateIndexFlyout: false,
reindexUniqueDocuments: false,
};
}

Expand Down Expand Up @@ -198,7 +200,7 @@ export default class Reindex extends Component<ReindexProps, ReindexState> {
};

onClickAction = async () => {
const { sourceQuery, destination, slices, selectedPipelines, ignoreConflicts, sources, subset } = this.state;
const { sourceQuery, destination, slices, selectedPipelines, ignoreConflicts, sources, subset, reindexUniqueDocuments } = this.state;

if (!(await this.validateSource(sources)) || !this.validateDestination(destination) || !this.validateSlices(slices)) {
return;
Expand All @@ -222,7 +224,7 @@ export default class Reindex extends Component<ReindexProps, ReindexState> {
},
dest: {
index: destination.map((item) => item.label)[0],
op_type: isDestAsDataStream ? "create" : "index",
op_type: isDestAsDataStream || reindexUniqueDocuments ? "create" : "index",
},
},
};
Expand Down Expand Up @@ -470,13 +472,17 @@ export default class Reindex extends Component<ReindexProps, ReindexState> {
this.setState({ ignoreConflicts: e.target.checked });
};

onReindexUniqueDocuments = (e: ChangeEvent<HTMLInputElement>): void => {
this.setState({ reindexUniqueDocuments: e.target.checked });
};

onSubsetChange = (event: EuiSwitchEvent) => {
this.setState({ subset: event.target.checked });
};

render() {
const { sources, destination, sourceQuery, destError, slices, sourceErr, advancedSettingsOpen, showCreateIndexFlyout } = this.state;
const { ignoreConflicts: ignoreConflicts, subset, executing, sourceQueryErr } = this.state;
const { ignoreConflicts: ignoreConflicts, reindexUniqueDocuments, subset, executing, sourceQueryErr } = this.state;

const advanceTitle = (
<EuiFlexGroup gutterSize="none" justifyContent="flexStart" alignItems="center">
Expand All @@ -500,14 +506,20 @@ export default class Reindex extends Component<ReindexProps, ReindexState> {
);

const subTitleText = (
<EuiText color="subdued" size="s" style={{ padding: "5px 0px" }}>
<p style={{ fontWeight: 200 }}>
Use reindex to make extensive changes to your index. Reindex will copy data of the source index into another index.{" "}
<EuiLink href={this.context.docLinks.links.opensearch.reindexData.base} target="_blank" rel="noopener noreferrer">
Learn more.
</EuiLink>
</p>
</EuiText>
<CustomFormRow
fullWidth
label=""
helpText={
<div>
Use reindex to make extensive changes to your index. Reindex will copy data of the source index into another index.{" "}
<EuiLink href={this.context.docLinks.links.opensearch.reindexData.base} target="_blank" rel="noopener noreferrer">
Learn more.
</EuiLink>
</div>
}
>
<></>
</CustomFormRow>
);

// expand data streams and aliases
Expand Down Expand Up @@ -632,6 +644,8 @@ export default class Reindex extends Component<ReindexProps, ReindexState> {
onSelectedPipelinesChange={this.onPipelineChange}
ignoreConflicts={ignoreConflicts}
onIgnoreConflictsChange={this.onIgnoreConflictsChange}
reindexUniqueDocuments={reindexUniqueDocuments}
onReindexUniqueDocumentsChange={this.onReindexUniqueDocuments}
/>
)}
</ContentPanel>
Expand Down
Loading

0 comments on commit 5c03c8f

Please sign in to comment.