Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

support reindex unique documents #546

Merged
merged 1 commit into from
Jan 9, 2023
Merged

support reindex unique documents #546

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions public/pages/Reindex/components/ReindexAdvancedOptions/ReindexAdvancedOptions.test.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ describe("<ReindexAdvancedOptions /> spec", () => {
ignoreConflicts={true}
onIgnoreConflictsChange={() => {}}
getAllPipelines={async () => [{ label: "pipeline" }]}
reindexUniqueDocuments={true}
onReindexUniqueDocumentsChange={() => {}}
/>
</CoreServicesContext.Provider>
);
Expand All @@ -41,6 +43,8 @@ describe("<ReindexAdvancedOptions /> spec", () => {
ignoreConflicts={false}
onIgnoreConflictsChange={() => {}}
getAllPipelines={async () => [{ label: "pipeline" }]}
reindexUniqueDocuments={true}
onReindexUniqueDocumentsChange={() => {}}
/>
</CoreServicesContext.Provider>
);
Expand All @@ -61,6 +65,8 @@ describe("<ReindexAdvancedOptions /> spec", () => {
ignoreConflicts={false}
onIgnoreConflictsChange={() => {}}
getAllPipelines={async () => Promise.reject("service not available")}
reindexUniqueDocuments={true}
onReindexUniqueDocumentsChange={() => {}}
/>
</CoreServicesContext.Provider>
);
Expand All @@ -82,6 +88,8 @@ describe("<ReindexAdvancedOptions /> spec", () => {
ignoreConflicts={false}
onIgnoreConflictsChange={() => {}}
getAllPipelines={getPipeline}
reindexUniqueDocuments={true}
onReindexUniqueDocumentsChange={() => {}}
/>
</CoreServicesContext.Provider>
);
Expand Down
29 changes: 27 additions & 2 deletions public/pages/Reindex/components/ReindexAdvancedOptions/ReindexAdvancedOptions.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ interface ReindexOptionsProps {
getAllPipelines: () => Promise<EuiComboBoxOptionOption[]>;
ignoreConflicts: boolean;
onIgnoreConflictsChange: (val: ChangeEvent<HTMLInputElement>) => void;
reindexUniqueDocuments: boolean;
onReindexUniqueDocumentsChange: (val: ChangeEvent<HTMLInputElement>) => void;
}

const ReindexAdvancedOptions = (props: ReindexOptionsProps) => {
Expand All @@ -34,6 +36,8 @@ const ReindexAdvancedOptions = (props: ReindexOptionsProps) => {
getAllPipelines,
ignoreConflicts,
onIgnoreConflictsChange,
reindexUniqueDocuments,
onReindexUniqueDocumentsChange,
} = props;

const sliceEnabled = slices !== undefined;
Expand All @@ -52,11 +56,32 @@ const ReindexAdvancedOptions = (props: ReindexOptionsProps) => {

return (
<div style={{ padding: "10px 10px" }}>
<CustomFormRow
label="Reindex only unique documents"
helpText={
<>
You can choose to copy only the documents that do not exist in the destination index. By default, OpenSearch will copy all
documents from the source index.{" "}
<EuiLink href={coreServices.docLinks.links.opensearch.reindexData.unique} target="_blank">
Learn more.
</EuiLink>
</>
}
>
<EuiCheckbox
id="uniqueCheckbox"
label="Reindex only unique documents"
checked={reindexUniqueDocuments}
onChange={onReindexUniqueDocumentsChange}
/>
</CustomFormRow>
<EuiSpacer />

<CustomFormRow
label="Version conflicts"
helpText={
<>
Instead of failing the reindexing operation, ignore any version conflicts during reindexing.
Instead of failing the reindexing operation, ignore any version conflicts during reindexing.{" "}
<EuiLink href={coreServices.docLinks.links.opensearch.reindexData.unique} target="_blank">
Learn more.
</EuiLink>
Expand Down Expand Up @@ -143,7 +168,7 @@ const ReindexAdvancedOptions = (props: ReindexOptionsProps) => {
}
helpText={
<>
Select an ingest pipeline if you need to transform documents before writing data to the destination index.
Select an ingest pipeline to transform documents before writing data to the destination.{" "}
<EuiLink href={coreServices.docLinks.links.opensearch.reindexData.transform} target="_blank">
Learn more.
</EuiLink>
Expand Down
116 changes: 114 additions & 2 deletions ...ndex/components/ReindexAdvancedOptions/__snapshots__/ReindexAdvancedOptions.test.tsx.snap
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,60 @@ Object {
<div
style="padding: 10px 10px;"
>
<div
class="euiFormRow"
id="some_html_id-row"
>
<div
class="euiFormRow__labelWrapper"
>
<label
class="euiFormLabel euiFormRow__label"
for="some_html_id"
>
Reindex only unique documents
</label>
</div>
<div
class="euiFormRow__fieldWrapper"
>
<div
class="euiFormHelpText euiFormRow__text"
style="padding-top: 0px; padding-bottom: 4px;"
>
You can choose to copy only the documents that do not exist in the destination index. By default, OpenSearch will copy all documents from the source index.

<button
class="euiLink euiLink--primary"
type="button"
>
Learn more.
</button>
</div>
<div
class="euiCheckbox"
>
<input
checked=""
class="euiCheckbox__input"
id="uniqueCheckbox"
type="checkbox"
/>
<div
class="euiCheckbox__square"
/>
<label
class="euiCheckbox__label"
for="uniqueCheckbox"
>
Reindex only unique documents
</label>
</div>
</div>
</div>
<div
class="euiSpacer euiSpacer--l"
/>
<div
class="euiFormRow"
id="some_html_id-row"
Expand All @@ -30,6 +84,7 @@ Object {
style="padding-top: 0px; padding-bottom: 4px;"
>
Instead of failing the reindexing operation, ignore any version conflicts during reindexing.

<button
class="euiLink euiLink--primary"
type="button"
Expand Down Expand Up @@ -223,7 +278,8 @@ Object {
class="euiFormHelpText euiFormRow__text"
style="padding-top: 0px; padding-bottom: 4px;"
>
Select an ingest pipeline if you need to transform documents before writing data to the destination index.
Select an ingest pipeline to transform documents before writing data to the destination.

<a
class="euiLink euiLink--primary"
href="https://opensearch.org/docs/latest/opensearch/reindex-data/#transform-documents-during-reindexing"
Expand Down Expand Up @@ -303,6 +359,60 @@ Object {
<div
style="padding: 10px 10px;"
>
<div
class="euiFormRow"
id="some_html_id-row"
>
<div
class="euiFormRow__labelWrapper"
>
<label
class="euiFormLabel euiFormRow__label"
for="some_html_id"
>
Reindex only unique documents
</label>
</div>
<div
class="euiFormRow__fieldWrapper"
>
<div
class="euiFormHelpText euiFormRow__text"
style="padding-top: 0px; padding-bottom: 4px;"
>
You can choose to copy only the documents that do not exist in the destination index. By default, OpenSearch will copy all documents from the source index.

<button
class="euiLink euiLink--primary"
type="button"
>
Learn more.
</button>
</div>
<div
class="euiCheckbox"
>
<input
checked=""
class="euiCheckbox__input"
id="uniqueCheckbox"
type="checkbox"
/>
<div
class="euiCheckbox__square"
/>
<label
class="euiCheckbox__label"
for="uniqueCheckbox"
>
Reindex only unique documents
</label>
</div>
</div>
</div>
<div
class="euiSpacer euiSpacer--l"
/>
<div
class="euiFormRow"
id="some_html_id-row"
Expand All @@ -325,6 +435,7 @@ Object {
style="padding-top: 0px; padding-bottom: 4px;"
>
Instead of failing the reindexing operation, ignore any version conflicts during reindexing.

<button
class="euiLink euiLink--primary"
type="button"
Expand Down Expand Up @@ -518,7 +629,8 @@ Object {
class="euiFormHelpText euiFormRow__text"
style="padding-top: 0px; padding-bottom: 4px;"
>
Select an ingest pipeline if you need to transform documents before writing data to the destination index.
Select an ingest pipeline to transform documents before writing data to the destination.

<a
class="euiLink euiLink--primary"
href="https://opensearch.org/docs/latest/opensearch/reindex-data/#transform-documents-during-reindexing"
Expand Down
36 changes: 25 additions & 11 deletions public/pages/Reindex/container/Reindex/Reindex.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ interface ReindexState {
pipelines: EuiComboBoxOptionOption[];
selectedPipelines?: EuiComboBoxOptionOption[];
ignoreConflicts: boolean;
reindexUniqueDocuments: boolean;
executing: boolean;
showCreateIndexFlyout: boolean;
}
Expand All @@ -82,6 +83,7 @@ export default class Reindex extends Component<ReindexProps, ReindexState> {
subset: false,
executing: false,
showCreateIndexFlyout: false,
reindexUniqueDocuments: false,
};
}

Expand Down Expand Up @@ -198,7 +200,7 @@ export default class Reindex extends Component<ReindexProps, ReindexState> {
};

onClickAction = async () => {
const { sourceQuery, destination, slices, selectedPipelines, ignoreConflicts, sources, subset } = this.state;
const { sourceQuery, destination, slices, selectedPipelines, ignoreConflicts, sources, subset, reindexUniqueDocuments } = this.state;

if (!(await this.validateSource(sources)) || !this.validateDestination(destination) || !this.validateSlices(slices)) {
return;
Expand All @@ -222,7 +224,7 @@ export default class Reindex extends Component<ReindexProps, ReindexState> {
},
dest: {
index: destination.map((item) => item.label)[0],
op_type: isDestAsDataStream ? "create" : "index",
op_type: isDestAsDataStream || reindexUniqueDocuments ? "create" : "index",
},
},
};
Expand Down Expand Up @@ -470,13 +472,17 @@ export default class Reindex extends Component<ReindexProps, ReindexState> {
this.setState({ ignoreConflicts: e.target.checked });
};

onReindexUniqueDocuments = (e: ChangeEvent<HTMLInputElement>): void => {
this.setState({ reindexUniqueDocuments: e.target.checked });
};

onSubsetChange = (event: EuiSwitchEvent) => {
this.setState({ subset: event.target.checked });
};

render() {
const { sources, destination, sourceQuery, destError, slices, sourceErr, advancedSettingsOpen, showCreateIndexFlyout } = this.state;
const { ignoreConflicts: ignoreConflicts, subset, executing, sourceQueryErr } = this.state;
const { ignoreConflicts: ignoreConflicts, reindexUniqueDocuments, subset, executing, sourceQueryErr } = this.state;

const advanceTitle = (
<EuiFlexGroup gutterSize="none" justifyContent="flexStart" alignItems="center">
Expand All @@ -500,14 +506,20 @@ export default class Reindex extends Component<ReindexProps, ReindexState> {
);

const subTitleText = (
<EuiText color="subdued" size="s" style={{ padding: "5px 0px" }}>
<p style={{ fontWeight: 200 }}>
Use reindex to make extensive changes to your index. Reindex will copy data of the source index into another index.{" "}
<EuiLink href={this.context.docLinks.links.opensearch.reindexData.base} target="_blank" rel="noopener noreferrer">
Learn more.
</EuiLink>
</p>
</EuiText>
<CustomFormRow
fullWidth
label=""
helpText={
<div>
Use reindex to make extensive changes to your index. Reindex will copy data of the source index into another index.{" "}
<EuiLink href={this.context.docLinks.links.opensearch.reindexData.base} target="_blank" rel="noopener noreferrer">
Learn more.
</EuiLink>
</div>
}
>
<></>
</CustomFormRow>
);

// expand data streams and aliases
Expand Down Expand Up @@ -632,6 +644,8 @@ export default class Reindex extends Component<ReindexProps, ReindexState> {
onSelectedPipelinesChange={this.onPipelineChange}
ignoreConflicts={ignoreConflicts}
onIgnoreConflictsChange={this.onIgnoreConflictsChange}
reindexUniqueDocuments={reindexUniqueDocuments}
onReindexUniqueDocumentsChange={this.onReindexUniqueDocuments}
/>
)}
</ContentPanel>
Expand Down
Loading