Skip to content

Commit

Permalink
Check load file for duplicates
Browse files Browse the repository at this point in the history
Check if load file has duplicate records

Show warning if duplicates exist and allow user to view duplicates in modal

resolves #171
  • Loading branch information
paustint committed Mar 3, 2023
1 parent 17b8ddd commit 697029f
Show file tree
Hide file tree
Showing 5 changed files with 177 additions and 3 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ import * as fromLoadRecordsState from './load-records.state';
import LoadRecordsFieldMapping from './steps/FieldMapping';
import LoadRecordsLoadAutomationDeploy from './steps/LoadRecordsAutomationDeploy';
import LoadRecordsLoadAutomationRollback from './steps/LoadRecordsAutomationRollback';
import LoadRecordsPerformLoad from './steps/PerformLoad';
import PerformLoad from './steps/PerformLoad';
import PerformLoadCustomMetadata from './steps/PerformLoadCustomMetadata';
import LoadRecordsSelectObjectAndFile from './steps/SelectObjectAndFile';
import { autoMapFields, getFieldMetadata } from './utils/load-records-utils';
Expand Down Expand Up @@ -456,10 +456,11 @@ export const LoadRecords: FunctionComponent<LoadRecordsProps> = ({ featureFlags
{currentStep.name === 'loadRecords' && selectedSObject && inputFileData && (
<span>
{!isCustomMetadataObject ? (
<LoadRecordsPerformLoad
<PerformLoad
selectedOrg={selectedOrg}
orgType={orgType}
selectedSObject={selectedSObject.name}
inputFileHeader={inputFileHeader}
loadType={loadType}
fieldMapping={fieldMapping}
inputFileData={inputFileData}
Expand All @@ -474,6 +475,7 @@ export const LoadRecords: FunctionComponent<LoadRecordsProps> = ({ featureFlags
selectedOrg={selectedOrg}
orgType={orgType}
selectedSObject={selectedSObject.name}
inputFileHeader={inputFileHeader}
fields={mappableFields}
fieldMapping={fieldMapping}
inputFileData={inputFileData}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,119 @@
import { InsertUpdateUpsertDelete, Maybe } from '@jetstream/types';
import { Alert, AutoFullHeightContainer, DataTable, getColumnsForGenericTable, Modal, RowWithKey } from '@jetstream/ui';
import { FunctionComponent, useEffect, useRef, useState } from 'react';
import { Column } from 'react-data-grid';
import { FieldMapping } from '../load-records-types';
import { checkForDuplicateRecords } from '../utils/load-records-utils';

const DUPE_COLUMN = '_DUPLICATE';

const getRowId = ({ _key }: any) => _key;

function getColumnDefinitions(headers: string[], duplicateKey: string): Column<RowWithKey>[] {
return getColumnsForGenericTable([
{ key: DUPE_COLUMN, label: `Duplicate Value (${duplicateKey})`, columnProps: { width: 200, filters: [] } },
...headers.map((header) => ({ key: header, label: header })),
]);
}

export interface LoadRecordsDuplicateWarningProps {
className?: string;
inputFileHeader: string[] | null;
fieldMapping: FieldMapping;
inputFileData: any[];
loadType: InsertUpdateUpsertDelete;
isCustomMetadata?: boolean;
externalId?: string;
}

export const LoadRecordsDuplicateWarning: FunctionComponent<LoadRecordsDuplicateWarningProps> = ({
className,
inputFileHeader,
fieldMapping,
inputFileData,
loadType,
isCustomMetadata = false,
externalId,
}) => {
const isMounted = useRef(true);
const modalRef = useRef();
const [columns, setColumns] = useState<Maybe<Column<RowWithKey>[]>>(null);
const [rows, setRows] = useState<Maybe<RowWithKey[]>>(null);
const [isOpen, setIsOpen] = useState(false);

const [duplicates] = useState(() => checkForDuplicateRecords(fieldMapping, inputFileData, loadType, isCustomMetadata, externalId));

useEffect(() => {
isMounted.current = true;
return () => {
isMounted.current = false;
};
}, []);

useEffect(() => {
if (duplicates && inputFileHeader) {
setRows(
duplicates.duplicateRecords
.flatMap(([key, value]) =>
value.map((row) => ({
[DUPE_COLUMN]: key,
...row,
}))
)
.map((row, i) => ({ ...row, _key: i }))
);
setColumns(getColumnDefinitions(inputFileHeader, duplicates.duplicateKey));
} else {
setColumns(null);
setRows(null);
}
}, [duplicates, inputFileHeader]);

if (!duplicates?.duplicateRecords.length) {
return null;
}

return (
<div className={className}>
{duplicates.duplicateRecords?.length && (
<Alert type="warning" leadingIcon="info">
<span>You have duplicate rows in your spreadsheet.</span>
<button className="slds-button slds-m-left_x-small" onClick={() => setIsOpen(true)}>
View Duplicates
</button>
.
</Alert>
)}

{isOpen && (
<Modal
ref={modalRef}
size="lg"
header="Duplicate Records"
closeOnBackdropClick
tagline={
isCustomMetadata
? 'You can still load your file, but only one of each set of duplicate records will be loaded.'
: 'You can still load your file, but you may get an error for these records.'
}
footer={
<div>
<button className="slds-button slds-button_neutral" onClick={() => setIsOpen(false)}>
Close
</button>
</div>
}
onClose={() => setIsOpen(false)}
>
<div className="slds-is-relative slds-scrollable_x">
<AutoFullHeightContainer fillHeight setHeightAttr bottomBuffer={300}>
{rows && columns && <DataTable columns={columns} data={rows} getRowKey={getRowId} />}
</AutoFullHeightContainer>
</div>
</Modal>
)}
</div>
);
};

export default LoadRecordsDuplicateWarning;
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import { useAmplitude } from '../../core/analytics';
import ConfirmPageChange from '../../core/ConfirmPageChange';
import LoadRecordsResults from '../components/load-results/LoadRecordsResults';
import LoadRecordsAssignmentRules from '../components/LoadRecordsAssignmentRules';
import LoadRecordsDuplicateWarning from '../components/LoadRecordsDuplicateWarning';
import { ApiMode, FieldMapping } from '../load-records-types';

const MAX_BULK = 10000;
Expand Down Expand Up @@ -59,6 +60,7 @@ export interface LoadRecordsPerformLoadProps {
orgType: Maybe<SalesforceOrgUiType>;
selectedSObject: string;
loadType: InsertUpdateUpsertDelete;
inputFileHeader: string[] | null;
fieldMapping: FieldMapping;
inputFileData: any[];
inputZipFileData: Maybe<ArrayBuffer>;
Expand All @@ -71,6 +73,7 @@ export const LoadRecordsPerformLoad: FunctionComponent<LoadRecordsPerformLoadPro
orgType,
selectedSObject,
loadType,
inputFileHeader,
fieldMapping,
inputFileData,
inputZipFileData,
Expand Down Expand Up @@ -193,6 +196,14 @@ export const LoadRecordsPerformLoad: FunctionComponent<LoadRecordsPerformLoadPro
return (
<div>
<ConfirmPageChange actionInProgress={loadInProgress} />
<LoadRecordsDuplicateWarning
className="slds-m-vertical_x-small"
inputFileHeader={inputFileHeader}
fieldMapping={fieldMapping}
inputFileData={inputFileData}
loadType={loadType}
externalId={externalId}
/>
<h1 className="slds-text-heading_medium">Options</h1>
<div className="slds-p-around_small">
<RadioGroup
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import * as fromJetstreamEvents from '../../core/jetstream-events';
import { DownloadType } from '../../shared/load-records-results/load-records-results-types';
import LoadRecordsCustomMetadataResultsTable from '../components/load-results/LoadRecordsCustomMetadataResultsTable';
import LoadRecordsResultsModal from '../components/load-results/LoadRecordsResultsModal';
import LoadRecordsDuplicateWarning from '../components/LoadRecordsDuplicateWarning';
import { DownloadModalData, FieldMapping, FieldWithRelatedEntities, MapOfCustomMetadataRecord, ViewModalData } from '../load-records-types';
import { useDeployMetadataPackage } from '../useDeployMetadataPackage';
import { convertCsvToCustomMetadata, prepareCustomMetadata } from '../utils/load-records-utils';
Expand All @@ -29,6 +30,7 @@ export interface PerformLoadCustomMetadataProps {
selectedOrg: SalesforceOrgUi;
orgType: Maybe<SalesforceOrgUiType>;
selectedSObject: string;
inputFileHeader: string[] | null;
fieldMapping: FieldMapping;
fields: FieldWithRelatedEntities[];
inputFileData: any[];
Expand All @@ -39,6 +41,7 @@ export const PerformLoadCustomMetadata: FunctionComponent<PerformLoadCustomMetad
selectedOrg,
orgType,
selectedSObject,
inputFileHeader,
fieldMapping,
fields,
inputFileData,
Expand Down Expand Up @@ -213,6 +216,14 @@ export const PerformLoadCustomMetadata: FunctionComponent<PerformLoadCustomMetad
onClose={handleViewModalClose}
/>
)}
<LoadRecordsDuplicateWarning
className="slds-m-vertical_x-small"
inputFileHeader={inputFileHeader}
fieldMapping={fieldMapping}
inputFileData={inputFileData}
loadType="UPSERT"
isCustomMetadata
/>
<h1 className="slds-text-heading_medium">Options</h1>
<div className="slds-p-around_small">
<Checkbox
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import { SFDC_BULK_API_NULL_VALUE } from '@jetstream/shared/constants';
import { queryAll, queryWithCache } from '@jetstream/shared/data';
import { describeSObjectWithExtendedTypes, formatNumber } from '@jetstream/shared/ui-utils';
import { REGEX, transformRecordForDataLoad } from '@jetstream/shared/utils';
import { EntityParticleRecord, FieldWithExtendedType, isNotNullish, MapOf, Maybe, SalesforceOrgUi } from '@jetstream/types';
import { EntityParticleRecord, FieldWithExtendedType, InsertUpdateUpsertDelete, MapOf, Maybe, SalesforceOrgUi } from '@jetstream/types';
import { DescribeGlobalSObjectResult } from 'jsforce';
import JSZip from 'jszip';
import groupBy from 'lodash/groupBy';
Expand Down Expand Up @@ -636,6 +636,7 @@ export function convertCsvToCustomMetadata(
logger.log({ metadataByFullName });
return metadataByFullName;
}

export function prepareCustomMetadata(apiVersion, metadata: MapOfCustomMetadataRecord): Promise<ArrayBuffer> {
const zip = new JSZip();
zip.file(
Expand All @@ -656,3 +657,33 @@ export function prepareCustomMetadata(apiVersion, metadata: MapOfCustomMetadataR
});
return zip.generateAsync({ type: 'arraybuffer' });
}

export function checkForDuplicateRecords(
fieldMapping: FieldMapping,
inputFileData: any[],
loadType: InsertUpdateUpsertDelete,
isCustomMetadata = false,
externalId?: string
): {
duplicateKey: string;
duplicateRecords: [string, any[]][];
} | null {
let mappingItem: FieldMappingItem | undefined;
if (isCustomMetadata) {
mappingItem = Object.values(fieldMapping).find(({ targetField }) => targetField === 'DeveloperName');
} else if (loadType === 'UPDATE' || loadType === 'DELETE') {
mappingItem = Object.values(fieldMapping).find(({ targetField }) => targetField === 'Id');
} else if (loadType === 'UPSERT' && externalId) {
mappingItem = Object.values(fieldMapping).find(({ targetField }) => targetField === externalId);
}

if (mappingItem && mappingItem.targetField) {
const rowsByMappedKeyField = groupBy(inputFileData, mappingItem.csvField);
return {
duplicateKey:
mappingItem.csvField === mappingItem.targetField ? mappingItem.csvField : `${mappingItem.csvField} -> ${mappingItem.targetField}`,
duplicateRecords: Object.entries(rowsByMappedKeyField).filter(([key, values]) => values.length > 1),
};
}
return null;
}

0 comments on commit 697029f

Please sign in to comment.