Skip to content

Commit

Permalink
Merge pull request #182 from jetstreamapp/feat/171
Browse files Browse the repository at this point in the history
Check load file for duplicates
  • Loading branch information
paustint authored Mar 4, 2023
2 parents 17b8ddd + 697029f commit e4b90aa
Show file tree
Hide file tree
Showing 5 changed files with 177 additions and 3 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ import * as fromLoadRecordsState from './load-records.state';
import LoadRecordsFieldMapping from './steps/FieldMapping';
import LoadRecordsLoadAutomationDeploy from './steps/LoadRecordsAutomationDeploy';
import LoadRecordsLoadAutomationRollback from './steps/LoadRecordsAutomationRollback';
import LoadRecordsPerformLoad from './steps/PerformLoad';
import PerformLoad from './steps/PerformLoad';
import PerformLoadCustomMetadata from './steps/PerformLoadCustomMetadata';
import LoadRecordsSelectObjectAndFile from './steps/SelectObjectAndFile';
import { autoMapFields, getFieldMetadata } from './utils/load-records-utils';
Expand Down Expand Up @@ -456,10 +456,11 @@ export const LoadRecords: FunctionComponent<LoadRecordsProps> = ({ featureFlags
{currentStep.name === 'loadRecords' && selectedSObject && inputFileData && (
<span>
{!isCustomMetadataObject ? (
<LoadRecordsPerformLoad
<PerformLoad
selectedOrg={selectedOrg}
orgType={orgType}
selectedSObject={selectedSObject.name}
inputFileHeader={inputFileHeader}
loadType={loadType}
fieldMapping={fieldMapping}
inputFileData={inputFileData}
Expand All @@ -474,6 +475,7 @@ export const LoadRecords: FunctionComponent<LoadRecordsProps> = ({ featureFlags
selectedOrg={selectedOrg}
orgType={orgType}
selectedSObject={selectedSObject.name}
inputFileHeader={inputFileHeader}
fields={mappableFields}
fieldMapping={fieldMapping}
inputFileData={inputFileData}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,119 @@
import { InsertUpdateUpsertDelete, Maybe } from '@jetstream/types';
import { Alert, AutoFullHeightContainer, DataTable, getColumnsForGenericTable, Modal, RowWithKey } from '@jetstream/ui';
import { FunctionComponent, useEffect, useRef, useState } from 'react';
import { Column } from 'react-data-grid';
import { FieldMapping } from '../load-records-types';
import { checkForDuplicateRecords } from '../utils/load-records-utils';

const DUPE_COLUMN = '_DUPLICATE';

const getRowId = ({ _key }: any) => _key;

function getColumnDefinitions(headers: string[], duplicateKey: string): Column<RowWithKey>[] {
return getColumnsForGenericTable([
{ key: DUPE_COLUMN, label: `Duplicate Value (${duplicateKey})`, columnProps: { width: 200, filters: [] } },
...headers.map((header) => ({ key: header, label: header })),
]);
}

export interface LoadRecordsDuplicateWarningProps {
className?: string;
inputFileHeader: string[] | null;
fieldMapping: FieldMapping;
inputFileData: any[];
loadType: InsertUpdateUpsertDelete;
isCustomMetadata?: boolean;
externalId?: string;
}

export const LoadRecordsDuplicateWarning: FunctionComponent<LoadRecordsDuplicateWarningProps> = ({
className,
inputFileHeader,
fieldMapping,
inputFileData,
loadType,
isCustomMetadata = false,
externalId,
}) => {
const isMounted = useRef(true);
const modalRef = useRef();
const [columns, setColumns] = useState<Maybe<Column<RowWithKey>[]>>(null);
const [rows, setRows] = useState<Maybe<RowWithKey[]>>(null);
const [isOpen, setIsOpen] = useState(false);

const [duplicates] = useState(() => checkForDuplicateRecords(fieldMapping, inputFileData, loadType, isCustomMetadata, externalId));

useEffect(() => {
isMounted.current = true;
return () => {
isMounted.current = false;
};
}, []);

useEffect(() => {
if (duplicates && inputFileHeader) {
setRows(
duplicates.duplicateRecords
.flatMap(([key, value]) =>
value.map((row) => ({
[DUPE_COLUMN]: key,
...row,
}))
)
.map((row, i) => ({ ...row, _key: i }))
);
setColumns(getColumnDefinitions(inputFileHeader, duplicates.duplicateKey));
} else {
setColumns(null);
setRows(null);
}
}, [duplicates, inputFileHeader]);

if (!duplicates?.duplicateRecords.length) {
return null;
}

return (
<div className={className}>
{duplicates.duplicateRecords?.length && (
<Alert type="warning" leadingIcon="info">
<span>You have duplicate rows in your spreadsheet.</span>
<button className="slds-button slds-m-left_x-small" onClick={() => setIsOpen(true)}>
View Duplicates
</button>
.
</Alert>
)}

{isOpen && (
<Modal
ref={modalRef}
size="lg"
header="Duplicate Records"
closeOnBackdropClick
tagline={
isCustomMetadata
? 'You can still load your file, but only one of each set of duplicate records will be loaded.'
: 'You can still load your file, but you may get an error for these records.'
}
footer={
<div>
<button className="slds-button slds-button_neutral" onClick={() => setIsOpen(false)}>
Close
</button>
</div>
}
onClose={() => setIsOpen(false)}
>
<div className="slds-is-relative slds-scrollable_x">
<AutoFullHeightContainer fillHeight setHeightAttr bottomBuffer={300}>
{rows && columns && <DataTable columns={columns} data={rows} getRowKey={getRowId} />}
</AutoFullHeightContainer>
</div>
</Modal>
)}
</div>
);
};

export default LoadRecordsDuplicateWarning;
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import { useAmplitude } from '../../core/analytics';
import ConfirmPageChange from '../../core/ConfirmPageChange';
import LoadRecordsResults from '../components/load-results/LoadRecordsResults';
import LoadRecordsAssignmentRules from '../components/LoadRecordsAssignmentRules';
import LoadRecordsDuplicateWarning from '../components/LoadRecordsDuplicateWarning';
import { ApiMode, FieldMapping } from '../load-records-types';

const MAX_BULK = 10000;
Expand Down Expand Up @@ -59,6 +60,7 @@ export interface LoadRecordsPerformLoadProps {
orgType: Maybe<SalesforceOrgUiType>;
selectedSObject: string;
loadType: InsertUpdateUpsertDelete;
inputFileHeader: string[] | null;
fieldMapping: FieldMapping;
inputFileData: any[];
inputZipFileData: Maybe<ArrayBuffer>;
Expand All @@ -71,6 +73,7 @@ export const LoadRecordsPerformLoad: FunctionComponent<LoadRecordsPerformLoadPro
orgType,
selectedSObject,
loadType,
inputFileHeader,
fieldMapping,
inputFileData,
inputZipFileData,
Expand Down Expand Up @@ -193,6 +196,14 @@ export const LoadRecordsPerformLoad: FunctionComponent<LoadRecordsPerformLoadPro
return (
<div>
<ConfirmPageChange actionInProgress={loadInProgress} />
<LoadRecordsDuplicateWarning
className="slds-m-vertical_x-small"
inputFileHeader={inputFileHeader}
fieldMapping={fieldMapping}
inputFileData={inputFileData}
loadType={loadType}
externalId={externalId}
/>
<h1 className="slds-text-heading_medium">Options</h1>
<div className="slds-p-around_small">
<RadioGroup
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import * as fromJetstreamEvents from '../../core/jetstream-events';
import { DownloadType } from '../../shared/load-records-results/load-records-results-types';
import LoadRecordsCustomMetadataResultsTable from '../components/load-results/LoadRecordsCustomMetadataResultsTable';
import LoadRecordsResultsModal from '../components/load-results/LoadRecordsResultsModal';
import LoadRecordsDuplicateWarning from '../components/LoadRecordsDuplicateWarning';
import { DownloadModalData, FieldMapping, FieldWithRelatedEntities, MapOfCustomMetadataRecord, ViewModalData } from '../load-records-types';
import { useDeployMetadataPackage } from '../useDeployMetadataPackage';
import { convertCsvToCustomMetadata, prepareCustomMetadata } from '../utils/load-records-utils';
Expand All @@ -29,6 +30,7 @@ export interface PerformLoadCustomMetadataProps {
selectedOrg: SalesforceOrgUi;
orgType: Maybe<SalesforceOrgUiType>;
selectedSObject: string;
inputFileHeader: string[] | null;
fieldMapping: FieldMapping;
fields: FieldWithRelatedEntities[];
inputFileData: any[];
Expand All @@ -39,6 +41,7 @@ export const PerformLoadCustomMetadata: FunctionComponent<PerformLoadCustomMetad
selectedOrg,
orgType,
selectedSObject,
inputFileHeader,
fieldMapping,
fields,
inputFileData,
Expand Down Expand Up @@ -213,6 +216,14 @@ export const PerformLoadCustomMetadata: FunctionComponent<PerformLoadCustomMetad
onClose={handleViewModalClose}
/>
)}
<LoadRecordsDuplicateWarning
className="slds-m-vertical_x-small"
inputFileHeader={inputFileHeader}
fieldMapping={fieldMapping}
inputFileData={inputFileData}
loadType="UPSERT"
isCustomMetadata
/>
<h1 className="slds-text-heading_medium">Options</h1>
<div className="slds-p-around_small">
<Checkbox
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import { SFDC_BULK_API_NULL_VALUE } from '@jetstream/shared/constants';
import { queryAll, queryWithCache } from '@jetstream/shared/data';
import { describeSObjectWithExtendedTypes, formatNumber } from '@jetstream/shared/ui-utils';
import { REGEX, transformRecordForDataLoad } from '@jetstream/shared/utils';
import { EntityParticleRecord, FieldWithExtendedType, isNotNullish, MapOf, Maybe, SalesforceOrgUi } from '@jetstream/types';
import { EntityParticleRecord, FieldWithExtendedType, InsertUpdateUpsertDelete, MapOf, Maybe, SalesforceOrgUi } from '@jetstream/types';
import { DescribeGlobalSObjectResult } from 'jsforce';
import JSZip from 'jszip';
import groupBy from 'lodash/groupBy';
Expand Down Expand Up @@ -636,6 +636,7 @@ export function convertCsvToCustomMetadata(
logger.log({ metadataByFullName });
return metadataByFullName;
}

export function prepareCustomMetadata(apiVersion, metadata: MapOfCustomMetadataRecord): Promise<ArrayBuffer> {
const zip = new JSZip();
zip.file(
Expand All @@ -656,3 +657,33 @@ export function prepareCustomMetadata(apiVersion, metadata: MapOfCustomMetadataR
});
return zip.generateAsync({ type: 'arraybuffer' });
}

export function checkForDuplicateRecords(
fieldMapping: FieldMapping,
inputFileData: any[],
loadType: InsertUpdateUpsertDelete,
isCustomMetadata = false,
externalId?: string
): {
duplicateKey: string;
duplicateRecords: [string, any[]][];
} | null {
let mappingItem: FieldMappingItem | undefined;
if (isCustomMetadata) {
mappingItem = Object.values(fieldMapping).find(({ targetField }) => targetField === 'DeveloperName');
} else if (loadType === 'UPDATE' || loadType === 'DELETE') {
mappingItem = Object.values(fieldMapping).find(({ targetField }) => targetField === 'Id');
} else if (loadType === 'UPSERT' && externalId) {
mappingItem = Object.values(fieldMapping).find(({ targetField }) => targetField === externalId);
}

if (mappingItem && mappingItem.targetField) {
const rowsByMappedKeyField = groupBy(inputFileData, mappingItem.csvField);
return {
duplicateKey:
mappingItem.csvField === mappingItem.targetField ? mappingItem.csvField : `${mappingItem.csvField} -> ${mappingItem.targetField}`,
duplicateRecords: Object.entries(rowsByMappedKeyField).filter(([key, values]) => values.length > 1),
};
}
return null;
}

0 comments on commit e4b90aa

Please sign in to comment.