diff --git a/CRM/Event/BAO/Participant.php b/CRM/Event/BAO/Participant.php index 90fbc73a1587..b48daaa4b348 100644 --- a/CRM/Event/BAO/Participant.php +++ b/CRM/Event/BAO/Participant.php @@ -578,6 +578,10 @@ public static function pendingToConfirmSpaces($eventId) { * @param bool $checkPermission * Is this a permissioned retrieval? * + * @deprecated only called from event search, but without most of the details + * returned. Event search should call stop using this & get the metadata + * a better way. + * * @return array * array of importable Fields */ diff --git a/CRM/Event/Import/Form/DataSource.php b/CRM/Event/Import/Form/DataSource.php index 3d96f26535a1..70d9fb68fd8c 100644 --- a/CRM/Event/Import/Form/DataSource.php +++ b/CRM/Event/Import/Form/DataSource.php @@ -41,7 +41,6 @@ public function getUserJobType(): string { public function buildQuickForm() { parent::buildQuickForm(); - $duplicateOptions = []; $this->addRadio('onDuplicate', ts('On Duplicate Entries'), [ CRM_Import_Parser::DUPLICATE_SKIP => ts('Skip'), CRM_Import_Parser::DUPLICATE_UPDATE => ts('Update'), @@ -52,22 +51,6 @@ public function buildQuickForm() { $this->addContactTypeSelector(); } - /** - * Process the uploaded file. - * - * @return void - */ - public function postProcess() { - $this->storeFormValues([ - 'onDuplicate', - 'contactType', - 'dateFormats', - 'savedMapping', - ]); - - $this->submitFileForMapping('CRM_Event_Import_Parser_Participant'); - } - /** * @return CRM_Event_Import_Parser_Participant */ diff --git a/CRM/Event/Import/Form/MapField.php b/CRM/Event/Import/Form/MapField.php index 2c260c1d8734..fe942b6f9eab 100644 --- a/CRM/Event/Import/Form/MapField.php +++ b/CRM/Event/Import/Form/MapField.php @@ -85,7 +85,7 @@ public static function formRule($fields, $files, $self) { } // FIXME: should use the schema titles, not redeclare them $requiredFields = array( - 'participant_contact_id' => ts('Contact ID'), + 'contact_id' => ts('Contact ID'), 'event_id' => ts('Event ID'), ); @@ -93,13 +93,13 @@ public static function formRule($fields, $files, $self) { foreach ($requiredFields as $field => $title) { if (!in_array($field, $importKeys)) { - if ($field == 'participant_contact_id') { + if ($field === 'contact_id') { if (!$contactFieldsBelowWeightMessage || in_array('external_identifier', $importKeys) || in_array('participant_id', $importKeys) ) { continue; } - if ($self->_onDuplicate == CRM_Import_Parser::DUPLICATE_UPDATE) { + if ($self->isUpdateExisting()) { $errors['_qf_default'] .= ts('Missing required field: Provide Participant ID') . '
'; } else { @@ -166,16 +166,15 @@ protected function getParser(): CRM_Event_Import_Parser_Participant { * Get the fields to highlight. * * @return array - * @throws \CRM_Core_Exception */ protected function getHighlightedFields(): array { $highlightedFields = []; - if ($this->getSubmittedValue('onDuplicate') == CRM_Import_Parser::DUPLICATE_UPDATE) { + if ($this->isUpdateExisting()) { $highlightedFieldsArray = [ - 'participant_id', + 'id', 'event_id', 'event_title', - 'participant_status_id', + 'status_id', ]; foreach ($highlightedFieldsArray as $name) { $highlightedFields[] = $name; @@ -184,14 +183,17 @@ protected function getHighlightedFields(): array { elseif ($this->getSubmittedValue('onDuplicate') == CRM_Import_Parser::DUPLICATE_SKIP || $this->getSubmittedValue('onDuplicate') == CRM_Import_Parser::DUPLICATE_NOCHECK ) { + // this should be retrieved from the parser. $highlightedFieldsArray = [ - 'participant_contact_id', + 'contact_id', 'event_id', 'email', 'first_name', 'last_name', + 'organization_name', + 'household_name', 'external_identifier', - 'participant_status_id', + 'status_id', ]; foreach ($highlightedFieldsArray as $name) { $highlightedFields[] = $name; diff --git a/CRM/Event/Import/Form/Preview.php b/CRM/Event/Import/Form/Preview.php index d287c1528b51..3912762f72b7 100644 --- a/CRM/Event/Import/Form/Preview.php +++ b/CRM/Event/Import/Form/Preview.php @@ -21,46 +21,6 @@ */ class CRM_Event_Import_Form_Preview extends CRM_Import_Form_Preview { - /** - * Process the mapped fields and map it into the uploaded file - * preview the file and extract some summary statistics - * - * @return void - */ - public function postProcess() { - $fileName = $this->controller->exportValue('DataSource', 'uploadFile'); - $separator = $this->controller->exportValue('DataSource', 'fieldSeparator'); - $invalidRowCount = $this->get('invalidRowCount'); - $onDuplicate = $this->get('onDuplicate'); - - $mapper = $this->controller->exportValue('MapField', 'mapper'); - $mapperKeys = []; - - foreach ($mapper as $key => $value) { - $mapperKeys[$key] = $mapper[$key][0]; - } - - $parser = new CRM_Event_Import_Parser_Participant($mapperKeys); - $parser->setUserJobID($this->getUserJobID()); - $mapFields = $this->get('fields'); - - foreach ($mapper as $key => $value) { - $header = []; - if (isset($mapFields[$mapper[$key][0]])) { - $header[] = $mapFields[$mapper[$key][0]]; - } - $mapperFields[] = implode(' - ', $header); - } - $parser->run($fileName, $separator, - $mapperFields, - $this->getSubmittedValue('skipColumnHeader'), - CRM_Import_Parser::MODE_IMPORT - ); - - // add all the necessary variables to the form - $parser->set($this, CRM_Import_Parser::MODE_IMPORT); - } - /** * @return CRM_Event_Import_Parser_Participant */ diff --git a/CRM/Event/Import/Parser/Participant.php b/CRM/Event/Import/Parser/Participant.php index 4dc061ee65dd..9c20b42a93b8 100644 --- a/CRM/Event/Import/Parser/Participant.php +++ b/CRM/Event/Import/Parser/Participant.php @@ -85,21 +85,6 @@ public function init() { } } - /** - * Get the metadata field for which importable fields does not key the actual field name. - * - * @return string[] - */ - protected function getOddlyMappedMetadataFields(): array { - $uniqueNames = ['participant_id', 'participant_campaign_id', 'participant_contact_id', 'participant_status_id', 'participant_role_id', 'participant_register_date', 'participant_source', 'participant_is_pay_later']; - $fields = []; - foreach ($uniqueNames as $name) { - $fields[$this->importableFieldsMetadata[$name]['name']] = $name; - } - // Include the parent fields as they could be present if required for matching ...in theory. - return array_merge($fields, parent::getOddlyMappedMetadataFields()); - } - /** * Handle the values in preview mode. * @@ -534,78 +519,6 @@ protected function deprecated_participant_check_params($params, $checkDuplicate return TRUE; } - /** - * @param string $fileName - * @param string $separator - * @param $mapper - * @param bool $skipColumnHeader - * @param int $mode - * - * @return mixed - * @throws Exception - */ - public function run( - $fileName, - $separator, - $mapper, - $skipColumnHeader = FALSE, - $mode = self::MODE_PREVIEW - ) { - if (!is_array($fileName)) { - throw new CRM_Core_Exception('Unable to determine import file'); - } - $fileName = $fileName['name']; - $this->getContactType(); - $this->init(); - - $this->_haveColumnHeader = $skipColumnHeader; - - $this->_separator = $separator; - - $fd = fopen($fileName, "r"); - if (!$fd) { - return FALSE; - } - - $this->_lineCount = 0; - $this->_invalidRowCount = $this->_validCount = 0; - $this->_totalCount = 0; - - $this->_errors = []; - $this->_warnings = []; - - $this->_fileSize = number_format(filesize($fileName) / 1024.0, 2); - - if ($mode == self::MODE_MAPFIELD) { - $this->_rows = []; - } - else { - $this->_activeFieldCount = count($this->_activeFields); - } - - $dataSource = $this->getDataSourceObject(); - $dataSource->setStatuses(['new']); - while ($row = $dataSource->getRow()) { - $this->_lineCount++; - $values = array_values($row); - - $this->_totalCount++; - - if ($mode == self::MODE_MAPFIELD) { - $returnCode = CRM_Import_Parser::VALID; - } - elseif ($mode == self::MODE_PREVIEW) { - $returnCode = $this->preview($values); - } - elseif ($mode == self::MODE_SUMMARY) { - $returnCode = $this->summary($values); - } - elseif ($mode == self::MODE_IMPORT) { - $returnCode = $this->import($values); - } - } - } - /** * Given a list of the importable field keys that the user has selected * set the active fields array to this list @@ -652,96 +565,6 @@ public function addField($name, $title, $type = CRM_Utils_Type::T_INT, $headerPa } } - /** - * Store parser values. - * - * @param CRM_Core_Session $store - * - * @param int $mode - * - * @return void - */ - public function set($store, $mode = self::MODE_SUMMARY) { - $store->set('fileSize', $this->_fileSize); - $store->set('lineCount', $this->_lineCount); - $store->set('separator', $this->_separator); - $store->set('fields', $this->getSelectValues()); - - $store->set('headerPatterns', $this->getHeaderPatterns()); - $store->set('dataPatterns', $this->getDataPatterns()); - $store->set('columnCount', $this->_activeFieldCount); - - $store->set('totalRowCount', $this->_totalCount); - $store->set('validRowCount', $this->_validCount); - $store->set('invalidRowCount', $this->_invalidRowCount); - - switch ($this->_contactType) { - case 'Individual': - $store->set('contactType', CRM_Import_Parser::CONTACT_INDIVIDUAL); - break; - - case 'Household': - $store->set('contactType', CRM_Import_Parser::CONTACT_HOUSEHOLD); - break; - - case 'Organization': - $store->set('contactType', CRM_Import_Parser::CONTACT_ORGANIZATION); - } - - if ($this->_invalidRowCount) { - $store->set('errorsFileName', $this->_errorFileName); - } - if (isset($this->_rows) && !empty($this->_rows)) { - $store->set('dataValues', $this->_rows); - } - - if ($mode == self::MODE_IMPORT) { - $store->set('duplicateRowCount', $this->_duplicateCount); - if ($this->_duplicateCount) { - $store->set('duplicatesFileName', $this->_duplicateFileName); - } - } - } - - /** - * Export data to a CSV file. - * - * @param string $fileName - * @param array $header - * @param array $data - * - * @return void - */ - public static function exportCSV($fileName, $header, $data) { - $output = []; - $fd = fopen($fileName, 'w'); - - foreach ($header as $key => $value) { - $header[$key] = "\"$value\""; - } - $config = CRM_Core_Config::singleton(); - $output[] = implode($config->fieldSeparator, $header); - - foreach ($data as $datum) { - foreach ($datum as $key => $value) { - if (is_array($value)) { - foreach ($value[0] as $k1 => $v1) { - if ($k1 == 'location_type_id') { - continue; - } - $datum[$k1] = $v1; - } - } - else { - $datum[$key] = "\"$value\""; - } - } - $output[] = implode($config->fieldSeparator, $datum); - } - fwrite($fd, implode("\n", $output)); - fclose($fd); - } - /** * Set up field metadata. * @@ -749,9 +572,33 @@ public static function exportCSV($fileName, $header, $data) { */ protected function setFieldMetadata(): void { if (empty($this->importableFieldsMetadata)) { - $fields = CRM_Event_BAO_Participant::importableFields($this->getContactType(), FALSE); - // We can't import event type, the other two duplicate id fields that work fine. - unset($fields['participant_role'], $fields['participant_status'], $fields['event_type']); + $fields = array_merge( + [ + '' => ['title' => ts('- do not import -')], + 'participant_note' => [ + 'title' => ts('Participant Note'), + 'name' => 'participant_note', + 'headerPattern' => '/(participant.)?note$/i', + 'data_type' => CRM_Utils_Type::T_TEXT, + 'options' => FALSE, + ], + ], + CRM_Event_DAO_Participant::import(), + CRM_Core_BAO_CustomField::getFieldsForImport('Participant'), + $this->getContactMatchingFields() + ); + + $fields['participant_contact_id']['title'] .= ' (match to contact)'; + $fields['participant_contact_id']['html']['select'] = $fields['participant_contact_id']['title']; + foreach ($fields as $index => $field) { + if (isset($field['name']) && $field['name'] !== $index) { + // undo unique names - participant is the primary + // entity and no others have conflicting unique names + // if we ever added them the should have unique names - v4api style + $fields[$field['name']] = $field; + unset($fields[$index]); + } + } $this->importableFieldsMetadata = $fields; } } diff --git a/CRM/Import/Form/MapField.php b/CRM/Import/Form/MapField.php index 97147731300a..98ed61af5719 100644 --- a/CRM/Import/Form/MapField.php +++ b/CRM/Import/Form/MapField.php @@ -14,6 +14,7 @@ * @copyright CiviCRM LLC https://civicrm.org/licensing */ +use Civi\Api4\Mapping; use Civi\Api4\MappingField; /** @@ -305,17 +306,16 @@ protected function saveMapping(string $mappingType): void { } //Saving Mapping Details and Records if ($this->getSubmittedValue('saveMapping')) { - $mappingParams = [ + $savedMappingID = Mapping::create(FALSE)->setValues([ 'name' => $this->getSubmittedValue('saveMappingName'), 'description' => $this->getSubmittedValue('saveMappingDesc'), - 'mapping_type_id' => CRM_Core_PseudoConstant::getKey('CRM_Core_BAO_Mapping', 'mapping_type_id', $mappingType), - ]; - $saveMapping = CRM_Core_BAO_Mapping::add($mappingParams); + 'mapping_type_id' => $this->getMappingTypeName(), + ])->execute()->first()['id']; foreach (array_keys($this->getColumnHeaders()) as $i) { - $this->saveMappingField($saveMapping->id, $i, FALSE); + $this->saveMappingField($savedMappingID, $i, FALSE); } - $this->set('savedMapping', $saveMapping->id); + $this->set('savedMapping', $savedMappingID); } } diff --git a/CRM/Import/Forms.php b/CRM/Import/Forms.php index 00b4a6fe3a0f..571ef0f87e8d 100644 --- a/CRM/Import/Forms.php +++ b/CRM/Import/Forms.php @@ -132,7 +132,6 @@ protected function getUserJobSubmittedValues(): array { * @param string $fieldName * * @return mixed|null - * @throws \CRM_Core_Exception */ public function getSubmittedValue(string $fieldName) { if ($fieldName === 'dataSource') { @@ -301,8 +300,6 @@ protected function getDataSourceObject(): ?CRM_Import_DataSource { * This is called as a snippet in DataSourceConfig and * also from DataSource::buildForm to add the fields such * that quick form picks them up. - * - * @throws \CRM_Core_Exception */ protected function getDataSourceFields(): array { $className = $this->getDataSourceClassName(); @@ -330,7 +327,6 @@ protected function getDefaultDataSource(): string { * all forms. * * @return string[] - * @throws \CRM_Core_Exception */ protected function getSubmittableFields(): array { $dataSourceFields = array_fill_keys($this->getDataSourceFields(), 'DataSource'); @@ -639,4 +635,12 @@ public function getHeaderPatterns(): array { return $this->getParser()->getHeaderPatterns(); } + /** + * Has the user chosen to update existing records. + * @return bool + */ + protected function isUpdateExisting(): bool { + return ((int) $this->getSubmittedValue('onDuplicate')) === CRM_Import_Parser::DUPLICATE_UPDATE; + } + } diff --git a/CRM/Import/Parser.php b/CRM/Import/Parser.php index 7bc181668b79..3f84b93ef7ae 100644 --- a/CRM/Import/Parser.php +++ b/CRM/Import/Parser.php @@ -1372,7 +1372,7 @@ protected function getTransformedFieldValue(string $fieldName, $importedValue) { // getOptions does not retrieve these fields with high potential results if ($fieldName === 'event_id') { if (!isset(Civi::$statics[__CLASS__][$fieldName][$importedValue])) { - $event = Event::get()->addWhere('title', '=', $importedValue)->addSelect('id')->execute()->first(); + $event = Event::get()->addClause('OR', ['title', '=', $importedValue], ['id', '=', $importedValue])->addSelect('id')->execute()->first(); Civi::$statics[__CLASS__][$fieldName][$importedValue] = $event['id'] ?? FALSE; } return Civi::$statics[__CLASS__][$fieldName][$importedValue] ?? 'invalid_import_value'; @@ -1486,6 +1486,39 @@ protected function getFieldMetadata(string $fieldName, bool $loadOptions = FALSE return $fieldMetadata; } + /** + * Get the field metadata for fields to be be offered to match the contact. + * + * @return array + * @noinspection PhpDocMissingThrowsInspection + */ + protected function getContactMatchingFields(): array { + $contactFields = CRM_Contact_BAO_Contact::importableFields($this->getContactType(), NULL); + $fields = ['external_identifier' => $contactFields['external_identifier']]; + $fields['external_identifier']['title'] .= ' (match to contact)'; + // Using new Dedupe rule. + $ruleParams = [ + 'contact_type' => $this->getContactType(), + 'used' => $this->getSubmittedValue('dedupe_rule_id') ?? 'Unsupervised', + ]; + $fieldsArray = CRM_Dedupe_BAO_DedupeRule::dedupeRuleFields($ruleParams); + + if (is_array($fieldsArray)) { + foreach ($fieldsArray as $value) { + $customFieldId = CRM_Core_DAO::getFieldValue('CRM_Core_DAO_CustomField', + $value, + 'id', + 'column_name' + ); + $value = trim($customFieldId ? 'custom_' . $customFieldId : $value); + $fields[$value] = $contactFields[$value] ?? NULL; + $title = $fields[$value]['title'] . ' (match to contact)'; + $fields[$value]['title'] = $title; + } + } + return $fields; + } + /** * @param $customFieldID * @param $value @@ -1773,7 +1806,8 @@ public function getMappedFieldLabel(array $mappedField): string { return ''; } $this->setFieldMetadata(); - return $this->getFieldMetadata($mappedField['name'])['title']; + $metadata = $this->getFieldMetadata($mappedField['name']); + return $metadata['html']['label'] ?? $metadata['title']; } /** diff --git a/CRM/Upgrade/Incremental/php/FiveFiftyOne.php b/CRM/Upgrade/Incremental/php/FiveFiftyOne.php index 67f346acf524..3d1a83f16b53 100644 --- a/CRM/Upgrade/Incremental/php/FiveFiftyOne.php +++ b/CRM/Upgrade/Incremental/php/FiveFiftyOne.php @@ -140,14 +140,14 @@ public static function convertMappingFieldLabelsToNames(): bool { $fieldMap = []; foreach ($fields as $fieldName => $field) { - $fieldMap[$field['title']] = $fieldName; + $fieldMap[$field['title']] = $field['name'] ?? $fieldName; if (!empty($field['html']['label'])) { - $fieldMap[$field['html']['label']] = $fieldName; + $fieldMap[$field['html']['label']] = $field['name'] ?? $fieldName; } } $fieldMap[ts('- do not import -')] = 'do_not_import'; - $fieldMap[ts('Participant Status')] = 'participant_status_id'; - $fieldMap[ts('Participant Role')] = 'participant_role_id'; + $fieldMap[ts('Participant Status')] = 'status_id'; + $fieldMap[ts('Participant Role')] = 'role_id'; $fieldMap[ts('Event Title')] = 'event_id'; foreach ($mappings as $mapping) { diff --git a/tests/phpunit/CRM/Event/Import/Parser/ParticipantTest.php b/tests/phpunit/CRM/Event/Import/Parser/ParticipantTest.php index ab521a555b24..9e7de0c17149 100644 --- a/tests/phpunit/CRM/Event/Import/Parser/ParticipantTest.php +++ b/tests/phpunit/CRM/Event/Import/Parser/ParticipantTest.php @@ -112,31 +112,44 @@ protected function getMapperFromFieldMappings(array $mappings): array { /** * Test the full form-flow import. + * + * @dataProvider importData */ - public function testImportCSV() :void { + public function testImportCSV($csv, $mapper) :void { $this->campaignCreate(['name' => 'Soccer cup']); $this->eventCreate(['title' => 'Rain-forest Cup Youth Soccer Tournament']); $this->individualCreate(['email' => 'mum@example.com']); - $this->importCSV('participant.csv', [ - ['name' => 'event_id'], - ['name' => 'participant_campaign_id'], - ['name' => 'email'], - ['name' => 'participant_fee_amount'], - ['name' => 'participant_fee_currency'], - ['name' => 'participant_fee_level'], - ['name' => 'participant_is_pay_later'], - ['name' => 'participant_role_id'], - ['name' => 'participant_source'], - ['name' => 'participant_status_id'], - ['name' => 'participant_register_date'], - ['name' => 'do_not_import'], - ]); + $this->importCSV($csv, $mapper); $dataSource = new CRM_Import_DataSource_CSV($this->userJobID); $row = $dataSource->getRow(); $this->assertEquals('IMPORTED', $row['_status']); $this->callAPISuccessGetSingle('Participant', ['campaign_id' => 'Soccer Cup']); } + /** + * Data provider for importCSV. + */ + public function importData(): array { + $defaultMapper = [ + ['name' => 'event_id'], + ['name' => 'campaign_id'], + ['name' => 'email'], + ['name' => 'fee_amount'], + ['name' => 'fee_currency'], + ['name' => 'fee_level'], + ['name' => 'is_pay_later'], + ['name' => 'role_id'], + ['name' => 'source'], + ['name' => 'status_id'], + ['name' => 'register_date'], + ['name' => 'do_not_import'], + ]; + return [ + ['csv' => 'participant.csv', 'mapper' => $defaultMapper], + ['csv' => 'participant_with_event_id.csv', 'mapper' => $defaultMapper], + ]; + } + /** * @param array $submittedValues *