Merge pull request #23736 from eileenmcnaughton/import_multi

dev/core#3498 Fix mishandled option values
civicrm · Jun 9, 2022 · 2e881e7 · 2e881e7
2 parents 333f329 + 5a9b0b7
commit 2e881e7
Show file tree

Hide file tree

Showing 7 changed files with 218 additions and 97 deletions.
diff --git a/CRM/Custom/Import/Parser/Api.php b/CRM/Custom/Import/Parser/Api.php
@@ -1,6 +1,6 @@
 <?php
 
-use Civi\Api4\CustomGroup;
+use Civi\Api4\CustomField;
 
 /**
  * Class CRM_Custom_Import_Parser_Api
@@ -81,8 +81,8 @@ public function setFieldMetadata(): void {
       $importableFields = $this->getGroupFieldsForImport($customGroupID);
       $this->importableFieldsMetadata = array_merge([
         'do_not_import' => ['title' => ts('- do not import -')],
-        'contact_id' => ['title' => ts('Contact ID'), 'name' => 'contact_id', 'type' => CRM_Utils_Type::T_INT, 'options' => FALSE],
-        'external_identifier' => ['title' => ts('External Identifier'), 'name' => 'external_identifier', 'type' => CRM_Utils_Type::T_INT, 'options' => FALSE],
+        'contact_id' => ['title' => ts('Contact ID'), 'name' => 'contact_id', 'type' => CRM_Utils_Type::T_INT, 'options' => FALSE, 'headerPattern' => '/contact?|id$/i'],
+        'external_identifier' => ['title' => ts('External Identifier'), 'name' => 'external_identifier', 'type' => CRM_Utils_Type::T_INT, 'options' => FALSE, 'headerPattern' => '/external\s?id/i'],
       ], $importableFields);
     }
   }
@@ -175,35 +175,40 @@ private function formatCommonData($params, &$formatted) {
    *
    * @return array
    *
+   * @noinspection PhpDocMissingThrowsInspection
+   * @noinspection PhpUnhandledExceptionInspection
    */
-  private function getGroupFieldsForImport($customGroupID) {
+  private function getGroupFieldsForImport(int $customGroupID): array {
     $importableFields = [];
-    $params = ['custom_group_id' => $customGroupID];
-    $group = CustomGroup::get(FALSE)->addSelect('extends')->addWhere('id', '=', $customGroupID)->execute()->first();
-    $allFields = civicrm_api3('custom_field', 'get', $params);
-    $fields = $allFields['values'];
-    foreach ($fields as $id => $values) {
+    $fields = (array) CustomField::get(FALSE)
+      ->addSelect('*', 'custom_group_id.is_multiple', 'custom_group_id.name', 'custom_group_id.extends')
+      ->addWhere('custom_group_id', '=', $customGroupID)->execute();
+
+    foreach ($fields as $values) {
       $datatype = $values['data_type'] ?? NULL;
       if ($datatype === 'File') {
         continue;
       }
       /* generate the key for the fields array */
-      $key = "custom_$id";
-      $regexp = preg_replace('/[.,;:!?]/', '', CRM_Utils_Array::value(0, $values));
+      $key = 'custom_' . $values['id'];
+      $regexp = preg_replace('/[.,;:!?]/', '', $values['label']);
       $importableFields[$key] = [
         'name' => $key,
         'title' => $values['label'] ?? NULL,
-        'headerPattern' => '/' . preg_quote($regexp, '/') . '/',
+        'headerPattern' => '/' . preg_quote($regexp, '/') . '/i',
         'import' => 1,
-        'custom_field_id' => $id,
-        'options_per_line' => $values['options_per_line'] ?? NULL,
-        'data_type' => $values['data_type'] ?? NULL,
-        'html_type' => $values['html_type'] ?? NULL,
+        'custom_field_id' => $values['id'],
+        'options_per_line' => $values['options_per_line'],
+        'data_type' => $values['data_type'],
+        'html_type' => $values['html_type'],
         'type' => CRM_Core_BAO_CustomField::dataToType()[$values['data_type']],
-        'is_search_range' => $values['is_search_range'] ?? NULL,
-        'date_format' => $values['date_format'] ?? NULL,
-        'time_format' => $values['time_format'] ?? NULL,
-        'extends' => $group['extends'],
+        'is_search_range' => $values['is_search_range'],
+        'date_format' => $values['date_format'],
+        'time_format' => $values['time_format'],
+        'extends' => $values['custom_group_id.extends'],
+        'custom_group_id' => $customGroupID,
+        'custom_group_id.name' => $values['custom_group_id.name'],
+        'is_multiple' => $values['custom_group_id.is_multiple'],
       ];
     }
     return $importableFields;

diff --git a/CRM/Import/Parser.php b/CRM/Import/Parser.php
@@ -571,9 +571,9 @@ public function getSelectTypes() {
    */
   public function getHeaderPatterns(): array {
     $values = [];
-    foreach ($this->_fields as $name => $field) {
-      if (isset($field->_headerPattern)) {
-        $values[$name] = $field->_headerPattern;
+    foreach ($this->importableFieldsMetadata as $name => $field) {
+      if (isset($field['headerPattern'])) {
+        $values[$name] = $field['headerPattern'] ?: '//';
       }
     }
     return $values;
@@ -1441,19 +1441,28 @@ protected function getFieldMetadata(string $fieldName, bool $loadOptions = FALSE
       }
       $optionFieldName = empty($fieldMap[$fieldName]) ? $fieldMetadata['name'] : $fieldName;
 
-      if (!empty($fieldMetadata['custom_group_id'])) {
-        $customField = CustomField::get(FALSE)
-          ->addWhere('id', '=', $fieldMetadata['custom_field_id'])
-          ->addSelect('name', 'custom_group_id.name')
-          ->execute()
-          ->first();
-        $optionFieldName = $customField['custom_group_id.name'] . '.' . $customField['name'];
-      }
-      $options = civicrm_api4($this->getFieldEntity($fieldName), 'getFields', [
-        'loadOptions' => ['id', 'name', 'label', 'abbr'],
-        'where' => [['name', '=', $optionFieldName]],
-        'select' => ['options'],
-      ])->first()['options'];
+      if (!empty($fieldMetadata['custom_field_id']) && !empty($fieldMetadata['is_multiple'])) {
+        $options = civicrm_api4('Custom_' . $fieldMetadata['custom_group_id.name'], 'getFields', [
+          'loadOptions' => ['id', 'name', 'label', 'abbr'],
+          'where' => [['custom_field_id', '=', $fieldMetadata['custom_field_id']]],
+          'select' => ['options'],
+        ])->first()['options'];
+      }
+      else {
+        if (!empty($fieldMetadata['custom_group_id'])) {
+          $customField = CustomField::get(FALSE)
+            ->addWhere('id', '=', $fieldMetadata['custom_field_id'])
+            ->addSelect('name', 'custom_group_id.name')
+            ->execute()
+            ->first();
+          $optionFieldName = $customField['custom_group_id.name'] . '.' . $customField['name'];
+        }
+        $options = civicrm_api4($this->getFieldEntity($fieldName), 'getFields', [
+          'loadOptions' => ['id', 'name', 'label', 'abbr'],
+          'where' => [['name', '=', $optionFieldName]],
+          'select' => ['options'],
+        ])->first()['options'];
+      }
       if (is_array($options)) {
         // We create an array of the possible variants - notably including
         // name AND label as either might be used. We also lower case before checking

diff --git a/tests/phpunit/CRM/Contribute/Import/Parser/ContributionTest.php b/tests/phpunit/CRM/Contribute/Import/Parser/ContributionTest.php
@@ -26,11 +26,6 @@ class CRM_Contribute_Import_Parser_ContributionTest extends CiviUnitTestCase {
    */
   protected $entity = 'Contribution';
 
-  /**
-   * @var int
-   */
-  protected $userJobID;
-
   /**
    * Cleanup function.
    *
@@ -363,21 +358,49 @@ protected function addRandomOption(string $optionGroup = 'payment_instrument'):
   }
 
   /**
-   * @param array $mappings
+   * Get the import's datasource form.
    *
-   * @return array
+   * Defaults to contribution - other classes should override.
+   *
+   * @param array $submittedValues
+   *
+   * @return \CRM_Contribute_Import_Form_DataSource|\CRM_Core_Form|\CRM_Custom_Import_Form_DataSource
+   * @noinspection PhpUnnecessaryLocalVariableInspection
    */
-  protected function getMapperFromFieldMappings(array $mappings): array {
-    $mapper = [];
-    foreach ($mappings as $mapping) {
-      $fieldInput = [$mapping['name']];
-      if (!empty($mapping['soft_credit_type_id'])) {
-        $fieldInput[1] = $mapping['soft_credit_match_field'];
-        $fieldInput[2] = $mapping['soft_credit_type_id'];
-      }
-      $mapper[] = $fieldInput;
-    }
-    return $mapper;
+  protected function getDataSourceForm(array $submittedValues) {
+    return $this->getFormObject('CRM_Contribute_Import_Form_DataSource', $submittedValues);
+  }
+
+  /**
+   * Get the import's mapField form.
+   *
+   * Defaults to contribution - other classes should override.
+   *
+   * @param array $submittedValues
+   *
+   * @return \CRM_Contribute_Import_Form_MapField
+   * @noinspection PhpUnnecessaryLocalVariableInspection
+   */
+  protected function getMapFieldForm(array $submittedValues): CRM_Contribute_Import_Form_MapField {
+    /* @var \CRM_Contribute_Import_Form_MapField $form */
+    $form = $this->getFormObject('CRM_Contribute_Import_Form_MapField', $submittedValues);
+    return $form;
+  }
+
+  /**
+   * Get the import's preview form.
+   *
+   * Defaults to contribution - other classes should override.
+   *
+   * @param array $submittedValues
+   *
+   * @return \CRM_Contribute_Import_Form_Preview
+   * @noinspection PhpUnnecessaryLocalVariableInspection
+   */
+  protected function getPreviewForm(array $submittedValues): CRM_Contribute_Import_Form_Preview {
+    /* @var CRM_Contribute_Import_Form_Preview $form */
+    $form = $this->getFormObject('CRM_Contribute_Import_Form_Preview', $submittedValues);
+    return $form;
   }
 
 }
diff --git a/tests/phpunit/CRM/Custom/Import/Parser/ApiTest.php b/tests/phpunit/CRM/Custom/Import/Parser/ApiTest.php
@@ -0,0 +1,89 @@
+<?php
+/**
+ * @file
+ * File for the CRM_Custom_Import_Parser_ContributionTest class.
+ */
+
+/**
+ *  Test Contribution import parser.
+ *
+ * @package CiviCRM
+ * @group headless
+ */
+class CRM_Custom_Import_Parser_ApiTest extends CiviUnitTestCase {
+
+  use CRMTraits_Custom_CustomDataTrait;
+  use CRMTraits_Import_ParserTrait;
+
+  /**
+   * Test the full form-flow import.
+   */
+  public function testImport(): void {
+    $this->individualCreate();
+    $this->createCustomGroupWithFieldOfType(['is_multiple' => TRUE, 'extends' => 'Contact'], 'select', 'level');
+    $customGroupID = $this->ids['CustomGroup']['level'];
+    $dateFieldID = $this->createDateCustomField(['date_format' => 'yy', 'custom_group_id' => $customGroupID])['id'];
+    $this->importCSV('custom_data_date_select.csv', [
+      ['name' => 'contact_id'],
+      ['name' => $this->getCustomFieldName('levelselect')],
+      ['name' => 'do_not_import'],
+      ['name' => 'custom_' . $dateFieldID],
+    ], ['multipleCustomData' => $customGroupID]);
+    $dataSource = new CRM_Import_DataSource_CSV($this->userJobID);
+    $row = $dataSource->getRow();
+    $this->assertEquals('IMPORTED', $row['_status']);
+    $row = $dataSource->getRow();
+    $this->assertEquals('IMPORTED', $row['_status']);
+    $row = $dataSource->getRow();
+    $this->assertEquals('ERROR', $row['_status']);
+  }
+
+  /**
+   * Get the import's datasource form.
+   *
+   * Defaults to contribution - other classes should override.
+   *
+   * @param array $submittedValues
+   *
+   * @return \CRM_Custom_Import_Form_DataSource
+   * @noinspection PhpUnnecessaryLocalVariableInspection
+   */
+  protected function getDataSourceForm(array $submittedValues): CRM_Custom_Import_Form_DataSource {
+    /* @var \CRM_Custom_Import_Form_DataSource $form */
+    $form = $this->getFormObject('CRM_Custom_Import_Form_DataSource', $submittedValues);
+    return $form;
+  }
+
+  /**
+   * Get the import's mapField form.
+   *
+   * Defaults to contribution - other classes should override.
+   *
+   * @param array $submittedValues
+   *
+   * @return \CRM_Custom_Import_Form_MapField
+   * @noinspection PhpUnnecessaryLocalVariableInspection
+   */
+  protected function getMapFieldForm(array $submittedValues): CRM_Custom_Import_Form_MapField {
+    /* @var \CRM_Custom_Import_Form_MapField $form */
+    $form = $this->getFormObject('CRM_Custom_Import_Form_MapField', $submittedValues);
+    return $form;
+  }
+
+  /**
+   * Get the import's preview form.
+   *
+   * Defaults to contribution - other classes should override.
+   *
+   * @param array $submittedValues
+   *
+   * @return \CRM_Custom_Import_Form_Preview
+   * @noinspection PhpUnnecessaryLocalVariableInspection
+   */
+  protected function getPreviewForm(array $submittedValues): CRM_Custom_Import_Form_Preview {
+    /* @var CRM_Custom_Import_Form_Preview $form */
+    $form = $this->getFormObject('CRM_Custom_Import_Form_Preview', $submittedValues);
+    return $form;
+  }
+
+}
diff --git a/tests/phpunit/CRM/Custom/Import/Parser/data/custom_data_date_select.csv b/tests/phpunit/CRM/Custom/Import/Parser/data/custom_data_date_select.csv
@@ -0,0 +1,4 @@
+The Contact ID,The Level,The School,The Start Year
+3,Red,Ah Ha,2010-12-01
+3,R,Bah bah,2022-01-02
+3,Aqua,Cah cah,2022-02-03
diff --git a/tests/phpunit/CRMTraits/Import/ParserTrait.php b/tests/phpunit/CRMTraits/Import/ParserTrait.php
@@ -16,6 +16,11 @@
  */
 trait CRMTraits_Import_ParserTrait {
 
+  /**
+   * @var int
+   */
+  protected $userJobID;
+
   /**
    * Import the csv file values.
    *
@@ -41,9 +46,13 @@ protected function importCSV(string $csv, array $fieldMappings, array $submitted
       'groups' => [],
     ], $submittedValues);
     $form = $this->getDataSourceForm($submittedValues);
+    $values = $_SESSION['_' . $form->controller->_name . '_container']['values'];
     $form->buildForm();
     $form->postProcess();
     $this->userJobID = $form->getUserJobID();
+    // This gets reset in DataSource so re-do....
+    $_SESSION['_' . $form->controller->_name . '_container']['values'] = $values;
+
     $form = $this->getMapFieldForm($submittedValues);
     $form->setUserJobID($this->userJobID);
     $form->buildForm();
@@ -57,51 +66,21 @@ protected function importCSV(string $csv, array $fieldMappings, array $submitted
   }
 
   /**
-   * Get the import's datasource form.
-   *
-   * Defaults to contribution - other classes should override.
-   *
-   * @param array $submittedValues
-   *
-   * @return \CRM_Contribute_Import_Form_DataSource
-   * @noinspection PhpUnnecessaryLocalVariableInspection
-   */
-  protected function getDataSourceForm(array $submittedValues): CRM_Contribute_Import_Form_DataSource {
-    /* @var \CRM_Contribute_Import_Form_DataSource $form */
-    $form = $this->getFormObject('CRM_Contribute_Import_Form_DataSource', $submittedValues);
-    return $form;
-  }
-
-  /**
-   * Get the import's mapField form.
-   *
-   * Defaults to contribution - other classes should override.
-   *
-   * @param array $submittedValues
-   *
-   * @return \CRM_Contribute_Import_Form_MapField
-   * @noinspection PhpUnnecessaryLocalVariableInspection
-   */
-  protected function getMapFieldForm(array $submittedValues): CRM_Contribute_Import_Form_MapField {
-    /* @var \CRM_Contribute_Import_Form_MapField $form */
-    $form = $this->getFormObject('CRM_Contribute_Import_Form_MapField', $submittedValues);
-    return $form;
-  }
-
-  /**
-   * Get the import's preview form.
-   *
-   * Defaults to contribution - other classes should override.
-   *
-   * @param array $submittedValues
+   * @param array $mappings
    *
-   * @return \CRM_Contribute_Import_Form_Preview
-   * @noinspection PhpUnnecessaryLocalVariableInspection
+   * @return array
    */
-  protected function getPreviewForm(array $submittedValues): CRM_Contribute_Import_Form_Preview {
-    /* @var CRM_Contribute_Import_Form_Preview $form */
-    $form = $this->getFormObject('CRM_Contribute_Import_Form_Preview', $submittedValues);
-    return $form;
+  protected function getMapperFromFieldMappings(array $mappings): array {
+    $mapper = [];
+    foreach ($mappings as $mapping) {
+      $fieldInput = [$mapping['name']];
+      if (!empty($mapping['soft_credit_type_id'])) {
+        $fieldInput[1] = $mapping['soft_credit_match_field'];
+        $fieldInput[2] = $mapping['soft_credit_type_id'];
+      }
+      $mapper[] = $fieldInput;
+    }
+    return $mapper;
   }
 
 }