Merge pull request #23712 from eileenmcnaughton/import_queue

Enable QueueRunner for all imports
civicrm · Jun 9, 2022 · a56a8ca · a56a8ca
2 parents 2e881e7 + 79091ae
commit a56a8ca
Show file tree

Hide file tree

Showing 10 changed files with 103 additions and 139 deletions.
diff --git a/CRM/Contact/Import/Form/Preview.php b/CRM/Contact/Import/Form/Preview.php
@@ -192,18 +192,7 @@ public function postProcess(): void {
     }
 
     CRM_Utils_Address_USPS::disable($this->getSubmittedValue('disableUSPS'));
-
-    // run the import
-
-    $parser = $this->getParser();
-    $parser->queue();
-    $queue = Civi::queue('user_job_' . $this->getUserJobID());
-    $runner = new CRM_Queue_Runner([
-      'queue' => $queue,
-      'errorMode' => CRM_Queue_Runner::ERROR_ABORT,
-      'onEndUrl' => CRM_Utils_System::url('civicrm/import/contact/summary', ['user_job_id' => $this->getUserJobID(), 'reset' => 1]),
-    ]);
-    $runner->runAllViaWeb();
+    $this->runTheImport();
   }
 
   /**

diff --git a/CRM/Contact/Import/Form/Summary.php b/CRM/Contact/Import/Form/Summary.php
@@ -30,7 +30,8 @@ class CRM_Contact_Import_Form_Summary extends CRM_Import_Form_Summary {
    */
   public function preProcess() {
     $userJobID = CRM_Utils_Request::retrieve('user_job_id', 'String', $this, TRUE);
-    $userJob = UserJob::get(TRUE)->addWhere('id', '=', $userJobID)->execute()->first();
+    $userJob = UserJob::get(TRUE)->addWhere('id', '=', $userJobID)->addSelect('metadata', 'type_id:label')->execute()->first();
+    $this->setTitle($userJob['type_id:label']);
     $onDuplicate = $userJob['metadata']['submitted_values']['onDuplicate'];
     $this->assign('dupeError', FALSE);
 
@@ -46,17 +47,11 @@ public function preProcess() {
       $this->assign('dupeError', TRUE);
     }
 
-    $this->assign('groupAdditions', $this->getUserJob()['metadata']['summary_info']['groups']);
-    $this->assign('tagAdditions', $this->getUserJob()['metadata']['summary_info']['tags']);
+    $this->assign('groupAdditions', $this->getUserJob()['metadata']['summary_info']['groups'] ?? []);
+    $this->assign('tagAdditions', $this->getUserJob()['metadata']['summary_info']['tags'] ?? []);
     $this->assignOutputURLs();
     $session = CRM_Core_Session::singleton();
     $session->pushUserContext(CRM_Utils_System::url('civicrm/import/contact', 'reset=1'));
   }
 
-  /**
-   * Clean up the import table we used.
-   */
-  public function postProcess() {
-  }
-
 }
diff --git a/CRM/Import/Form/Preview.php b/CRM/Import/Form/Preview.php
@@ -124,7 +124,25 @@ protected function assignPreviewVariables(): void {
    * @return void
    */
   public function postProcess() {
-    CRM_Import_Parser::runImport(NULL, $this->getUserJobID(), 0);
+    $this->runTheImport();
+  }
+
+  /**
+   * Run the import.
+   */
+  protected function runTheImport(): void {
+    $parser = $this->getParser();
+    $parser->queue();
+    $queue = Civi::queue('user_job_' . $this->getUserJobID());
+    $runner = new CRM_Queue_Runner([
+      'queue' => $queue,
+      'errorMode' => CRM_Queue_Runner::ERROR_ABORT,
+      'onEndUrl' => CRM_Utils_System::url('civicrm/import/contact/summary', [
+        'user_job_id' => $this->getUserJobID(),
+        'reset' => 1,
+      ]),
+    ]);
+    $runner->runAllViaWeb();
   }
 
 }
diff --git a/CRM/Member/Import/Form/MapField.php b/CRM/Member/Import/Form/MapField.php
@@ -45,8 +45,6 @@ public function buildQuickForm() {
       unset($sel1['membership_id']);
     }
 
-    $sel2[''] = NULL;
-
     $js = "<script type='text/javascript'>\n";
     $formName = 'document.forms.' . $this->_name;
 
@@ -113,7 +111,7 @@ public function buildQuickForm() {
           );
         }
       }
-      $sel->setOptions(array($sel1, $sel2));
+      $sel->setOptions([$sel1]);
     }
     $js .= "</script>\n";
     $this->assign('initHideBoxes', $js);

diff --git a/CRM/Member/Import/Form/Summary.php b/CRM/Member/Import/Form/Summary.php
@@ -17,6 +17,9 @@
 
 /**
  * This class summarizes the import results
+ * @todo - this class is no longer used - it just needs to
+ * be removed when the other summary classes are removed from the
+ * import controller.
  */
 class CRM_Member_Import_Form_Summary extends CRM_Import_Form_Summary {
 

diff --git a/templates/CRM/Member/Import/Form/Preview.tpl b/templates/CRM/Member/Import/Form/Preview.tpl
@@ -28,7 +28,7 @@
     <p>{ts}Click 'Import Now' if you are ready to proceed.{/ts}</p>
  </div>
  <div class="crm-submit-buttons">{include file="CRM/common/formButtons.tpl" location="top"}</div>
- {include file="CRM/common/importProgress.tpl"}
+
  {* Summary Preview (record counts) *}
  <table id="preview-counts" class="report">
     <tr><td class="label crm-grid-cell">{ts}Total Rows{/ts}</td>

diff --git a/templates/CRM/Member/Import/Form/Summary.tpl b/templates/CRM/Member/Import/Form/Summary.tpl
@@ -7,72 +7,6 @@
  | and copyright information, see https://civicrm.org/licensing       |
  +--------------------------------------------------------------------+
 *}
-{* Membership Import Wizard - Step 4 (summary of import results AFTER actual data loading) *}
+{* Membership Import Wizard - no longer used - needs to be removed form the
+{* controller & this can be deleted *}
 {* @var $form Contains the array for the form elements and other form associated information assigned to the template by the controller *}
-
-<div class="crm-block crm-form-block crm-member-import-summary-form-block">
- {* WizardHeader.tpl provides visual display of steps thru the wizard as well as title for current step *}
- {include file="CRM/common/WizardHeader.tpl"}
-
- <div class="help">
-    <p>
-    <strong>{ts}Import has completed successfully.{/ts}</strong> {ts}The information below summarizes the results.{/ts}
-    </p>
-
-    {if $invalidRowCount }
-        <p class="error">
-        {ts count=$invalidRowCount plural='CiviCRM has detected invalid data and/or formatting errors in %count records. These records have not been imported.'}CiviCRM has detected invalid data and/or formatting errors in one record. This record has not been imported.{/ts}
-        </p>
-        <p class="error">
-        {ts 1=$downloadErrorRecordsUrl}You can <a href="%1">Download Errors</a>. You may then correct them, and import the new file with the corrected data.{/ts}
-        </p>
-    {/if}
-
-    {if $duplicateRowCount}
-        <p {if $dupeError}class="error"{/if}>
-        {ts count=$duplicateRowCount plural='CiviCRM has detected %count records which are duplicates of existing CiviCRM membership records.'}CiviCRM has detected one record which is a duplicate of existing CiviCRM membership record.{/ts} {$dupeActionString}
-        </p>
-        <p {if $dupeError}class="error"{/if}>
-        {ts 1=$downloadDuplicateRecordsUrl}You can <a href="%1">Download Duplicates</a>. You may then review these records to determine if they are actually duplicates, and correct the transaction IDs for those that are not.{/ts}
-        </p>
-    {/if}
- </div>
- <div class="crm-submit-buttons">{include file="CRM/common/formButtons.tpl" location="top"}</div>
- {* Summary of Import Results (record counts) *}
- <table id="summary-counts" class="report">
-    <tr><td class="label crm-grid-cell">{ts}Total Rows{/ts}</td>
-        <td class="data">{$totalRowCount}</td>
-        <td class="explanation">{ts}Total rows (membership records) in uploaded file.{/ts}</td>
-    </tr>
-
-    {if $invalidRowCount }
-    <tr class="error"><td class="label crm-grid-cell">{ts}Invalid Rows (skipped){/ts}</td>
-        <td class="data">{$invalidRowCount}</td>
-        <td class="explanation">{ts}Rows with invalid data in one or more fields. These rows will be skipped (not imported).{/ts}
-            {if $invalidRowCount}
-                <div class="action-link"><a href="{$downloadErrorRecordsUrl}"><i class="crm-i fa-download" aria-hidden="true"></i> {ts}Download Errors{/ts}</a></div>
-            {/if}
-        </td>
-    </tr>
-    {/if}
-
-    {if $duplicateRowCount}
-    <tr class="error"><td class="label crm-grid-cell">{ts}Duplicate Rows{/ts}</td>
-        <td class="data">{$duplicateRowCount}</td>
-        <td class="explanation">{ts}Rows which are duplicates of existing CiviCRM membership records.{/ts} {$dupeActionString}
-            {if $duplicateRowCount}
-                <p><a href="{$downloadDuplicateRecordsUrl}">{ts}Download Duplicates{/ts}</a></p>
-            {/if}
-        </td>
-    </tr>
-    {/if}
-
-    <tr><td class="label crm-grid-cell">{ts}Records Imported{/ts}</td>
-        <td class="data">{$validRowCount}</td>
-        <td class="explanation">{ts}Rows imported successfully.{/ts}</td>
-    </tr>
-
- </table>
-
- <div class="crm-submit-buttons">{include file="CRM/common/formButtons.tpl" location="bottom"}</div>
- </div>
diff --git a/tests/phpunit/CRM/Activity/Import/Parser/ActivityTest.php b/tests/phpunit/CRM/Activity/Import/Parser/ActivityTest.php
@@ -28,6 +28,7 @@
  */
 class CRM_Activity_Import_Parser_ActivityTest extends CiviUnitTestCase {
   use CRMTraits_Custom_CustomDataTrait;
+  use CRMTraits_Import_ParserTrait;
 
   /**
    * @var int|null
@@ -315,49 +316,6 @@ public function activityImportValidationProvider(): array {
     ];
   }
 
-  /**
-   * Import the csv file values.
-   *
-   * This function uses a flow that mimics the UI flow.
-   *
-   * @param string $csv Name of csv file.
-   * @param array $fieldMappings
-   * @param array $submittedValues
-   */
-  protected function importCSV(string $csv, array $fieldMappings, array $submittedValues = []): void {
-    $submittedValues = array_merge([
-      'uploadFile' => ['name' => __DIR__ . '/data/' . $csv],
-      'skipColumnHeader' => TRUE,
-      'fieldSeparator' => ',',
-      'contactType' => CRM_Import_Parser::CONTACT_INDIVIDUAL,
-      'mapper' => $this->getMapperFromFieldMappings($fieldMappings),
-      'dataSource' => 'CRM_Import_DataSource_CSV',
-      'file' => ['name' => $csv],
-      'dateFormats' => CRM_Core_Form_Date::DATE_yyyy_mm_dd,
-      'onDuplicate' => CRM_Import_Parser::DUPLICATE_UPDATE,
-      'groups' => [],
-    ], $submittedValues);
-    /* @var \CRM_Activity_Import_Form_DataSource $form */
-    $form = $this->getFormObject('CRM_Activity_Import_Form_DataSource', $submittedValues);
-    $values = $_SESSION['_' . $form->controller->_name . '_container']['values'];
-    $form->buildForm();
-    $form->postProcess();
-    // This gets reset in DataSource so re-do....
-    $_SESSION['_' . $form->controller->_name . '_container']['values'] = $values;
-
-    $this->userJobID = $form->getUserJobID();
-    /* @var CRM_Activity_Import_Form_MapField $form */
-    $form = $this->getFormObject('CRM_Activity_Import_Form_MapField', $submittedValues);
-    $form->setUserJobID($this->userJobID);
-    $form->buildForm();
-    $form->postProcess();
-    /* @var CRM_Activity_Import_Form_Preview $form */
-    $form = $this->getFormObject('CRM_Activity_Import_Form_Preview', $submittedValues);
-    $form->setUserJobID($this->userJobID);
-    $form->buildForm();
-    $form->postProcess();
-  }
-
   /**
    * @param array $mappings
    *
@@ -427,4 +385,52 @@ protected function getUserJobID(array $submittedValues = []): int {
     return $userJobID;
   }
 
+  /**
+   * Get the import's datasource form.
+   *
+   * Defaults to contribution - other classes should override.
+   *
+   * @param array $submittedValues
+   *
+   * @return \CRM_Activity_Import_Form_DataSource
+   * @noinspection PhpUnnecessaryLocalVariableInspection
+   */
+  protected function getDataSourceForm(array $submittedValues): CRM_Activity_Import_Form_DataSource {
+    /* @var \CRM_Activity_Import_Form_DataSource $form */
+    $form = $this->getFormObject('CRM_Activity_Import_Form_DataSource', $submittedValues);
+    return $form;
+  }
+
+  /**
+   * Get the import's mapField form.
+   *
+   * Defaults to contribution - other classes should override.
+   *
+   * @param array $submittedValues
+   *
+   * @return \CRM_Activity_Import_Form_MapField
+   * @noinspection PhpUnnecessaryLocalVariableInspection
+   */
+  protected function getMapFieldForm(array $submittedValues): CRM_Activity_Import_Form_MapField {
+    /* @var \CRM_Activity_Import_Form_MapField $form */
+    $form = $this->getFormObject('CRM_Activity_Import_Form_MapField', $submittedValues);
+    return $form;
+  }
+
+  /**
+   * Get the import's preview form.
+   *
+   * Defaults to contribution - other classes should override.
+   *
+   * @param array $submittedValues
+   *
+   * @return \CRM_Activity_Import_Form_Preview
+   * @noinspection PhpUnnecessaryLocalVariableInspection
+   */
+  protected function getPreviewForm(array $submittedValues): CRM_Activity_Import_Form_Preview {
+    /* @var CRM_Activity_Import_Form_Preview $form */
+    $form = $this->getFormObject('CRM_Activity_Import_Form_Preview', $submittedValues);
+    return $form;
+  }
+
 }
diff --git a/tests/phpunit/CRM/Member/Import/Parser/MembershipTest.php b/tests/phpunit/CRM/Member/Import/Parser/MembershipTest.php
@@ -492,7 +492,17 @@ protected function importCSV(string $csv, array $fieldMappings, array $submitted
     $form = $this->getFormObject('CRM_Member_Import_Form_Preview', $submittedValues);
     $form->setUserJobID($this->userJobID);
     $form->buildForm();
-    $form->postProcess();
+    try {
+      $form->postProcess();
+    }
+    catch (CRM_Core_Exception_PrematureExitException $e) {
+      $queue = Civi::queue('user_job_' . $this->userJobID);
+      $runner = new CRM_Queue_Runner([
+        'queue' => $queue,
+        'errorMode' => CRM_Queue_Runner::ERROR_ABORT,
+      ]);
+      $runner->runAll();
+    }
   }
 
   /**

diff --git a/tests/phpunit/CRMTraits/Import/ParserTrait.php b/tests/phpunit/CRMTraits/Import/ParserTrait.php
@@ -62,7 +62,18 @@ protected function importCSV(string $csv, array $fieldMappings, array $submitted
     $form->setUserJobID($this->userJobID);
     $form->buildForm();
     $this->assertTrue($form->validate());
-    $form->postProcess();
+    try {
+      $form->postProcess();
+      $this->fail('Expected a redirect');
+    }
+    catch (CRM_Core_Exception_PrematureExitException $e) {
+      $queue = Civi::queue('user_job_' . $this->userJobID);
+      $runner = new CRM_Queue_Runner([
+        'queue' => $queue,
+        'errorMode' => CRM_Queue_Runner::ERROR_ABORT,
+      ]);
+      $runner->runAll();
+    }
   }
 
   /**