DKAN-4291 Add empty row remover.

modules/datastore/modules/datastore_mysql_import/config/schema/datastore_mysql_import.schema.yml

@@ -0,0 +1,7 @@
+datastore_mysql_import.settings:
+  type: config_object
+  label: 'Datastore Mysql Import settings'
+  mapping:
+    remove_empty_rows:
+      type: boolean
+      label: 'Remove empty rows in dataset'

modules/datastore/modules/datastore_mysql_import/src/Form/DatastoreMysqlImportSettingsForm.php

@@ -2,10 +2,8 @@
 
 namespace Drupal\datastore_mysql_import\Form;
 
-use Drupal\Core\Config\ConfigFactoryInterface;
 use Drupal\Core\Form\ConfigFormBase;
 use Drupal\Core\Form\FormStateInterface;
-use Symfony\Component\DependencyInjection\ContainerInterface;
 
 /**
  * Datastore MySQL Import settings form.
@@ -15,26 +13,6 @@
  */
 class DatastoreMysqlImportSettingsForm extends ConfigFormBase {
 
-
-  /**
-   * Constructs form.
-   *
-   * @param \Drupal\Core\Config\ConfigFactoryInterface $configFactory
-   *   The factory for configuration objects.
-   */
-  public function __construct(ConfigFactoryInterface $config_factory) {
-    parent::__construct($config_factory);
-  }
-
-  /**
-   * {@inheritdoc}
-   */
-  public static function create(ContainerInterface $container) {
-    return new static(
-      $container->get('config.factory'),
-    );
-  }
-
   /**
    * {@inheritdoc}
    */
@@ -46,14 +24,14 @@ public function getFormId() {
    * {@inheritdoc}
    */
   protected function getEditableConfigNames() {
-    return ['datastore.mysql_import.settings'];
+    return ['datastore_mysql_import.settings'];
   }
 
   /**
    * {@inheritdoc}
    */
   public function buildForm(array $form, FormStateInterface $form_state) {
-    $config = $this->config('datastore.mysql_import.settings');
+    $config = $this->config('datastore_mysql_import.settings');
     $form['remove_empty_rows'] = [
       '#type' => 'checkbox',
       '#title' => $this->t('Enable removal of empty rows in dataset.'),
@@ -67,7 +45,7 @@ public function buildForm(array $form, FormStateInterface $form_state) {
    * {@inheritdoc}
    */
   public function submitForm(array &$form, FormStateInterface $form_state) {
-    $this->config('datastore.mysql_import.settings')
+    $this->config('datastore_mysql_import.settings')
       ->set('remove_empty_rows', $form_state->getValue('remove_empty_rows'))
       ->save();
     parent::submitForm($form, $form_state);

modules/datastore/modules/datastore_mysql_import/src/Service/MysqlImport.php

@@ -108,6 +108,7 @@ protected function runIt() {
     Database::setActiveConnection();
 
     $this->setStatus(Result::DONE);
+    $this->runOptionalPostImportCleanup($this->dataStorage->getTableName(), $spec);
     return NULL;
   }
 
@@ -262,4 +263,41 @@ protected function getSqlStatement(string $file_path, string $table_name, array
     ]);
   }
 
+  /**
+   * Performs optional cleanup steps on imported data.
+   *
+   * @param string $table_name
+   *   The name of the table to clean.
+   * @param array $spec
+   *   The sanitized headers for the imported data.
+   */
+  protected function runOptionalPostImportCleanup(string $table_name, array $spec): void {
+    // @todo this should use dependency injection, but this is not a service.
+    $config = \Drupal::config('datastore_mysql_import.settings');
+    if ($config->get('remove_empty_rows')) {
+      $this->removeEmptyRows($table_name, $spec);
+    }
+  }
+
+  /**
+   * Remove empty rows from the imported data.
+   *
+   * @param string $table_name
+   *   The name of the table to remove empty rows from.
+   * @param array $fields
+   *   The sanitized headers for the imported data.
+   */
+  protected function removeEmptyRows(string $table_name, array $fields): void {
+    $connection = \Drupal::database();
+    $query = $connection->delete($table_name);
+    foreach ($fields as $field) {
+      // Build query to look for records with all empty values for all fields.
+      $query->condition($field['description'], '', '=');
+    }
+    $num_deleted = $query->execute();
+    if ($num_deleted > 0) {
+      // @todo We should log how many were deleted... no logger :(
+    }
+  }
+
 }

modules/datastore/modules/datastore_mysql_import/tests/data/multiple_empty_rows.csv

@@ -0,0 +1,6 @@
+"id","name","score"
+1,"Yessenia",67.1
+,,
+2,"Roberto",2.2
+"","",""
+,,

modules/datastore/modules/datastore_mysql_import/tests/src/Kernel/Service/MysqlImportTest.php

@@ -30,6 +30,14 @@ class MysqlImportTest extends KernelTestBase {
     'metastore',
   ];
 
+  /**
+   * {@inheritdoc}
+   */
+  protected function setUp(): void {
+    // This setUp method is used so we have access to $this->config.
+    parent::setUp();
+  }
+
   public function testTableDuplicateException() {
     $identifier = 'my_id';
     $file_path = dirname(__FILE__, 4) . '/data/columnspaces.csv';
@@ -68,7 +76,6 @@ public function testTableDuplicateException() {
    * Test MysqlImport importer.
    */
   public function testMysqlImporter() {
-
     $identifier = 'my_id';
     $file_path = dirname(__FILE__, 7) . '/tests/data/countries.csv';
     $data_resource = new DataResource($file_path, 'text/csv');
@@ -174,6 +181,80 @@ public function testHasBeenImported() {
     $this->assertEquals(Result::DONE, $result->getStatus(), $result->getError());
   }
 
+  /**
+   * Test MysqlImport with a CSV file with multiple empty rows and removes.
+   */
+  public function testMysqlImporterWithRemoveMultipleEmptyRows() {
+    $identifier = 'my_id';
+    $file_path = dirname(__FILE__, 7) . '/modules/datastore_mysql_import/tests/data/multiple_empty_rows.csv';
+    $data_resource = new DataResource($file_path, 'text/csv');
+
+    $import_factory = $this->container->get('dkan.datastore.service.factory.import');
+    $this->assertInstanceOf(MysqlImportFactory::class, $import_factory);
+
+    /** @var \Drupal\datastore_mysql_import\Service\MysqlImport $mysql_import */
+    $import_service = $import_factory->getInstance(
+      $identifier,
+      ['resource' => $data_resource]
+    );
+    $this->assertInstanceOf(ImportService::class, $import_service);
+    $import_service->setImporterClass(MockQueryVisibilityImport::class);
+    $mysql_import = $import_service->getImporter();
+    $this->assertInstanceOf(MockQueryVisibilityImport::class, $mysql_import);
+    $this->assertInstanceOf(MySqlDatabaseTable::class, $mysql_import->getStorage());
+    $this->toggleEmptyRowRemoval(TRUE);
+
+    // Store the table.
+    $result = $mysql_import->run();
+    // The row cleaner runs as part of run().
+    // Set the config back to off.
+    $this->toggleEmptyRowRemoval(FALSE);
+    $sql_row_count = $mysql_import->getStorage()->count();
+
+    $this->assertEquals(2, $sql_row_count, "There should be only 2 rows.");
+  }
+
+  /**
+   * Test MysqlImport with a CSV file with multiple empty rows no removal.
+   */
+  public function testMysqlImporterWithoutRemoveMultipleEmptyRows() {
+    $identifier = 'my_id';
+    $file_path = dirname(__FILE__, 7) . '/modules/datastore_mysql_import/tests/data/multiple_empty_rows.csv';
+    $data_resource = new DataResource($file_path, 'text/csv');
+
+    $import_factory = $this->container->get('dkan.datastore.service.factory.import');
+    $this->assertInstanceOf(MysqlImportFactory::class, $import_factory);
+
+    /** @var \Drupal\datastore_mysql_import\Service\MysqlImport $mysql_import */
+    $import_service = $import_factory->getInstance(
+      $identifier,
+      ['resource' => $data_resource]
+    );
+    $this->assertInstanceOf(ImportService::class, $import_service);
+    $import_service->setImporterClass(MockQueryVisibilityImport::class);
+    $mysql_import = $import_service->getImporter();
+    $this->assertInstanceOf(MockQueryVisibilityImport::class, $mysql_import);
+    $this->assertInstanceOf(MySqlDatabaseTable::class, $mysql_import->getStorage());
+    $this->toggleEmptyRowRemoval(FALSE);
+    // Store the table.
+    $result = $mysql_import->run();
+    $sql_row_count = $mysql_import->getStorage()->count();
+
+    $this->assertEquals(5, $sql_row_count, "There should be 5 rows.");
+  }
+
+  /**
+   * Toggle the empty row remover setting on or off.
+   *
+   * @param bool $on
+   * The value TRUE or FALSE. TRUE to enable the row removal.
+   */
+  protected function toggleEmptyRowRemoval(bool $on):void {
+    $config = $this->config('datastore_mysql_import.settings');
+    $config->set('remove_empty_rows', $on);
+    $config->save();
+  }
+
   /**
    * Test MysqlImport importer with an empty CSV file.
    */