Moved cache tag invalidation from ImportQueueWorker to ImportService (#…

…4345)

modules/datastore/datastore.services.yml

@@ -60,6 +60,7 @@ services:
       - '@dkan.datastore.database_table_factory'
       - '@dkan.datastore.logger_channel'
       - '@event_dispatcher'
+      - '@dkan.metastore.reference_lookup'
 
   dkan.datastore.logger_channel:
     parent: logger.channel_base

modules/datastore/modules/datastore_mysql_import/datastore_mysql_import.services.yml

@@ -7,6 +7,7 @@ services:
       - '@dkan.datastore_mysql_import.database_table_factory'
       - '@dkan.datastore.logger_channel'
       - '@event_dispatcher'
+      - '@dkan.metastore.reference_lookup'
 
   dkan.datastore_mysql_import.database_table_factory:
     class: \Drupal\datastore_mysql_import\Storage\MySqlDatabaseTableFactory

modules/datastore/modules/datastore_mysql_import/src/Factory/MysqlImportFactory.php

@@ -7,6 +7,7 @@
 use Drupal\datastore\Storage\ImportJobStoreFactory;
 use Drupal\datastore_mysql_import\Service\MysqlImport;
 use Drupal\datastore_mysql_import\Storage\MySqlDatabaseTableFactory;
+use Drupal\metastore\Reference\ReferenceLookup;
 use Psr\Log\LoggerInterface;
 use Symfony\Component\EventDispatcher\EventDispatcherInterface;
 
@@ -43,6 +44,13 @@ class MysqlImportFactory implements ImportFactoryInterface {
    */
   protected EventDispatcherInterface $eventDispatcher;
 
+  /**
+   * Reference lookup service.
+   *
+   * @var \Drupal\metastore\Reference\ReferenceLookup
+   */
+  protected $referenceLookup;
+
   /**
    * Constructor.
    */
@@ -51,11 +59,13 @@ public function __construct(
     MySqlDatabaseTableFactory $databaseTableFactory,
     LoggerInterface $loggerChannel,
     EventDispatcherInterface $eventDispatcher,
+    ReferenceLookup $referenceLookup,
   ) {
     $this->importJobStoreFactory = $importJobStoreFactory;
     $this->databaseTableFactory = $databaseTableFactory;
     $this->logger = $loggerChannel;
     $this->eventDispatcher = $eventDispatcher;
+    $this->referenceLookup = $referenceLookup;
   }
 
   /**
@@ -72,7 +82,8 @@ public function getInstance(string $identifier, array $config = []) {
       $this->importJobStoreFactory,
       $this->databaseTableFactory,
       $this->logger,
-      $this->eventDispatcher
+      $this->eventDispatcher,
+      $this->referenceLookup
     );
     $importer->setImporterClass(MysqlImport::class);
     return $importer;

modules/datastore/src/Plugin/QueueWorker/ImportQueueWorker.php

@@ -8,7 +8,6 @@
 use Drupal\common\Storage\DatabaseConnectionFactoryInterface;
 use Drupal\common\Storage\ImportedItemInterface;
 use Drupal\datastore\DatastoreService;
-use Drupal\metastore\Reference\ReferenceLookup;
 use Procrastinator\Result;
 use Psr\Log\LoggerInterface;
 use Symfony\Component\DependencyInjection\ContainerInterface;
@@ -41,13 +40,6 @@ class ImportQueueWorker extends QueueWorkerBase implements ContainerFactoryPlugi
    */
   protected $datastore;
 
-  /**
-   * Reference lookup service.
-   *
-   * @var \Drupal\metastore\Reference\ReferenceLookup
-   */
-  protected $referenceLookup;
-
   /**
    * Datastore config settings.
    *
@@ -84,8 +76,6 @@ class ImportQueueWorker extends QueueWorkerBase implements ContainerFactoryPlugi
    *   A DKAN datastore service instance.
    * @param \Psr\Log\LoggerInterface $loggerChannel
    *   A logger channel factory instance.
-   * @param \Drupal\metastore\Reference\ReferenceLookup $referenceLookup
-   *   The reference lookup service.
    * @param \Drupal\common\Storage\DatabaseConnectionFactoryInterface $defaultConnectionFactory
    *   Default database connection factory.
    * @param \Drupal\common\Storage\DatabaseConnectionFactoryInterface $datastoreConnectionFactory
@@ -98,13 +88,11 @@ public function __construct(
     ConfigFactoryInterface $configFactory,
     DatastoreService $datastore,
     LoggerInterface $loggerChannel,
-    ReferenceLookup $referenceLookup,
     DatabaseConnectionFactoryInterface $defaultConnectionFactory,
     DatabaseConnectionFactoryInterface $datastoreConnectionFactory
   ) {
     parent::__construct($configuration, $plugin_id, $plugin_definition);
     $this->datastore = $datastore;
-    $this->referenceLookup = $referenceLookup;
     $this->datastoreConfig = $configFactory->get('datastore.settings');
     $this->databaseQueue = $datastore->getQueueFactory()->get($plugin_id);
     $this->fileSystem = $datastore->getResourceLocalizer()->getFileSystem();
@@ -128,7 +116,6 @@ public static function create(ContainerInterface $container, array $configuratio
       $container->get('config.factory'),
       $container->get('dkan.datastore.service'),
       $container->get('dkan.datastore.logger_channel'),
-      $container->get('dkan.metastore.reference_lookup'),
       $container->get('dkan.common.database_connection_factory'),
       $container->get('dkan.datastore.database_connection_factory')
     );
@@ -244,23 +231,12 @@ protected function processResult(Result $result, mixed $data, bool $queued = FAL
 
       case Result::DONE:
         $this->logger->notice($label . ' for ' . $uid . ' completed.');
-        $this->invalidateCacheTags($uid . '__source');
         break;
     }
 
     return $queued;
   }
 
-  /**
-   * Invalidate all appropriate cache tags for this resource.
-   *
-   * @param mixed $resourceId
-   *   A resource ID.
-   */
-  protected function invalidateCacheTags(mixed $resourceId) {
-    $this->referenceLookup->invalidateReferencerCacheTags('distribution', $resourceId, 'downloadURL');
-  }
-
   /**
    * Requeues the job with extra state information.
    *

modules/datastore/src/Service/Factory/ImportServiceFactory.php

@@ -5,6 +5,7 @@
 use Drupal\datastore\Service\ImportService;
 use Drupal\datastore\Storage\DatabaseTableFactory;
 use Drupal\datastore\Storage\ImportJobStoreFactory;
+use Drupal\metastore\Reference\ReferenceLookup;
 use Psr\Log\LoggerInterface;
 use Symfony\Component\EventDispatcher\EventDispatcherInterface;
 
@@ -41,6 +42,13 @@ class ImportServiceFactory implements ImportFactoryInterface {
    */
   private EventDispatcherInterface $eventDispatcher;
 
+  /**
+   * Reference lookup service.
+   *
+   * @var \Drupal\metastore\Reference\ReferenceLookup
+   */
+  protected $referenceLookup;
+
   /**
    * Constructor.
    */
@@ -49,11 +57,13 @@ public function __construct(
     DatabaseTableFactory $databaseTableFactory,
     LoggerInterface $loggerChannel,
     EventDispatcherInterface $eventDispatcher,
+    ReferenceLookup $referenceLookup,
   ) {
     $this->importJobStoreFactory = $importJobStoreFactory;
     $this->databaseTableFactory = $databaseTableFactory;
     $this->logger = $loggerChannel;
     $this->eventDispatcher = $eventDispatcher;
+    $this->referenceLookup = $referenceLookup;
   }
 
   /**
@@ -67,6 +77,7 @@ public function getInstance(string $identifier, array $config = []) {
         $this->databaseTableFactory,
         $this->logger,
         $this->eventDispatcher,
+        $this->referenceLookup,
       );
     }
     throw new \Exception("config['resource'] is required");

modules/datastore/src/Service/ImportService.php

@@ -10,6 +10,7 @@
 use Drupal\datastore\Storage\DatabaseTable;
 use Drupal\datastore\Storage\DatabaseTableFactory;
 use Drupal\datastore\Storage\ImportJobStoreFactory;
+use Drupal\metastore\Reference\ReferenceLookup;
 use Procrastinator\Result;
 use Psr\Log\LoggerInterface;
 use Symfony\Component\EventDispatcher\EventDispatcherInterface;
@@ -98,6 +99,13 @@ class ImportService {
    */
   private EventDispatcherInterface $eventDispatcher;
 
+  /**
+   * Reference lookup service.
+   *
+   * @var \Drupal\metastore\Reference\ReferenceLookup
+   */
+  protected $referenceLookup;
+
   /**
    * Create a resource service instance.
    *
@@ -111,19 +119,23 @@ class ImportService {
    *   DKAN logger channel service.
    * @param \Symfony\Component\EventDispatcher\EventDispatcherInterface $eventDispatcher
    *   Event dispatcher service.
+   * @param \Drupal\metastore\Reference\ReferenceLookup $referenceLookup
+   *   The reference lookup service.
    */
   public function __construct(
     DataResource $resource,
     ImportJobStoreFactory $importJobStoreFactory,
     DatabaseTableFactory $databaseTableFactory,
     LoggerInterface $loggerChannel,
     EventDispatcherInterface $eventDispatcher,
+    ReferenceLookup $referenceLookup,
   ) {
     $this->resource = $resource;
     $this->importJobStoreFactory = $importJobStoreFactory;
     $this->databaseTableFactory = $databaseTableFactory;
     $this->logger = $loggerChannel;
     $this->eventDispatcher = $eventDispatcher;
+    $this->referenceLookup = $referenceLookup;
   }
 
   /**
@@ -168,6 +180,10 @@ public function import() {
       // Queue the imported resource for post-import processing.
       $post_import_queue = \Drupal::service('queue')->get('post_import');
       $post_import_queue->createItem($data_resource);
+
+      // Invalidate cache tag.
+      $uid = $data_resource->getIdentifier() . '__' . $data_resource->getVersion();
+      $this->invalidateCacheTags($uid . '__source');
     }
   }
 
@@ -245,4 +261,14 @@ public function getStorage(): DatabaseTable {
     return $this->databaseTableFactory->getInstance($datastore_resource->getId(), ['resource' => $datastore_resource]);
   }
 
+  /**
+   * Invalidate all appropriate cache tags for this resource.
+   *
+   * @param mixed $resourceId
+   *   A resource ID.
+   */
+  protected function invalidateCacheTags(mixed $resourceId) {
+    $this->referenceLookup->invalidateReferencerCacheTags('distribution', $resourceId, 'downloadURL');
+  }
+
 }

modules/datastore/tests/src/Kernel/Plugin/QueueWorker/ImportQueueWorkerTest.php

@@ -150,7 +150,6 @@ public function testProcessItemAlreadyImported() {
       $this->container->get('config.factory'),
       $this->container->get('dkan.datastore.service'),
       $this->container->get('dkan.datastore.logger_channel'),
-      $this->container->get('dkan.metastore.reference_lookup'),
       $this->container->get('dkan.common.database_connection_factory'),
       $this->container->get('dkan.datastore.database_connection_factory')
     );
@@ -192,7 +191,6 @@ public function testProcessItemImportException() {
       $this->container->get('config.factory'),
       $this->container->get('dkan.datastore.service'),
       $this->container->get('dkan.datastore.logger_channel'),
-      $this->container->get('dkan.metastore.reference_lookup'),
       $this->container->get('dkan.common.database_connection_factory'),
       $this->container->get('dkan.datastore.database_connection_factory')
     );

modules/datastore/tests/src/Kernel/Service/ImportServiceEventsTest.php

@@ -22,10 +22,17 @@
  */
 class ImportServiceEventsTest extends KernelTestBase implements EventSubscriberInterface {
 
+  protected $strictConfigSchema = FALSE;
+
   protected static $modules = [
     'common',
     'datastore',
     'metastore',
+    'node',
+    'user',
+    'field',
+    'text',
+    'system',
   ];
 
   /**
@@ -63,6 +70,9 @@ public function register(ContainerBuilder $container) {
   }
 
   public function testEvents() {
+    $this->installEntitySchema('node');
+    $this->installConfig(['node']);
+    $this->installConfig(['metastore']);
     // Our result will be DONE.
     $result = new Result();
     $result->setStatus(Result::DONE);
@@ -85,6 +95,7 @@ public function testEvents() {
         $this->container->get('dkan.datastore.database_table_factory'),
         $this->container->get('dkan.datastore.logger_channel'),
         $this->container->get('event_dispatcher'),
+        $this->container->get('dkan.metastore.reference_lookup'),
       ])
       ->onlyMethods(['getImporter', 'getResource'])
       ->getMock();

modules/datastore/tests/src/Kernel/Service/ImportServiceTest.php

@@ -20,10 +20,17 @@
  */
 class ImportServiceTest extends KernelTestBase {
 
+  protected $strictConfigSchema = FALSE;
+
   protected static $modules = [
     'common',
     'datastore',
     'metastore',
+    'node',
+    'user',
+    'field',
+    'text',
+    'system',
   ];
 
   /**
@@ -32,6 +39,9 @@ class ImportServiceTest extends KernelTestBase {
    * @covers ::import
    */
   public function testImport() {
+    $this->installEntitySchema('node');
+    $this->installConfig(['node']);
+    $this->installConfig(['metastore']);
     // Mock some services and statuses.
     $result = new Result();
     $result->setStatus(Result::DONE);
@@ -54,6 +64,7 @@ public function testImport() {
         $this->container->get('dkan.datastore.database_table_factory'),
         $this->container->get('dkan.datastore.logger_channel'),
         $this->container->get('event_dispatcher'),
+        $this->container->get('dkan.metastore.reference_lookup'), 
       ])
       ->getMock();
     $import_service->method('getImporter')
@@ -83,9 +94,14 @@ public function testLogImportError() {
     // Tell the logger channel factory to use a buffering logger.
     $logger = new BufferingLogger();
     $logger_factory = $this->createMock(LoggerChannelFactory::class);
-    $logger_factory->expects($this->once())
+    $logger_factory->expects($this->exactly(2))
       ->method('get')
-      ->with('datastore')
+      ->with(
+        $this->logicalOr(
+          $this->equalTo('datastore'),
+          $this->equalTo('dkan')
+        )
+      )
       ->willReturn($logger);
     $this->container->set('logger.factory', $logger_factory);
 

modules/datastore/tests/src/Unit/Service/Factory/ImportServiceFactoryTest.php

@@ -7,6 +7,7 @@
 use Drupal\datastore\Service\Factory\ImportServiceFactory;
 use Drupal\datastore\Storage\DatabaseTableFactory;
 use Drupal\datastore\Storage\ImportJobStoreFactory;
+use Drupal\metastore\Reference\ReferenceLookup;
 use PHPUnit\Framework\TestCase;
 use Psr\Log\LoggerInterface;
 use Symfony\Component\EventDispatcher\EventDispatcherInterface;
@@ -33,7 +34,8 @@ public function testGetInstanceException() {
         ->disableOriginalConstructor()
         ->getMock(),
       $this->createStub(LoggerInterface::class),
-      $this->createStub(EventDispatcherInterface::class)
+      $this->createStub(EventDispatcherInterface::class),
+      $this->createStub(ReferenceLookup::class)
     );
 
     $this->expectException(\Exception::class);