Merge pull request #14032 from craftcms/feature/acc-349-bulk-element-…

…operation-tracking

Bulk element operation tracking + event

CHANGELOG-WIP.md

@@ -61,6 +61,7 @@
 - Entry type names and handles must now be unique globally, rather than just within a single section. Existing entry type names and handles will be renamed automatically where needed, to ensure uniqueness.
 - Assets, categories, entries, and tags now support eager-loading paths prefixed with a field layout provider’s handle (e.g. `myEntryType:myField`).
 - Element queries now have an `eagerly` param, which can be used to lazily eager-load the resulting elements for all peer elements, when `all()`, `collect()`, `one()`, `nth()`, or `count()` is called.
+- Element queries now have an `inBulkOp` param, which limits the results to elements which were involved in a bulk operation. ([#14032](https://github.com/craftcms/cms/pull/14032))
 - Address queries now have `addressLine1`, `addressLine2`, `administrativeArea`, `countryCode`, `dependentLocality`, `firstName`, `fullName`, `lastName`, `locality`, `organizationTaxId`, `organization`, `postalCode`, and `sortingCode` params.
 - Entry queries now have `field`, `fieldId`, `primaryOwner`, `primaryOwnerId`, `owner`, `ownerId`, `allowOwnerDrafts`, and `allowOwnerRevisions` params.
 - Entries’ GraphQL type names are now formatted as `<entryTypeHandle>_Entry`, and are no longer prefixed with their section’s handle. (That goes for Matrix-nested entries as well.)
@@ -171,6 +172,7 @@
 - Added `craft\enums\PropagationMethod`.
 - Added `craft\enums\TimePeriod`.
 - Added `craft\events\BulkElementsEvent`.
+- Added `craft\events\BulkOpEvent`. ([#14032](https://github.com/craftcms/cms/pull/14032))
 - Added `craft\events\DefineEntryTypesForFieldEvent`.
 - Added `craft\events\DefineFieldHtmlEvent::$inline`.
 - Added `craft\fieldlayoutelements\BaseField::$includeInCards`.
@@ -226,6 +228,11 @@
 - Added `craft\models\Section::getCpEditUrl()`.
 - Added `craft\models\Volume::getSubpath()`.
 - Added `craft\models\Volume::setSubpath()`.
+- Added `craft\queue\BaseBatchedElementJob`. ([#14032](https://github.com/craftcms/cms/pull/14032))
+- Added `craft\queue\BaseBatchedJob::after()`.
+- Added `craft\queue\BaseBatchedJob::afterBatch()`.
+- Added `craft\queue\BaseBatchedJob::before()`.
+- Added `craft\queue\BaseBatchedJob::beforeBatch()`.
 - Added `craft\services\Auth`.
 - Added `craft\services\Entries::refreshEntryTypes()`.
 - Added `craft\services\Fields::$fieldContext`, which replaces `craft\services\Content::$fieldContext`.

src/config/app.php

@@ -4,7 +4,7 @@
     'id' => 'CraftCMS',
     'name' => 'Craft CMS',
     'version' => '5.0.0-alpha',
-    'schemaVersion' => '5.0.0.10',
+    'schemaVersion' => '5.0.0.11',
     'minVersionRequired' => '4.4.0',
     'basePath' => dirname(__DIR__), // Defines the @app alias
     'runtimePath' => '@storage/runtime', // Defines the @runtime alias

src/db/Table.php

@@ -46,6 +46,8 @@ abstract class Table
     public const ELEMENTACTIVITY = '{{%elementactivity}}';
     public const ELEMENTS = '{{%elements}}';
     /** @since 5.0.0 */
+    public const ELEMENTS_BULKOPS = '{{%elements_bulkops}}';
+    /** @since 5.0.0 */
     public const ELEMENTS_OWNERS = '{{%elements_owners}}';
     public const ELEMENTS_SITES = '{{%elements_sites}}';
     public const RESOURCEPATHS = '{{%resourcepaths}}';

src/elements/db/ElementQuery.php

@@ -101,6 +101,9 @@ class ElementQuery extends Query implements ElementQueryInterface
      */
     public const EVENT_AFTER_POPULATE_ELEMENTS = 'afterPopulateElements';
 
+    // Base config attributes
+    // -------------------------------------------------------------------------
+
     /**
      * @var string The name of the [[ElementInterface]] class.
      * @phpstan-var class-string<ElementInterface>
@@ -343,6 +346,14 @@ class ElementQuery extends Query implements ElementQueryInterface
      */
     public mixed $search = null;
 
+    /**
+     * @var string|null The bulk element operation key that the resulting elements were involved in.
+     *
+     * @used-by ElementQuery::inBulkOp()
+     * @since 5.0.0
+     */
+    public ?string $inBulkOp = null;
+
     /**
      * @var mixed The reference code(s) used to identify the element(s).
      *
@@ -1075,6 +1086,16 @@ public function search($value): static
         return $this;
     }
 
+    /**
+     * @inheritdoc
+     * @uses $inBulkOp
+     */
+    public function inBulkOp(?string $value): static
+    {
+        $this->inBulkOp = $value;
+        return $this;
+    }
+
     /**
      * @inheritdoc
      * @uses $ref
@@ -1531,6 +1552,7 @@ public function prepare($builder): Query
         $this->_applyStructureParams($class);
         $this->_applyRevisionParams();
         $this->_applySearchParam($db);
+        $this->_applyInBulkOpParam();
         $this->_applyOrderByParams($db);
         $this->_applySelectParam();
         $this->_applyJoinParams();
@@ -2843,6 +2865,18 @@ private function _applySearchParam(Connection $db): void
         }
     }
 
+    /**
+     * Applies the 'inBulkOp' param to the query being prepared.
+     */
+    private function _applyInBulkOpParam(): void
+    {
+        if ($this->inBulkOp) {
+            $this->subQuery
+                ->innerJoin(['elements_bulkops' => Table::ELEMENTS_BULKOPS], '[[elements_bulkops.elementId]] = [[elements.id]]')
+                ->andWhere(['elements_bulkops.key' => $this->inBulkOp]);
+        }
+    }
+
     /**
      * Applies the 'fixedOrder' and 'orderBy' params to the query being prepared.
      *

src/elements/db/ElementQueryInterface.php

@@ -999,6 +999,15 @@ public function uri(mixed $value): static;
      */
     public function search(mixed $value): static;
 
+    /**
+     * Narrows the query results to only {elements} that were involved in a bulk element operation.
+     *
+     * @param string|null $value The property value
+     * @return static self reference
+     * @since 5.0.0
+     */
+    public function inBulkOp(?string $value): static;
+
     /**
      * Narrows the query results based on a reference string.
      *

src/events/BulkOpEvent.php

@@ -0,0 +1,22 @@
+<?php
+/**
+ * @link https://craftcms.com/
+ * @copyright Copyright (c) Pixel & Tonic, Inc.
+ * @license https://craftcms.github.io/license/
+ */
+
+namespace craft\events;
+
+/**
+ * Bulk operation event class.
+ *
+ * @author Pixel & Tonic, Inc. <[email protected]>
+ * @since 5.0.0
+ */
+class BulkOpEvent extends ElementQueryEvent
+{
+    /**
+     * @var string The bulk operation key.
+     */
+    public string $key;
+}

src/migrations/Install.php

@@ -323,6 +323,12 @@ public function createTables(): void
             'deletedWithOwner' => $this->boolean()->null(),
             'uid' => $this->uid(),
         ]);
+        $this->createTable(Table::ELEMENTS_BULKOPS, [
+            'elementId' => $this->integer(),
+            'key' => $this->char(10)->notNull(),
+            'timestamp' => $this->dateTime()->notNull(),
+            'PRIMARY KEY([[elementId]], [[key]])',
+        ]);
         $this->createTable(Table::ELEMENTS_OWNERS, [
             'elementId' => $this->integer()->notNull(),
             'ownerId' => $this->integer()->notNull(),
@@ -811,6 +817,7 @@ public function createIndexes(): void
         $this->createIndex(null, Table::ELEMENTS, ['archived', 'dateCreated'], false);
         $this->createIndex(null, Table::ELEMENTS, ['archived', 'dateDeleted', 'draftId', 'revisionId', 'canonicalId'], false);
         $this->createIndex(null, Table::ELEMENTS, ['archived', 'dateDeleted', 'draftId', 'revisionId', 'canonicalId', 'enabled'], false);
+        $this->createIndex(null, Table::ELEMENTS_BULKOPS, ['timestamp'], false);
         $this->createIndex(null, Table::ELEMENTS_SITES, ['elementId', 'siteId'], true);
         $this->createIndex(null, Table::ELEMENTS_SITES, ['siteId'], false);
         $this->createIndex(null, Table::ELEMENTS_SITES, ['title', 'siteId'], false);
@@ -986,6 +993,7 @@ public function addForeignKeys(): void
         $this->addForeignKey(null, Table::ELEMENTS, ['draftId'], Table::DRAFTS, ['id'], 'CASCADE', null);
         $this->addForeignKey(null, Table::ELEMENTS, ['revisionId'], Table::REVISIONS, ['id'], 'CASCADE', null);
         $this->addForeignKey(null, Table::ELEMENTS, ['fieldLayoutId'], Table::FIELDLAYOUTS, ['id'], 'SET NULL', null);
+        $this->addForeignKey(null, Table::ELEMENTS_BULKOPS, ['elementId'], Table::ELEMENTS, ['id'], 'CASCADE', null);
         $this->addForeignKey(null, Table::ELEMENTS_OWNERS, ['elementId'], Table::ELEMENTS, ['id'], 'CASCADE', null);
         $this->addForeignKey(null, Table::ELEMENTS_OWNERS, ['ownerId'], Table::ELEMENTS, ['id'], 'CASCADE', null);
         $this->addForeignKey(null, Table::ELEMENTS_SITES, ['elementId'], Table::ELEMENTS, ['id'], 'CASCADE', null);

src/migrations/m231213_030600_element_bulk_ops.php

@@ -0,0 +1,37 @@
+<?php
+
+namespace craft\migrations;
+
+use craft\db\Migration;
+use craft\db\Table;
+
+/**
+ * m231213_030600_element_bulk_ops migration.
+ */
+class m231213_030600_element_bulk_ops extends Migration
+{
+    /**
+     * @inheritdoc
+     */
+    public function safeUp(): bool
+    {
+        $this->createTable(Table::ELEMENTS_BULKOPS, [
+            'elementId' => $this->integer(),
+            'key' => $this->char(10)->notNull(),
+            'timestamp' => $this->dateTime()->notNull(),
+            'PRIMARY KEY([[elementId]], [[key]])',
+        ]);
+        $this->addForeignKey(null, Table::ELEMENTS_BULKOPS, ['elementId'], Table::ELEMENTS, ['id'], 'CASCADE', null);
+        $this->createIndex(null, Table::ELEMENTS_BULKOPS, ['timestamp'], false);
+        return true;
+    }
+
+    /**
+     * @inheritdoc
+     */
+    public function safeDown(): bool
+    {
+        echo "m231213_030600_element_bulk_ops cannot be reverted.\n";
+        return false;
+    }
+}

src/queue/BaseBatchedElementJob.php

@@ -0,0 +1,47 @@
+<?php
+/**
+ * @link https://craftcms.com/
+ * @copyright Copyright (c) Pixel & Tonic, Inc.
+ * @license https://craftcms.github.io/license/
+ */
+
+namespace craft\queue;
+
+use Craft;
+
+/**
+ * BaseBatchedElementJob is the base class for large jobs that may need to spawn
+ * additional jobs to complete the workload, which perform actions on elements.
+ *
+ * @author Pixel & Tonic, Inc. <[email protected]>
+ * @since 5.0.0
+ */
+abstract class BaseBatchedElementJob extends BaseBatchedJob
+{
+    /** @internal */
+    public string $bulkOpKey;
+
+    /**
+     * @inheritdoc
+     */
+    protected function before(): void
+    {
+        $this->bulkOpKey = Craft::$app->getElements()->beginBulkOp();
+    }
+
+    /**
+     * @inheritdoc
+     */
+    protected function beforeBatch(): void
+    {
+        Craft::$app->getElements()->resumeBulkOp($this->bulkOpKey);
+    }
+
+    /**
+     * @inheritdoc
+     */
+    protected function after(): void
+    {
+        Craft::$app->getElements()->endBulkOp($this->bulkOpKey);
+    }
+}

src/queue/BaseBatchedJob.php

@@ -120,6 +120,12 @@ public function execute($queue): void
         $memoryLimit = ConfigHelper::sizeInBytes(ini_get('memory_limit'));
         $startMemory = $memoryLimit != -1 ? memory_get_usage() : null;
 
+        if ($this->itemOffset === 0) {
+            $this->before();
+        }
+
+        $this->beforeBatch();
+
         $i = 0;
 
         foreach ($items as $item) {
@@ -141,11 +147,15 @@ public function execute($queue): void
             }
         }
 
+        $this->afterBatch();
+
         // Spawn another job if there are more items
         if ($this->itemOffset < $this->totalItems()) {
             $nextJob = clone $this;
             $nextJob->batchIndex++;
             QueueHelper::push($nextJob, $this->priority, 0, $this->ttr, $queue);
+        } else {
+            $this->after();
         }
     }
 
@@ -156,6 +166,42 @@ public function execute($queue): void
      */
     abstract protected function processItem(mixed $item): void;
 
+    /**
+     * Does things before the first item of the first batch.
+     *
+     * @since 5.0.0
+     */
+    protected function before(): void
+    {
+    }
+
+    /**
+     * Does things after the last item of the last batch.
+     *
+     * @since 5.0.0
+     */
+    protected function after(): void
+    {
+    }
+
+    /**
+     * Does things before the first item of the current batch.
+     *
+     * @since 5.0.0
+     */
+    protected function beforeBatch(): void
+    {
+    }
+
+    /**
+     * Does things after the last item of the current batch.
+     *
+     * @since 5.0.0
+     */
+    protected function afterBatch(): void
+    {
+    }
+
     /**
      * @inheritdoc
      */

src/queue/jobs/ApplyNewPropagationMethod.php

@@ -18,7 +18,7 @@
 use craft\helpers\Db;
 use craft\helpers\ElementHelper;
 use craft\i18n\Translation;
-use craft\queue\BaseBatchedJob;
+use craft\queue\BaseBatchedElementJob;
 use craft\services\Structures;
 use Throwable;
 
@@ -30,7 +30,7 @@
  * @author Pixel & Tonic, Inc. <[email protected]>
  * @since 3.4.8
  */
-class ApplyNewPropagationMethod extends BaseBatchedJob
+class ApplyNewPropagationMethod extends BaseBatchedElementJob
 {
     /**
      * @var string The element type to use

src/queue/jobs/PropagateElements.php

@@ -14,15 +14,15 @@
 use craft\db\QueryBatcher;
 use craft\helpers\ElementHelper;
 use craft\i18n\Translation;
-use craft\queue\BaseBatchedJob;
+use craft\queue\BaseBatchedElementJob;
 
 /**
  * PropagateElements job
  *
  * @author Pixel & Tonic, Inc. <[email protected]>
  * @since 3.0.13
  */
-class PropagateElements extends BaseBatchedJob
+class PropagateElements extends BaseBatchedElementJob
 {
     /**
      * @var string The element type that should be propagated

src/queue/jobs/ResaveElements.php

@@ -15,7 +15,7 @@
 use craft\db\QueryBatcher;
 use craft\helpers\ElementHelper;
 use craft\i18n\Translation;
-use craft\queue\BaseBatchedJob;
+use craft\queue\BaseBatchedElementJob;
 use Throwable;
 
 /**
@@ -24,7 +24,7 @@
  * @author Pixel & Tonic, Inc. <[email protected]>
  * @since 3.0.0
  */
-class ResaveElements extends BaseBatchedJob
+class ResaveElements extends BaseBatchedElementJob
 {
     /**
      * @var string The element type that should be resaved