Skip to content

Commit

Permalink
Merged PR 59272: Update cleaner configs in admin
Browse files Browse the repository at this point in the history
## What's being changed

This PR moves cleaner configs to a new section in Dotdigital > Developer Settings. The setting for `table_cleaner_interval` is now editable. There's a patch to migrate the configs, and I've added some small improvements to the Cleaner class also.

## Why it's being changed

Merchants have requested full admin access to certain configs, including `table_cleaner_interval`, so it made sense to move these out of 'Manage Cron Timings' and into their own section.

## How to review / test this change

- Before switching to this branch, set configs for the old paths 'connector_developer_settings/cron_schedules/cleaner' and 'connector_developer_settings/cron_schedules/table_cleaner_interval'
- Switch to this branch and run setup:upgrade
- Confirm the old path values are moved to the new paths
- Review the new admin section, check your saved values appear in core_config_data
- Test amending the configs, see new values update in the table
- Run bin/magento dotdigital:task Cleaner
- Confirm it still works i.e. data is purged older than x date
- Check email_sms_message_queue is also purged via its plugin

Related work items: #242257
  • Loading branch information
sta1r committed Oct 31, 2024
1 parent 24e8db7 commit 1b9301b
Show file tree
Hide file tree
Showing 5 changed files with 175 additions and 21 deletions.
4 changes: 2 additions & 2 deletions Helper/Config.php
Original file line number Diff line number Diff line change
Expand Up @@ -452,9 +452,9 @@ class Config extends AbstractHelper
public const XML_PATH_CRON_SCHEDULE_CONSENT =
'connector_developer_settings/cron_schedules/consent';
public const XML_PATH_CRON_SCHEDULE_CLEANER =
'connector_developer_settings/cron_schedules/cleaner';
'connector_developer_settings/cleaner/schedule';
public const XML_PATH_CRON_SCHEDULE_TABLE_CLEANER_INTERVAL =
'connector_developer_settings/cron_schedules/table_cleaner_interval';
'connector_developer_settings/cleaner/table_cleaner_interval';

/**
* API and portal endpoints
Expand Down
45 changes: 32 additions & 13 deletions Model/Cron/Cleaner.php
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
<?php

declare(strict_types=1);

namespace Dotdigitalgroup\Email\Model\Cron;

use Dotdigitalgroup\Email\Helper\File;
Expand Down Expand Up @@ -83,12 +85,13 @@ public function __construct(
public function run(): void
{
$tables = $this->getTablesForCleanUp();
$olderThanDateString = $this->getOlderThanDateString();

foreach ($tables as $table) {
$dateColumn = $table === SchemaInterface::EMAIL_CONTACT_CONSENT_TABLE ?
'consent_datetime' :
'created_at';
$this->cleanTable($table, $dateColumn);
$this->cleanTable($table, $dateColumn, $olderThanDateString);
}

$this->cleanUpCsvArchiveFolder();
Expand All @@ -106,30 +109,37 @@ public function getTablesForCleanUp(array $additionalTables = [])
return $this->tables + $additionalTables;
}

/**
* Get table cleaner interval.
*
* @return string
*/
public function getTableCleanerInterval(): string
{
return (string) $this->scopeConfig->getValue(Config::XML_PATH_CRON_SCHEDULE_TABLE_CLEANER_INTERVAL);
}

/**
* Delete records older than x days from the provided table.
*
* @param string $tableName
* @param string $dateColumn
* @param string $olderThanDateString
*
* @return void
*/
private function cleanTable(string $tableName, string $dateColumn)
private function cleanTable(string $tableName, string $dateColumn, string $olderThanDateString): void
{
try {
$now = $this->dateTimeFactory->create('now', new \DateTimeZone('UTC'));
$interval = new \DateInterval(sprintf('P%sD', $this->getTableCleanerInterval()));
$date = $now->sub($interval)->format('Y-m-d H:i:s');
$conn = $this->resourceConnection->getConnection();

$numRows = 0;
while (true) {
$select = $conn->select()
->from(
['table_to_clean' => $this->resourceConnection->getTableName($tableName)],
['table_to_clean.id']
)->where(
$conn->quoteInto($dateColumn . ' < ?', $date)
$conn->quoteInto($dateColumn . ' < ?', $olderThanDateString)
)->limit(self::BATCH_SIZE);

$ids = $conn->fetchCol($select);
Expand All @@ -155,27 +165,36 @@ private function cleanTable(string $tableName, string $dateColumn)
}

/**
* Get table cleaner interval.
* Get the from x date string.
*
* @return string
* @throws \Exception
*/
public function getTableCleanerInterval(): string
private function getOlderThanDateString(): string
{
return (string) $this->scopeConfig->getValue(Config::XML_PATH_CRON_SCHEDULE_TABLE_CLEANER_INTERVAL);
$now = $this->dateTimeFactory->create('now', new \DateTimeZone('UTC'));
$interval = new \DateInterval(sprintf('P%sD', $this->getTableCleanerInterval()));
return $now->sub($interval)->format('Y-m-d H:i:s');
}

/**
* Clean up CSV archive folder.
*
* @return void
* @throws FileSystemException
*
* @deprecated CSV files are no longer used.
* @see \Dotdigitalgroup\Email\Model\Sync\Importer\Type\Contact\BulkJson;
*/
private function cleanUpCsvArchiveFolder()
{
$archivedFolder = $this->fileHelper->getArchiveFolder();
$this->fileHelper->deleteDir($archivedFolder);
try {
$archivedFolder = $this->fileHelper->getArchiveFolder();
$this->fileHelper->deleteDir($archivedFolder);
} catch (\Exception $e) {
$this->logger->debug(
'Could not clean up the CSV archive folder.',
[$e->getMessage()]
);
}
}
}
126 changes: 126 additions & 0 deletions Setup/Patch/Data/UpdateCleanerPaths.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,126 @@
<?php

declare(strict_types=1);

namespace Dotdigitalgroup\Email\Setup\Patch\Data;

use Dotdigitalgroup\Email\Helper\Config;
use Magento\Framework\Setup\ModuleDataSetupInterface;
use Magento\Framework\Setup\Patch\DataPatchInterface;

/**
* Patch is mechanism, that allows to do atomic upgrade data changes
*/
class UpdateCleanerPaths implements DataPatchInterface
{
/**
* @var ModuleDataSetupInterface $moduleDataSetup
*/
private $moduleDataSetup;

/**
* @param ModuleDataSetupInterface $moduleDataSetup
*/
public function __construct(ModuleDataSetupInterface $moduleDataSetup)
{
$this->moduleDataSetup = $moduleDataSetup;
}

/**
* Do Upgrade
*
* @return void
*/
public function apply()
{
$oldCleanerSchedulePath = 'connector_developer_settings/cron_schedules/cleaner';
$oldCleanerIntervalPath = 'connector_developer_settings/cron_schedules/table_cleaner_interval';

$this->moduleDataSetup->getConnection()->startSetup();
$select = $this->moduleDataSetup->getConnection()->select()->from(
$this->moduleDataSetup->getTable('core_config_data'),
['*']
)->where('path in (?)', [$oldCleanerSchedulePath, $oldCleanerIntervalPath]);

foreach ($this->moduleDataSetup->getConnection()->fetchAll($select) as $configRow) {
switch ($configRow['path']) {
case $oldCleanerSchedulePath:
$path = Config::XML_PATH_CRON_SCHEDULE_CLEANER;
break;
case $oldCleanerIntervalPath:
$path = Config::XML_PATH_CRON_SCHEDULE_TABLE_CLEANER_INTERVAL;
break;
}

if (isset($path)) {
$this->updateRow($configRow, $path);
}
}
$this->moduleDataSetup->getConnection()->endSetup();
}

/**
* @inheritdoc
*/
public function getAliases()
{
return [];
}

/**
* @inheritdoc
*/
public static function getDependencies()
{
return [];
}

/**
* Update consent record row paths.
*
* @param array $configRow
* @param string $path
* @return void
*/
private function updateRow($configRow, $path)
{
if ($this->keyAlreadyExists($path)) {
$this->moduleDataSetup->getConnection()->delete(
$this->moduleDataSetup->getTable('core_config_data'),
['path = ?' => $configRow['path']]
);
return;
}

$this->moduleDataSetup->getConnection()->update(
$this->moduleDataSetup->getTable('core_config_data'),
[
'path' => $path,
],
[
'path = ?' => $configRow['path']
]
);
}

/**
* Check if newer path name equivalent already exists.
*
* @param string $path
*
* @return bool
*/
private function keyAlreadyExists($path)
{
$existingKey = $this->moduleDataSetup->getConnection()->select()->from(
$this->moduleDataSetup->getTable('core_config_data'),
['*']
)->where(
'path = ?',
$path
);

$result = $this->moduleDataSetup->getConnection()->fetchAll($existingKey);
return count($result) > 0;
}
}
15 changes: 11 additions & 4 deletions etc/adminhtml/system.xml
Original file line number Diff line number Diff line change
Expand Up @@ -1983,16 +1983,23 @@
<source_model>Dotdigitalgroup\Email\Model\Config\Developer\Cronexpressionsthree</source_model>
<backend_model>Dotdigitalgroup\Email\Model\Adminhtml\Backend\CronOffset</backend_model>
</field>
<field id="cleaner" translate="label" sortOrder="90" type="select" showInStore="0" showInWebsite="0" showInDefault="1">
</group>
<group id="cleaner" translate="label" sortOrder="80" showInDefault="1" showInWebsite="0" showInStore="0">
<label>Cleaner</label>
<comment><![CDATA[The cleaner cron purges data from the email_automation, email_importer, email_campaign and email_contact_consent tables on a schedule.]]></comment>

<field id="schedule" translate="label" sortOrder="10" type="select" showInDefault="1" showInStore="0" showInWebsite="0" canRestore="1">
<label>Cleaner</label>
<comment><![CDATA[Cron schedule for cleaner. Defaults to the first day of the month.]]></comment>
<source_model>Dotdigitalgroup\Email\Model\Config\Developer\Cronexpressionsfour</source_model>
<backend_model>Dotdigitalgroup\Email\Model\Adminhtml\Backend\DailyIntervalCron</backend_model>
</field>
<field id="table_cleaner_interval" translate="label" sortOrder="100" showInStore="0" showInWebsite="0" showInDefault="0">
<field id="table_cleaner_interval" translate="label" sortOrder="20" showInDefault="1" showInStore="0" showInWebsite="0" canRestore="1">
<label>Table Cleaner Interval</label>
<comment><![CDATA[Delete data older than x days. Defaults to 30.]]></comment>
</field>
</group>
<group id="system_alerts" translate="label" sortOrder="80" showInDefault="1" showInWebsite="0" showInStore="0">
<group id="system_alerts" translate="label" sortOrder="90" showInDefault="1" showInWebsite="0" showInStore="0">
<label>System Alerts</label>
<field id="system_messages" translate="label" type="select" sortOrder="10" showInDefault="1" showInStore="0" showInWebsite="0">
<label>Enable System Messages</label>
Expand All @@ -2015,7 +2022,7 @@
<comment><![CDATA[This setting controls both email notification frequency and the time period in which we report errors. e.g. Select "24 Hours" to receive an email every 24 hours reporting any errors that may have occurred in the last 24 hours.]]></comment>
</field>
</group>
<group id="pwa_settings" translate="label" sortOrder="90" showInDefault="1" showInWebsite="1" showInStore="0">
<group id="pwa_settings" translate="label" sortOrder="100" showInDefault="1" showInWebsite="1" showInStore="0">
<label>PWA Settings</label>
<field id="pwa_url" translate="label" type="text" sortOrder="10" showInDefault="1" showInStore="0" showInWebsite="1">
<label>PWA Storefront Base URL</label>
Expand Down
6 changes: 4 additions & 2 deletions etc/config.xml
Original file line number Diff line number Diff line change
Expand Up @@ -328,9 +328,11 @@
<catalog>*/15 * * * *</catalog>
<review_wishlist>*/15 * * * *</review_wishlist>
<consent>*/15 * * * *</consent>
<cleaner>0 0 1 * *</cleaner>
<table_cleaner_interval>30</table_cleaner_interval>
</cron_schedules>
<cleaner>
<schedule>0 0 1 * *</schedule>
<table_cleaner_interval>30</table_cleaner_interval>
</cleaner>
<system_alerts>
<system_messages>1</system_messages>
<email_notifications>0</email_notifications>
Expand Down

0 comments on commit 1b9301b

Please sign in to comment.