Skip to content

Commit

Permalink
Rework commands for more consistent reporting.
Browse files Browse the repository at this point in the history
  • Loading branch information
adam-vessey committed Nov 20, 2024
1 parent b80ec95 commit b070ebf
Show file tree
Hide file tree
Showing 4 changed files with 419 additions and 101 deletions.
71 changes: 71 additions & 0 deletions src/Drush/Commands/QueueDrushCommands.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
<?php

namespace Drupal\islandora_drush_utils\Drush\Commands;

use Drupal\Core\DependencyInjection\AutowireTrait;
use Drupal\Core\Queue\QueueFactory;
use Drupal\islandora_drush_utils\Drush\Commands\Traits\WrappedCommandVerbosityTrait;
use Drush\Attributes as CLI;
use Drush\Commands\DrushCommands;
use Drush\Commands\core\QueueCommands;
use Drush\Drush;
use Symfony\Component\DependencyInjection\Attribute\Autowire;

/**
* Extended queue-running command.
*/
class QueueDrushCommands extends DrushCommands {

use AutowireTrait;
use WrappedCommandVerbosityTrait;

/**
* Construct.
*/
public function __construct(
#[Autowire(service: 'queue')]
protected QueueFactory $queueFactory,
) {
parent::__construct();
}

/**
* Command callback; wrap queue running to run to completion.
*
* NOTE: `time-per-iteration` and `items-per-iteration` should be selected
* with awareness of the environments memory constraints.
*/
#[CLI\Command(name: 'islandora_drush_utils:queue:run')]
#[CLI\Argument(name: 'name', description: 'The name of the queue to run.')]
#[CLI\Option(name: 'time-per-iteration', description: 'The time limit we will provide to the wrapped `queue:run` invocation.')]
#[CLI\Option(name: 'items-per-invocation', description: 'The item limit we will provide to the wrapped `queue:run` invocation.')]
#[CLI\ValidateQueueName(argumentName: 'name')]
public function runQueue(
string $name,
array $options = [
'time-per-iteration' => 300,
'items-per-iteration' => 100,
],
) : void {
$queue = $this->queueFactory->get($name, TRUE);

while ($queue->numberOfItems() > 0) {
$process = Drush::drush(
Drush::aliasManager()->getSelf(),
QueueCommands::RUN,
[$name],
[
'time-limit' => $options['time-per-iteration'],
'items-limit' => $options['items-per-iteration'],
] + $this->getVerbosityOptions(),
);
// We expect sane exit from time * items.
$process->setTimeout(NULL);
$process->run(static::directOutputCallback(...));
if (!$process->isSuccessful()) {
throw new \Exception('Subprocess failed.');
}
}
}

}
191 changes: 90 additions & 101 deletions src/Drush/Commands/Sec873DrushCommands.php
Original file line number Diff line number Diff line change
Expand Up @@ -5,16 +5,20 @@
use Consolidation\AnnotatedCommand\Attributes\HookSelector;
use Drupal\Core\Database\Connection;
use Drupal\Core\Database\Query\SelectInterface;
use Drupal\Core\DependencyInjection\DependencySerializationTrait;
use Drupal\Core\Entity\EntityFieldManagerInterface;
use Drupal\Core\Entity\EntityTypeManagerInterface;
use Drupal\Core\Entity\RevisionLogInterface;
use Drupal\Core\Entity\Sql\SqlContentEntityStorage;
use Drupal\Core\Queue\QueueFactory;
use Drupal\Core\Queue\ReliableQueueInterface;
use Drupal\Core\Session\AccountInterface;
use Drupal\field\Entity\FieldStorageConfig;
use Drupal\islandora_drush_utils\Drush\Commands\Traits\WrappedCommandVerbosityTrait;
use Drupal\user\UserInterface;
use Drush\Attributes as CLI;
use Drush\Commands\AutowireTrait;
use Drush\Commands\DrushCommands;
use Drush\Drush;
use Psr\Log\LoggerInterface;
use Symfony\Component\DependencyInjection\Attribute\Autowire;

Expand All @@ -29,6 +33,8 @@
class Sec873DrushCommands extends DrushCommands {

use AutowireTrait;
use DependencySerializationTrait;
use WrappedCommandVerbosityTrait;

const IDS_META = 'sec-873-ids-alias';
const COUNT_META = 'sec-873-count';
Expand All @@ -54,6 +60,8 @@ public function __construct(
protected ?AccountInterface $currentUser,
#[Autowire(service: 'logger.islandora_drush_utils.sec_873')]
LoggerInterface $logger,
#[Autowire(service: 'queue')]
protected QueueFactory $queueFactory,
) {
parent::__construct();
$this->setLogger($logger);
Expand Down Expand Up @@ -268,128 +276,109 @@ public function getRevisions(array $options = []) : void {
* Options, see attributes for details.
*/
#[CLI\Command(name: 'islandora_drush_utils:sec-873:repair')]
#[CLI\Help(description: 'Given CSV to process representing paragraphs which are referenced across different entities, create entity-specific instances in the newest revisions.')]
#[CLI\Help(description: 'Given CSV to process representing paragraphs which are referenced across different entities, create entity-specific instances in the newest revisions. Thin wrapper around the enqueue and batch commands.')]
#[CLI\Option(name: 'dry-run', description: 'Flag to avoid making changes.')]
#[CLI\Option(name: 'batch-size', description: 'The number of items which are processed per batch.')]
#[CLI\Usage(name: 'drush islandora_drush_utils:sec-873:repair --user=1 < current.csv', description: 'Consume from pre-run CSV.')]
#[CLI\Usage(name: 'drush islandora_drush_utils:sec-873:get-current | drush islandora_drush_utils:sec-873:repair --user=1', description: 'Consume CSV from pipe.')]
#[HookSelector(name: 'islandora-drush-utils-user-wrap')]
#[CLI\ValidateModulesEnabled(modules: ['paragraphs'])]
public function repair(
array $options = [
'dry-run' => self::OPT,
'batch-size' => 100,
],
) : void {
/** @var \Drupal\Core\Entity\RevisionableStorageInterface $paragraph_storage */
$paragraph_storage = $this->entityTypeManager->getStorage('paragraph');
while ($row = fgetcsv(STDIN)) {
[$entity_type, $field_name, $paragraph_id, , $_id_csv] = $row;
$ids = explode(',', $_id_csv);
$this->enqueue([
'batch-size' => $options['batch-size'] ?? 100,
]);
$this->processQueue([
'dry-run' => $options['dry-run'] ?? FALSE,
]);
}

$transaction = $this->database->startTransaction();
try {
/** @var \Drupal\Core\Entity\ContentEntityInterface[] $entities */
$entities = $this->entityTypeManager->getStorage($entity_type)->loadMultiple($ids);
foreach ($entities as $entity) {
$this->logger->debug('Processing {entity_type}:{entity_id}:{field_name}:{paragraph_id}', [
'entity_type' => $entity->getEntityTypeId(),
'entity_id' => $entity->id(),
'field_name' => $field_name,
'paragraph_id' => $paragraph_id,
]);
/** @var \Drupal\entity_reference_revisions\EntityReferenceRevisionsFieldItemList $paragraph_list */
$paragraph_list = $entity->get($field_name);
/**
* Helper; get queue to use.
*
* @return \Drupal\Core\Queue\ReliableQueueInterface
* The queue to use.
*/
protected function getQueue() : ReliableQueueInterface {
return $this->queueFactory->get('islandora_drush_utils__sec873', TRUE);
}

$to_replace = NULL;
$to_delete = [];
/** @var \Drupal\entity_reference_revisions\Plugin\Field\FieldType\EntityReferenceRevisionsItem $item */
foreach ($paragraph_list as $index => $item) {
if ($item->get('target_id')->getValue() !== $paragraph_id) {
continue;
}
if ($to_replace === NULL) {
$to_replace = $index;
$this->logger->debug('Replacing {entity_type}:{entity_id}:{field_name}:target_id {paragraph_id} @ delta {delta}', [
'entity_type' => $entity->getEntityTypeId(),
'entity_id' => $entity->id(),
'field_name' => $field_name,
'paragraph_id' => $paragraph_id,
'delta' => $index,
]);
}
else {
// XXX: Unlikely to encounter, but _if_ there were somehow
// multiple references to the same paragraph in a given field,
// this would handle de-duping them (which is to say, deleting the
// extra references).
$to_delete[] = $index;
$this->logger->debug('Deleting {entity_type}:{entity_id}:{field_name}:target_id {paragraph_id} @ delta {delta}', [
'entity_type' => $entity->getEntityTypeId(),
'entity_id' => $entity->id(),
'field_name' => $field_name,
'paragraph_id' => $paragraph_id,
'delta' => $index,
]);
}
}
/**
* Given CSV, populate queue.
*
* @param array $options
* Options, see attributes for details.
*/
#[CLI\Command(name: 'islandora_drush_utils:sec-873:repair:enqueue')]
#[CLI\Help(description: 'Given CSV to process representing paragraphs which are referenced across different entities, populate queue to be processed.')]
#[CLI\Option(name: 'batch-size', description: 'The number of items which are processed per batch.')]
#[CLI\Usage(name: 'drush islandora_drush_utils:sec-873:repair:enqueue --user=1 < current.csv', description: 'Consume from pre-run CSV.')]
#[CLI\Usage(name: 'drush islandora_drush_utils:sec-873:get-current | drush islandora_drush_utils:sec-873:repair:enqueue --user=1', description: 'Consume CSV from pipe.')]
#[HookSelector(name: 'islandora-drush-utils-user-wrap')]
#[CLI\ValidateModulesEnabled(modules: ['paragraphs'])]
public function enqueue(
array $options = [
'batch-size' => 100,
],
) : void {
$queue = $this->getQueue();

$info = $paragraph_list->get($to_replace)->getValue();
/** @var \Drupal\paragraphs\Entity\Paragraph $item */
$item = $paragraph_storage->loadRevision($info['target_revision_id']);
/** @var \Drupal\paragraphs\Entity\Paragraph $dupe */
$dupe = $item->createDuplicate();
$dupe->setParentEntity($entity, $field_name);
$paragraph_list->set($to_replace, $dupe);
// Wipe all items/recreate queue.
$queue->deleteQueue();
$queue->createQueue();

// XXX: Need to unset from end to start, as the list will rekey itself
// automatically.
foreach (array_reverse($to_delete) as $index_to_delete) {
unset($paragraph_list[$index_to_delete]);
}
while ($row = fgetcsv(STDIN)) {
[$entity_type, $field_name, $paragraph_id, , $_id_csv] = $row;
$ids = explode(',', $_id_csv);

$entity->setNewRevision();
if ($entity instanceof RevisionLogInterface) {
if ($this->currentUser) {
$entity->setRevisionUser($this->getCurrentUser());
}
$entity->setRevisionLogMessage("Reworked away from the shared cross-entity paragraph entity {$paragraph_id}.");
}
if (!$options['dry-run']) {
$entity->save();
$this->logger->info('Updated {entity_id} away from {paragraph_id}.', [
'entity_id' => $entity->id(),
'paragraph_id' => $paragraph_id,
]);
}
else {
$this->logger->info('Would update {entity_id} away from {paragraph_id}.', [
'entity_id' => $entity->id(),
'paragraph_id' => $paragraph_id,
]);
}
}
}
catch (\Exception $e) {
$transaction->rollBack();
throw new \Exception("Encountered exception, rolled back transaction.", previous: $e);
}
if ($options['dry-run']) {
$transaction->rollBack();
$this->logger->debug('Dry run, rolling back paragraphs.');
foreach (array_chunk($ids, $options['batch-size']) as $chunk) {
$queue->createItem([
'entity_type' => $entity_type,
'field_name' => $field_name,
'paragraph_id' => $paragraph_id,
'ids' => $chunk,
'uid' => $this->currentUser->id(),
]);
}
}
}

/**
* Load actual entity of the current user.
* Given populated queue, process it.
*
* @return \Drupal\user\UserInterface
* User entity for the current user.
*
* @throws \Drupal\Component\Plugin\Exception\InvalidPluginDefinitionException
* @throws \Drupal\Component\Plugin\Exception\PluginNotFoundException
* @param array $options
* Options, see attributes for details.
*/
protected function getCurrentUser() : UserInterface {
return $this->currentUserAsUser ??= $this->entityTypeManager->getStorage('user')->load($this->currentUser->id());
#[CLI\Command(name: 'islandora_drush_utils:sec-873:repair:batch')]
#[CLI\Help(description: 'Given populated queue, batch process it.')]
#[CLI\Option(name: 'dry-run', description: 'Flag to avoid making changes. NOTE: The queue will still be consumed.')]
#[CLI\Usage(name: 'drush islandora_drush_utils:sec-873:repair:batch', description: 'Process the queue.')]
#[CLI\ValidateModulesEnabled(modules: ['paragraphs'])]
public function processQueue(
array $options = [
'dry-run' => self::OPT,
],
) : void {
$process = Drush::drush(
Drush::aliasManager()->getSelf(),
'islandora_drush_utils:queue:run',
['islandora_drush_utils__sec873'],
$this->getVerbosityOptions(),
);
$process->setTimeout(NULL);
$process->run(
static::directOutputCallback(...),
env: [
'ISLANDORA_DRUSH_UTILS_SEC_783__DRY_RUN' => $options['dry-run'],
],
);
if (!$process->isSuccessful()) {
throw new \Exception('Subprocess failed');
}
}

}
69 changes: 69 additions & 0 deletions src/Drush/Commands/Traits/WrappedCommandVerbosityTrait.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
<?php

namespace Drupal\islandora_drush_utils\Drush\Commands\Traits;

use Drush\Commands\DrushCommands;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Process\Process;

/**
* Facilitate forwarding of verbosity options to wrapped commands.
*/
trait WrappedCommandVerbosityTrait {

/**
* Helper; build map of verbosity options from the current instance.
*
* @return bool[]
* An associative array mapping options as strings to booleans representing
* the state of the given options.
*
* @see static::mapVerbosityOptions()
*/
protected function getVerbosityOptions() : array {
$input = match(TRUE) {
$this instanceof DrushCommands => $this->input(),
};

return static::mapVerbosityOptions($input);
}

/**
* Helper; build map of verbosity options from the given input.
*
* @param \Symfony\Component\Console\Input\InputInterface $input
* The input from which to map the options.
*
* @return bool[]
* An associative array mapping options as strings to booleans representing
* the state of the given options.
*/
protected static function mapVerbosityOptions(InputInterface $input) : array {
return [
// Adapted from https://github.com/drush-ops/drush/blob/7fe0a492d5126c457c5fb184c4668a132b0aaac6/src/Application.php#L291-L302
'verbose' => $input->getParameterOption(['--verbose', '-v'], FALSE, TRUE) !== FALSE,
'vv' => $input->getParameterOption(['-vv'], FALSE, TRUE) !== FALSE,
'vvv' => $input->getParameterOption(['--debug', '-d', '-vvv'], FALSE, TRUE) !== FALSE,
];
}

/**
* Helper to directly output from wrapped commands.
*
* @param string $type
* The type of output, one of Process::OUT and Process::ERR.
* @param string $output
* The output to output.
*
* @return false|int
* The number of bytes written; otherwise, FALSE.
*/
protected static function directOutputCallback(string $type, string $output) : false|int {
$fp = match($type) {
Process::OUT => STDOUT,
Process::ERR => STDERR,
};
return fwrite($fp, $output);
}

}
Loading

0 comments on commit b070ebf

Please sign in to comment.