From a0f9050c85f6981a92c27e2927180c2302d09355 Mon Sep 17 00:00:00 2001 From: Mathieu De Keyzer Date: Fri, 29 Nov 2024 20:29:58 +0100 Subject: [PATCH 1/9] fix: was working as only used in twigland so far --- EMS/core-bundle/src/Repository/FormSubmissionRepository.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/EMS/core-bundle/src/Repository/FormSubmissionRepository.php b/EMS/core-bundle/src/Repository/FormSubmissionRepository.php index 8e74613e1..2ce93f1d2 100644 --- a/EMS/core-bundle/src/Repository/FormSubmissionRepository.php +++ b/EMS/core-bundle/src/Repository/FormSubmissionRepository.php @@ -57,7 +57,7 @@ public function findAllUnprocessed(): array ->andWhere($qb->expr()->isNotNull('fs.data')) ->orderBy('fs.created', 'desc'); - return $qb->getQuery()->getArrayResult(); + return $qb->getQuery()->execute(); } public function countAllUnprocessed(string $searchValue): int @@ -105,7 +105,7 @@ public function findFormSubmissions(?string $formInstance = null): array ->andWhere($qb->expr()->isNotNull('fs.data')) ->orderBy('fs.created', 'desc'); - return $qb->getQuery()->getArrayResult(); + return $qb->getQuery()->execute(); } public function persist(FormSubmission $formSubmission): void From aa6d680022ae50355a9cfb03fd0d1a559a04d2ee Mon Sep 17 00:00:00 2001 From: Mathieu De Keyzer Date: Fri, 29 Nov 2024 20:34:24 +0100 Subject: [PATCH 2/9] feat: first version of the export --- .../src/Command/Submission/ExportCommand.php | 72 +++++++++++++++++++ EMS/core-bundle/src/Commands.php | 1 + .../src/Resources/config/command.xml | 5 ++ 3 files changed, 78 insertions(+) create mode 100644 EMS/core-bundle/src/Command/Submission/ExportCommand.php diff --git a/EMS/core-bundle/src/Command/Submission/ExportCommand.php b/EMS/core-bundle/src/Command/Submission/ExportCommand.php new file mode 100644 index 000000000..2087dc0df --- /dev/null +++ b/EMS/core-bundle/src/Command/Submission/ExportCommand.php @@ -0,0 +1,72 @@ +setDescription('Extract form submissions') + ->addArgument( + self::ARG_FIELDS, + InputArgument::IS_ARRAY, + 'Fields to export' + )->addOption( + self::OPTIONS_FILTER, + 'f', + InputOption::VALUE_OPTIONAL, + 'Expression to filter submissions' + ); + } + + protected function initialize(InputInterface $input, OutputInterface $output): void + { + parent::initialize($input, $output); + $this->fields = $this->getArgumentStringArray(self::ARG_FIELDS); + $this->filter = $this->getOptionStringNull(self::OPTIONS_FILTER); + } + + protected function execute(InputInterface $input, OutputInterface $output): int + { + $this->io->section('Export the form submissions'); + + foreach ($this->formSubmissionService->getUnprocessed() as $submission) { + $data = \array_merge([ + 'instance' => $submission->getInstance(), + 'name' => $submission->getName(), + 'locale' => $submission->getLocale(), + 'submission_date' => $submission->getCreated()->format('c'), + ], $submission->getData() ?? []); + if (null !== $this->filter && !$this->expressionService->evaluateToBool($this->filter, $data)) { + continue; + } + foreach ($this->fields as $field) { + echo $data[$field] ?? ''; + } + } + + return self::EXECUTE_SUCCESS; + } +} diff --git a/EMS/core-bundle/src/Commands.php b/EMS/core-bundle/src/Commands.php index 22c239381..0f4d5a769 100644 --- a/EMS/core-bundle/src/Commands.php +++ b/EMS/core-bundle/src/Commands.php @@ -43,4 +43,5 @@ final class Commands public const MANAGED_ALIAS_ADD_ENVIRONMENT = 'emsco:managed-alias:add-environment'; public const ASSET_REFRESH_FILE_FIELD = 'emsco:asset:refresh-file-fields'; final public const SUBMISSION_FORWARD = 'emsco:submissions:forward'; + final public const SUBMISSION_EXPORT = 'emsco:submissions:export'; } diff --git a/EMS/core-bundle/src/Resources/config/command.xml b/EMS/core-bundle/src/Resources/config/command.xml index 9ecf67348..a49bdb2ba 100644 --- a/EMS/core-bundle/src/Resources/config/command.xml +++ b/EMS/core-bundle/src/Resources/config/command.xml @@ -260,6 +260,11 @@ + + + + + From 440da558ec60f8e9a6849b4b01ea7c831195610e Mon Sep 17 00:00:00 2001 From: Mathieu De Keyzer Date: Fri, 29 Nov 2024 20:53:10 +0100 Subject: [PATCH 3/9] feat: generate an excel --- .../src/Command/Submission/ExportCommand.php | 34 ++++++++++++++++--- .../src/Resources/config/command.xml | 1 + 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/EMS/core-bundle/src/Command/Submission/ExportCommand.php b/EMS/core-bundle/src/Command/Submission/ExportCommand.php index 2087dc0df..6347c4c5c 100644 --- a/EMS/core-bundle/src/Command/Submission/ExportCommand.php +++ b/EMS/core-bundle/src/Command/Submission/ExportCommand.php @@ -3,6 +3,7 @@ namespace EMS\CoreBundle\Command\Submission; use EMS\CommonBundle\Common\Command\AbstractCommand; +use EMS\CommonBundle\Contracts\SpreadsheetGeneratorServiceInterface; use EMS\CommonBundle\Service\ExpressionService; use EMS\CoreBundle\Commands; use EMS\CoreBundle\Service\Form\Submission\FormSubmissionService; @@ -16,12 +17,17 @@ class ExportCommand extends AbstractCommand protected static $defaultName = Commands::SUBMISSION_EXPORT; public const ARG_FIELDS = 'fields'; public const OPTIONS_FILTER = 'filter'; + public const OPTIONS_FORMAT = 'format'; /** @var string[] */ private array $fields; private ?string $filter; + private string $format; - public function __construct(private readonly FormSubmissionService $formSubmissionService, private readonly ExpressionService $expressionService) - { + public function __construct( + private readonly FormSubmissionService $formSubmissionService, + private readonly ExpressionService $expressionService, + private readonly SpreadsheetGeneratorServiceInterface $spreadsheetGeneratorService, + ) { parent::__construct(); } @@ -35,9 +41,15 @@ protected function configure(): void 'Fields to export' )->addOption( self::OPTIONS_FILTER, - 'f', + null, InputOption::VALUE_OPTIONAL, 'Expression to filter submissions' + )->addOption( + self::OPTIONS_FORMAT, + null, + InputOption::VALUE_OPTIONAL, + 'File format of the export, xlsx or csv are supported', + 'xlsx' ); } @@ -46,11 +58,13 @@ protected function initialize(InputInterface $input, OutputInterface $output): v parent::initialize($input, $output); $this->fields = $this->getArgumentStringArray(self::ARG_FIELDS); $this->filter = $this->getOptionStringNull(self::OPTIONS_FILTER); + $this->format = $this->getOptionString(self::OPTIONS_FORMAT); } protected function execute(InputInterface $input, OutputInterface $output): int { $this->io->section('Export the form submissions'); + $sheet = [[...$this->fields]]; foreach ($this->formSubmissionService->getUnprocessed() as $submission) { $data = \array_merge([ @@ -62,11 +76,23 @@ protected function execute(InputInterface $input, OutputInterface $output): int if (null !== $this->filter && !$this->expressionService->evaluateToBool($this->filter, $data)) { continue; } + $line = []; foreach ($this->fields as $field) { - echo $data[$field] ?? ''; + $line[] = $data[$field] ?? ''; } + $sheet[] = $line; } + $config = [ + SpreadsheetGeneratorServiceInterface::SHEETS => [[ + 'rows' => $sheet, + 'name' => 'submissions', + ]], + SpreadsheetGeneratorServiceInterface::CONTENT_FILENAME => 'submissions', + SpreadsheetGeneratorServiceInterface::WRITER => $this->format, + ]; + $this->spreadsheetGeneratorService->generateSpreadsheetFile($config, 'submission.xlsx'); + return self::EXECUTE_SUCCESS; } } diff --git a/EMS/core-bundle/src/Resources/config/command.xml b/EMS/core-bundle/src/Resources/config/command.xml index a49bdb2ba..4b3465b08 100644 --- a/EMS/core-bundle/src/Resources/config/command.xml +++ b/EMS/core-bundle/src/Resources/config/command.xml @@ -263,6 +263,7 @@ + From b0d1b357afd847c78c33ef59aebe38eaeb17a652 Mon Sep 17 00:00:00 2001 From: Mathieu De Keyzer Date: Fri, 29 Nov 2024 21:53:41 +0100 Subject: [PATCH 4/9] feat: generate an Excel, or a csv, if a filename has been provided. A table output otherwise --- .../SpreadsheetGeneratorServiceInterface.php | 1 + .../src/Command/Submission/ExportCommand.php | 38 ++++++++++++------- 2 files changed, 26 insertions(+), 13 deletions(-) diff --git a/EMS/common-bundle/src/Contracts/SpreadsheetGeneratorServiceInterface.php b/EMS/common-bundle/src/Contracts/SpreadsheetGeneratorServiceInterface.php index 959cf515c..96b05466a 100644 --- a/EMS/common-bundle/src/Contracts/SpreadsheetGeneratorServiceInterface.php +++ b/EMS/common-bundle/src/Contracts/SpreadsheetGeneratorServiceInterface.php @@ -12,6 +12,7 @@ interface SpreadsheetGeneratorServiceInterface public const WRITER = 'writer'; public const XLSX_WRITER = 'xlsx'; public const CSV_WRITER = 'csv'; + public const FORMAT_WRITERS = [self::CSV_WRITER, self::XLSX_WRITER]; public const CSV_SEPARATOR = 'csv_separator'; public const SHEETS = 'sheets'; public const CONTENT_FILENAME = 'filename'; diff --git a/EMS/core-bundle/src/Command/Submission/ExportCommand.php b/EMS/core-bundle/src/Command/Submission/ExportCommand.php index 6347c4c5c..f4aa78d75 100644 --- a/EMS/core-bundle/src/Command/Submission/ExportCommand.php +++ b/EMS/core-bundle/src/Command/Submission/ExportCommand.php @@ -16,12 +16,13 @@ class ExportCommand extends AbstractCommand { protected static $defaultName = Commands::SUBMISSION_EXPORT; public const ARG_FIELDS = 'fields'; - public const OPTIONS_FILTER = 'filter'; - public const OPTIONS_FORMAT = 'format'; + public const OPTION_FILTER = 'filter'; + public const OPTION_FILENAME = 'filename'; + /** @var string[] */ private array $fields; private ?string $filter; - private string $format; + private ?string $filename; public function __construct( private readonly FormSubmissionService $formSubmissionService, @@ -40,16 +41,15 @@ protected function configure(): void InputArgument::IS_ARRAY, 'Fields to export' )->addOption( - self::OPTIONS_FILTER, + self::OPTION_FILTER, null, InputOption::VALUE_OPTIONAL, 'Expression to filter submissions' )->addOption( - self::OPTIONS_FORMAT, + self::OPTION_FILENAME, null, InputOption::VALUE_OPTIONAL, - 'File format of the export, xlsx or csv are supported', - 'xlsx' + 'Export filename, xlsx or csv formats are supported', ); } @@ -57,14 +57,14 @@ protected function initialize(InputInterface $input, OutputInterface $output): v { parent::initialize($input, $output); $this->fields = $this->getArgumentStringArray(self::ARG_FIELDS); - $this->filter = $this->getOptionStringNull(self::OPTIONS_FILTER); - $this->format = $this->getOptionString(self::OPTIONS_FORMAT); + $this->filter = $this->getOptionStringNull(self::OPTION_FILTER); + $this->filename = $this->getOptionStringNull(self::OPTION_FILENAME); } protected function execute(InputInterface $input, OutputInterface $output): int { $this->io->section('Export the form submissions'); - $sheet = [[...$this->fields]]; + $sheet = []; foreach ($this->formSubmissionService->getUnprocessed() as $submission) { $data = \array_merge([ @@ -83,15 +83,27 @@ protected function execute(InputInterface $input, OutputInterface $output): int $sheet[] = $line; } + if (null === $this->filename) { + $this->io->table([...$this->fields], $sheet); + + return self::EXECUTE_SUCCESS; + } + + $extension = \pathinfo($this->filename)['extension'] ?? ''; + if (!\in_array($extension, SpreadsheetGeneratorServiceInterface::FORMAT_WRITERS)) { + $this->io->error(\sprintf('File format %s is not supported', $extension)); + } + $config = [ SpreadsheetGeneratorServiceInterface::SHEETS => [[ - 'rows' => $sheet, + 'rows' => [[...$this->fields], ...$sheet], 'name' => 'submissions', ]], SpreadsheetGeneratorServiceInterface::CONTENT_FILENAME => 'submissions', - SpreadsheetGeneratorServiceInterface::WRITER => $this->format, + SpreadsheetGeneratorServiceInterface::WRITER => $extension, ]; - $this->spreadsheetGeneratorService->generateSpreadsheetFile($config, 'submission.xlsx'); + $this->spreadsheetGeneratorService->generateSpreadsheetFile($config, $this->filename); + $this->io->success(\sprintf('The file %s has been successfully generated', $this->filename)); return self::EXECUTE_SUCCESS; } From 2e62bd9bcbbd766d357feb264153d4da075fc147 Mon Sep 17 00:00:00 2001 From: Mathieu De Keyzer Date: Fri, 29 Nov 2024 21:56:06 +0100 Subject: [PATCH 5/9] doc: --- EMS/core-bundle/src/Command/Submission/ExportCommand.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EMS/core-bundle/src/Command/Submission/ExportCommand.php b/EMS/core-bundle/src/Command/Submission/ExportCommand.php index f4aa78d75..69b3e78ed 100644 --- a/EMS/core-bundle/src/Command/Submission/ExportCommand.php +++ b/EMS/core-bundle/src/Command/Submission/ExportCommand.php @@ -44,7 +44,7 @@ protected function configure(): void self::OPTION_FILTER, null, InputOption::VALUE_OPTIONAL, - 'Expression to filter submissions' + 'Expression to filter submissions, e.g. "locale==\'en\'"' )->addOption( self::OPTION_FILENAME, null, From 836639f658107f105589bb41ad2d34737dee4e50 Mon Sep 17 00:00:00 2001 From: Mathieu De Keyzer Date: Sat, 30 Nov 2024 09:59:52 +0100 Subject: [PATCH 6/9] feat: progress bar --- EMS/core-bundle/src/Command/Submission/ExportCommand.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/EMS/core-bundle/src/Command/Submission/ExportCommand.php b/EMS/core-bundle/src/Command/Submission/ExportCommand.php index 69b3e78ed..f0e476961 100644 --- a/EMS/core-bundle/src/Command/Submission/ExportCommand.php +++ b/EMS/core-bundle/src/Command/Submission/ExportCommand.php @@ -66,6 +66,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int $this->io->section('Export the form submissions'); $sheet = []; + $this->io->progressStart($this->formSubmissionService->count()); foreach ($this->formSubmissionService->getUnprocessed() as $submission) { $data = \array_merge([ 'instance' => $submission->getInstance(), @@ -74,6 +75,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int 'submission_date' => $submission->getCreated()->format('c'), ], $submission->getData() ?? []); if (null !== $this->filter && !$this->expressionService->evaluateToBool($this->filter, $data)) { + $this->io->progressAdvance(); continue; } $line = []; @@ -81,7 +83,9 @@ protected function execute(InputInterface $input, OutputInterface $output): int $line[] = $data[$field] ?? ''; } $sheet[] = $line; + $this->io->progressAdvance(); } + $this->io->progressFinish(); if (null === $this->filename) { $this->io->table([...$this->fields], $sheet); From 10d1adea49a2424b15e13041b740382a629f3ed5 Mon Sep 17 00:00:00 2001 From: Mathieu De Keyzer Date: Sun, 1 Dec 2024 17:38:08 +0100 Subject: [PATCH 7/9] fix: don't merge data and metadata into an array. As expression language doesn't support variable with dashes a data variable allows access to dashed variables e.g. `data['var-with-dashes']` --- EMS/core-bundle/src/Command/Submission/ExportCommand.php | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/EMS/core-bundle/src/Command/Submission/ExportCommand.php b/EMS/core-bundle/src/Command/Submission/ExportCommand.php index f0e476961..29944c057 100644 --- a/EMS/core-bundle/src/Command/Submission/ExportCommand.php +++ b/EMS/core-bundle/src/Command/Submission/ExportCommand.php @@ -44,7 +44,7 @@ protected function configure(): void self::OPTION_FILTER, null, InputOption::VALUE_OPTIONAL, - 'Expression to filter submissions, e.g. "locale==\'en\'"' + 'Expression to filter submissions, e.g. "\'true\' == (data[\'recontact-optin\'] ?? \'false\')"' )->addOption( self::OPTION_FILENAME, null, @@ -68,19 +68,20 @@ protected function execute(InputInterface $input, OutputInterface $output): int $this->io->progressStart($this->formSubmissionService->count()); foreach ($this->formSubmissionService->getUnprocessed() as $submission) { - $data = \array_merge([ + $data = [ 'instance' => $submission->getInstance(), 'name' => $submission->getName(), 'locale' => $submission->getLocale(), 'submission_date' => $submission->getCreated()->format('c'), - ], $submission->getData() ?? []); + 'data' => $submission->getData() ?? [], + ]; if (null !== $this->filter && !$this->expressionService->evaluateToBool($this->filter, $data)) { $this->io->progressAdvance(); continue; } $line = []; foreach ($this->fields as $field) { - $line[] = $data[$field] ?? ''; + $line[] = $data['data'][$field] ?? $data[$field] ?? ''; } $sheet[] = $line; $this->io->progressAdvance(); From 655568ea6f01ae5931c14ad9bca8e29bab6b4bc2 Mon Sep 17 00:00:00 2001 From: Mathieu De Keyzer Date: Sun, 1 Dec 2024 17:42:09 +0100 Subject: [PATCH 8/9] doc: filter expression lnaguage available variables --- EMS/core-bundle/src/Command/Submission/ExportCommand.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EMS/core-bundle/src/Command/Submission/ExportCommand.php b/EMS/core-bundle/src/Command/Submission/ExportCommand.php index 29944c057..4c8ac92b2 100644 --- a/EMS/core-bundle/src/Command/Submission/ExportCommand.php +++ b/EMS/core-bundle/src/Command/Submission/ExportCommand.php @@ -44,7 +44,7 @@ protected function configure(): void self::OPTION_FILTER, null, InputOption::VALUE_OPTIONAL, - 'Expression to filter submissions, e.g. "\'true\' == (data[\'recontact-optin\'] ?? \'false\')"' + 'Expression to filter submissions, e.g. "\'true\' == (data[\'recontact-optin\'] ?? \'false\')". The following variables are available: data (array), instance (string), name (string), locale (string), submission_date (date in the ISO 8601 format)' )->addOption( self::OPTION_FILENAME, null, From e300afa1b5b73f2abe104eb3f8b8a8cb68df00b4 Mon Sep 17 00:00:00 2001 From: Mathieu De Keyzer Date: Sun, 1 Dec 2024 17:43:44 +0100 Subject: [PATCH 9/9] chore: declare(strict_types=1); --- EMS/core-bundle/src/Command/Submission/ExportCommand.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/EMS/core-bundle/src/Command/Submission/ExportCommand.php b/EMS/core-bundle/src/Command/Submission/ExportCommand.php index 4c8ac92b2..c52b46a2b 100644 --- a/EMS/core-bundle/src/Command/Submission/ExportCommand.php +++ b/EMS/core-bundle/src/Command/Submission/ExportCommand.php @@ -1,5 +1,7 @@