From 151a3b6c45dcdfbb951b886b46e37054b05ecaa6 Mon Sep 17 00:00:00 2001 From: Ingo Schommer Date: Mon, 15 Apr 2019 15:15:02 +1200 Subject: [PATCH] NEW Legacy thumbnail migration task See https://github.com/silverstripe/silverstripe-assets/issues/235 Makes a start at https://github.com/silverstripe/silverstripe-assets/issues/219 as well --- .../14_Files/04_File_Storage.md | 2 +- .../14_Files/05_File_Migration.md | 30 +++- docs/en/04_Changelogs/4.4.0.md | 27 +++- src/Dev/Tasks/MigrateFileTask.php | 151 +++++++++++++++--- src/Logging/PreformattedEchoHandler.php | 28 ++++ 5 files changed, 209 insertions(+), 29 deletions(-) create mode 100644 src/Logging/PreformattedEchoHandler.php diff --git a/docs/en/02_Developer_Guides/14_Files/04_File_Storage.md b/docs/en/02_Developer_Guides/14_Files/04_File_Storage.md index ab01ae2b7dd..73d41fcad51 100644 --- a/docs/en/02_Developer_Guides/14_Files/04_File_Storage.md +++ b/docs/en/02_Developer_Guides/14_Files/04_File_Storage.md @@ -68,7 +68,7 @@ assets/ The URL for this file will match the physical location on disk: `http://www.example.com/assets/my-public-folder/my-public-file.jpg`. -## Variant file paths (e.g. resized images) +## Variant file paths (e.g. resized images) {#variant-file-paths} Each file can have variants, most commonly resized versions of an image. These can be generated by resizing an image in the CMS rich text editor, diff --git a/docs/en/02_Developer_Guides/14_Files/05_File_Migration.md b/docs/en/02_Developer_Guides/14_Files/05_File_Migration.md index 478f02d046d..b03e0fab039 100644 --- a/docs/en/02_Developer_Guides/14_Files/05_File_Migration.md +++ b/docs/en/02_Developer_Guides/14_Files/05_File_Migration.md @@ -19,10 +19,26 @@ $ ./vendor/bin/sake dev/tasks/MigrateFileTask This task will also support migration of existing File objects to file versioning. Any pre-existing File objects will be automatically published to the live stage, to ensure that previously visible assets remain visible to the public site. - If additional security or visibility rules should be applied to File dataobjects, then make sure to correctly extend `canView` via extensions. +Imports all files referenced by File dataobjects into the new Asset Persistence Layer introduced in 4.0. +Moves existing thumbnails, and generates new thumbnail sizes for the CMS UI. +If the task fails or times out, run it again and it will start where it left off. + +Arguments: + + - `only`: Comma separated list of tasks to run on the multi-step migration (see "Available subtasks"). + Example: `only=move-files,move-thumbnails` + +Availabile subtasks: + + - `move-files`: The main task, moves database and filesystem data + - `move-thumbnails`: Move existing thumbnails, rather than have them generated on the fly. + This task is optional, but helps to avoid growing your asset folder (no duplicate thumbnails) + - `generate-cms-thumbnails`: The new CMS UI needs different thumbnail sizes, which can be pregenerated. + This can be a CPU and memory intensive task for large asset stores. + ## Automatic migration Migration can be invoked by either this task, or can be configured to automatically run during dev build @@ -37,14 +53,12 @@ SilverStripe\Assets\File: You can also run this task without CLI access through the [queuedjobs](https://github.com/symbiote/silverstripe-queuedjobs) module. -## Migration of thumbnails - -If you have the [asset admin](https://github.com/silverstripe/silverstripe-asset-admin) module installed -this will also ensure that thumbnails for these images are generated when running 'MigrateFileTask'. -Existing thumbnails will not be migrated however, and must be re-generated for use in the CMS. +## Migration of existing thumbnails -Note: Thumbnails can be regenerated on a one-by-one basis within the CMS by re-saving it -within the file edit details form. +Thumbnails generated through SilverStripe's image manipulation layer can be created by authors +resizing images in the rich text editor, through template or PHP code, or by SilverStripe's built-in CMS logic. +They are now called "variants", and are placed in a different folder structure. In order to avoid re-generating those thumbnails, +and cluttering up your asset store with orphaned files, the task will move them to the new location by default. ## Discarded files during migration diff --git a/docs/en/04_Changelogs/4.4.0.md b/docs/en/04_Changelogs/4.4.0.md index 9ab0bfec73b..da023088bb3 100644 --- a/docs/en/04_Changelogs/4.4.0.md +++ b/docs/en/04_Changelogs/4.4.0.md @@ -2,6 +2,8 @@ ## Overview {#overview} + - [Optional migration to hash-less public asset URLs](#hash-less) + - [Optional migration of legacy thumbnail locations](#legacy-thumb) - [Correct PHP types are now returned from database queries](/developer_guides/model/sql_select#data-types) - [Server Requirements](/getting_started/server_requirements/#web-server-software-requirements) have been refined: MySQL 5.5 end of life reached in December 2018, thus SilverStripe 4.4 requires MySQL 5.6+. @@ -11,7 +13,7 @@ ## Upgrading {#upgrading} -### Optional migration to hash-less public asset URLs +### Optional migration to hash-less public asset URLs {#hash-less} SilverStripe 4.x introduced an [asset abstraction](https://docs.silverstripe.org/en/4/developer_guides/files/file_storage/) system which required a [file migration](https://docs.silverstripe.org/en/4/developer_guides/files/file_migration/) task. @@ -42,6 +44,29 @@ the [queuedjobs](https://github.com/symbiote/silverstripe-queuedjobs) module. Further information is provided in the [Hash-less Public Asset URLs FAQ](#hashless-faq) below. +### Optional migration of legacy thumbnail locations {#legacy-thumb} + +Alongside the new [asset abstraction](https://docs.silverstripe.org/en/4/developer_guides/files/file_storage/), +we've also changed where thumbnails are [stored](/developer_guides/files/file_storage#variant-file-paths). +The file format is now generalised as a "variant", and no longer lives in a `_resampled` folder. +Prior to this release, these thumbnails were left in place. For thumbnails generated +through PHP or template code, these are generated on demand in their new location, +causing duplication and unnecessary storage space. + +You can opt-in to moving these legacy thumbnails to their new locations +with a subtask of `dev/tasks/MigrateFileTask`. If newer thumbnails have been generated, +it'll keep those. New migrations from 3.x to 4.x will include this migration step by default. + +``` +vendor/bin/sake dev/tasks/MigrateFileTask only=generate-cms-thumbnails +``` + +Note that as part of the [hash-less public asset URLs](#hash-less) +introduced in this release, requests to these legacy thumbnails will automatically redirect to +their new locations. + +TODO Add reference to shortcode rewriting task + ### Adopting to new `_resources` directory The name of the directory where vendor module resources are exposed can now be configured by defining a `extra.resources-dir` key in your `composer.json` file. If the key is not set, it will automatically default to `resources`. New projects will be preset to `_resources`. diff --git a/src/Dev/Tasks/MigrateFileTask.php b/src/Dev/Tasks/MigrateFileTask.php index a7ca519f5e8..2669a8aeec0 100644 --- a/src/Dev/Tasks/MigrateFileTask.php +++ b/src/Dev/Tasks/MigrateFileTask.php @@ -2,7 +2,19 @@ namespace SilverStripe\Dev\Tasks; +use Monolog\Formatter\FormatterInterface; +use Monolog\Handler\StreamHandler; +use Monolog\Logger; +use Psr\Log\LoggerInterface; +use Psr\Log\NullLogger; use SilverStripe\AssetAdmin\Helper\ImageThumbnailHelper; +use SilverStripe\Assets\LegacyThumbnailMigrationHelper; +use SilverStripe\Assets\Storage\AssetStore; +use SilverStripe\Control\Director; +use SilverStripe\Core\Injector\Injector; +use SilverStripe\Core\Tests\Injector\InjectorTest; +use SilverStripe\Logging\HTTPOutputHandler; +use SilverStripe\Logging\PreformattedEchoHandler; use SilverStripe\ORM\DB; use SilverStripe\Assets\FileMigrationHelper; use SilverStripe\Dev\BuildTask; @@ -17,33 +29,134 @@ class MigrateFileTask extends BuildTask protected $title = 'Migrate File dataobjects from 3.x'; - protected $description = - 'Imports all files referenced by File dataobjects into the new Asset Persistence Layer introduced in 4.0. ' . - 'If the task fails or times out, run it again and it will start where it left off.'; + protected $defaultSubtasks = [ + 'move-files', + 'move-thumbnails', + 'generate-cms-thumbnails' + ]; + + private static $dependencies = [ + 'logger' => '%$' . LoggerInterface::class, + ]; + + /** @var Logger */ + private $logger; public function run($request) { - if (!class_exists(FileMigrationHelper::class)) { - DB::alteration_message("No file migration helper detected", "notice"); - return; + // TODO Refactor this whole mess into Symfony Console on a TaskRunner level, + // with a thin wrapper to show coloured console output via a browser: + // https://github.com/silverstripe/silverstripe-framework/issues/5542 + if (Director::is_cli()) { + $this->logger->pushHandler(new StreamHandler('php://stdout')); + $this->logger->pushHandler(new StreamHandler('php://stderr', Logger::WARNING)); + } else { + $this->logger->pushHandler(new PreformattedEchoHandler()); } - DB::alteration_message( - 'If the task fails or times out, run it again and it will start where it left off.', - "info" - ); + $args = $request->getVars(); + $this->validateArgs($args); - $migrated = FileMigrationHelper::singleton()->run(); - if ($migrated) { - DB::alteration_message("{$migrated} File DataObjects upgraded", "changed"); - } else { - DB::alteration_message("No File DataObjects need upgrading", "notice"); + $subtasks = !empty($args['only']) ? explode(',', $args['only']) : $this->defaultSubtasks; + + if (in_array('move-files', $subtasks)) { + if (!class_exists(FileMigrationHelper::class)) { + $this->logger->error("No file migration helper detected"); + return; + } + + $this->logger->info('### Migrating filesystem and database records (move-files)'); + + $this->logger->info('If the task fails or times out, run it again and it will start where it left off.'); + + $migrated = FileMigrationHelper::singleton()->run(); + if ($migrated) { + $this->logger->info("{$migrated} File DataObjects upgraded"); + } else { + $this->logger->info("No File DataObjects need upgrading"); + } } - if (!class_exists(ImageThumbnailHelper::class)) { - DB::alteration_message("No image thumbnail helper detected", "notice"); - return; + if (in_array('move-thumbnails', $subtasks)) { + if (!class_exists(LegacyThumbnailMigrationHelper::class)) { + $this->logger->error("LegacyThumbnailMigrationHelper not found"); + return; + } + + $this->logger->info('### Migrating existing thumbnails (move-thumbnails)'); + + $moved = LegacyThumbnailMigrationHelper::singleton() + ->setLogger($this->logger) + ->run($this->getStore()); + + if ($moved) { + $this->logger->info(sprintf("%d thumbnails moved", count($moved))); + } else { + $this->logger->info("No thumbnails moved"); + } + } + + if (in_array('generate-cms-thumbnails', $subtasks)) { + if (!class_exists(ImageThumbnailHelper::class)) { + $this->logger->error("ImageThumbnailHelper not found"); + return; + } + + $this->logger->info('### Generating new CMS UI thumbnails (generate-cms-thumbnails)'); + + ImageThumbnailHelper::singleton()->run(); + } + } + + public function getDescription() + { + return <<logger = $logger; + + return $this; + } + + /** + * @return AssetStore + */ + protected function getStore() + { + return singleton(AssetStore::class); + } + + /** + * @param array $args + * @throws \InvalidArgumentException + */ + protected function validateArgs($args) + { + if (!empty($args['only'])) { + if (array_diff(explode(',', $args['only']), $this->defaultSubtasks)) { + throw new \InvalidArgumentException('Invalid subtasks detected: ' . $args['only']); + } } - ImageThumbnailHelper::singleton()->run(); } } diff --git a/src/Logging/PreformattedEchoHandler.php b/src/Logging/PreformattedEchoHandler.php new file mode 100644 index 00000000000..c4c65aab2fc --- /dev/null +++ b/src/Logging/PreformattedEchoHandler.php @@ -0,0 +1,28 @@ +%s', htmlspecialchars($record['formatted'], ENT_QUOTES, 'UTF-8')); + } +}