diff --git a/composer.json b/composer.json index 3407229e5d..bd0a5d1e1d 100755 --- a/composer.json +++ b/composer.json @@ -1,7 +1,7 @@ { "name": "magento/ece-tools", "description": "Provides tools to build and deploy Magento 2 Enterprise Edition", - "version": "2002.0.7", + "version": "2002.0.8", "license": [ "OSL-3.0", "AFL-3.0" diff --git a/patches.json b/patches.json new file mode 100644 index 0000000000..409c3c1824 --- /dev/null +++ b/patches.json @@ -0,0 +1,28 @@ +{ + "colinmollenhour/credis": { + "Fix Redis issue": { + "1.6": "patches/redis-pipeline.patch" + } + }, + "magento/framework": { + "Fix locker process": { + "101.0.*": "patches/locker-process.patch" + }, + "Remove permissions check": { + "101.0.*": "patches/remove-permission-checks.patch" + }, + "Fix redis session locking for 2.2.0 and 2.2.1": { + "101.0.0||101.0.1": "patches/fix-redis-session-manager-locking.patch" + } + }, + "magento/module-customer-import-export": { + "Fix out of memory during import of customers and addresses": { + ">=100.2.0": "patches/fix-oom-during-import-customers-and-addresses.patch" + } + }, + "magento/module-config": { + "Fix app:config:import for Magento 2.2.2": { + "=101.0.2": "patches/fix-app-config-import.patch" + } + } +} diff --git a/patches/fix-app-config-import.patch b/patches/fix-app-config-import.patch new file mode 100644 index 0000000000..01b465a750 --- /dev/null +++ b/patches/fix-app-config-import.patch @@ -0,0 +1,14 @@ +--- a/vendor/magento/module-config/Model/Config/Importer.php ++++ b/vendor/magento/module-config/Model/Config/Importer.php +@@ -129,8 +129,10 @@ class Importer implements ImporterInterface + + // Invoke saving of new values. + $this->saveProcessor->process($changedData); +- $this->flagManager->saveFlag(static::FLAG_CODE, $data); + }); ++ ++ $this->scope->setCurrentScope($currentScope); ++ $this->flagManager->saveFlag(static::FLAG_CODE, $data); + } catch (\Exception $e) { + throw new InvalidTransitionException(__('%1', $e->getMessage()), $e); + } finally { diff --git a/patches/fix-oom-during-import-customers-and-addresses.patch b/patches/fix-oom-during-import-customers-and-addresses.patch new file mode 100644 index 0000000000..63b9bc1b33 --- /dev/null +++ b/patches/fix-oom-during-import-customers-and-addresses.patch @@ -0,0 +1,110 @@ +commit 4ee8443a262e18c08b942aef313710b2c070a7a4 +Author: Viktor Paladiichuk +Date: Thu Nov 16 18:55:15 2017 +0200 + + SET-36: Memory limit exhausted during import of customers and addresses + +diff --git a/vendor/magento/module-customer-import-export/Model/Import/Address.php b/vendor/magento/module-customer-import-export/Model/Import/Address.php +index eb5742d24c7..70b8c34ef41 100644 +--- a/vendor/magento/module-customer-import-export/Model/Import/Address.php ++++ b/vendor/magento/module-customer-import-export/Model/Import/Address.php +@@ -238,6 +238,11 @@ class Address extends AbstractCustomer + protected $postcodeValidator; + + /** ++ * @var array ++ */ ++ private $loadedAddresses; ++ ++ /** + * @param \Magento\Framework\Stdlib\StringUtils $string + * @param \Magento\Framework\App\Config\ScopeConfigInterface $scopeConfig + * @param \Magento\ImportExport\Model\ImportFactory $importFactory +@@ -368,21 +373,50 @@ class Address extends AbstractCustomer + */ + protected function _initAddresses() + { +- /** @var $address \Magento\Customer\Model\Address */ +- foreach ($this->_addressCollection as $address) { +- $customerId = $address->getParentId(); +- if (!isset($this->_addresses[$customerId])) { +- $this->_addresses[$customerId] = []; ++ if ($this->_addressCollection->isLoaded()) { ++ /** @var $address \Magento\Customer\Model\Address */ ++ foreach ($this->_addressCollection as $address) { ++ $customerId = $address->getParentId(); ++ if (!isset($this->_addresses[$customerId])) { ++ $this->_addresses[$customerId] = []; ++ } ++ $addressId = $address->getId(); ++ if (!in_array($addressId, $this->_addresses[$customerId])) { ++ $this->_addresses[$customerId][] = $addressId; ++ } + } +- $addressId = $address->getId(); +- if (!in_array($addressId, $this->_addresses[$customerId])) { +- $this->_addresses[$customerId][] = $addressId; ++ } else { ++ foreach ($this->getLoadedAddresses() as $addressId => $address) { ++ $customerId = $address['parent_id']; ++ if (!isset($this->_addresses[$customerId])) { ++ $this->_addresses[$customerId] = []; ++ } ++ if (!in_array($addressId, $this->_addresses[$customerId])) { ++ $this->_addresses[$customerId][] = $addressId; ++ } + } + } + return $this; + } + + /** ++ * @return array ++ */ ++ private function getLoadedAddresses() ++ { ++ if (empty($this->loadedAddresses)) { ++ $collection = clone $this->_addressCollection; ++ $table = $collection->getMainTable(); ++ $select = $collection->getSelect(); ++ $select->reset('columns'); ++ $select->reset('from'); ++ $select->from($table, ['entity_id', 'parent_id']); ++ $this->loadedAddresses = $collection->getResource()->getConnection()->fetchAssoc($select); ++ } ++ return $this->loadedAddresses; ++ } ++ ++ /** + * Initialize country regions hash for clever recognition + * + * @return $this +diff --git a/vendor/magento/module-customer-import-export/Model/ResourceModel/Import/Customer/Storage.php b/vendor/magento/module-customer-import-export/Model/ResourceModel/Import/Customer/Storage.php +index 4e6687bff28..359822df6d9 100644 +--- a/vendor/magento/module-customer-import-export/Model/ResourceModel/Import/Customer/Storage.php ++++ b/vendor/magento/module-customer-import-export/Model/ResourceModel/Import/Customer/Storage.php +@@ -117,13 +117,18 @@ class Storage + */ + public function getCustomerId($email, $websiteId) + { +- // lazy loading +- $this->load(); ++ if (!isset($this->_customerIds[$email][$websiteId])) { ++ $collection = clone $this->_customerCollection; ++ $mainTable = $collection->getResource()->getEntityTable(); + +- if (isset($this->_customerIds[$email][$websiteId])) { +- return $this->_customerIds[$email][$websiteId]; +- } ++ $select = $collection->getSelect(); ++ $select->reset(); ++ $select->from($mainTable, ['entity_id']); ++ $select->where($mainTable . '.email = ?', $email); ++ $select->where($mainTable . '.website_id = ?', $websiteId); + +- return false; ++ $this->_customerIds[$email][$websiteId] = $collection->getResource()->getConnection()->fetchOne($select); ++ } ++ return $this->_customerIds[$email][$websiteId]; + } + } diff --git a/patches/fix-redis-session-manager-locking.patch b/patches/fix-redis-session-manager-locking.patch new file mode 100644 index 0000000000..88f3dff4c4 --- /dev/null +++ b/patches/fix-redis-session-manager-locking.patch @@ -0,0 +1,24 @@ +diff --git a/vendor/magento/framework/Session/SessionManager.php b/vendor/magento/framework/Session/SessionManager.php +index 2cea02f..272d3d9 100644 +--- a/vendor/magento/framework/Session/SessionManager.php ++++ b/vendor/magento/framework/Session/SessionManager.php +@@ -504,18 +504,8 @@ class SessionManager implements SessionManagerInterface + return $this; + } + +- //@see http://php.net/manual/en/function.session-regenerate-id.php#53480 workaround + if ($this->isSessionExists()) { +- $oldSessionId = session_id(); +- session_regenerate_id(); +- $newSessionId = session_id(); +- session_id($oldSessionId); +- session_destroy(); +- +- $oldSession = $_SESSION; +- session_id($newSessionId); +- session_start(); +- $_SESSION = $oldSession; ++ session_regenerate_id(true); + } else { + session_start(); + } diff --git a/patches/locker-process.patch b/patches/locker-process.patch new file mode 100644 index 0000000000..44142f5677 --- /dev/null +++ b/patches/locker-process.patch @@ -0,0 +1,76 @@ +MDVA-2470 +--- a/vendor/magento/framework/View/Asset/LockerProcess.php 2017-02-13 17:38:15.000000000 +0000 ++++ b/vendor/magento/framework/View/Asset/LockerProcess.php 2017-02-13 18:53:22.000000000 +0000 +@@ -68,12 +68,26 @@ + + $this->tmpDirectory = $this->filesystem->getDirectoryWrite(DirectoryList::VAR_DIR); + $this->lockFilePath = $this->getFilePath($lockName); ++ $this->waitForLock(); + +- while ($this->isProcessLocked()) { +- usleep(1000); ++ try { ++ $this->tmpDirectory->writeFile($this->lockFilePath, time(), 'x+'); ++ }catch (\Exception $e) { ++ $this->waitForLock(); ++ try { ++ $this->tmpDirectory->writeFile($this->lockFilePath, time(), 'x+'); ++ }catch (\Exception $e) { ++ throw new \Exception($e->getMessage()); ++ } + } + +- $this->tmpDirectory->writeFile($this->lockFilePath, time()); ++ } ++ ++ public function waitForLock() ++ { ++ while ($this->isProcessLocked() ) { ++ usleep(500); ++ } + } + + /** +--- a/vendor/magento/framework/View/Asset/PreProcessor/AlternativeSource.php 2017-02-14 20:49:33.000000000 +0000 ++++ b/vendor/magento/framework/View/Asset/PreProcessor/AlternativeSource.php 2017-02-15 15:00:41.000000000 +0000 +@@ -106,7 +106,7 @@ + } + + try { +- $this->lockerProcess->lockProcess($this->lockName); ++ $this->lockerProcess->lockProcess($chain->getAsset()->getPath()); + + $module = $chain->getAsset()->getModule(); + +--- a/vendor/magento/framework/View/Asset/LockerProcess.php 2017-02-14 21:50:57.000000000 +0000 ++++ b/vendor/magento/framework/View/Asset/LockerProcess.php 2017-02-15 15:00:41.000000000 +0000 +@@ -67,7 +67,7 @@ + } + + $this->tmpDirectory = $this->filesystem->getDirectoryWrite(DirectoryList::VAR_DIR); +- $this->lockFilePath = $this->getFilePath($lockName); ++ $this->lockFilePath = $this->getFilePath(str_replace(DIRECTORY_SEPARATOR, '_', $lockName)); + $this->waitForLock(); + + try { +@@ -77,7 +77,8 @@ + try { + $this->tmpDirectory->writeFile($this->lockFilePath, time(), 'x+'); + }catch (\Exception $e) { +- throw new \Exception($e->getMessage()); ++ echo($this->lockFilePath); ++ throw new \Exception("In exception for lock process" . $e->getMessage()); + } + } + +--- a/vendor/magento/module-developer/Model/View/Asset/PreProcessor/FrontendCompilation.php 2017-02-15 16:24:07.000000000 +0000 ++++ b/vendor/magento/module-developer/Model/View/Asset/PreProcessor/FrontendCompilation.php 2017-02-15 16:24:07.000000000 +0000 +@@ -76,7 +76,7 @@ + { + + try { +- $this->lockerProcess->lockProcess($this->lockName); ++ $this->lockerProcess->lockProcess($chain->getAsset()->getPath()); + + $path = $chain->getAsset()->getFilePath(); + $module = $chain->getAsset()->getModule(); diff --git a/patches/redis-pipeline.patch b/patches/redis-pipeline.patch new file mode 100644 index 0000000000..6b1ca9186c --- /dev/null +++ b/patches/redis-pipeline.patch @@ -0,0 +1,12 @@ +diff --git a/vendor/colinmollenhour/credis/Client.php b/vendor/colinmollenhour/credis/Client.php +index afbc85d..8368b32 100755 +--- a/vendor/colinmollenhour/credis/Client.php ++++ b/vendor/colinmollenhour/credis/Client.php +@@ -1017,6 +1017,7 @@ class Credis_Client { + } else { + $this->isMulti = TRUE; + $this->redisMulti = call_user_func_array(array($this->redis, $name), $args); ++ return $this; + } + } + else if($name == 'exec' || $name == 'discard') { diff --git a/patches/remove-permission-checks.patch b/patches/remove-permission-checks.patch new file mode 100644 index 0000000000..7592814d8e --- /dev/null +++ b/patches/remove-permission-checks.patch @@ -0,0 +1,43 @@ +diff -Naur b/vendor/magento/framework/Setup/FilePermissions.php a/vendor/magento/framework/Setup/FilePermissions.php +--- b/vendor/magento/framework/Setup/FilePermissions.php 2016-09-23 16:01:12.000000000 -0500 ++++ a/vendor/magento/framework/Setup/FilePermissions.php 2016-09-23 16:22:09.000000000 -0500 +@@ -233,26 +233,8 @@ + */ + public function getMissingWritablePathsForInstallation($associative = false) + { +- $required = $this->getInstallationWritableDirectories(); +- $current = $this->getInstallationCurrentWritableDirectories(); +- $missingPaths = []; +- foreach (array_diff($required, $current) as $missingPath) { +- if (isset($this->nonWritablePathsInDirectories[$missingPath])) { +- if ($associative) { +- $missingPaths[$missingPath] = $this->nonWritablePathsInDirectories[$missingPath]; +- } else { +- $missingPaths = array_merge( +- $missingPaths, +- $this->nonWritablePathsInDirectories[$missingPath] +- ); +- } +- } +- } +- if ($associative) { +- $required = array_flip($required); +- $missingPaths = array_merge($required, $missingPaths); +- } +- return $missingPaths; ++ // Unnecessary check in controlled environment ++ return []; + } + + /** +@@ -275,8 +257,7 @@ + */ + public function getUnnecessaryWritableDirectoriesForApplication() + { +- $required = $this->getApplicationNonWritableDirectories(); +- $current = $this->getApplicationCurrentNonWritableDirectories(); +- return array_diff($required, $current); ++ // Unnecessary check in controlled environment ++ return []; + } + } diff --git a/src/App/Logger.php b/src/App/Logger.php index 49eef302dd..865b0d282c 100644 --- a/src/App/Logger.php +++ b/src/App/Logger.php @@ -66,7 +66,10 @@ private function prepare() $this->file->createDirectory($this->directoryList->getLog()); - if ($deployLogFileExists && !$this->isBuildLogApplied($deployLogPath, $buildPhaseLogContent)) { + if ($deployLogFileExists + && $buildPhaseLogContent + && !$this->isBuildLogApplied($deployLogPath, $buildPhaseLogContent) + ) { $this->file->filePutContents($deployLogPath, $buildPhaseLogContent, FILE_APPEND); } elseif (!$deployLogFileExists && $buildLogFileExists) { $this->file->copy($buildPhaseLogPath, $deployLogPath); diff --git a/src/Application.php b/src/Application.php index 42d0209c1e..a03860759a 100644 --- a/src/Application.php +++ b/src/Application.php @@ -6,6 +6,7 @@ namespace Magento\MagentoCloud; use Composer\Composer; +use Magento\MagentoCloud\Command\ApplyPatches; use Magento\MagentoCloud\Command\BackupList; use Magento\MagentoCloud\Command\BackupRestore; use Magento\MagentoCloud\Command\Build; @@ -65,6 +66,7 @@ protected function getDefaultCommands() $this->container->get(CronUnlock::class), $this->container->get(BackupRestore::class), $this->container->get(BackupList::class), + $this->container->get(ApplyPatches::class), ] ); } diff --git a/src/Command/ApplyPatches.php b/src/Command/ApplyPatches.php new file mode 100644 index 0000000000..84ec7e3a14 --- /dev/null +++ b/src/Command/ApplyPatches.php @@ -0,0 +1,71 @@ +logger = $logger; + $this->manager = $manager; + + parent::__construct(); + } + + /** + * @inheritdoc + */ + protected function configure() + { + $this->setName(static::NAME) + ->setDescription('Applies custom patches'); + + parent::configure(); + } + + /** + * @inheritdoc + */ + public function execute(InputInterface $input, OutputInterface $output) + { + try { + $this->logger->info('Patching started.'); + $this->manager->applyAll(); + $this->logger->info('Patching finished.'); + } catch (\Exception $exception) { + $this->logger->critical($exception->getMessage()); + + throw $exception; + } + } +} diff --git a/src/Config/Stage/Build.php b/src/Config/Stage/Build.php index 0b4a4ea8b0..5913473af4 100644 --- a/src/Config/Stage/Build.php +++ b/src/Config/Stage/Build.php @@ -130,7 +130,7 @@ private function getDeprecatedConfig(): array } if (isset($buildConfig['skip_scd'])) { - $result[self::VAR_SKIP_SCD] = $buildConfig['skip_scd'] === 'yes'; + $result[self::VAR_SKIP_SCD] = $buildConfig['skip_scd'] === '1'; } if (isset($buildConfig['VERBOSE_COMMANDS'])) { diff --git a/src/Config/Stage/Deploy.php b/src/Config/Stage/Deploy.php index 05715577ca..3eca68640f 100644 --- a/src/Config/Stage/Deploy.php +++ b/src/Config/Stage/Deploy.php @@ -111,7 +111,6 @@ private function getEnvironmentConfig(): array } $disabledFlow = [ - self::VAR_REDIS_SESSION_DISABLE_LOCKING, self::VAR_CLEAN_STATIC_FILES, self::VAR_STATIC_CONTENT_SYMLINK, self::VAR_UPDATE_URLS, @@ -167,7 +166,6 @@ private function getDefault(): array return [ self::VAR_SCD_STRATEGY => '', self::VAR_SCD_COMPRESSION_LEVEL => 4, - self::VAR_REDIS_SESSION_DISABLE_LOCKING => true, self::VAR_SEARCH_CONFIGURATION => [], self::VAR_QUEUE_CONFIGURATION => [], self::VAR_VERBOSE_COMMANDS => '', diff --git a/src/Config/Stage/DeployInterface.php b/src/Config/Stage/DeployInterface.php index e819ae7d18..9184c9cf8d 100644 --- a/src/Config/Stage/DeployInterface.php +++ b/src/Config/Stage/DeployInterface.php @@ -14,7 +14,6 @@ interface DeployInterface extends StageConfigInterface { const VAR_QUEUE_CONFIGURATION = 'QUEUE_CONFIGURATION'; const VAR_SEARCH_CONFIGURATION = 'SEARCH_CONFIGURATION'; - const VAR_REDIS_SESSION_DISABLE_LOCKING = 'REDIS_SESSION_DISABLE_LOCKING'; const VAR_CRON_CONSUMERS_RUNNER = 'CRON_CONSUMERS_RUNNER'; const VAR_CLEAN_STATIC_FILES = 'CLEAN_STATIC_FILES'; const VAR_STATIC_CONTENT_SYMLINK = 'STATIC_CONTENT_SYMLINK'; diff --git a/src/Filesystem/FileList.php b/src/Filesystem/FileList.php index 2362e746e2..7af691f1c6 100644 --- a/src/Filesystem/FileList.php +++ b/src/Filesystem/FileList.php @@ -105,4 +105,12 @@ public function getInstallUpgradeLog(): string { return $this->directoryList->getLog() . '/install_upgrade.log'; } + + /** + * @return string + */ + public function getPatches(): string + { + return $this->directoryList->getRoot() . '/patches.json'; + } } diff --git a/src/Patch/Applier.php b/src/Patch/Applier.php new file mode 100644 index 0000000000..e3202a52c0 --- /dev/null +++ b/src/Patch/Applier.php @@ -0,0 +1,125 @@ +composer = $composer; + $this->repository = $composer->getRepositoryManager()->getLocalRepository(); + $this->shell = $shell; + $this->logger = $logger; + $this->directoryList = $directoryList; + $this->file = $file; + } + + /** + * Applies patch, using 'git apply' command. + * + * @param string $path Path to patch + * @param string|null $name Name of patch + * @param string|null $packageName Name of package to be patched + * @param string|null $constraint Specific constraint of package to be fixed + * @return void + * @throws \RuntimeException + */ + public function apply(string $path, $name, $packageName, $constraint) + { + $name = $name ?: $path; + + /** + * Support for relative paths. + */ + if (!$this->file->isExists($path)) { + $path = $this->directoryList->getRoot() . '/' . $path; + } + + $this->logger->info(sprintf( + 'Applying patch %s %s.', + $name, + $constraint + )); + + if ($packageName && !$this->matchConstraint($packageName, $constraint)) { + $this->logger->notice(sprintf( + 'Constraint %s %s was not found.', + $packageName, + $constraint + )); + + return; + } + + $this->shell->execute('git apply ' . $path); + $this->logger->info('Done.'); + } + + /** + * Checks whether package with specific constraint exists in the system. + * + * @param string $packageName + * @param string $constraint + * @return bool True if patch with provided constraint exists, false otherwise. + */ + private function matchConstraint(string $packageName, string $constraint): bool + { + return $this->repository->findPackage($packageName, $constraint) instanceof PackageInterface; + } +} diff --git a/src/Patch/Manager.php b/src/Patch/Manager.php new file mode 100644 index 0000000000..442d4f4f37 --- /dev/null +++ b/src/Patch/Manager.php @@ -0,0 +1,177 @@ +applier = $applier; + $this->logger = $logger; + $this->file = $file; + $this->fileList = $fileList; + $this->directoryList = $directoryList; + } + + /** + * Applies all needed patches. + * + * @throws \RuntimeException + * @throws FileSystemException + */ + public function applyAll() + { + $this->copyStaticFile(); + $this->applyComposerPatches(); + $this->applyHotFixes(); + } + + /** + * Copying static file endpoint. + * This resolves issue MAGECLOUD-314 + * + * @return void + * @throws FileSystemException + */ + private function copyStaticFile() + { + $magentoRoot = $this->directoryList->getMagentoRoot(); + + if (!$this->file->isExists($magentoRoot . '/pub/static.php')) { + $this->logger->notice('File static.php was not found.'); + + return; + } + + $this->file->copy($magentoRoot . '/pub/static.php', $magentoRoot . '/pub/front-static.php'); + $this->logger->info('File static.php was copied.'); + } + + /** + * Applies patches from composer.json file. + * Patches are applying from top to bottom of config list. + * + * ``` + * "colinmollenhour/credis" : { + * "Fix Redis issue": { + * "1.6": "patches/redis-pipeline.patch" + * } + * } + * + * Each patch must have corresponding constraint of target package, + * in one of the following format: + * - 1.6 + * - 1.6.* + * - ^1.6 + * + * @return void + * @throws \RuntimeException + * @throws FileSystemException + */ + private function applyComposerPatches() + { + $patches = json_decode( + $this->file->fileGetContents($this->fileList->getPatches()), + true + ); + + if (!$patches) { + $this->logger->notice('Patching skipped.'); + + return; + } + + foreach ($patches as $packageName => $patchesInfo) { + foreach ($patchesInfo as $patchName => $packageInfo) { + if (is_string($packageInfo)) { + $this->applier->apply($packageInfo, $patchName, $packageName, '*'); + } elseif (is_array($packageInfo)) { + foreach ($packageInfo as $constraint => $path) { + $this->applier->apply($path, $patchName, $packageName, $constraint); + } + } + } + } + } + + /** + * Applies patches from root directory m2-hotfixes. + * + * @return void + * @throws \RuntimeException + * @throws FileSystemException + */ + private function applyHotFixes() + { + $hotFixesDir = $this->directoryList->getMagentoRoot() . '/' . static::HOTFIXES_DIR; + + if (!$this->file->isDirectory($hotFixesDir)) { + $this->logger->notice('Hot-fixes directory was not found. Skipping.'); + + return; + } + + $this->logger->info('Applying hot-fixes.'); + + $files = glob($hotFixesDir . '/*.patch'); + sort($files); + + foreach ($files as $file) { + $this->applier->apply($file, null, null, null); + } + } +} diff --git a/src/Process/Build/ApplyPatches.php b/src/Process/Build/ApplyPatches.php index 746d985252..ff72cf0680 100644 --- a/src/Process/Build/ApplyPatches.php +++ b/src/Process/Build/ApplyPatches.php @@ -5,10 +5,8 @@ */ namespace Magento\MagentoCloud\Process\Build; -use Magento\MagentoCloud\Filesystem\DirectoryList; -use Magento\MagentoCloud\Filesystem\Driver\File; +use Magento\MagentoCloud\Patch\Manager; use Magento\MagentoCloud\Process\ProcessInterface; -use Magento\MagentoCloud\Shell\ShellInterface; use Psr\Log\LoggerInterface; /** @@ -16,42 +14,26 @@ */ class ApplyPatches implements ProcessInterface { - /** - * @var ShellInterface - */ - private $shell; - /** * @var LoggerInterface */ private $logger; /** - * @var File - */ - private $file; - - /** - * @var DirectoryList + * @var Manager */ - private $directoryList; + private $manager; /** - * @param ShellInterface $shell * @param LoggerInterface $logger - * @param File $file - * @param DirectoryList $directoryList + * @param Manager $manager */ public function __construct( - ShellInterface $shell, LoggerInterface $logger, - File $file, - DirectoryList $directoryList + Manager $manager ) { - $this->shell = $shell; $this->logger = $logger; - $this->file = $file; - $this->directoryList = $directoryList; + $this->manager = $manager; } /** @@ -60,15 +42,6 @@ public function __construct( public function execute() { $this->logger->info('Applying patches.'); - - $patchScript = $this->directoryList->getMagentoRoot() . '/vendor/bin/m2-apply-patches'; - - if (!$this->file->isExists($patchScript)) { - $this->logger->warning('Package with patches was not found.'); - - return; - } - - $this->shell->execute('php ' . $patchScript); + $this->manager->applyAll(); } } diff --git a/src/Process/Deploy/InstallUpdate/ConfigUpdate/Redis.php b/src/Process/Deploy/InstallUpdate/ConfigUpdate/Redis.php index 05485dc7d6..52ef0ab7e6 100644 --- a/src/Process/Deploy/InstallUpdate/ConfigUpdate/Redis.php +++ b/src/Process/Deploy/InstallUpdate/ConfigUpdate/Redis.php @@ -96,7 +96,6 @@ public function execute() 'host' => $redisConfig[0]['host'], 'port' => $redisConfig[0]['port'], 'database' => 0, - 'disable_locking' => (int)$this->isLockingDisabled(), ]; $config['session'] = [ 'save' => 'redis', @@ -105,6 +104,11 @@ public function execute() $redisSessionConfig ), ]; + + if (isset($config['session']['redis']['disable_locking'])) { + $this->logger->info('Removing disable_locking env.php.'); + unset($config['session']['redis']['disable_locking']); + } } else { $config = $this->removeRedisConfiguration($config); } @@ -139,16 +143,4 @@ private function removeRedisConfiguration($config) return $config; } - - /** - * Checks if disable_locking options is enabled. - * By default this method returns true and disable_locking options will be set to 1. - * For turning this option off environment variable 'REDIS_SESSION_DISABLE_LOCKING' should have value 'disabled'. - * - * @return bool - */ - private function isLockingDisabled(): bool - { - return $this->stageConfig->get(DeployInterface::VAR_REDIS_SESSION_DISABLE_LOCKING); - } } diff --git a/src/Test/Integration/Bootstrap.php b/src/Test/Integration/Bootstrap.php index c6a805fdcf..8ed774085b 100644 --- a/src/Test/Integration/Bootstrap.php +++ b/src/Test/Integration/Bootstrap.php @@ -135,7 +135,7 @@ public function createApplication(array $environment): Application ]); $container = new Container(new DirectoryList( - $this->getSandboxDir(), + ECE_BP, $this->getSandboxDir(), [] )); diff --git a/src/Test/Unit/App/LoggerTest.php b/src/Test/Unit/App/LoggerTest.php index 4105828bfa..724e739ee9 100644 --- a/src/Test/Unit/App/LoggerTest.php +++ b/src/Test/Unit/App/LoggerTest.php @@ -139,6 +139,15 @@ public function executeDataProvider(): array 'deployLogFileExists' => true, 'fileMockFilePutContentsExpects' => 1, 'fileMockCopyExpects' => 0, + ], + [ + 'fileMockFileGetContentsExpects' => 0, + 'buildPhaseLogContent' => '', + 'buildLogFileExists' => false, + 'deployLogContent' => 'some log the build phase log was applied some log', + 'deployLogFileExists' => true, + 'fileMockFilePutContentsExpects' => 0, + 'fileMockCopyExpects' => 0, ] ]; } diff --git a/src/Test/Unit/ApplicationTest.php b/src/Test/Unit/ApplicationTest.php index 5fb41c4ca6..9947d2bf6d 100644 --- a/src/Test/Unit/ApplicationTest.php +++ b/src/Test/Unit/ApplicationTest.php @@ -3,11 +3,12 @@ * Copyright © Magento, Inc. All rights reserved. * See COPYING.txt for license details. */ -namespace Unit; +namespace Magento\MagentoCloud\Test\Unit; use Composer\Composer; use Composer\Package\PackageInterface; use Magento\MagentoCloud\Application; +use Magento\MagentoCloud\Command\ApplyPatches; use Magento\MagentoCloud\Command\BackupList; use Magento\MagentoCloud\Command\Build; use Magento\MagentoCloud\Command\ConfigDump; @@ -77,6 +78,7 @@ public function setUp() $postDeployCommandMock = $this->createMock(PostDeploy::class); $backupRestoreCommandMock = $this->createMock(BackupRestore::class); $backupListCommandMock = $this->createMock(BackupList::class); + $applyPatchesCommandMock = $this->createMock(ApplyPatches::class); $this->mockCommand($buildCommandMock, Build::NAME); $this->mockCommand($configDumpCommandMock, ConfigDump::NAME); @@ -87,6 +89,7 @@ public function setUp() $this->mockCommand($prestartCommandMock, Prestart::NAME); $this->mockCommand($backupRestoreCommandMock, BackupRestore::class); $this->mockCommand($backupListCommandMock, BackupList::class); + $this->mockCommand($applyPatchesCommandMock, ApplyPatches::class); $this->containerMock->method('get') ->willReturnMap([ @@ -100,6 +103,7 @@ public function setUp() [CronUnlock::class, $cronUnlockCommandMock], [BackupRestore::class, $backupRestoreCommandMock], [BackupList::class, $backupListCommandMock], + [ApplyPatches::class, $applyPatchesCommandMock], ]); $this->composerMock->method('getPackage') ->willReturn($this->packageMock); diff --git a/src/Test/Unit/Command/ApplyPatchesTest.php b/src/Test/Unit/Command/ApplyPatchesTest.php new file mode 100644 index 0000000000..9092c10aa9 --- /dev/null +++ b/src/Test/Unit/Command/ApplyPatchesTest.php @@ -0,0 +1,86 @@ +loggerMock = $this->getMockForAbstractClass(LoggerInterface::class); + $this->managerMock = $this->createMock(Manager::class); + + $this->command = new ApplyPatches( + $this->loggerMock, + $this->managerMock + ); + } + + public function testExecute() + { + $this->loggerMock->expects($this->exactly(2)) + ->method('info') + ->withConsecutive( + ['Patching started.'], + ['Patching finished.'] + ); + $this->managerMock->expects($this->once()) + ->method('applyAll'); + + $tester = new CommandTester( + $this->command + ); + $tester->execute([]); + + $this->assertSame(0, $tester->getStatusCode()); + } + + /** + * @expectedException \Exception + * @expectedExceptionMessage Some error + */ + public function testExecuteWithException() + { + $this->loggerMock->expects($this->once()) + ->method('info') + ->with('Patching started.'); + $this->loggerMock->expects($this->once()) + ->method('critical') + ->with('Some error'); + $this->managerMock->expects($this->once()) + ->method('applyAll') + ->willThrowException(new \Exception('Some error')); + + $tester = new CommandTester( + $this->command + ); + $tester->execute([]); + } +} diff --git a/src/Test/Unit/Command/_files/m2-hotfixes/patch1.patch b/src/Test/Unit/Command/_files/m2-hotfixes/patch1.patch new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/Test/Unit/Command/_files/m2-hotfixes/patch2.patch b/src/Test/Unit/Command/_files/m2-hotfixes/patch2.patch new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/Test/Unit/Command/_files/m2-hotfixes/readme.md b/src/Test/Unit/Command/_files/m2-hotfixes/readme.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/Test/Unit/Config/Stage/BuildTest.php b/src/Test/Unit/Config/Stage/BuildTest.php index f0185c7f03..9fb1b667ce 100644 --- a/src/Test/Unit/Config/Stage/BuildTest.php +++ b/src/Test/Unit/Config/Stage/BuildTest.php @@ -318,7 +318,7 @@ public function getDataProvider(): array ], ], [ - 'skip_scd' => 'yes', + 'skip_scd' => '1', ], true, ], diff --git a/src/Test/Unit/Config/Stage/DeployTest.php b/src/Test/Unit/Config/Stage/DeployTest.php index 78fdd1ee27..6098d472c9 100644 --- a/src/Test/Unit/Config/Stage/DeployTest.php +++ b/src/Test/Unit/Config/Stage/DeployTest.php @@ -141,14 +141,6 @@ public function getDataProvider(): array [], '{"SOME_CONFIG": "some value', ], - 'disabled flow 1' => [ - Deploy::VAR_REDIS_SESSION_DISABLE_LOCKING, - [], - [ - Deploy::VAR_REDIS_SESSION_DISABLE_LOCKING => EnvironmentConfig::VAL_DISABLED, - ], - false, - ], 'disabled flow 2' => [ Deploy::VAR_UPDATE_URLS, [], diff --git a/src/Test/Unit/Patch/ApplierTest.php b/src/Test/Unit/Patch/ApplierTest.php new file mode 100644 index 0000000000..21ba4406ee --- /dev/null +++ b/src/Test/Unit/Patch/ApplierTest.php @@ -0,0 +1,175 @@ +composerMock = $this->createMock(Composer::class); + $this->shellMock = $this->getMockForAbstractClass(ShellInterface::class); + $this->loggerMock = $this->getMockForAbstractClass(LoggerInterface::class); + $this->localRepositoryMock = $this->getMockForAbstractClass(WritableRepositoryInterface::class); + $repositoryManagerMock = $this->createMock(RepositoryManager::class); + $this->directoryListMock = $this->createMock(DirectoryList::class); + $this->fileMock = $this->createMock(File::class); + + $repositoryManagerMock->expects($this->once()) + ->method('getLocalRepository') + ->willReturn($this->localRepositoryMock); + $this->composerMock->expects($this->once()) + ->method('getRepositoryManager') + ->willReturn($repositoryManagerMock); + + $this->applier = new Applier( + $this->composerMock, + $this->shellMock, + $this->loggerMock, + $this->directoryListMock, + $this->fileMock + ); + } + + public function testApply() + { + $path = 'path/to/patch'; + $name = 'patchName'; + $packageName = 'packageName'; + $constraint = '1.0'; + + $this->fileMock->expects($this->once()) + ->method('isExists') + ->with($path) + ->willReturn(true); + $this->localRepositoryMock->expects($this->once()) + ->method('findPackage') + ->with($packageName, $constraint) + ->willReturn($this->getMockForAbstractClass(PackageInterface::class)); + $this->shellMock->expects($this->once()) + ->method('execute') + ->with('git apply ' . $path); + $this->loggerMock->expects($this->exactly(2)) + ->method('info') + ->withConsecutive( + ['Applying patch patchName 1.0.'], + ['Done.'] + ); + $this->loggerMock->expects($this->never()) + ->method('notice'); + + $this->applier->apply($path, $name, $packageName, $constraint); + } + + public function testApplyPathNotExists() + { + $path = 'path/to/patch'; + $name = 'patchName'; + $packageName = 'packageName'; + $constraint = '1.0'; + + $this->fileMock->expects($this->once()) + ->method('isExists') + ->with($path) + ->willReturn(false); + $this->localRepositoryMock->expects($this->once()) + ->method('findPackage') + ->with($packageName, $constraint) + ->willReturn($this->getMockForAbstractClass(PackageInterface::class)); + $this->shellMock->expects($this->once()) + ->method('execute') + ->with('git apply root/' . $path); + $this->loggerMock->expects($this->never()) + ->method('notice'); + $this->loggerMock->expects($this->exactly(2)) + ->method('info') + ->withConsecutive( + ['Applying patch patchName 1.0.'], + ['Done.'] + ); + $this->directoryListMock->expects($this->once()) + ->method('getRoot') + ->willReturn('root'); + + $this->applier->apply($path, $name, $packageName, $constraint); + } + + public function testApplyPathNotExistsAndNotMatchedConstraints() + { + $path = 'path/to/patch'; + $name = 'patchName'; + $packageName = 'packageName'; + $constraint = '1.0'; + + $this->fileMock->expects($this->once()) + ->method('isExists') + ->with($path) + ->willReturn(false); + $this->localRepositoryMock->expects($this->once()) + ->method('findPackage') + ->with($packageName, $constraint) + ->willReturn(null); + $this->loggerMock->expects($this->once()) + ->method('notice') + ->with('Constraint packageName 1.0 was not found.'); + $this->loggerMock->expects($this->once()) + ->method('info') + ->with('Applying patch patchName 1.0.'); + $this->shellMock->expects($this->never()) + ->method('execute'); + + $this->applier->apply($path, $name, $packageName, $constraint); + } +} diff --git a/src/Test/Unit/Patch/ManagerTest.php b/src/Test/Unit/Patch/ManagerTest.php new file mode 100644 index 0000000000..55c3b732cc --- /dev/null +++ b/src/Test/Unit/Patch/ManagerTest.php @@ -0,0 +1,157 @@ +applierMock = $this->createMock(Applier::class); + $this->loggerMock = $this->getMockForAbstractClass(LoggerInterface::class); + $this->composerPackageMock = $this->getMockForAbstractClass(RootPackageInterface::class); + $this->fileMock = $this->createMock(File::class); + $this->directoryListMock = $this->createMock(DirectoryList::class); + $this->fileListMock = $this->createMock(FileList::class); + + $this->manager = new Manager( + $this->applierMock, + $this->loggerMock, + $this->fileMock, + $this->fileListMock, + $this->directoryListMock + ); + } + + public function testExecuteCopyStaticFiles() + { + $this->fileMock->expects($this->once()) + ->method('isExists') + ->with('/pub/static.php') + ->willReturn(true); + $this->fileMock->expects($this->once()) + ->method('copy') + ->with('/pub/static.php', '/pub/front-static.php') + ->willReturn(true); + + $this->loggerMock->expects($this->once()) + ->method('info') + ->withConsecutive( + ['File static.php was copied.'] + ); + + $this->manager->applyAll(); + } + + public function testExecuteApplyComposerPatches() + { + $this->fileMock->expects($this->once()) + ->method('fileGetContents') + ->willReturn(json_encode( + [ + 'package1' => [ + 'patchName1' => [ + '100' => 'patchPath1', + ], + ], + 'package2' => [ + 'patchName2' => [ + '101.*' => 'patchPath2', + ], + 'patchName3' => [ + '102.*' => 'patchPath3', + ], + ], + 'package3' => [ + 'patchName4' => 'patchPath4', + ], + ] + )); + $this->applierMock->expects($this->exactly(4)) + ->method('apply') + ->withConsecutive( + ['patchPath1', 'patchName1', 'package1', '100'], + ['patchPath2', 'patchName2', 'package2', '101.*'], + ['patchPath3', 'patchName3', 'package2', '102.*'], + ['patchPath4', 'patchName4', 'package3', '*'] + ); + + $this->manager->applyAll(); + } + + public function testExecuteApplyHotFixes() + { + $this->directoryListMock->expects($this->any()) + ->method('getMagentoRoot') + ->willReturn(__DIR__ . '/_files'); + $this->fileMock->expects($this->once()) + ->method('isDirectory') + ->willReturn(true); + $this->applierMock->expects($this->exactly(2)) + ->method('apply') + ->withConsecutive( + [__DIR__ . '/_files/' . Manager::HOTFIXES_DIR . '/patch1.patch'], + [__DIR__ . '/_files/' . Manager::HOTFIXES_DIR . '/patch2.patch'] + ); + $this->loggerMock->expects($this->once()) + ->method('info') + ->withConsecutive( + ['Applying hot-fixes.'] + ); + + $this->manager->applyAll(); + } +} diff --git a/src/Test/Unit/Patch/_files/m2-hotfixes/patch1.patch b/src/Test/Unit/Patch/_files/m2-hotfixes/patch1.patch new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/Test/Unit/Patch/_files/m2-hotfixes/patch2.patch b/src/Test/Unit/Patch/_files/m2-hotfixes/patch2.patch new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/Test/Unit/Patch/_files/m2-hotfixes/readme.md b/src/Test/Unit/Patch/_files/m2-hotfixes/readme.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/Test/Unit/Process/Build/ApplyPatchesTest.php b/src/Test/Unit/Process/Build/ApplyPatchesTest.php index 1e9c78fb9d..06b39c7f86 100644 --- a/src/Test/Unit/Process/Build/ApplyPatchesTest.php +++ b/src/Test/Unit/Process/Build/ApplyPatchesTest.php @@ -5,10 +5,8 @@ */ namespace Magento\MagentoCloud\Test\Unit\Process\Build; -use Magento\MagentoCloud\Filesystem\DirectoryList; -use Magento\MagentoCloud\Filesystem\Driver\File; +use Magento\MagentoCloud\Patch\Manager; use Magento\MagentoCloud\Process\Build\ApplyPatches; -use Magento\MagentoCloud\Shell\ShellInterface; use PHPUnit\Framework\TestCase; use Psr\Log\LoggerInterface; use PHPUnit_Framework_MockObject_MockObject as Mock; @@ -29,19 +27,9 @@ class ApplyPatchesTest extends TestCase private $loggerMock; /** - * @var ShellInterface|Mock + * @var Manager|Mock */ - private $shellMock; - - /** - * @var File|Mock - */ - private $fileMock; - - /** - * @var DirectoryList|Mock - */ - private $directoryListMock; + private $managerMock; /** * @inheritdoc @@ -50,18 +38,11 @@ protected function setUp() { $this->loggerMock = $this->getMockBuilder(LoggerInterface::class) ->getMockForAbstractClass(); - $this->shellMock = $this->getMockBuilder(ShellInterface::class) - ->getMockForAbstractClass(); - $this->fileMock = $this->getMockBuilder(File::class) - ->disableOriginalConstructor() - ->getMock(); - $this->directoryListMock = $this->createMock(DirectoryList::class); + $this->managerMock = $this->createMock(Manager::class); $this->process = new ApplyPatches( - $this->shellMock, $this->loggerMock, - $this->fileMock, - $this->directoryListMock + $this->managerMock ); parent::setUp(); @@ -69,36 +50,11 @@ protected function setUp() public function testExecute() { - $this->directoryListMock->method('getMagentoRoot') - ->willReturn('magento_root'); - $this->loggerMock->expects($this->once()) - ->method('info') - ->with('Applying patches.'); - $this->shellMock->expects($this->once()) - ->method('execute') - ->with('php magento_root/vendor/bin/m2-apply-patches'); - $this->fileMock->method('isExists') - ->with('magento_root/vendor/bin/m2-apply-patches') - ->willReturn(true); - - $this->process->execute(); - } - - public function testExecuteWithoutPatches() - { - $this->directoryListMock->method('getMagentoRoot') - ->willReturn('magento_root'); $this->loggerMock->expects($this->once()) ->method('info') ->with('Applying patches.'); - $this->fileMock->method('isExists') - ->with('magento_root/vendor/bin/m2-apply-patches') - ->willReturn(false); - $this->loggerMock->expects($this->once()) - ->method('warning') - ->with('Package with patches was not found.'); - $this->shellMock->expects($this->never()) - ->method('execute'); + $this->managerMock->expects($this->once()) + ->method('applyAll'); $this->process->execute(); } diff --git a/src/Test/Unit/Process/Deploy/InstallUpdate/ConfigUpdate/RedisTest.php b/src/Test/Unit/Process/Deploy/InstallUpdate/ConfigUpdate/RedisTest.php index 74cf199b77..22fcf944f3 100644 --- a/src/Test/Unit/Process/Deploy/InstallUpdate/ConfigUpdate/RedisTest.php +++ b/src/Test/Unit/Process/Deploy/InstallUpdate/ConfigUpdate/RedisTest.php @@ -73,12 +73,7 @@ protected function setUp() ); } - /** - * @param $envSessionLocking - * @param int $expectedDisableLocking - * @dataProvider executeDataProvider - */ - public function testExecute($envSessionLocking, int $expectedDisableLocking) + public function testExecute() { $this->loggerMock->expects($this->once()) ->method('info') @@ -96,10 +91,6 @@ public function testExecute($envSessionLocking, int $expectedDisableLocking) $this->environmentMock->expects($this->any()) ->method('getAdminUrl') ->willReturn('admin'); - $this->stageConfigMock->expects($this->once()) - ->method('get') - ->with(DeployInterface::VAR_REDIS_SESSION_DISABLE_LOCKING) - ->willReturn($envSessionLocking); $this->configReaderMock->expects($this->once()) ->method('read') ->willReturn([]); @@ -133,7 +124,6 @@ public function testExecute($envSessionLocking, int $expectedDisableLocking) 'host' => '127.0.0.1', 'port' => '6379', 'database' => 0, - 'disable_locking' => $expectedDisableLocking, ], ], ]); @@ -141,27 +131,6 @@ public function testExecute($envSessionLocking, int $expectedDisableLocking) $this->process->execute(); } - /** - * @return array - */ - public function executeDataProvider(): array - { - return [ - [ - true, - 1, - ], - [ - false, - 0, - ], - [ - true, - 1, - ], - ]; - } - public function testExecuteRemovingRedis() { $this->loggerMock->expects($this->once()) @@ -239,10 +208,6 @@ public function testExecuteWithDifferentRedisOptions() $this->environmentMock->expects($this->any()) ->method('getAdminUrl') ->willReturn('admin'); - $this->stageConfigMock->expects($this->once()) - ->method('get') - ->with(DeployInterface::VAR_REDIS_SESSION_DISABLE_LOCKING) - ->willReturn(true); $this->configReaderMock->expects($this->once()) ->method('read') ->willReturn([ @@ -286,7 +251,6 @@ public function testExecuteWithDifferentRedisOptions() 'host' => '127.0.0.1', 'port' => '6379', 'database' => 0, - 'disable_locking' => 1, 'max_concurrency' => 10, 'bot_first_lifetime' => 100, 'bot_lifetime' => 10000, @@ -298,4 +262,75 @@ public function testExecuteWithDifferentRedisOptions() $this->process->execute(); } + + public function testRemoveDisableLocking() + { + $this->loggerMock->expects($this->exactly(2)) + ->method('info') + ->withConsecutive( + [ + 'Updating env.php Redis cache configuration.', + ], + [ + 'Removing disable_locking env.php.' + ] + ); + $this->environmentMock->expects($this->any()) + ->method('getRelationships') + ->willReturn([ + 'redis' => [ + 0 => [ + 'host' => '127.0.0.1', + 'port' => '6379', + ], + ], + ]); + $this->environmentMock->expects($this->any()) + ->method('getAdminUrl') + ->willReturn('admin'); + $this->configReaderMock->expects($this->once()) + ->method('read') + ->willReturn([ + 'session' => [ + 'redis' => [ + 'disable_locking' => '1', + ], + ], + ]); + + $this->configWriterMock->expects($this->once()) + ->method('write') + ->with([ + 'cache' => [ + 'frontend' => [ + 'default' => [ + 'backend' => 'Cm_Cache_Backend_Redis', + 'backend_options' => [ + 'server' => '127.0.0.1', + 'port' => '6379', + 'database' => 1, + ], + ], + 'page_cache' => [ + 'backend' => 'Cm_Cache_Backend_Redis', + 'backend_options' => [ + 'server' => '127.0.0.1', + 'port' => '6379', + 'database' => 1, + ], + ], + ], + ], + 'session' => [ + 'save' => 'redis', + 'redis' => [ + 'host' => '127.0.0.1', + 'port' => '6379', + 'database' => 0, + ], + ], + ]); + + $this->process->execute(); + } }