From 9a39b30887efb18a2b0d0437aab25fd7f5c91ece Mon Sep 17 00:00:00 2001 From: Stanislav Idolov Date: Tue, 21 Feb 2017 14:55:05 +0200 Subject: [PATCH 01/43] MAGETWO-64184: Integration tests for price indexer --- .../Model/Product/BundlePriceAbstract.php | 8 +- .../DynamicBundlePriceCalculatorTest.php | 10 ++ ...icBundleWithSpecialPriceCalculatorTest.php | 33 +++- ...namicBundleWithTierPriceCalculatorTest.php | 46 ++++- .../FixedBundlePriceCalculatorTest.php | 53 +++++- ...edBundleWithSpecialPriceCalculatorTest.php | 157 +++++++++++------ ...FixedBundleWithTierPriceCalculatorTest.php | 161 ++++++++++++------ .../Price/SimpleWithOptionsFinalPriceTest.php | 60 +++++++ .../Price/SimpleWithOptionsTierPriceTest.php | 73 ++++++++ .../Model/ResourceModel/Indexer/PriceTest.php | 79 +++++++++ .../Product/Indexer/Price/GroupedTest.php | 75 ++++++++ 11 files changed, 634 insertions(+), 121 deletions(-) create mode 100644 dev/tests/integration/testsuite/Magento/Catalog/Model/Indexer/Product/Price/SimpleWithOptionsFinalPriceTest.php create mode 100644 dev/tests/integration/testsuite/Magento/Catalog/Model/Indexer/Product/Price/SimpleWithOptionsTierPriceTest.php create mode 100644 dev/tests/integration/testsuite/Magento/Downloadable/Model/ResourceModel/Indexer/PriceTest.php create mode 100644 dev/tests/integration/testsuite/Magento/GroupedProduct/Model/ResourceModel/Product/Indexer/Price/GroupedTest.php diff --git a/dev/tests/integration/testsuite/Magento/Bundle/Model/Product/BundlePriceAbstract.php b/dev/tests/integration/testsuite/Magento/Bundle/Model/Product/BundlePriceAbstract.php index 07cb90765a5e1..241e3e4f833c1 100644 --- a/dev/tests/integration/testsuite/Magento/Bundle/Model/Product/BundlePriceAbstract.php +++ b/dev/tests/integration/testsuite/Magento/Bundle/Model/Product/BundlePriceAbstract.php @@ -23,10 +23,17 @@ abstract class BundlePriceAbstract extends \PHPUnit_Framework_TestCase /** @var \Magento\Catalog\Api\ProductRepositoryInterface */ protected $productRepository; + /** + * @var \Magento\Catalog\Model\ResourceModel\Product\CollectionFactory + */ + protected $productCollectionFactory; + protected function setUp() { $this->objectManager = \Magento\TestFramework\Helper\Bootstrap::getObjectManager(); $this->productRepository = $this->objectManager->create(\Magento\Catalog\Api\ProductRepositoryInterface::class); + $this->productCollectionFactory = + $this->objectManager->create(\Magento\Catalog\Model\ResourceModel\Product\CollectionFactory::class); } /** @@ -58,7 +65,6 @@ protected function prepareFixture($strategyModifiers, $productSku) ); } } - $this->productRepository->save($bundleProduct); } diff --git a/dev/tests/integration/testsuite/Magento/Bundle/Model/Product/DynamicBundlePriceCalculatorTest.php b/dev/tests/integration/testsuite/Magento/Bundle/Model/Product/DynamicBundlePriceCalculatorTest.php index 55f6d26f5a712..58afa281fe971 100644 --- a/dev/tests/integration/testsuite/Magento/Bundle/Model/Product/DynamicBundlePriceCalculatorTest.php +++ b/dev/tests/integration/testsuite/Magento/Bundle/Model/Product/DynamicBundlePriceCalculatorTest.php @@ -27,6 +27,12 @@ public function testPriceForDynamicBundle(array $strategyModifiers, array $expec $priceInfo = $bundleProduct->getPriceInfo(); $priceCode = \Magento\Catalog\Pricing\Price\FinalPrice::PRICE_CODE; + $priceInfoFromIndexer = $this->productCollectionFactory->create() + ->addFieldToFilter('sku', 'bundle_product') + ->addPriceData() + ->load() + ->getFirstItem(); + $this->assertEquals( $expectedResults['minimalPrice'], $priceInfo->getPrice($priceCode)->getMinimalPrice()->getValue(), @@ -38,6 +44,10 @@ public function testPriceForDynamicBundle(array $strategyModifiers, array $expec $priceInfo->getPrice($priceCode)->getMaximalPrice()->getValue(), 'Failed to check maximal price on product' ); + + $this->assertEquals($expectedResults['minimalPrice'], $priceInfoFromIndexer->getMinimalPrice()); + + $this->assertEquals($expectedResults['maximalPrice'], $priceInfoFromIndexer->getMaxPrice()); } /** diff --git a/dev/tests/integration/testsuite/Magento/Bundle/Model/Product/DynamicBundleWithSpecialPriceCalculatorTest.php b/dev/tests/integration/testsuite/Magento/Bundle/Model/Product/DynamicBundleWithSpecialPriceCalculatorTest.php index 8c5b1fbfea734..b08d5f6f83193 100644 --- a/dev/tests/integration/testsuite/Magento/Bundle/Model/Product/DynamicBundleWithSpecialPriceCalculatorTest.php +++ b/dev/tests/integration/testsuite/Magento/Bundle/Model/Product/DynamicBundleWithSpecialPriceCalculatorTest.php @@ -27,6 +27,11 @@ public function testPriceForDynamicBundle(array $strategyModifiers, array $expec $priceInfo = $bundleProduct->getPriceInfo(); $priceCode = \Magento\Catalog\Pricing\Price\FinalPrice::PRICE_CODE; + $priceInfoFromIndexer = $this->productCollectionFactory->create() + ->addFieldToFilter('sku', 'bundle_product') + ->addPriceData() + ->load() + ->getFirstItem(); $this->assertEquals( $expectedResults['minimalPrice'], $priceInfo->getPrice($priceCode)->getMinimalPrice()->getValue(), @@ -56,6 +61,11 @@ public function testPriceForDynamicBundle(array $strategyModifiers, array $expec 'Failed to check maximal regular price on product' ); } + + $this->assertEquals($expectedResults['indexerMinimalPrice'], $priceInfoFromIndexer->getMinimalPrice()); + + $this->assertEquals($expectedResults['indexerMaximumPrice'], $priceInfoFromIndexer->getMaxPrice()); + } /** @@ -72,7 +82,9 @@ public function getTestCases() // 0.5 * 10 'minimalPrice' => 5, // 0.5 * 10 - 'maximalPrice' => 5 + 'maximalPrice' => 5, + 'indexerMinimalPrice' => 10, + 'indexerMaximumPrice' => 10 ] ], @@ -82,8 +94,11 @@ public function getTestCases() // 0.5 * 2 * 10 'minimalPrice' => 10, // 0.5 * 2 * 10 - 'maximalPrice' => 10 + 'maximalPrice' => 10, + 'indexerMinimalPrice' => 20, + 'indexerMaximumPrice' => 20 ] + ], ' @@ -95,7 +110,9 @@ public function getTestCases() // 0.5 * 1 * 10 'minimalPrice' => 5, // 0.5 * (1 * 10 + 3 * 30) - 'maximalPrice' => 50 + 'maximalPrice' => 50, + 'indexerMinimalPrice' => 10, + 'indexerMaximumPrice' => 100 ] ], @@ -108,7 +125,9 @@ public function getTestCases() // 0.5 * (min (1 * 9.9, 2.5 * 4)) 'minimalPrice' => 4.95, // 0.5 * ( 1 * 9.9 + 2.5 * 4) - 'maximalPrice' => 9.95 + 'maximalPrice' => 9.95, + 'indexerMinimalPrice' => 9.9, + 'indexerMaximumPrice' => 19.9 ] ], @@ -123,6 +142,8 @@ public function getTestCases() 'regularMinimalPrice' => '10', // 3 * 20 + (30 * 1 + 13 * 3) 'regularMaximalPrice' => '129', + 'indexerMinimalPrice' => 7.5, + 'indexerMaximumPrice' => 79 ] ], @@ -132,7 +153,9 @@ public function getTestCases() // 0.5 * min(4 * 2.5, 1 * 9.9) 'minimalPrice' => 4.95, // 0.5 * max(4 * 2.5, 1 * 9.9) - 'maximalPrice' => 5 + 'maximalPrice' => 5, + 'indexerMinimalPrice' => 9.9, + 'indexerMaximumPrice' => 10 ] ], ]; diff --git a/dev/tests/integration/testsuite/Magento/Bundle/Model/Product/DynamicBundleWithTierPriceCalculatorTest.php b/dev/tests/integration/testsuite/Magento/Bundle/Model/Product/DynamicBundleWithTierPriceCalculatorTest.php index 620fd72278435..63903932004d2 100644 --- a/dev/tests/integration/testsuite/Magento/Bundle/Model/Product/DynamicBundleWithTierPriceCalculatorTest.php +++ b/dev/tests/integration/testsuite/Magento/Bundle/Model/Product/DynamicBundleWithTierPriceCalculatorTest.php @@ -38,6 +38,11 @@ public function testPriceForDynamicBundle(array $strategyModifiers, array $expec $priceInfo = $bundleProduct->getPriceInfo(); $priceCode = \Magento\Catalog\Pricing\Price\FinalPrice::PRICE_CODE; + $priceInfoFromIndexer = $this->productCollectionFactory->create() + ->addFieldToFilter('sku', 'bundle_product') + ->addPriceData() + ->load() + ->getFirstItem(); $this->assertEquals( $expectedResults['minimalPrice'], $priceInfo->getPrice($priceCode)->getMinimalPrice()->getValue(), @@ -49,6 +54,10 @@ public function testPriceForDynamicBundle(array $strategyModifiers, array $expec $priceInfo->getPrice($priceCode)->getMaximalPrice()->getValue(), 'Failed to check maximal price on product' ); + + $this->assertEquals($expectedResults['indexerMinimalPrice'], $priceInfoFromIndexer->getMinimalPrice()); + + $this->assertEquals($expectedResults['indexerMaximumPrice'], $priceInfoFromIndexer->getMaxPrice()); } /** @@ -68,7 +77,9 @@ public function getTestCases() // 0.5 * 10 'minimalPrice' => 5, // 0.5 * 10 - 'maximalPrice' => 5 + 'maximalPrice' => 5, + 'indexerMinimalPrice' => 0, + 'indexerMaximumPrice' => 10 ] ], @@ -81,7 +92,9 @@ public function getTestCases() // 0.5 * 2 * 10 'minimalPrice' => 10, // 0.5 * 2 * 10 - 'maximalPrice' => 10 + 'maximalPrice' => 10, + 'indexerMinimalPrice' => 0, + 'indexerMaximumPrice' => 20 ] ], @@ -94,7 +107,9 @@ public function getTestCases() // 0.5 * 1 * 10 'minimalPrice' => 5, // 0.5 * (1 * 10 + 3 * 20) - 'maximalPrice' => 35 + 'maximalPrice' => 35, + 'indexerMinimalPrice' => 0, + 'indexerMaximumPrice' => 70 ] ], @@ -107,7 +122,9 @@ public function getTestCases() // 0.5 * 1 * 10 'minimalPrice' => 5, // 0.5 * (1 * 10 + 3 * 20) - 'maximalPrice' => 35 + 'maximalPrice' => 35, + 'indexerMinimalPrice' => 0, + 'indexerMaximumPrice' => 70 ] ], @@ -120,7 +137,10 @@ public function getTestCases() // 0.5 * 1 * 10 'minimalPrice' => 5, // 0.5 * 3 * 20 - 'maximalPrice' => 30 + 'maximalPrice' => 30, + 'indexerMinimalPrice' => 0, + 'indexerMaximumPrice' => 60 + ] ], @@ -133,7 +153,9 @@ public function getTestCases() // 0.5 * (1 * 10 + 1 * 10) 'minimalPrice' => 10, // 0.5 * (3 * 20 + 1 * 10 + 3 * 20) - 'maximalPrice' => 65 + 'maximalPrice' => 65, + 'indexerMinimalPrice' => 0, + 'indexerMaximumPrice' => 130 ] ], @@ -146,7 +168,9 @@ public function getTestCases() // 0.5 * (1 * 10) 'minimalPrice' => 5, // 0.5 * (3 * 20 + 1 * 10 + 3 * 20) - 'maximalPrice' => 65 + 'maximalPrice' => 65, + 'indexerMinimalPrice' => 0, + 'indexerMaximumPrice' => 130 ] ], @@ -159,7 +183,9 @@ public function getTestCases() // 0.5 * (1 * 10) 'minimalPrice' => 5, // 0.5 * (3 * 20 + 1 * 10 + 3 * 20) - 'maximalPrice' => 65 + 'maximalPrice' => 65, + 'indexerMinimalPrice' => 0, + 'indexerMaximumPrice' => 130 ] ], @@ -172,7 +198,9 @@ public function getTestCases() // 0.5 * 1 * 2.5 'minimalPrice' => 1.25, // 0.5 * 3 * 20 - 'maximalPrice' => 30 + 'maximalPrice' => 30, + 'indexerMinimalPrice' => 0, + 'indexerMaximumPrice' => 60 ] ], ]; diff --git a/dev/tests/integration/testsuite/Magento/Bundle/Model/Product/FixedBundlePriceCalculatorTest.php b/dev/tests/integration/testsuite/Magento/Bundle/Model/Product/FixedBundlePriceCalculatorTest.php index ed394fc523fc1..342a7a4ccb0e7 100644 --- a/dev/tests/integration/testsuite/Magento/Bundle/Model/Product/FixedBundlePriceCalculatorTest.php +++ b/dev/tests/integration/testsuite/Magento/Bundle/Model/Product/FixedBundlePriceCalculatorTest.php @@ -29,6 +29,11 @@ public function testPriceForFixedBundle(array $strategyModifiers, array $expecte $priceInfo = $bundleProduct->getPriceInfo(); $priceCode = \Magento\Catalog\Pricing\Price\FinalPrice::PRICE_CODE; + $priceInfoFromIndexer = $this->productCollectionFactory->create() + ->addIdFilter([42]) + ->addPriceData() + ->load() + ->getFirstItem(); $this->assertEquals( $expectedResults['minimalPrice'], $priceInfo->getPrice($priceCode)->getMinimalPrice()->getValue(), @@ -40,6 +45,11 @@ public function testPriceForFixedBundle(array $strategyModifiers, array $expecte $priceInfo->getPrice($priceCode)->getMaximalPrice()->getValue(), 'Failed to check maximal price on product' ); + $this->assertEquals($expectedResults['indexerMinimalPrice'], $priceInfoFromIndexer->getMinimalPrice()); + //This verification is skipped due to MAGETWO-64406, so in some cases 'indexerMaximumPrice' key was commented. + if (isset($expectedResults['indexerMaximumPrice'])) { + $this->assertEquals($expectedResults['indexerMaximumPrice'], $priceInfoFromIndexer->getMaxPrice()); + } } /** @@ -58,6 +68,11 @@ public function testPriceForFixedBundleInWebsiteScope(array $strategyModifiers, $priceInfo = $bundleProduct->getPriceInfo(); $priceCode = \Magento\Catalog\Pricing\Price\FinalPrice::PRICE_CODE; + $priceInfoFromIndexer = $this->productCollectionFactory->create() + ->addFieldToFilter('sku', 'bundle_product') + ->addPriceData() + ->load() + ->getFirstItem(); $this->assertEquals( $expectedResults['minimalPrice'], $priceInfo->getPrice($priceCode)->getMinimalPrice()->getValue(), @@ -69,6 +84,12 @@ public function testPriceForFixedBundleInWebsiteScope(array $strategyModifiers, $priceInfo->getPrice($priceCode)->getMaximalPrice()->getValue(), 'Failed to check maximal price on product' ); + $this->assertEquals($expectedResults['indexerMinimalPrice'], $priceInfoFromIndexer->getMinimalPrice()); + + //This verification is skipped due to MAGETWO-64406, so in some cases 'indexerMaximumPrice' key was commented. + if (isset($expectedResults['indexerMaximumPrice'])) { + $this->assertEquals($expectedResults['indexerMaximumPrice'], $priceInfoFromIndexer->getMaxPrice()); + } } /** @@ -85,7 +106,9 @@ public function getTestCases() // 110 + 10 (price from simple1) 'minimalPrice' => 120, // 110 + 10 (sum of simple price) - 'maximalPrice' => 120 + 'maximalPrice' => 120, + 'indexerMinimalPrice' => 120, + 'indexerMaximumPrice' => 120 ] ], @@ -95,7 +118,9 @@ public function getTestCases() // 110 + 10 (min price from simples) 'minimalPrice' => 120, // 110 + (3 * 10) + (2 * 10) + 10 - 'maximalPrice' => 170 + 'maximalPrice' => 170, + 'indexerMinimalPrice' => 120, + 'indexerMaximumPrice' => 170 ] ], @@ -105,7 +130,9 @@ public function getTestCases() // 110 + 10 'minimalPrice' => 120, // 110 + 60 - 'maximalPrice' => 170 + 'maximalPrice' => 170, + 'indexerMinimalPrice' => 120, + 'indexerMaximumPrice' => 170 ] ], @@ -115,7 +142,9 @@ public function getTestCases() // 110 + 10 'minimalPrice' => 120, // 110 + 30 - 'maximalPrice' => 140 + 'maximalPrice' => 140, + 'indexerMinimalPrice' => 120, + 'indexerMaximumPrice' => 140 ] ], @@ -132,7 +161,9 @@ public function getTestCases() 'minimalPrice' => 230, // 110 + 1 * 20 + 100 - 'maximalPrice' => 230 + 'maximalPrice' => 230, + 'indexerMinimalPrice' => 130, + //'indexerMaximumPrice' => 230 ] ], @@ -149,7 +180,9 @@ public function getTestCases() 'minimalPrice' => 242, // 110 + 110 * 0.2 + 110 * 1 - 'maximalPrice' => 242 + 'maximalPrice' => 242, + 'indexerMinimalPrice' => 132, + //'indexerMaximumPrice' => 242 ] ], @@ -166,7 +199,9 @@ public function getTestCases() 'minimalPrice' => 240, // 110 + 1 * 20 + 110 * 1 - 'maximalPrice' => 240 + 'maximalPrice' => 240, + 'indexerMinimalPrice' => 130, + //'indexerMaximumPrice' => 240 ] ], @@ -183,7 +218,9 @@ public function getTestCases() 'minimalPrice' => 232, // 110 + 110 * 0.2 + 100 - 'maximalPrice' => 232 + 'maximalPrice' => 232, + 'indexerMinimalPrice' => 132, + //'indexerMaximumPrice' => 232 ] ], ]; diff --git a/dev/tests/integration/testsuite/Magento/Bundle/Model/Product/FixedBundleWithSpecialPriceCalculatorTest.php b/dev/tests/integration/testsuite/Magento/Bundle/Model/Product/FixedBundleWithSpecialPriceCalculatorTest.php index 9fc01c1354347..4e422fa4e6d1b 100644 --- a/dev/tests/integration/testsuite/Magento/Bundle/Model/Product/FixedBundleWithSpecialPriceCalculatorTest.php +++ b/dev/tests/integration/testsuite/Magento/Bundle/Model/Product/FixedBundleWithSpecialPriceCalculatorTest.php @@ -29,6 +29,11 @@ public function testPriceForFixedBundle(array $strategyModifiers, array $expecte $priceInfo = $bundleProduct->getPriceInfo(); $priceCode = \Magento\Catalog\Pricing\Price\FinalPrice::PRICE_CODE; + $priceInfoFromIndexer = $this->productCollectionFactory->create() + ->addFieldToFilter('sku', 'bundle_product') + ->addPriceData() + ->load() + ->getFirstItem(); $this->assertEquals( $expectedResults['minimalPrice'], $priceInfo->getPrice($priceCode)->getMinimalPrice()->getValue(), @@ -40,6 +45,12 @@ public function testPriceForFixedBundle(array $strategyModifiers, array $expecte $priceInfo->getPrice($priceCode)->getMaximalPrice()->getValue(), 'Failed to check maximal price on product' ); + $this->assertEquals($expectedResults['indexerMinimalPrice'], $priceInfoFromIndexer->getMinimalPrice()); + + //This verification is skipped due to MAGETWO-64406, so in some cases 'indexerMaximumPrice' key was commented. + if (isset($expectedResults['indexerMaximumPrice'])) { + $this->assertEquals($expectedResults['indexerMaximumPrice'], $priceInfoFromIndexer->getMaxPrice()); + } } /** @@ -51,7 +62,7 @@ public function getTestCases() { return [ ' - #1 Testing price for fixed bundle product + #1 Testing price for fixed bundle product with special price and without any sub items and options ' => [ 'strategy' => $this->getBundleConfiguration1(), @@ -60,12 +71,14 @@ public function getTestCases() 'minimalPrice' => 55, // 110 * 0.5 - 'maximalPrice' => 55 + 'maximalPrice' => 55, + 'indexerMinimalPrice' => null, + 'indexerMaximumPrice' => null ] ], ' - #2 Testing price for fixed bundle product + #2 Testing price for fixed bundle product with special price, fixed sub items and fixed options ' => [ 'strategy' => $this->getBundleConfiguration2( @@ -77,12 +90,14 @@ public function getTestCases() 'minimalPrice' => 165, // 0.5 * (110 + 1 * 20) + 100 - 'maximalPrice' => 165 + 'maximalPrice' => 165, + 'indexerMinimalPrice' => 130, + //'indexerMaximumPrice' => 230 ] ], ' - #3 Testing price for fixed bundle product + #3 Testing price for fixed bundle product with special price, percent sub items and percent options ' => [ 'strategy' => $this->getBundleConfiguration2( @@ -94,12 +109,14 @@ public function getTestCases() 'minimalPrice' => 121, // 0.5 * (110 + 110 * 0.2 + 110 * 1) - 'maximalPrice' => 121 + 'maximalPrice' => 121, + 'indexerMinimalPrice' => 132, + //'indexerMaximumPrice' => 242 ] ], ' - #4 Testing price for fixed bundle product + #4 Testing price for fixed bundle product with special price, fixed sub items and percent options ' => [ 'strategy' => $this->getBundleConfiguration2( @@ -111,12 +128,14 @@ public function getTestCases() 'minimalPrice' => 120, // 0.5 * (110 + 1 * 20 + 110 * 1) - 'maximalPrice' => 120 + 'maximalPrice' => 120, + 'indexerMinimalPrice' => 130, + //'indexerMaximumPrice' => 240 ] ], ' - #5 Testing price for fixed bundle product + #5 Testing price for fixed bundle product with special price, percent sub items and fixed options ' => [ 'strategy' => $this->getBundleConfiguration2( @@ -128,12 +147,14 @@ public function getTestCases() 'minimalPrice' => 166, // 0.5 * (110 + 110 * 0.2) + 100 - 'maximalPrice' => 166 + 'maximalPrice' => 166, + 'indexerMinimalPrice' => 132, + //'indexerMaximumPrice' => 232 ] ], ' - #6 Testing price for fixed bundle product + #6 Testing price for fixed bundle product with special price, fixed sub items and fixed options ' => [ 'strategy' => $this->getBundleConfiguration3( @@ -145,12 +166,14 @@ public function getTestCases() 'minimalPrice' => 155, // 0.5 * (110 + 2 * 20) + 100 - 'maximalPrice' => 175 + 'maximalPrice' => 175, + 'indexerMinimalPrice' => 150, + //'indexerMaximumPrice' => 250 ] ], ' - #7 Testing price for fixed bundle product + #7 Testing price for fixed bundle product with special price, percent sub items and percent options ' => [ 'strategy' => $this->getBundleConfiguration3( @@ -162,12 +185,14 @@ public function getTestCases() 'minimalPrice' => 110, // 0.5 * (110 + 2 * 110 * 0.2 + 1 * 110) - 'maximalPrice' => 132 + 'maximalPrice' => 132, + 'indexerMinimalPrice' => 154, + //'indexerMaximumPrice' => 264 ] ], ' - #8 Testing price for fixed bundle product + #8 Testing price for fixed bundle product with special price, fixed sub items and percent options ' => [ 'strategy' => $this->getBundleConfiguration3( @@ -179,12 +204,14 @@ public function getTestCases() 'minimalPrice' => 110, // 0.5 * (110 + 2 * 20 + 1 * 110) - 'maximalPrice' => 130 + 'maximalPrice' => 130, + 'indexerMinimalPrice' => 150, + //'indexerMaximumPrice' => 260 ] ], ' - #9 Testing price for fixed bundle product + #9 Testing price for fixed bundle product with special price, percent sub items and fixed options ' => [ 'strategy' => $this->getBundleConfiguration3( @@ -196,12 +223,14 @@ public function getTestCases() 'minimalPrice' => 155, // 0.5 * (110 + 2 * 0.2 * 110) + 100 - 'maximalPrice' => 177 + 'maximalPrice' => 177, + 'indexerMinimalPrice' => 154, + //'indexerMaximumPrice' => 254 ] ], ' - #10 Testing price for fixed bundle product + #10 Testing price for fixed bundle product with special price, fixed sub items and fixed options ' => [ 'strategy' => $this->getBundleConfiguration4( @@ -213,12 +242,14 @@ public function getTestCases() 'minimalPrice' => 170, // 0.5 * (110 + 3 * 10 + 1 * 40) + 100 - 'maximalPrice' => 190 + 'maximalPrice' => 190, + 'indexerMinimalPrice' => 140, + //'indexerMaximumPrice' => 280 ] ], ' - #11 Testing price for fixed bundle product + #11 Testing price for fixed bundle product with special price, percent sub items and percent options ' => [ 'strategy' => $this->getBundleConfiguration4( @@ -230,12 +261,14 @@ public function getTestCases() 'minimalPrice' => 126.5, // 0.5 * (110 + 3 * 110 * 0.1 + 1 * 110 * 0.4 + 110 * 1) - 'maximalPrice' => 148.5 + 'maximalPrice' => 148.5, + 'indexerMinimalPrice' => 143, + //'indexerMaximumPrice' => 297 ] ], ' - #12 Testing price for fixed bundle product + #12 Testing price for fixed bundle product with special price, fixed sub items and percent options ' => [ 'strategy' => $this->getBundleConfiguration4( @@ -247,12 +280,14 @@ public function getTestCases() 'minimalPrice' => 125, // 0.5 * (110 + 3 * 10 + 1 * 40 + 1 * 110) - 'maximalPrice' => 145 + 'maximalPrice' => 145, + 'indexerMinimalPrice' => 140, + //'indexerMaximumPrice' => 290 ] ], ' - #13 Testing price for fixed bundle product + #13 Testing price for fixed bundle product with special price, percent sub items and fixed options ' => [ 'strategy' => $this->getBundleConfiguration4( @@ -264,12 +299,14 @@ public function getTestCases() 'minimalPrice' => 171.5, // 0.5 * (110 + 3 * 110 * 0.1 + 1 * 110 * 0.4) + 100 - 'maximalPrice' => 193.5 + 'maximalPrice' => 193.5, + 'indexerMinimalPrice' => 143, + //'indexerMaximumPrice' => 287 ] ], ' - #14 Testing price for fixed bundle product + #14 Testing price for fixed bundle product with special price, fixed sub items and fixed options ' => [ 'strategy' => $this->getBundleConfiguration5( @@ -281,12 +318,14 @@ public function getTestCases() 'minimalPrice' => 175, // 0.5 * (110 + 1 * 40 + 3 * 15) + 100 - 'maximalPrice' => 197.5 + 'maximalPrice' => 197.5, + 'indexerMinimalPrice' => 150, + //'indexerMaximumPrice' => 295 ] ], ' - #15 Testing price for fixed bundle product + #15 Testing price for fixed bundle product with special price, percent sub items and percent options ' => [ 'strategy' => $this->getBundleConfiguration5( @@ -298,12 +337,14 @@ public function getTestCases() 'minimalPrice' => 132, // 0.5 * (110 + 1 * 110 * 0.4 + 3 * 110 * 0.15 + 110 * 1) - 'maximalPrice' => 156.75 + 'maximalPrice' => 156.75, + 'indexerMinimalPrice' => 154, + //'indexerMaximumPrice' => 313.5 ] ], ' - #16 Testing price for fixed bundle product + #16 Testing price for fixed bundle product with special price, fixed sub items and percent options ' => [ 'strategy' => $this->getBundleConfiguration5( @@ -315,12 +356,14 @@ public function getTestCases() 'minimalPrice' => 130, // 0.5 * (110 + 1 * 40 + 3 * 15 + 1 * 110) - 'maximalPrice' => 152.5 + 'maximalPrice' => 152.5, + 'indexerMinimalPrice' => 150, + //'indexerMaximumPrice' => 305 ] ], ' - #17 Testing price for fixed bundle product + #17 Testing price for fixed bundle product with special price, percent sub items and fixed options ' => [ 'strategy' => $this->getBundleConfiguration5( @@ -332,12 +375,14 @@ public function getTestCases() 'minimalPrice' => 177, // 0.5 * (110 + 1 * 110 * 0.4 + 3 * 110 * 0.15) + 100 - 'maximalPrice' => 201.75 + 'maximalPrice' => 201.75, + 'indexerMinimalPrice' => 154, + //'indexerMaximumPrice' => 303.5 ] ], ' - #18 Testing price for fixed bundle product + #18 Testing price for fixed bundle product with special price, fixed sub items and fixed options ' => [ 'strategy' => $this->getBundleConfiguration6( @@ -349,12 +394,14 @@ public function getTestCases() 'minimalPrice' => 175, // 0.5 * (110 + 3 * 15) + 100 - 'maximalPrice' => 177.5 + 'maximalPrice' => 177.5, + 'indexerMinimalPrice' => 150, + //'indexerMaximumPrice' => 255 ] ], ' - #19 Testing price for fixed bundle product + #19 Testing price for fixed bundle product with special price, percent sub items and percent options ' => [ 'strategy' => $this->getBundleConfiguration6( @@ -366,12 +413,14 @@ public function getTestCases() 'minimalPrice' => 132, // 0.5 * (110 + 3 * 110 * 0.15 + 1 * 110) - 'maximalPrice' => 134.75 + 'maximalPrice' => 134.75, + 'indexerMinimalPrice' => 154, + //'indexerMaximumPrice' => 269.5 ] ], ' - #20 Testing price for fixed bundle product + #20 Testing price for fixed bundle product with special price, fixed sub items and percent options ' => [ 'strategy' => $this->getBundleConfiguration6( @@ -383,12 +432,14 @@ public function getTestCases() 'minimalPrice' => 130, // 0.5 * (110 + 3 * 15 + 110 * 1) - 'maximalPrice' => 132.5 + 'maximalPrice' => 132.5, + 'indexerMinimalPrice' => 150, + //'indexerMaximumPrice' => 265 ] ], ' - #21 Testing price for fixed bundle product + #21 Testing price for fixed bundle product with special price, percent sub items and fixed options ' => [ 'strategy' => $this->getBundleConfiguration6( @@ -400,12 +451,14 @@ public function getTestCases() 'minimalPrice' => 177, // 0.5 * (110 + 3 * 110 * 0.15) + 100 - 'maximalPrice' => 179.75 + 'maximalPrice' => 179.75, + 'indexerMinimalPrice' => 154, + //'indexerMaximumPrice' => 259.5 ] ], ' - #22 Testing price for fixed bundle product + #22 Testing price for fixed bundle product with special price, fixed sub items and fixed options ' => [ 'strategy' => $this->getBundleConfiguration7( @@ -417,12 +470,14 @@ public function getTestCases() 'minimalPrice' => 185, // 0.5 * (110 + 3 * 15 + 1 * 20 + 3 * 10) + 100 - 'maximalPrice' => 202.5 + 'maximalPrice' => 202.5, + 'indexerMinimalPrice' => 170, + //'indexerMaximumPrice' => 305 ] ], ' - #23 Testing price for fixed bundle product + #23 Testing price for fixed bundle product with special price, percent sub items and percent options ' => [ 'strategy' => $this->getBundleConfiguration7( @@ -434,7 +489,9 @@ public function getTestCases() 'minimalPrice' => 143, // 0.5 * (110 + 3 * 110 * 0.15 + 1 * 110 * 0.2 + 3 * 110 * 0.1 + 110 * 1) - 'maximalPrice' => 162.25 + 'maximalPrice' => 162.25, + 'indexerMinimalPrice' => 176, + //'indexerMaximumPrice' => 324.5 ] ], @@ -451,7 +508,9 @@ public function getTestCases() 'minimalPrice' => 140, // 0.5 * (110 + 3 * 15 + 1 * 20 + 3 * 10 + 1 * 110) - 'maximalPrice' => 157.5 + 'maximalPrice' => 157.5, + 'indexerMinimalPrice' => 170, + //'indexerMaximumPrice' => 315 ] ], @@ -468,7 +527,9 @@ public function getTestCases() 'minimalPrice' => 188, // 0.5 * (110 + 3 * 110 * 0.15 + 1 * 110 * 0.2 + 3 * 110 * 0.1) + 100 - 'maximalPrice' => 207.25 + 'maximalPrice' => 207.25, + 'indexerMinimalPrice' => 176, + //'indexerMaximumPrice' => 314.5 ] ], ]; diff --git a/dev/tests/integration/testsuite/Magento/Bundle/Model/Product/FixedBundleWithTierPriceCalculatorTest.php b/dev/tests/integration/testsuite/Magento/Bundle/Model/Product/FixedBundleWithTierPriceCalculatorTest.php index 1b217a7d080d3..d5e6c8815a03c 100644 --- a/dev/tests/integration/testsuite/Magento/Bundle/Model/Product/FixedBundleWithTierPriceCalculatorTest.php +++ b/dev/tests/integration/testsuite/Magento/Bundle/Model/Product/FixedBundleWithTierPriceCalculatorTest.php @@ -40,7 +40,11 @@ public function testPriceForFixedBundle(array $strategyModifiers, array $expecte /** @var \Magento\Framework\Pricing\PriceInfo\Base $priceInfo */ $priceInfo = $bundleProduct->getPriceInfo(); $priceCode = \Magento\Catalog\Pricing\Price\FinalPrice::PRICE_CODE; - + $priceInfoFromIndexer = $this->productCollectionFactory->create() + ->addFieldToFilter('sku', 'bundle_product') + ->addPriceData() + ->load() + ->getFirstItem(); $this->assertEquals( $expectedResults['minimalPrice'], $priceInfo->getPrice($priceCode)->getMinimalPrice()->getValue(), @@ -52,6 +56,12 @@ public function testPriceForFixedBundle(array $strategyModifiers, array $expecte $priceInfo->getPrice($priceCode)->getMaximalPrice()->getValue(), 'Failed to check maximal price on product' ); + $this->assertEquals($expectedResults['indexerMinimalPrice'], $priceInfoFromIndexer->getMinimalPrice()); + + //This verification is skipped due to MAGETWO-64406, so in some cases 'indexerMaximumPrice' key was commented. + if (isset($expectedResults['indexerMaximumPrice'])) { + $this->assertEquals($expectedResults['indexerMaximumPrice'], $priceInfoFromIndexer->getMaxPrice()); + } } /** @@ -63,7 +73,7 @@ public function getTestCases() { return [ ' - #1 Testing product price + #1 Testing product price with tier price and without any sub items and options ' => [ 'strategy' => $this->getBundleConfiguration1(), @@ -72,12 +82,14 @@ public function getTestCases() 'minimalPrice' => 55, // 110 * 0.5 - 'maximalPrice' => 55 + 'maximalPrice' => 55, + 'indexerMinimalPrice' => null, + 'indexerMaximumPrice' => null ] ], ' - #2 Testing product price + #2 Testing product price with tier price, fixed sub items and fixed options ' => [ 'strategy' => $this->getProductConfiguration2( @@ -89,12 +101,14 @@ public function getTestCases() 'minimalPrice' => 165, // 0.5 * (110 + 1 * 20) + 100 - 'maximalPrice' => 165 + 'maximalPrice' => 165, + 'indexerMinimalPrice' => 65, + //'indexerMaximumPrice' => 165 ] ], ' - #3 Testing product price + #3 Testing product price with tier price, percent sub items and percent options ' => [ 'strategy' => $this->getProductConfiguration2( @@ -106,12 +120,14 @@ public function getTestCases() 'minimalPrice' => 121, // 0.5 * (110 + 110 * 0.2 + 110 * 1) - 'maximalPrice' => 121 + 'maximalPrice' => 121, + 'indexerMinimalPrice' => 66, + //'indexerMaximumPrice' => 121 ] ], ' - #4 Testing product price + #4 Testing product price with tier price, fixed sub items and percent options ' => [ 'strategy' => $this->getProductConfiguration2( @@ -123,12 +139,14 @@ public function getTestCases() 'minimalPrice' => 120, // 0.5 * (110 + 1 * 20 + 110 * 1) - 'maximalPrice' => 120 + 'maximalPrice' => 120, + 'indexerMinimalPrice' => 65, + //'indexerMaximumPrice' => 120 ] ], ' - #5 Testing product price + #5 Testing product price with tier price, percent sub items and fixed options ' => [ 'strategy' => $this->getProductConfiguration2( @@ -140,12 +158,14 @@ public function getTestCases() 'minimalPrice' => 166, // 0.5 * (110 + 110 * 0.2) + 100 - 'maximalPrice' => 166 + 'maximalPrice' => 166, + 'indexerMinimalPrice' => 66, + //'indexerMaximumPrice' => 166 ] ], ' - #6 Testing product price + #6 Testing product price with tier price, fixed sub items and fixed options ' => [ 'strategy' => $this->getProductConfiguration3( @@ -157,7 +177,9 @@ public function getTestCases() 'minimalPrice' => 155, // 0.5 * (110 + 2 * 20) + 100 - 'maximalPrice' => 175 + 'maximalPrice' => 175, + 'indexerMinimalPrice' => 75, + //'indexerMaximumPrice' => 175 ] ], @@ -174,12 +196,14 @@ public function getTestCases() 'minimalPrice' => 110, // 0.5 * (110 + 2 * 110 * 0.2 + 1 * 110) - 'maximalPrice' => 132 + 'maximalPrice' => 132, + 'indexerMinimalPrice' => 77, + //'indexerMaximumPrice' => 132 ] ], ' - #8 Testing product price + #8 Testing product price with tier price, fixed sub items and percent options ' => [ 'strategy' => $this->getProductConfiguration3( @@ -191,12 +215,15 @@ public function getTestCases() 'minimalPrice' => 110, // 0.5 * (110 + 2 * 20 + 1 * 110) - 'maximalPrice' => 130 + 'maximalPrice' => 130, + 'indexerMinimalPrice' => 75, + //'indexerMaximumPrice' => 130 + ] ], ' - #9 Testing product price + #9 Testing product price with tier price, percent sub items and fixed options ' => [ 'strategy' => $this->getProductConfiguration3( @@ -208,12 +235,14 @@ public function getTestCases() 'minimalPrice' => 155, // 0.5 * (110 + 2 * 0.2 * 110) + 100 - 'maximalPrice' => 177 + 'maximalPrice' => 177, + 'indexerMinimalPrice' => 77, + //'indexerMaximumPrice' => 177 ] ], ' - #10 Testing product price + #10 Testing product price with tier price, fixed sub items and fixed options ' => [ 'strategy' => $this->getProductConfiguration4( @@ -225,12 +254,14 @@ public function getTestCases() 'minimalPrice' => 170, // 0.5 * (110 + 3 * 10 + 1 * 40) + 100 - 'maximalPrice' => 190 + 'maximalPrice' => 190, + 'indexerMinimalPrice' => 70, + //'indexerMaximumPrice' => 190 ] ], ' - #11 Testing product price + #11 Testing product price with tier price, percent sub items and percent options ' => [ 'strategy' => $this->getProductConfiguration4( @@ -242,12 +273,14 @@ public function getTestCases() 'minimalPrice' => 126.5, // 0.5 * (110 + 3 * 110 * 0.1 + 1 * 110 * 0.4 + 110 * 1) - 'maximalPrice' => 148.5 + 'maximalPrice' => 148.5, + 'indexerMinimalPrice' => 71.5, + //'indexerMaximumPrice' => 148.5 ] ], ' - #12 Testing product price + #12 Testing product price with tier price, fixed sub items and percent options ' => [ 'strategy' => $this->getProductConfiguration4( @@ -259,12 +292,14 @@ public function getTestCases() 'minimalPrice' => 125, // 0.5 * (110 + 3 * 10 + 1 * 40 + 1 * 110) - 'maximalPrice' => 145 + 'maximalPrice' => 145, + 'indexerMinimalPrice' => 70, + //'indexerMaximumPrice' => 145 ] ], ' - #13 Testing product price + #13 Testing product price with tier price, percent sub items and fixed options ' => [ 'strategy' => $this->getProductConfiguration4( @@ -276,12 +311,14 @@ public function getTestCases() 'minimalPrice' => 171.5, // 0.5 * (110 + 3 * 110 * 0.1 + 1 * 110 * 0.4) + 100 - 'maximalPrice' => 193.5 + 'maximalPrice' => 193.5, + 'indexerMinimalPrice' => 71.5, + //'indexerMaximumPrice' => 193.5 ] ], ' - #14 Testing product price + #14 Testing product price with tier price, fixed sub items and fixed options ' => [ 'strategy' => $this->getProductConfiguration5( @@ -293,12 +330,14 @@ public function getTestCases() 'minimalPrice' => 175, // 0.5 * (110 + 1 * 40 + 3 * 15) + 100 - 'maximalPrice' => 197.5 + 'maximalPrice' => 197.5, + 'indexerMinimalPrice' => 75, + //'indexerMaximumPrice' => 197.5 ] ], ' - #15 Testing product price + #15 Testing product price with tier price, percent sub items and percent options ' => [ 'strategy' => $this->getProductConfiguration5( @@ -310,12 +349,14 @@ public function getTestCases() 'minimalPrice' => 132, // 0.5 * (110 + 1 * 110 * 0.4 + 3 * 110 * 0.15 + 110 * 1) - 'maximalPrice' => 156.75 + 'maximalPrice' => 156.75, + 'indexerMinimalPrice' => 77, + //'indexerMaximumPrice' => 156.75 ] ], ' - #16 Testing product price + #16 Testing product price with tier price, fixed sub items and percent options ' => [ 'strategy' => $this->getProductConfiguration5( @@ -327,12 +368,14 @@ public function getTestCases() 'minimalPrice' => 130, // 0.5 * (110 + 1 * 40 + 3 * 15 + 1 * 110) - 'maximalPrice' => 152.5 + 'maximalPrice' => 152.5, + 'indexerMinimalPrice' => 75, + //'indexerMaximumPrice' => 152.5 ] ], ' - #17 Testing product price + #17 Testing product price with tier price, percent sub items and fixed options ' => [ 'strategy' => $this->getProductConfiguration5( @@ -344,12 +387,14 @@ public function getTestCases() 'minimalPrice' => 177, // 0.5 * (110 + 1 * 110 * 0.4 + 3 * 110 * 0.15) + 100 - 'maximalPrice' => 201.75 + 'maximalPrice' => 201.75, + 'indexerMinimalPrice' => 77, + //'indexerMaximumPrice' => 201.75 ] ], ' - #18 Testing product price + #18 Testing product price with tier price, fixed sub items and fixed options ' => [ 'strategy' => $this->getProductConfiguration6( @@ -361,12 +406,14 @@ public function getTestCases() 'minimalPrice' => 175, // 0.5 * (110 + 3 * 15) + 100 - 'maximalPrice' => 177.5 + 'maximalPrice' => 177.5, + 'indexerMinimalPrice' => 75, + //'indexerMaximumPrice' => 177.5 ] ], ' - #19 Testing product price + #19 Testing product price with tier price, percent sub items and percent options ' => [ 'strategy' => $this->getProductConfiguration6( @@ -378,12 +425,14 @@ public function getTestCases() 'minimalPrice' => 132, // 0.5 * (110 + 3 * 110 * 0.15 + 1 * 110) - 'maximalPrice' => 134.75 + 'maximalPrice' => 134.75, + 'indexerMinimalPrice' => 77, + //'indexerMaximumPrice' => 134.75 ] ], ' - #20 Testing product price + #20 Testing product price with tier price, fixed sub items and percent options ' => [ 'strategy' => $this->getProductConfiguration6( @@ -395,12 +444,14 @@ public function getTestCases() 'minimalPrice' => 130, // 0.5 * (110 + 3 * 15 + 110 * 1) - 'maximalPrice' => 132.5 + 'maximalPrice' => 132.5, + 'indexerMinimalPrice' => 75, + //'indexerMaximumPrice' => 132.5 ] ], ' - #21 Testing product price + #21 Testing product price with tier price, percent sub items and fixed options ' => [ 'strategy' => $this->getProductConfiguration6( @@ -412,12 +463,14 @@ public function getTestCases() 'minimalPrice' => 177, // 0.5 * (110 + 3 * 110 * 0.15) + 100 - 'maximalPrice' => 179.75 + 'maximalPrice' => 179.75, + 'indexerMinimalPrice' => 77, + //'indexerMaximumPrice' => 179.75 ] ], ' - #22 Testing product price + #22 Testing product price with tier price, fixed sub items and fixed options ' => [ 'strategy' => $this->getProductConfiguration7( @@ -429,12 +482,14 @@ public function getTestCases() 'minimalPrice' => 185, // 0.5 * (110 + 3 * 15 + 1 * 20 + 3 * 10) + 100 - 'maximalPrice' => 202.5 + 'maximalPrice' => 202.5, + 'indexerMinimalPrice' => 85, + //'indexerMaximumPrice' => 202.5 ] ], ' - #23 Testing product price + #23 Testing product price with tier price, percent sub items and percent options ' => [ 'strategy' => $this->getProductConfiguration7( @@ -446,12 +501,14 @@ public function getTestCases() 'minimalPrice' => 143, // 0.5 * (110 + 3 * 110 * 0.15 + 1 * 110 * 0.2 + 3 * 110 * 0.1 + 110 * 1) - 'maximalPrice' => 162.25 + 'maximalPrice' => 162.25, + 'indexerMinimalPrice' => 88, + //'indexerMaximumPrice' => 162.25 ] ], ' - #24 Testing product price + #24 Testing product price with tier price, fixed sub items and percent options ' => [ 'strategy' => $this->getProductConfiguration7( @@ -463,12 +520,14 @@ public function getTestCases() 'minimalPrice' => 140, // 0.5 * (110 + 3 * 15 + 1 * 20 + 3 * 10 + 1 * 110) - 'maximalPrice' => 157.5 + 'maximalPrice' => 157.5, + 'indexerMinimalPrice' => 85, + //'indexerMaximumPrice' => 157.5 ] ], ' - #25 Testing product price + #25 Testing product price with tier price, percent sub items and fixed options ' => [ 'strategy' => $this->getProductConfiguration7( @@ -480,7 +539,9 @@ public function getTestCases() 'minimalPrice' => 188, // 0.5 * (110 + 3 * 110 * 0.15 + 1 * 110 * 0.2 + 3 * 110 * 0.1) + 100 - 'maximalPrice' => 207.25 + 'maximalPrice' => 207.25, + 'indexerMinimalPrice' => 88, + //'indexerMaximumPrice' => 207.25 ] ], ]; diff --git a/dev/tests/integration/testsuite/Magento/Catalog/Model/Indexer/Product/Price/SimpleWithOptionsFinalPriceTest.php b/dev/tests/integration/testsuite/Magento/Catalog/Model/Indexer/Product/Price/SimpleWithOptionsFinalPriceTest.php new file mode 100644 index 0000000000000..70dffe25551a8 --- /dev/null +++ b/dev/tests/integration/testsuite/Magento/Catalog/Model/Indexer/Product/Price/SimpleWithOptionsFinalPriceTest.php @@ -0,0 +1,60 @@ +objectManager = \Magento\TestFramework\Helper\Bootstrap::getObjectManager(); + $this->productRepository = $this->objectManager->create(\Magento\Catalog\Api\ProductRepositoryInterface::class); + $this->productCollectionFactory = $this->objectManager->create( + \Magento\Catalog\Model\ResourceModel\Product\CollectionFactory::class + ); + } + + /** + * @magentoDbIsolation enabled + * @magentoAppIsolation enabled + * @magentoDataFixture Magento/Catalog/_files/product_with_options.php + */ + public function testFinalPrice() + { + $this->markTestSkipped('MAGETWO-64406'); + + $product = $this->productRepository->get('simple', false, null, true); + + /** @var \Magento\Catalog\Model\Indexer\Product\Price\Processor $priceIndexer */ + $priceIndexer = $this->objectManager->create(\Magento\Catalog\Model\Indexer\Product\Price\Processor::class); + $priceIndexer->reindexRow($product->getId()); + + /** @var \Magento\Catalog\Model\ResourceModel\Product\Collection $productCollection */ + $productCollection = $this->productCollectionFactory->create(); + $productCollection->addIdFilter([$product->getId()]); + $productCollection->addPriceData(); + $productCollection->load(); + $indexPriceInfo = $productCollection->getFirstItem(); + + $this->assertEquals(395, $indexPriceInfo->getMaxPrice()); + $this->assertEquals(50, $indexPriceInfo->getMinimalPrice()); + } +} diff --git a/dev/tests/integration/testsuite/Magento/Catalog/Model/Indexer/Product/Price/SimpleWithOptionsTierPriceTest.php b/dev/tests/integration/testsuite/Magento/Catalog/Model/Indexer/Product/Price/SimpleWithOptionsTierPriceTest.php new file mode 100644 index 0000000000000..431114d19212a --- /dev/null +++ b/dev/tests/integration/testsuite/Magento/Catalog/Model/Indexer/Product/Price/SimpleWithOptionsTierPriceTest.php @@ -0,0 +1,73 @@ +objectManager = \Magento\TestFramework\Helper\Bootstrap::getObjectManager(); + $this->productRepository = $this->objectManager->create(\Magento\Catalog\Api\ProductRepositoryInterface::class); + $this->productCollectionFactory = $this->objectManager->create( + \Magento\Catalog\Model\ResourceModel\Product\CollectionFactory::class + ); + } + + /** + * @magentoDbIsolation enabled + * @magentoDataFixture Magento/Catalog/_files/product_with_options.php + */ + public function testTierPrice() + { + $tierPriceValue = 20.00; + + /** @var \Magento\Catalog\Model\Product $product */ + $product = $this->productRepository->get('simple'); + + /** @var \Magento\Catalog\Api\ScopedProductTierPriceManagementInterface $tierPriceManagement */ + $tierPriceManagement = $this->objectManager->create( + \Magento\Catalog\Api\ScopedProductTierPriceManagementInterface::class + ); + + /** @var \Magento\Catalog\Api\Data\ProductTierPriceInterface $tierPrice */ + $tierPrice = $this->objectManager->create(\Magento\Catalog\Api\Data\ProductTierPriceInterfaceFactory::class) + ->create(); + + $tierPrice->setCustomerGroupId(\Magento\Customer\Model\Group::CUST_GROUP_ALL); + $tierPrice->setQty(1.00); + $tierPrice->setValue($tierPriceValue); + + $tierPriceManagement->add($product->getSku(), $tierPrice); + + /** @var \Magento\Catalog\Model\ResourceModel\Product\Collection $productCollection */ + $productCollection = $this->productCollectionFactory->create(); + $productCollection->addIdFilter([$product->getId()]); + $productCollection->addPriceData(); + $productCollection->load(); + $indexPriceInfo = $productCollection->getFirstItem(); + + $tierPriceModel = $indexPriceInfo->getPriceInfo() + ->getPrice(\Magento\Catalog\Pricing\Price\TierPrice::PRICE_CODE); + + $this->assertEquals($tierPriceValue, $tierPriceModel->getValue()); + } +} diff --git a/dev/tests/integration/testsuite/Magento/Downloadable/Model/ResourceModel/Indexer/PriceTest.php b/dev/tests/integration/testsuite/Magento/Downloadable/Model/ResourceModel/Indexer/PriceTest.php new file mode 100644 index 0000000000000..8e7ab13954a52 --- /dev/null +++ b/dev/tests/integration/testsuite/Magento/Downloadable/Model/ResourceModel/Indexer/PriceTest.php @@ -0,0 +1,79 @@ +indexer = Bootstrap::getObjectManager()->get( + \Magento\Catalog\Model\Indexer\Product\Price\Processor::class + ); + $this->productCollectionFactory = Bootstrap::getObjectManager()->get(CollectionFactory::class); + $this->productRepository = Bootstrap::getObjectManager()->get(\Magento\Catalog\Model\ProductRepository::class); + } + + /** + * Steps: + * 1. Add custom tier prices to the product from fixture. + * 2. Run reindexing. + * 3. Load the product again and check all the prices. + * + * @magentoDbIsolation enabled + * @magentoAppIsolation enabled + * @magentoDataFixture Magento/Downloadable/_files/product_downloadable_with_files.php + */ + public function testReindexEntity() + { + $specialPrice = 7.90; + $product = $this->productRepository->get('downloadable-product'); + $tierData = [ + ['website_id' => 0, 'cust_group' => 1, 'price_qty' => 11, 'price' => 8.20], + ['website_id' => 0, 'cust_group' => 1, 'price_qty' => 21, 'price' => 7.55], + ]; + $product->setData('tier_price', $tierData); + $product->setData('special_price', $specialPrice); + $this->productRepository->save($product); + + $this->indexer->reindexAll(); + + /** @var \Magento\Catalog\Model\ResourceModel\Product\Collection $collection */ + $collection = $this->productCollectionFactory->create(); + $collection->addPriceData()->addFieldToFilter('sku', 'downloadable-product'); + /** @var \Magento\Catalog\Model\Product $product */ + $product = $collection->getFirstItem(); + + $this->assertEquals(10, $product->getPrice(), 'Wrong downloadable product price'); + $this->assertEquals($specialPrice, $product->getMinimalPrice()); + + $resultTiers = $product->getTierPrices(); + $this->assertTrue(is_array($resultTiers), 'Tiers not found'); + $this->assertEquals(count($tierData), count($resultTiers), 'Incorrect number of result tiers'); + + for ($i = 0; $i < count($tierData); $i++) { + $this->assertEquals($tierData[$i]['price_qty'], $resultTiers[$i]->getQty(), 'Wrong tier price quantity'); + $this->assertEquals($tierData[$i]['price'], $resultTiers[$i]->getValue(), 'Wrong tier price value'); + } + } +} diff --git a/dev/tests/integration/testsuite/Magento/GroupedProduct/Model/ResourceModel/Product/Indexer/Price/GroupedTest.php b/dev/tests/integration/testsuite/Magento/GroupedProduct/Model/ResourceModel/Product/Indexer/Price/GroupedTest.php new file mode 100644 index 0000000000000..4a26ad5126467 --- /dev/null +++ b/dev/tests/integration/testsuite/Magento/GroupedProduct/Model/ResourceModel/Product/Indexer/Price/GroupedTest.php @@ -0,0 +1,75 @@ +productRepository = Bootstrap::getObjectManager()->get(ProductRepositoryInterface::class); + $this->productCollectionFactory = Bootstrap::getObjectManager()->get(CollectionFactory::class); + $this->indexerProcessor = Bootstrap::getObjectManager()->get(Processor::class); + $this->tierPriceFactory = Bootstrap::getObjectManager()->get(ProductTierPriceInterfaceFactory::class); + } + + /** + * @magentoDataFixture Magento/GroupedProduct/_files/product_grouped.php + * @magentoAppIsolation enabled + */ + public function testReindex() + { + $simpleProductPrice = 15; + $virtualProductPrice = 5; + /** @var \Magento\Catalog\Model\Product $simpleProduct */ + $simpleProduct = $this->productRepository->get('simple', true); + /** @var \Magento\Catalog\Model\Product $virtualProduct */ + $virtualProduct = $this->productRepository->get('virtual-product', true); + $simpleProduct->setData('price', $simpleProductPrice); + $virtualProduct->setData('price', $virtualProductPrice); + $this->productRepository->save($simpleProduct); + $this->productRepository->save($virtualProduct); + $this->indexerProcessor->reindexAll(); + /** @var ProductCollection $collection */ + $collection = $this->productCollectionFactory->create(); + $collection->addPriceData()->addFieldToFilter(ProductInterface::SKU, 'grouped-product'); + /** @var \Magento\Catalog\Model\Product $item */ + $item = $collection->getFirstItem(); + $this->assertEquals($virtualProductPrice, $item->getData('min_price')); + $this->assertEquals($simpleProductPrice, $item->getData('max_price')); + } +} From 456a66e3d1b31ebf119e7bab6355a9ea9400e3ab Mon Sep 17 00:00:00 2001 From: Stanislav Idolov Date: Tue, 21 Feb 2017 14:55:42 +0200 Subject: [PATCH 02/43] MAGETWO-64187: Batch size algorithm --- .../Framework/Indexer/BatchProvider.php | 64 +++++++++++++++++++ .../Indexer/BatchProviderInterface.php | 39 +++++++++++ .../Framework/Indexer/BatchSizeCalculator.php | 45 +++++++++++++ .../Indexer/BatchSizeCalculatorInterface.php | 27 ++++++++ .../Indexer/Test/Unit/BatchProviderTest.php | 60 +++++++++++++++++ .../Test/Unit/BatchSizeCalculatorTest.php | 63 ++++++++++++++++++ 6 files changed, 298 insertions(+) create mode 100644 lib/internal/Magento/Framework/Indexer/BatchProvider.php create mode 100644 lib/internal/Magento/Framework/Indexer/BatchProviderInterface.php create mode 100644 lib/internal/Magento/Framework/Indexer/BatchSizeCalculator.php create mode 100644 lib/internal/Magento/Framework/Indexer/BatchSizeCalculatorInterface.php create mode 100644 lib/internal/Magento/Framework/Indexer/Test/Unit/BatchProviderTest.php create mode 100644 lib/internal/Magento/Framework/Indexer/Test/Unit/BatchSizeCalculatorTest.php diff --git a/lib/internal/Magento/Framework/Indexer/BatchProvider.php b/lib/internal/Magento/Framework/Indexer/BatchProvider.php new file mode 100644 index 0000000000000..0a2e71d4c0af6 --- /dev/null +++ b/lib/internal/Magento/Framework/Indexer/BatchProvider.php @@ -0,0 +1,64 @@ +fetchOne( + $adapter->select()->from( + ['entity' => $tableName], + [ + 'max_value' => new \Zend_Db_Expr('MAX(entity.' . $linkField . ')') + ] + ) + ); + + /** @var int $truncatedBatchSize size of the last batch that is smaller than expected batch size */ + $truncatedBatchSize = $maxLinkFieldValue % $batchSize; + /** @var int $fullBatchCount count of the batches that have expected batch size */ + $fullBatchCount = ($maxLinkFieldValue - $truncatedBatchSize) / $batchSize; + + for ($batchIndex = 0; $batchIndex < $fullBatchCount; $batchIndex ++) { + yield ['from' => $batchIndex * $batchSize + 1, 'to' => ($batchIndex + 1) * $batchSize]; + } + // return the last batch if it has smaller size + if ($truncatedBatchSize > 0) { + yield ['from' => $fullBatchCount * $batchSize + 1, 'to' => $maxLinkFieldValue]; + } + } + + /** + * @inheritdoc + */ + public function getBatchIds( + \Magento\Framework\DB\Adapter\AdapterInterface $connection, + \Magento\Framework\DB\Select $select, + array $batch + ) { + $betweenCondition = sprintf( + '(%s BETWEEN %s AND %s)', + 'entity_id', + $connection->quote($batch['from']), + $connection->quote($batch['to']) + ); + + $ids = $connection->fetchCol($select->where($betweenCondition)); + return array_map('intval', $ids); + } +} diff --git a/lib/internal/Magento/Framework/Indexer/BatchProviderInterface.php b/lib/internal/Magento/Framework/Indexer/BatchProviderInterface.php new file mode 100644 index 0000000000000..14739941e4a14 --- /dev/null +++ b/lib/internal/Magento/Framework/Indexer/BatchProviderInterface.php @@ -0,0 +1,39 @@ + ..., 'to' => ...] + */ + public function getBatches(AdapterInterface $adapter, $tableName, $linkField, $batchSize); + + /** + * Get list of entity ids based on batch + * + * @param AdapterInterface $connection + * @param Select $select + * @param array $batch + * @return array + */ + public function getBatchIds(AdapterInterface $connection, Select $select, array $batch); +} diff --git a/lib/internal/Magento/Framework/Indexer/BatchSizeCalculator.php b/lib/internal/Magento/Framework/Indexer/BatchSizeCalculator.php new file mode 100644 index 0000000000000..8b10413826506 --- /dev/null +++ b/lib/internal/Magento/Framework/Indexer/BatchSizeCalculator.php @@ -0,0 +1,45 @@ +fetchOne('SELECT @@max_heap_table_size;'); + $tmpTableSize = $adapter->fetchOne('SELECT @@tmp_table_size;'); + $maxMemoryTableSize = min($maxHeapTableSize, $tmpTableSize); + + /** + * According to MySQL documentation minimum value of the max_heap_table_size is 16384. + * (This system variable sets the maximum size to which user-created MEMORY tables are permitted to grow) + */ + return (int)($maxMemoryTableSize * ($memoryTableMinRows / 16384)); + } +} diff --git a/lib/internal/Magento/Framework/Indexer/BatchSizeCalculatorInterface.php b/lib/internal/Magento/Framework/Indexer/BatchSizeCalculatorInterface.php new file mode 100644 index 0000000000000..b5b182cafa357 --- /dev/null +++ b/lib/internal/Magento/Framework/Indexer/BatchSizeCalculatorInterface.php @@ -0,0 +1,27 @@ +model = new BatchProvider(); + } + + /** + * @param int $batchSize preferable batch size + * @param int $maxLinkFieldValue maximum value of the entity identifier in the table + * @param int $expectedResult list of expected consecutive entity ID ranges (batches) + * + * @dataProvider getBatchesDataProvider + */ + public function testGetBatches($batchSize, $maxLinkFieldValue, $expectedResult) + { + $tableName = 'test_table'; + $linkField = 'id'; + + $selectMock = $this->getMock(Select::class, [], [], '', false); + $adapterMock = $this->getMock(AdapterInterface::class); + + $selectMock->expects($this->once())->method('from')->willReturnSelf(); + $adapterMock->expects($this->once())->method('select')->willReturn($selectMock); + $adapterMock->expects($this->once())->method('fetchOne')->with($selectMock, [])->willReturn($maxLinkFieldValue); + $batches = $this->model->getBatches($adapterMock, $tableName, $linkField, $batchSize); + foreach ($batches as $index => $batch) { + $this->assertEquals($expectedResult[$index], $batch); + } + } + + /** + * @return array + */ + public function getBatchesDataProvider() + { + return [ + [200, 600, [['from' => 1, 'to' => 200], ['from' => 201, 'to' => 400], ['from' => 401, 'to' => 600]]], + [200, 555, [['from' => 1, 'to' => 200], ['from' => 201, 'to' => 400], ['from' => 401, 'to' => 555]]], + [200, 10, [['from' => 1, 'to' => 10]]], + [200, 0, []], + ]; + } +} diff --git a/lib/internal/Magento/Framework/Indexer/Test/Unit/BatchSizeCalculatorTest.php b/lib/internal/Magento/Framework/Indexer/Test/Unit/BatchSizeCalculatorTest.php new file mode 100644 index 0000000000000..29477ddc9b29d --- /dev/null +++ b/lib/internal/Magento/Framework/Indexer/Test/Unit/BatchSizeCalculatorTest.php @@ -0,0 +1,63 @@ +model = new BatchSizeCalculator(); + } + + /** + * @param int $memoryTableMinRows number of records that can be inserted into MEMORY table when its maximum size + * @param int $maxHeapTableSize max_heap_table_size MySQL value + * @param int $tmpTableSize tmp_table_size MySQL value + * @param int $expectedResult + * + * @dataProvider estimateBatchSizeDataProvider + */ + public function testEstimateBatchSize($memoryTableMinRows, $maxHeapTableSize, $tmpTableSize, $expectedResult) + { + $adapterMock = $this->getMock(AdapterInterface::class); + $adapterMock->expects($this->at(0)) + ->method('fetchOne') + ->with('SELECT @@max_heap_table_size;', []) + ->willReturn($maxHeapTableSize); + $adapterMock->expects($this->at(1)) + ->method('fetchOne') + ->with('SELECT @@tmp_table_size;', []) + ->willReturn($tmpTableSize); + + $this->assertEquals($expectedResult, $this->model->estimateBatchSize($adapterMock, $memoryTableMinRows)); + } + + /** + * @return array + */ + public function estimateBatchSizeDataProvider() + { + /** + * The maximum size for in-memory temporary tables is determined from whichever of the values of + * tmp_table_size and max_heap_table_size is smaller. + */ + return [ + [200, 16384, 16384, 200], + [200, 16384, 20000, 200], + [200, 20000, 16384, 200], + [200, 20000, 20000, 244], + [200, 2000, 2000, 24], + ]; + } +} From 99982cac530f75e9f465de3597a4e90c01326c0c Mon Sep 17 00:00:00 2001 From: Stanislav Idolov Date: Tue, 21 Feb 2017 14:58:17 +0200 Subject: [PATCH 03/43] MAGETWO-64193: Integrate the batch algorithm to DefaultPrice indexer --- .../Model/ResourceModel/Indexer/Price.php | 32 +--- app/code/Magento/Bundle/etc/di.xml | 7 + .../Indexer/Product/Price/Action/Full.php | 153 +++++++++++++++++- .../Product/Indexer/Price/DefaultPrice.php | 30 ++-- app/code/Magento/Catalog/etc/di.xml | 9 ++ .../Product/Indexer/Price/Configurable.php | 41 +---- .../Model/ResourceModel/Indexer/Price.php | 31 ---- .../Product/Indexer/Price/Grouped.php | 30 +--- 8 files changed, 193 insertions(+), 140 deletions(-) diff --git a/app/code/Magento/Bundle/Model/ResourceModel/Indexer/Price.php b/app/code/Magento/Bundle/Model/ResourceModel/Indexer/Price.php index 39db57e94438c..a559cee731405 100644 --- a/app/code/Magento/Bundle/Model/ResourceModel/Indexer/Price.php +++ b/app/code/Magento/Bundle/Model/ResourceModel/Indexer/Price.php @@ -15,38 +15,12 @@ class Price extends \Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\DefaultPrice { /** - * Reindex temporary (price result data) for all products - * - * @return $this - * @throws \Exception + * {@inheritdoc} + * @param null|array $entityIds */ - public function reindexAll() - { - $this->tableStrategy->setUseIdxTable(true); - - $this->beginTransaction(); - try { - $this->_prepareBundlePrice(); - $this->commit(); - } catch (\Exception $e) { - $this->rollBack(); - throw $e; - } - - return $this; - } - - /** - * Reindex temporary (price result data) for defined product(s) - * - * @param int|array $entityIds - * @return $this - */ - public function reindexEntity($entityIds) + protected function reindex($entityIds = null) { $this->_prepareBundlePrice($entityIds); - - return $this; } /** diff --git a/app/code/Magento/Bundle/etc/di.xml b/app/code/Magento/Bundle/etc/di.xml index e2a913acd50b7..d9d1a75f43822 100644 --- a/app/code/Magento/Bundle/etc/di.xml +++ b/app/code/Magento/Bundle/etc/di.xml @@ -137,4 +137,11 @@ + + + + 136 + + + diff --git a/app/code/Magento/Catalog/Model/Indexer/Product/Price/Action/Full.php b/app/code/Magento/Catalog/Model/Indexer/Product/Price/Action/Full.php index 8148703b1777c..acfe8900598e1 100644 --- a/app/code/Magento/Catalog/Model/Indexer/Product/Price/Action/Full.php +++ b/app/code/Magento/Catalog/Model/Indexer/Product/Price/Action/Full.php @@ -7,31 +7,172 @@ /** * Class Full reindex action - * + * @SuppressWarnings(PHPMD.CouplingBetweenObjects) */ class Full extends \Magento\Catalog\Model\Indexer\Product\Price\AbstractAction { + /** + * @var array + */ + private $memoryTablesMinRows; + + /** + * @var \Magento\Framework\EntityManager\MetadataPool + */ + private $metadataPool; + + /** + * @var \Magento\Framework\Indexer\BatchSizeCalculatorInterface + */ + private $batchSizeCalculator; + + /** + * @var \Magento\Framework\Indexer\BatchProviderInterface + */ + private $batchProvider; + + /** + * @param \Magento\Framework\App\Config\ScopeConfigInterface $config + * @param \Magento\Store\Model\StoreManagerInterface $storeManager + * @param \Magento\Directory\Model\CurrencyFactory $currencyFactory + * @param \Magento\Framework\Stdlib\DateTime\TimezoneInterface $localeDate + * @param \Magento\Framework\Stdlib\DateTime $dateTime + * @param \Magento\Catalog\Model\Product\Type $catalogProductType + * @param \Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\Factory $indexerPriceFactory + * @param \Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\DefaultPrice $defaultIndexerResource + * @param \Magento\Framework\EntityManager\MetadataPool|null $metadataPool + * @param \Magento\Framework\Indexer\BatchSizeCalculatorInterface|null $batchSizeCalculator + * @param \Magento\Framework\Indexer\BatchProviderInterface|null $batchProvider + * @param array $memoryTablesMinRows + * @SuppressWarnings(PHPMD.ExcessiveParameterList) + */ + public function __construct( + \Magento\Framework\App\Config\ScopeConfigInterface $config, + \Magento\Store\Model\StoreManagerInterface $storeManager, + \Magento\Directory\Model\CurrencyFactory $currencyFactory, + \Magento\Framework\Stdlib\DateTime\TimezoneInterface $localeDate, + \Magento\Framework\Stdlib\DateTime $dateTime, + \Magento\Catalog\Model\Product\Type $catalogProductType, + \Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\Factory $indexerPriceFactory, + \Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\DefaultPrice $defaultIndexerResource, + \Magento\Framework\EntityManager\MetadataPool $metadataPool = null, + \Magento\Framework\Indexer\BatchSizeCalculatorInterface $batchSizeCalculator = null, + \Magento\Framework\Indexer\BatchProviderInterface $batchProvider = null, + array $memoryTablesMinRows = [] + ) { + $objectManager = \Magento\Framework\App\ObjectManager::getInstance(); + $this->metadataPool = $metadataPool ?: $objectManager->get( + \Magento\Framework\EntityManager\MetadataPool::class + ); + $this->batchSizeCalculator = $batchSizeCalculator ?: $objectManager->get( + \Magento\Framework\Indexer\BatchSizeCalculatorInterface::class + ); + $this->batchProvider = $batchProvider ?: $objectManager->get( + \Magento\Framework\Indexer\BatchProviderInterface::class + ); + $this->memoryTablesMinRows = $memoryTablesMinRows; + parent::__construct( + $config, + $storeManager, + $currencyFactory, + $localeDate, + $dateTime, + $catalogProductType, + $indexerPriceFactory, + $defaultIndexerResource + ); + } + /** * Execute Full reindex * * @param array|int|null $ids * @return void * @throws \Magento\Framework\Exception\LocalizedException + * @SuppressWarnings(PHPMD.UnusedFormalParameter) */ public function execute($ids = null) { try { - $this->_defaultIndexerResource->getTableStrategy()->setUseIdxTable(true); - $this->_emptyTable($this->_defaultIndexerResource->getIdxTable()); + $this->_defaultIndexerResource->beginTransaction(); + $this->_defaultIndexerResource->getTableStrategy()->setUseIdxTable(false); $this->_prepareWebsiteDateTable(); - $this->_prepareTierPriceIndex(); + $this->_emptyTable($this->_defaultIndexerResource->getIdxTable()); + $this->emptyPriceIndexTable(); + + $entityMetadata = $this->metadataPool->getMetadata(\Magento\Catalog\Api\Data\ProductInterface::class); + $notComposite = []; + + /** @var \Magento\Catalog\Model\ResourceModel\Product\Indexer\AbstractIndexer $indexer */ foreach ($this->getTypeIndexers() as $indexer) { - $indexer->reindexAll(); + + $memoryTableMinRows = isset($this->memoryTablesMinRows[$indexer->getTypeId()]) + ? $this->memoryTablesMinRows[$indexer->getTypeId()] + : $this->memoryTablesMinRows['default']; + + $connection = $indexer->getConnection(); + $batches = $this->batchProvider->getBatches( + $connection, + $entityMetadata->getEntityTable(), + $entityMetadata->getIdentifierField(), + $this->batchSizeCalculator->estimateBatchSize($connection, $memoryTableMinRows) + ); + + foreach ($batches as $batch) { + // Get entity ids from batch + $select = $connection->select(); + $select->distinct(true); + $select->from(['e' => $entityMetadata->getEntityTable()], $entityMetadata->getIdentifierField()); + $select->where('type_id = ?', $indexer->getTypeId()); + + $entityIds = $this->batchProvider->getBatchIds($connection, $select, $batch); + + if (!empty($entityIds)) { + $this->_emptyTable($this->_defaultIndexerResource->getIdxTable()); + if ($indexer->getIsComposite()) { + $this->_copyRelationIndexData($entityIds, $notComposite); + } else { + $notComposite = array_merge($notComposite, $entityIds); + } + $this->_prepareTierPriceIndex($entityIds); + // Reindex entities by id + $indexer->reindexEntity($entityIds); + + // Sync data from temp table to index table + $this->_insertFromTable( + $this->_defaultIndexerResource->getIdxTable(), + $this->_defaultIndexerResource->getTable('catalog_product_index_price') + ); + } + } } - $this->_syncData(); + $this->_defaultIndexerResource->commit(); } catch (\Exception $e) { + $this->_defaultIndexerResource->rollBack(); throw new \Magento\Framework\Exception\LocalizedException(__($e->getMessage()), $e); } } + + /** + * Remove price index data + * + * Remove all price data from index table for current website + * @return void + */ + private function emptyPriceIndexTable() + { + $select = $this->_connection->select()->from( + ['index_price' => $this->_defaultIndexerResource->getTable('catalog_product_index_price')], + null + )->joinLeft( + ['ip_tmp' => $this->_defaultIndexerResource->getIdxTable()], + 'index_price.entity_id = ip_tmp.entity_id AND index_price.website_id = ip_tmp.website_id', + [] + )->where( + 'ip_tmp.entity_id IS NULL' + ); + $sql = $select->deleteFromSelect('index_price'); + $this->_connection->query($sql); + } } diff --git a/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/DefaultPrice.php b/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/DefaultPrice.php index 8a97d43a18d7e..554dc6ac4c90d 100644 --- a/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/DefaultPrice.php +++ b/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/DefaultPrice.php @@ -44,7 +44,12 @@ class DefaultPrice extends AbstractIndexer implements PriceInterface protected $_eventManager = null; /** - * Class constructor + * @var bool|null + */ + private $hasEntity = null; + + /** + * DefaultPrice constructor. * * @param \Magento\Framework\Model\ResourceModel\Db\Context $context * @param \Magento\Framework\Indexer\Table\StrategyInterface $tableStrategy @@ -677,16 +682,19 @@ public function getIdxTable($table = null) */ protected function hasEntity() { - $reader = $this->getConnection(); - - $select = $reader->select()->from( - [$this->getTable('catalog_product_entity')], - ['count(entity_id)'] - )->where( - 'type_id=?', - $this->getTypeId() - ); + if ($this->hasEntity === null) { + $reader = $this->getConnection(); + + $select = $reader->select()->from( + [$this->getTable('catalog_product_entity')], + ['count(entity_id)'] + )->where( + 'type_id=?', + $this->getTypeId() + ); + $this->hasEntity = (int)$reader->fetchOne($select) > 0; + } - return (int)$reader->fetchOne($select) > 0; + return $this->hasEntity; } } diff --git a/app/code/Magento/Catalog/etc/di.xml b/app/code/Magento/Catalog/etc/di.xml index 8ce6b1785171c..ce910c560a248 100644 --- a/app/code/Magento/Catalog/etc/di.xml +++ b/app/code/Magento/Catalog/etc/di.xml @@ -62,6 +62,8 @@ + + @@ -906,4 +908,11 @@ + + + + 200 + + + diff --git a/app/code/Magento/ConfigurableProduct/Model/ResourceModel/Product/Indexer/Price/Configurable.php b/app/code/Magento/ConfigurableProduct/Model/ResourceModel/Product/Indexer/Price/Configurable.php index 77536521e10a3..6ca2162bbd223 100644 --- a/app/code/Magento/ConfigurableProduct/Model/ResourceModel/Product/Indexer/Price/Configurable.php +++ b/app/code/Magento/ConfigurableProduct/Model/ResourceModel/Product/Indexer/Price/Configurable.php @@ -31,8 +31,8 @@ class Configurable extends \Magento\Catalog\Model\ResourceModel\Product\Indexer\ * @param \Magento\Eav\Model\Config $eavConfig * @param \Magento\Framework\Event\ManagerInterface $eventManager * @param \Magento\Framework\Module\Manager $moduleManager - * @param string $connectionName - * @param StoreResolverInterface $storeResolver + * @param string|null $connectionName + * @param StoreResolverInterface|null $storeResolver */ public function __construct( \Magento\Framework\Model\ResourceModel\Db\Context $context, @@ -44,40 +44,9 @@ public function __construct( StoreResolverInterface $storeResolver = null ) { parent::__construct($context, $tableStrategy, $eavConfig, $eventManager, $moduleManager, $connectionName); - $this->storeResolver = $storeResolver ?: - \Magento\Framework\App\ObjectManager::getInstance()->get(StoreResolverInterface::class); - } - - /** - * Reindex temporary (price result data) for all products - * - * @return $this - * @throws \Exception - */ - public function reindexAll() - { - $this->tableStrategy->setUseIdxTable(true); - $this->beginTransaction(); - try { - $this->reindex(); - $this->commit(); - } catch (\Exception $e) { - $this->rollBack(); - throw $e; - } - return $this; - } - - /** - * Reindex temporary (price result data) for defined product(s) - * - * @param int|array $entityIds - * @return \Magento\ConfigurableProduct\Model\ResourceModel\Product\Indexer\Price\Configurable - */ - public function reindexEntity($entityIds) - { - $this->reindex($entityIds); - return $this; + $this->storeResolver = $storeResolver ?: \Magento\Framework\App\ObjectManager::getInstance()->get( + StoreResolverInterface::class + ); } /** diff --git a/app/code/Magento/Downloadable/Model/ResourceModel/Indexer/Price.php b/app/code/Magento/Downloadable/Model/ResourceModel/Indexer/Price.php index 843e690e34213..9fe4da939a369 100644 --- a/app/code/Magento/Downloadable/Model/ResourceModel/Indexer/Price.php +++ b/app/code/Magento/Downloadable/Model/ResourceModel/Indexer/Price.php @@ -14,37 +14,6 @@ */ class Price extends \Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\DefaultPrice { - /** - * Reindex temporary (price result data) for all products - * - * @throws \Exception - * @return $this - */ - public function reindexAll() - { - $this->tableStrategy->setUseIdxTable(true); - $this->beginTransaction(); - try { - $this->reindex(); - $this->commit(); - } catch (\Exception $e) { - $this->rollBack(); - throw $e; - } - return $this; - } - - /** - * Reindex temporary (price result data) for defined product(s) - * - * @param int|array $entityIds - * @return $this - */ - public function reindexEntity($entityIds) - { - return $this->reindex($entityIds); - } - /** * @param null|int|array $entityIds * @return \Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\DefaultPrice diff --git a/app/code/Magento/GroupedProduct/Model/ResourceModel/Product/Indexer/Price/Grouped.php b/app/code/Magento/GroupedProduct/Model/ResourceModel/Product/Indexer/Price/Grouped.php index eaed75a967cc0..0dc6e525f4b26 100644 --- a/app/code/Magento/GroupedProduct/Model/ResourceModel/Product/Indexer/Price/Grouped.php +++ b/app/code/Magento/GroupedProduct/Model/ResourceModel/Product/Indexer/Price/Grouped.php @@ -13,36 +13,12 @@ class Grouped extends DefaultPrice implements GroupedInterface { /** - * Reindex temporary (price result data) for all products - * - * @throws \Exception - * @return \Magento\GroupedProduct\Model\ResourceModel\Product\Indexer\Price\Grouped + * {@inheritdoc} + * @param null|array $entityIds */ - public function reindexAll() - { - $this->tableStrategy->setUseIdxTable(true); - $this->beginTransaction(); - try { - $this->_prepareGroupedProductPriceData(); - $this->commit(); - } catch (\Exception $e) { - $this->rollBack(); - throw $e; - } - return $this; - } - - /** - * Reindex temporary (price result data) for defined product(s) - * - * @param int|array $entityIds - * @return \Magento\GroupedProduct\Model\ResourceModel\Product\Indexer\Price\Grouped - */ - public function reindexEntity($entityIds) + protected function reindex($entityIds = null) { $this->_prepareGroupedProductPriceData($entityIds); - - return $this; } /** From 3c175e08fc75d1fc5154d9ed2a4513c851154823 Mon Sep 17 00:00:00 2001 From: Stanislav Idolov Date: Tue, 21 Feb 2017 15:27:15 +0200 Subject: [PATCH 04/43] MAGETWO-64466: [Indexer optimizations] Indexation process in non-locking way for price indexer --- .../Indexer/Product/Price/AbstractAction.php | 19 ++- .../Indexer/Product/Price/Action/Full.php | 4 +- .../ResourceModel/Layer/Filter/Price.php | 25 +++- .../ResourceModel/Product/Collection.php | 13 +- ...LinkedProductSelectBuilderByIndexPrice.php | 16 ++- .../Product/Indexer/Price/DefaultPrice.php | 32 ++++- .../Magento/Catalog/Setup/InstallSchema.php | 10 ++ .../Magento/Catalog/Setup/UpgradeSchema.php | 22 ++++ .../Unit/Model/Product/Type/PriceTest.php | 13 +- .../ResourceModel/Layer/Filter/PriceTest.php | 44 +++++++ ...edProductSelectBuilderByIndexPriceTest.php | 109 +++++++++++++++++ .../Indexer/Price/DefaultPriceTest.php | 60 +++++++++ app/code/Magento/Catalog/etc/di.xml | 15 +++ app/code/Magento/Catalog/etc/module.xml | 2 +- .../Mysql/Aggregation/DataProvider.php | 16 ++- .../Adapter/Mysql/Dynamic/DataProvider.php | 16 ++- .../ResourceModel/Advanced/Collection.php | 7 +- .../ResourceModel/Fulltext/Collection.php | 7 +- .../Search/FilterMapper/ExclusionStrategy.php | 17 ++- .../Mysql/Aggregation/DataProviderTest.php | 114 ++++++++++++++++++ .../Mysql/Dynamic/DataProviderTest.php | 101 ++++++++++++++++ .../FilterMapper/ExclusionStrategyTest.php | 88 ++++++++++++++ app/code/Magento/CatalogSearch/composer.json | 1 + app/code/Magento/CatalogSearch/etc/di.xml | 11 ++ app/code/Magento/Indexer/Model/Indexer.php | 7 +- .../Magento/Indexer/Model/Indexer/State.php | 21 ++++ .../Model/ResourceModel/FrontendResource.php | 80 ++++++++++++ .../Magento/Indexer/Setup/InstallSchema.php | 8 ++ .../Magento/Indexer/Setup/UpgradeSchema.php | 53 ++++++++ app/code/Magento/Indexer/etc/module.xml | 2 +- .../Framework/Indexer/StateInterface.php | 2 + 31 files changed, 905 insertions(+), 30 deletions(-) create mode 100644 app/code/Magento/Catalog/Test/Unit/Model/ResourceModel/Layer/Filter/PriceTest.php create mode 100644 app/code/Magento/Catalog/Test/Unit/Model/ResourceModel/Product/Indexer/LinkedProductSelectBuilderByIndexPriceTest.php create mode 100644 app/code/Magento/Catalog/Test/Unit/Model/ResourceModel/Product/Indexer/Price/DefaultPriceTest.php create mode 100644 app/code/Magento/CatalogSearch/Test/Unit/Model/Adapter/Mysql/Aggregation/DataProviderTest.php create mode 100644 app/code/Magento/CatalogSearch/Test/Unit/Model/Adapter/Mysql/Dynamic/DataProviderTest.php create mode 100644 app/code/Magento/CatalogSearch/Test/Unit/Model/Search/FilterMapper/ExclusionStrategyTest.php create mode 100644 app/code/Magento/Indexer/Model/ResourceModel/FrontendResource.php create mode 100644 app/code/Magento/Indexer/Setup/UpgradeSchema.php diff --git a/app/code/Magento/Catalog/Model/Indexer/Product/Price/AbstractAction.php b/app/code/Magento/Catalog/Model/Indexer/Product/Price/AbstractAction.php index 3ba84ee11d143..f4693a91f8cea 100644 --- a/app/code/Magento/Catalog/Model/Indexer/Product/Price/AbstractAction.php +++ b/app/code/Magento/Catalog/Model/Indexer/Product/Price/AbstractAction.php @@ -75,6 +75,11 @@ abstract class AbstractAction */ private $productResource; + /** + * @var \Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\FrontendResource + */ + private $indexerFrontendResource; + /** * @param \Magento\Framework\App\Config\ScopeConfigInterface $config * @param \Magento\Store\Model\StoreManagerInterface $storeManager @@ -84,6 +89,7 @@ abstract class AbstractAction * @param \Magento\Catalog\Model\Product\Type $catalogProductType * @param \Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\Factory $indexerPriceFactory * @param \Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\DefaultPrice $defaultIndexerResource + * @param \Magento\Indexer\Model\ResourceModel\FrontendResource|null $indexerFrontendResource */ public function __construct( \Magento\Framework\App\Config\ScopeConfigInterface $config, @@ -93,7 +99,8 @@ public function __construct( \Magento\Framework\Stdlib\DateTime $dateTime, \Magento\Catalog\Model\Product\Type $catalogProductType, \Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\Factory $indexerPriceFactory, - \Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\DefaultPrice $defaultIndexerResource + \Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\DefaultPrice $defaultIndexerResource, + \Magento\Indexer\Model\ResourceModel\FrontendResource $indexerFrontendResource = null ) { $this->_config = $config; $this->_storeManager = $storeManager; @@ -104,6 +111,10 @@ public function __construct( $this->_indexerPriceFactory = $indexerPriceFactory; $this->_defaultIndexerResource = $defaultIndexerResource; $this->_connection = $this->_defaultIndexerResource->getConnection(); + $this->indexerFrontendResource = $indexerFrontendResource ?: \Magento\Framework\App\ObjectManager::getInstance() + ->get( + \Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\FrontendResource::class + ); } /** @@ -124,7 +135,7 @@ protected function _syncData(array $processIds = []) { // delete invalid rows $select = $this->_connection->select()->from( - ['index_price' => $this->_defaultIndexerResource->getTable('catalog_product_index_price')], + ['index_price' => $this->indexerFrontendResource->getMainTable()], null )->joinLeft( ['ip_tmp' => $this->_defaultIndexerResource->getIdxTable()], @@ -141,7 +152,7 @@ protected function _syncData(array $processIds = []) $this->_insertFromTable( $this->_defaultIndexerResource->getIdxTable(), - $this->_defaultIndexerResource->getTable('catalog_product_index_price') + $this->indexerFrontendResource->getMainTable() ); return $this; } @@ -460,7 +471,7 @@ protected function _copyRelationIndexData($parentIds, $excludeIds = null) if ($children) { $select = $this->_connection->select()->from( - $this->_defaultIndexerResource->getTable('catalog_product_index_price') + $this->indexerFrontendResource->getMainTable() )->where( 'entity_id IN(?)', $children diff --git a/app/code/Magento/Catalog/Model/Indexer/Product/Price/Action/Full.php b/app/code/Magento/Catalog/Model/Indexer/Product/Price/Action/Full.php index acfe8900598e1..80abc97ebbd1e 100644 --- a/app/code/Magento/Catalog/Model/Indexer/Product/Price/Action/Full.php +++ b/app/code/Magento/Catalog/Model/Indexer/Product/Price/Action/Full.php @@ -142,7 +142,7 @@ public function execute($ids = null) // Sync data from temp table to index table $this->_insertFromTable( $this->_defaultIndexerResource->getIdxTable(), - $this->_defaultIndexerResource->getTable('catalog_product_index_price') + $this->_defaultIndexerResource->getMainTable() ); } } @@ -163,7 +163,7 @@ public function execute($ids = null) private function emptyPriceIndexTable() { $select = $this->_connection->select()->from( - ['index_price' => $this->_defaultIndexerResource->getTable('catalog_product_index_price')], + ['index_price' => $this->_defaultIndexerResource->getMainTable()], null )->joinLeft( ['ip_tmp' => $this->_defaultIndexerResource->getIdxTable()], diff --git a/app/code/Magento/Catalog/Model/ResourceModel/Layer/Filter/Price.php b/app/code/Magento/Catalog/Model/ResourceModel/Layer/Filter/Price.php index a8b7bf2960185..608e2bfbad6cd 100644 --- a/app/code/Magento/Catalog/Model/ResourceModel/Layer/Filter/Price.php +++ b/app/code/Magento/Catalog/Model/ResourceModel/Layer/Filter/Price.php @@ -7,6 +7,7 @@ /** * Catalog Layer Price Filter resource model + * @SuppressWarnings(PHPMD.CouplingBetweenObjects) */ class Price extends \Magento\Framework\Model\ResourceModel\Db\AbstractDb { @@ -37,13 +38,19 @@ class Price extends \Magento\Framework\Model\ResourceModel\Db\AbstractDb */ private $storeManager; + /** + * @var \Magento\Indexer\Model\ResourceModel\FrontendResource + */ + private $indexerFrontendResource; + /** * @param \Magento\Framework\Model\ResourceModel\Db\Context $context * @param \Magento\Framework\Event\ManagerInterface $eventManager * @param \Magento\Catalog\Model\Layer\Resolver $layerResolver * @param \Magento\Customer\Model\Session $session * @param \Magento\Store\Model\StoreManagerInterface $storeManager - * @param string $connectionName + * @param null $connectionName + * @param \Magento\Indexer\Model\ResourceModel\FrontendResource|null $stateFactory */ public function __construct( \Magento\Framework\Model\ResourceModel\Db\Context $context, @@ -51,12 +58,15 @@ public function __construct( \Magento\Catalog\Model\Layer\Resolver $layerResolver, \Magento\Customer\Model\Session $session, \Magento\Store\Model\StoreManagerInterface $storeManager, - $connectionName = null + $connectionName = null, + \Magento\Indexer\Model\ResourceModel\FrontendResource $stateFactory = null ) { $this->layer = $layerResolver->get(); $this->session = $session; $this->storeManager = $storeManager; $this->_eventManager = $eventManager; + $this->indexerFrontendResource = $stateFactory ?: \Magento\Framework\App\ObjectManager::getInstance() + ->get(\Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\FrontendResource::class); parent::__construct($context, $connectionName); } @@ -381,4 +391,15 @@ protected function _getIndexTableAlias() { return 'price_index'; } + + /** + * @inheritdoc + * Returns main table name in depends of the suffix stored in the 'indexer_state' table + * + * @return string + */ + public function getMainTable() + { + return $this->indexerFrontendResource->getMainTable(); + } } diff --git a/app/code/Magento/Catalog/Model/ResourceModel/Product/Collection.php b/app/code/Magento/Catalog/Model/ResourceModel/Product/Collection.php index 8b32219bef32f..3cec8504a9213 100644 --- a/app/code/Magento/Catalog/Model/ResourceModel/Product/Collection.php +++ b/app/code/Magento/Catalog/Model/ResourceModel/Product/Collection.php @@ -261,6 +261,11 @@ class Collection extends \Magento\Catalog\Model\ResourceModel\Collection\Abstrac */ private $metadataPool; + /** + * @var \Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\FrontendResource + */ + private $indexerFrontendResource; + /** * Collection constructor * @@ -286,6 +291,7 @@ class Collection extends \Magento\Catalog\Model\ResourceModel\Collection\Abstrac * @param \Magento\Framework\DB\Adapter\AdapterInterface|null $connection * @param ProductLimitationFactory|null $productLimitationFactory * @param MetadataPool|null $metadataPool + * @param null|\Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\FrontendResource $indexerFrontendResource * * @SuppressWarnings(PHPMD.ExcessiveParameterList) */ @@ -311,7 +317,8 @@ public function __construct( GroupManagementInterface $groupManagement, \Magento\Framework\DB\Adapter\AdapterInterface $connection = null, ProductLimitationFactory $productLimitationFactory = null, - MetadataPool $metadataPool = null + MetadataPool $metadataPool = null, + \Magento\Indexer\Model\ResourceModel\FrontendResource $indexerFrontendResource = null ) { $this->moduleManager = $moduleManager; $this->_catalogProductFlatState = $catalogProductFlatState; @@ -328,6 +335,8 @@ public function __construct( ); $this->_productLimitationFilters = $productLimitationFactory->create(); $this->metadataPool = $metadataPool ?: ObjectManager::getInstance()->get(MetadataPool::class); + $this->indexerFrontendResource = $indexerFrontendResource ?: \Magento\Framework\App\ObjectManager::getInstance() + ->get(\Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\FrontendResource::class); parent::__construct( $entityFactory, $logger, @@ -1917,7 +1926,7 @@ protected function _productLimitationPrice($joinLeft = false) 'max_price', 'tier_price', ]; - $tableName = ['price_index' => $this->getTable('catalog_product_index_price')]; + $tableName = ['price_index' => $this->indexerFrontendResource->getMainTable()]; if ($joinLeft) { $select->joinLeft($tableName, $joinCond, $colls); } else { diff --git a/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/LinkedProductSelectBuilderByIndexPrice.php b/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/LinkedProductSelectBuilderByIndexPrice.php index ffaf8a5d100d3..8e183dadd70e7 100644 --- a/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/LinkedProductSelectBuilderByIndexPrice.php +++ b/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/LinkedProductSelectBuilderByIndexPrice.php @@ -39,18 +39,26 @@ class LinkedProductSelectBuilderByIndexPrice implements LinkedProductSelectBuild private $baseSelectProcessor; /** + * @var \Magento\Indexer\Model\ResourceModel\FrontendResource + */ + private $indexerFrontendResource; + + /** + * LinkedProductSelectBuilderByIndexPrice constructor. * @param \Magento\Store\Model\StoreManagerInterface $storeManager * @param \Magento\Framework\App\ResourceConnection $resourceConnection * @param \Magento\Customer\Model\Session $customerSession * @param \Magento\Framework\EntityManager\MetadataPool $metadataPool - * @param BaseSelectProcessorInterface $baseSelectProcessor + * @param BaseSelectProcessorInterface|null $baseSelectProcessor + * @param \Magento\Indexer\Model\ResourceModel\FrontendResource|null $indexerFrontendResource */ public function __construct( \Magento\Store\Model\StoreManagerInterface $storeManager, \Magento\Framework\App\ResourceConnection $resourceConnection, \Magento\Customer\Model\Session $customerSession, \Magento\Framework\EntityManager\MetadataPool $metadataPool, - BaseSelectProcessorInterface $baseSelectProcessor = null + BaseSelectProcessorInterface $baseSelectProcessor = null, + \Magento\Indexer\Model\ResourceModel\FrontendResource $indexerFrontendResource = null ) { $this->storeManager = $storeManager; $this->resource = $resourceConnection; @@ -58,6 +66,8 @@ public function __construct( $this->metadataPool = $metadataPool; $this->baseSelectProcessor = (null !== $baseSelectProcessor) ? $baseSelectProcessor : ObjectManager::getInstance()->get(BaseSelectProcessorInterface::class); + $this->indexerFrontendResource = $indexerFrontendResource ?: ObjectManager::getInstance() + ->get(\Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\FrontendResource::class); } /** @@ -79,7 +89,7 @@ public function build($productId) sprintf('%s.entity_id = link.child_id', BaseSelectProcessorInterface::PRODUCT_TABLE_ALIAS), ['entity_id'] )->joinInner( - ['t' => $this->resource->getTableName('catalog_product_index_price')], + ['t' => $this->indexerFrontendResource->getMainTable()], sprintf('t.entity_id = %s.entity_id', BaseSelectProcessorInterface::PRODUCT_TABLE_ALIAS), [] )->where('parent.entity_id = ?', $productId) diff --git a/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/DefaultPrice.php b/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/DefaultPrice.php index 554dc6ac4c90d..425900ed316a3 100644 --- a/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/DefaultPrice.php +++ b/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/DefaultPrice.php @@ -12,6 +12,7 @@ * For correctly work need define product type id * * @author Magento Core Team + * @SuppressWarnings(PHPMD.CouplingBetweenObjects) */ class DefaultPrice extends AbstractIndexer implements PriceInterface { @@ -48,6 +49,11 @@ class DefaultPrice extends AbstractIndexer implements PriceInterface */ private $hasEntity = null; + /** + * @var \Magento\Indexer\Model\Indexer\StateFactory + */ + private $indexerStateFactory; + /** * DefaultPrice constructor. * @@ -56,7 +62,8 @@ class DefaultPrice extends AbstractIndexer implements PriceInterface * @param \Magento\Eav\Model\Config $eavConfig * @param \Magento\Framework\Event\ManagerInterface $eventManager * @param \Magento\Framework\Module\Manager $moduleManager - * @param string $connectionName + * @param string|null $connectionName + * @param null|\Magento\Indexer\Model\Indexer\StateFactory $stateFactory */ public function __construct( \Magento\Framework\Model\ResourceModel\Db\Context $context, @@ -64,10 +71,13 @@ public function __construct( \Magento\Eav\Model\Config $eavConfig, \Magento\Framework\Event\ManagerInterface $eventManager, \Magento\Framework\Module\Manager $moduleManager, - $connectionName = null + $connectionName = null, + \Magento\Indexer\Model\Indexer\StateFactory $stateFactory = null ) { $this->_eventManager = $eventManager; $this->moduleManager = $moduleManager; + $this->indexerStateFactory = $stateFactory ?: \Magento\Framework\App\ObjectManager::getInstance() + ->get(\Magento\Indexer\Model\Indexer\StateFactory::class); parent::__construct($context, $tableStrategy, $eavConfig, $connectionName); } @@ -697,4 +707,22 @@ protected function hasEntity() return $this->hasEntity; } + + /** + * @inheritdoc + * Returns main table name in depends of the suffix stored in the 'indexer_state' table + * + * @return string + */ + public function getMainTable() + { + $table = parent::getMainTable(); + $indexerState = $this->indexerStateFactory->create()->loadByIndexer( + \Magento\Catalog\Model\Indexer\Product\Price\Processor::INDEXER_ID + ); + $destinationTableSuffix = ($indexerState->getTableSuffix() === '') + ? \Magento\Framework\Indexer\StateInterface::ADDITIONAL_TABLE_SUFFIX + : ''; + return $table . $destinationTableSuffix; + } } diff --git a/app/code/Magento/Catalog/Setup/InstallSchema.php b/app/code/Magento/Catalog/Setup/InstallSchema.php index 31cb143396dd4..ec82953245e20 100644 --- a/app/code/Magento/Catalog/Setup/InstallSchema.php +++ b/app/code/Magento/Catalog/Setup/InstallSchema.php @@ -3043,6 +3043,16 @@ public function install(SchemaSetupInterface $setup, ModuleContextInterface $con $installer->getConnection() ->createTable($table); + /** + * Create table 'catalog_product_index_price_replica' + */ + $installer->getConnection()->createTable( + $installer->getConnection()->createTableByDdl( + $installer->getTable('catalog_product_index_price'), + $installer->getTable('catalog_product_index_price_replica') + ) + ); + /** * Create table 'catalog_product_index_tier_price' */ diff --git a/app/code/Magento/Catalog/Setup/UpgradeSchema.php b/app/code/Magento/Catalog/Setup/UpgradeSchema.php index db0aaa3a010e3..62d76c78edb32 100755 --- a/app/code/Magento/Catalog/Setup/UpgradeSchema.php +++ b/app/code/Magento/Catalog/Setup/UpgradeSchema.php @@ -60,6 +60,9 @@ public function upgrade(SchemaSetupInterface $setup, ModuleContextInterface $con $this->recreateCatalogCategoryProductIndexTmpTable($setup); } + if (version_compare($context->getVersion(), '2.1.5', '<')) { + $this->addProductPriceIndexReplicaTable($setup); + } $setup->endSetup(); } @@ -446,4 +449,23 @@ private function recreateCatalogCategoryProductIndexTmpTable(SchemaSetupInterfac $setup->getConnection()->createTable($table); } + + /** + * Add Replica for Catalog Product Price Index Table. + * + * By adding 'catalog_product_index_price_replica' we provide separation of tables used for indexation write + * and read operations and affected models. + * + * @param SchemaSetupInterface $setup + * @return void + */ + private function addProductPriceIndexReplicaTable(SchemaSetupInterface $setup) + { + $setup->getConnection()->createTable( + $setup->getConnection()->createTableByDdl( + $setup->getTable('catalog_product_index_price'), + $setup->getTable('catalog_product_index_price_replica') + ) + ); + } } diff --git a/app/code/Magento/Catalog/Test/Unit/Model/Product/Type/PriceTest.php b/app/code/Magento/Catalog/Test/Unit/Model/Product/Type/PriceTest.php index a18c2aee1ef9d..68b5cb7b98fc0 100644 --- a/app/code/Magento/Catalog/Test/Unit/Model/Product/Type/PriceTest.php +++ b/app/code/Magento/Catalog/Test/Unit/Model/Product/Type/PriceTest.php @@ -59,6 +59,11 @@ class PriceTest extends \PHPUnit_Framework_TestCase */ protected $websiteMock; + /** + * @var \Magento\Indexer\Model\ResourceModel\FrontendResource + */ + private $idxFrontendResourceMock; + protected function setUp() { $this->objectManagerHelper = new ObjectManagerHelper($this); @@ -110,14 +115,18 @@ protected function setUp() $this->getMock(\Magento\Customer\Api\GroupManagementInterface::class, [], [], '', false); $this->groupManagementMock->expects($this->any())->method('getAllCustomersGroup') ->will($this->returnValue($group)); - + $this->idxFrontendResourceMock = + $this->getMockBuilder(\Magento\Indexer\Model\ResourceModel\FrontendResource::class) + ->disableOriginalConstructor() + ->getMock(); $this->model = $this->objectManagerHelper->getObject( \Magento\Catalog\Model\Product\Type\Price::class, [ 'tierPriceFactory' => $this->tpFactory, 'config' => $this->scopeConfigMock, 'storeManager' => $storeMangerMock, - 'groupManagement' => $this->groupManagementMock + 'groupManagement' => $this->groupManagementMock, + 'indexerFrontendResource' => $this->idxFrontendResourceMock ] ); } diff --git a/app/code/Magento/Catalog/Test/Unit/Model/ResourceModel/Layer/Filter/PriceTest.php b/app/code/Magento/Catalog/Test/Unit/Model/ResourceModel/Layer/Filter/PriceTest.php new file mode 100644 index 0000000000000..31e9391936cdf --- /dev/null +++ b/app/code/Magento/Catalog/Test/Unit/Model/ResourceModel/Layer/Filter/PriceTest.php @@ -0,0 +1,44 @@ +idxFrontendResourceMock = + $this->getMockBuilder(\Magento\Indexer\Model\ResourceModel\FrontendResource::class) + ->disableOriginalConstructor() + ->getMock(); + $this->model = $objectManagerHelper->getObject( + \Magento\Catalog\Model\ResourceModel\Layer\Filter\Price::class, + [ + 'indexerFrontendResource' => $this->idxFrontendResourceMock + ] + ); + } + + public function testGetMainTable() + { + $expectedTableName = 'expectedTableName'; + $this->idxFrontendResourceMock->expects($this->once())->method('getMainTable')->willReturn($expectedTableName); + $this->assertEquals($expectedTableName, $this->model->getMainTable()); + } +} diff --git a/app/code/Magento/Catalog/Test/Unit/Model/ResourceModel/Product/Indexer/LinkedProductSelectBuilderByIndexPriceTest.php b/app/code/Magento/Catalog/Test/Unit/Model/ResourceModel/Product/Indexer/LinkedProductSelectBuilderByIndexPriceTest.php new file mode 100644 index 0000000000000..cad914763bbeb --- /dev/null +++ b/app/code/Magento/Catalog/Test/Unit/Model/ResourceModel/Product/Indexer/LinkedProductSelectBuilderByIndexPriceTest.php @@ -0,0 +1,109 @@ +storeManagerMock = $this->getMockBuilder(\Magento\Store\Model\StoreManagerInterface::class) + ->disableOriginalConstructor() + ->getMockForAbstractClass(); + $this->resourceMock = $this->getMockBuilder(\Magento\Framework\App\ResourceConnection::class) + ->disableOriginalConstructor()->getMock(); + $this->customerSessionMock = $this->getMockBuilder(\Magento\Customer\Model\Session::class) + ->disableOriginalConstructor() + ->getMock(); + $this->metadataPoolMock = $this->getMockBuilder(\Magento\Framework\EntityManager\MetadataPool::class) + ->disableOriginalConstructor() + ->getMock(); + $this->baseSelectProcessorMock = + $this->getMockBuilder(\Magento\Catalog\Model\ResourceModel\Product\BaseSelectProcessorInterface::class) + ->disableOriginalConstructor() + ->getMockForAbstractClass(); + $this->indexerFrontendResourceMock = + $this->getMockBuilder(\Magento\Indexer\Model\ResourceModel\FrontendResource::class) + ->disableOriginalConstructor() + ->getMock(); + $this->model = new \Magento\Catalog\Model\ResourceModel\Product\Indexer\LinkedProductSelectBuilderByIndexPrice( + $this->storeManagerMock, + $this->resourceMock, + $this->customerSessionMock, + $this->metadataPoolMock, + $this->baseSelectProcessorMock, + $this->indexerFrontendResourceMock + ); + } + + public function testBuild() + { + $productId = 10; + $idxTable = 'catalog_product_index_price'; + $metadata = $this->getMockBuilder(\Magento\Framework\EntityManager\EntityMetadataInterface::class) + ->disableOriginalConstructor() + ->getMockForAbstractClass(); + $connection = $this->getMockBuilder(\Magento\Framework\DB\Adapter\AdapterInterface::class) + ->getMockForAbstractClass(); + $select = $this->getMockBuilder(\Magento\Framework\DB\Select::class) + ->disableOriginalConstructor() + ->getMock(); + $storeMock = $this->getMockBuilder(\Magento\Store\Api\Data\StoreInterface::class) + ->getMockForAbstractClass(); + $this->storeManagerMock->expects($this->once())->method('getStore')->willReturn($storeMock); + $this->customerSessionMock->expects($this->once())->method('getCustomerGroupId'); + $connection->expects($this->any())->method('select')->willReturn($select); + $select->expects($this->any())->method('from')->willReturnSelf(); + $select->expects($this->any())->method('joinInner')->willReturnSelf(); + $select->expects($this->any())->method('where')->willReturnSelf(); + $select->expects($this->once())->method('order')->willReturnSelf(); + $select->expects($this->once())->method('limit')->willReturnSelf(); + $this->resourceMock->expects($this->any())->method('getConnection')->willReturn($connection); + $this->metadataPoolMock->expects($this->once())->method('getMetadata')->willReturn($metadata); + $metadata->expects($this->once())->method('getLinkField')->willReturn('row_id'); + $this->resourceMock->expects($this->any())->method('getTableName'); + + $this->indexerFrontendResourceMock->expects($this->once())->method('getMainTable')->willReturn($idxTable); + $this->baseSelectProcessorMock->expects($this->once())->method('process')->willReturnSelf(); + $this->model->build($productId); + } +} diff --git a/app/code/Magento/Catalog/Test/Unit/Model/ResourceModel/Product/Indexer/Price/DefaultPriceTest.php b/app/code/Magento/Catalog/Test/Unit/Model/ResourceModel/Product/Indexer/Price/DefaultPriceTest.php new file mode 100644 index 0000000000000..09b27b0a59ea4 --- /dev/null +++ b/app/code/Magento/Catalog/Test/Unit/Model/ResourceModel/Product/Indexer/Price/DefaultPriceTest.php @@ -0,0 +1,60 @@ +resourceMock = $this->getMockBuilder(\Magento\Framework\App\ResourceConnection::class) + ->disableOriginalConstructor() + ->getMock(); + $this->indexerStateFactory = $this->getMockBuilder(\Magento\Indexer\Model\Indexer\StateFactory::class) + ->disableOriginalConstructor() + ->setMethods(['create']) + ->getMock(); + $this->model = $objectManagerHelper->getObject( + \Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\DefaultPrice::class, + [ + 'indexerStateFactory' => $this->indexerStateFactory, + 'resources' => $this->resourceMock + ] + ); + } + + public function testGetMainTable() + { + $indexerStateModel = $this->getMockBuilder(\Magento\Indexer\Model\Indexer\State::class) + ->disableOriginalConstructor() + ->getMock(); + $this->resourceMock->expects($this->once())->method('getTableName')->willReturn('catalog_product_index_price'); + $this->indexerStateFactory->expects($this->once())->method('create')->willReturn($indexerStateModel); + $indexerStateModel->expects($this->once()) + ->method('loadByIndexer') + ->with(\Magento\Catalog\Model\Indexer\Product\Price\Processor::INDEXER_ID) + ->willReturnSelf(); + $indexerStateModel->expects($this->once())->method('getTableSuffix')->willReturn(''); + $this->assertEquals('catalog_product_index_price_replica', $this->model->getMainTable()); + } +} diff --git a/app/code/Magento/Catalog/etc/di.xml b/app/code/Magento/Catalog/etc/di.xml index ce910c560a248..db38d7f270671 100644 --- a/app/code/Magento/Catalog/etc/di.xml +++ b/app/code/Magento/Catalog/etc/di.xml @@ -132,6 +132,7 @@ Magento\Catalog\Model\ResourceModel\Url\Proxy Magento\Customer\Model\Session\Proxy + Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\FrontendResource @@ -226,6 +227,7 @@ Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\DefaultPrice + Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\FrontendResource @@ -871,6 +873,7 @@ Magento\Catalog\Model\ResourceModel\Product\CompositeBaseSelectProcessor + Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\FrontendResource @@ -915,4 +918,16 @@ + + + Magento\Catalog\Model\Indexer\Product\Price\Processor::INDEXER_ID + catalog_product_index_price + entity_id + + + + + Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\FrontendResource + + diff --git a/app/code/Magento/Catalog/etc/module.xml b/app/code/Magento/Catalog/etc/module.xml index 571176694547a..383ab6dd26625 100644 --- a/app/code/Magento/Catalog/etc/module.xml +++ b/app/code/Magento/Catalog/etc/module.xml @@ -6,7 +6,7 @@ */ --> - + diff --git a/app/code/Magento/CatalogSearch/Model/Adapter/Mysql/Aggregation/DataProvider.php b/app/code/Magento/CatalogSearch/Model/Adapter/Mysql/Aggregation/DataProvider.php index 0c8c26904d148..2ed3215b8bdb1 100644 --- a/app/code/Magento/CatalogSearch/Model/Adapter/Mysql/Aggregation/DataProvider.php +++ b/app/code/Magento/CatalogSearch/Model/Adapter/Mysql/Aggregation/DataProvider.php @@ -16,6 +16,8 @@ use Magento\Framework\DB\Select; use Magento\Framework\Search\Adapter\Mysql\Aggregation\DataProviderInterface; use Magento\Framework\Search\Request\BucketInterface; +use Magento\Framework\App\ObjectManager; +use Magento\Indexer\Model\ResourceModel\FrontendResource; /** * @SuppressWarnings(PHPMD.CouplingBetweenObjects) @@ -47,23 +49,33 @@ class DataProvider implements DataProviderInterface */ private $connection; + /** + * @var FrontendResource + */ + private $indexerFrontendResource; + /** * @param Config $eavConfig * @param ResourceConnection $resource * @param ScopeResolverInterface $scopeResolver * @param Session $customerSession + * @param FrontendResource $indexerFrontendResource */ public function __construct( Config $eavConfig, ResourceConnection $resource, ScopeResolverInterface $scopeResolver, - Session $customerSession + Session $customerSession, + FrontendResource $indexerFrontendResource = null ) { $this->eavConfig = $eavConfig; $this->resource = $resource; $this->connection = $resource->getConnection(); $this->scopeResolver = $scopeResolver; $this->customerSession = $customerSession; + $this->indexerFrontendResource = $indexerFrontendResource ?: ObjectManager::getInstance()->get( + Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\FrontendResource::class + ); } /** @@ -92,7 +104,7 @@ public function getDataSet( if (!$store instanceof \Magento\Store\Model\Store) { throw new \RuntimeException('Illegal scope resolved'); } - $table = $this->resource->getTableName('catalog_product_index_price'); + $table = $this->indexerFrontendResource->getMainTable(); $select->from(['main_table' => $table], null) ->columns([BucketInterface::FIELD_VALUE => 'main_table.min_price']) ->where('main_table.customer_group_id = ?', $this->customerSession->getCustomerGroupId()) diff --git a/app/code/Magento/CatalogSearch/Model/Adapter/Mysql/Dynamic/DataProvider.php b/app/code/Magento/CatalogSearch/Model/Adapter/Mysql/Dynamic/DataProvider.php index a82bc173253f4..1a9644f7ec372 100644 --- a/app/code/Magento/CatalogSearch/Model/Adapter/Mysql/Dynamic/DataProvider.php +++ b/app/code/Magento/CatalogSearch/Model/Adapter/Mysql/Dynamic/DataProvider.php @@ -15,6 +15,8 @@ use Magento\Framework\Search\Dynamic\DataProviderInterface; use Magento\Framework\Search\Dynamic\IntervalFactory; use Magento\Framework\Search\Request\BucketInterface; +use Magento\Framework\App\ObjectManager; +use Magento\Indexer\Model\ResourceModel\FrontendResource; /** * @SuppressWarnings(PHPMD.CouplingBetweenObjects) @@ -51,19 +53,26 @@ class DataProvider implements DataProviderInterface */ private $connection; + /** + * @var FrontendResource + */ + private $indexerFrontendResource; + /** * @param ResourceConnection $resource * @param Range $range * @param Session $customerSession * @param MysqlDataProviderInterface $dataProvider * @param IntervalFactory $intervalFactory + * @param FrontendResource $indexerFrontendResource */ public function __construct( ResourceConnection $resource, Range $range, Session $customerSession, MysqlDataProviderInterface $dataProvider, - IntervalFactory $intervalFactory + IntervalFactory $intervalFactory, + FrontendResource $indexerFrontendResource = null ) { $this->resource = $resource; $this->connection = $resource->getConnection(); @@ -71,6 +80,9 @@ public function __construct( $this->customerSession = $customerSession; $this->dataProvider = $dataProvider; $this->intervalFactory = $intervalFactory; + $this->indexerFrontendResource = $indexerFrontendResource ?: ObjectManager::getInstance()->get( + \Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\FrontendResource::class + ); } /** @@ -95,7 +107,7 @@ public function getAggregations(\Magento\Framework\Search\Dynamic\EntityStorage $select = $this->getSelect(); - $tableName = $this->resource->getTableName('catalog_product_index_price'); + $tableName = $this->indexerFrontendResource->getMainTable(); /** @var Table $table */ $table = $entityStorage->getSource(); $select->from(['main_table' => $tableName], []) diff --git a/app/code/Magento/CatalogSearch/Model/ResourceModel/Advanced/Collection.php b/app/code/Magento/CatalogSearch/Model/ResourceModel/Advanced/Collection.php index 75f3cc4a196f9..3cc383eae6d25 100644 --- a/app/code/Magento/CatalogSearch/Model/ResourceModel/Advanced/Collection.php +++ b/app/code/Magento/CatalogSearch/Model/ResourceModel/Advanced/Collection.php @@ -84,6 +84,7 @@ class Collection extends \Magento\Catalog\Model\ResourceModel\Product\Collection * @param SearchResultFactory|null $searchResultFactory * @param ProductLimitationFactory|null $productLimitationFactory * @param MetadataPool|null $metadataPool + * @param \Magento\Indexer\Model\ResourceModel\FrontendResource $indexerFrontendResource * * @SuppressWarnings(PHPMD.ExcessiveParameterList) */ @@ -113,7 +114,8 @@ public function __construct( \Magento\Framework\DB\Adapter\AdapterInterface $connection = null, SearchResultFactory $searchResultFactory = null, ProductLimitationFactory $productLimitationFactory = null, - MetadataPool $metadataPool = null + MetadataPool $metadataPool = null, + \Magento\Indexer\Model\ResourceModel\FrontendResource $indexerFrontendResource = null ) { $this->requestBuilder = $requestBuilder; $this->searchEngine = $searchEngine; @@ -144,7 +146,8 @@ public function __construct( $groupManagement, $connection, $productLimitationFactory, - $metadataPool + $metadataPool, + $indexerFrontendResource ); } diff --git a/app/code/Magento/CatalogSearch/Model/ResourceModel/Fulltext/Collection.php b/app/code/Magento/CatalogSearch/Model/ResourceModel/Fulltext/Collection.php index d478b2dcf941d..18f340f6e3c4d 100644 --- a/app/code/Magento/CatalogSearch/Model/ResourceModel/Fulltext/Collection.php +++ b/app/code/Magento/CatalogSearch/Model/ResourceModel/Fulltext/Collection.php @@ -126,6 +126,7 @@ class Collection extends \Magento\Catalog\Model\ResourceModel\Product\Collection * @param SearchResultFactory|null $searchResultFactory * @param ProductLimitationFactory|null $productLimitationFactory * @param MetadataPool|null $metadataPool + * @param \Magento\Indexer\Model\ResourceModel\FrontendResource $indexerFrontendResource * * @SuppressWarnings(PHPMD.ExcessiveParameterList) */ @@ -157,7 +158,8 @@ public function __construct( $searchRequestName = 'catalog_view_container', SearchResultFactory $searchResultFactory = null, ProductLimitationFactory $productLimitationFactory = null, - MetadataPool $metadataPool = null + MetadataPool $metadataPool = null, + \Magento\Indexer\Model\ResourceModel\FrontendResource $indexerFrontendResource = null ) { $this->queryFactory = $catalogSearchData; if ($searchResultFactory === null) { @@ -186,7 +188,8 @@ public function __construct( $groupManagement, $connection, $productLimitationFactory, - $metadataPool + $metadataPool, + $indexerFrontendResource ); $this->requestBuilder = $requestBuilder; $this->searchEngine = $searchEngine; diff --git a/app/code/Magento/CatalogSearch/Model/Search/FilterMapper/ExclusionStrategy.php b/app/code/Magento/CatalogSearch/Model/Search/FilterMapper/ExclusionStrategy.php index 9575f6efe51d4..aab8b57208f6c 100644 --- a/app/code/Magento/CatalogSearch/Model/Search/FilterMapper/ExclusionStrategy.php +++ b/app/code/Magento/CatalogSearch/Model/Search/FilterMapper/ExclusionStrategy.php @@ -7,6 +7,8 @@ namespace Magento\CatalogSearch\Model\Search\FilterMapper; use Magento\CatalogSearch\Model\Adapter\Mysql\Filter\AliasResolver; +use Magento\Framework\App\ObjectManager; +use Magento\Indexer\Model\ResourceModel\FrontendResource; /** * Strategy which processes exclusions from general rules @@ -28,19 +30,29 @@ class ExclusionStrategy implements FilterStrategyInterface */ private $storeManager; + /** + * @var FrontendResource + */ + private $indexerFrontendResource; + /** * @param \Magento\Framework\App\ResourceConnection $resourceConnection * @param \Magento\Store\Model\StoreManagerInterface $storeManager * @param AliasResolver $aliasResolver + * @param FrontendResource $indexerFrontendResource */ public function __construct( \Magento\Framework\App\ResourceConnection $resourceConnection, \Magento\Store\Model\StoreManagerInterface $storeManager, - AliasResolver $aliasResolver + AliasResolver $aliasResolver, + FrontendResource $indexerFrontendResource = null ) { $this->resourceConnection = $resourceConnection; $this->storeManager = $storeManager; $this->aliasResolver = $aliasResolver; + $this->indexerFrontendResource = $indexerFrontendResource ?: ObjectManager::getInstance()->get( + \Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\FrontendResource::class + ); } /** @@ -52,9 +64,10 @@ public function apply( ) { $isApplied = false; $field = $filter->getField(); + if ('price' === $field) { $alias = $this->aliasResolver->getAlias($filter); - $tableName = $this->resourceConnection->getTableName('catalog_product_index_price'); + $tableName = $this->indexerFrontendResource->getMainTable(); $select->joinInner( [ $alias => $tableName diff --git a/app/code/Magento/CatalogSearch/Test/Unit/Model/Adapter/Mysql/Aggregation/DataProviderTest.php b/app/code/Magento/CatalogSearch/Test/Unit/Model/Adapter/Mysql/Aggregation/DataProviderTest.php new file mode 100644 index 0000000000000..8637c2097d02f --- /dev/null +++ b/app/code/Magento/CatalogSearch/Test/Unit/Model/Adapter/Mysql/Aggregation/DataProviderTest.php @@ -0,0 +1,114 @@ +eavConfigMock = $this->getMock(Config::class, [], [], '', false); + $this->resourceConnectionMock = $this->getMock(ResourceConnection::class, [], [], '', false); + $this->scopeResolverMock = $this->getMock(ScopeResolverInterface::class); + $this->sessionMock = $this->getMock(Session::class, [], [], '', false); + $this->frontendResourceMock = $this->getMock(FrontendResource::class, [], [], '', false); + $this->adapterMock = $this->getMock(AdapterInterface::class); + $this->resourceConnectionMock->expects($this->once())->method('getConnection')->willReturn($this->adapterMock); + + $this->model = new DataProvider( + $this->eavConfigMock, + $this->resourceConnectionMock, + $this->scopeResolverMock, + $this->sessionMock, + $this->frontendResourceMock + ); + } + + public function testGetDataSetUsesFrontendPriceIndexerTableIfAttributeIsPrice() + { + $storeId = 1; + $attributeCode = 'price'; + + $scopeMock = $this->getMock(Store::class, [], [], '', false); + $scopeMock->expects($this->any())->method('getId')->willReturn($storeId); + $dimensionMock = $this->getMock(Dimension::class, [], [], '', false); + $dimensionMock->expects($this->any())->method('getValue')->willReturn($storeId); + $this->scopeResolverMock->expects($this->any())->method('getScope')->with($storeId)->willReturn($scopeMock); + + $bucketMock = $this->getMock(BucketInterface::class); + $bucketMock->expects($this->once())->method('getField')->willReturn($attributeCode); + $attributeMock = $this->getMock(Attribute::class, [], [], '', false); + $attributeMock->expects($this->any())->method('getAttributeCode')->willReturn($attributeCode); + $this->eavConfigMock->expects($this->once()) + ->method('getAttribute')->with(Product::ENTITY, $attributeCode) + ->willReturn($attributeMock); + + $selectMock = $this->getMock(Select::class, [], [], '', false); + $selectMock->expects($this->any())->method('from')->willReturnSelf(); + $selectMock->expects($this->any())->method('where')->willReturnSelf(); + $selectMock->expects($this->any())->method('columns')->willReturnSelf(); + $this->adapterMock->expects($this->once())->method('select')->willReturn($selectMock); + $tableMock = $this->getMock(Table::class); + + // verify that frontend indexer table is used + $this->frontendResourceMock->expects($this->once())->method('getMainTable'); + + $this->model->getDataSet($bucketMock, ['scope' => $dimensionMock], $tableMock); + } +} diff --git a/app/code/Magento/CatalogSearch/Test/Unit/Model/Adapter/Mysql/Dynamic/DataProviderTest.php b/app/code/Magento/CatalogSearch/Test/Unit/Model/Adapter/Mysql/Dynamic/DataProviderTest.php new file mode 100644 index 0000000000000..9afc262a994e5 --- /dev/null +++ b/app/code/Magento/CatalogSearch/Test/Unit/Model/Adapter/Mysql/Dynamic/DataProviderTest.php @@ -0,0 +1,101 @@ +resourceConnectionMock = $this->getMock(ResourceConnection::class, [], [], '', false); + $this->sessionMock = $this->getMock(Session::class, [], [], '', false); + $this->frontendResourceMock = $this->getMock(FrontendResource::class, [], [], '', false); + $this->adapterMock = $this->getMock(AdapterInterface::class); + $this->resourceConnectionMock->expects($this->once())->method('getConnection')->willReturn($this->adapterMock); + $this->rangeMock = $this->getMock(Range::class, [], [], '', false); + $this->mysqlDataProviderMock = $this->getMock(DataProviderInterface::class); + $this->intervalFactoryMock = $this->getMock(IntervalFactory::class, [], [], '', false); + + $this->model = new DataProvider( + $this->resourceConnectionMock, + $this->rangeMock, + $this->sessionMock, + $this->mysqlDataProviderMock, + $this->intervalFactoryMock, + $this->frontendResourceMock + ); + } + + public function testGetAggregationsUsesFrontendPriceIndexerTable() + { + $selectMock = $this->getMock(Select::class, [], [], '', false); + $selectMock->expects($this->any())->method('from')->willReturnSelf(); + $selectMock->expects($this->any())->method('joinInner')->willReturnSelf(); + $selectMock->expects($this->any())->method('columns')->willReturnSelf(); + $this->adapterMock->expects($this->once())->method('select')->willReturn($selectMock); + $tableMock = $this->getMock(Table::class, [], [], '', false); + + $entityStorageMock = $this->getMock(EntityStorage::class, [], [], '', false); + $entityStorageMock->expects($this->any())->method('getSource')->willReturn($tableMock); + + // verify that frontend indexer table is used + $this->frontendResourceMock->expects($this->once())->method('getMainTable'); + + $this->model->getAggregations($entityStorageMock); + } +} diff --git a/app/code/Magento/CatalogSearch/Test/Unit/Model/Search/FilterMapper/ExclusionStrategyTest.php b/app/code/Magento/CatalogSearch/Test/Unit/Model/Search/FilterMapper/ExclusionStrategyTest.php new file mode 100644 index 0000000000000..a5d080324c464 --- /dev/null +++ b/app/code/Magento/CatalogSearch/Test/Unit/Model/Search/FilterMapper/ExclusionStrategyTest.php @@ -0,0 +1,88 @@ +resourceConnectionMock = $this->getMock(ResourceConnection::class, [], [], '', false); + + $this->frontendResourceMock = $this->getMock(FrontendResource::class, [], [], '', false); + $this->adapterMock = $this->getMock(AdapterInterface::class); + $this->resourceConnectionMock->expects($this->any())->method('getConnection')->willReturn($this->adapterMock); + $this->storeManagerMock = $this->getMock(StoreManagerInterface::class); + $this->aliasResolverMock = $this->getMock(AliasResolver::class, [], [], '', false); + + $this->model = new ExclusionStrategy( + $this->resourceConnectionMock, + $this->storeManagerMock, + $this->aliasResolverMock, + $this->frontendResourceMock + ); + } + + public function testApplyUsesFrontendPriceIndexerTableIfAttributeCodeIsPrice() + { + $attributeCode = 'price'; + $websiteId = 1; + $selectMock = $this->getMock(Select::class, [], [], '', false); + $selectMock->expects($this->any())->method('joinInner')->willReturnSelf(); + + $searchFilterMock = $this->getMock(Term::class, [], [], '', false); + $searchFilterMock->expects($this->any())->method('getField')->willReturn($attributeCode); + + $websiteMock = $this->getMock(WebsiteInterface::class); + $websiteMock->expects($this->any())->method('getId')->willReturn($websiteId); + $this->storeManagerMock->expects($this->any())->method('getWebsite')->willReturn($websiteMock); + + // verify that frontend indexer table is used + $this->frontendResourceMock->expects($this->once())->method('getMainTable'); + + $this->assertTrue($this->model->apply($searchFilterMock, $selectMock)); + } +} diff --git a/app/code/Magento/CatalogSearch/composer.json b/app/code/Magento/CatalogSearch/composer.json index 313cc99881a31..71c8f6f17bdc7 100644 --- a/app/code/Magento/CatalogSearch/composer.json +++ b/app/code/Magento/CatalogSearch/composer.json @@ -12,6 +12,7 @@ "magento/module-backend": "100.2.*", "magento/module-theme": "100.2.*", "magento/module-catalog-inventory": "100.2.*", + "magento/module-indexer": "100.2.*", "magento/framework": "100.2.*" }, "type": "magento2-module", diff --git a/app/code/Magento/CatalogSearch/etc/di.xml b/app/code/Magento/CatalogSearch/etc/di.xml index 47c4152e92a1f..2c4d5091d070e 100644 --- a/app/code/Magento/CatalogSearch/etc/di.xml +++ b/app/code/Magento/CatalogSearch/etc/di.xml @@ -205,6 +205,7 @@ Magento\Catalog\Model\Layer\Filter\Price\Range\Proxy + Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\FrontendResource @@ -279,4 +280,14 @@ + + + Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\FrontendResource + + + + + Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\FrontendResource + + diff --git a/app/code/Magento/Indexer/Model/Indexer.php b/app/code/Magento/Indexer/Model/Indexer.php index feea37a263664..492ca22e41ef7 100644 --- a/app/code/Magento/Indexer/Model/Indexer.php +++ b/app/code/Magento/Indexer/Model/Indexer.php @@ -59,6 +59,7 @@ class Indexer extends \Magento\Framework\DataObject implements IdxInterface protected $indexersFactory; /** + * Indexer constructor. * @param ConfigInterface $config * @param ActionFactory $actionFactory * @param StructureFactory $structureFactory @@ -410,6 +411,10 @@ public function reindexAll() } try { $this->getActionInstance()->executeFull(); + $tableSuffix = ($state->getTableSuffix() === '') + ? StateInterface::ADDITIONAL_TABLE_SUFFIX + : ''; + $state->setTableSuffix($tableSuffix); $state->setStatus(StateInterface::STATUS_VALID); $state->save(); $this->getView()->resume(); @@ -436,7 +441,7 @@ public function reindexRow($id) /** * Regenerate rows in index by ID list - *5 + * * @param int[] $ids * @return void */ diff --git a/app/code/Magento/Indexer/Model/Indexer/State.php b/app/code/Magento/Indexer/Model/Indexer/State.php index 96e0153bcac16..fa46c7c684206 100644 --- a/app/code/Magento/Indexer/Model/Indexer/State.php +++ b/app/code/Magento/Indexer/Model/Indexer/State.php @@ -121,6 +121,27 @@ public function setStatus($status) return parent::setStatus($status); } + /** + * Get suffix for indexer table. + * + * @return string + */ + public function getTableSuffix() + { + return $this->getData('table_suffix'); + } + + /** + * Get suffix for indexer table. + * + * @param string $tableSuffix + * @return $this + */ + public function setTableSuffix($tableSuffix) + { + return ($this->setData('table_suffix', $tableSuffix)); + } + /** * Processing object before save data * diff --git a/app/code/Magento/Indexer/Model/ResourceModel/FrontendResource.php b/app/code/Magento/Indexer/Model/ResourceModel/FrontendResource.php new file mode 100644 index 0000000000000..f07fc41b718fa --- /dev/null +++ b/app/code/Magento/Indexer/Model/ResourceModel/FrontendResource.php @@ -0,0 +1,80 @@ +indexerId = $indexerId; + $this->indexerBaseTable = $indexerBaseTable; + $this->indexerStateFactory = $indexerStateFactory; + $this->_idFieldName = $idFieldName; + } + + /** + * Retrieve indexer frontend table name. + * The table that is used for read operations only. + * + * @return string + */ + public function getMainTable() + { + $indexerState = $this->indexerStateFactory->create()->loadByIndexer($this->indexerId); + return $this->getTable($this->indexerBaseTable . $indexerState->getTableSuffix()); + } + + /** + * No resource initialization is required for this model. + * @return void + */ + protected function _construct() + { + // nothing to initialize + } +} diff --git a/app/code/Magento/Indexer/Setup/InstallSchema.php b/app/code/Magento/Indexer/Setup/InstallSchema.php index 7afd9381abfc4..a2410f159ac05 100644 --- a/app/code/Magento/Indexer/Setup/InstallSchema.php +++ b/app/code/Magento/Indexer/Setup/InstallSchema.php @@ -65,6 +65,14 @@ public function install(SchemaSetupInterface $setup, ModuleContextInterface $con ['nullable' => false], 'Hash of indexer config' ) + ->addColumn( + 'table_suffix', + \Magento\Framework\DB\Ddl\Table::TYPE_TEXT, + 255, + ['nullable' => false], + ['default' => ''], + 'Suffix for indexer table used in the read operations' + ) ->addIndex( $installer->getIdxName('indexer_state', ['indexer_id']), ['indexer_id'] diff --git a/app/code/Magento/Indexer/Setup/UpgradeSchema.php b/app/code/Magento/Indexer/Setup/UpgradeSchema.php new file mode 100644 index 0000000000000..e8c4c36e9a998 --- /dev/null +++ b/app/code/Magento/Indexer/Setup/UpgradeSchema.php @@ -0,0 +1,53 @@ +startSetup(); + + if (version_compare($context->getVersion(), '2.0.1', '<')) { + $this->addTableSuffixColumn($setup); + } + $setup->endSetup(); + } + + /** + * Add the column 'table_suffix' to the Indexer State table. + * It allows to identify which indexer table is used for data retrieve on storefront. + * + * @param SchemaSetupInterface $setup + * @return void + */ + private function addTableSuffixColumn(SchemaSetupInterface $setup) + { + $connection = $setup->getConnection(); + $connection->addColumn( + $setup->getTable('indexer_state'), + 'table_suffix', + [ + 'type' => \Magento\Framework\DB\Ddl\Table::TYPE_TEXT, + 'length' => 255, + 'nullable' => false, + 'default' => '', + 'comment' => 'Suffix for indexer table used for read operations', + ] + ); + } +} diff --git a/app/code/Magento/Indexer/etc/module.xml b/app/code/Magento/Indexer/etc/module.xml index a2beac159902b..93fc95c7cce06 100644 --- a/app/code/Magento/Indexer/etc/module.xml +++ b/app/code/Magento/Indexer/etc/module.xml @@ -6,7 +6,7 @@ */ --> - + diff --git a/lib/internal/Magento/Framework/Indexer/StateInterface.php b/lib/internal/Magento/Framework/Indexer/StateInterface.php index 7f501aed3bc50..830d9a3594b6a 100644 --- a/lib/internal/Magento/Framework/Indexer/StateInterface.php +++ b/lib/internal/Magento/Framework/Indexer/StateInterface.php @@ -14,6 +14,8 @@ interface StateInterface const STATUS_VALID = 'valid'; const STATUS_INVALID = 'invalid'; + const ADDITIONAL_TABLE_SUFFIX = '_replica'; + /** * Return indexer id * From 03d6eb4142ebfc17c186b5e80be68ffcd566bc5a Mon Sep 17 00:00:00 2001 From: Stanislav Idolov Date: Tue, 21 Feb 2017 16:54:04 +0200 Subject: [PATCH 05/43] MAGETWO-64193: Integrate the batch algorithm to DefaultPrice indexer --- .../Catalog/Model/Indexer/Product/Price/Action/Full.php | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/app/code/Magento/Catalog/Model/Indexer/Product/Price/Action/Full.php b/app/code/Magento/Catalog/Model/Indexer/Product/Price/Action/Full.php index acfe8900598e1..1ef4e051ea1cb 100644 --- a/app/code/Magento/Catalog/Model/Indexer/Product/Price/Action/Full.php +++ b/app/code/Magento/Catalog/Model/Indexer/Product/Price/Action/Full.php @@ -102,8 +102,6 @@ public function execute($ids = null) $entityMetadata = $this->metadataPool->getMetadata(\Magento\Catalog\Api\Data\ProductInterface::class); - $notComposite = []; - /** @var \Magento\Catalog\Model\ResourceModel\Product\Indexer\AbstractIndexer $indexer */ foreach ($this->getTypeIndexers() as $indexer) { @@ -131,9 +129,7 @@ public function execute($ids = null) if (!empty($entityIds)) { $this->_emptyTable($this->_defaultIndexerResource->getIdxTable()); if ($indexer->getIsComposite()) { - $this->_copyRelationIndexData($entityIds, $notComposite); - } else { - $notComposite = array_merge($notComposite, $entityIds); + $this->_copyRelationIndexData($entityIds); } $this->_prepareTierPriceIndex($entityIds); // Reindex entities by id From f7a3ca1121b3fee9405bff127c3ccb1c0ad81541 Mon Sep 17 00:00:00 2001 From: Stanislav Idolov Date: Wed, 22 Feb 2017 14:41:08 +0200 Subject: [PATCH 06/43] MAGETWO-64466: [Indexer optimizations] Indexation process in non-locking way for price indexer --- .../Indexer/Product/Price/Action/Full.php | 5 +- .../Product/Indexer/Price/DefaultPrice.php | 2 +- .../Magento/Indexer/Model/Indexer/State.php | 2 +- .../ResourceModel/FrontendResourceTest.php | 98 +++++++++++++++++++ 4 files changed, 104 insertions(+), 3 deletions(-) create mode 100644 app/code/Magento/Indexer/Test/Unit/Model/ResourceModel/FrontendResourceTest.php diff --git a/app/code/Magento/Catalog/Model/Indexer/Product/Price/Action/Full.php b/app/code/Magento/Catalog/Model/Indexer/Product/Price/Action/Full.php index 98a19b976e043..3c7babd96b7a0 100644 --- a/app/code/Magento/Catalog/Model/Indexer/Product/Price/Action/Full.php +++ b/app/code/Magento/Catalog/Model/Indexer/Product/Price/Action/Full.php @@ -40,6 +40,7 @@ class Full extends \Magento\Catalog\Model\Indexer\Product\Price\AbstractAction * @param \Magento\Catalog\Model\Product\Type $catalogProductType * @param \Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\Factory $indexerPriceFactory * @param \Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\DefaultPrice $defaultIndexerResource + * @param \Magento\Indexer\Model\ResourceModel\FrontendResource|null $indexerFrontendResource * @param \Magento\Framework\EntityManager\MetadataPool|null $metadataPool * @param \Magento\Framework\Indexer\BatchSizeCalculatorInterface|null $batchSizeCalculator * @param \Magento\Framework\Indexer\BatchProviderInterface|null $batchProvider @@ -55,6 +56,7 @@ public function __construct( \Magento\Catalog\Model\Product\Type $catalogProductType, \Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\Factory $indexerPriceFactory, \Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\DefaultPrice $defaultIndexerResource, + \Magento\Indexer\Model\ResourceModel\FrontendResource $indexerFrontendResource = null, \Magento\Framework\EntityManager\MetadataPool $metadataPool = null, \Magento\Framework\Indexer\BatchSizeCalculatorInterface $batchSizeCalculator = null, \Magento\Framework\Indexer\BatchProviderInterface $batchProvider = null, @@ -79,7 +81,8 @@ public function __construct( $dateTime, $catalogProductType, $indexerPriceFactory, - $defaultIndexerResource + $defaultIndexerResource, + $indexerFrontendResource ); } diff --git a/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/DefaultPrice.php b/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/DefaultPrice.php index 425900ed316a3..fea611e441ad0 100644 --- a/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/DefaultPrice.php +++ b/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/DefaultPrice.php @@ -710,7 +710,7 @@ protected function hasEntity() /** * @inheritdoc - * Returns main table name in depends of the suffix stored in the 'indexer_state' table + * Returns main table name based on the suffix stored in the 'indexer_state' table * * @return string */ diff --git a/app/code/Magento/Indexer/Model/Indexer/State.php b/app/code/Magento/Indexer/Model/Indexer/State.php index fa46c7c684206..c71ef88d11742 100644 --- a/app/code/Magento/Indexer/Model/Indexer/State.php +++ b/app/code/Magento/Indexer/Model/Indexer/State.php @@ -139,7 +139,7 @@ public function getTableSuffix() */ public function setTableSuffix($tableSuffix) { - return ($this->setData('table_suffix', $tableSuffix)); + return $this->setData('table_suffix', $tableSuffix); } /** diff --git a/app/code/Magento/Indexer/Test/Unit/Model/ResourceModel/FrontendResourceTest.php b/app/code/Magento/Indexer/Test/Unit/Model/ResourceModel/FrontendResourceTest.php new file mode 100644 index 0000000000000..1a01a631f8ff9 --- /dev/null +++ b/app/code/Magento/Indexer/Test/Unit/Model/ResourceModel/FrontendResourceTest.php @@ -0,0 +1,98 @@ +indexerId = 'indexer_id'; + $this->indexerBaseTable = 'indexer_base_table'; + $this->connectionName = 'connectionName'; + + $this->stateFactoryMock = $this->getMock( + \Magento\Indexer\Model\Indexer\StateFactory::class, + ['create'], + [], + '', + false + ); + + $this->resourceMock = $this->getMock(\Magento\Framework\App\ResourceConnection::class, [], [], '', false); + $contextMock = $this->getMock(\Magento\Framework\Model\ResourceModel\Db\Context::class, [], [], '', false); + $contextMock->expects($this->once())->method('getResources')->willReturn($this->resourceMock); + + $this->model = new \Magento\Indexer\Model\ResourceModel\FrontendResource( + $contextMock, + $this->indexerId, + $this->indexerBaseTable, + 'idFieldName', + $this->stateFactoryMock, + $this->connectionName + ); + } + + public function testGetMainTable() + { + $mainTable = $this->indexerBaseTable . StateInterface::ADDITIONAL_TABLE_SUFFIX; + $stateMock = $this->getMock( + \Magento\Indexer\Model\Indexer\State::class, + ['loadByIndexer', 'getTableSuffix'], + [], + '', + false + ); + $this->stateFactoryMock->expects($this->once())->method('create')->willReturn($stateMock); + + $stateMock->expects($this->once())->method('loadByIndexer')->with($this->indexerId)->willReturnSelf(); + $stateMock->expects($this->once()) + ->method('getTableSuffix') + ->willReturn(StateInterface::ADDITIONAL_TABLE_SUFFIX); + + $this->resourceMock->expects($this->once()) + ->method('getTableName') + ->with($mainTable, $this->connectionName) + ->willReturn($mainTable); + + $this->assertEquals( + $mainTable, + $this->model->getMainTable() + ); + } +} From 290e9322f1fe772654a18208e0d5cdef8ee06826 Mon Sep 17 00:00:00 2001 From: Iryna Lagno Date: Fri, 24 Feb 2017 14:32:07 +0200 Subject: [PATCH 07/43] MAGETWO-64184: [Indexer optimizations] Batch data processing for price indexer - remove not relevant integration tests --- .../FixedBundlePriceCalculatorTest.php | 33 ++------ ...edBundleWithSpecialPriceCalculatorTest.php | 78 ++++++------------- ...FixedBundleWithTierPriceCalculatorTest.php | 78 ++++++------------- .../Price/SimpleWithOptionsFinalPriceTest.php | 60 -------------- 4 files changed, 56 insertions(+), 193 deletions(-) delete mode 100644 dev/tests/integration/testsuite/Magento/Catalog/Model/Indexer/Product/Price/SimpleWithOptionsFinalPriceTest.php diff --git a/dev/tests/integration/testsuite/Magento/Bundle/Model/Product/FixedBundlePriceCalculatorTest.php b/dev/tests/integration/testsuite/Magento/Bundle/Model/Product/FixedBundlePriceCalculatorTest.php index 342a7a4ccb0e7..f65bd37e0c8e4 100644 --- a/dev/tests/integration/testsuite/Magento/Bundle/Model/Product/FixedBundlePriceCalculatorTest.php +++ b/dev/tests/integration/testsuite/Magento/Bundle/Model/Product/FixedBundlePriceCalculatorTest.php @@ -46,10 +46,6 @@ public function testPriceForFixedBundle(array $strategyModifiers, array $expecte 'Failed to check maximal price on product' ); $this->assertEquals($expectedResults['indexerMinimalPrice'], $priceInfoFromIndexer->getMinimalPrice()); - //This verification is skipped due to MAGETWO-64406, so in some cases 'indexerMaximumPrice' key was commented. - if (isset($expectedResults['indexerMaximumPrice'])) { - $this->assertEquals($expectedResults['indexerMaximumPrice'], $priceInfoFromIndexer->getMaxPrice()); - } } /** @@ -85,11 +81,6 @@ public function testPriceForFixedBundleInWebsiteScope(array $strategyModifiers, 'Failed to check maximal price on product' ); $this->assertEquals($expectedResults['indexerMinimalPrice'], $priceInfoFromIndexer->getMinimalPrice()); - - //This verification is skipped due to MAGETWO-64406, so in some cases 'indexerMaximumPrice' key was commented. - if (isset($expectedResults['indexerMaximumPrice'])) { - $this->assertEquals($expectedResults['indexerMaximumPrice'], $priceInfoFromIndexer->getMaxPrice()); - } } /** @@ -107,8 +98,7 @@ public function getTestCases() 'minimalPrice' => 120, // 110 + 10 (sum of simple price) 'maximalPrice' => 120, - 'indexerMinimalPrice' => 120, - 'indexerMaximumPrice' => 120 + 'indexerMinimalPrice' => 120 ] ], @@ -119,8 +109,7 @@ public function getTestCases() 'minimalPrice' => 120, // 110 + (3 * 10) + (2 * 10) + 10 'maximalPrice' => 170, - 'indexerMinimalPrice' => 120, - 'indexerMaximumPrice' => 170 + 'indexerMinimalPrice' => 120 ] ], @@ -131,8 +120,7 @@ public function getTestCases() 'minimalPrice' => 120, // 110 + 60 'maximalPrice' => 170, - 'indexerMinimalPrice' => 120, - 'indexerMaximumPrice' => 170 + 'indexerMinimalPrice' => 120 ] ], @@ -143,8 +131,7 @@ public function getTestCases() 'minimalPrice' => 120, // 110 + 30 'maximalPrice' => 140, - 'indexerMinimalPrice' => 120, - 'indexerMaximumPrice' => 140 + 'indexerMinimalPrice' => 120 ] ], @@ -162,8 +149,7 @@ public function getTestCases() // 110 + 1 * 20 + 100 'maximalPrice' => 230, - 'indexerMinimalPrice' => 130, - //'indexerMaximumPrice' => 230 + 'indexerMinimalPrice' => 130 ] ], @@ -181,8 +167,7 @@ public function getTestCases() // 110 + 110 * 0.2 + 110 * 1 'maximalPrice' => 242, - 'indexerMinimalPrice' => 132, - //'indexerMaximumPrice' => 242 + 'indexerMinimalPrice' => 132 ] ], @@ -200,8 +185,7 @@ public function getTestCases() // 110 + 1 * 20 + 110 * 1 'maximalPrice' => 240, - 'indexerMinimalPrice' => 130, - //'indexerMaximumPrice' => 240 + 'indexerMinimalPrice' => 130 ] ], @@ -219,8 +203,7 @@ public function getTestCases() // 110 + 110 * 0.2 + 100 'maximalPrice' => 232, - 'indexerMinimalPrice' => 132, - //'indexerMaximumPrice' => 232 + 'indexerMinimalPrice' => 132 ] ], ]; diff --git a/dev/tests/integration/testsuite/Magento/Bundle/Model/Product/FixedBundleWithSpecialPriceCalculatorTest.php b/dev/tests/integration/testsuite/Magento/Bundle/Model/Product/FixedBundleWithSpecialPriceCalculatorTest.php index 4e422fa4e6d1b..68c77c37ca007 100644 --- a/dev/tests/integration/testsuite/Magento/Bundle/Model/Product/FixedBundleWithSpecialPriceCalculatorTest.php +++ b/dev/tests/integration/testsuite/Magento/Bundle/Model/Product/FixedBundleWithSpecialPriceCalculatorTest.php @@ -46,11 +46,6 @@ public function testPriceForFixedBundle(array $strategyModifiers, array $expecte 'Failed to check maximal price on product' ); $this->assertEquals($expectedResults['indexerMinimalPrice'], $priceInfoFromIndexer->getMinimalPrice()); - - //This verification is skipped due to MAGETWO-64406, so in some cases 'indexerMaximumPrice' key was commented. - if (isset($expectedResults['indexerMaximumPrice'])) { - $this->assertEquals($expectedResults['indexerMaximumPrice'], $priceInfoFromIndexer->getMaxPrice()); - } } /** @@ -72,8 +67,7 @@ public function getTestCases() // 110 * 0.5 'maximalPrice' => 55, - 'indexerMinimalPrice' => null, - 'indexerMaximumPrice' => null + 'indexerMinimalPrice' => null ] ], @@ -91,8 +85,7 @@ public function getTestCases() // 0.5 * (110 + 1 * 20) + 100 'maximalPrice' => 165, - 'indexerMinimalPrice' => 130, - //'indexerMaximumPrice' => 230 + 'indexerMinimalPrice' => 130 ] ], @@ -110,8 +103,7 @@ public function getTestCases() // 0.5 * (110 + 110 * 0.2 + 110 * 1) 'maximalPrice' => 121, - 'indexerMinimalPrice' => 132, - //'indexerMaximumPrice' => 242 + 'indexerMinimalPrice' => 132 ] ], @@ -129,8 +121,7 @@ public function getTestCases() // 0.5 * (110 + 1 * 20 + 110 * 1) 'maximalPrice' => 120, - 'indexerMinimalPrice' => 130, - //'indexerMaximumPrice' => 240 + 'indexerMinimalPrice' => 130 ] ], @@ -148,8 +139,7 @@ public function getTestCases() // 0.5 * (110 + 110 * 0.2) + 100 'maximalPrice' => 166, - 'indexerMinimalPrice' => 132, - //'indexerMaximumPrice' => 232 + 'indexerMinimalPrice' => 132 ] ], @@ -167,8 +157,7 @@ public function getTestCases() // 0.5 * (110 + 2 * 20) + 100 'maximalPrice' => 175, - 'indexerMinimalPrice' => 150, - //'indexerMaximumPrice' => 250 + 'indexerMinimalPrice' => 150 ] ], @@ -186,8 +175,7 @@ public function getTestCases() // 0.5 * (110 + 2 * 110 * 0.2 + 1 * 110) 'maximalPrice' => 132, - 'indexerMinimalPrice' => 154, - //'indexerMaximumPrice' => 264 + 'indexerMinimalPrice' => 154 ] ], @@ -205,8 +193,7 @@ public function getTestCases() // 0.5 * (110 + 2 * 20 + 1 * 110) 'maximalPrice' => 130, - 'indexerMinimalPrice' => 150, - //'indexerMaximumPrice' => 260 + 'indexerMinimalPrice' => 150 ] ], @@ -224,8 +211,7 @@ public function getTestCases() // 0.5 * (110 + 2 * 0.2 * 110) + 100 'maximalPrice' => 177, - 'indexerMinimalPrice' => 154, - //'indexerMaximumPrice' => 254 + 'indexerMinimalPrice' => 154 ] ], @@ -243,8 +229,7 @@ public function getTestCases() // 0.5 * (110 + 3 * 10 + 1 * 40) + 100 'maximalPrice' => 190, - 'indexerMinimalPrice' => 140, - //'indexerMaximumPrice' => 280 + 'indexerMinimalPrice' => 140 ] ], @@ -262,8 +247,7 @@ public function getTestCases() // 0.5 * (110 + 3 * 110 * 0.1 + 1 * 110 * 0.4 + 110 * 1) 'maximalPrice' => 148.5, - 'indexerMinimalPrice' => 143, - //'indexerMaximumPrice' => 297 + 'indexerMinimalPrice' => 143 ] ], @@ -281,8 +265,7 @@ public function getTestCases() // 0.5 * (110 + 3 * 10 + 1 * 40 + 1 * 110) 'maximalPrice' => 145, - 'indexerMinimalPrice' => 140, - //'indexerMaximumPrice' => 290 + 'indexerMinimalPrice' => 140 ] ], @@ -300,8 +283,7 @@ public function getTestCases() // 0.5 * (110 + 3 * 110 * 0.1 + 1 * 110 * 0.4) + 100 'maximalPrice' => 193.5, - 'indexerMinimalPrice' => 143, - //'indexerMaximumPrice' => 287 + 'indexerMinimalPrice' => 143 ] ], @@ -319,8 +301,7 @@ public function getTestCases() // 0.5 * (110 + 1 * 40 + 3 * 15) + 100 'maximalPrice' => 197.5, - 'indexerMinimalPrice' => 150, - //'indexerMaximumPrice' => 295 + 'indexerMinimalPrice' => 150 ] ], @@ -338,8 +319,7 @@ public function getTestCases() // 0.5 * (110 + 1 * 110 * 0.4 + 3 * 110 * 0.15 + 110 * 1) 'maximalPrice' => 156.75, - 'indexerMinimalPrice' => 154, - //'indexerMaximumPrice' => 313.5 + 'indexerMinimalPrice' => 154 ] ], @@ -357,8 +337,7 @@ public function getTestCases() // 0.5 * (110 + 1 * 40 + 3 * 15 + 1 * 110) 'maximalPrice' => 152.5, - 'indexerMinimalPrice' => 150, - //'indexerMaximumPrice' => 305 + 'indexerMinimalPrice' => 150 ] ], @@ -376,8 +355,7 @@ public function getTestCases() // 0.5 * (110 + 1 * 110 * 0.4 + 3 * 110 * 0.15) + 100 'maximalPrice' => 201.75, - 'indexerMinimalPrice' => 154, - //'indexerMaximumPrice' => 303.5 + 'indexerMinimalPrice' => 154 ] ], @@ -395,8 +373,7 @@ public function getTestCases() // 0.5 * (110 + 3 * 15) + 100 'maximalPrice' => 177.5, - 'indexerMinimalPrice' => 150, - //'indexerMaximumPrice' => 255 + 'indexerMinimalPrice' => 150 ] ], @@ -414,8 +391,7 @@ public function getTestCases() // 0.5 * (110 + 3 * 110 * 0.15 + 1 * 110) 'maximalPrice' => 134.75, - 'indexerMinimalPrice' => 154, - //'indexerMaximumPrice' => 269.5 + 'indexerMinimalPrice' => 154 ] ], @@ -433,8 +409,7 @@ public function getTestCases() // 0.5 * (110 + 3 * 15 + 110 * 1) 'maximalPrice' => 132.5, - 'indexerMinimalPrice' => 150, - //'indexerMaximumPrice' => 265 + 'indexerMinimalPrice' => 150 ] ], @@ -452,8 +427,7 @@ public function getTestCases() // 0.5 * (110 + 3 * 110 * 0.15) + 100 'maximalPrice' => 179.75, - 'indexerMinimalPrice' => 154, - //'indexerMaximumPrice' => 259.5 + 'indexerMinimalPrice' => 154 ] ], @@ -471,8 +445,7 @@ public function getTestCases() // 0.5 * (110 + 3 * 15 + 1 * 20 + 3 * 10) + 100 'maximalPrice' => 202.5, - 'indexerMinimalPrice' => 170, - //'indexerMaximumPrice' => 305 + 'indexerMinimalPrice' => 170 ] ], @@ -490,8 +463,7 @@ public function getTestCases() // 0.5 * (110 + 3 * 110 * 0.15 + 1 * 110 * 0.2 + 3 * 110 * 0.1 + 110 * 1) 'maximalPrice' => 162.25, - 'indexerMinimalPrice' => 176, - //'indexerMaximumPrice' => 324.5 + 'indexerMinimalPrice' => 176 ] ], @@ -510,7 +482,6 @@ public function getTestCases() // 0.5 * (110 + 3 * 15 + 1 * 20 + 3 * 10 + 1 * 110) 'maximalPrice' => 157.5, 'indexerMinimalPrice' => 170, - //'indexerMaximumPrice' => 315 ] ], @@ -528,8 +499,7 @@ public function getTestCases() // 0.5 * (110 + 3 * 110 * 0.15 + 1 * 110 * 0.2 + 3 * 110 * 0.1) + 100 'maximalPrice' => 207.25, - 'indexerMinimalPrice' => 176, - //'indexerMaximumPrice' => 314.5 + 'indexerMinimalPrice' => 176 ] ], ]; diff --git a/dev/tests/integration/testsuite/Magento/Bundle/Model/Product/FixedBundleWithTierPriceCalculatorTest.php b/dev/tests/integration/testsuite/Magento/Bundle/Model/Product/FixedBundleWithTierPriceCalculatorTest.php index d5e6c8815a03c..f14c22a87e2ab 100644 --- a/dev/tests/integration/testsuite/Magento/Bundle/Model/Product/FixedBundleWithTierPriceCalculatorTest.php +++ b/dev/tests/integration/testsuite/Magento/Bundle/Model/Product/FixedBundleWithTierPriceCalculatorTest.php @@ -57,11 +57,6 @@ public function testPriceForFixedBundle(array $strategyModifiers, array $expecte 'Failed to check maximal price on product' ); $this->assertEquals($expectedResults['indexerMinimalPrice'], $priceInfoFromIndexer->getMinimalPrice()); - - //This verification is skipped due to MAGETWO-64406, so in some cases 'indexerMaximumPrice' key was commented. - if (isset($expectedResults['indexerMaximumPrice'])) { - $this->assertEquals($expectedResults['indexerMaximumPrice'], $priceInfoFromIndexer->getMaxPrice()); - } } /** @@ -83,8 +78,7 @@ public function getTestCases() // 110 * 0.5 'maximalPrice' => 55, - 'indexerMinimalPrice' => null, - 'indexerMaximumPrice' => null + 'indexerMinimalPrice' => null ] ], @@ -102,8 +96,7 @@ public function getTestCases() // 0.5 * (110 + 1 * 20) + 100 'maximalPrice' => 165, - 'indexerMinimalPrice' => 65, - //'indexerMaximumPrice' => 165 + 'indexerMinimalPrice' => 65 ] ], @@ -121,8 +114,7 @@ public function getTestCases() // 0.5 * (110 + 110 * 0.2 + 110 * 1) 'maximalPrice' => 121, - 'indexerMinimalPrice' => 66, - //'indexerMaximumPrice' => 121 + 'indexerMinimalPrice' => 66 ] ], @@ -140,8 +132,7 @@ public function getTestCases() // 0.5 * (110 + 1 * 20 + 110 * 1) 'maximalPrice' => 120, - 'indexerMinimalPrice' => 65, - //'indexerMaximumPrice' => 120 + 'indexerMinimalPrice' => 65 ] ], @@ -159,8 +150,7 @@ public function getTestCases() // 0.5 * (110 + 110 * 0.2) + 100 'maximalPrice' => 166, - 'indexerMinimalPrice' => 66, - //'indexerMaximumPrice' => 166 + 'indexerMinimalPrice' => 66 ] ], @@ -179,7 +169,6 @@ public function getTestCases() // 0.5 * (110 + 2 * 20) + 100 'maximalPrice' => 175, 'indexerMinimalPrice' => 75, - //'indexerMaximumPrice' => 175 ] ], @@ -197,8 +186,7 @@ public function getTestCases() // 0.5 * (110 + 2 * 110 * 0.2 + 1 * 110) 'maximalPrice' => 132, - 'indexerMinimalPrice' => 77, - //'indexerMaximumPrice' => 132 + 'indexerMinimalPrice' => 77 ] ], @@ -216,8 +204,7 @@ public function getTestCases() // 0.5 * (110 + 2 * 20 + 1 * 110) 'maximalPrice' => 130, - 'indexerMinimalPrice' => 75, - //'indexerMaximumPrice' => 130 + 'indexerMinimalPrice' => 75 ] ], @@ -236,8 +223,7 @@ public function getTestCases() // 0.5 * (110 + 2 * 0.2 * 110) + 100 'maximalPrice' => 177, - 'indexerMinimalPrice' => 77, - //'indexerMaximumPrice' => 177 + 'indexerMinimalPrice' => 77 ] ], @@ -255,8 +241,7 @@ public function getTestCases() // 0.5 * (110 + 3 * 10 + 1 * 40) + 100 'maximalPrice' => 190, - 'indexerMinimalPrice' => 70, - //'indexerMaximumPrice' => 190 + 'indexerMinimalPrice' => 70 ] ], @@ -274,8 +259,7 @@ public function getTestCases() // 0.5 * (110 + 3 * 110 * 0.1 + 1 * 110 * 0.4 + 110 * 1) 'maximalPrice' => 148.5, - 'indexerMinimalPrice' => 71.5, - //'indexerMaximumPrice' => 148.5 + 'indexerMinimalPrice' => 71.5 ] ], @@ -293,8 +277,7 @@ public function getTestCases() // 0.5 * (110 + 3 * 10 + 1 * 40 + 1 * 110) 'maximalPrice' => 145, - 'indexerMinimalPrice' => 70, - //'indexerMaximumPrice' => 145 + 'indexerMinimalPrice' => 70 ] ], @@ -312,8 +295,7 @@ public function getTestCases() // 0.5 * (110 + 3 * 110 * 0.1 + 1 * 110 * 0.4) + 100 'maximalPrice' => 193.5, - 'indexerMinimalPrice' => 71.5, - //'indexerMaximumPrice' => 193.5 + 'indexerMinimalPrice' => 71.5 ] ], @@ -331,8 +313,7 @@ public function getTestCases() // 0.5 * (110 + 1 * 40 + 3 * 15) + 100 'maximalPrice' => 197.5, - 'indexerMinimalPrice' => 75, - //'indexerMaximumPrice' => 197.5 + 'indexerMinimalPrice' => 75 ] ], @@ -350,8 +331,7 @@ public function getTestCases() // 0.5 * (110 + 1 * 110 * 0.4 + 3 * 110 * 0.15 + 110 * 1) 'maximalPrice' => 156.75, - 'indexerMinimalPrice' => 77, - //'indexerMaximumPrice' => 156.75 + 'indexerMinimalPrice' => 77 ] ], @@ -369,8 +349,7 @@ public function getTestCases() // 0.5 * (110 + 1 * 40 + 3 * 15 + 1 * 110) 'maximalPrice' => 152.5, - 'indexerMinimalPrice' => 75, - //'indexerMaximumPrice' => 152.5 + 'indexerMinimalPrice' => 75 ] ], @@ -388,8 +367,7 @@ public function getTestCases() // 0.5 * (110 + 1 * 110 * 0.4 + 3 * 110 * 0.15) + 100 'maximalPrice' => 201.75, - 'indexerMinimalPrice' => 77, - //'indexerMaximumPrice' => 201.75 + 'indexerMinimalPrice' => 77 ] ], @@ -407,8 +385,7 @@ public function getTestCases() // 0.5 * (110 + 3 * 15) + 100 'maximalPrice' => 177.5, - 'indexerMinimalPrice' => 75, - //'indexerMaximumPrice' => 177.5 + 'indexerMinimalPrice' => 75 ] ], @@ -426,8 +403,7 @@ public function getTestCases() // 0.5 * (110 + 3 * 110 * 0.15 + 1 * 110) 'maximalPrice' => 134.75, - 'indexerMinimalPrice' => 77, - //'indexerMaximumPrice' => 134.75 + 'indexerMinimalPrice' => 77 ] ], @@ -445,8 +421,7 @@ public function getTestCases() // 0.5 * (110 + 3 * 15 + 110 * 1) 'maximalPrice' => 132.5, - 'indexerMinimalPrice' => 75, - //'indexerMaximumPrice' => 132.5 + 'indexerMinimalPrice' => 75 ] ], @@ -464,8 +439,7 @@ public function getTestCases() // 0.5 * (110 + 3 * 110 * 0.15) + 100 'maximalPrice' => 179.75, - 'indexerMinimalPrice' => 77, - //'indexerMaximumPrice' => 179.75 + 'indexerMinimalPrice' => 77 ] ], @@ -483,8 +457,7 @@ public function getTestCases() // 0.5 * (110 + 3 * 15 + 1 * 20 + 3 * 10) + 100 'maximalPrice' => 202.5, - 'indexerMinimalPrice' => 85, - //'indexerMaximumPrice' => 202.5 + 'indexerMinimalPrice' => 85 ] ], @@ -502,8 +475,7 @@ public function getTestCases() // 0.5 * (110 + 3 * 110 * 0.15 + 1 * 110 * 0.2 + 3 * 110 * 0.1 + 110 * 1) 'maximalPrice' => 162.25, - 'indexerMinimalPrice' => 88, - //'indexerMaximumPrice' => 162.25 + 'indexerMinimalPrice' => 88 ] ], @@ -521,8 +493,7 @@ public function getTestCases() // 0.5 * (110 + 3 * 15 + 1 * 20 + 3 * 10 + 1 * 110) 'maximalPrice' => 157.5, - 'indexerMinimalPrice' => 85, - //'indexerMaximumPrice' => 157.5 + 'indexerMinimalPrice' => 85 ] ], @@ -540,8 +511,7 @@ public function getTestCases() // 0.5 * (110 + 3 * 110 * 0.15 + 1 * 110 * 0.2 + 3 * 110 * 0.1) + 100 'maximalPrice' => 207.25, - 'indexerMinimalPrice' => 88, - //'indexerMaximumPrice' => 207.25 + 'indexerMinimalPrice' => 88 ] ], ]; diff --git a/dev/tests/integration/testsuite/Magento/Catalog/Model/Indexer/Product/Price/SimpleWithOptionsFinalPriceTest.php b/dev/tests/integration/testsuite/Magento/Catalog/Model/Indexer/Product/Price/SimpleWithOptionsFinalPriceTest.php deleted file mode 100644 index 70dffe25551a8..0000000000000 --- a/dev/tests/integration/testsuite/Magento/Catalog/Model/Indexer/Product/Price/SimpleWithOptionsFinalPriceTest.php +++ /dev/null @@ -1,60 +0,0 @@ -objectManager = \Magento\TestFramework\Helper\Bootstrap::getObjectManager(); - $this->productRepository = $this->objectManager->create(\Magento\Catalog\Api\ProductRepositoryInterface::class); - $this->productCollectionFactory = $this->objectManager->create( - \Magento\Catalog\Model\ResourceModel\Product\CollectionFactory::class - ); - } - - /** - * @magentoDbIsolation enabled - * @magentoAppIsolation enabled - * @magentoDataFixture Magento/Catalog/_files/product_with_options.php - */ - public function testFinalPrice() - { - $this->markTestSkipped('MAGETWO-64406'); - - $product = $this->productRepository->get('simple', false, null, true); - - /** @var \Magento\Catalog\Model\Indexer\Product\Price\Processor $priceIndexer */ - $priceIndexer = $this->objectManager->create(\Magento\Catalog\Model\Indexer\Product\Price\Processor::class); - $priceIndexer->reindexRow($product->getId()); - - /** @var \Magento\Catalog\Model\ResourceModel\Product\Collection $productCollection */ - $productCollection = $this->productCollectionFactory->create(); - $productCollection->addIdFilter([$product->getId()]); - $productCollection->addPriceData(); - $productCollection->load(); - $indexPriceInfo = $productCollection->getFirstItem(); - - $this->assertEquals(395, $indexPriceInfo->getMaxPrice()); - $this->assertEquals(50, $indexPriceInfo->getMinimalPrice()); - } -} From 367183caa4697b96e7009e91a0d66cfc4cb08c67 Mon Sep 17 00:00:00 2001 From: Iryna Lagno Date: Mon, 27 Feb 2017 11:10:06 +0200 Subject: [PATCH 08/43] MAGETWO-64588: Create upgrade scripts --- app/code/Magento/Catalog/Setup/InstallSchema.php | 12 +----------- app/code/Magento/Catalog/Setup/UpgradeSchema.php | 2 +- app/code/Magento/Catalog/etc/module.xml | 2 +- app/code/Magento/Indexer/Setup/InstallSchema.php | 8 -------- app/code/Magento/Indexer/Setup/UpgradeSchema.php | 2 +- app/code/Magento/Indexer/etc/module.xml | 2 +- 6 files changed, 5 insertions(+), 23 deletions(-) diff --git a/app/code/Magento/Catalog/Setup/InstallSchema.php b/app/code/Magento/Catalog/Setup/InstallSchema.php index ec82953245e20..e408d170dd06b 100644 --- a/app/code/Magento/Catalog/Setup/InstallSchema.php +++ b/app/code/Magento/Catalog/Setup/InstallSchema.php @@ -3043,16 +3043,6 @@ public function install(SchemaSetupInterface $setup, ModuleContextInterface $con $installer->getConnection() ->createTable($table); - /** - * Create table 'catalog_product_index_price_replica' - */ - $installer->getConnection()->createTable( - $installer->getConnection()->createTableByDdl( - $installer->getTable('catalog_product_index_price'), - $installer->getTable('catalog_product_index_price_replica') - ) - ); - /** * Create table 'catalog_product_index_tier_price' */ @@ -4282,7 +4272,7 @@ public function install(SchemaSetupInterface $setup, ModuleContextInterface $con $installer->getConnection() ->createTable($table); - $installer->endSetup(); + $installer->endSetup(); } } diff --git a/app/code/Magento/Catalog/Setup/UpgradeSchema.php b/app/code/Magento/Catalog/Setup/UpgradeSchema.php index 62d76c78edb32..b9f2b0adebdf4 100755 --- a/app/code/Magento/Catalog/Setup/UpgradeSchema.php +++ b/app/code/Magento/Catalog/Setup/UpgradeSchema.php @@ -60,7 +60,7 @@ public function upgrade(SchemaSetupInterface $setup, ModuleContextInterface $con $this->recreateCatalogCategoryProductIndexTmpTable($setup); } - if (version_compare($context->getVersion(), '2.1.5', '<')) { + if (version_compare($context->getVersion(), '2.2.0', '<')) { $this->addProductPriceIndexReplicaTable($setup); } $setup->endSetup(); diff --git a/app/code/Magento/Catalog/etc/module.xml b/app/code/Magento/Catalog/etc/module.xml index 383ab6dd26625..ff25ca46ea39c 100644 --- a/app/code/Magento/Catalog/etc/module.xml +++ b/app/code/Magento/Catalog/etc/module.xml @@ -6,7 +6,7 @@ */ --> - + diff --git a/app/code/Magento/Indexer/Setup/InstallSchema.php b/app/code/Magento/Indexer/Setup/InstallSchema.php index a2410f159ac05..7afd9381abfc4 100644 --- a/app/code/Magento/Indexer/Setup/InstallSchema.php +++ b/app/code/Magento/Indexer/Setup/InstallSchema.php @@ -65,14 +65,6 @@ public function install(SchemaSetupInterface $setup, ModuleContextInterface $con ['nullable' => false], 'Hash of indexer config' ) - ->addColumn( - 'table_suffix', - \Magento\Framework\DB\Ddl\Table::TYPE_TEXT, - 255, - ['nullable' => false], - ['default' => ''], - 'Suffix for indexer table used in the read operations' - ) ->addIndex( $installer->getIdxName('indexer_state', ['indexer_id']), ['indexer_id'] diff --git a/app/code/Magento/Indexer/Setup/UpgradeSchema.php b/app/code/Magento/Indexer/Setup/UpgradeSchema.php index e8c4c36e9a998..291a8dabbd39a 100644 --- a/app/code/Magento/Indexer/Setup/UpgradeSchema.php +++ b/app/code/Magento/Indexer/Setup/UpgradeSchema.php @@ -22,7 +22,7 @@ public function upgrade(SchemaSetupInterface $setup, ModuleContextInterface $con { $setup->startSetup(); - if (version_compare($context->getVersion(), '2.0.1', '<')) { + if (version_compare($context->getVersion(), '2.2.0', '<')) { $this->addTableSuffixColumn($setup); } $setup->endSetup(); diff --git a/app/code/Magento/Indexer/etc/module.xml b/app/code/Magento/Indexer/etc/module.xml index 93fc95c7cce06..b7535a2ff566f 100644 --- a/app/code/Magento/Indexer/etc/module.xml +++ b/app/code/Magento/Indexer/etc/module.xml @@ -6,7 +6,7 @@ */ --> - + From 0e9045dfa4b05139641cc8f78e4a2c41a493b306 Mon Sep 17 00:00:00 2001 From: Stanislav Idolov Date: Thu, 2 Mar 2017 09:58:58 +0200 Subject: [PATCH 09/43] MAGETWO-65303: Catalog displays wrong price for simple product if its part of configurable product and was changed by editing configurable product configurations --- .../Indexer/Product/Price/AbstractAction.php | 21 +++++++++++++++---- .../Indexer/Product/Price/Action/Full.php | 9 ++++++++ 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/app/code/Magento/Catalog/Model/Indexer/Product/Price/AbstractAction.php b/app/code/Magento/Catalog/Model/Indexer/Product/Price/AbstractAction.php index f4693a91f8cea..3a857693acf88 100644 --- a/app/code/Magento/Catalog/Model/Indexer/Product/Price/AbstractAction.php +++ b/app/code/Magento/Catalog/Model/Indexer/Product/Price/AbstractAction.php @@ -76,7 +76,7 @@ abstract class AbstractAction private $productResource; /** - * @var \Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\FrontendResource + * @var \Magento\Indexer\Model\ResourceModel\FrontendResource */ private $indexerFrontendResource; @@ -135,7 +135,7 @@ protected function _syncData(array $processIds = []) { // delete invalid rows $select = $this->_connection->select()->from( - ['index_price' => $this->indexerFrontendResource->getMainTable()], + ['index_price' => $this->getIndexerDefaultTable()], null )->joinLeft( ['ip_tmp' => $this->_defaultIndexerResource->getIdxTable()], @@ -152,7 +152,7 @@ protected function _syncData(array $processIds = []) $this->_insertFromTable( $this->_defaultIndexerResource->getIdxTable(), - $this->indexerFrontendResource->getMainTable() + $this->getIndexerDefaultTable() ); return $this; } @@ -471,7 +471,7 @@ protected function _copyRelationIndexData($parentIds, $excludeIds = null) if ($children) { $select = $this->_connection->select()->from( - $this->indexerFrontendResource->getMainTable() + $this->getIndexerDefaultTable() )->where( 'entity_id IN(?)', $children @@ -483,6 +483,19 @@ protected function _copyRelationIndexData($parentIds, $excludeIds = null) return $this; } + /** + * Returns default table name for writing + * + * Method support writing to frontend table in case with partial reindex + * and case with writing to replica table during full reindex + * + * @return string + */ + protected function getIndexerDefaultTable() + { + return $this->indexerFrontendResource->getMainTable(); + } + /** * @return string */ diff --git a/app/code/Magento/Catalog/Model/Indexer/Product/Price/Action/Full.php b/app/code/Magento/Catalog/Model/Indexer/Product/Price/Action/Full.php index 3c7babd96b7a0..569ead4dbd7c9 100644 --- a/app/code/Magento/Catalog/Model/Indexer/Product/Price/Action/Full.php +++ b/app/code/Magento/Catalog/Model/Indexer/Product/Price/Action/Full.php @@ -153,6 +153,15 @@ public function execute($ids = null) } } + /** + * @inheritdoc + * @return string + */ + protected function getIndexerDefaultTable() + { + return $this->_defaultIndexerResource->getMainTable(); + } + /** * Remove price index data * From 442533cea5333029c383d88e38b99f7a6b699362 Mon Sep 17 00:00:00 2001 From: Stanislav Idolov Date: Mon, 6 Mar 2017 16:51:10 +0200 Subject: [PATCH 10/43] MAGETWO-64187: Batch size algorithm --- app/code/Magento/Bundle/etc/di.xml | 6 +- .../Indexer/Product/Price/Action/Full.php | 63 +++--------- .../Indexer/Price/BatchSizeCalculator.php | 52 ++++++++++ .../CompositeProductBatchSizeCalculator.php | 96 +++++++++++++++++++ .../Indexer/Price/BatchSizeCalculatorTest.php | 49 ++++++++++ app/code/Magento/Catalog/etc/di.xml | 5 +- .../Magento/ConfigurableProduct/etc/di.xml | 7 ++ app/code/Magento/GroupedProduct/etc/di.xml | 7 ++ 8 files changed, 231 insertions(+), 54 deletions(-) create mode 100644 app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/BatchSizeCalculator.php create mode 100644 app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/CompositeProductBatchSizeCalculator.php create mode 100644 app/code/Magento/Catalog/Test/Unit/Model/ResourceModel/Product/Indexer/Price/BatchSizeCalculatorTest.php diff --git a/app/code/Magento/Bundle/etc/di.xml b/app/code/Magento/Bundle/etc/di.xml index d9d1a75f43822..32246ade94efb 100644 --- a/app/code/Magento/Bundle/etc/di.xml +++ b/app/code/Magento/Bundle/etc/di.xml @@ -137,10 +137,10 @@ - + - - 136 + + \Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\CompositeProductBatchSizeCalculator diff --git a/app/code/Magento/Catalog/Model/Indexer/Product/Price/Action/Full.php b/app/code/Magento/Catalog/Model/Indexer/Product/Price/Action/Full.php index 569ead4dbd7c9..cdea22d10ae9f 100644 --- a/app/code/Magento/Catalog/Model/Indexer/Product/Price/Action/Full.php +++ b/app/code/Magento/Catalog/Model/Indexer/Product/Price/Action/Full.php @@ -5,24 +5,21 @@ */ namespace Magento\Catalog\Model\Indexer\Product\Price\Action; +use Magento\Framework\App\ObjectManager; + /** * Class Full reindex action * @SuppressWarnings(PHPMD.CouplingBetweenObjects) */ class Full extends \Magento\Catalog\Model\Indexer\Product\Price\AbstractAction { - /** - * @var array - */ - private $memoryTablesMinRows; - /** * @var \Magento\Framework\EntityManager\MetadataPool */ private $metadataPool; /** - * @var \Magento\Framework\Indexer\BatchSizeCalculatorInterface + * @var \Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\BatchSizeCalculator */ private $batchSizeCalculator; @@ -42,9 +39,8 @@ class Full extends \Magento\Catalog\Model\Indexer\Product\Price\AbstractAction * @param \Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\DefaultPrice $defaultIndexerResource * @param \Magento\Indexer\Model\ResourceModel\FrontendResource|null $indexerFrontendResource * @param \Magento\Framework\EntityManager\MetadataPool|null $metadataPool - * @param \Magento\Framework\Indexer\BatchSizeCalculatorInterface|null $batchSizeCalculator + * @param \Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\BatchSizeCalculator|null $batchSizeCalculator * @param \Magento\Framework\Indexer\BatchProviderInterface|null $batchProvider - * @param array $memoryTablesMinRows * @SuppressWarnings(PHPMD.ExcessiveParameterList) */ public function __construct( @@ -58,21 +54,18 @@ public function __construct( \Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\DefaultPrice $defaultIndexerResource, \Magento\Indexer\Model\ResourceModel\FrontendResource $indexerFrontendResource = null, \Magento\Framework\EntityManager\MetadataPool $metadataPool = null, - \Magento\Framework\Indexer\BatchSizeCalculatorInterface $batchSizeCalculator = null, - \Magento\Framework\Indexer\BatchProviderInterface $batchProvider = null, - array $memoryTablesMinRows = [] + \Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\BatchSizeCalculator $batchSizeCalculator = null, + \Magento\Framework\Indexer\BatchProviderInterface $batchProvider = null ) { - $objectManager = \Magento\Framework\App\ObjectManager::getInstance(); - $this->metadataPool = $metadataPool ?: $objectManager->get( + $this->metadataPool = $metadataPool ?: ObjectManager::getInstance()->get( \Magento\Framework\EntityManager\MetadataPool::class ); - $this->batchSizeCalculator = $batchSizeCalculator ?: $objectManager->get( - \Magento\Framework\Indexer\BatchSizeCalculatorInterface::class + $this->batchSizeCalculator = $batchSizeCalculator ?: ObjectManager::getInstance()->get( + \Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\BatchSizeCalculator::class ); - $this->batchProvider = $batchProvider ?: $objectManager->get( + $this->batchProvider = $batchProvider ?: ObjectManager::getInstance()->get( \Magento\Framework\Indexer\BatchProviderInterface::class ); - $this->memoryTablesMinRows = $memoryTablesMinRows; parent::__construct( $config, $storeManager, @@ -97,27 +90,21 @@ public function __construct( public function execute($ids = null) { try { - $this->_defaultIndexerResource->beginTransaction(); $this->_defaultIndexerResource->getTableStrategy()->setUseIdxTable(false); $this->_prepareWebsiteDateTable(); - $this->_emptyTable($this->_defaultIndexerResource->getIdxTable()); - $this->emptyPriceIndexTable(); $entityMetadata = $this->metadataPool->getMetadata(\Magento\Catalog\Api\Data\ProductInterface::class); /** @var \Magento\Catalog\Model\ResourceModel\Product\Indexer\AbstractIndexer $indexer */ foreach ($this->getTypeIndexers() as $indexer) { - - $memoryTableMinRows = isset($this->memoryTablesMinRows[$indexer->getTypeId()]) - ? $this->memoryTablesMinRows[$indexer->getTypeId()] - : $this->memoryTablesMinRows['default']; - + $indexer->getTableStrategy()->setUseIdxTable(false); $connection = $indexer->getConnection(); + $batches = $this->batchProvider->getBatches( $connection, $entityMetadata->getEntityTable(), $entityMetadata->getIdentifierField(), - $this->batchSizeCalculator->estimateBatchSize($connection, $memoryTableMinRows) + $this->batchSizeCalculator->estimateBatchSize($connection, $indexer->getTypeId()) ); foreach ($batches as $batch) { @@ -146,9 +133,7 @@ public function execute($ids = null) } } } - $this->_defaultIndexerResource->commit(); } catch (\Exception $e) { - $this->_defaultIndexerResource->rollBack(); throw new \Magento\Framework\Exception\LocalizedException(__($e->getMessage()), $e); } } @@ -161,26 +146,4 @@ protected function getIndexerDefaultTable() { return $this->_defaultIndexerResource->getMainTable(); } - - /** - * Remove price index data - * - * Remove all price data from index table for current website - * @return void - */ - private function emptyPriceIndexTable() - { - $select = $this->_connection->select()->from( - ['index_price' => $this->_defaultIndexerResource->getMainTable()], - null - )->joinLeft( - ['ip_tmp' => $this->_defaultIndexerResource->getIdxTable()], - 'index_price.entity_id = ip_tmp.entity_id AND index_price.website_id = ip_tmp.website_id', - [] - )->where( - 'ip_tmp.entity_id IS NULL' - ); - $sql = $select->deleteFromSelect('index_price'); - $this->_connection->query($sql); - } } diff --git a/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/BatchSizeCalculator.php b/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/BatchSizeCalculator.php new file mode 100644 index 0000000000000..38acfd701d834 --- /dev/null +++ b/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/BatchSizeCalculator.php @@ -0,0 +1,52 @@ +memoryTablesMinRows = $memoryTablesMinRows; + $this->calculators = $calculators; + } + + /** + * Composite object for batch size calculators + * + * @param \Magento\Framework\DB\Adapter\AdapterInterface $connection + * @param $indexerTypeId + * @return int + */ + public function estimateBatchSize(\Magento\Framework\DB\Adapter\AdapterInterface $connection, $indexerTypeId) + { + $memoryTableMinRows = isset($this->memoryTablesMinRows[$indexerTypeId]) + ? $this->memoryTablesMinRows[$indexerTypeId] + : $this->memoryTablesMinRows['default']; + + /** @var \Magento\Framework\Indexer\BatchSizeCalculatorInterface $calculator */ + $calculator = isset($this->calculators[$indexerTypeId]) + ? $this->calculators[$indexerTypeId] + : $this->calculators['default']; + + return $calculator->estimateBatchSize($connection, $memoryTableMinRows); + } +} diff --git a/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/CompositeProductBatchSizeCalculator.php b/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/CompositeProductBatchSizeCalculator.php new file mode 100644 index 0000000000000..3197f85111562 --- /dev/null +++ b/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/CompositeProductBatchSizeCalculator.php @@ -0,0 +1,96 @@ +websiteManagement = $websiteManagement; + $this->collectionFactory = $collectionFactory; + } + + /** + * @inheritdoc + */ + public function estimateBatchSize(\Magento\Framework\DB\Adapter\AdapterInterface $connection, $memoryTableMinRows) + { + // Override param, define 100000 records and change memory table size according the calculations + $memoryTableMinRows = 100000; + + // Calculate memory table size for largest composite product + $memoryForLargeComposite = $this->calculateMemorySize($connection); + + $maxHeapTableSize = $connection->fetchOne('SELECT @@max_heap_table_size;'); + $tmpTableSize = $connection->fetchOne('SELECT @@tmp_table_size;'); + $maxMemoryTableSize = min($maxHeapTableSize, $tmpTableSize); + + $size = (int) ($memoryForLargeComposite * $memoryTableMinRows); + + if ($maxMemoryTableSize < $size) { + $connection->query('SET SESSION tmp_table_size = ' . $size); + $connection->query('SET SESSION max_heap_table_size = ' . $size); + } + return $memoryTableMinRows; + } + + /** + * Calculate memory size for largest composite product in database. + * + * @param \Magento\Framework\DB\Adapter\AdapterInterface $connection + * @return float + */ + private function calculateMemorySize(\Magento\Framework\DB\Adapter\AdapterInterface $connection) + { + $relationSelect = $connection->select(); + $relationSelect->from( + ['relation' => $connection->getTableName('catalog_product_relation')], + ['count' => new \Zend_Db_Expr('count(relation.child_id)')] + ); + $relationSelect->group('parent_id'); + + $maxSelect = $connection->select(); + $maxSelect->from( + ['max_value' => $relationSelect], + ['count' => new \Zend_Db_Expr('MAX(count)')] + ); + + $maxRelatedProductCount = $connection->fetchOne($maxSelect); + $websitesCount = $this->websiteManagement->getCount(); + + /** @var \Magento\Customer\Model\ResourceModel\Group\Collection $collection */ + $collection = $this->collectionFactory->create(); + $customerGroupCount = $collection->getSize(); + + /** + * Calculate memory size for largest composite product in database. + * + * $maxRelatedProductCount - maximum number of related products + * $websitesCount - active websites + * $customerGroupCount - active customer groups + * 90 - calculated memory size for one record in catalog_product_index_price table + */ + return ceil($maxRelatedProductCount * $websitesCount * $customerGroupCount * 90); + } +} \ No newline at end of file diff --git a/app/code/Magento/Catalog/Test/Unit/Model/ResourceModel/Product/Indexer/Price/BatchSizeCalculatorTest.php b/app/code/Magento/Catalog/Test/Unit/Model/ResourceModel/Product/Indexer/Price/BatchSizeCalculatorTest.php new file mode 100644 index 0000000000000..e611d53e46af5 --- /dev/null +++ b/app/code/Magento/Catalog/Test/Unit/Model/ResourceModel/Product/Indexer/Price/BatchSizeCalculatorTest.php @@ -0,0 +1,49 @@ +calculatorMock = $this->getMock(\Magento\Framework\Indexer\BatchSizeCalculatorInterface::class); + $this->memoryTablesMinRows = 200; + $this->model = new \Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\BatchSizeCalculator( + ['default' => $this->memoryTablesMinRows], + ['default' => $this->calculatorMock] + ); + } + + public function testEstimateBatchSize() + { + $connectionMock = $this->getMock(\Magento\Framework\DB\Adapter\AdapterInterface::class); + $typeId = 'default'; + $batchSize = 100500; + + $this->calculatorMock->expects($this->once()) + ->method('estimateBatchSize') + ->with($connectionMock, $this->memoryTablesMinRows) + ->willReturn($batchSize); + + $this->model->estimateBatchSize($connectionMock, $typeId); + } +} diff --git a/app/code/Magento/Catalog/etc/di.xml b/app/code/Magento/Catalog/etc/di.xml index c26cb92f7dfd2..2fcc4ed576b85 100644 --- a/app/code/Magento/Catalog/etc/di.xml +++ b/app/code/Magento/Catalog/etc/di.xml @@ -916,11 +916,14 @@ - + 200 + + \Magento\Framework\Indexer\BatchSizeCalculator + diff --git a/app/code/Magento/ConfigurableProduct/etc/di.xml b/app/code/Magento/ConfigurableProduct/etc/di.xml index 00509f1d506ca..890c79ff431ea 100644 --- a/app/code/Magento/ConfigurableProduct/etc/di.xml +++ b/app/code/Magento/ConfigurableProduct/etc/di.xml @@ -171,4 +171,11 @@ + + + + \Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\CompositeProductBatchSizeCalculator + + + diff --git a/app/code/Magento/GroupedProduct/etc/di.xml b/app/code/Magento/GroupedProduct/etc/di.xml index 788052d07fca5..16705699f17cf 100644 --- a/app/code/Magento/GroupedProduct/etc/di.xml +++ b/app/code/Magento/GroupedProduct/etc/di.xml @@ -89,4 +89,11 @@ + + + + \Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\CompositeProductBatchSizeCalculator + + + From a4dd1219043c6694d9c51ee61ee54a3cb08bde97 Mon Sep 17 00:00:00 2001 From: Stanislav Idolov Date: Tue, 7 Mar 2017 08:15:48 +0200 Subject: [PATCH 11/43] MAGETWO-64187: Batch size algorithm --- .../ResourceModel/Product/Indexer/Price/BatchSizeCalculator.php | 2 +- .../Indexer/Price/CompositeProductBatchSizeCalculator.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/BatchSizeCalculator.php b/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/BatchSizeCalculator.php index 38acfd701d834..4459dbc984cd2 100644 --- a/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/BatchSizeCalculator.php +++ b/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/BatchSizeCalculator.php @@ -33,7 +33,7 @@ public function __construct(array $memoryTablesMinRows, array $calculators) * Composite object for batch size calculators * * @param \Magento\Framework\DB\Adapter\AdapterInterface $connection - * @param $indexerTypeId + * @param string $indexerTypeId * @return int */ public function estimateBatchSize(\Magento\Framework\DB\Adapter\AdapterInterface $connection, $indexerTypeId) diff --git a/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/CompositeProductBatchSizeCalculator.php b/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/CompositeProductBatchSizeCalculator.php index 3197f85111562..0485eb4df5586 100644 --- a/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/CompositeProductBatchSizeCalculator.php +++ b/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/CompositeProductBatchSizeCalculator.php @@ -93,4 +93,4 @@ private function calculateMemorySize(\Magento\Framework\DB\Adapter\AdapterInterf */ return ceil($maxRelatedProductCount * $websitesCount * $customerGroupCount * 90); } -} \ No newline at end of file +} From 1ff8713150681d5bbd0912a6f8c2dc27375cf610 Mon Sep 17 00:00:00 2001 From: Stanislav Idolov Date: Tue, 7 Mar 2017 09:45:06 +0200 Subject: [PATCH 12/43] MAGETWO-64187: Batch size algorithm --- .../Price/CompositeProductBatchSizeCalculator.php | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/CompositeProductBatchSizeCalculator.php b/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/CompositeProductBatchSizeCalculator.php index 0485eb4df5586..d841115e33308 100644 --- a/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/CompositeProductBatchSizeCalculator.php +++ b/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/CompositeProductBatchSizeCalculator.php @@ -63,9 +63,15 @@ public function estimateBatchSize(\Magento\Framework\DB\Adapter\AdapterInterface */ private function calculateMemorySize(\Magento\Framework\DB\Adapter\AdapterInterface $connection) { + $websitesCount = $this->websiteManagement->getCount(); + + /** @var \Magento\Customer\Model\ResourceModel\Group\Collection $collection */ + $collection = $this->collectionFactory->create(); + $customerGroupCount = $collection->getSize(); + $relationSelect = $connection->select(); $relationSelect->from( - ['relation' => $connection->getTableName('catalog_product_relation')], + ['relation' => $collection->getResource()->getTable('catalog_product_relation')], ['count' => new \Zend_Db_Expr('count(relation.child_id)')] ); $relationSelect->group('parent_id'); @@ -75,13 +81,7 @@ private function calculateMemorySize(\Magento\Framework\DB\Adapter\AdapterInterf ['max_value' => $relationSelect], ['count' => new \Zend_Db_Expr('MAX(count)')] ); - $maxRelatedProductCount = $connection->fetchOne($maxSelect); - $websitesCount = $this->websiteManagement->getCount(); - - /** @var \Magento\Customer\Model\ResourceModel\Group\Collection $collection */ - $collection = $this->collectionFactory->create(); - $customerGroupCount = $collection->getSize(); /** * Calculate memory size for largest composite product in database. From 8d9110578ef063ce908bd14a25efa683f4382e6c Mon Sep 17 00:00:00 2001 From: Stanislav Idolov Date: Thu, 9 Mar 2017 11:08:11 +0200 Subject: [PATCH 13/43] MAGETWO-64187: Batch size algorithm --- app/code/Magento/Bundle/etc/di.xml | 7 +- .../Indexer/Price/BatchSizeCalculator.php | 34 +++---- .../CompositeProductBatchSizeCalculator.php | 96 ------------------- .../CompositeProductRowSizeEstimator.php | 62 ++++++++++++ .../Indexer/Price/BatchSizeCalculatorTest.php | 20 ++-- app/code/Magento/Catalog/etc/di.xml | 13 ++- .../Magento/ConfigurableProduct/etc/di.xml | 7 +- app/code/Magento/GroupedProduct/etc/di.xml | 7 +- .../Model/IndexTableRowSizeEstimator.php | 54 +++++++++++ app/code/Magento/Indexer/etc/di.xml | 6 ++ .../Framework/Indexer/BatchSizeCalculator.php | 45 --------- .../Indexer/BatchSizeCalculatorInterface.php | 27 ------ .../Framework/Indexer/BatchSizeManagement.php | 45 +++++++++ .../Indexer/BatchSizeManagementInterface.php | 22 +++++ .../IndexTableRowSizeEstimatorInterface.php | 17 ++++ .../Test/Unit/BatchSizeCalculatorTest.php | 63 ------------ .../Test/Unit/BatchSizeManagementTest.php | 75 +++++++++++++++ 17 files changed, 333 insertions(+), 267 deletions(-) delete mode 100644 app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/CompositeProductBatchSizeCalculator.php create mode 100644 app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/CompositeProductRowSizeEstimator.php create mode 100644 app/code/Magento/Indexer/Model/IndexTableRowSizeEstimator.php delete mode 100644 lib/internal/Magento/Framework/Indexer/BatchSizeCalculator.php delete mode 100644 lib/internal/Magento/Framework/Indexer/BatchSizeCalculatorInterface.php create mode 100644 lib/internal/Magento/Framework/Indexer/BatchSizeManagement.php create mode 100644 lib/internal/Magento/Framework/Indexer/BatchSizeManagementInterface.php create mode 100644 lib/internal/Magento/Framework/Indexer/IndexTableRowSizeEstimatorInterface.php delete mode 100644 lib/internal/Magento/Framework/Indexer/Test/Unit/BatchSizeCalculatorTest.php create mode 100644 lib/internal/Magento/Framework/Indexer/Test/Unit/BatchSizeManagementTest.php diff --git a/app/code/Magento/Bundle/etc/di.xml b/app/code/Magento/Bundle/etc/di.xml index 32246ade94efb..017c7bd04e748 100644 --- a/app/code/Magento/Bundle/etc/di.xml +++ b/app/code/Magento/Bundle/etc/di.xml @@ -139,8 +139,11 @@ - - \Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\CompositeProductBatchSizeCalculator + + 100000 + + + Magento\Catalog\Model\Indexer\CompositeProductBatchSizeManagement diff --git a/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/BatchSizeCalculator.php b/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/BatchSizeCalculator.php index 4459dbc984cd2..abba22333a57f 100644 --- a/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/BatchSizeCalculator.php +++ b/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/BatchSizeCalculator.php @@ -11,22 +11,22 @@ class BatchSizeCalculator /** * @var array */ - private $memoryTablesMinRows; + private $batchRowsCount; /** - * @var \Magento\Framework\Indexer\BatchSizeCalculatorInterface[] + * @var \Magento\Framework\Indexer\BatchSizeManagementInterface[] */ - private $calculators; + private $estimators; /** * BatchSizeCalculator constructor. - * @param array $memoryTablesMinRows - * @param array $calculators + * @param array $batchRowsCount + * @param array $estimators */ - public function __construct(array $memoryTablesMinRows, array $calculators) + public function __construct(array $batchRowsCount, array $estimators) { - $this->memoryTablesMinRows = $memoryTablesMinRows; - $this->calculators = $calculators; + $this->batchRowsCount = $batchRowsCount; + $this->estimators = $estimators; } /** @@ -38,15 +38,17 @@ public function __construct(array $memoryTablesMinRows, array $calculators) */ public function estimateBatchSize(\Magento\Framework\DB\Adapter\AdapterInterface $connection, $indexerTypeId) { - $memoryTableMinRows = isset($this->memoryTablesMinRows[$indexerTypeId]) - ? $this->memoryTablesMinRows[$indexerTypeId] - : $this->memoryTablesMinRows['default']; + $batchRowsCount = isset($this->batchRowsCount[$indexerTypeId]) + ? $this->batchRowsCount[$indexerTypeId] + : $this->batchRowsCount['default']; - /** @var \Magento\Framework\Indexer\BatchSizeCalculatorInterface $calculator */ - $calculator = isset($this->calculators[$indexerTypeId]) - ? $this->calculators[$indexerTypeId] - : $this->calculators['default']; + /** @var \Magento\Framework\Indexer\BatchSizeManagementInterface $calculator */ + $calculator = isset($this->estimators[$indexerTypeId]) + ? $this->estimators[$indexerTypeId] + : $this->estimators['default']; - return $calculator->estimateBatchSize($connection, $memoryTableMinRows); + $calculator->ensureBatchSize($connection, $batchRowsCount); + + return $batchRowsCount; } } diff --git a/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/CompositeProductBatchSizeCalculator.php b/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/CompositeProductBatchSizeCalculator.php deleted file mode 100644 index d841115e33308..0000000000000 --- a/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/CompositeProductBatchSizeCalculator.php +++ /dev/null @@ -1,96 +0,0 @@ -websiteManagement = $websiteManagement; - $this->collectionFactory = $collectionFactory; - } - - /** - * @inheritdoc - */ - public function estimateBatchSize(\Magento\Framework\DB\Adapter\AdapterInterface $connection, $memoryTableMinRows) - { - // Override param, define 100000 records and change memory table size according the calculations - $memoryTableMinRows = 100000; - - // Calculate memory table size for largest composite product - $memoryForLargeComposite = $this->calculateMemorySize($connection); - - $maxHeapTableSize = $connection->fetchOne('SELECT @@max_heap_table_size;'); - $tmpTableSize = $connection->fetchOne('SELECT @@tmp_table_size;'); - $maxMemoryTableSize = min($maxHeapTableSize, $tmpTableSize); - - $size = (int) ($memoryForLargeComposite * $memoryTableMinRows); - - if ($maxMemoryTableSize < $size) { - $connection->query('SET SESSION tmp_table_size = ' . $size); - $connection->query('SET SESSION max_heap_table_size = ' . $size); - } - return $memoryTableMinRows; - } - - /** - * Calculate memory size for largest composite product in database. - * - * @param \Magento\Framework\DB\Adapter\AdapterInterface $connection - * @return float - */ - private function calculateMemorySize(\Magento\Framework\DB\Adapter\AdapterInterface $connection) - { - $websitesCount = $this->websiteManagement->getCount(); - - /** @var \Magento\Customer\Model\ResourceModel\Group\Collection $collection */ - $collection = $this->collectionFactory->create(); - $customerGroupCount = $collection->getSize(); - - $relationSelect = $connection->select(); - $relationSelect->from( - ['relation' => $collection->getResource()->getTable('catalog_product_relation')], - ['count' => new \Zend_Db_Expr('count(relation.child_id)')] - ); - $relationSelect->group('parent_id'); - - $maxSelect = $connection->select(); - $maxSelect->from( - ['max_value' => $relationSelect], - ['count' => new \Zend_Db_Expr('MAX(count)')] - ); - $maxRelatedProductCount = $connection->fetchOne($maxSelect); - - /** - * Calculate memory size for largest composite product in database. - * - * $maxRelatedProductCount - maximum number of related products - * $websitesCount - active websites - * $customerGroupCount - active customer groups - * 90 - calculated memory size for one record in catalog_product_index_price table - */ - return ceil($maxRelatedProductCount * $websitesCount * $customerGroupCount * 90); - } -} diff --git a/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/CompositeProductRowSizeEstimator.php b/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/CompositeProductRowSizeEstimator.php new file mode 100644 index 0000000000000..54d842a153fd8 --- /dev/null +++ b/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/CompositeProductRowSizeEstimator.php @@ -0,0 +1,62 @@ +resourceConnection = $resourceConnection; + $this->indexTableRowSizeEstimator = $indexTableRowSizeEstimator; + } + + /** + * Calculate memory size for largest composite product in database. + * + * @inheritdoc + */ + public function estimateRowSize() + { + $connection = $this->resourceConnection->getConnection(); + $relationSelect = $connection->select(); + $relationSelect->from( + ['relation' => $connection->getTableName('catalog_product_relation')], + ['count' => new \Zend_Db_Expr('count(relation.child_id)')] + ); + $relationSelect->group('parent_id'); + + $maxSelect = $connection->select(); + $maxSelect->from( + ['max_value' => $relationSelect], + ['count' => new \Zend_Db_Expr('MAX(count)')] + ); + $maxRelatedProductCount = $connection->fetchOne($maxSelect); + + /** + * Calculate memory size for largest composite product in database. + * + * $maxRelatedProductCount - maximum number of related products + */ + return ceil($maxRelatedProductCount * $this->indexTableRowSizeEstimator->estimateRowSize()); + } +} diff --git a/app/code/Magento/Catalog/Test/Unit/Model/ResourceModel/Product/Indexer/Price/BatchSizeCalculatorTest.php b/app/code/Magento/Catalog/Test/Unit/Model/ResourceModel/Product/Indexer/Price/BatchSizeCalculatorTest.php index e611d53e46af5..926803d4077f9 100644 --- a/app/code/Magento/Catalog/Test/Unit/Model/ResourceModel/Product/Indexer/Price/BatchSizeCalculatorTest.php +++ b/app/code/Magento/Catalog/Test/Unit/Model/ResourceModel/Product/Indexer/Price/BatchSizeCalculatorTest.php @@ -14,22 +14,22 @@ class BatchSizeCalculatorTest extends \PHPUnit_Framework_TestCase private $model; /** - * @var \Magento\Framework\Indexer\BatchSizeCalculatorInterface|\PHPUnit_Framework_MockObject_MockObject + * @var \Magento\Framework\Indexer\BatchSizeManagementInterface|\PHPUnit_Framework_MockObject_MockObject */ - private $calculatorMock; + private $estimatorMock; /** * @var int */ - private $memoryTablesMinRows; + private $batchRowsCount; protected function setUp() { - $this->calculatorMock = $this->getMock(\Magento\Framework\Indexer\BatchSizeCalculatorInterface::class); - $this->memoryTablesMinRows = 200; + $this->estimatorMock = $this->getMock(\Magento\Framework\Indexer\BatchSizeManagementInterface::class); + $this->batchRowsCount = 200; $this->model = new \Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\BatchSizeCalculator( - ['default' => $this->memoryTablesMinRows], - ['default' => $this->calculatorMock] + ['default' => $this->batchRowsCount], + ['default' => $this->estimatorMock] ); } @@ -39,9 +39,9 @@ public function testEstimateBatchSize() $typeId = 'default'; $batchSize = 100500; - $this->calculatorMock->expects($this->once()) - ->method('estimateBatchSize') - ->with($connectionMock, $this->memoryTablesMinRows) + $this->estimatorMock->expects($this->once()) + ->method('ensureBatchSize') + ->with($connectionMock, $this->batchRowsCount) ->willReturn($batchSize); $this->model->estimateBatchSize($connectionMock, $typeId); diff --git a/app/code/Magento/Catalog/etc/di.xml b/app/code/Magento/Catalog/etc/di.xml index 2fcc4ed576b85..3b1aa120d00df 100644 --- a/app/code/Magento/Catalog/etc/di.xml +++ b/app/code/Magento/Catalog/etc/di.xml @@ -918,14 +918,19 @@ - - 200 + + 200000 - - \Magento\Framework\Indexer\BatchSizeCalculator + + \Magento\Framework\Indexer\BatchSizeManagement + + + Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\CompositeProductRowSizeEstimator + + Magento\Catalog\Model\Indexer\Product\Price\Processor::INDEXER_ID diff --git a/app/code/Magento/ConfigurableProduct/etc/di.xml b/app/code/Magento/ConfigurableProduct/etc/di.xml index 890c79ff431ea..f555e5f5cd963 100644 --- a/app/code/Magento/ConfigurableProduct/etc/di.xml +++ b/app/code/Magento/ConfigurableProduct/etc/di.xml @@ -173,8 +173,11 @@ - - \Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\CompositeProductBatchSizeCalculator + + 100000 + + + Magento\Catalog\Model\Indexer\CompositeProductBatchSizeManagement diff --git a/app/code/Magento/GroupedProduct/etc/di.xml b/app/code/Magento/GroupedProduct/etc/di.xml index 16705699f17cf..c3346656ef9b8 100644 --- a/app/code/Magento/GroupedProduct/etc/di.xml +++ b/app/code/Magento/GroupedProduct/etc/di.xml @@ -91,8 +91,11 @@ type="Magento\GroupedProduct\Model\ResourceModel\Product\Indexer\Price\Grouped"/> - - \Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\CompositeProductBatchSizeCalculator + + 100000 + + + Magento\Catalog\Model\Indexer\CompositeProductBatchSizeManagement diff --git a/app/code/Magento/Indexer/Model/IndexTableRowSizeEstimator.php b/app/code/Magento/Indexer/Model/IndexTableRowSizeEstimator.php new file mode 100644 index 0000000000000..0c4402436b58f --- /dev/null +++ b/app/code/Magento/Indexer/Model/IndexTableRowSizeEstimator.php @@ -0,0 +1,54 @@ +websiteManagement = $websiteManagement; + $this->collectionFactory = $collectionFactory; + } + + /** + * @inheritdoc + */ + public function estimateRowSize() + { + $websitesCount = $this->websiteManagement->getCount(); + + /** @var \Magento\Customer\Model\ResourceModel\Group\Collection $collection */ + $collection = $this->collectionFactory->create(); + $customerGroupCount = $collection->getSize(); + + /** + * Calculate memory size for product in database. + * + * $websitesCount - active websites + * $customerGroupCount - active customer groups + * 90 - calculated memory size for one record in catalog_product_index_price table + */ + return ceil($websitesCount * $customerGroupCount * 90); + } +} diff --git a/app/code/Magento/Indexer/etc/di.xml b/app/code/Magento/Indexer/etc/di.xml index 1d3f125406f7d..fc4b0991a54ff 100644 --- a/app/code/Magento/Indexer/etc/di.xml +++ b/app/code/Magento/Indexer/etc/di.xml @@ -55,4 +55,10 @@ + + + + Magento\Indexer\Model\IndexTableRowSizeEstimator + + diff --git a/lib/internal/Magento/Framework/Indexer/BatchSizeCalculator.php b/lib/internal/Magento/Framework/Indexer/BatchSizeCalculator.php deleted file mode 100644 index 8b10413826506..0000000000000 --- a/lib/internal/Magento/Framework/Indexer/BatchSizeCalculator.php +++ /dev/null @@ -1,45 +0,0 @@ -fetchOne('SELECT @@max_heap_table_size;'); - $tmpTableSize = $adapter->fetchOne('SELECT @@tmp_table_size;'); - $maxMemoryTableSize = min($maxHeapTableSize, $tmpTableSize); - - /** - * According to MySQL documentation minimum value of the max_heap_table_size is 16384. - * (This system variable sets the maximum size to which user-created MEMORY tables are permitted to grow) - */ - return (int)($maxMemoryTableSize * ($memoryTableMinRows / 16384)); - } -} diff --git a/lib/internal/Magento/Framework/Indexer/BatchSizeCalculatorInterface.php b/lib/internal/Magento/Framework/Indexer/BatchSizeCalculatorInterface.php deleted file mode 100644 index b5b182cafa357..0000000000000 --- a/lib/internal/Magento/Framework/Indexer/BatchSizeCalculatorInterface.php +++ /dev/null @@ -1,27 +0,0 @@ -rowSizeEstimator = $rowSizeEstimator; + } + + /** + * @inheritdoc + */ + public function ensureBatchSize(\Magento\Framework\DB\Adapter\AdapterInterface $connection, $batchSize) + { + // Calculate memory table size for product + $memoryForLargeComposite = $this->rowSizeEstimator->estimateRowSize(); + + $maxHeapTableSize = $connection->fetchOne('SELECT @@max_heap_table_size;'); + $tmpTableSize = $connection->fetchOne('SELECT @@tmp_table_size;'); + $maxMemoryTableSize = min($maxHeapTableSize, $tmpTableSize); + + $size = (int) ($memoryForLargeComposite * $batchSize); + + if ($maxMemoryTableSize < $size) { + $connection->query('SET SESSION tmp_table_size = ' . $size . ';'); + $connection->query('SET SESSION max_heap_table_size = ' . $size . ';'); + } + } +} diff --git a/lib/internal/Magento/Framework/Indexer/BatchSizeManagementInterface.php b/lib/internal/Magento/Framework/Indexer/BatchSizeManagementInterface.php new file mode 100644 index 0000000000000..33a5a0d1f95bc --- /dev/null +++ b/lib/internal/Magento/Framework/Indexer/BatchSizeManagementInterface.php @@ -0,0 +1,22 @@ +model = new BatchSizeCalculator(); - } - - /** - * @param int $memoryTableMinRows number of records that can be inserted into MEMORY table when its maximum size - * @param int $maxHeapTableSize max_heap_table_size MySQL value - * @param int $tmpTableSize tmp_table_size MySQL value - * @param int $expectedResult - * - * @dataProvider estimateBatchSizeDataProvider - */ - public function testEstimateBatchSize($memoryTableMinRows, $maxHeapTableSize, $tmpTableSize, $expectedResult) - { - $adapterMock = $this->getMock(AdapterInterface::class); - $adapterMock->expects($this->at(0)) - ->method('fetchOne') - ->with('SELECT @@max_heap_table_size;', []) - ->willReturn($maxHeapTableSize); - $adapterMock->expects($this->at(1)) - ->method('fetchOne') - ->with('SELECT @@tmp_table_size;', []) - ->willReturn($tmpTableSize); - - $this->assertEquals($expectedResult, $this->model->estimateBatchSize($adapterMock, $memoryTableMinRows)); - } - - /** - * @return array - */ - public function estimateBatchSizeDataProvider() - { - /** - * The maximum size for in-memory temporary tables is determined from whichever of the values of - * tmp_table_size and max_heap_table_size is smaller. - */ - return [ - [200, 16384, 16384, 200], - [200, 16384, 20000, 200], - [200, 20000, 16384, 200], - [200, 20000, 20000, 244], - [200, 2000, 2000, 24], - ]; - } -} diff --git a/lib/internal/Magento/Framework/Indexer/Test/Unit/BatchSizeManagementTest.php b/lib/internal/Magento/Framework/Indexer/Test/Unit/BatchSizeManagementTest.php new file mode 100644 index 0000000000000..87f09d0b74aaf --- /dev/null +++ b/lib/internal/Magento/Framework/Indexer/Test/Unit/BatchSizeManagementTest.php @@ -0,0 +1,75 @@ +rowSizeEstimatorMock = $this->getMock( + \Magento\Framework\Indexer\IndexTableRowSizeEstimatorInterface::class + ); + $this->model = new BatchSizeManagement($this->rowSizeEstimatorMock); + } + + /** + * @param int $batchSize number of records in the batch + * @param int $maxHeapTableSize max_heap_table_size MySQL value + * @param int $tmpTableSize tmp_table_size MySQL value + * @param int $size + * + * @dataProvider estimateBatchSizeDataProvider + */ + public function testEnsureBatchSize($batchSize, $maxHeapTableSize, $tmpTableSize, $size) + { + $this->rowSizeEstimatorMock->expects($this->once())->method('estimateRowSize')->willReturn(100); + $adapterMock = $this->getMock(AdapterInterface::class); + $adapterMock->expects($this->at(0)) + ->method('fetchOne') + ->with('SELECT @@max_heap_table_size;', []) + ->willReturn($maxHeapTableSize); + $adapterMock->expects($this->at(1)) + ->method('fetchOne') + ->with('SELECT @@tmp_table_size;', []) + ->willReturn($tmpTableSize); + + $adapterMock->expects($this->at(2)) + ->method('query') + ->with('SET SESSION tmp_table_size = ' . $size . ';', []); + + $adapterMock->expects($this->at(3)) + ->method('query') + ->with('SET SESSION max_heap_table_size = ' . $size . ';', []); + + $this->model->ensureBatchSize($adapterMock, $batchSize); + } + + /** + * @return array + */ + public function estimateBatchSizeDataProvider() + { + return [ + [200, 16384, 16384, 20000], + [300, 16384, 20000, 30000], + [400, 20000, 16384, 40000], + [500, 2000, 2000, 50000], + ]; + } +} From bf07aebbf711605034da0a1d9273395fc9411c8b Mon Sep 17 00:00:00 2001 From: Stanislav Idolov Date: Thu, 9 Mar 2017 11:29:13 +0200 Subject: [PATCH 14/43] MAGETWO-64187: Batch size algorithm --- .../Price/CompositeProductRowSizeEstimator.php | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/CompositeProductRowSizeEstimator.php b/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/CompositeProductRowSizeEstimator.php index 54d842a153fd8..1a7e3d4a9a351 100644 --- a/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/CompositeProductRowSizeEstimator.php +++ b/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/CompositeProductRowSizeEstimator.php @@ -14,19 +14,19 @@ class CompositeProductRowSizeEstimator implements \Magento\Framework\Indexer\Ind private $indexTableRowSizeEstimator; /** - * @var \Magento\Framework\App\ResourceConnection + * @var \Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\DefaultPrice */ - private $resourceConnection; + private $indexerResource; /** - * @param \Magento\Framework\App\ResourceConnection $resourceConnection + * @param DefaultPrice $indexerResource * @param \Magento\Indexer\Model\IndexTableRowSizeEstimator $indexTableRowSizeEstimator */ public function __construct( - \Magento\Framework\App\ResourceConnection $resourceConnection, + \Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\DefaultPrice $indexerResource, \Magento\Indexer\Model\IndexTableRowSizeEstimator $indexTableRowSizeEstimator ) { - $this->resourceConnection = $resourceConnection; + $this->indexerResource = $indexerResource; $this->indexTableRowSizeEstimator = $indexTableRowSizeEstimator; } @@ -37,10 +37,10 @@ public function __construct( */ public function estimateRowSize() { - $connection = $this->resourceConnection->getConnection(); + $connection = $this->indexerResource->getConnection(); $relationSelect = $connection->select(); $relationSelect->from( - ['relation' => $connection->getTableName('catalog_product_relation')], + ['relation' => $this->indexerResource->getTable('catalog_product_relation')], ['count' => new \Zend_Db_Expr('count(relation.child_id)')] ); $relationSelect->group('parent_id'); From 67924ec9e6f1d2407791dccb5e42f30747366494 Mon Sep 17 00:00:00 2001 From: Stanislav Idolov Date: Thu, 9 Mar 2017 11:38:33 +0200 Subject: [PATCH 15/43] MAGETWO-64187: Batch size algorithm --- .../Product/Indexer}/IndexTableRowSizeEstimator.php | 2 +- .../Indexer/Price/CompositeProductRowSizeEstimator.php | 6 +++--- app/code/Magento/Catalog/etc/di.xml | 5 +++++ app/code/Magento/Indexer/etc/di.xml | 5 ----- 4 files changed, 9 insertions(+), 9 deletions(-) rename app/code/Magento/{Indexer/Model => Catalog/Model/ResourceModel/Product/Indexer}/IndexTableRowSizeEstimator.php (96%) diff --git a/app/code/Magento/Indexer/Model/IndexTableRowSizeEstimator.php b/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/IndexTableRowSizeEstimator.php similarity index 96% rename from app/code/Magento/Indexer/Model/IndexTableRowSizeEstimator.php rename to app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/IndexTableRowSizeEstimator.php index 0c4402436b58f..706d4400b3319 100644 --- a/app/code/Magento/Indexer/Model/IndexTableRowSizeEstimator.php +++ b/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/IndexTableRowSizeEstimator.php @@ -4,7 +4,7 @@ * See COPYING.txt for license details. */ -namespace Magento\Indexer\Model; +namespace Magento\Catalog\Model\ResourceModel\Product\Indexer; class IndexTableRowSizeEstimator implements \Magento\Framework\Indexer\IndexTableRowSizeEstimatorInterface { diff --git a/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/CompositeProductRowSizeEstimator.php b/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/CompositeProductRowSizeEstimator.php index 1a7e3d4a9a351..3e3decf4fd1d0 100644 --- a/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/CompositeProductRowSizeEstimator.php +++ b/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/CompositeProductRowSizeEstimator.php @@ -9,7 +9,7 @@ class CompositeProductRowSizeEstimator implements \Magento\Framework\Indexer\IndexTableRowSizeEstimatorInterface { /** - * @var \Magento\Indexer\Model\IndexTableRowSizeEstimator + * @var \Magento\Catalog\Model\ResourceModel\Product\Indexer\IndexTableRowSizeEstimator */ private $indexTableRowSizeEstimator; @@ -20,11 +20,11 @@ class CompositeProductRowSizeEstimator implements \Magento\Framework\Indexer\Ind /** * @param DefaultPrice $indexerResource - * @param \Magento\Indexer\Model\IndexTableRowSizeEstimator $indexTableRowSizeEstimator + * @param \Magento\Catalog\Model\ResourceModel\Product\Indexer\IndexTableRowSizeEstimator $indexTableRowSizeEstimator */ public function __construct( \Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\DefaultPrice $indexerResource, - \Magento\Indexer\Model\IndexTableRowSizeEstimator $indexTableRowSizeEstimator + \Magento\Catalog\Model\ResourceModel\Product\Indexer\IndexTableRowSizeEstimator $indexTableRowSizeEstimator ) { $this->indexerResource = $indexerResource; $this->indexTableRowSizeEstimator = $indexTableRowSizeEstimator; diff --git a/app/code/Magento/Catalog/etc/di.xml b/app/code/Magento/Catalog/etc/di.xml index 3b1aa120d00df..752fe4e79d721 100644 --- a/app/code/Magento/Catalog/etc/di.xml +++ b/app/code/Magento/Catalog/etc/di.xml @@ -943,4 +943,9 @@ Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\FrontendResource + + + \Magento\Catalog\Model\ResourceModel\Product\Indexer\IndexTableRowSizeEstimator + + diff --git a/app/code/Magento/Indexer/etc/di.xml b/app/code/Magento/Indexer/etc/di.xml index fc4b0991a54ff..7cb8971dfdacf 100644 --- a/app/code/Magento/Indexer/etc/di.xml +++ b/app/code/Magento/Indexer/etc/di.xml @@ -56,9 +56,4 @@ - - - Magento\Indexer\Model\IndexTableRowSizeEstimator - - From b88d1bf2bc2d83d9d46ea2dba057809a6ca8c8f9 Mon Sep 17 00:00:00 2001 From: Stanislav Idolov Date: Thu, 9 Mar 2017 12:55:24 +0200 Subject: [PATCH 16/43] MAGETWO-64187: Batch size algorithm --- .../CompositeProductRowSizeEstimator.php | 15 ++-- .../IndexTableRowSizeEstimatorTest.php | 62 +++++++++++++ .../CompositeProductRowSizeEstimatorTest.php | 90 +++++++++++++++++++ 3 files changed, 161 insertions(+), 6 deletions(-) create mode 100644 app/code/Magento/Catalog/Test/Unit/Model/ResourceModel/Product/Indexer/IndexTableRowSizeEstimatorTest.php create mode 100644 app/code/Magento/Catalog/Test/Unit/Model/ResourceModel/Product/Indexer/Price/CompositeProductRowSizeEstimatorTest.php diff --git a/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/CompositeProductRowSizeEstimator.php b/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/CompositeProductRowSizeEstimator.php index 3e3decf4fd1d0..47f5443094381 100644 --- a/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/CompositeProductRowSizeEstimator.php +++ b/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/CompositeProductRowSizeEstimator.php @@ -6,25 +6,28 @@ namespace Magento\Catalog\Model\ResourceModel\Product\Indexer\Price; -class CompositeProductRowSizeEstimator implements \Magento\Framework\Indexer\IndexTableRowSizeEstimatorInterface +use Magento\Catalog\Model\ResourceModel\Product\Indexer\IndexTableRowSizeEstimator; +use Magento\Framework\Indexer\IndexTableRowSizeEstimatorInterface; + +class CompositeProductRowSizeEstimator implements IndexTableRowSizeEstimatorInterface { /** - * @var \Magento\Catalog\Model\ResourceModel\Product\Indexer\IndexTableRowSizeEstimator + * @var IndexTableRowSizeEstimator */ private $indexTableRowSizeEstimator; /** - * @var \Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\DefaultPrice + * @var DefaultPrice */ private $indexerResource; /** * @param DefaultPrice $indexerResource - * @param \Magento\Catalog\Model\ResourceModel\Product\Indexer\IndexTableRowSizeEstimator $indexTableRowSizeEstimator + * @param IndexTableRowSizeEstimator $indexTableRowSizeEstimator */ public function __construct( - \Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\DefaultPrice $indexerResource, - \Magento\Catalog\Model\ResourceModel\Product\Indexer\IndexTableRowSizeEstimator $indexTableRowSizeEstimator + DefaultPrice $indexerResource, + IndexTableRowSizeEstimator $indexTableRowSizeEstimator ) { $this->indexerResource = $indexerResource; $this->indexTableRowSizeEstimator = $indexTableRowSizeEstimator; diff --git a/app/code/Magento/Catalog/Test/Unit/Model/ResourceModel/Product/Indexer/IndexTableRowSizeEstimatorTest.php b/app/code/Magento/Catalog/Test/Unit/Model/ResourceModel/Product/Indexer/IndexTableRowSizeEstimatorTest.php new file mode 100644 index 0000000000000..de66f83333daa --- /dev/null +++ b/app/code/Magento/Catalog/Test/Unit/Model/ResourceModel/Product/Indexer/IndexTableRowSizeEstimatorTest.php @@ -0,0 +1,62 @@ +websiteManagementMock = $this->getMock(\Magento\Store\Api\WebsiteManagementInterface::class); + $this->collectionFactoryMock = $this->getMock( + \Magento\Customer\Model\ResourceModel\Group\CollectionFactory::class, + ['create'], + [], + '', + false + ); + $this->model = new \Magento\Catalog\Model\ResourceModel\Product\Indexer\IndexTableRowSizeEstimator( + $this->websiteManagementMock, + $this->collectionFactoryMock + ); + } + + public function testEstimateRowSize() + { + $expectedValue = 1800000; + + $this->websiteManagementMock->expects($this->once())->method('getCount')->willReturn(100); + $collectionMock = $this->getMock( + \Magento\Customer\Model\ResourceModel\Group\Collection::class, + [], + [], + '', + false + ); + $this->collectionFactoryMock->expects($this->once())->method('create')->willReturn($collectionMock); + $collectionMock->expects($this->once())->method('getSize')->willReturn(200); + + $this->assertEquals( + $expectedValue, + $this->model->estimateRowSize() + ); + } +} diff --git a/app/code/Magento/Catalog/Test/Unit/Model/ResourceModel/Product/Indexer/Price/CompositeProductRowSizeEstimatorTest.php b/app/code/Magento/Catalog/Test/Unit/Model/ResourceModel/Product/Indexer/Price/CompositeProductRowSizeEstimatorTest.php new file mode 100644 index 0000000000000..4e7483644afcf --- /dev/null +++ b/app/code/Magento/Catalog/Test/Unit/Model/ResourceModel/Product/Indexer/Price/CompositeProductRowSizeEstimatorTest.php @@ -0,0 +1,90 @@ +rowSizeEstimatorMock = $this->getMock( + \Magento\Catalog\Model\ResourceModel\Product\Indexer\IndexTableRowSizeEstimator::class, + [], + [], + '', + false + ); + $this->defaultPriceMock = $this->getMock( + \Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\DefaultPrice::class, + [], + [], + '', + false + ); + $this->model = new CompositeProductRowSizeEstimator($this->defaultPriceMock, $this->rowSizeEstimatorMock); + } + + public function testEstimateRowSize() + { + $expectedResult = 2000; + $tableName = 'catalog_product_relation'; + $maxRelatedProductCount = 10; + + $this->rowSizeEstimatorMock->expects($this->once())->method('estimateRowSize')->willReturn(200); + + $connectionMock = $this->getMock(\Magento\Framework\DB\Adapter\AdapterInterface::class); + $this->defaultPriceMock->expects($this->once())->method('getConnection')->willReturn($connectionMock); + $this->defaultPriceMock->expects($this->once())->method('getTable')->with($tableName)->willReturn($tableName); + + $relationSelectMock = $this->getMock(\Magento\Framework\DB\Select::class, [], [], '', false); + $relationSelectMock->expects($this->once()) + ->method('from') + ->with( + ['relation' => $tableName], + ['count' => 'count(relation.child_id)'] + ) + ->willReturnSelf(); + $relationSelectMock->expects($this->once())->method('group')->with('parent_id')->willReturnSelf(); + $connectionMock->expects($this->at(0))->method('select')->willReturn($relationSelectMock); + + $maxSelectMock = $this->getMock(\Magento\Framework\DB\Select::class, [], [], '', false); + $maxSelectMock->expects($this->once()) + ->method('from') + ->with( + ['max_value' => $relationSelectMock], + ['count' => 'MAX(count)'] + ) + ->willReturnSelf(); + $connectionMock->expects($this->at(1))->method('select')->willReturn($maxSelectMock); + + $connectionMock->expects($this->at(2)) + ->method('fetchOne') + ->with($maxSelectMock) + ->willReturn($maxRelatedProductCount); + + $this->assertEquals( + $expectedResult, + $this->model->estimateRowSize() + ); + } +} From d8b5f70eb305aded0ce6f3dcb9c84bf35f06568d Mon Sep 17 00:00:00 2001 From: Stanislav Idolov Date: Thu, 9 Mar 2017 13:00:22 +0200 Subject: [PATCH 17/43] MAGETWO-64187: Batch size algorithm --- .../Magento/Framework/Indexer/BatchSizeManagementInterface.php | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/internal/Magento/Framework/Indexer/BatchSizeManagementInterface.php b/lib/internal/Magento/Framework/Indexer/BatchSizeManagementInterface.php index 33a5a0d1f95bc..8ff4e65ddb5f1 100644 --- a/lib/internal/Magento/Framework/Indexer/BatchSizeManagementInterface.php +++ b/lib/internal/Magento/Framework/Indexer/BatchSizeManagementInterface.php @@ -16,6 +16,7 @@ interface BatchSizeManagementInterface * Ensure memory size for data in batch. * * @param AdapterInterface $adapter database adapter. + * @param int $batchSize * @return void */ public function ensureBatchSize(\Magento\Framework\DB\Adapter\AdapterInterface $adapter, $batchSize); From 757fd496be8e15c2c3d1008c42228c5aa57f4a9b Mon Sep 17 00:00:00 2001 From: Stanislav Idolov Date: Thu, 9 Mar 2017 15:46:38 +0200 Subject: [PATCH 18/43] MAGETWO-64187: Batch size algorithm --- app/code/Magento/Bundle/etc/di.xml | 2 +- .../Price/CompositeProductRowSizeEstimator.php | 1 - .../{ => Price}/IndexTableRowSizeEstimator.php | 2 +- .../Price/CompositeProductRowSizeEstimatorTest.php | 2 +- .../{ => Price}/IndexTableRowSizeEstimatorTest.php | 6 +++--- app/code/Magento/Catalog/etc/di.xml | 14 +++++++------- app/code/Magento/ConfigurableProduct/etc/di.xml | 2 +- app/code/Magento/GroupedProduct/etc/di.xml | 2 +- 8 files changed, 15 insertions(+), 16 deletions(-) rename app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/{ => Price}/IndexTableRowSizeEstimator.php (96%) rename app/code/Magento/Catalog/Test/Unit/Model/ResourceModel/Product/Indexer/{ => Price}/IndexTableRowSizeEstimatorTest.php (94%) diff --git a/app/code/Magento/Bundle/etc/di.xml b/app/code/Magento/Bundle/etc/di.xml index 017c7bd04e748..3cbe4fcd00590 100644 --- a/app/code/Magento/Bundle/etc/di.xml +++ b/app/code/Magento/Bundle/etc/di.xml @@ -143,7 +143,7 @@ 100000 - Magento\Catalog\Model\Indexer\CompositeProductBatchSizeManagement + Magento\Catalog\Model\Indexer\Price\CompositeProductBatchSizeManagement diff --git a/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/CompositeProductRowSizeEstimator.php b/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/CompositeProductRowSizeEstimator.php index 47f5443094381..254b8d3115e36 100644 --- a/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/CompositeProductRowSizeEstimator.php +++ b/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/CompositeProductRowSizeEstimator.php @@ -6,7 +6,6 @@ namespace Magento\Catalog\Model\ResourceModel\Product\Indexer\Price; -use Magento\Catalog\Model\ResourceModel\Product\Indexer\IndexTableRowSizeEstimator; use Magento\Framework\Indexer\IndexTableRowSizeEstimatorInterface; class CompositeProductRowSizeEstimator implements IndexTableRowSizeEstimatorInterface diff --git a/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/IndexTableRowSizeEstimator.php b/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/IndexTableRowSizeEstimator.php similarity index 96% rename from app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/IndexTableRowSizeEstimator.php rename to app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/IndexTableRowSizeEstimator.php index 706d4400b3319..a6e0ac2dbae31 100644 --- a/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/IndexTableRowSizeEstimator.php +++ b/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/IndexTableRowSizeEstimator.php @@ -4,7 +4,7 @@ * See COPYING.txt for license details. */ -namespace Magento\Catalog\Model\ResourceModel\Product\Indexer; +namespace Magento\Catalog\Model\ResourceModel\Product\Indexer\Price; class IndexTableRowSizeEstimator implements \Magento\Framework\Indexer\IndexTableRowSizeEstimatorInterface { diff --git a/app/code/Magento/Catalog/Test/Unit/Model/ResourceModel/Product/Indexer/Price/CompositeProductRowSizeEstimatorTest.php b/app/code/Magento/Catalog/Test/Unit/Model/ResourceModel/Product/Indexer/Price/CompositeProductRowSizeEstimatorTest.php index 4e7483644afcf..78c69f0a7fcfd 100644 --- a/app/code/Magento/Catalog/Test/Unit/Model/ResourceModel/Product/Indexer/Price/CompositeProductRowSizeEstimatorTest.php +++ b/app/code/Magento/Catalog/Test/Unit/Model/ResourceModel/Product/Indexer/Price/CompositeProductRowSizeEstimatorTest.php @@ -28,7 +28,7 @@ class CompositeProductRowSizeEstimatorTest extends \PHPUnit_Framework_TestCase protected function setUp() { $this->rowSizeEstimatorMock = $this->getMock( - \Magento\Catalog\Model\ResourceModel\Product\Indexer\IndexTableRowSizeEstimator::class, + \Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\IndexTableRowSizeEstimator::class, [], [], '', diff --git a/app/code/Magento/Catalog/Test/Unit/Model/ResourceModel/Product/Indexer/IndexTableRowSizeEstimatorTest.php b/app/code/Magento/Catalog/Test/Unit/Model/ResourceModel/Product/Indexer/Price/IndexTableRowSizeEstimatorTest.php similarity index 94% rename from app/code/Magento/Catalog/Test/Unit/Model/ResourceModel/Product/Indexer/IndexTableRowSizeEstimatorTest.php rename to app/code/Magento/Catalog/Test/Unit/Model/ResourceModel/Product/Indexer/Price/IndexTableRowSizeEstimatorTest.php index de66f83333daa..180ba229dbf0f 100644 --- a/app/code/Magento/Catalog/Test/Unit/Model/ResourceModel/Product/Indexer/IndexTableRowSizeEstimatorTest.php +++ b/app/code/Magento/Catalog/Test/Unit/Model/ResourceModel/Product/Indexer/Price/IndexTableRowSizeEstimatorTest.php @@ -4,12 +4,12 @@ * See COPYING.txt for license details. */ -namespace Magento\Catalog\Test\Unit\Model\ResourceModel\Product\Indexer; +namespace Magento\Catalog\Test\Unit\Model\ResourceModel\Product\Indexer\Price; class IndexTableRowSizeEstimatorTest extends \PHPUnit_Framework_TestCase { /** - * @var \Magento\Catalog\Model\ResourceModel\Product\Indexer\IndexTableRowSizeEstimator + * @var \Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\IndexTableRowSizeEstimator */ private $model; @@ -33,7 +33,7 @@ protected function setUp() '', false ); - $this->model = new \Magento\Catalog\Model\ResourceModel\Product\Indexer\IndexTableRowSizeEstimator( + $this->model = new \Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\IndexTableRowSizeEstimator( $this->websiteManagementMock, $this->collectionFactoryMock ); diff --git a/app/code/Magento/Catalog/etc/di.xml b/app/code/Magento/Catalog/etc/di.xml index 752fe4e79d721..d48e54d874a56 100644 --- a/app/code/Magento/Catalog/etc/di.xml +++ b/app/code/Magento/Catalog/etc/di.xml @@ -922,15 +922,20 @@ 200000 - \Magento\Framework\Indexer\BatchSizeManagement + Magento\Catalog\Model\Indexer\Price\BatchSizeManagement - + Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\CompositeProductRowSizeEstimator + + + \Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\IndexTableRowSizeEstimator + + Magento\Catalog\Model\Indexer\Product\Price\Processor::INDEXER_ID @@ -943,9 +948,4 @@ Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\FrontendResource - - - \Magento\Catalog\Model\ResourceModel\Product\Indexer\IndexTableRowSizeEstimator - - diff --git a/app/code/Magento/ConfigurableProduct/etc/di.xml b/app/code/Magento/ConfigurableProduct/etc/di.xml index f555e5f5cd963..f86760c7e348d 100644 --- a/app/code/Magento/ConfigurableProduct/etc/di.xml +++ b/app/code/Magento/ConfigurableProduct/etc/di.xml @@ -177,7 +177,7 @@ 100000 - Magento\Catalog\Model\Indexer\CompositeProductBatchSizeManagement + Magento\Catalog\Model\Indexer\Price\CompositeProductBatchSizeManagement diff --git a/app/code/Magento/GroupedProduct/etc/di.xml b/app/code/Magento/GroupedProduct/etc/di.xml index c3346656ef9b8..24bf2fa0cbad5 100644 --- a/app/code/Magento/GroupedProduct/etc/di.xml +++ b/app/code/Magento/GroupedProduct/etc/di.xml @@ -95,7 +95,7 @@ 100000 - Magento\Catalog\Model\Indexer\CompositeProductBatchSizeManagement + Magento\Catalog\Model\Indexer\Price\CompositeProductBatchSizeManagement From f837cac8c57dd9acf1b57515e62005a927b44b01 Mon Sep 17 00:00:00 2001 From: Stanislav Idolov Date: Fri, 10 Mar 2017 09:47:11 +0200 Subject: [PATCH 19/43] MAGETWO-64588: Create upgrade scripts --- app/code/Magento/Indexer/Setup/UpgradeSchema.php | 2 +- app/code/Magento/Indexer/etc/module.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/code/Magento/Indexer/Setup/UpgradeSchema.php b/app/code/Magento/Indexer/Setup/UpgradeSchema.php index 291a8dabbd39a..615519b2e1a23 100644 --- a/app/code/Magento/Indexer/Setup/UpgradeSchema.php +++ b/app/code/Magento/Indexer/Setup/UpgradeSchema.php @@ -22,7 +22,7 @@ public function upgrade(SchemaSetupInterface $setup, ModuleContextInterface $con { $setup->startSetup(); - if (version_compare($context->getVersion(), '2.2.0', '<')) { + if (version_compare($context->getVersion(), '2.1.0', '<')) { $this->addTableSuffixColumn($setup); } $setup->endSetup(); diff --git a/app/code/Magento/Indexer/etc/module.xml b/app/code/Magento/Indexer/etc/module.xml index b7535a2ff566f..60726e4379e55 100644 --- a/app/code/Magento/Indexer/etc/module.xml +++ b/app/code/Magento/Indexer/etc/module.xml @@ -6,7 +6,7 @@ */ --> - + From 393630cd5c2ff973b378ee8a57cc8c2fb69bbd37 Mon Sep 17 00:00:00 2001 From: Viktor Tymchynskyi Date: Fri, 3 Mar 2017 17:07:27 +0200 Subject: [PATCH 20/43] MAGETWO-65119: Create/update automated functional tests - Move Accept Payment and Deny Payment steps from Braintree to Sales module - Add Unhold order step --- .../Magento/Braintree/Test/etc/testcase.xml | 4 +- .../TestStep/PlaceOrderWithHostedProStep.php | 16 ++++- .../Test/Block/Adminhtml/Order/Actions.php | 20 ++++++ .../Test/TestStep/AcceptPaymentStep.php | 2 +- .../Test/TestStep/DenyPaymentStep.php | 2 +- .../Sales/Test/TestStep/UnholdOrderStep.php | 64 +++++++++++++++++++ 6 files changed, 103 insertions(+), 5 deletions(-) rename dev/tests/functional/tests/app/Magento/{Braintree => Sales}/Test/TestStep/AcceptPaymentStep.php (96%) rename dev/tests/functional/tests/app/Magento/{Braintree => Sales}/Test/TestStep/DenyPaymentStep.php (96%) create mode 100644 dev/tests/functional/tests/app/Magento/Sales/Test/TestStep/UnholdOrderStep.php diff --git a/dev/tests/functional/tests/app/Magento/Braintree/Test/etc/testcase.xml b/dev/tests/functional/tests/app/Magento/Braintree/Test/etc/testcase.xml index ffe67f1979ee0..36a4ecff2c6be 100644 --- a/dev/tests/functional/tests/app/Magento/Braintree/Test/etc/testcase.xml +++ b/dev/tests/functional/tests/app/Magento/Braintree/Test/etc/testcase.xml @@ -155,7 +155,7 @@ - + @@ -169,7 +169,7 @@ - + diff --git a/dev/tests/functional/tests/app/Magento/Paypal/Test/TestStep/PlaceOrderWithHostedProStep.php b/dev/tests/functional/tests/app/Magento/Paypal/Test/TestStep/PlaceOrderWithHostedProStep.php index 11bc0dd225e75..140c1a6b3e039 100644 --- a/dev/tests/functional/tests/app/Magento/Paypal/Test/TestStep/PlaceOrderWithHostedProStep.php +++ b/dev/tests/functional/tests/app/Magento/Paypal/Test/TestStep/PlaceOrderWithHostedProStep.php @@ -7,6 +7,7 @@ namespace Magento\Paypal\Test\TestStep; use Magento\Checkout\Test\Page\CheckoutOnepage; +use Magento\Checkout\Test\Page\CheckoutOnepageSuccess; use Magento\Mtf\Fixture\FixtureFactory; use Magento\Mtf\Fixture\FixtureInterface; use Magento\Mtf\TestStep\TestStepInterface; @@ -53,10 +54,16 @@ class PlaceOrderWithHostedProStep implements TestStepInterface */ private $creditCard; + /** + * @var CheckoutOnepageSuccess + */ + private $checkoutOnepageSuccess; + /** * @param CheckoutOnepage $checkoutOnepage * @param FixtureFactory $fixtureFactory * @param CreditCard $creditCard + * @param CheckoutOnepageSuccess $checkoutOnepageSuccess * @param array $payment * @param array $products */ @@ -64,6 +71,7 @@ public function __construct( CheckoutOnepage $checkoutOnepage, FixtureFactory $fixtureFactory, CreditCard $creditCard, + CheckoutOnepageSuccess $checkoutOnepageSuccess, array $payment, array $products ) { @@ -72,6 +80,7 @@ public function __construct( $this->creditCard = $creditCard; $this->payment = $payment; $this->products = $products; + $this->checkoutOnepageSuccess = $checkoutOnepageSuccess; } /** @@ -90,6 +99,8 @@ public function run() $this->checkoutOnepage->getHostedProBlock()->fillPaymentData($this->creditCard); $attempts++; } + + $orderId = $this->checkoutOnepageSuccess->getSuccessBlock()->getGuestOrderId(); /** @var OrderInjectable $order */ $order = $this->fixtureFactory->createByCode( 'orderInjectable', @@ -99,6 +110,9 @@ public function run() ] ] ); - return ['order' => $order]; + return [ + 'orderId' => $orderId, + 'order' => $order + ]; } } diff --git a/dev/tests/functional/tests/app/Magento/Sales/Test/Block/Adminhtml/Order/Actions.php b/dev/tests/functional/tests/app/Magento/Sales/Test/Block/Adminhtml/Order/Actions.php index d13855db0cc76..9008526b0f81d 100644 --- a/dev/tests/functional/tests/app/Magento/Sales/Test/Block/Adminhtml/Order/Actions.php +++ b/dev/tests/functional/tests/app/Magento/Sales/Test/Block/Adminhtml/Order/Actions.php @@ -12,6 +12,9 @@ /** * Order actions block. + * + * @SuppressWarnings(PHPMD.TooManyFields) + * @SuppressWarnings(PHPMD.TooManyPublicMethods) */ class Actions extends Block { @@ -57,6 +60,13 @@ class Actions extends Block */ protected $hold = '[id$=hold-button]'; + /** + * 'Unhold' button. + * + * @var string + */ + protected $unhold = '[id$=unhold-button]'; + /** * 'Invoice' button. * @@ -212,6 +222,16 @@ public function hold() $this->_rootElement->find($this->hold)->click(); } + /** + * Unhold order. + * + * @return void + */ + public function unhold() + { + $this->_rootElement->find($this->unhold)->click(); + } + /** * Order credit memo. * diff --git a/dev/tests/functional/tests/app/Magento/Braintree/Test/TestStep/AcceptPaymentStep.php b/dev/tests/functional/tests/app/Magento/Sales/Test/TestStep/AcceptPaymentStep.php similarity index 96% rename from dev/tests/functional/tests/app/Magento/Braintree/Test/TestStep/AcceptPaymentStep.php rename to dev/tests/functional/tests/app/Magento/Sales/Test/TestStep/AcceptPaymentStep.php index 9da200a2eee8f..05d53179fd145 100644 --- a/dev/tests/functional/tests/app/Magento/Braintree/Test/TestStep/AcceptPaymentStep.php +++ b/dev/tests/functional/tests/app/Magento/Sales/Test/TestStep/AcceptPaymentStep.php @@ -3,7 +3,7 @@ * Copyright © 2013-2017 Magento, Inc. All rights reserved. * See COPYING.txt for license details. */ -namespace Magento\Braintree\Test\TestStep; +namespace Magento\Sales\Test\TestStep; use Magento\Sales\Test\Fixture\OrderInjectable; use Magento\Sales\Test\Page\Adminhtml\OrderIndex; diff --git a/dev/tests/functional/tests/app/Magento/Braintree/Test/TestStep/DenyPaymentStep.php b/dev/tests/functional/tests/app/Magento/Sales/Test/TestStep/DenyPaymentStep.php similarity index 96% rename from dev/tests/functional/tests/app/Magento/Braintree/Test/TestStep/DenyPaymentStep.php rename to dev/tests/functional/tests/app/Magento/Sales/Test/TestStep/DenyPaymentStep.php index e47a5af23a0d3..2f1747c57f494 100644 --- a/dev/tests/functional/tests/app/Magento/Braintree/Test/TestStep/DenyPaymentStep.php +++ b/dev/tests/functional/tests/app/Magento/Sales/Test/TestStep/DenyPaymentStep.php @@ -3,7 +3,7 @@ * Copyright © 2013-2017 Magento, Inc. All rights reserved. * See COPYING.txt for license details. */ -namespace Magento\Braintree\Test\TestStep; +namespace Magento\Sales\Test\TestStep; use Magento\Sales\Test\Fixture\OrderInjectable; use Magento\Sales\Test\Page\Adminhtml\OrderIndex; diff --git a/dev/tests/functional/tests/app/Magento/Sales/Test/TestStep/UnholdOrderStep.php b/dev/tests/functional/tests/app/Magento/Sales/Test/TestStep/UnholdOrderStep.php new file mode 100644 index 0000000000000..7ccd77a6ddf25 --- /dev/null +++ b/dev/tests/functional/tests/app/Magento/Sales/Test/TestStep/UnholdOrderStep.php @@ -0,0 +1,64 @@ +orderIndex = $orderIndex; + $this->salesOrderView = $salesOrderView; + $this->order = $order; + } + + /** + * On hold order in admin. + * + * @return void + */ + public function run() + { + $this->orderIndex->open(); + $this->orderIndex->getSalesOrderGrid()->searchAndOpen(['id' => $this->order->getId()]); + $this->salesOrderView->getPageActions()->unhold(); + } +} From 59ce6dda9e4a63354e10fad66657a1fcf9d5b4f4 Mon Sep 17 00:00:00 2001 From: Stanislav Idolov Date: Mon, 13 Mar 2017 09:57:27 +0200 Subject: [PATCH 21/43] MAGETWO-64187: Batch size algorithm --- app/code/Magento/Catalog/etc/di.xml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/app/code/Magento/Catalog/etc/di.xml b/app/code/Magento/Catalog/etc/di.xml index 7aadad5befb1e..8d4000377c7ad 100644 --- a/app/code/Magento/Catalog/etc/di.xml +++ b/app/code/Magento/Catalog/etc/di.xml @@ -63,7 +63,6 @@ - @@ -926,14 +925,14 @@ - + - Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\CompositeProductRowSizeEstimator + Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\CompositeProductRowSizeEstimator - + - \Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\IndexTableRowSizeEstimator + Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\IndexTableRowSizeEstimator From 01da0004a6a5191607556efb38b4eca5829a2a24 Mon Sep 17 00:00:00 2001 From: Stanislav Idolov Date: Mon, 13 Mar 2017 10:57:37 +0200 Subject: [PATCH 22/43] MAGETWO-64187: Batch size algorithm --- .../Product/Indexer/Price/BatchSizeCalculator.php | 3 +++ .../Indexer/Price/CompositeProductRowSizeEstimator.php | 3 +++ .../Product/Indexer/Price/IndexTableRowSizeEstimator.php | 4 ++++ .../Magento/Framework/Indexer/BatchSizeManagement.php | 8 +++++--- .../Framework/Indexer/BatchSizeManagementInterface.php | 1 + .../Indexer/IndexTableRowSizeEstimatorInterface.php | 6 +++++- 6 files changed, 21 insertions(+), 4 deletions(-) diff --git a/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/BatchSizeCalculator.php b/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/BatchSizeCalculator.php index abba22333a57f..e5b8b45679fac 100644 --- a/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/BatchSizeCalculator.php +++ b/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/BatchSizeCalculator.php @@ -6,6 +6,9 @@ namespace Magento\Catalog\Model\ResourceModel\Product\Indexer\Price; +/** + * Ensure that size of index MEMORY table is enough for configured rows count in batch. + */ class BatchSizeCalculator { /** diff --git a/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/CompositeProductRowSizeEstimator.php b/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/CompositeProductRowSizeEstimator.php index 254b8d3115e36..043ee1cc24a8f 100644 --- a/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/CompositeProductRowSizeEstimator.php +++ b/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/CompositeProductRowSizeEstimator.php @@ -8,6 +8,9 @@ use Magento\Framework\Indexer\IndexTableRowSizeEstimatorInterface; +/** + * Estimate index memory size for largest composite product in catalog. + */ class CompositeProductRowSizeEstimator implements IndexTableRowSizeEstimatorInterface { /** diff --git a/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/IndexTableRowSizeEstimator.php b/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/IndexTableRowSizeEstimator.php index a6e0ac2dbae31..44c486046c8db 100644 --- a/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/IndexTableRowSizeEstimator.php +++ b/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/IndexTableRowSizeEstimator.php @@ -6,6 +6,10 @@ namespace Magento\Catalog\Model\ResourceModel\Product\Indexer\Price; +/** + * Estimate index memory size for simple product. + * Size depends on websites and customer groups count. + */ class IndexTableRowSizeEstimator implements \Magento\Framework\Indexer\IndexTableRowSizeEstimatorInterface { /** diff --git a/lib/internal/Magento/Framework/Indexer/BatchSizeManagement.php b/lib/internal/Magento/Framework/Indexer/BatchSizeManagement.php index e06c27ab10aaf..23eb7b8ac4f05 100644 --- a/lib/internal/Magento/Framework/Indexer/BatchSizeManagement.php +++ b/lib/internal/Magento/Framework/Indexer/BatchSizeManagement.php @@ -6,6 +6,9 @@ namespace Magento\Framework\Indexer; +/** + * Class set MEMORY table size for indexer processes according batch size and index row size. + */ class BatchSizeManagement implements \Magento\Framework\Indexer\BatchSizeManagementInterface { /** @@ -28,14 +31,13 @@ public function __construct( */ public function ensureBatchSize(\Magento\Framework\DB\Adapter\AdapterInterface $connection, $batchSize) { - // Calculate memory table size for product - $memoryForLargeComposite = $this->rowSizeEstimator->estimateRowSize(); + $rowMemory = $this->rowSizeEstimator->estimateRowSize(); $maxHeapTableSize = $connection->fetchOne('SELECT @@max_heap_table_size;'); $tmpTableSize = $connection->fetchOne('SELECT @@tmp_table_size;'); $maxMemoryTableSize = min($maxHeapTableSize, $tmpTableSize); - $size = (int) ($memoryForLargeComposite * $batchSize); + $size = (int) ($rowMemory * $batchSize); if ($maxMemoryTableSize < $size) { $connection->query('SET SESSION tmp_table_size = ' . $size . ';'); diff --git a/lib/internal/Magento/Framework/Indexer/BatchSizeManagementInterface.php b/lib/internal/Magento/Framework/Indexer/BatchSizeManagementInterface.php index 8ff4e65ddb5f1..28f9281c95f11 100644 --- a/lib/internal/Magento/Framework/Indexer/BatchSizeManagementInterface.php +++ b/lib/internal/Magento/Framework/Indexer/BatchSizeManagementInterface.php @@ -9,6 +9,7 @@ /** * Batch size manager can be used to ensure that MEMORY table has enough memory for data in batch. + * @api */ interface BatchSizeManagementInterface { diff --git a/lib/internal/Magento/Framework/Indexer/IndexTableRowSizeEstimatorInterface.php b/lib/internal/Magento/Framework/Indexer/IndexTableRowSizeEstimatorInterface.php index 4c920e1ee0f91..acede2f9a03ed 100644 --- a/lib/internal/Magento/Framework/Indexer/IndexTableRowSizeEstimatorInterface.php +++ b/lib/internal/Magento/Framework/Indexer/IndexTableRowSizeEstimatorInterface.php @@ -6,10 +6,14 @@ namespace Magento\Framework\Indexer; +/** + * Calculate memory size for entity according different dimensions. + * @api + */ interface IndexTableRowSizeEstimatorInterface { /** - * Calculate memory size for product row. + * Calculate memory size for entity row. * * @return float */ From 02e96a94fccbfb4eab693b8369e4b7aa22aeb37f Mon Sep 17 00:00:00 2001 From: Stanislav Idolov Date: Tue, 14 Mar 2017 12:51:11 +0200 Subject: [PATCH 23/43] MAGETWO-64187: Batch size algorithm --- .../Model/ResourceModel/Indexer/Price.php | 3 +-- .../Indexer/Product/Price/AbstractAction.php | 13 ++++++------ .../Indexer/Product/Price/Action/Full.php | 21 +++++++++---------- .../Indexer/Price/BatchSizeCalculator.php | 4 +++- .../CompositeProductRowSizeEstimator.php | 2 +- .../Product/Indexer/Price/DefaultPrice.php | 3 +-- 6 files changed, 22 insertions(+), 24 deletions(-) diff --git a/app/code/Magento/Bundle/Model/ResourceModel/Indexer/Price.php b/app/code/Magento/Bundle/Model/ResourceModel/Indexer/Price.php index a559cee731405..bf2ca58e2babd 100644 --- a/app/code/Magento/Bundle/Model/ResourceModel/Indexer/Price.php +++ b/app/code/Magento/Bundle/Model/ResourceModel/Indexer/Price.php @@ -15,8 +15,7 @@ class Price extends \Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\DefaultPrice { /** - * {@inheritdoc} - * @param null|array $entityIds + * @inheritdoc */ protected function reindex($entityIds = null) { diff --git a/app/code/Magento/Catalog/Model/Indexer/Product/Price/AbstractAction.php b/app/code/Magento/Catalog/Model/Indexer/Product/Price/AbstractAction.php index 3a857693acf88..d66bc1b116f99 100644 --- a/app/code/Magento/Catalog/Model/Indexer/Product/Price/AbstractAction.php +++ b/app/code/Magento/Catalog/Model/Indexer/Product/Price/AbstractAction.php @@ -135,7 +135,7 @@ protected function _syncData(array $processIds = []) { // delete invalid rows $select = $this->_connection->select()->from( - ['index_price' => $this->getIndexerDefaultTable()], + ['index_price' => $this->getIndexTargetTable()], null )->joinLeft( ['ip_tmp' => $this->_defaultIndexerResource->getIdxTable()], @@ -152,7 +152,7 @@ protected function _syncData(array $processIds = []) $this->_insertFromTable( $this->_defaultIndexerResource->getIdxTable(), - $this->getIndexerDefaultTable() + $this->getIndexTargetTable() ); return $this; } @@ -471,7 +471,7 @@ protected function _copyRelationIndexData($parentIds, $excludeIds = null) if ($children) { $select = $this->_connection->select()->from( - $this->getIndexerDefaultTable() + $this->getIndexTargetTable() )->where( 'entity_id IN(?)', $children @@ -484,14 +484,13 @@ protected function _copyRelationIndexData($parentIds, $excludeIds = null) } /** - * Returns default table name for writing + * Retrieve index table that will be used for write operations. * - * Method support writing to frontend table in case with partial reindex - * and case with writing to replica table during full reindex + * This method is used to during both partial and full reindex to identify the the table. * * @return string */ - protected function getIndexerDefaultTable() + protected function getIndexTargetTable() { return $this->indexerFrontendResource->getMainTable(); } diff --git a/app/code/Magento/Catalog/Model/Indexer/Product/Price/Action/Full.php b/app/code/Magento/Catalog/Model/Indexer/Product/Price/Action/Full.php index cdea22d10ae9f..8176451a7c327 100644 --- a/app/code/Magento/Catalog/Model/Indexer/Product/Price/Action/Full.php +++ b/app/code/Magento/Catalog/Model/Indexer/Product/Price/Action/Full.php @@ -57,15 +57,6 @@ public function __construct( \Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\BatchSizeCalculator $batchSizeCalculator = null, \Magento\Framework\Indexer\BatchProviderInterface $batchProvider = null ) { - $this->metadataPool = $metadataPool ?: ObjectManager::getInstance()->get( - \Magento\Framework\EntityManager\MetadataPool::class - ); - $this->batchSizeCalculator = $batchSizeCalculator ?: ObjectManager::getInstance()->get( - \Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\BatchSizeCalculator::class - ); - $this->batchProvider = $batchProvider ?: ObjectManager::getInstance()->get( - \Magento\Framework\Indexer\BatchProviderInterface::class - ); parent::__construct( $config, $storeManager, @@ -77,6 +68,15 @@ public function __construct( $defaultIndexerResource, $indexerFrontendResource ); + $this->metadataPool = $metadataPool ?: ObjectManager::getInstance()->get( + \Magento\Framework\EntityManager\MetadataPool::class + ); + $this->batchSizeCalculator = $batchSizeCalculator ?: ObjectManager::getInstance()->get( + \Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\BatchSizeCalculator::class + ); + $this->batchProvider = $batchProvider ?: ObjectManager::getInstance()->get( + \Magento\Framework\Indexer\BatchProviderInterface::class + ); } /** @@ -140,9 +140,8 @@ public function execute($ids = null) /** * @inheritdoc - * @return string */ - protected function getIndexerDefaultTable() + protected function getIndexTargetTable() { return $this->_defaultIndexerResource->getMainTable(); } diff --git a/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/BatchSizeCalculator.php b/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/BatchSizeCalculator.php index e5b8b45679fac..68f09b66cf08a 100644 --- a/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/BatchSizeCalculator.php +++ b/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/BatchSizeCalculator.php @@ -33,7 +33,9 @@ public function __construct(array $batchRowsCount, array $estimators) } /** - * Composite object for batch size calculators + * Retrieve batch size for the given indexer. + * + * Ensure that the database will be able to handle provided batch size correctly. * * @param \Magento\Framework\DB\Adapter\AdapterInterface $connection * @param string $indexerTypeId diff --git a/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/CompositeProductRowSizeEstimator.php b/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/CompositeProductRowSizeEstimator.php index 043ee1cc24a8f..758435b3fae12 100644 --- a/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/CompositeProductRowSizeEstimator.php +++ b/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/CompositeProductRowSizeEstimator.php @@ -38,7 +38,7 @@ public function __construct( /** * Calculate memory size for largest composite product in database. * - * @inheritdoc + * {@inheritdoc} */ public function estimateRowSize() { diff --git a/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/DefaultPrice.php b/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/DefaultPrice.php index fea611e441ad0..2da75f81c68ec 100644 --- a/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/DefaultPrice.php +++ b/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/DefaultPrice.php @@ -709,10 +709,9 @@ protected function hasEntity() } /** - * @inheritdoc * Returns main table name based on the suffix stored in the 'indexer_state' table * - * @return string + * {@inheritdoc} */ public function getMainTable() { From f33b393a265f6feeb132d54441512a78aa2e13aa Mon Sep 17 00:00:00 2001 From: Stanislav Idolov Date: Tue, 14 Mar 2017 15:20:09 +0200 Subject: [PATCH 24/43] MAGETWO-64187: Batch size algorithm --- .../Catalog/Model/Indexer/Product/Price/AbstractAction.php | 2 +- .../Magento/Catalog/Model/ResourceModel/Layer/Filter/Price.php | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/app/code/Magento/Catalog/Model/Indexer/Product/Price/AbstractAction.php b/app/code/Magento/Catalog/Model/Indexer/Product/Price/AbstractAction.php index d66bc1b116f99..09e36953529bf 100644 --- a/app/code/Magento/Catalog/Model/Indexer/Product/Price/AbstractAction.php +++ b/app/code/Magento/Catalog/Model/Indexer/Product/Price/AbstractAction.php @@ -486,7 +486,7 @@ protected function _copyRelationIndexData($parentIds, $excludeIds = null) /** * Retrieve index table that will be used for write operations. * - * This method is used to during both partial and full reindex to identify the the table. + * This method is used during both partial and full reindex to identify the the table. * * @return string */ diff --git a/app/code/Magento/Catalog/Model/ResourceModel/Layer/Filter/Price.php b/app/code/Magento/Catalog/Model/ResourceModel/Layer/Filter/Price.php index 608e2bfbad6cd..d572ee79fa79f 100644 --- a/app/code/Magento/Catalog/Model/ResourceModel/Layer/Filter/Price.php +++ b/app/code/Magento/Catalog/Model/ResourceModel/Layer/Filter/Price.php @@ -394,9 +394,6 @@ protected function _getIndexTableAlias() /** * @inheritdoc - * Returns main table name in depends of the suffix stored in the 'indexer_state' table - * - * @return string */ public function getMainTable() { From 04bc651f1b7bdf324d67b94631ea18866b8e3855 Mon Sep 17 00:00:00 2001 From: Dmytro Yushkin Date: Fri, 17 Mar 2017 18:07:41 +0200 Subject: [PATCH 25/43] MAGETWO-66178: Update marketing information, order page and order list with new labels --- .../web/css/source/module/_config.less | 20 ++++++++++--------- .../web/css/source/module/_order.less | 12 +++++------ 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/app/design/adminhtml/Magento/backend/Magento_Signifyd/web/css/source/module/_config.less b/app/design/adminhtml/Magento/backend/Magento_Signifyd/web/css/source/module/_config.less index fca1da0f6ffa1..5aee1d2792be7 100644 --- a/app/design/adminhtml/Magento/backend/Magento_Signifyd/web/css/source/module/_config.less +++ b/app/design/adminhtml/Magento/backend/Magento_Signifyd/web/css/source/module/_config.less @@ -9,15 +9,17 @@ // Outer slider of configuration // .signifyd-logo-header { - > .entry-edit-head { - > a:after {content: url(@image-signifyd-logo-path)} - } + > .entry-edit-head { + > a:after { + content: url(@image-signifyd-logo-path); + } + } - ul { - margin-left: 5em; - } + ul { + margin-left: 5em; + } - .webhook-url { - word-break: break-all; - } + .webhook-url { + word-break: break-all; + } } diff --git a/app/design/adminhtml/Magento/backend/Magento_Signifyd/web/css/source/module/_order.less b/app/design/adminhtml/Magento/backend/Magento_Signifyd/web/css/source/module/_order.less index 994a26c12095c..da9f1335a5e2d 100644 --- a/app/design/adminhtml/Magento/backend/Magento_Signifyd/web/css/source/module/_order.less +++ b/app/design/adminhtml/Magento/backend/Magento_Signifyd/web/css/source/module/_order.less @@ -8,16 +8,16 @@ // --------------------------------------------- .order-case-table { - &:extend(.abs-order-tables all); - &:extend(.abs-order-tbody-border all); + &:extend(.abs-order-tables all); + &:extend(.abs-order-tbody-border all); } // // Layout // --------------------------------------------- .media-width(@extremum, @break) when (@extremum = 'min') and (@break = @screen__m) { - .case-information { - float: left; - #mix-grid .width(6,12); - } + .case-information { + float: left; + #mix-grid .width(6,12); + } } From f33c0ff17f09529ef06ae90863cda49135266292 Mon Sep 17 00:00:00 2001 From: Stanislav Idolov Date: Mon, 20 Mar 2017 09:50:52 +0200 Subject: [PATCH 26/43] MAGETWO-66195: SQL Error during price reindexation process for custom profile --- app/code/Magento/Bundle/etc/di.xml | 6 ++ .../Indexer/Product/Price/Action/Full.php | 13 +-- .../CompositeProductRowSizeEstimator.php | 32 +++++-- .../Price/IndexTableRowSizeEstimator.php | 4 +- .../Indexer/TemporaryTableStrategy.php | 81 +++++++++++++++++ .../Indexer/TemporaryTableStrategyTest.php | 90 +++++++++++++++++++ app/code/Magento/Catalog/etc/di.xml | 7 ++ .../Magento/ConfigurableProduct/etc/di.xml | 6 ++ app/code/Magento/Downloadable/etc/di.xml | 6 ++ 9 files changed, 230 insertions(+), 15 deletions(-) create mode 100644 app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/TemporaryTableStrategy.php create mode 100644 app/code/Magento/Catalog/Test/Unit/Model/ResourceModel/Product/Indexer/TemporaryTableStrategyTest.php diff --git a/app/code/Magento/Bundle/etc/di.xml b/app/code/Magento/Bundle/etc/di.xml index 3cbe4fcd00590..37be75d727e7a 100644 --- a/app/code/Magento/Bundle/etc/di.xml +++ b/app/code/Magento/Bundle/etc/di.xml @@ -147,4 +147,10 @@ + + + Magento\Catalog\Model\ResourceModel\Product\Indexer\TemporaryTableStrategy + indexer + + diff --git a/app/code/Magento/Catalog/Model/Indexer/Product/Price/Action/Full.php b/app/code/Magento/Catalog/Model/Indexer/Product/Price/Action/Full.php index 8176451a7c327..70dbb68fb9762 100644 --- a/app/code/Magento/Catalog/Model/Indexer/Product/Price/Action/Full.php +++ b/app/code/Magento/Catalog/Model/Indexer/Product/Price/Action/Full.php @@ -117,19 +117,22 @@ public function execute($ids = null) $entityIds = $this->batchProvider->getBatchIds($connection, $select, $batch); if (!empty($entityIds)) { - $this->_emptyTable($this->_defaultIndexerResource->getIdxTable()); + // Temporary table will created + $idxTableName = $this->_defaultIndexerResource->getIdxTable(); + if ($indexer->getIsComposite()) { $this->_copyRelationIndexData($entityIds); } $this->_prepareTierPriceIndex($entityIds); + // Reindex entities by id $indexer->reindexEntity($entityIds); // Sync data from temp table to index table - $this->_insertFromTable( - $this->_defaultIndexerResource->getIdxTable(), - $this->_defaultIndexerResource->getMainTable() - ); + $this->_insertFromTable($idxTableName, $this->_defaultIndexerResource->getMainTable()); + + // Drop temporary index table + $connection->dropTable($idxTableName); } } } diff --git a/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/CompositeProductRowSizeEstimator.php b/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/CompositeProductRowSizeEstimator.php index 758435b3fae12..7f3ad52b4c094 100644 --- a/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/CompositeProductRowSizeEstimator.php +++ b/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/CompositeProductRowSizeEstimator.php @@ -7,6 +7,8 @@ namespace Magento\Catalog\Model\ResourceModel\Product\Indexer\Price; use Magento\Framework\Indexer\IndexTableRowSizeEstimatorInterface; +use Magento\Store\Api\WebsiteManagementInterface; +use Magento\Customer\Model\ResourceModel\Group\CollectionFactory; /** * Estimate index memory size for largest composite product in catalog. @@ -14,25 +16,33 @@ class CompositeProductRowSizeEstimator implements IndexTableRowSizeEstimatorInterface { /** - * @var IndexTableRowSizeEstimator + * @var DefaultPrice */ - private $indexTableRowSizeEstimator; + private $indexerResource; /** - * @var DefaultPrice + * @var WebsiteManagementInterface */ - private $indexerResource; + private $websiteManagement; + + /** + * @var CollectionFactory + */ + private $collectionFactory; /** * @param DefaultPrice $indexerResource - * @param IndexTableRowSizeEstimator $indexTableRowSizeEstimator + * @param WebsiteManagementInterface $websiteManagement + * @param CollectionFactory $collectionFactory */ public function __construct( DefaultPrice $indexerResource, - IndexTableRowSizeEstimator $indexTableRowSizeEstimator + WebsiteManagementInterface $websiteManagement, + CollectionFactory $collectionFactory ) { $this->indexerResource = $indexerResource; - $this->indexTableRowSizeEstimator = $indexTableRowSizeEstimator; + $this->websiteManagement = $websiteManagement; + $this->collectionFactory = $collectionFactory; } /** @@ -42,6 +52,9 @@ public function __construct( */ public function estimateRowSize() { + $websitesCount = $this->websiteManagement->getCount(); + $customerGroupCount = $this->collectionFactory->create()->getSize(); + $connection = $this->indexerResource->getConnection(); $relationSelect = $connection->select(); $relationSelect->from( @@ -61,7 +74,10 @@ public function estimateRowSize() * Calculate memory size for largest composite product in database. * * $maxRelatedProductCount - maximum number of related products + * $websitesCount - active websites + * $customerGroupCount - active customer groups + * 200 - calculated memory size for one record in catalog_product_index_price table */ - return ceil($maxRelatedProductCount * $this->indexTableRowSizeEstimator->estimateRowSize()); + return ceil($maxRelatedProductCount * $websitesCount * $customerGroupCount * 200); } } diff --git a/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/IndexTableRowSizeEstimator.php b/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/IndexTableRowSizeEstimator.php index 44c486046c8db..ad8f0464c0271 100644 --- a/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/IndexTableRowSizeEstimator.php +++ b/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/IndexTableRowSizeEstimator.php @@ -51,8 +51,8 @@ public function estimateRowSize() * * $websitesCount - active websites * $customerGroupCount - active customer groups - * 90 - calculated memory size for one record in catalog_product_index_price table + * 120 - calculated memory size for one record in catalog_product_index_price table */ - return ceil($websitesCount * $customerGroupCount * 90); + return ceil($websitesCount * $customerGroupCount * 120); } } diff --git a/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/TemporaryTableStrategy.php b/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/TemporaryTableStrategy.php new file mode 100644 index 0000000000000..afff681286a11 --- /dev/null +++ b/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/TemporaryTableStrategy.php @@ -0,0 +1,81 @@ +strategy = $strategy; + $this->resource = $resource; + } + + /** + * @inheritdoc + */ + public function getUseIdxTable() + { + return $this->strategy->getUseIdxTable(); + } + + /** + * @inheritdoc + */ + public function setUseIdxTable($value = false) + { + return $this->strategy->setUseIdxTable($value); + } + + /** + * @inheritdoc + */ + public function getTableName($tablePrefix) + { + return $this->resource->getTableName($this->prepareTableName($tablePrefix)); + } + + /** + * Create temporary index table based on memory table + * + * {@inheritdoc} + */ + public function prepareTableName($tablePrefix) + { + if ($this->getUseIdxTable()) { + return $tablePrefix . self::IDX_SUFFIX; + } + + // Create temporary table + $this->resource->getConnection('indexer')->createTemporaryTableLike( + $this->resource->getTableName($tablePrefix . self::TEMP_SUFFIX), + $this->resource->getTableName($tablePrefix . self::TMP_SUFFIX), + true + ); + return $tablePrefix . self::TEMP_SUFFIX; + } +} diff --git a/app/code/Magento/Catalog/Test/Unit/Model/ResourceModel/Product/Indexer/TemporaryTableStrategyTest.php b/app/code/Magento/Catalog/Test/Unit/Model/ResourceModel/Product/Indexer/TemporaryTableStrategyTest.php new file mode 100644 index 0000000000000..257e65b48b22b --- /dev/null +++ b/app/code/Magento/Catalog/Test/Unit/Model/ResourceModel/Product/Indexer/TemporaryTableStrategyTest.php @@ -0,0 +1,90 @@ +tableStrategyMock = $this->getMock(\Magento\Framework\Indexer\Table\Strategy::class, [], [], '', false); + $this->resourceMock = $this->getMock(\Magento\Framework\App\ResourceConnection::class, [], [], '', false); + + $this->model = new \Magento\Catalog\Model\ResourceModel\Product\Indexer\TemporaryTableStrategy( + $this->tableStrategyMock, + $this->resourceMock + ); + } + + public function testGetUseIdxTable() + { + $this->tableStrategyMock->expects($this->once())->method('getUseIdxTable')->willReturn(true); + $this->assertTrue($this->model->getUseIdxTable()); + } + + public function testSetUseIdxTable() + { + $this->tableStrategyMock->expects($this->once())->method('setUseIdxTable')->with(true)->willReturnSelf(); + $this->assertEquals($this->tableStrategyMock, $this->model->setUseIdxTable(true)); + } + + public function testGetTableName() + { + $tablePrefix = 'prefix'; + $expectedResult = $tablePrefix . \Magento\Framework\Indexer\Table\StrategyInterface::IDX_SUFFIX; + $this->tableStrategyMock->expects($this->once())->method('getUseIdxTable')->willReturn(true); + $this->resourceMock->expects($this->once()) + ->method('getTableName') + ->with($expectedResult) + ->willReturn($expectedResult); + $this->assertEquals($expectedResult, $this->model->getTableName($tablePrefix)); + } + + public function testPrepareTableName() + { + $tablePrefix = 'prefix'; + $expectedResult = $tablePrefix . TemporaryTableStrategy::TEMP_SUFFIX; + $tempTableName = $tablePrefix . \Magento\Framework\Indexer\Table\StrategyInterface::TMP_SUFFIX; + + $this->tableStrategyMock->expects($this->once())->method('getUseIdxTable')->willReturn(false); + $connectionMock = $this->getMock(\Magento\Framework\DB\Adapter\AdapterInterface::class); + + $this->resourceMock->expects($this->once()) + ->method('getConnection') + ->with('indexer') + ->willReturn($connectionMock); + $this->resourceMock->expects($this->at(1)) + ->method('getTableName') + ->with($expectedResult) + ->willReturn($expectedResult); + $this->resourceMock->expects($this->at(2)) + ->method('getTableName') + ->with($tempTableName) + ->willReturn($tempTableName); + $connectionMock->expects($this->once()) + ->method('createTemporaryTableLike') + ->with($expectedResult, $tempTableName, true); + + $this->assertEquals($expectedResult, $this->model->prepareTableName($tablePrefix)); + } +} diff --git a/app/code/Magento/Catalog/etc/di.xml b/app/code/Magento/Catalog/etc/di.xml index 8d4000377c7ad..4f17dff6ba7a4 100644 --- a/app/code/Magento/Catalog/etc/di.xml +++ b/app/code/Magento/Catalog/etc/di.xml @@ -947,4 +947,11 @@ Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\FrontendResource + + + + Magento\Catalog\Model\ResourceModel\Product\Indexer\TemporaryTableStrategy + indexer + + diff --git a/app/code/Magento/ConfigurableProduct/etc/di.xml b/app/code/Magento/ConfigurableProduct/etc/di.xml index f86760c7e348d..113ce13f86ccf 100644 --- a/app/code/Magento/ConfigurableProduct/etc/di.xml +++ b/app/code/Magento/ConfigurableProduct/etc/di.xml @@ -181,4 +181,10 @@ + + + Magento\Catalog\Model\ResourceModel\Product\Indexer\TemporaryTableStrategy + indexer + + diff --git a/app/code/Magento/Downloadable/etc/di.xml b/app/code/Magento/Downloadable/etc/di.xml index 3f2958b8240b1..0429cacc8999b 100644 --- a/app/code/Magento/Downloadable/etc/di.xml +++ b/app/code/Magento/Downloadable/etc/di.xml @@ -151,4 +151,10 @@ + + + Magento\Catalog\Model\ResourceModel\Product\Indexer\TemporaryTableStrategy + indexer + + From 845ddbc3ad8c616f6a4fa8bd9d464113f8db6237 Mon Sep 17 00:00:00 2001 From: Stanislav Idolov Date: Mon, 20 Mar 2017 12:26:41 +0200 Subject: [PATCH 27/43] MAGETWO-66195: SQL Error during price reindexation process for custom profile --- .../Indexer/Product/Price/Action/Full.php | 3 +- .../Indexer/TemporaryTableStrategy.php | 3 ++ .../CompositeProductRowSizeEstimatorTest.php | 37 ++++++++++++++++--- .../Price/IndexTableRowSizeEstimatorTest.php | 2 +- 4 files changed, 37 insertions(+), 8 deletions(-) diff --git a/app/code/Magento/Catalog/Model/Indexer/Product/Price/Action/Full.php b/app/code/Magento/Catalog/Model/Indexer/Product/Price/Action/Full.php index 70dbb68fb9762..97d05419ce289 100644 --- a/app/code/Magento/Catalog/Model/Indexer/Product/Price/Action/Full.php +++ b/app/code/Magento/Catalog/Model/Indexer/Product/Price/Action/Full.php @@ -117,8 +117,9 @@ public function execute($ids = null) $entityIds = $this->batchProvider->getBatchIds($connection, $select, $batch); if (!empty($entityIds)) { - // Temporary table will created + // Temporary table will created if not exists $idxTableName = $this->_defaultIndexerResource->getIdxTable(); + $this->_emptyTable($idxTableName); if ($indexer->getIsComposite()) { $this->_copyRelationIndexData($entityIds); diff --git a/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/TemporaryTableStrategy.php b/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/TemporaryTableStrategy.php index afff681286a11..7c9bea2f5d9ab 100644 --- a/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/TemporaryTableStrategy.php +++ b/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/TemporaryTableStrategy.php @@ -6,6 +6,9 @@ namespace Magento\Catalog\Model\ResourceModel\Product\Indexer; +/** + * Provided logic will create temporary table based on memory table and will return new index table name. + */ class TemporaryTableStrategy implements \Magento\Framework\Indexer\Table\StrategyInterface { const TEMP_SUFFIX = '_temp'; diff --git a/app/code/Magento/Catalog/Test/Unit/Model/ResourceModel/Product/Indexer/Price/CompositeProductRowSizeEstimatorTest.php b/app/code/Magento/Catalog/Test/Unit/Model/ResourceModel/Product/Indexer/Price/CompositeProductRowSizeEstimatorTest.php index 78c69f0a7fcfd..024cd74deb5e1 100644 --- a/app/code/Magento/Catalog/Test/Unit/Model/ResourceModel/Product/Indexer/Price/CompositeProductRowSizeEstimatorTest.php +++ b/app/code/Magento/Catalog/Test/Unit/Model/ResourceModel/Product/Indexer/Price/CompositeProductRowSizeEstimatorTest.php @@ -18,22 +18,34 @@ class CompositeProductRowSizeEstimatorTest extends \PHPUnit_Framework_TestCase /** * @var \PHPUnit_Framework_MockObject_MockObject */ - private $rowSizeEstimatorMock; + private $websiteManagementMock; /** * @var \PHPUnit_Framework_MockObject_MockObject */ private $defaultPriceMock; + /** + * @var \PHPUnit_Framework_MockObject_MockObject + */ + private $collectionFactoryMock; + protected function setUp() { - $this->rowSizeEstimatorMock = $this->getMock( - \Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\IndexTableRowSizeEstimator::class, + $this->websiteManagementMock = $this->getMock( + \Magento\Store\Api\WebsiteManagementInterface::class, [], [], '', false ); + $this->collectionFactoryMock = $this->getMock( + \Magento\Customer\Model\ResourceModel\Group\CollectionFactory::class, + ['create'], + [], + '', + false + ); $this->defaultPriceMock = $this->getMock( \Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\DefaultPrice::class, [], @@ -41,16 +53,29 @@ protected function setUp() '', false ); - $this->model = new CompositeProductRowSizeEstimator($this->defaultPriceMock, $this->rowSizeEstimatorMock); + $this->model = new CompositeProductRowSizeEstimator( + $this->defaultPriceMock, + $this->websiteManagementMock, + $this->collectionFactoryMock + ); } public function testEstimateRowSize() { - $expectedResult = 2000; + $expectedResult = 40000000; $tableName = 'catalog_product_relation'; $maxRelatedProductCount = 10; - $this->rowSizeEstimatorMock->expects($this->once())->method('estimateRowSize')->willReturn(200); + $this->websiteManagementMock->expects($this->once())->method('getCount')->willReturn(100); + $collectionMock = $this->getMock( + \Magento\Customer\Model\ResourceModel\Group\Collection::class, + [], + [], + '', + false + ); + $this->collectionFactoryMock->expects($this->once())->method('create')->willReturn($collectionMock); + $collectionMock->expects($this->once())->method('getSize')->willReturn(200); $connectionMock = $this->getMock(\Magento\Framework\DB\Adapter\AdapterInterface::class); $this->defaultPriceMock->expects($this->once())->method('getConnection')->willReturn($connectionMock); diff --git a/app/code/Magento/Catalog/Test/Unit/Model/ResourceModel/Product/Indexer/Price/IndexTableRowSizeEstimatorTest.php b/app/code/Magento/Catalog/Test/Unit/Model/ResourceModel/Product/Indexer/Price/IndexTableRowSizeEstimatorTest.php index 180ba229dbf0f..a37fa7c512d99 100644 --- a/app/code/Magento/Catalog/Test/Unit/Model/ResourceModel/Product/Indexer/Price/IndexTableRowSizeEstimatorTest.php +++ b/app/code/Magento/Catalog/Test/Unit/Model/ResourceModel/Product/Indexer/Price/IndexTableRowSizeEstimatorTest.php @@ -41,7 +41,7 @@ protected function setUp() public function testEstimateRowSize() { - $expectedValue = 1800000; + $expectedValue = 2400000; $this->websiteManagementMock->expects($this->once())->method('getCount')->willReturn(100); $collectionMock = $this->getMock( From 5b5969f4281ea769c77762c583f091bdbcf83e91 Mon Sep 17 00:00:00 2001 From: Dmytro Yushkin Date: Mon, 20 Mar 2017 15:37:52 +0200 Subject: [PATCH 28/43] MAGETWO-66240: Add tags for tests that require single flow run --- .../InjectableTests/3rd_party_single_flow.xml | 20 +++++++++++++++++++ .../Mtf/TestSuite/InjectableTests/basic.xml | 1 + .../TestSuite/InjectableTests/basic_green.xml | 1 + 3 files changed, 22 insertions(+) create mode 100644 dev/tests/functional/testsuites/Magento/Mtf/TestSuite/InjectableTests/3rd_party_single_flow.xml diff --git a/dev/tests/functional/testsuites/Magento/Mtf/TestSuite/InjectableTests/3rd_party_single_flow.xml b/dev/tests/functional/testsuites/Magento/Mtf/TestSuite/InjectableTests/3rd_party_single_flow.xml new file mode 100644 index 0000000000000..f360afff7cfa2 --- /dev/null +++ b/dev/tests/functional/testsuites/Magento/Mtf/TestSuite/InjectableTests/3rd_party_single_flow.xml @@ -0,0 +1,20 @@ + + + + + + + + + + + + + + diff --git a/dev/tests/functional/testsuites/Magento/Mtf/TestSuite/InjectableTests/basic.xml b/dev/tests/functional/testsuites/Magento/Mtf/TestSuite/InjectableTests/basic.xml index a4c3de08d5d66..9c591e485202b 100644 --- a/dev/tests/functional/testsuites/Magento/Mtf/TestSuite/InjectableTests/basic.xml +++ b/dev/tests/functional/testsuites/Magento/Mtf/TestSuite/InjectableTests/basic.xml @@ -10,6 +10,7 @@ + diff --git a/dev/tests/functional/testsuites/Magento/Mtf/TestSuite/InjectableTests/basic_green.xml b/dev/tests/functional/testsuites/Magento/Mtf/TestSuite/InjectableTests/basic_green.xml index 6dfbc16f4bec4..4827696dc500f 100644 --- a/dev/tests/functional/testsuites/Magento/Mtf/TestSuite/InjectableTests/basic_green.xml +++ b/dev/tests/functional/testsuites/Magento/Mtf/TestSuite/InjectableTests/basic_green.xml @@ -22,6 +22,7 @@ + From 3b5840b01a12a86a74cdad91a5852406c09fab9d Mon Sep 17 00:00:00 2001 From: Iryna Lagno Date: Tue, 21 Mar 2017 14:44:15 +0200 Subject: [PATCH 29/43] MAGETWO-66195: SQL Error during price reindexation process for custom profile --- .../Product/Indexer/Eav/Source.php | 43 ++++++++++++++++ app/code/Magento/Catalog/etc/di.xml | 6 +++ .../Product/Indexer/Price/Grouped.php | 51 ++++++++++++++++--- 3 files changed, 92 insertions(+), 8 deletions(-) diff --git a/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Eav/Source.php b/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Eav/Source.php index 7a698ae595d19..65f8429165b0e 100644 --- a/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Eav/Source.php +++ b/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Eav/Source.php @@ -15,6 +15,8 @@ */ class Source extends AbstractEav { + const TRANSIT_PREFIX = 'transit_'; + /** * Catalog resource helper * @@ -361,4 +363,45 @@ private function saveDataFromSelect(\Magento\Framework\DB\Select $select, array $this->_saveIndexData($data); } + + /** + * @inheritdoc + */ + protected function _prepareRelationIndex($parentIds = null) + { + $connection = $this->getConnection(); + $idxTable = $this->getIdxTable(); + + if (!$this->tableStrategy->getUseIdxTable()) { + $additionalIdxTable = $connection->getTableName(self::TRANSIT_PREFIX . $this->getIdxTable()); + $connection->createTemporaryTableLike($additionalIdxTable, $idxTable); + + $query = $connection->insertFromSelect( + $this->_prepareRelationIndexSelect($parentIds), + $additionalIdxTable, + [] + ); + $connection->query($query); + + $select = $connection->select()->from($additionalIdxTable); + $query = $connection->insertFromSelect( + $select, + $idxTable, + [], + \Magento\Framework\DB\Adapter\AdapterInterface::INSERT_IGNORE + ); + $connection->query($query); + + $connection->dropTemporaryTable($additionalIdxTable); + } else { + $query = $connection->insertFromSelect( + $this->_prepareRelationIndexSelect($parentIds), + $idxTable, + [], + \Magento\Framework\DB\Adapter\AdapterInterface::INSERT_IGNORE + ); + $connection->query($query); + } + return $this; + } } diff --git a/app/code/Magento/Catalog/etc/di.xml b/app/code/Magento/Catalog/etc/di.xml index 4f17dff6ba7a4..8d0c7f817d6c5 100644 --- a/app/code/Magento/Catalog/etc/di.xml +++ b/app/code/Magento/Catalog/etc/di.xml @@ -954,4 +954,10 @@ indexer + + + Magento\Catalog\Model\ResourceModel\Product\Indexer\TemporaryTableStrategy + indexer + + diff --git a/app/code/Magento/GroupedProduct/Model/ResourceModel/Product/Indexer/Price/Grouped.php b/app/code/Magento/GroupedProduct/Model/ResourceModel/Product/Indexer/Price/Grouped.php index 0dc6e525f4b26..0861aafad5bf6 100644 --- a/app/code/Magento/GroupedProduct/Model/ResourceModel/Product/Indexer/Price/Grouped.php +++ b/app/code/Magento/GroupedProduct/Model/ResourceModel/Product/Indexer/Price/Grouped.php @@ -12,9 +12,10 @@ class Grouped extends DefaultPrice implements GroupedInterface { + const TRANSIT_PREFIX = 'transit_'; + /** - * {@inheritdoc} - * @param null|array $entityIds + * @inheritdoc */ protected function reindex($entityIds = null) { @@ -26,13 +27,51 @@ protected function reindex($entityIds = null) * Use calculated price for relation products * * @param int|array $entityIds the parent entity ids limitation - * @return \Magento\GroupedProduct\Model\ResourceModel\Product\Indexer\Price\Grouped + * @return $this */ protected function _prepareGroupedProductPriceData($entityIds = null) { if (!$this->hasEntity() && empty($entityIds)) { return $this; } + + $connection = $this->getConnection(); + $table = $this->getIdxTable(); + + if (!$this->tableStrategy->getUseIdxTable()) { + $additionalIdxTable = $connection->getTableName(self::TRANSIT_PREFIX . $this->getIdxTable()); + $connection->createTemporaryTableLike($additionalIdxTable, $table); + $query = $connection->insertFromSelect( + $this->_prepareGroupedProductPriceDataSelect($entityIds), + $additionalIdxTable, + [] + ); + $connection->query($query); + + $select = $connection->select()->from($additionalIdxTable); + $query = $connection->insertFromSelect( + $select, + $table, + [], + \Magento\Framework\DB\Adapter\AdapterInterface::INSERT_ON_DUPLICATE + ); + $connection->query($query); + $connection->dropTemporaryTable($additionalIdxTable); + } else { + $query = $this->_prepareGroupedProductPriceDataSelect($entityIds)->insertFromSelect($table); + $connection->query($query); + } + return $this; + } + + /** + * Prepare data index select for Grouped products prices + * + * @param int|array $entityIds the parent entity ids limitation + * @return \Magento\Framework\DB\Select + */ + protected function _prepareGroupedProductPriceDataSelect($entityIds = null) + { $connection = $this->getConnection(); $table = $this->getIdxTable(); $linkField = $this->getMetadataPool()->getMetadata(ProductInterface::class)->getLinkField(); @@ -99,10 +138,6 @@ protected function _prepareGroupedProductPriceData($entityIds = null) 'store_field' => new \Zend_Db_Expr('cs.store_id') ] ); - - $query = $select->insertFromSelect($table); - $connection->query($query); - - return $this; + return $select; } } From aebf18bc0fa30471e316e07cb5db08ede63a2175 Mon Sep 17 00:00:00 2001 From: Dmytro Yushkin Date: Wed, 22 Mar 2017 11:59:28 +0200 Subject: [PATCH 30/43] MAGETWO-65119: Create/update automated functional tests - Added delete customer step to functional test --- .../Test/TestStep/DeleteCustomerStep.php | 59 +++++++++++++++++++ .../TestStep/PlaceOrderWithHostedProStep.php | 1 + 2 files changed, 60 insertions(+) create mode 100644 dev/tests/functional/tests/app/Magento/Customer/Test/TestStep/DeleteCustomerStep.php diff --git a/dev/tests/functional/tests/app/Magento/Customer/Test/TestStep/DeleteCustomerStep.php b/dev/tests/functional/tests/app/Magento/Customer/Test/TestStep/DeleteCustomerStep.php new file mode 100644 index 0000000000000..d8b3832a1cfeb --- /dev/null +++ b/dev/tests/functional/tests/app/Magento/Customer/Test/TestStep/DeleteCustomerStep.php @@ -0,0 +1,59 @@ +customer = $customer; + $this->customerIndexPage = $customerIndexPage; + $this->customerIndexEditPage = $customerIndexEditPage; + } + + /** + * @inheritdoc + */ + public function run() + { + $filter = ['email' => $this->customer->getEmail()]; + $this->customerIndexPage->open(); + $this->customerIndexPage->getCustomerGridBlock()->searchAndOpen($filter); + $this->customerIndexEditPage->getPageActionsBlock()->delete(); + $this->customerIndexEditPage->getModalBlock()->acceptAlert(); + } +} diff --git a/dev/tests/functional/tests/app/Magento/Paypal/Test/TestStep/PlaceOrderWithHostedProStep.php b/dev/tests/functional/tests/app/Magento/Paypal/Test/TestStep/PlaceOrderWithHostedProStep.php index 140c1a6b3e039..decf5e704fc88 100644 --- a/dev/tests/functional/tests/app/Magento/Paypal/Test/TestStep/PlaceOrderWithHostedProStep.php +++ b/dev/tests/functional/tests/app/Magento/Paypal/Test/TestStep/PlaceOrderWithHostedProStep.php @@ -106,6 +106,7 @@ public function run() 'orderInjectable', [ 'data' => [ + 'id' => $orderId, 'entity_id' => ['products' => $this->products] ] ] From bdc1f9b8baa938d953365c154d43a54235b4d075 Mon Sep 17 00:00:00 2001 From: Dmytro Yushkin Date: Mon, 27 Mar 2017 12:26:19 +0300 Subject: [PATCH 31/43] MAGETWO-65119: Create/update automated functional tests - Added deny to testsuite --- .../Magento/Mtf/TestSuite/InjectableTests/3rd_party.xml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/dev/tests/functional/testsuites/Magento/Mtf/TestSuite/InjectableTests/3rd_party.xml b/dev/tests/functional/testsuites/Magento/Mtf/TestSuite/InjectableTests/3rd_party.xml index 8d67f87de77f3..c4583a6c432c5 100644 --- a/dev/tests/functional/testsuites/Magento/Mtf/TestSuite/InjectableTests/3rd_party.xml +++ b/dev/tests/functional/testsuites/Magento/Mtf/TestSuite/InjectableTests/3rd_party.xml @@ -11,10 +11,16 @@ + + + + + + From 31cc41987fd8b2c3bbad3a20f99ded3c6f62dc57 Mon Sep 17 00:00:00 2001 From: Dmytro Yushkin Date: Tue, 28 Mar 2017 15:33:09 +0300 Subject: [PATCH 32/43] MAGETWO-65119: Create/update automated functional tests - Added order id in step returned result --- .../Paypal/Test/TestStep/PlaceOrderWithHostedProStep.php | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dev/tests/functional/tests/app/Magento/Paypal/Test/TestStep/PlaceOrderWithHostedProStep.php b/dev/tests/functional/tests/app/Magento/Paypal/Test/TestStep/PlaceOrderWithHostedProStep.php index 4f8c271549518..ed1edbc2f0890 100644 --- a/dev/tests/functional/tests/app/Magento/Paypal/Test/TestStep/PlaceOrderWithHostedProStep.php +++ b/dev/tests/functional/tests/app/Magento/Paypal/Test/TestStep/PlaceOrderWithHostedProStep.php @@ -122,6 +122,9 @@ public function run() ['data' => array_merge($data, $orderData)] ); - return ['order' => $order]; + return [ + 'orderId' => $orderId, + 'order' => $order, + ]; } } From 56a7d43f4d94af452e66f26881cebf6da283c130 Mon Sep 17 00:00:00 2001 From: Dmytro Yushkin Date: Thu, 30 Mar 2017 13:23:23 +0300 Subject: [PATCH 33/43] MAGETWO-65119: Create/update automated functional tests - Fixed test runner options --- .../testsuites/Magento/Mtf/TestSuite/InjectableTests/basic.xml | 3 +-- .../Magento/Mtf/TestSuite/InjectableTests/basic_green.xml | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/dev/tests/functional/testsuites/Magento/Mtf/TestSuite/InjectableTests/basic.xml b/dev/tests/functional/testsuites/Magento/Mtf/TestSuite/InjectableTests/basic.xml index 9c591e485202b..4840c695a5f15 100644 --- a/dev/tests/functional/testsuites/Magento/Mtf/TestSuite/InjectableTests/basic.xml +++ b/dev/tests/functional/testsuites/Magento/Mtf/TestSuite/InjectableTests/basic.xml @@ -9,8 +9,7 @@ xsi:noNamespaceSchemaLocation="../../../../../vendor/magento/mtf/Magento/Mtf/TestRunner/etc/testRunner.xsd"> - - + diff --git a/dev/tests/functional/testsuites/Magento/Mtf/TestSuite/InjectableTests/basic_green.xml b/dev/tests/functional/testsuites/Magento/Mtf/TestSuite/InjectableTests/basic_green.xml index 4827696dc500f..4fc6cca5598cd 100644 --- a/dev/tests/functional/testsuites/Magento/Mtf/TestSuite/InjectableTests/basic_green.xml +++ b/dev/tests/functional/testsuites/Magento/Mtf/TestSuite/InjectableTests/basic_green.xml @@ -21,8 +21,7 @@ - - + From f4bc33a48c4f821b6d906561d32803c35f2a5de8 Mon Sep 17 00:00:00 2001 From: Iryna Lagno Date: Thu, 30 Mar 2017 15:24:19 +0300 Subject: [PATCH 34/43] MAGETWO-64182: [Indexer optimizations] Batch data processing for price indexer --- app/code/Magento/Bundle/etc/di.xml | 2 +- .../Indexer/Product/Price/AbstractAction.php | 121 +++++++++++------- app/code/Magento/Catalog/etc/di.xml | 2 +- .../Magento/ConfigurableProduct/etc/di.xml | 2 +- app/code/Magento/GroupedProduct/etc/di.xml | 2 +- 5 files changed, 79 insertions(+), 50 deletions(-) diff --git a/app/code/Magento/Bundle/etc/di.xml b/app/code/Magento/Bundle/etc/di.xml index 37be75d727e7a..d2e568da30eeb 100644 --- a/app/code/Magento/Bundle/etc/di.xml +++ b/app/code/Magento/Bundle/etc/di.xml @@ -140,7 +140,7 @@ - 100000 + 5000 Magento\Catalog\Model\Indexer\Price\CompositeProductBatchSizeManagement diff --git a/app/code/Magento/Catalog/Model/Indexer/Product/Price/AbstractAction.php b/app/code/Magento/Catalog/Model/Indexer/Product/Price/AbstractAction.php index 09e36953529bf..5d5b911110d9d 100644 --- a/app/code/Magento/Catalog/Model/Indexer/Product/Price/AbstractAction.php +++ b/app/code/Magento/Catalog/Model/Indexer/Product/Price/AbstractAction.php @@ -228,61 +228,90 @@ protected function _prepareTierPriceIndex($entityIds = null) { $table = $this->_defaultIndexerResource->getTable('catalog_product_index_tier_price'); $this->_emptyTable($table); - - $tierPriceExpression = $this->_connection->getCheckSql( - 'tp.value = 0', - 'product_price.value * (1 - tp.percentage_value / 100)', - 'tp.value' - ); - $websiteExpression = $this->_connection->getCheckSql( - 'tp.website_id = 0', - 'ROUND(' . $tierPriceExpression . ' * cwd.rate, 4)', - $tierPriceExpression - ); + if (empty($entityIds)) { + return $this; + } $linkField = $this->getProductIdFieldName(); $priceAttribute = $this->getProductResource()->getAttribute('price'); - - $select = $this->_connection->select()->from( + $baseColumns = [ + 'cpe.entity_id', + 'tp.customer_group_id', + 'tp.website_id' + ]; + if ($linkField !== 'entity_id') { + $baseColumns[] = 'cpe.' . $linkField; + }; + $subSelect = $this->_connection->select()->from( ['cpe' => $this->_defaultIndexerResource->getTable('catalog_product_entity')], - ['cpe.entity_id'] - )->join( + array_merge_recursive( + $baseColumns, + [ + 'min(tp.value) AS value', + 'min(tp.percentage_value) AS percentage_value' + ] + ) + )->joinInner( ['tp' => $this->_defaultIndexerResource->getTable(['catalog_product_entity', 'tier_price'])], 'tp.' . $linkField . ' = cpe.' . $linkField, [] - )->join( - ['cg' => $this->_defaultIndexerResource->getTable('customer_group')], - 'tp.all_groups = 1 OR (tp.all_groups = 0 AND tp.customer_group_id = cg.customer_group_id)', - ['customer_group_id'] - )->join( - ['cw' => $this->_defaultIndexerResource->getTable('store_website')], - 'tp.website_id = 0 OR tp.website_id = cw.website_id', - ['website_id'] - )->join( - ['cwd' => $this->_defaultIndexerResource->getTable('catalog_product_index_website')], - 'cw.website_id = cwd.website_id', - [] - )->join( - ['product_price' => $priceAttribute->getBackend()->getTable()], - 'tp.' . $linkField . ' = product_price.' . $linkField, - [] - )->where( - 'cw.website_id != 0' - )->where( - 'product_price.attribute_id = ?', - $priceAttribute->getAttributeId() - )->columns( - new \Zend_Db_Expr("MIN({$websiteExpression})") - )->group( - ['cpe.entity_id', 'cg.customer_group_id', 'cw.website_id'] - ); + )->where("cpe.entity_id IN(?)", $entityIds) + ->where("tp.website_id != 0") + ->group(['cpe.entity_id', 'tp.customer_group_id', 'tp.website_id']); + + $subSelect2 = $this->_connection->select() + ->from( + ['cpe' => $this->_defaultIndexerResource->getTable('catalog_product_entity')], + array_merge_recursive( + $baseColumns, + [ + 'MIN(ROUND(tp.value * cwd.rate, 4)) AS value', + 'MIN(ROUND(tp.percentage_value * cwd.rate, 4)) AS percentage_value' + + ] + ) + ) + ->joinInner( + ['tp' => $this->_defaultIndexerResource->getTable(['catalog_product_entity', 'tier_price'])], + 'tp.' . $linkField . ' = cpe.' . $linkField, + [] + )->join( + ['cw' => $this->_defaultIndexerResource->getTable('store_website')], + true, + [] + ) + ->joinInner( + ['cwd' => $this->_defaultIndexerResource->getTable('catalog_product_index_website')], + 'cw.website_id = cwd.website_id', + [] + ) + ->where("cpe.entity_id IN(?)", $entityIds) + ->where("tp.website_id = 0") + ->group( + ['cpe.entity_id', 'tp.customer_group_id', 'tp.website_id'] + ); - if (!empty($entityIds)) { - $select->where("cpe.entity_id IN(?)", $entityIds); - } + $unionSelect = $this->_connection->select() + ->union([$subSelect, $subSelect2], \Magento\Framework\DB\Select::SQL_UNION_ALL); + $select = $this->_connection->select() + ->from( + ['b' => new \Zend_Db_Expr(sprintf('(%s)', $unionSelect->assemble()))], + [ + 'b.entity_id', + 'b.customer_group_id', + 'b.website_id', + 'MIN(IF(b.value = 0, product_price.value * (1 - b.percentage_value / 100), b.value))' + ] + ) + ->joinInner( + ['product_price' => $priceAttribute->getBackend()->getTable()], + 'b.' . $linkField . ' = product_price.' . $linkField, + [] + ) + ->group(['b.entity_id', 'b.customer_group_id', 'b.website_id']); + + $query = $select->insertFromSelect($table, [], false); - $query = $select->insertFromSelect($table); $this->_connection->query($query); - return $this; } diff --git a/app/code/Magento/Catalog/etc/di.xml b/app/code/Magento/Catalog/etc/di.xml index 8d0c7f817d6c5..7faba3d223dba 100644 --- a/app/code/Magento/Catalog/etc/di.xml +++ b/app/code/Magento/Catalog/etc/di.xml @@ -918,7 +918,7 @@ - 200000 + 5000 Magento\Catalog\Model\Indexer\Price\BatchSizeManagement diff --git a/app/code/Magento/ConfigurableProduct/etc/di.xml b/app/code/Magento/ConfigurableProduct/etc/di.xml index 113ce13f86ccf..3bb8381650d2b 100644 --- a/app/code/Magento/ConfigurableProduct/etc/di.xml +++ b/app/code/Magento/ConfigurableProduct/etc/di.xml @@ -174,7 +174,7 @@ - 100000 + 5000 Magento\Catalog\Model\Indexer\Price\CompositeProductBatchSizeManagement diff --git a/app/code/Magento/GroupedProduct/etc/di.xml b/app/code/Magento/GroupedProduct/etc/di.xml index e2ffc8c8bebd8..80b3a77b7fbe2 100644 --- a/app/code/Magento/GroupedProduct/etc/di.xml +++ b/app/code/Magento/GroupedProduct/etc/di.xml @@ -92,7 +92,7 @@ - 100000 + 5000 Magento\Catalog\Model\Indexer\Price\CompositeProductBatchSizeManagement From 7935acad6247fde755b1f68fda3c2bf9de65d5c2 Mon Sep 17 00:00:00 2001 From: Iryna Lagno Date: Fri, 31 Mar 2017 09:47:14 +0300 Subject: [PATCH 35/43] MAGETWO-64182: [Indexer optimizations] Batch data processing for price indexer - fix static tests --- .../ResourceModel/Product/Indexer/Price/BatchSizeCalculator.php | 2 +- .../Product/Indexer/Price/CompositeProductRowSizeEstimator.php | 2 +- .../Product/Indexer/Price/IndexTableRowSizeEstimator.php | 2 +- .../ResourceModel/Product/Indexer/TemporaryTableStrategy.php | 2 +- .../Test/Unit/Model/ResourceModel/Layer/Filter/PriceTest.php | 2 +- .../Indexer/LinkedProductSelectBuilderByIndexPriceTest.php | 2 +- .../Product/Indexer/Price/BatchSizeCalculatorTest.php | 2 +- .../Indexer/Price/CompositeProductRowSizeEstimatorTest.php | 2 +- .../ResourceModel/Product/Indexer/Price/DefaultPriceTest.php | 2 +- .../Product/Indexer/Price/IndexTableRowSizeEstimatorTest.php | 2 +- .../Product/Indexer/TemporaryTableStrategyTest.php | 2 +- .../Unit/Model/Adapter/Mysql/Aggregation/DataProviderTest.php | 2 +- .../Test/Unit/Model/Adapter/Mysql/Dynamic/DataProviderTest.php | 2 +- .../Unit/Model/Search/FilterMapper/ExclusionStrategyTest.php | 2 +- .../Magento/Indexer/Model/ResourceModel/FrontendResource.php | 2 +- app/code/Magento/Indexer/Setup/UpgradeSchema.php | 2 +- .../Test/Unit/Model/ResourceModel/FrontendResourceTest.php | 2 +- .../Indexer/Product/Price/SimpleWithOptionsTierPriceTest.php | 2 +- .../Downloadable/Model/ResourceModel/Indexer/PriceTest.php | 2 +- .../Model/ResourceModel/Product/Indexer/Price/GroupedTest.php | 2 +- lib/internal/Magento/Framework/Indexer/BatchProvider.php | 2 +- .../Magento/Framework/Indexer/BatchProviderInterface.php | 2 +- lib/internal/Magento/Framework/Indexer/BatchSizeManagement.php | 2 +- .../Magento/Framework/Indexer/BatchSizeManagementInterface.php | 2 +- .../Framework/Indexer/IndexTableRowSizeEstimatorInterface.php | 2 +- .../Magento/Framework/Indexer/Test/Unit/BatchProviderTest.php | 2 +- .../Framework/Indexer/Test/Unit/BatchSizeManagementTest.php | 2 +- 27 files changed, 27 insertions(+), 27 deletions(-) diff --git a/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/BatchSizeCalculator.php b/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/BatchSizeCalculator.php index 68f09b66cf08a..b31147af2342f 100644 --- a/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/BatchSizeCalculator.php +++ b/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/BatchSizeCalculator.php @@ -1,6 +1,6 @@ Date: Fri, 31 Mar 2017 12:58:07 +0300 Subject: [PATCH 36/43] MAGETWO-64182: [Indexer optimizations] Batch data processing for price indexer --- .../Framework/Indexer/BatchSizeManagement.php | 19 ++++++- .../Test/Unit/BatchSizeManagementTest.php | 52 ++++++++++--------- 2 files changed, 45 insertions(+), 26 deletions(-) diff --git a/lib/internal/Magento/Framework/Indexer/BatchSizeManagement.php b/lib/internal/Magento/Framework/Indexer/BatchSizeManagement.php index e3da081275a38..940e7ab73e98e 100644 --- a/lib/internal/Magento/Framework/Indexer/BatchSizeManagement.php +++ b/lib/internal/Magento/Framework/Indexer/BatchSizeManagement.php @@ -16,14 +16,22 @@ class BatchSizeManagement implements \Magento\Framework\Indexer\BatchSizeManagem */ private $rowSizeEstimator; + /** + * @var \Psr\Log\LoggerInterface + */ + private $logger; + /** * CompositeProductBatchSizeCalculator constructor. * @param \Magento\Framework\Indexer\IndexTableRowSizeEstimatorInterface $rowSizeEstimator + * @param \Psr\Log\LoggerInterface $logger */ public function __construct( - \Magento\Framework\Indexer\IndexTableRowSizeEstimatorInterface $rowSizeEstimator + \Magento\Framework\Indexer\IndexTableRowSizeEstimatorInterface $rowSizeEstimator, + \Psr\Log\LoggerInterface $logger ) { $this->rowSizeEstimator = $rowSizeEstimator; + $this->logger = $logger; } /** @@ -35,10 +43,19 @@ public function ensureBatchSize(\Magento\Framework\DB\Adapter\AdapterInterface $ $maxHeapTableSize = $connection->fetchOne('SELECT @@max_heap_table_size;'); $tmpTableSize = $connection->fetchOne('SELECT @@tmp_table_size;'); + $bufferPoolSize = $connection->fetchOne('SELECT @@innodb_buffer_pool_size;'); $maxMemoryTableSize = min($maxHeapTableSize, $tmpTableSize); $size = (int) ($rowMemory * $batchSize); + // Log warning if allocated memory for temp table greater than 20% of innodb_buffer_pool_size + if ($size > $bufferPoolSize * .2) { + $this->logger->warning(__( + 'Memory size allocated for temporary table is more than 20% innodb_buffer_pool_size. ' . + 'Please update innodb_buffer_pool_size or decrease batch size value.' + )); + } + if ($maxMemoryTableSize < $size) { $connection->query('SET SESSION tmp_table_size = ' . $size . ';'); $connection->query('SET SESSION max_heap_table_size = ' . $size . ';'); diff --git a/lib/internal/Magento/Framework/Indexer/Test/Unit/BatchSizeManagementTest.php b/lib/internal/Magento/Framework/Indexer/Test/Unit/BatchSizeManagementTest.php index 55b8482296896..40b1189c4d71e 100644 --- a/lib/internal/Magento/Framework/Indexer/Test/Unit/BatchSizeManagementTest.php +++ b/lib/internal/Magento/Framework/Indexer/Test/Unit/BatchSizeManagementTest.php @@ -20,24 +20,28 @@ class BatchSizeManagementTest extends \PHPUnit_Framework_TestCase */ private $rowSizeEstimatorMock; + /** + * @var \Psr\Log\LoggerInterface|\PHPUnit_Framework_MockObject_MockObject + */ + private $loggerMock; + protected function setUp() { $this->rowSizeEstimatorMock = $this->getMock( \Magento\Framework\Indexer\IndexTableRowSizeEstimatorInterface::class ); - $this->model = new BatchSizeManagement($this->rowSizeEstimatorMock); + $this->loggerMock = $this->getMock(\Psr\Log\LoggerInterface::class); + $this->model = new BatchSizeManagement($this->rowSizeEstimatorMock, $this->loggerMock); } - /** - * @param int $batchSize number of records in the batch - * @param int $maxHeapTableSize max_heap_table_size MySQL value - * @param int $tmpTableSize tmp_table_size MySQL value - * @param int $size - * - * @dataProvider estimateBatchSizeDataProvider - */ - public function testEnsureBatchSize($batchSize, $maxHeapTableSize, $tmpTableSize, $size) + public function testEnsureBatchSize() { + $batchSize = 200; + $maxHeapTableSize = 16384; + $tmpTableSize = 16384; + $size = 20000; + $innodbPollSize = 100; + $this->rowSizeEstimatorMock->expects($this->once())->method('estimateRowSize')->willReturn(100); $adapterMock = $this->getMock(AdapterInterface::class); $adapterMock->expects($this->at(0)) @@ -48,28 +52,26 @@ public function testEnsureBatchSize($batchSize, $maxHeapTableSize, $tmpTableSize ->method('fetchOne') ->with('SELECT @@tmp_table_size;', []) ->willReturn($tmpTableSize); - $adapterMock->expects($this->at(2)) + ->method('fetchOne') + ->with('SELECT @@innodb_buffer_pool_size;', []) + ->willReturn($innodbPollSize); + + $this->loggerMock->expects($this->once()) + ->method('warning') + ->with(__( + 'Memory size allocated for temporary table is more than 20% innodb_buffer_pool_size. ' . + 'Please update innodb_buffer_pool_size or decrease batch size value.' + )); + + $adapterMock->expects($this->at(3)) ->method('query') ->with('SET SESSION tmp_table_size = ' . $size . ';', []); - $adapterMock->expects($this->at(3)) + $adapterMock->expects($this->at(4)) ->method('query') ->with('SET SESSION max_heap_table_size = ' . $size . ';', []); $this->model->ensureBatchSize($adapterMock, $batchSize); } - - /** - * @return array - */ - public function estimateBatchSizeDataProvider() - { - return [ - [200, 16384, 16384, 20000], - [300, 16384, 20000, 30000], - [400, 20000, 16384, 40000], - [500, 2000, 2000, 50000], - ]; - } } From 732abab74fa0582a3cbbf9d36b016353cdeb43b1 Mon Sep 17 00:00:00 2001 From: Dmytro Yushkin Date: Fri, 31 Mar 2017 14:10:47 +0300 Subject: [PATCH 37/43] MAGETWO-63642: Update Order Status Based on Signifyd Guarantee Status - Fixed copyrights --- .../app/Magento/Customer/Test/TestStep/DeleteCustomerStep.php | 2 +- .../tests/app/Magento/Sales/Test/TestStep/UnholdOrderStep.php | 2 +- .../Mtf/TestSuite/InjectableTests/3rd_party_single_flow.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dev/tests/functional/tests/app/Magento/Customer/Test/TestStep/DeleteCustomerStep.php b/dev/tests/functional/tests/app/Magento/Customer/Test/TestStep/DeleteCustomerStep.php index d8b3832a1cfeb..c17db90b1e346 100644 --- a/dev/tests/functional/tests/app/Magento/Customer/Test/TestStep/DeleteCustomerStep.php +++ b/dev/tests/functional/tests/app/Magento/Customer/Test/TestStep/DeleteCustomerStep.php @@ -1,6 +1,6 @@ From 3ed9a1c2b5fe6d9b3d3a4ba3f3f9f867438ac26d Mon Sep 17 00:00:00 2001 From: Stanislav Idolov Date: Fri, 31 Mar 2017 15:15:09 +0300 Subject: [PATCH 38/43] MAGETWO-66195: SQL Error during price reindexation process for custom profile --- .../Indexer/Price/CompositeProductRowSizeEstimator.php | 9 +++++++-- .../Product/Indexer/Price/IndexTableRowSizeEstimator.php | 9 +++++++-- .../Product/Indexer/TemporaryTableStrategy.php | 5 ++++- app/code/Magento/Catalog/etc/di.xml | 6 +++++- 4 files changed, 23 insertions(+), 6 deletions(-) diff --git a/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/CompositeProductRowSizeEstimator.php b/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/CompositeProductRowSizeEstimator.php index 8a790b85167f9..a3425bfc9d729 100644 --- a/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/CompositeProductRowSizeEstimator.php +++ b/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/CompositeProductRowSizeEstimator.php @@ -15,6 +15,11 @@ */ class CompositeProductRowSizeEstimator implements IndexTableRowSizeEstimatorInterface { + /** + * Calculated memory size for one record in catalog_product_index_price table + */ + const MEMORY_SIZE_FOR_ONE_ROW = 200; + /** * @var DefaultPrice */ @@ -76,8 +81,8 @@ public function estimateRowSize() * $maxRelatedProductCount - maximum number of related products * $websitesCount - active websites * $customerGroupCount - active customer groups - * 200 - calculated memory size for one record in catalog_product_index_price table + * MEMORY_SIZE_FOR_ONE_ROW - calculated memory size for one record in catalog_product_index_price table */ - return ceil($maxRelatedProductCount * $websitesCount * $customerGroupCount * 200); + return ceil($maxRelatedProductCount * $websitesCount * $customerGroupCount * self::MEMORY_SIZE_FOR_ONE_ROW); } } diff --git a/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/IndexTableRowSizeEstimator.php b/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/IndexTableRowSizeEstimator.php index 1c21cff9916dd..89df6677f2490 100644 --- a/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/IndexTableRowSizeEstimator.php +++ b/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/IndexTableRowSizeEstimator.php @@ -12,6 +12,11 @@ */ class IndexTableRowSizeEstimator implements \Magento\Framework\Indexer\IndexTableRowSizeEstimatorInterface { + /** + * Calculated memory size for one record in catalog_product_index_price table + */ + const MEMORY_SIZE_FOR_ONE_ROW = 120; + /** * @var \Magento\Store\Api\WebsiteManagementInterface */ @@ -51,8 +56,8 @@ public function estimateRowSize() * * $websitesCount - active websites * $customerGroupCount - active customer groups - * 120 - calculated memory size for one record in catalog_product_index_price table + * MEMORY_SIZE_FOR_ONE_ROW - calculated memory size for one record in catalog_product_index_price table */ - return ceil($websitesCount * $customerGroupCount * 120); + return ceil($websitesCount * $customerGroupCount * self::MEMORY_SIZE_FOR_ONE_ROW); } } diff --git a/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/TemporaryTableStrategy.php b/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/TemporaryTableStrategy.php index a8306de26ffa3..54673cb01bb1d 100644 --- a/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/TemporaryTableStrategy.php +++ b/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/TemporaryTableStrategy.php @@ -11,6 +11,9 @@ */ class TemporaryTableStrategy implements \Magento\Framework\Indexer\Table\StrategyInterface { + /** + * Suffix for new temporary table + */ const TEMP_SUFFIX = '_temp'; /** @@ -31,7 +34,7 @@ class TemporaryTableStrategy implements \Magento\Framework\Indexer\Table\Strateg * @param \Magento\Framework\App\ResourceConnection $resource */ public function __construct( - \Magento\Framework\Indexer\Table\Strategy $strategy, + \Magento\Framework\Indexer\Table\StrategyInterface $strategy, \Magento\Framework\App\ResourceConnection $resource ) { $this->strategy = $strategy; diff --git a/app/code/Magento/Catalog/etc/di.xml b/app/code/Magento/Catalog/etc/di.xml index a9edad6efacee..7da263648c69e 100644 --- a/app/code/Magento/Catalog/etc/di.xml +++ b/app/code/Magento/Catalog/etc/di.xml @@ -947,7 +947,11 @@ Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\FrontendResource - + + + Magento\Framework\Indexer\Table\Strategy + + Magento\Catalog\Model\ResourceModel\Product\Indexer\TemporaryTableStrategy From 1ddde99b7caeb14a81449ff0749bf2c3237079fa Mon Sep 17 00:00:00 2001 From: Stanislav Idolov Date: Fri, 31 Mar 2017 15:38:25 +0300 Subject: [PATCH 39/43] MAGETWO-66195: SQL Error during price reindexation process for custom profile --- .../Product/Indexer/Eav/Source.php | 43 ------------------- app/code/Magento/Catalog/etc/di.xml | 6 --- .../Product/Indexer/Price/Grouped.php | 3 ++ 3 files changed, 3 insertions(+), 49 deletions(-) diff --git a/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Eav/Source.php b/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Eav/Source.php index e428ff6f317c9..d5ef80c798fde 100644 --- a/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Eav/Source.php +++ b/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Eav/Source.php @@ -15,8 +15,6 @@ */ class Source extends AbstractEav { - const TRANSIT_PREFIX = 'transit_'; - /** * Catalog resource helper * @@ -363,45 +361,4 @@ private function saveDataFromSelect(\Magento\Framework\DB\Select $select, array $this->_saveIndexData($data); } - - /** - * @inheritdoc - */ - protected function _prepareRelationIndex($parentIds = null) - { - $connection = $this->getConnection(); - $idxTable = $this->getIdxTable(); - - if (!$this->tableStrategy->getUseIdxTable()) { - $additionalIdxTable = $connection->getTableName(self::TRANSIT_PREFIX . $this->getIdxTable()); - $connection->createTemporaryTableLike($additionalIdxTable, $idxTable); - - $query = $connection->insertFromSelect( - $this->_prepareRelationIndexSelect($parentIds), - $additionalIdxTable, - [] - ); - $connection->query($query); - - $select = $connection->select()->from($additionalIdxTable); - $query = $connection->insertFromSelect( - $select, - $idxTable, - [], - \Magento\Framework\DB\Adapter\AdapterInterface::INSERT_IGNORE - ); - $connection->query($query); - - $connection->dropTemporaryTable($additionalIdxTable); - } else { - $query = $connection->insertFromSelect( - $this->_prepareRelationIndexSelect($parentIds), - $idxTable, - [], - \Magento\Framework\DB\Adapter\AdapterInterface::INSERT_IGNORE - ); - $connection->query($query); - } - return $this; - } } diff --git a/app/code/Magento/Catalog/etc/di.xml b/app/code/Magento/Catalog/etc/di.xml index 7da263648c69e..f53a99754ad57 100644 --- a/app/code/Magento/Catalog/etc/di.xml +++ b/app/code/Magento/Catalog/etc/di.xml @@ -958,10 +958,4 @@ indexer - - - Magento\Catalog\Model\ResourceModel\Product\Indexer\TemporaryTableStrategy - indexer - - diff --git a/app/code/Magento/GroupedProduct/Model/ResourceModel/Product/Indexer/Price/Grouped.php b/app/code/Magento/GroupedProduct/Model/ResourceModel/Product/Indexer/Price/Grouped.php index d2eb54869d22d..cbbb58d3c24c8 100644 --- a/app/code/Magento/GroupedProduct/Model/ResourceModel/Product/Indexer/Price/Grouped.php +++ b/app/code/Magento/GroupedProduct/Model/ResourceModel/Product/Indexer/Price/Grouped.php @@ -12,6 +12,9 @@ class Grouped extends DefaultPrice implements GroupedInterface { + /** + * Prefix for temporary table support. + */ const TRANSIT_PREFIX = 'transit_'; /** From 7395bae1410c4d07c5031ce30f427a907f1a85e1 Mon Sep 17 00:00:00 2001 From: Stanislav Idolov Date: Fri, 31 Mar 2017 16:24:24 +0300 Subject: [PATCH 40/43] MAGETWO-64182: [Indexer optimizations] Batch data processing for price indexer --- lib/internal/Magento/Framework/Indexer/BatchSizeManagement.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/internal/Magento/Framework/Indexer/BatchSizeManagement.php b/lib/internal/Magento/Framework/Indexer/BatchSizeManagement.php index 940e7ab73e98e..e341d03d83df4 100644 --- a/lib/internal/Magento/Framework/Indexer/BatchSizeManagement.php +++ b/lib/internal/Magento/Framework/Indexer/BatchSizeManagement.php @@ -50,7 +50,7 @@ public function ensureBatchSize(\Magento\Framework\DB\Adapter\AdapterInterface $ // Log warning if allocated memory for temp table greater than 20% of innodb_buffer_pool_size if ($size > $bufferPoolSize * .2) { - $this->logger->warning(__( + $this->logger->warning(new \Magento\Framework\Phrase( 'Memory size allocated for temporary table is more than 20% innodb_buffer_pool_size. ' . 'Please update innodb_buffer_pool_size or decrease batch size value.' )); From 48df3462b718f7a3ea09d2822e7d79cdf7c1633c Mon Sep 17 00:00:00 2001 From: Stanislav Idolov Date: Mon, 3 Apr 2017 10:51:12 +0300 Subject: [PATCH 41/43] MAGETWO-64182: [Indexer optimizations] Batch data processing for price indexer --- .../Magento/Framework/Indexer/BatchSizeManagement.php | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/internal/Magento/Framework/Indexer/BatchSizeManagement.php b/lib/internal/Magento/Framework/Indexer/BatchSizeManagement.php index e341d03d83df4..461e173ff4124 100644 --- a/lib/internal/Magento/Framework/Indexer/BatchSizeManagement.php +++ b/lib/internal/Magento/Framework/Indexer/BatchSizeManagement.php @@ -51,8 +51,9 @@ public function ensureBatchSize(\Magento\Framework\DB\Adapter\AdapterInterface $ // Log warning if allocated memory for temp table greater than 20% of innodb_buffer_pool_size if ($size > $bufferPoolSize * .2) { $this->logger->warning(new \Magento\Framework\Phrase( - 'Memory size allocated for temporary table is more than 20% innodb_buffer_pool_size. ' . - 'Please update innodb_buffer_pool_size or decrease batch size value.' + 'Memory size allocated for the temporary table is more than 20% of innodb_buffer_pool_size. ' . + 'Please update innodb_buffer_pool_size or decrease batch size value '. + '(which decreases memory usages for the temporary table).' )); } From 6fe6d2f105f15a6560d49591932f6046eecf2eea Mon Sep 17 00:00:00 2001 From: Stanislav Idolov Date: Mon, 3 Apr 2017 15:28:02 +0300 Subject: [PATCH 42/43] MAGETWO-64182: [Indexer optimizations] Batch data processing for price indexer --- app/code/Magento/Bundle/etc/di.xml | 3 --- app/code/Magento/ConfigurableProduct/etc/di.xml | 3 --- app/code/Magento/GroupedProduct/etc/di.xml | 3 --- 3 files changed, 9 deletions(-) diff --git a/app/code/Magento/Bundle/etc/di.xml b/app/code/Magento/Bundle/etc/di.xml index 9848d7299a144..a4c2fe1b88485 100644 --- a/app/code/Magento/Bundle/etc/di.xml +++ b/app/code/Magento/Bundle/etc/di.xml @@ -139,9 +139,6 @@ - - 5000 - Magento\Catalog\Model\Indexer\Price\CompositeProductBatchSizeManagement diff --git a/app/code/Magento/ConfigurableProduct/etc/di.xml b/app/code/Magento/ConfigurableProduct/etc/di.xml index a77007b5ab48f..ca097898f1c57 100644 --- a/app/code/Magento/ConfigurableProduct/etc/di.xml +++ b/app/code/Magento/ConfigurableProduct/etc/di.xml @@ -173,9 +173,6 @@ - - 5000 - Magento\Catalog\Model\Indexer\Price\CompositeProductBatchSizeManagement diff --git a/app/code/Magento/GroupedProduct/etc/di.xml b/app/code/Magento/GroupedProduct/etc/di.xml index cbbdb705c2f1a..ff2cac0bd3186 100644 --- a/app/code/Magento/GroupedProduct/etc/di.xml +++ b/app/code/Magento/GroupedProduct/etc/di.xml @@ -91,9 +91,6 @@ type="Magento\GroupedProduct\Model\ResourceModel\Product\Indexer\Price\Grouped"/> - - 5000 - Magento\Catalog\Model\Indexer\Price\CompositeProductBatchSizeManagement From cdeb9808b2b0723048d64f655a81d276061604c4 Mon Sep 17 00:00:00 2001 From: Stanislav Idolov Date: Mon, 3 Apr 2017 16:34:55 +0300 Subject: [PATCH 43/43] MAGETWO-64182: [Indexer optimizations] Batch data processing for price indexer --- .../Framework/Indexer/Test/Unit/BatchSizeManagementTest.php | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/internal/Magento/Framework/Indexer/Test/Unit/BatchSizeManagementTest.php b/lib/internal/Magento/Framework/Indexer/Test/Unit/BatchSizeManagementTest.php index 40b1189c4d71e..ea19c752d0f62 100644 --- a/lib/internal/Magento/Framework/Indexer/Test/Unit/BatchSizeManagementTest.php +++ b/lib/internal/Magento/Framework/Indexer/Test/Unit/BatchSizeManagementTest.php @@ -60,8 +60,9 @@ public function testEnsureBatchSize() $this->loggerMock->expects($this->once()) ->method('warning') ->with(__( - 'Memory size allocated for temporary table is more than 20% innodb_buffer_pool_size. ' . - 'Please update innodb_buffer_pool_size or decrease batch size value.' + 'Memory size allocated for the temporary table is more than 20% of innodb_buffer_pool_size. ' . + 'Please update innodb_buffer_pool_size or decrease batch size value '. + '(which decreases memory usages for the temporary table).' )); $adapterMock->expects($this->at(3))