diff --git a/Helper/Data.php b/Helper/Data.php index 4e62e7706..b84ec85ae 100644 --- a/Helper/Data.php +++ b/Helper/Data.php @@ -128,6 +128,16 @@ class Data extends AbstractHelper */ public const XML_PATH_INDEXER_MEMORY = 'nosto/flags/indexer_memory'; + /** + * Product per request + */ + public const XML_PATH_PRODUCT_PER_REQUEST = 'nosto/flags/product_per_request'; + + /** + * Request timeout + */ + public const XML_PATH_REQUEST_TIMEOUT = 'nosto/flags/request_timeout'; + /** * Path to the configuration object that stores the preference for indexing disabled products */ @@ -441,6 +451,28 @@ public function getIndexerMemory(StoreInterface $store = null) return $this->getStoreConfig(self::XML_PATH_INDEXER_MEMORY, $store); } + /** + * Products per request + * + * @param StoreInterface|null $store the store model or null. + * @return int the configuration value + */ + public function getProductsPerRequest(StoreInterface $store = null) + { + return $this->getStoreConfig(self::XML_PATH_PRODUCT_PER_REQUEST, $store); + } + + /** + * Request timeout + * + * @param StoreInterface|null $store the store model or null. + * @return int the configuration value + */ + public function getRequestTimeout(StoreInterface $store = null) + { + return $this->getStoreConfig(self::XML_PATH_REQUEST_TIMEOUT, $store); + } + /** * Returns maximum percentage of PHP available memory that indexer should use * diff --git a/Model/Config/Source/ProductsPerRequest.php b/Model/Config/Source/ProductsPerRequest.php new file mode 100644 index 000000000..9225e6613 --- /dev/null +++ b/Model/Config/Source/ProductsPerRequest.php @@ -0,0 +1,59 @@ + + * @copyright 2020 Nosto Solutions Ltd + * @license http://opensource.org/licenses/BSD-3-Clause BSD 3-Clause + * + */ + +namespace Nosto\Tagging\Model\Config\Source; + +use Magento\Framework\Data\OptionSourceInterface; +use Magento\Framework\Phrase; + +class ProductsPerRequest implements OptionSourceInterface +{ + /** + * Options getter + * + * @return array + */ + public function toOptionArray() + { + return [ + ['value' => 100, 'label' => new Phrase('100')], + ['value' => 200, 'label' => new Phrase('200')], + ['value' => 300, 'label' => new Phrase('300')], + ['value' => 400, 'label' => new Phrase('400')], + ['value' => 500, 'label' => new Phrase('500')], + ]; + } +} diff --git a/Model/Config/Source/TimeBetweenBatchOfRequests.php b/Model/Config/Source/TimeBetweenBatchOfRequests.php new file mode 100644 index 000000000..95db5a8da --- /dev/null +++ b/Model/Config/Source/TimeBetweenBatchOfRequests.php @@ -0,0 +1,59 @@ + + * @copyright 2020 Nosto Solutions Ltd + * @license http://opensource.org/licenses/BSD-3-Clause BSD 3-Clause + * + */ + +namespace Nosto\Tagging\Model\Config\Source; + +use Magento\Framework\Data\OptionSourceInterface; +use Magento\Framework\Phrase; + +class TimeBetweenBatchOfRequests implements OptionSourceInterface +{ + /** + * Options getter + * + * @return array + */ + public function toOptionArray() + { + return [ + ['value' => 60, 'label' => new Phrase('60')], + ['value' => 120, 'label' => new Phrase('120')], + ['value' => 180, 'label' => new Phrase('180')], + ['value' => 240, 'label' => new Phrase('240')], + ['value' => 300, 'label' => new Phrase('300')], + ]; + } +} diff --git a/Model/Service/Sync/Recrawl/RecrawlService.php b/Model/Service/Sync/Recrawl/RecrawlService.php index 4eb8bd4f9..b30870fd6 100644 --- a/Model/Service/Sync/Recrawl/RecrawlService.php +++ b/Model/Service/Sync/Recrawl/RecrawlService.php @@ -61,7 +61,6 @@ class RecrawlService extends AbstractService public const BENCHMARK_RECRAWL_NAME = 'nosto_product_recrawl'; public const BENCHMARK_RECRAWL_BREAKPOINT = 1; - public const PRODUCT_RECRAWL_BATCH_SIZE = 100; /** @var CacheService */ private CacheService $cacheService; @@ -78,9 +77,6 @@ class RecrawlService extends AbstractService /** @var ProductRepository */ private ProductRepository $productRepository; - /** @var int */ - private int $recrawlBatchSize; - /** * RecrawlService constructor. * @param CacheService $cacheService @@ -90,7 +86,6 @@ class RecrawlService extends AbstractService * @param NostoHelperUrl $nostoHelperUrl * @param ProductRepository $productRepository * @param NostoLogger $logger - * @param int $recrawlBatchSize */ public function __construct( CacheService $cacheService, @@ -99,15 +94,13 @@ public function __construct( NostoHelperData $nostoHelperData, NostoHelperUrl $nostoHelperUrl, ProductRepository $productRepository, - NostoLogger $logger, - int $recrawlBatchSize = self::PRODUCT_RECRAWL_BATCH_SIZE + NostoLogger $logger ) { $this->cacheService = $cacheService; $this->productService = $productService; $this->nostoHelperAccount = $nostoHelperAccount; $this->nostoHelperUrl = $nostoHelperUrl; $this->productRepository = $productRepository; - $this->recrawlBatchSize = $recrawlBatchSize; parent::__construct($nostoHelperData, $nostoHelperAccount, $logger); } @@ -132,8 +125,11 @@ public function recrawl(ProductCollection $collection, Store $store) $account = $this->nostoHelperAccount->findAccount($store); $this->startBenchmark(self::BENCHMARK_RECRAWL_NAME, self::BENCHMARK_RECRAWL_BREAKPOINT); + $timeBetweenBatchOfRequests = $this->getDataHelper()->getRequestTimeout(); + $productsPerRequest = $this->getDataHelper()->getProductsPerRequest(); + $index = 0; - $collection->setPageSize($this->recrawlBatchSize); + $collection->setPageSize($productsPerRequest); $iterator = new PagingIterator($collection); /** @var ProductCollection $page */ @@ -144,8 +140,8 @@ public function recrawl(ProductCollection $collection, Store $store) $op->setResponseTimeout(60); $products = $this->productRepository->getByIds( $page->getAllIds( - $this->recrawlBatchSize, - ($iterator->getCurrentPageNumber() - 1) * $this->recrawlBatchSize + $productsPerRequest, + ($iterator->getCurrentPageNumber() - 1) * $productsPerRequest ) ); /** @var Product $product */ @@ -167,13 +163,14 @@ public function recrawl(ProductCollection $collection, Store $store) $this->logDebugWithStore( sprintf( 'Upserting batch of %d (%s) - API timeout is set to %d seconds', - $this->recrawlBatchSize, + $productsPerRequest, implode(',', $productIdsInBatch), 60 ), $store ); $op->requestRecrawl(); + sleep($timeBetweenBatchOfRequests); } $this->logBenchmarkSummary(self::BENCHMARK_RECRAWL_NAME, $store, $this); } diff --git a/etc/adminhtml/system.xml b/etc/adminhtml/system.xml index 44e6efee5..5515e3001 100644 --- a/etc/adminhtml/system.xml +++ b/etc/adminhtml/system.xml @@ -191,6 +191,22 @@ Nosto\Tagging\Model\Config\Source\Memory + + + + + + Nosto\Tagging\Model\Config\Source\ProductsPerRequest + + + + + + + Nosto\Tagging\Model\Config\Source\TimeBetweenBatchOfRequests + diff --git a/etc/config.xml b/etc/config.xml index 8848eaa1d..8e735381a 100644 --- a/etc/config.xml +++ b/etc/config.xml @@ -56,6 +56,8 @@ 0 1 50 + 100 + 60 0 0 1 diff --git a/etc/indexer.xml b/etc/indexer.xml index b0a77060d..4a068dc37 100644 --- a/etc/indexer.xml +++ b/etc/indexer.xml @@ -40,7 +40,7 @@ - Nosto Product Indexer + Nosto Product Light Indexer Populates message queue with product ids to be sent to Nosto