Create product statuses jobs for each page token

src/API/Google/MerchantReport.php

@@ -51,9 +51,9 @@
 	 * @param ShoppingContent $service
 	 * @param ProductHelper   $product_helper
 	 */
 	public function __construct( ShoppingContent $service, ProductHelper $product_helper ) {
 		$this->service        = $service;
 		$this->product_helper = $product_helper;
 	}
 
 	/**
@@ -64,19 +64,19 @@
 	 *
 	 * @throws Exception If the product view report data can't be retrieved.
 	 */
 	public function get_product_view_report( $next_page_token = null ): array {
 		$batch_size = apply_filters( 'woocommerce_gla_product_view_report_page_size', 1000 );
 
 		try {
 			$product_view_data = [
 				'statuses'  => [],
 				'next_page' => null,
 			];
 
 			$query = new MerchantProductViewReportQuery(
 				[
 					'next_page' => $next_page_token,
 					'per_page'  => $batch_size,
 				]
 			);
 
@@ -84,33 +84,33 @@
 			->set_client( $this->service, $this->options->get_merchant_id() )
 			->get_results();
 
 			foreach ( $response->getResults() as $row ) {
 
 				/** @var ProductView $product_view  */
 				$product_view = $row->getProductView();
 
 				$wc_product_id     = $this->product_helper->get_wc_product_id( $product_view->getId() );
 				$mc_product_status = $this->convert_aggregated_status_to_mc_status( $product_view->getAggregatedDestinationStatus() );
 
 				// Skip if the product id does not exist
 				if ( ! $wc_product_id ) {
 					continue;
 				}
 
 				$product_view_data['statuses'][] = [
 					'product_id'      => $wc_product_id,
 					'status'          => $mc_product_status,
 					'expiration_date' => $this->convert_shopping_content_date( $product_view->getExpirationDate() ),
 				];
 
 			}
 
-			$product_view_data['next_page'] = $response->getNextPageToken();
+			$product_view_data['next_page_token'] = $response->getNextPageToken();
 
 			return $product_view_data;
 		} catch ( GoogleException $e ) {
 			do_action( 'woocommerce_gla_mc_client_exception', $e, __METHOD__ );
 			throw new Exception( __( 'Unable to retrieve Product View Report.', 'google-listings-and-ads' ) . $e->getMessage(), $e->getCode() );
 		}
 	}
 
@@ -121,18 +121,18 @@
 	 *
 	 * @return string The MC status.
 	 */
 	protected function convert_aggregated_status_to_mc_status( string $status ): string {
 		switch ( $status ) {
 			case 'ELIGIBLE':
 				return MCStatus::APPROVED;
 			case 'ELIGIBLE_LIMITED':
 				return MCStatus::PARTIALLY_APPROVED;
 			case 'NOT_ELIGIBLE_OR_DISAPPROVED':
 				return MCStatus::DISAPPROVED;
 			case 'PENDING':
 				return MCStatus::PENDING;
 			default:
 				return MCStatus::NOT_SYNCED;
 		}
 	}
 

src/API/Site/Controllers/MerchantCenter/ProductStatisticsController.php

@@ -106,11 +106,11 @@
 	 *
 	 * @return Response
 	 */
 	protected function get_product_status_stats( Request $request, bool $force_refresh = false ): Response {
 		try {
 			$response = $this->merchant_statuses->get_product_statistics( $force_refresh );
 
 			$response['scheduled_sync'] = $this->sync_stats->get_count();
 
 			return $this->prepare_item_for_response( $response, $request );
 		} catch ( Exception $e ) {
@@ -125,13 +125,13 @@
 	 */
 	protected function get_schema_properties(): array {
 		return [
 			'timestamp'      => [
 				'type'        => 'number',
 				'description' => __( 'Timestamp reflecting when the product status statistics were last generated.', 'google-listings-and-ads' ),
 				'context'     => [ 'view' ],
 				'readonly'    => true,
 			],
 			'statistics'     => [
 				'type'        => 'object',
 				'description' => __( 'Merchant Center product status statistics.', 'google-listings-and-ads' ),
 				'context'     => [ 'view' ],
@@ -164,12 +164,18 @@
 					],
 				],
 			],
 			'scheduled_sync' => [
 				'type'        => 'number',
 				'description' => __( 'Amount of scheduled jobs which will sync products to Google.', 'google-listings-and-ads' ),
 				'context'     => [ 'view' ],
 				'readonly'    => true,
 			],
+			'loading'        => [
+				'type'        => 'boolean',
+				'description' => __( 'Whether the product statistics are loading.', 'google-listings-and-ads' ),
+				'context'     => [ 'view' ],
+				'readonly'    => true,
+			],
 		];
 	}
 

src/Jobs/UpdateMerchantProductStatuses.php

@@ -82,18 +82,24 @@
 	 * @throws JobException If the shipping settings cannot be synced.
 	 */
 	public function process_items( array $items ) {
-		$next_page_token = null;
+		$next_page_token = $items['next_page_token'] ?? null;
 
-		do {
-			$results = $this->merchant_report->get_product_view_report( $next_page_token );
-
-			$this->merchant_statuses->process_product_statuses( $results['statuses'] );
+		// Clear the cache if we're starting from the beginning.
+		if ( ! $next_page_token ) {
+			$this->merchant_statuses->clear_cache();
+			$this->merchant_statuses->delete_product_statuses_count_intermediate_data();
+		}
 
-			$next_page_token = $results['next_page'];
+		$results         = $this->merchant_report->get_product_view_report( $next_page_token );
+		$next_page_token = $results['next_page_token'];
 
-		} while ( $next_page_token );
+		$this->merchant_statuses->update_product_stats( $results['statuses'] );
 
-		$this->merchant_statuses->update_product_stats();
+		if ( $next_page_token ) {
+			$this->schedule( [ [ 'next_page_token' => $next_page_token ] ] );
+		} else {
+			$this->merchant_statuses->handle_complete_mc_statuses_fetching();
+		}
 	}
 
 	/**
@@ -102,17 +108,18 @@
 	 * @param array $args - arguments.
 	 */
 	public function schedule( array $args = [] ) {
-		if ( $this->can_schedule() ) {
-			$this->action_scheduler->schedule_immediate( $this->get_process_item_hook() );
+		if ( $this->can_schedule( $args ) ) {
+			$this->action_scheduler->schedule_immediate( $this->get_process_item_hook(), $args );
 		}
 	}
 
 	/**
-	 * The job is considered to be scheduled if the "process_item" action is currently pending or in-progress.
+	 * The job is considered to be scheduled if the "process_item" action is currently pending or in-progress regardless of the arguments.
 	 *
 	 * @return bool
 	 */
 	public function is_scheduled(): bool {
-		return $this->is_running();
+		// We set 'args' to null so it matches any arguments. This is because it's possible to have multiple instances of the job running with different page tokens
+		return $this->is_running( null );
 	}
 }

src/MerchantCenter/MerchantStatuses.php

@@ -22,6 +22,9 @@
 use Automattic\WooCommerce\GoogleListingsAndAds\Value\MCStatus;
 use Automattic\WooCommerce\GoogleListingsAndAds\Vendor\Google\Service\ShoppingContent\ProductStatus as GoogleProductStatus;
 use Automattic\WooCommerce\GoogleListingsAndAds\Jobs\UpdateMerchantProductStatuses;
+use Automattic\WooCommerce\GoogleListingsAndAds\Options\OptionsAwareInterface;
+use Automattic\WooCommerce\GoogleListingsAndAds\Options\OptionsAwareTrait;
+use Automattic\WooCommerce\GoogleListingsAndAds\Options\OptionsInterface;
 use DateTime;
 use Exception;
 
@@ -37,11 +40,13 @@
  * - ProductHelper
  * - ProductRepository
  * - TransientsInterface
+ * - UpdateMerchantProductStatuses
  *
  * @package Automattic\WooCommerce\GoogleListingsAndAds\MerchantCenter
  */
-class MerchantStatuses implements Service, ContainerAwareInterface {
+class MerchantStatuses implements Service, ContainerAwareInterface, OptionsAwareInterface {
 
+	use OptionsAwareTrait;
 	use ContainerAwareTrait;
 	use PluginHelper;
 
@@ -105,9 +110,11 @@
 	 * @param bool $force_refresh Force refresh of all product status data.
 	 *
 	 * @return array The product status statistics.
-	 * @throws Exception If the Merchant Center can't be polled for the statuses.
+	 * @throws Exception If no Merchant Center account is connected, or account status is not retrievable.
 	 */
 	public function get_product_statistics( bool $force_refresh = false ): array {
+		$this->check_mc_is_connected();
+
 		$this->mc_statuses = $this->container->get( TransientsInterface::class )->get( Transients::MC_STATUSES );
 		$job               = $this->container->get( UpdateMerchantProductStatuses::class );
 
@@ -118,10 +125,10 @@
 		}
 
 		if ( $job->is_scheduled() || null === $this->mc_statuses ) {
-			// TODO: Add a notice to the client to inform that the statuses are being updated, or maybe we can pass the is_scheduled to the client.
 			return [
 				'timestamp'  => $this->cache_created_time->getTimestamp(),
 				'statistics' => [],
+				'loading'    => true,
 			];
 		}
 
@@ -173,6 +180,36 @@
 		$this->container->get( TransientsInterface::class )->delete( TransientsInterface::MC_STATUSES );
 	}
 
+	/**
+	 * Delete the intermediate product status count data.
+	 *
+	 * @since x.x.x
+	 */
+	public function delete_product_statuses_count_intermediate_data(): void {
+		$this->options->delete( OptionsInterface::PRODUCT_STATUSES_COUNT_INTERMEDIATE_DATA );
+	}
+
+	/**
+	 * Check if the Merchant Center account is connected and throw an exception if it's not.
+	 *
+	 * @since x.x.x
+	 *
+	 * @throws Exception If the Merchant Center account is not connected.
+	 */
+	protected function check_mc_is_connected() {
+		$mc_service = $this->container->get( MerchantCenterService::class );
+
+		if ( ! $mc_service->is_connected() ) {
+
+			// Return a 401 to redirect to reconnect flow if the Google account is not connected.
+			if ( ! $mc_service->is_google_connected() ) {
+				throw new Exception( __( 'Google account is not connected.', 'google-listings-and-ads' ), 401 );
+			}
+
+			throw new Exception( __( 'Merchant Center account is not set up.', 'google-listings-and-ads' ) );
+		}
+	}
+
 	/**
 	 * Update stale status-related data - account issues, product issues, products status stats.
 	 *
@@ -184,20 +221,11 @@
 		// Only refresh if the current data has expired.
 		$this->mc_statuses = $this->container->get( TransientsInterface::class )->get( Transients::MC_STATUSES );
 		if ( ! $force_refresh && null !== $this->mc_statuses ) {
 			return;
 		}
 
 		// Save a request if accounts are not connected.
-		$mc_service = $this->container->get( MerchantCenterService::class );
-		if ( ! $mc_service->is_connected() ) {
-
-			// Return a 401 to redirect to reconnect flow if the Google account is not connected.
-			if ( ! $mc_service->is_google_connected() ) {
-				throw new Exception( __( 'Google account is not connected.', 'google-listings-and-ads' ), 401 );
-			}
-
-			throw new Exception( __( 'Merchant Center account is not set up.', 'google-listings-and-ads' ) );
-		}
+		$this->check_mc_is_connected();
 
 		// Update account-level issues.
 		$this->refresh_account_issues();
@@ -588,63 +616,75 @@
 	 * @param array[] $statuses statuses.
 	 * @see MerchantReport::get_product_view_report
 	 */
-	public function process_product_statuses( $statuses ): void {
+	public function process_product_statuses( array $statuses ): void {
+		$this->product_statuses = [
+			'products' => [],
+			'parents'  => [],
+		];
+
 		/** @var ProductHelper $product_helper */
 		$product_helper      = $this->container->get( ProductHelper::class );
 		$visibility_meta_key = $this->prefix_meta_key( ProductMetaHandler::KEY_VISIBILITY );
 
 		foreach ( $statuses as $product_status ) {
 
 			$wc_product_id     = $product_status['product_id'];
 			$mc_product_status = $product_status['status'];
 
 			// Skip if the product does not exist or if the product previously found/validated.
 			if ( ! $wc_product_id || ! empty( $this->product_data_lookup[ $wc_product_id ] ) ) {
 				continue;
 			}
 
 			if ( $this->product_is_expiring( $product_status['expiration_date'] ) ) {
 				$mc_product_status = MCStatus::EXPIRING;
 			}
 
 			$wc_product = $product_helper->get_wc_product_by_wp_post( $wc_product_id );
 			if ( ! $wc_product || 'product' !== substr( $wc_product->post_type, 0, 7 ) ) {
 				// Should never reach here since the products IDS are retrieved from postmeta.
 				do_action(
 					'woocommerce_gla_debug_message',
 					sprintf( 'Merchant Center product %s not found in this WooCommerce store.', $wc_product_id ),
 					__METHOD__,
 				);
 				continue;
 			}
 
 			$this->product_data_lookup[ $wc_product_id ] = [
 				'name'       => get_the_title( $wc_product ),
 				'visibility' => get_post_meta( $wc_product_id, $visibility_meta_key ),
 				'parent_id'  => $wc_product->post_parent,
 			];
 
 			// Products is used later for global product status statistics.
 			$this->product_statuses['products'][ $wc_product_id ][ $mc_product_status ] = 1 + ( $this->product_statuses['products'][ $wc_product_id ][ $mc_product_status ] ?? 0 );
 
 			// Aggregate parent statuses for mc_status postmeta.
 			$wc_parent_id = $this->product_data_lookup[ $wc_product_id ]['parent_id'];
 			if ( ! $wc_parent_id ) {
 				continue;
 			}
 			$this->product_statuses['parents'][ $wc_parent_id ][ $mc_product_status ] = 1 + ( $this->product_statuses['parents'][ $wc_parent_id ][ $mc_product_status ] ?? 0 );
 
 		}
 	}
 
 	/**
-	 * Update the product status statistics.
+	 * Update the product status statistics for a list of products statuses.
+	 *
+	 * @since x.x.x
+	 *
+	 * @param array[] $statuses statuses. See statuses format in MerchantReport::get_product_view_report.
 	 */
-	public function update_product_stats() {
+	public function update_product_stats( array $statuses ): void {
 		$this->mc_statuses = [];
+
+		$this->process_product_statuses( $statuses );
+
 		// Update each product's mc_status and then update the global statistics.
 		$this->update_products_meta_with_mc_status();
-		$this->update_mc_status_statistics();
+		$this->update_intermediate_product_statistics();
 	}
 
 	/**
@@ -654,17 +694,17 @@
 	 *
 	 * @return bool Whether the product is expiring.
 	 */
 	protected function product_is_expiring( DateTime $expiration_date ): bool {
 		if ( ! $expiration_date ) {
 			return false;
 		}
 
 		// Products are considered expiring if they will expire within 3 days.
 		return time() + 3 * DAY_IN_SECONDS > $expiration_date->getTimestamp();
 	}
 
 	/**
-	 * Calculate the synced product status statistics. It will group
+	 * Sum and update the intermediate product status statistics. It will group
 	 * the variations for the same parent.
 	 *
 	 * For the case that one variation is approved and the other disapproved:
@@ -675,7 +715,7 @@
 	 *
 	 * @return array Product status statistics.
 	 */
-	protected function calculate_synced_product_statistics(): array {
+	protected function update_intermediate_product_statistics(): array {
 		$product_statistics = [
 			MCStatus::APPROVED           => 0,
 			MCStatus::PARTIALLY_APPROVED => 0,
@@ -685,6 +725,12 @@
 			MCStatus::NOT_SYNCED         => 0,
 		];
 
+		// If the option is set, use it to sum the total quantity.
+		$product_statistics_intermediate_data = $this->options->get( OptionsInterface::PRODUCT_STATUSES_COUNT_INTERMEDIATE_DATA );
+		if ( $product_statistics_intermediate_data ) {
+			$product_statistics = $product_statistics_intermediate_data;
+		}
+
 		$product_statistics_priority = [
 			MCStatus::APPROVED           => 6,
 			MCStatus::PARTIALLY_APPROVED => 5,
@@ -716,36 +762,69 @@
 			$product_statistics[ $parent_status ] += 1;
 		}
 
+		$this->options->update( OptionsInterface::PRODUCT_STATUSES_COUNT_INTERMEDIATE_DATA, $product_statistics );
+
 		return $product_statistics;
 	}
 
 	/**
-	 * Calculate the product status statistics and update the transient.
+	 * Calculate the total count of products in the MC using the statistics.
+	 *
+	 * @since x.x.x
+	 *
+	 * @param array $statistics
+	 *
+	 * @return int
 	 */
-	protected function update_mc_status_statistics() {
-		$product_statistics = $this->calculate_synced_product_statistics();
+	protected function calculate_total_synced_product_statistics( array $statistics ): int {
+		if ( ! count( $statistics ) ) {
+			return 0;
+		}
 
-		/** @var ProductRepository $product_repository */
-		$product_repository                         = $this->container->get( ProductRepository::class );
-		$product_statistics[ MCStatus::NOT_SYNCED ] = count( $product_repository->find_mc_not_synced_product_ids() );
+		$synced_status_values = array_values( array_diff( $statistics, [ $statistics[ MCStatus::NOT_SYNCED ] ] ) );
+		return array_sum( $synced_status_values );
+	}
 
-		$this->mc_statuses = [
-			'timestamp'  => $this->cache_created_time->getTimestamp(),
-			'statistics' => $product_statistics,
-		];
 
-		// Update the cached values
-		$this->container->get( TransientsInterface::class )->set(
-			Transients::MC_STATUSES,
-			$this->mc_statuses,
-			$this->get_status_lifetime()
-		);
+	/**
+	 * Handle the completion of the Merchant Center statuses fetching.
+	 *
+	 * @since x.x.x
+	 */
+	public function handle_complete_mc_statuses_fetching() {
+		$intermediate_data = $this->options->get( OptionsInterface::PRODUCT_STATUSES_COUNT_INTERMEDIATE_DATA );
+
+		if ( $intermediate_data ) {
+
+			$total_synced_products = $this->calculate_total_synced_product_statistics( $intermediate_data );
+
+			/** @var ProductRepository $product_repository */
+			$product_repository                        = $this->container->get( ProductRepository::class );
+			$intermediate_data[ MCStatus::NOT_SYNCED ] = count(
+				$product_repository->find_all_product_ids()
+			) - $total_synced_products;
+
+			$mc_statuses = [
+				'timestamp'  => $this->cache_created_time->getTimestamp(),
+				'statistics' => $intermediate_data,
+				'loading'    => false,
+			];
+
+			$this->container->get( TransientsInterface::class )->set(
+				Transients::MC_STATUSES,
+				$mc_statuses,
+				$this->get_status_lifetime()
+			);
+
+			$this->delete_product_statuses_count_intermediate_data();
+		}
 	}
 
+
 	/**
 	 * Update the Merchant Center status for each product.
 	 */
 	protected function update_products_meta_with_mc_status() {
 		// Generate a product_id=>mc_status array.
 		$new_product_statuses = [];
 		foreach ( $this->product_statuses as $types ) {
@@ -767,16 +846,11 @@
 		}
 		ksort( $new_product_statuses );
 
-		/** @var ProductRepository $product_repository */
-		$product_repository = $this->container->get( ProductRepository::class );
-
 		/** @var ProductMetaQueryHelper $product_meta_query_helper */
 		$product_meta_query_helper = $this->container->get( ProductMetaQueryHelper::class );
 
 		// Get all MC statuses.
 		$current_product_statuses = $product_meta_query_helper->get_all_values( ProductMetaHandler::KEY_MC_STATUS );
-		// Get all product IDs.
-		$all_product_ids = array_flip( $product_repository->find_ids() );
 
 		// Format: product_id=>status.
 		$to_insert = [];
@@ -791,21 +865,8 @@
 				// MC status not same as WC, update.
 				$to_update[ $new_status ][] = intval( $product_id );
 			}
-			// Unset all found statuses from WC products array.
-			unset( $all_product_ids[ $product_id ] );
 		}
 
-		// Set products NOT FOUND in MC to NOT_SYNCED.
-		foreach ( array_keys( $all_product_ids ) as $product_id ) {
-			if ( empty( $current_product_statuses[ $product_id ] ) ) {
-				$to_insert[ $product_id ] = MCStatus::NOT_SYNCED;
-			} elseif ( $current_product_statuses[ $product_id ] !== MCStatus::NOT_SYNCED ) {
-				$to_update[ MCStatus::NOT_SYNCED ][] = $product_id;
-			}
-		}
-
-		unset( $all_product_ids, $current_product_statuses, $new_product_statuses );
-
 		// Insert and update changed MC Status postmeta.
 		$product_meta_query_helper->batch_insert_values( ProductMetaHandler::KEY_MC_STATUS, $to_insert );
 		foreach ( $to_update as $status => $product_ids ) {

src/Options/OptionsInterface.php

@@ -37,6 +37,7 @@ interface OptionsInterface {
 	public const SITE_VERIFICATION                         = 'site_verification';
 	public const SYNCABLE_PRODUCTS_COUNT                   = 'syncable_products_count';
 	public const SYNCABLE_PRODUCTS_COUNT_INTERMEDIATE_DATA = 'syncable_products_count_intermediate_data';
+	public const PRODUCT_STATUSES_COUNT_INTERMEDIATE_DATA  = 'product_statuses_count_intermediate_data';
 	public const TARGET_AUDIENCE                           = 'target_audience';
 	public const TOURS                                     = 'tours';
 	public const UPDATE_ALL_PRODUCTS_LAST_SYNC             = 'update_all_products_last_sync';
@@ -68,6 +69,7 @@ interface OptionsInterface {
 		self::SITE_VERIFICATION                         => true,
 		self::SYNCABLE_PRODUCTS_COUNT                   => true,
 		self::SYNCABLE_PRODUCTS_COUNT_INTERMEDIATE_DATA => true,
+		self::PRODUCT_STATUSES_COUNT_INTERMEDIATE_DATA  => true,
 		self::TARGET_AUDIENCE                           => true,
 		self::TOURS                                     => true,
 		self::UPDATE_ALL_PRODUCTS_LAST_SYNC             => true,