Fix handling sync status by harmonizing SDK and custom sync statuses

android/engine/src/main/java/org/smartregister/fhircore/engine/configuration/ConfigurationRegistry.kt

@@ -25,6 +25,7 @@
 import com.google.android.fhir.db.ResourceNotFoundException
 import com.google.android.fhir.get
 import com.google.android.fhir.knowledge.KnowledgeManager
+import com.google.android.fhir.sync.ParamMap
 import com.google.android.fhir.sync.SyncDataParams.LAST_UPDATED_KEY
 import com.google.android.fhir.sync.download.ResourceSearchParams
 import dagger.hilt.android.qualifiers.ApplicationContext
@@ -55,7 +56,6 @@
 import org.jetbrains.annotations.VisibleForTesting
 import org.json.JSONObject
 import org.smartregister.fhircore.engine.BuildConfig
-import org.smartregister.fhircore.engine.configuration.app.ApplicationConfiguration
 import org.smartregister.fhircore.engine.configuration.app.ConfigService
 import org.smartregister.fhircore.engine.data.remote.fhir.resource.FhirResourceDataSource
 import org.smartregister.fhircore.engine.di.NetworkModule
@@ -565,7 +565,7 @@
     return resultBundle
   }
 
-  suspend fun fetchResources(
+  private suspend fun fetchResources(
     gatewayModeHeaderValue: String? = null,
     url: String,
   ) {
@@ -581,11 +581,8 @@
           Timber.e("Error occurred while retrieving resource via URL $url", throwable)
         }
         .getOrThrow()
-
     val nextPageUrl = resultBundle.getLink(PAGINATION_NEXT)?.url
-
     processResultBundleEntries(resultBundle.entry)
-
     if (!nextPageUrl.isNullOrEmpty()) {
       fetchResources(
         gatewayModeHeaderValue = gatewayModeHeaderValue,
@@ -594,7 +591,7 @@
     }
   }
 
-  private suspend fun processResultBundleEntries(
+  suspend fun processResultBundleEntries(
     resultBundleEntries: List<Bundle.BundleEntryComponent>,
   ) {
     resultBundleEntries.forEach { bundleEntryComponent ->
@@ -784,10 +781,8 @@
     }
   }
 
-  suspend fun loadResourceSearchParams():
-    Pair<Map<String, Map<String, String>>, ResourceSearchParams> {
+  suspend fun loadResourceSearchParams(): Pair<Map<String, ParamMap>, ResourceSearchParams> {
     val syncConfig = retrieveResourceConfiguration<Parameters>(ConfigType.Sync)
-    val appConfig = retrieveConfiguration<ApplicationConfiguration>(ConfigType.Application)
     val customResourceSearchParams = mutableMapOf<String, MutableMap<String, String>>()
     val fhirResourceSearchParams = mutableMapOf<ResourceType, MutableMap<String, String>>()
     val organizationResourceTag =
@@ -847,9 +842,32 @@
             }
           }
       }
+
+    // If there are custom resources to be synced return 2 otherwise 1
+    sharedPreferencesHelper.write(
+      SharedPreferenceKey.TOTAL_SYNC_COUNT.name,
+      if (customResourceSearchParams.isEmpty()) "1" else "2",
+    )
     return Pair(customResourceSearchParams, fhirResourceSearchParams)
   }
 
+  /**
+   * This function returns either '1' or '2' depending on whether there are custom resources (not
+   * included in ResourceType enum) in the sync configuration. The custom resources are configured
+   * in the sync configuration JSON file as valid FHIR SearchParameter of type 'special'. If there
+   * are custom resources to be synced with the data, the application will first download the custom
+   * resources then the rest of the app data.
+   */
+  fun retrieveTotalSyncCount(): Int {
+    val totalSyncCount = sharedPreferencesHelper.read(SharedPreferenceKey.TOTAL_SYNC_COUNT.name, "")
+    return if (totalSyncCount.isNullOrBlank()) {
+      retrieveResourceConfiguration<Parameters>(ConfigType.Sync)
+        .parameter
+        .map { it.resource as SearchParameter }
+        .count { it.hasType() && it.type == Enumerations.SearchParamType.SPECIAL }
+    } else totalSyncCount.toInt()
+  }
+
   companion object {
     const val BASE_CONFIG_PATH = "configs/%s"
     const val COMPOSITION_CONFIG_PATH = "configs/%s/composition_config.json"

android/engine/src/main/java/org/smartregister/fhircore/engine/sync/AppSyncWorker.kt

@@ -41,11 +41,14 @@
 import com.ibm.icu.util.Calendar
 import dagger.assisted.Assisted
 import dagger.assisted.AssistedInject
+import java.time.OffsetDateTime
 import kotlinx.coroutines.runBlocking
 import org.smartregister.fhircore.engine.R
 import org.smartregister.fhircore.engine.configuration.app.ConfigService
 import org.smartregister.fhircore.engine.util.NotificationConstants
 import org.smartregister.fhircore.engine.util.SharedPreferenceKey
+import retrofit2.HttpException
+import timber.log.Timber
 
 @HiltWorker
 class AppSyncWorker
@@ -57,6 +60,7 @@
   private val openSrpFhirEngine: FhirEngine,
   private val appTimeStampContext: AppTimeStampContext,
   private val configService: ConfigService,
+  private val customResourceSyncService: CustomResourceSyncService,
 ) : FhirSyncWorker(appContext, workerParams), OnSyncListener {
   private val notificationManager =
     appContext.getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager
@@ -74,11 +78,39 @@
     )
 
   override suspend fun doWork(): Result {
-    saveSyncStartTimestamp()
-    setForeground(getForegroundInfo())
-    return super.doWork()
+    kotlin
+      .runCatching {
+        saveSyncStartTimestamp()
+        setForeground(getForegroundInfo())
+        customResourceSyncService.runCustomResourceSync()
+      }
+      .onSuccess {
+        return super.doWork()
+      }
+      .onFailure { exception ->
+        when (exception) {
+          is HttpException -> {
+            val response = exception.response()
+            if (response != null && (400..503).contains(response.code())) {
+              Timber.e("HTTP exception ${response.code()} -> ${response.errorBody()}")
+            }
+          }
+          else -> Timber.e(exception)
+        }
+        syncListenerManager.emitSyncStatus(
+          SyncState(
+            counter = SYNC_COUNTER_1,
+            currentSyncJobStatus = CurrentSyncJobStatus.Failed(OffsetDateTime.now()),
+          ),
+        )
+        return result()
+      }
+    return Result.success()
   }
 
+  private fun result(): Result =
+    if (inputData.getInt(MAX_RETRIES, 3) > runAttemptCount) Result.retry() else Result.failure()
+
   private fun saveSyncStartTimestamp() {
     syncListenerManager.sharedPreferencesHelper.write(
       SharedPreferenceKey.SYNC_START_TIMESTAMP.name,
@@ -131,8 +163,8 @@
   private fun getSyncProgress(completed: Int, total: Int) =
     completed * 100 / if (total > 0) total else 1
 
-  override fun onSync(syncJobStatus: CurrentSyncJobStatus) {
-    when (syncJobStatus) {
+  override fun onSync(syncState: SyncState) {
+    when (val syncJobStatus = syncState.currentSyncJobStatus) {
       is CurrentSyncJobStatus.Running -> {
         if (syncJobStatus.inProgressSyncJob is SyncJobStatus.InProgress) {
           val inProgressSyncJob = syncJobStatus.inProgressSyncJob as SyncJobStatus.InProgress
@@ -175,4 +207,8 @@
       buildNotification(progress = progress, isSyncUpload = isSyncUpload, isInitial = false)
     notificationManager.notify(NotificationConstants.NotificationId.DATA_SYNC, notification)
   }
+
+  companion object {
+    const val MAX_RETRIES = "max_retires"
+  }
 }

android/engine/src/main/java/org/smartregister/fhircore/engine/sync/CustomResourceSyncService.kt

@@ -0,0 +1,151 @@
+/*
+ * Copyright 2021-2024 Ona Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.smartregister.fhircore.engine.sync
+
+import com.google.android.fhir.sync.CurrentSyncJobStatus
+import com.google.android.fhir.sync.SyncJobStatus
+import com.google.android.fhir.sync.SyncOperation
+import com.google.android.fhir.sync.concatParams
+import java.time.OffsetDateTime
+import javax.inject.Inject
+import javax.inject.Singleton
+import org.smartregister.fhircore.engine.configuration.ConfigurationRegistry
+import org.smartregister.fhircore.engine.configuration.ConfigurationRegistry.Companion.PAGINATION_NEXT
+import org.smartregister.fhircore.engine.data.remote.fhir.resource.FhirResourceDataSource
+import org.smartregister.fhircore.engine.util.DispatcherProvider
+import timber.log.Timber
+
+@Singleton
+class CustomResourceSyncService
+@Inject
+constructor(
+  val configurationRegistry: ConfigurationRegistry,
+  val dispatcherProvider: DispatcherProvider,
+  val fhirResourceDataSource: FhirResourceDataSource,
+  val syncListenerManager: SyncListenerManager,
+) {
+  suspend fun runCustomResourceSync() {
+    val (resourceSearchParams, _) = configurationRegistry.loadResourceSearchParams()
+    if (resourceSearchParams.isEmpty()) return
+
+    val resourceUrls =
+      resourceSearchParams
+        .asIterable()
+        .filter { it.value.isNotEmpty() }
+        .map { "${it.key}?${it.value.concatParams()}" }
+
+    val summaryCount = fetchSummaryCount(resourceUrls).values.sumOf { it ?: 0 }
+
+    resourceUrls.forEach { url ->
+      fetchCustomResources(
+        gatewayModeHeaderValue = ConfigurationRegistry.FHIR_GATEWAY_MODE_HEADER_VALUE,
+        url = url,
+        totalCounts = summaryCount,
+      )
+    }
+  }
+
+  private suspend fun fetchCustomResources(
+    gatewayModeHeaderValue: String? = null,
+    url: String,
+    totalCounts: Int = 0,
+    completedRecords: Int = 0,
+  ) {
+    runCatching {
+        Timber.d("Setting state: Running")
+        syncListenerManager.emitSyncStatus(
+          SyncState(
+            counter = SYNC_COUNTER_1,
+            currentSyncJobStatus =
+              CurrentSyncJobStatus.Running(
+                SyncJobStatus.InProgress(
+                  syncOperation = SyncOperation.DOWNLOAD,
+                  total = totalCounts,
+                  completed = completedRecords,
+                ),
+              ),
+          ),
+        )
+        Timber.d("Fetching page with URL: $url")
+        if (gatewayModeHeaderValue.isNullOrEmpty()) {
+          fhirResourceDataSource.getResource(url)
+        } else {
+          fhirResourceDataSource.getResourceWithGatewayModeHeader(gatewayModeHeaderValue, url)
+        }
+      }
+      .onFailure { throwable ->
+        Timber.e("Error occurred while retrieving resource via URL $url", throwable)
+        syncListenerManager.emitSyncStatus(
+          SyncState(
+            counter = SYNC_COUNTER_1,
+            currentSyncJobStatus = CurrentSyncJobStatus.Failed(OffsetDateTime.now()),
+          ),
+        )
+        return
+      }
+      .onSuccess { resultBundle ->
+        configurationRegistry.processResultBundleEntries(resultBundle.entry)
+        val newCompletedRecords = completedRecords + resultBundle.entry.size
+        syncListenerManager.emitSyncStatus(
+          SyncState(
+            counter = SYNC_COUNTER_1,
+            currentSyncJobStatus =
+              CurrentSyncJobStatus.Running(
+                SyncJobStatus.InProgress(
+                  syncOperation = SyncOperation.DOWNLOAD,
+                  total = totalCounts,
+                  completed = newCompletedRecords,
+                ),
+              ),
+          ),
+        )
+
+        val nextPageUrl = resultBundle.getLink(PAGINATION_NEXT)?.url
+
+        if (!nextPageUrl.isNullOrEmpty()) {
+          fetchCustomResources(
+            gatewayModeHeaderValue = gatewayModeHeaderValue,
+            url = nextPageUrl,
+            totalCounts = totalCounts,
+            completedRecords = newCompletedRecords,
+          )
+        } else {
+          Timber.d("Fetch complete. Emitting SyncStatus.Succeeded.")
+          syncListenerManager.emitSyncStatus(
+            SyncState(
+              counter = SYNC_COUNTER_1,
+              currentSyncJobStatus = CurrentSyncJobStatus.Succeeded(OffsetDateTime.now()),
+            ),
+          )
+        }
+      }
+  }
+
+  /** Fetch summary counts for the provided [resourceUrls] */
+  private suspend fun fetchSummaryCount(resourceUrls: List<String>): Map<String, Int?> =
+    resourceUrls
+      .associate { url ->
+        val summaryUrl = "$url&summary=count"
+        val total: Int? =
+          runCatching { fhirResourceDataSource.getResource(summaryUrl) }
+            .onFailure { Timber.e(it, "Failed to fetch summary for $summaryUrl") }
+            .getOrNull()
+            ?.total
+        summaryUrl to total
+      }
+      .also { summaries -> Timber.i("Summary fetch results: $summaries") }
+}