WIP sync

danielyrovas · Jun 19, 2024 · 5997261 · 5997261
1 parent 2a57881
commit 5997261
Show file tree

Hide file tree

Showing 9 changed files with 125 additions and 66 deletions.
diff --git a/app/src/main/java/org/yrovas/linklater/data/Bookmark.kt b/app/src/main/java/org/yrovas/linklater/data/Bookmark.kt
@@ -16,6 +16,7 @@ data class Bookmark(
     val notes: String? = null,
     val website_title: String? = null,
     val website_description: String? = null,
+//    val web_archive_snapshot_url: String? = null,
     val is_archived: Boolean = false,
     val unread: Boolean = false,
     val shared: Boolean = false,

diff --git a/app/src/main/java/org/yrovas/linklater/data/local/BookmarkDataSourceImpl.kt b/app/src/main/java/org/yrovas/linklater/data/local/BookmarkDataSourceImpl.kt
@@ -3,7 +3,6 @@ package org.yrovas.linklater.data.local
 import android.util.Log
 import app.cash.sqldelight.coroutines.asFlow
 import app.cash.sqldelight.coroutines.mapToList
-import app.cash.sqldelight.coroutines.mapToOne
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.flow.Flow
 import kotlinx.coroutines.flow.map
@@ -12,14 +11,14 @@ import org.yrovas.linklater.Database
 import org.yrovas.linklater.data.Bookmark
 import org.yrovas.linklater.data.toBookmark
 import org.yrovas.linklater.domain.BookmarkDataSource
-import kotlin.math.log
 
 const val TAG = "DEBUG"
 
 class BookmarkDataSourceImpl(db: Database) : BookmarkDataSource {
     init {
         Log.d("DEBUG/create", "BookmarkDataSourceImpl: CREATE")
     }
+
     private val q = db.bookmarkTagsQueries
 
     override suspend fun getBookmark(id: Long): Bookmark? {
@@ -29,9 +28,8 @@ class BookmarkDataSourceImpl(db: Database) : BookmarkDataSource {
     }
 
     override fun getBookmarks(): Flow<List<Bookmark>> {
-        return q.getBookmarksWithTags().asFlow().mapToList(Dispatchers.IO)
-            .map { list ->
-                list.map { it.toBookmark() }
+        return q.getBookmarksWithTags().asFlow().mapToList(Dispatchers.IO).map { list ->
+            list.map { it.toBookmark() }
         }
     }
 
@@ -106,4 +104,15 @@ class BookmarkDataSourceImpl(db: Database) : BookmarkDataSource {
             q.deleteBookmarkByID(id)
         }
     }
+
+    override suspend fun upsertOrDeleteWithinRange(bookmarks: List<Bookmark>) {
+        withContext(Dispatchers.IO) {
+            val keepIds = bookmarks.map { it.id }
+            q.deleteBookmarksCreatedWithinRangeExcluding(
+                start_date = bookmarks.first().date_added,
+                end_date = bookmarks.last().date_added,
+                keep = keepIds
+            )
+        }
+    }
 }
diff --git a/app/src/main/java/org/yrovas/linklater/data/remote/EmptyBookmarkAPI.kt b/app/src/main/java/org/yrovas/linklater/data/remote/EmptyBookmarkAPI.kt
@@ -1,8 +1,10 @@
 package org.yrovas.linklater.data.remote
 
 import kotlinx.coroutines.delay
+import kotlinx.coroutines.flow.Flow
 import kotlinx.coroutines.flow.MutableStateFlow
 import kotlinx.coroutines.flow.StateFlow
+import kotlinx.coroutines.flow.flowOf
 import org.yrovas.linklater.data.Bookmark
 import org.yrovas.linklater.data.BookmarkMetadata
 import org.yrovas.linklater.data.LocalBookmark
@@ -29,13 +31,16 @@ class EmptyBookmarkAPI : BookmarkAPI {
 
     override suspend fun getBookmarks(
         page: Int,
-        query: String?,
     ): Res<List<Bookmark>, APIError> {
         delay(4300)
         return Ok(emptyList())
 //        return Err(APIError.AUTH)
     }
 
+    override suspend fun getAllBookmarks(): Flow<Res<List<Bookmark>, APIError>> {
+        return flowOf()
+    }
+
     override suspend fun saveBookmark(bookmark: LocalBookmark): Res<Bookmark, APIError> {
         return Err(APIError.NO_CONNECTION)
     }

diff --git a/app/src/main/java/org/yrovas/linklater/data/remote/LinkDingAPI.kt b/app/src/main/java/org/yrovas/linklater/data/remote/LinkDingAPI.kt
@@ -8,9 +8,13 @@ import io.ktor.client.request.header
 import io.ktor.client.request.parameter
 import io.ktor.client.request.post
 import io.ktor.client.request.setBody
+import kotlinx.coroutines.awaitCancellation
+import kotlinx.coroutines.flow.Flow
 import kotlinx.coroutines.flow.MutableStateFlow
 import kotlinx.coroutines.flow.StateFlow
 import kotlinx.coroutines.flow.asStateFlow
+import kotlinx.coroutines.flow.flow
+import kotlinx.coroutines.flow.flowOf
 import kotlinx.coroutines.flow.update
 import kotlinx.serialization.Serializable
 import me.tatarka.inject.annotations.Inject
@@ -25,27 +29,32 @@ import org.yrovas.linklater.domain.BookmarkAPI
 import org.yrovas.linklater.domain.Err
 import org.yrovas.linklater.domain.Ok
 import org.yrovas.linklater.domain.Res
+import org.yrovas.linklater.domain.getOrThrow
+import org.yrovas.linklater.domain.ifOk
+import org.yrovas.linklater.domain.isOk
+import org.yrovas.linklater.domain.mapData
+import org.yrovas.linklater.domain.ok
+import org.yrovas.linklater.domain.then
+
+const val TAG = "DEBUG/net"
+const val MAX_PAGE_COUNT = 10000
 
 @AppScope
 @Inject
 class LinkDingAPI(
     private val client: HttpClient,
     private var endpoint: String? = null,
     private var token: String? = null,
-    private val pageSize: Int = 20,
+    private val pageSize: Int = 100,
 ) : BookmarkAPI {
-    init {
-        Log.d("DEBUG/create", "LinkDingAPI: CREATE")
-    }
-
     private val _authProvided = MutableStateFlow(false)
     override val authProvided: StateFlow<Boolean> = _authProvided.asStateFlow()
 
     override fun authenticate(
         endpoint: String?,
         token: String?,
     ): Res<Unit, APIError> {
-        Log.d("DEBUG", "authenticate: with endpoint: $endpoint")
+        Log.d(TAG, "authenticate: with endpoint: $endpoint")
         if (!endpoint.isNullOrBlank()) {
             if (!checkURL(endpoint)) return Err(APIError.INCORRECT_ENDPOINT)
             this.endpoint = endpoint
@@ -69,33 +78,63 @@ class LinkDingAPI(
         }
     }
 
-    override suspend fun getBookmarks(
+    private suspend fun fetchBookmarks(
         page: Int,
-        query: String?,
-    ): Res<List<Bookmark>, APIError> {
-//        delay(3200)
+        archived: Boolean = false,
+        sortByAddedAsc: Boolean = false,
+    ): Res<BookmarkResponse, APIError> {
         if (!authProvided.value) return Err(APIError.INCORRECT_AUTH)
         return try {
-            Log.d("DEBUG/net", "getBookmarks: starting request")
-            val response = client.get("${endpoint!!}/bookmarks/") {
-                header("Authorization", "Token ${token!!}")
-                if (page > 0) {
-//                    parameter("offset", pageSize * page)
-                    url.parameters.append(
-                        "offset", (pageSize * page).toString()
-                    )
-                }
-                if (!query.isNullOrBlank()) {
-                    url.parameters.append("q", query)
+            val response =
+                client.get("${endpoint!!}/bookmarks/${if (archived) "archived/" else ""}") {
+                    header("Authorization", "Token ${token!!}")
+                    if (sortByAddedAsc) {
+                        parameter("sort", "added_asc")
+                    }
+                    if (page > 0) {
+                        parameter("offset", (pageSize * page).toString())
+                    }
                 }
-            }
-            Ok(response.body<BookmarkResponse>().results)
+            Ok(response.body<BookmarkResponse>())
         } catch (e: Exception) {
-            Log.i("DEBUG/net", "getBookmarks: ${e.message}")
+            Log.i(TAG, "getBookmarks: ${e.message}")
             Err(APIError.NO_CONNECTION)
         }
     }
 
+    override suspend fun getBookmarks(page: Int): Res<List<Bookmark>, APIError> {
+        return fetchBookmarks(page).mapData { it.results }
+    }
+
+    override suspend fun getAllBookmarks(): Flow<Res<List<Bookmark>, APIError>> {
+        return flow {
+            Log.d(TAG, "getAllBookmarks: Flow Created")
+            var page = 0
+            var archived = false
+
+            // NOTE: we might not crawl archived pages if there are more than 1000 x 10000 bookmarks
+            while (page < MAX_PAGE_COUNT) {
+                Log.d(TAG, "getAllBookmarks: Fetching page $page")
+                val res = fetchBookmarks(page, archived, sortByAddedAsc = true)
+                page++
+                if (res.isOk) {
+                    emit(res.mapData { it.results })
+
+                    // exit when finished archived
+                    if (res.getOrThrow().next.isNullOrBlank() && archived) break
+
+                    // crawl archived after completing unarchived.
+                    if (res.getOrThrow().next.isNullOrBlank() && !archived) {
+                        archived = true
+                        page = 0
+                    }
+                } else {
+                    break
+                }
+            }
+        }
+    }
+
     override suspend fun saveBookmark(bookmark: LocalBookmark): Res<Bookmark, APIError> {
         if (!authProvided.value) return Err(APIError.INCORRECT_AUTH)
         return try {
@@ -115,11 +154,11 @@ class LinkDingAPI(
     override suspend fun getTags(page: Int): Res<List<String>, APIError> {
         if (!authProvided.value) return Err(APIError.INCORRECT_AUTH)
         return try {
-            Log.d("DEBUG/net", "getTags: starting request")
+            Log.d(TAG, "getTags: starting request")
             val response = client.get("${endpoint!!}/tags/") {
                 header("Authorization", "Token ${token!!}")
                 if (page > 0) {
-                    url.parameters.append("offset", (pageSize * page).toString())
+                    parameter("offset", (pageSize * page).toString())
                 }
             }
             Ok(response.body<TagResponse>().results)
@@ -133,12 +172,11 @@ class LinkDingAPI(
         if (url.isBlank()) return null to null
 
         return try {
-            Log.d("DEBUG/net", "checkExists")
-            val response =
-                client.get("${endpoint!!}/bookmarks/check/") {
-                    header("Authorization", "Token ${token!!}")
-                    parameter("url", url)
-                }
+            Log.d(TAG, "checkExists")
+            val response = client.get("${endpoint!!}/bookmarks/check/") {
+                header("Authorization", "Token ${token!!}")
+                parameter("url", url)
+            }
             val r = response.body<BookmarkExistsResponse>()
             r.bookmark to r.metadata
         } catch (e: Exception) {

diff --git a/app/src/main/java/org/yrovas/linklater/domain/BookmarkAPI.kt b/app/src/main/java/org/yrovas/linklater/domain/BookmarkAPI.kt
@@ -1,5 +1,6 @@
 package org.yrovas.linklater.domain
 
+import kotlinx.coroutines.flow.Flow
 import kotlinx.coroutines.flow.StateFlow
 import org.yrovas.linklater.data.Bookmark
 import org.yrovas.linklater.data.BookmarkMetadata
@@ -11,7 +12,8 @@ interface BookmarkAPI {
     val authProvided: StateFlow<Boolean>
     fun authenticate(endpoint: String? = null, token: String? = null): Res<Unit, APIError>
     suspend fun checkConnection(): Res<Unit, APIError>
-    suspend fun getBookmarks(page: Int, query: String? = null): Res<List<Bookmark>, APIError>
+    suspend fun getBookmarks(page: Int): Res<List<Bookmark>, APIError>
+    suspend fun getAllBookmarks(): Flow<Res<List<Bookmark>, APIError>>
     suspend fun saveBookmark(bookmark: LocalBookmark): Res<Bookmark, APIError>
     suspend fun getTags(page: Int): Res<List<String>, APIError>
     suspend fun checkExists(url: String): Pair<Bookmark?, BookmarkMetadata?>

diff --git a/app/src/main/java/org/yrovas/linklater/domain/BookmarkDataSource.kt b/app/src/main/java/org/yrovas/linklater/domain/BookmarkDataSource.kt
@@ -10,4 +10,5 @@ interface BookmarkDataSource {
     suspend fun insertBookmark(bookmark: Bookmark)
     suspend fun insertBookmarks(bookmarks: List<Bookmark>)
     suspend fun deleteBookmark(id: Long)
+    suspend fun upsertOrDeleteWithinRange(bookmarks: List<Bookmark>)
 }
diff --git a/app/src/main/java/org/yrovas/linklater/domain/Res.kt b/app/src/main/java/org/yrovas/linklater/domain/Res.kt
@@ -56,16 +56,16 @@ fun <D, T : Any, E : Error> Res<D, E>.then(ok: (data: D) -> T, err: (error: E) -
     }
 }
 
-fun <D, T, E : Error> Res<T, E>.into(withData: D): Res<D, E> {
+fun <D, T, E : Error> Res<T, E>.mapData(mapData: (T) -> D): Res<D, E> {
     return when (this) {
         is Res.Err -> Res.Err(this.error)
-        is Res.Ok -> Res.Ok(withData)
+        is Res.Ok -> Res.Ok(mapData(this.data))
     }
 }
 
-fun <T, E : Error> Res<T, E>.into(withError: E): Res<T, E> {
+fun <T, E : Error> Res<T, E>.mapError(mapError: (E) -> E): Res<T, E> {
     return when (this) {
-        is Res.Err -> Err(withError)
+        is Res.Err -> Err(mapError(this.error))
         is Res.Ok -> Ok(this.data)
     }
 }

diff --git a/app/src/main/java/org/yrovas/linklater/ui/screens/preferences/PreferencesState.kt b/app/src/main/java/org/yrovas/linklater/ui/screens/preferences/PreferencesState.kt
@@ -2,7 +2,9 @@ package org.yrovas.linklater.ui.screens.preferences
 
 import android.util.Log
 import androidx.lifecycle.viewModelScope
+import androidx.lifecycle.viewmodel.compose.viewModel
 import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.cancel
 import kotlinx.coroutines.flow.MutableStateFlow
 import kotlinx.coroutines.flow.asStateFlow
 import kotlinx.coroutines.flow.update
@@ -15,6 +17,10 @@ import org.yrovas.linklater.data.local.Prefs
 import org.yrovas.linklater.domain.BookmarkAPI
 import org.yrovas.linklater.domain.BookmarkDataSource
 import org.yrovas.linklater.domain.Res
+import org.yrovas.linklater.domain.errorOrThrow
+import org.yrovas.linklater.domain.getOrThrow
+import org.yrovas.linklater.domain.isErr
+import org.yrovas.linklater.domain.isOk
 import org.yrovas.linklater.intoTags
 import org.yrovas.linklater.ui.screens.home.TAG
 import org.yrovas.linklater.ui.screens.preferences.PreferencesState.Effect
@@ -118,34 +124,25 @@ class PreferencesState(
     }
 
     private fun fetchAllRemoteBookmarks() {
-        // guard against going back to home by launching from application scope
-        appScope.launch(Dispatchers.IO) {
-            var res = api.getBookmarks(0)
-            var page = 0
-            while (true) when (res) {
-                is Res.Err -> {
-                    Log.d(
-                        TAG, "fetchAllRemoteBookmarks: stopping due to ${res.error}"
-                    )
-                    break
-                }
-
-                is Res.Ok -> {
-                    if (res.data.isEmpty()) {
-                        Log.d(
-                            TAG, "fetchAllRemoteBookmarks: stopping due to empty result set"
-                        )
-                        break
-                    }
+        viewModelScope.launch {
+            api.getAllBookmarks().collect {
+                if (it.isErr) {
+                    Log.d(TAG, "fetchAllRemoteBookmarks: ${it.errorOrThrow()}")
+                    cancel()
+                } else {
                     Log.d(
-                        TAG, "fetchAllRemoteBookmarks: FETCHED ${res.data} from page $page"
+                        TAG, "fetchAllRemoteBookmarks: collected bookmarks ${it.getOrThrow().size}"
                     )
-                    bookmarkSource.insertBookmarks(res.data)
-                    res = api.getBookmarks(page++)
+                    // delete from db where date_added older than newest on page,
+                    // but younger than oldest on page - ie is in date range of page AND
+                    // id not in the set of bookmarks returned from API.
+                    bookmarkSource.upsertOrDeleteWithinRange(it.getOrThrow())
                 }
             }
-            appScope.showSnackbar("Refresh Complete")
         }
+        // guard against going back to home by launching from application scope
+//        appScope.launch(Dispatchers.IO) {
+//            appScope.showSnackbar("Refresh Complete")
     }
 
 

diff --git a/app/src/main/sqldelight/linklater/BookmarkTags.sq b/app/src/main/sqldelight/linklater/BookmarkTags.sq
@@ -79,6 +79,12 @@ SELECT COUNT(*) FROM bookmarkEntity
 deleteBookmarkByID:
 DELETE FROM bookmarkEntity WHERE id = :id;
 
+deleteBookmarksCreatedWithinRangeExcluding:
+DELETE FROM bookmarkEntity
+WHERE date_added BETWEEN :start_date AND :end_date
+AND id NOT IN :keep
+;
+
 getTagByID:
 SELECT (name) FROM tagEntity WHERE id = :id;