Ignore index not found error in file-backed index polling task (#4474)

quickwit/quickwit-metastore/src/metastore/file_backed/lazy_file_backed_index.rs

@@ -20,14 +20,14 @@
 use std::sync::{Arc, Weak};
 use std::time::Duration;
 
-use quickwit_proto::metastore::MetastoreResult;
+use quickwit_proto::metastore::{EntityKind, MetastoreError, MetastoreResult};
 use quickwit_proto::types::IndexId;
 use quickwit_storage::Storage;
 use tokio::sync::{Mutex, OnceCell};
 use tracing::error;
 
 use super::file_backed_index::FileBackedIndex;
-use super::store_operations::load_index;
+use super::store_operations::{load_index, METASTORE_FILE_NAME};
 
 /// Lazy [`FileBackedIndex`]. It loads a `FileBackedIndex`
 /// on demand and optionally spawns a task to poll
@@ -86,19 +86,30 @@ impl LazyFileBackedIndex {
 async fn poll_index_metadata_once(
     storage: &dyn Storage,
     index_id: &str,
-    metadata_mutex: &Mutex<FileBackedIndex>,
+    index_mutex: &Mutex<FileBackedIndex>,
 ) {
-    let mut metadata_lock = metadata_mutex.lock().await;
-    if metadata_lock.flip_recently_modified_down() {
+    let mut locked_index = index_mutex.lock().await;
+    if locked_index.flip_recently_modified_down() {
         return;
     }
-    let index_fetch_res = load_index(storage, index_id).await;
-    match index_fetch_res {
+    let load_index_result = load_index(storage, index_id).await;
+
+    match load_index_result {
         Ok(index) => {
-            *metadata_lock = index;
+            *locked_index = index;
+        }
+        Err(MetastoreError::NotFound(EntityKind::Index { .. })) => {
+            // The index has been deleted by the file-backed metastore holding a reference to this
+            // index. When it removes an index, it does so without holding the lock on the target
+            // index. As a result, the associated polling task may run for one
+            // more iteration before exiting and `load_index` returns a `NotFound` error.
         }
-        Err(fetch_error) => {
-            error!(error=?fetch_error, "fetch-metadata-error");
+        Err(metastore_error) => {
+            error!(
+                error=%metastore_error,
+                "failed to load index metadata from metastore file located at `{}/{index_id}/{METASTORE_FILE_NAME}`",
+                storage.uri()
+            );
         }
     }
 }
@@ -112,6 +123,7 @@ fn spawn_index_metadata_polling_task(
     tokio::task::spawn(async move {
         let mut interval = tokio::time::interval(polling_interval);
         interval.tick().await; //< this is to prevent fetch right after the first population of the data.
+
         while let Some(metadata_mutex) = metastore_weak.upgrade() {
             interval.tick().await;
             poll_index_metadata_once(&*storage, &index_id, &metadata_mutex).await;

quickwit/quickwit-metastore/src/metastore/file_backed/store_operations.rs

@@ -25,7 +25,7 @@ use quickwit_storage::{Storage, StorageError, StorageErrorKind};
 use crate::metastore::file_backed::file_backed_index::FileBackedIndex;
 
 /// Index metastore file managed by [`FileBackedMetastore`](crate::FileBackedMetastore).
-const METASTORE_FILE_NAME: &str = "metastore.json";
+pub(super) const METASTORE_FILE_NAME: &str = "metastore.json";
 
 /// Path to the metadata file from the given index ID.
 pub(super) fn metastore_filepath(index_id: &str) -> PathBuf {