From cdf3db5e0d013b7e2837fab8843aa92282738e40 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Wed, 23 Oct 2019 11:32:58 +0100 Subject: [PATCH] Cleanup Concurrent RepositoryData Loading (#48329) The loading of `RepositoryData` is not an atomic operation. It uses a list + get combination of calls. This lead to accidentally returning an empty repository data for generations >=0 which can never not exist unless the repository is corrupted. In the test #48122 (and other SLM tests) there was a low chance of running into this concurrent modification scenario and the repository actually moving two index generations between listing out the index-N and loading the latest version of it. Since we only keep two index-N around at a time this lead to unexpectedly absent snapshots in status APIs. Fixing the behavior to be more resilient is non-trivial but in the works. For now I think we should simply throw in this scenario. This will also help prevent corruption in the unlikely event but possible of running into this issue in a snapshot create or delete operation on master failover on a repository like S3 which doesn't have the "no overwrites" protection on writing a new index-N. Fixes #48122 --- .../repositories/blobstore/BlobStoreRepository.java | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java index 8a9c12f9f4c74..2aeeef57996d4 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java @@ -767,9 +767,6 @@ public void endVerification(String seed) { public RepositoryData getRepositoryData() { try { return getRepositoryData(latestIndexBlobId()); - } catch (NoSuchFileException ex) { - // repository doesn't have an index blob, its a new blank repo - return RepositoryData.EMPTY; } catch (IOException ioe) { throw new RepositoryException(metadata.name(), "Could not determine repository generation from root blobs", ioe); } @@ -782,17 +779,12 @@ private RepositoryData getRepositoryData(long indexGen) { try { final String snapshotsIndexBlobName = INDEX_FILE_PREFIX + Long.toString(indexGen); - RepositoryData repositoryData; // EMPTY is safe here because RepositoryData#fromXContent calls namedObject try (InputStream blob = blobContainer().readBlob(snapshotsIndexBlobName); XContentParser parser = XContentType.JSON.xContent().createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, blob)) { - repositoryData = RepositoryData.snapshotsFromXContent(parser, indexGen); + return RepositoryData.snapshotsFromXContent(parser, indexGen); } - return repositoryData; - } catch (NoSuchFileException ex) { - // repository doesn't have an index blob, its a new blank repo - return RepositoryData.EMPTY; } catch (IOException ioe) { throw new RepositoryException(metadata.name(), "could not read repository data from index blob", ioe); }