[BUG] [Remote Store] Segment upload fails with FileAlreadyExistsException

[linuxpi]

Describe the bug
Local files created during segment upload are not cleaned up in some cases due to errors in upload flow. which leads to failures during next retry

[2023-05-02T08:00:44,571][WARN ][o.o.i.s.RemoteStoreRefreshListener] [node-1] Exception while uploading new segments to the remote segment store
java.nio.file.FileAlreadyExistsException: /home/ec2-user/opensearch-3.0.0-SNAPSHOT/data/nodes/0/indices/MYce-mgETOGu23bJrxMY7w/29/index/segment_infos_snapshot_filename__4
        at sun.nio.fs.UnixException.translateToIOException(UnixException.java:94) ~[?:?]
        at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:106) ~[?:?]
        at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:111) ~[?:?]
        at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:218) ~[?:?]
        at java.nio.file.spi.FileSystemProvider.newOutputStream(FileSystemProvider.java:484) ~[?:?]
        at java.nio.file.Files.newOutputStream(Files.java:228) ~[?:?]
        at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:394) ~[lucene-core-9.6.0-snapshot-a3ae27f.jar:9.6.0-snapshot-a3ae27f a3ae27fd28135a80da36364592a72fb9d54c6cbe - 2023-03-30 19:18:02]
        at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:387) ~[lucene-core-9.6.0-snapshot-a3ae27f.jar:9.6.0-snapshot-a3ae27f a3ae27fd28135a80da36364592a72fb9d54c6cbe - 2023-03-30 19:18:02]
        at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:220) ~[lucene-core-9.6.0-snapshot-a3ae27f.jar:9.6.0-snapshot-a3ae27f a3ae27fd28135a80da36364592a72fb9d54c6cbe - 2023-03-30 19:18:02]
        at org.apache.lucene.store.FilterDirectory.createOutput(FilterDirectory.java:75) ~[lucene-core-9.6.0-snapshot-a3ae27f.jar:9.6.0-snapshot-a3ae27f a3ae27fd28135a80da36364592a72fb9d54c6cbe - 2023-03-30 19:18:02]
        at org.opensearch.index.store.ByteSizeCachingDirectory.createOutput(ByteSizeCachingDirectory.java:153) ~[opensearch-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
        at org.apache.lucene.store.FilterDirectory.createOutput(FilterDirectory.java:75) ~[lucene-core-9.6.0-snapshot-a3ae27f.jar:9.6.0-snapshot-a3ae27f a3ae27fd28135a80da36364592a72fb9d54c6cbe - 2023-03-30 19:18:02]
        at org.opensearch.index.shard.RemoteStoreRefreshListener.uploadSegmentInfosSnapshot(RemoteStoreRefreshListener.java:191) ~[opensearch-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
        at org.opensearch.index.shard.RemoteStoreRefreshListener.afterRefresh(RemoteStoreRefreshListener.java:133) [opensearch-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
        at org.apache.lucene.search.ReferenceManager.notifyRefreshListenersRefreshed(ReferenceManager.java:275) [lucene-core-9.6.0-snapshot-a3ae27f.jar:9.6.0-snapshot-a3ae27f a3ae27fd28135a80da36364592a72fb9d54c6cbe - 2023-03-30 19:18:02]
        at org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:182) [lucene-core-9.6.0-snapshot-a3ae27f.jar:9.6.0-snapshot-a3ae27f a3ae27fd28135a80da36364592a72fb9d54c6cbe - 2023-03-30 19:18:02]
        at org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:240) [lucene-core-9.6.0-snapshot-a3ae27f.jar:9.6.0-snapshot-a3ae27f a3ae27fd28135a80da36364592a72fb9d54c6cbe - 2023-03-30 19:18:02]
        at org.opensearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:432) [opensearch-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
        at org.opensearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:412) [opensearch-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
        at org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:167) [lucene-core-9.6.0-snapshot-a3ae27f.jar:9.6.0-snapshot-a3ae27f a3ae27fd28135a80da36364592a72fb9d54c6cbe - 2023-03-30 19:18:02]
        at org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:213) [lucene-core-9.6.0-snapshot-a3ae27f.jar:9.6.0-snapshot-a3ae27f a3ae27fd28135a80da36364592a72fb9d54c6cbe - 2023-03-30 19:18:02]
        at org.opensearch.index.engine.InternalEngine.refresh(InternalEngine.java:1755) [opensearch-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
        at org.opensearch.index.engine.InternalEngine.maybeRefresh(InternalEngine.java:1734) [opensearch-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
        at org.opensearch.index.shard.IndexShard.scheduledRefresh(IndexShard.java:4163) [opensearch-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
        at org.opensearch.index.IndexService.maybeRefreshEngine(IndexService.java:983) [opensearch-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
        at org.opensearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:1116) [opensearch-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
        at org.opensearch.common.util.concurrent.AbstractAsyncTask.run(AbstractAsyncTask.java:159) [opensearch-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
        at org.opensearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:747) [opensearch-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) [?:?]
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) [?:?]
        at java.lang.Thread.run(Thread.java:1589) [?:?]

To Reproduce
Steps to reproduce the behavior:

1. Go to '...'
2. Click on '....'
3. Scroll down to '....'
4. See error

Expected behavior
A clear and concise description of what you expected to happen.

Plugins
Please list all plugins currently enabled.

Screenshots
If applicable, add screenshots to help explain your problem.

Host/Environment (please complete the following information):

• OS: [e.g. iOS]
• Version [e.g. 22]

Additional context
Add any other context about the problem here.

[sachinpkale]

@linuxpi I have assigned this to myself.

[linuxpi]

We can repro this by simulating an exception in uploadSegmentInfosSnapshot after the following block

try (IndexOutput indexOutput = storeDirectory.createOutput(segmentInfoSnapshotFilename, IOContext.DEFAULT)) {
            segmentInfosSnapshot.write(indexOutput);
}

This would cause error but the file opened at segmentInfoSnapshotFilename wont be deleted.

[linuxpi]

Saw failures in metadata upload flow as well.

[2023-05-09T04:31:21,448][WARN ][o.o.i.s.RemoteStoreRefreshListener] [node-1] Exception while uploading new segments to the remote segment store
java.nio.file.FileAlreadyExistsException: /home/ec2-user/opensearch-3.0.0-SNAPSHOT/data/nodes/0/indices/WirstdfARjqr39JBH7x6_w/36/index/metadata__1__4__SX-R_ocBMrWdwgxOHbFe
        at sun.nio.fs.UnixException.translateToIOException(UnixException.java:94) ~[?:?]
        at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:106) ~[?:?]
        at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:111) ~[?:?]
        at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:218) ~[?:?]
        at java.nio.file.spi.FileSystemProvider.newOutputStream(FileSystemProvider.java:484) ~[?:?]
        at java.nio.file.Files.newOutputStream(Files.java:228) ~[?:?]
        at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:394) ~[lucene-core-9.6.0-snapshot-a3ae27f.jar:9.6.0-snapshot-a3ae27f a3ae27fd28135a80da36364592a72fb9d54c6cbe - 2023-03-30 19:18:02]
        at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:387) ~[lucene-core-9.6.0-snapshot-a3ae27f.jar:9.6.0-snapshot-a3ae27f a3ae27fd28135a80da36364592a72fb9d54c6cbe - 2023-03-30 19:18:02]
        at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:220) ~[lucene-core-9.6.0-snapshot-a3ae27f.jar:9.6.0-snapshot-a3ae27f a3ae27fd28135a80da36364592a72fb9d54c6cbe - 2023-03-30 19:18:02]
        at org.apache.lucene.store.FilterDirectory.createOutput(FilterDirectory.java:75) ~[lucene-core-9.6.0-snapshot-a3ae27f.jar:9.6.0-snapshot-a3ae27f a3ae27fd28135a80da36364592a72fb9d54c6cbe - 2023-03-30 19:18:02]
        at org.opensearch.index.store.ByteSizeCachingDirectory.createOutput(ByteSizeCachingDirectory.java:153) ~[opensearch-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
        at org.apache.lucene.store.FilterDirectory.createOutput(FilterDirectory.java:75) ~[lucene-core-9.6.0-snapshot-a3ae27f.jar:9.6.0-snapshot-a3ae27f a3ae27fd28135a80da36364592a72fb9d54c6cbe - 2023-03-30 19:18:02]
        at org.opensearch.index.store.RemoteSegmentStoreDirectory.uploadMetadata(RemoteSegmentStoreDirectory.java:367) ~[opensearch-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
        at org.opensearch.index.shard.RemoteStoreRefreshListener.afterRefresh(RemoteStoreRefreshListener.java:140) [opensearch-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
        at org.apache.lucene.search.ReferenceManager.notifyRefreshListenersRefreshed(ReferenceManager.java:275) [lucene-core-9.6.0-snapshot-a3ae27f.jar:9.6.0-snapshot-a3ae27f a3ae27fd28135a80da36364592a72fb9d54c6cbe - 2023-03-30 19:18:02]
        at org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:182) [lucene-core-9.6.0-snapshot-a3ae27f.jar:9.6.0-snapshot-a3ae27f a3ae27fd28135a80da36364592a72fb9d54c6cbe - 2023-03-30 19:18:02]
        at org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:240) [lucene-core-9.6.0-snapshot-a3ae27f.jar:9.6.0-snapshot-a3ae27f a3ae27fd28135a80da36364592a72fb9d54c6cbe - 2023-03-30 19:18:02]
        at org.opensearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:432) [opensearch-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
        at org.opensearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:412) [opensearch-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
        at org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:167) [lucene-core-9.6.0-snapshot-a3ae27f.jar:9.6.0-snapshot-a3ae27f a3ae27fd28135a80da36364592a72fb9d54c6cbe - 2023-03-30 19:18:02]
        at org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:213) [lucene-core-9.6.0-snapshot-a3ae27f.jar:9.6.0-snapshot-a3ae27f a3ae27fd28135a80da36364592a72fb9d54c6cbe - 2023-03-30 19:18:02]
        at org.opensearch.index.engine.InternalEngine.refresh(InternalEngine.java:1755) [opensearch-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
        at org.opensearch.index.engine.InternalEngine.maybeRefresh(InternalEngine.java:1734) [opensearch-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
        at org.opensearch.index.shard.IndexShard.scheduledRefresh(IndexShard.java:4170) [opensearch-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
        at org.opensearch.index.IndexService.maybeRefreshEngine(IndexService.java:983) [opensearch-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
        at org.opensearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:1116) [opensearch-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
        at org.opensearch.common.util.concurrent.AbstractAsyncTask.run(AbstractAsyncTask.java:159) [opensearch-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
        at org.opensearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:747) [opensearch-3.0.0-SNAPSHOT.jar:3.0.0-SNAPSHOT]
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) [?:?]
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) [?:?]
        at java.lang.Thread.run(Thread.java:1589) [?:?]