[Backport 2.x] Backports 6558 and 6537 (#6584)

* Implement equals/hashcode for named DocValueFormat inner classes (#6357) Signed-off-by: Bryan Burkholder <[email protected]> (cherry picked from commit 5a7fd4f) Signed-off-by: Kunal Kotwani <[email protected]> * Fix compiler warning and base blob path case (#6558) Signed-off-by: Samuel Herman <[email protected]> Co-authored-by: Samuel Herman <[email protected]> (cherry picked from commit 8c0e5d0) Signed-off-by: Kunal Kotwani <[email protected]> --------- Signed-off-by: Bryan Burkholder <[email protected]> Signed-off-by: Kunal Kotwani <[email protected]> Signed-off-by: Samuel Herman <[email protected]> Co-authored-by: Bryan Burkholder <[email protected]> Co-authored-by: samuel-oci <[email protected]> Co-authored-by: Samuel Herman <[email protected]>
opensearch-project · Mar 8, 2023 · f647c41 · f647c41
1 parent 59f38d2
commit f647c41
Show file tree

Hide file tree

Showing 9 changed files with 44 additions and 4 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -8,6 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add GeoTile and GeoHash Grid aggregations on GeoShapes. ([#5589](https://github.com/opensearch-project/OpenSearch/pull/5589))
 - Disallow multiple data paths for search nodes ([#6427](https://github.com/opensearch-project/OpenSearch/pull/6427))
 - [Segment Replication] Allocation and rebalancing based on average primary shard count per index ([#6422](https://github.com/opensearch-project/OpenSearch/pull/6422))
+- Add 'base_path' setting to File System Repository ([#6558](https://github.com/opensearch-project/OpenSearch/pull/6558))
 
 ### Dependencies
 - Bump `org.apache.logging.log4j:log4j-core` from 2.18.0 to 2.20.0 ([#6490](https://github.com/opensearch-project/OpenSearch/pull/6490))
@@ -25,7 +26,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ### Fixed
 - Added depth check in doc parser for deep nested document ([#5199](https://github.com/opensearch-project/OpenSearch/pull/5199))
+- Added equals/hashcode for named DocValueFormat.DateTime inner class ([#6357](https://github.com/opensearch-project/OpenSearch/pull/6357))
+- Fixed bug for searchable snapshot to take 'base_path' of blob into account ([#6558](https://github.com/opensearch-project/OpenSearch/pull/6558))
 
 ### Security
 
+[Unreleased 3.0]: https://github.com/opensearch-project/OpenSearch/compare/2.x...HEAD
 [Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.5...2.x
diff --git a/...alClusterTest/java/org/opensearch/indices/replication/SegmentReplicationAllocationIT.java b/...alClusterTest/java/org/opensearch/indices/replication/SegmentReplicationAllocationIT.java
@@ -193,7 +193,7 @@ public void testAllocationWithDisruption() throws Exception {
 
     /**
      * Utility method which ensures cluster has balanced primary shard distribution across a single index.
-     * @throws Exception
+     * @throws Exception exception
      */
     private void verifyPerIndexPrimaryBalance() throws Exception {
         assertBusy(() -> {

diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/SearchableSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/SearchableSnapshotIT.java
@@ -72,6 +72,7 @@ protected Settings featureFlagSettings() {
     protected Settings.Builder randomRepositorySettings() {
         final Settings.Builder settings = Settings.builder();
         settings.put("location", randomRepoPath()).put("compress", randomBoolean());
+        settings.put(FsRepository.BASE_PATH_SETTING.getKey(), "my_base_path");
         return settings;
     }
 

diff --git a/...main/java/org/opensearch/index/store/remote/directory/RemoteSnapshotDirectoryFactory.java b/...main/java/org/opensearch/index/store/remote/directory/RemoteSnapshotDirectoryFactory.java
@@ -73,7 +73,8 @@ private Future<RemoteSnapshotDirectory> createRemoteSnapshotDirectoryFromSnapsho
         ShardPath localShardPath,
         BlobStoreRepository blobStoreRepository
     ) throws IOException {
-        final BlobPath blobPath = new BlobPath().add("indices")
+        final BlobPath blobPath = blobStoreRepository.basePath()
+            .add("indices")
             .add(IndexSettings.SEARCHABLE_SNAPSHOT_INDEX_ID.get(indexSettings.getSettings()))
             .add(Integer.toString(localShardPath.getShardId().getId()));
         final SnapshotId snapshotId = new SnapshotId(

diff --git a/server/src/main/java/org/opensearch/repositories/fs/FsRepository.java b/server/src/main/java/org/opensearch/repositories/fs/FsRepository.java
@@ -36,6 +36,7 @@
 import org.apache.logging.log4j.Logger;
 import org.opensearch.cluster.metadata.RepositoryMetadata;
 import org.opensearch.cluster.service.ClusterService;
+import org.opensearch.common.Strings;
 import org.opensearch.common.blobstore.BlobPath;
 import org.opensearch.common.blobstore.BlobStore;
 import org.opensearch.common.blobstore.fs.FsBlobStore;
@@ -98,6 +99,9 @@ public class FsRepository extends BlobStoreRepository {
         Property.NodeScope,
         Property.Deprecated
     );
+
+    public static final Setting<String> BASE_PATH_SETTING = Setting.simpleString("base_path");
+
     private final Environment environment;
 
     private ByteSizeValue chunkSize;
@@ -154,7 +158,12 @@ public FsRepository(
         } else {
             this.chunkSize = REPOSITORIES_CHUNK_SIZE_SETTING.get(environment.settings());
         }
-        this.basePath = BlobPath.cleanPath();
+        final String basePath = BASE_PATH_SETTING.get(metadata.settings());
+        if (Strings.hasLength(basePath)) {
+            this.basePath = new BlobPath().add(basePath);
+        } else {
+            this.basePath = BlobPath.cleanPath();
+        }
     }
 
     private static boolean calculateCompress(RepositoryMetadata metadata, Environment environment) {

diff --git a/server/src/main/java/org/opensearch/search/DocValueFormat.java b/server/src/main/java/org/opensearch/search/DocValueFormat.java
@@ -302,6 +302,27 @@ public double parseDouble(String value, boolean roundUp, LongSupplier now) {
         public String toString() {
             return "DocValueFormat.DateTime(" + formatter + ", " + timeZone + ", " + resolution + ")";
         }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) {
+                return true;
+            }
+            if (o == null || getClass() != o.getClass()) {
+                return false;
+            }
+
+            DateTime that = (DateTime) o;
+
+            return Objects.equals(formatter, that.formatter)
+                && Objects.equals(timeZone, that.timeZone)
+                && Objects.equals(resolution, that.resolution);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(formatter, timeZone, resolution);
+        }
     }
 
     DocValueFormat GEOHASH = new DocValueFormat() {

diff --git a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java
@@ -162,7 +162,7 @@ private Map<String, String> getDummyMetadata(String prefix, int commitGeneration
      * Prepares metadata file bytes with header and footer
      * @param segmentFilesMap: actual metadata content
      * @return ByteArrayIndexInput: metadata file bytes with header and footer
-     * @throws IOException
+     * @throws IOException IOException
      */
     private ByteArrayIndexInput createMetadataFileBytes(Map<String, String> segmentFilesMap) throws IOException {
         BytesStreamOutput output = new BytesStreamOutput();

diff --git a/server/src/test/java/org/opensearch/repositories/fs/FsRepositoryTests.java b/server/src/test/java/org/opensearch/repositories/fs/FsRepositoryTests.java
@@ -104,6 +104,7 @@ public void testSnapshotAndRestore() throws IOException, InterruptedException {
                 .put("location", repo)
                 .put("compress", randomBoolean())
                 .put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES)
+                .put(FsRepository.BASE_PATH_SETTING.getKey(), "my_base_path")
                 .build();
 
             int numDocs = indexDocs(directory);

diff --git a/server/src/test/java/org/opensearch/search/DocValueFormatTests.java b/server/src/test/java/org/opensearch/search/DocValueFormatTests.java
@@ -76,6 +76,7 @@ public void testSerialization() throws Exception {
         DocValueFormat vf = in.readNamedWriteable(DocValueFormat.class);
         assertEquals(DocValueFormat.Decimal.class, vf.getClass());
         assertEquals("###.##", ((DocValueFormat.Decimal) vf).pattern);
+        assertEquals(decimalFormat, vf);
 
         DateFormatter formatter = DateFormatter.forPattern("epoch_second");
         DocValueFormat.DateTime dateFormat = new DocValueFormat.DateTime(formatter, ZoneOffset.ofHours(1), Resolution.MILLISECONDS);
@@ -87,6 +88,7 @@ public void testSerialization() throws Exception {
         assertEquals("epoch_second", ((DocValueFormat.DateTime) vf).formatter.pattern());
         assertEquals(ZoneOffset.ofHours(1), ((DocValueFormat.DateTime) vf).timeZone);
         assertEquals(Resolution.MILLISECONDS, ((DocValueFormat.DateTime) vf).resolution);
+        assertEquals(dateFormat, vf);
 
         DocValueFormat.DateTime nanosDateFormat = new DocValueFormat.DateTime(formatter, ZoneOffset.ofHours(1), Resolution.NANOSECONDS);
         out = new BytesStreamOutput();
@@ -97,6 +99,7 @@ public void testSerialization() throws Exception {
         assertEquals("epoch_second", ((DocValueFormat.DateTime) vf).formatter.pattern());
         assertEquals(ZoneOffset.ofHours(1), ((DocValueFormat.DateTime) vf).timeZone);
         assertEquals(Resolution.NANOSECONDS, ((DocValueFormat.DateTime) vf).resolution);
+        assertEquals(nanosDateFormat, vf);
 
         out = new BytesStreamOutput();
         out.writeNamedWriteable(DocValueFormat.GEOHASH);