From 0d01d1755e282763e5ea020112617ba25459863c Mon Sep 17 00:00:00 2001 From: bowenlan-amzn Date: Tue, 25 Jun 2024 08:21:33 -0700 Subject: [PATCH 01/11] Fix flaky test in range aggregation yaml test (#14486) Signed-off-by: bowenlan-amzn --- .../rest-api-spec/test/search.aggregation/10_histogram.yml | 5 +++++ .../rest-api-spec/test/search.aggregation/230_composite.yml | 6 ++++++ .../test/search.aggregation/330_auto_date_histogram.yml | 5 +++++ .../rest-api-spec/test/search.aggregation/40_range.yml | 6 ++++++ 4 files changed, 22 insertions(+) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/10_histogram.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/10_histogram.yml index 996c2aae8cfe4..a75b1d0eac793 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/10_histogram.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/10_histogram.yml @@ -678,6 +678,11 @@ setup: - '{"index": {}}' - '{"date": "2016-03-01"}' + - do: + indices.forcemerge: + index: test_2 + max_num_segments: 1 + - do: search: index: test_2 diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/230_composite.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/230_composite.yml index 78e2e6858c6ff..ade9eb3eee0dc 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/230_composite.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/230_composite.yml @@ -1101,6 +1101,12 @@ setup: - '{"date": "2016-02-01"}' - '{"index": {}}' - '{"date": "2016-03-01"}' + + - do: + indices.forcemerge: + index: test_2 + max_num_segments: 1 + - do: search: index: test_2 diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/330_auto_date_histogram.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/330_auto_date_histogram.yml index fc82517788c91..0897e0bdd894b 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/330_auto_date_histogram.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/330_auto_date_histogram.yml @@ -133,6 +133,11 @@ setup: - '{"index": {}}' - '{"date": "2020-03-09", "v": 4}' + - do: + indices.forcemerge: + index: test_profile + max_num_segments: 1 + - do: search: index: test_profile diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/40_range.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/40_range.yml index 2fd926276d0b4..80aad96ce1f6b 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/40_range.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/40_range.yml @@ -544,6 +544,7 @@ setup: body: settings: number_of_replicas: 0 + number_of_shards: 1 refresh_interval: -1 mappings: properties: @@ -567,6 +568,11 @@ setup: - '{"index": {}}' - '{"double" : 50}' + - do: + indices.forcemerge: + index: test_profile + max_num_segments: 1 + - do: search: index: test_profile From aa83733c3ffdf6ef1ff98abd9519001e5aa71509 Mon Sep 17 00:00:00 2001 From: Prudhvi Godithi Date: Tue, 25 Jun 2024 11:29:41 -0700 Subject: [PATCH 02/11] Use CODECOV_TOKEN (#14536) Signed-off-by: Prudhvi Godithi --- .github/workflows/gradle-check.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/gradle-check.yml b/.github/workflows/gradle-check.yml index 2909ee95349ce..89d894403ff1a 100644 --- a/.github/workflows/gradle-check.yml +++ b/.github/workflows/gradle-check.yml @@ -113,6 +113,7 @@ jobs: if: success() uses: codecov/codecov-action@v4 with: + token: ${{ secrets.CODECOV_TOKEN }} files: ./codeCoverage.xml - name: Create Comment Success From 563375de28b16870ab42b9fb4260127598d47d91 Mon Sep 17 00:00:00 2001 From: Sagar <99425694+sgup432@users.noreply.github.com> Date: Tue, 25 Jun 2024 12:04:17 -0700 Subject: [PATCH 03/11] [Tiered Caching] Moving query recomputation logic outside of write lock (#14187) * Moving query recompute out of write lock Signed-off-by: Sagar Upadhyaya * [Tiered Caching] Moving query recomputation logic outside of write lock Signed-off-by: Sagar Upadhyaya * Adding java doc for the completable map Signed-off-by: Sagar Upadhyaya * Changes to call future handler only once per key Signed-off-by: Sagar Upadhyaya * Fixing spotless check Signed-off-by: Sagar Upadhyaya * Added changelog Signed-off-by: Sagar Upadhyaya * Addressing comments Signed-off-by: Sagar Upadhyaya * Fixing gradle fail Signed-off-by: Sagar Upadhyaya * Addressing comments to refactor unit test Signed-off-by: Sagar Upadhyaya * minor UT refactor Signed-off-by: Sagar Upadhyaya --------- Signed-off-by: Sagar Upadhyaya Signed-off-by: Sagar <99425694+sgup432@users.noreply.github.com> Co-authored-by: Sagar Upadhyaya --- CHANGELOG.md | 1 + .../common/tier/TieredSpilloverCache.java | 85 ++++- .../tier/TieredSpilloverCacheTests.java | 339 +++++++++++++++++- 3 files changed, 406 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cafe9c20e7ff4..f71ba46745ef1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `azure-identity` from 1.11.4 to 1.13.0, Bump `msal4j` from 1.14.3 to 1.15.1, Bump `msal4j-persistence-extension` from 1.2.0 to 1.3.0 ([#14506](https://github.com/opensearch-project/OpenSearch/pull/14506)) ### Changed +- [Tiered Caching] Move query recomputation logic outside write lock ([#14187](https://github.com/opensearch-project/OpenSearch/pull/14187)) - unsignedLongRangeQuery now returns MatchNoDocsQuery if the lower bounds are greater than the upper bounds ([#14416](https://github.com/opensearch-project/OpenSearch/pull/14416)) - Updated the `indices.query.bool.max_clause_count` setting from being static to dynamically updateable ([#13568](https://github.com/opensearch-project/OpenSearch/pull/13568)) - Make the class CommunityIdProcessor final ([#14448](https://github.com/opensearch-project/OpenSearch/pull/14448)) diff --git a/modules/cache-common/src/main/java/org/opensearch/cache/common/tier/TieredSpilloverCache.java b/modules/cache-common/src/main/java/org/opensearch/cache/common/tier/TieredSpilloverCache.java index 63cdbca101f2a..b6d6913a9f8d4 100644 --- a/modules/cache-common/src/main/java/org/opensearch/cache/common/tier/TieredSpilloverCache.java +++ b/modules/cache-common/src/main/java/org/opensearch/cache/common/tier/TieredSpilloverCache.java @@ -8,6 +8,8 @@ package org.opensearch.cache.common.tier; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.opensearch.cache.common.policy.TookTimePolicy; import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.common.cache.CacheType; @@ -35,9 +37,13 @@ import java.util.Map; import java.util.NoSuchElementException; import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.function.BiFunction; import java.util.function.Function; import java.util.function.Predicate; import java.util.function.ToLongBiFunction; @@ -61,6 +67,7 @@ public class TieredSpilloverCache implements ICache { // Used to avoid caching stale entries in lower tiers. private static final List SPILLOVER_REMOVAL_REASONS = List.of(RemovalReason.EVICTED, RemovalReason.CAPACITY); + private static final Logger logger = LogManager.getLogger(TieredSpilloverCache.class); private final ICache diskCache; private final ICache onHeapCache; @@ -86,6 +93,12 @@ public class TieredSpilloverCache implements ICache { private final Map, TierInfo> caches; private final List> policies; + /** + * This map is used to handle concurrent requests for same key in computeIfAbsent() to ensure we load the value + * only once. + */ + Map, CompletableFuture, V>>> completableFutureMap = new ConcurrentHashMap<>(); + TieredSpilloverCache(Builder builder) { Objects.requireNonNull(builder.onHeapCacheFactory, "onHeap cache builder can't be null"); Objects.requireNonNull(builder.diskCacheFactory, "disk cache builder can't be null"); @@ -190,10 +203,7 @@ public V computeIfAbsent(ICacheKey key, LoadAwareCacheLoader, V> // Add the value to the onHeap cache. We are calling computeIfAbsent which does another get inside. // This is needed as there can be many requests for the same key at the same time and we only want to load // the value once. - V value = null; - try (ReleasableLock ignore = writeLock.acquire()) { - value = onHeapCache.computeIfAbsent(key, loader); - } + V value = compute(key, loader); // Handle stats if (loader.isLoaded()) { // The value was just computed and added to the cache by this thread. Register a miss for the heap cache, and the disk cache @@ -222,6 +232,57 @@ public V computeIfAbsent(ICacheKey key, LoadAwareCacheLoader, V> return cacheValueTuple.v1(); } + private V compute(ICacheKey key, LoadAwareCacheLoader, V> loader) throws Exception { + // Only one of the threads will succeed putting a future into map for the same key. + // Rest will fetch existing future and wait on that to complete. + CompletableFuture, V>> future = completableFutureMap.putIfAbsent(key, new CompletableFuture<>()); + // Handler to handle results post processing. Takes a tuple or exception as an input and returns + // the value. Also before returning value, puts the value in cache. + BiFunction, V>, Throwable, Void> handler = (pair, ex) -> { + if (pair != null) { + try (ReleasableLock ignore = writeLock.acquire()) { + onHeapCache.put(pair.v1(), pair.v2()); + } catch (Exception e) { + // TODO: Catch specific exceptions to know whether this resulted from cache or underlying removal + // listeners/stats. Needs better exception handling at underlying layers.For now swallowing + // exception. + logger.warn("Exception occurred while putting item onto heap cache", e); + } + } else { + if (ex != null) { + logger.warn("Exception occurred while trying to compute the value", ex); + } + } + completableFutureMap.remove(key); // Remove key from map as not needed anymore. + return null; + }; + V value = null; + if (future == null) { + future = completableFutureMap.get(key); + future.handle(handler); + try { + value = loader.load(key); + } catch (Exception ex) { + future.completeExceptionally(ex); + throw new ExecutionException(ex); + } + if (value == null) { + NullPointerException npe = new NullPointerException("Loader returned a null value"); + future.completeExceptionally(npe); + throw new ExecutionException(npe); + } else { + future.complete(new Tuple<>(key, value)); + } + } else { + try { + value = future.get().v2(); + } catch (InterruptedException ex) { + throw new IllegalStateException(ex); + } + } + return value; + } + @Override public void invalidate(ICacheKey key) { // We are trying to invalidate the key from all caches though it would be present in only of them. @@ -328,12 +389,22 @@ void handleRemovalFromHeapTier(RemovalNotification, V> notification ICacheKey key = notification.getKey(); boolean wasEvicted = SPILLOVER_REMOVAL_REASONS.contains(notification.getRemovalReason()); boolean countEvictionTowardsTotal = false; // Don't count this eviction towards the cache's total if it ends up in the disk tier - if (caches.get(diskCache).isEnabled() && wasEvicted && evaluatePolicies(notification.getValue())) { + boolean exceptionOccurredOnDiskCachePut = false; + boolean canCacheOnDisk = caches.get(diskCache).isEnabled() && wasEvicted && evaluatePolicies(notification.getValue()); + if (canCacheOnDisk) { try (ReleasableLock ignore = writeLock.acquire()) { diskCache.put(key, notification.getValue()); // spill over to the disk tier and increment its stats + } catch (Exception ex) { + // TODO: Catch specific exceptions. Needs better exception handling. We are just swallowing exception + // in this case as it shouldn't cause upstream request to fail. + logger.warn("Exception occurred while putting item to disk cache", ex); + exceptionOccurredOnDiskCachePut = true; } - updateStatsOnPut(TIER_DIMENSION_VALUE_DISK, key, notification.getValue()); - } else { + if (!exceptionOccurredOnDiskCachePut) { + updateStatsOnPut(TIER_DIMENSION_VALUE_DISK, key, notification.getValue()); + } + } + if (!canCacheOnDisk || exceptionOccurredOnDiskCachePut) { // If the value is not going to the disk cache, send this notification to the TSC's removal listener // as the value is leaving the TSC entirely removalListener.onRemoval(notification); diff --git a/modules/cache-common/src/test/java/org/opensearch/cache/common/tier/TieredSpilloverCacheTests.java b/modules/cache-common/src/test/java/org/opensearch/cache/common/tier/TieredSpilloverCacheTests.java index 54b15f236a418..b9c7bbdb77d3d 100644 --- a/modules/cache-common/src/test/java/org/opensearch/cache/common/tier/TieredSpilloverCacheTests.java +++ b/modules/cache-common/src/test/java/org/opensearch/cache/common/tier/TieredSpilloverCacheTests.java @@ -44,8 +44,12 @@ import java.util.UUID; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.concurrent.Phaser; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; import java.util.function.Predicate; @@ -56,6 +60,10 @@ import static org.opensearch.cache.common.tier.TieredSpilloverCacheStatsHolder.TIER_DIMENSION_VALUE_DISK; import static org.opensearch.cache.common.tier.TieredSpilloverCacheStatsHolder.TIER_DIMENSION_VALUE_ON_HEAP; import static org.opensearch.common.cache.store.settings.OpenSearchOnHeapCacheSettings.MAXIMUM_SIZE_IN_BYTES_KEY; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class TieredSpilloverCacheTests extends OpenSearchTestCase { static final List dimensionNames = List.of("dim1", "dim2", "dim3"); @@ -408,6 +416,7 @@ public void testComputeIfAbsentWithEvictionsFromOnHeapCache() throws Exception { assertEquals(onHeapCacheHit, getHitsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP)); assertEquals(cacheMiss + numOfItems1, getMissesForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_DISK)); assertEquals(diskCacheHit, getHitsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_DISK)); + assertEquals(0, tieredSpilloverCache.completableFutureMap.size()); } public void testComputeIfAbsentWithEvictionsFromTieredCache() throws Exception { @@ -802,7 +811,7 @@ public String load(ICacheKey key) { }; loadAwareCacheLoaderList.add(loadAwareCacheLoader); phaser.arriveAndAwaitAdvance(); - tieredSpilloverCache.computeIfAbsent(key, loadAwareCacheLoader); + assertEquals(value, tieredSpilloverCache.computeIfAbsent(key, loadAwareCacheLoader)); } catch (Exception e) { throw new RuntimeException(e); } @@ -811,7 +820,7 @@ public String load(ICacheKey key) { threads[i].start(); } phaser.arriveAndAwaitAdvance(); - countDownLatch.await(); // Wait for rest of tasks to be cancelled. + countDownLatch.await(); int numberOfTimesKeyLoaded = 0; assertEquals(numberOfSameKeys, loadAwareCacheLoaderList.size()); for (int i = 0; i < loadAwareCacheLoaderList.size(); i++) { @@ -824,6 +833,215 @@ public String load(ICacheKey key) { // We should see only one heap miss, and the rest hits assertEquals(1, getMissesForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP)); assertEquals(numberOfSameKeys - 1, getHitsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP)); + assertEquals(0, tieredSpilloverCache.completableFutureMap.size()); + } + + public void testComputIfAbsentConcurrentlyWithMultipleKeys() throws Exception { + int onHeapCacheSize = randomIntBetween(300, 500); + int diskCacheSize = randomIntBetween(600, 700); + int keyValueSize = 50; + + MockCacheRemovalListener removalListener = new MockCacheRemovalListener<>(); + Settings settings = Settings.builder() + .put( + OpenSearchOnHeapCacheSettings.getSettingListForCacheType(CacheType.INDICES_REQUEST_CACHE) + .get(MAXIMUM_SIZE_IN_BYTES_KEY) + .getKey(), + onHeapCacheSize * keyValueSize + "b" + ) + .build(); + + TieredSpilloverCache tieredSpilloverCache = initializeTieredSpilloverCache( + keyValueSize, + diskCacheSize, + removalListener, + settings, + 0 + ); + + int iterations = 10; + int numberOfKeys = 20; + List> iCacheKeyList = new ArrayList<>(); + for (int i = 0; i < numberOfKeys; i++) { + ICacheKey key = getICacheKey(UUID.randomUUID().toString()); + iCacheKeyList.add(key); + } + ExecutorService executorService = Executors.newFixedThreadPool(8); + CountDownLatch countDownLatch = new CountDownLatch(iterations * numberOfKeys); // To wait for all threads to finish. + + List, String>> loadAwareCacheLoaderList = new CopyOnWriteArrayList<>(); + for (int j = 0; j < numberOfKeys; j++) { + int finalJ = j; + for (int i = 0; i < iterations; i++) { + executorService.submit(() -> { + try { + LoadAwareCacheLoader, String> loadAwareCacheLoader = new LoadAwareCacheLoader<>() { + boolean isLoaded = false; + + @Override + public boolean isLoaded() { + return isLoaded; + } + + @Override + public String load(ICacheKey key) { + isLoaded = true; + return iCacheKeyList.get(finalJ).key; + } + }; + loadAwareCacheLoaderList.add(loadAwareCacheLoader); + tieredSpilloverCache.computeIfAbsent(iCacheKeyList.get(finalJ), loadAwareCacheLoader); + } catch (Exception e) { + throw new RuntimeException(e); + } finally { + countDownLatch.countDown(); + } + }); + } + } + countDownLatch.await(); + int numberOfTimesKeyLoaded = 0; + assertEquals(iterations * numberOfKeys, loadAwareCacheLoaderList.size()); + for (int i = 0; i < loadAwareCacheLoaderList.size(); i++) { + LoadAwareCacheLoader, String> loader = loadAwareCacheLoaderList.get(i); + if (loader.isLoaded()) { + numberOfTimesKeyLoaded++; + } + } + assertEquals(numberOfKeys, numberOfTimesKeyLoaded); // It should be loaded only once. + // We should see only one heap miss, and the rest hits + assertEquals(numberOfKeys, getMissesForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP)); + assertEquals((iterations * numberOfKeys) - numberOfKeys, getHitsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP)); + assertEquals(0, tieredSpilloverCache.completableFutureMap.size()); + executorService.shutdownNow(); + } + + public void testComputeIfAbsentConcurrentlyAndThrowsException() throws Exception { + LoadAwareCacheLoader, String> loadAwareCacheLoader = new LoadAwareCacheLoader<>() { + boolean isLoaded = false; + + @Override + public boolean isLoaded() { + return isLoaded; + } + + @Override + public String load(ICacheKey key) { + throw new RuntimeException("Testing"); + } + }; + verifyComputeIfAbsentThrowsException(RuntimeException.class, loadAwareCacheLoader, "Testing"); + } + + @SuppressWarnings({ "rawtypes", "unchecked" }) + public void testComputeIfAbsentWithOnHeapCacheThrowingExceptionOnPut() throws Exception { + int onHeapCacheSize = randomIntBetween(100, 300); + int diskCacheSize = randomIntBetween(200, 400); + int keyValueSize = 50; + + MockCacheRemovalListener removalListener = new MockCacheRemovalListener<>(); + Settings settings = Settings.builder() + .put( + OpenSearchOnHeapCacheSettings.getSettingListForCacheType(CacheType.INDICES_REQUEST_CACHE) + .get(MAXIMUM_SIZE_IN_BYTES_KEY) + .getKey(), + onHeapCacheSize * keyValueSize + "b" + ) + .build(); + ICache.Factory onHeapCacheFactory = mock(OpenSearchOnHeapCache.OpenSearchOnHeapCacheFactory.class); + ICache mockOnHeapCache = mock(ICache.class); + when(onHeapCacheFactory.create(any(), any(), any())).thenReturn(mockOnHeapCache); + doThrow(new RuntimeException("Testing")).when(mockOnHeapCache).put(any(), any()); + CacheConfig cacheConfig = getCacheConfig(keyValueSize, settings, removalListener); + ICache.Factory mockDiskCacheFactory = new MockDiskCache.MockDiskCacheFactory(0, diskCacheSize, false); + + TieredSpilloverCache tieredSpilloverCache = getTieredSpilloverCache( + onHeapCacheFactory, + mockDiskCacheFactory, + cacheConfig, + null, + removalListener + ); + String value = ""; + value = tieredSpilloverCache.computeIfAbsent(getICacheKey("test"), new LoadAwareCacheLoader<>() { + @Override + public boolean isLoaded() { + return false; + } + + @Override + public String load(ICacheKey key) { + return "test"; + } + }); + assertEquals("test", value); + assertEquals(0, tieredSpilloverCache.completableFutureMap.size()); + } + + @SuppressWarnings({ "rawtypes", "unchecked" }) + public void testComputeIfAbsentWithDiskCacheThrowingExceptionOnPut() throws Exception { + int onHeapCacheSize = 0; + int keyValueSize = 50; + + MockCacheRemovalListener removalListener = new MockCacheRemovalListener<>(); + Settings settings = Settings.builder() + .put( + OpenSearchOnHeapCacheSettings.getSettingListForCacheType(CacheType.INDICES_REQUEST_CACHE) + .get(MAXIMUM_SIZE_IN_BYTES_KEY) + .getKey(), + onHeapCacheSize * keyValueSize + "b" + ) + .build(); + ICache.Factory onHeapCacheFactory = new OpenSearchOnHeapCache.OpenSearchOnHeapCacheFactory(); + CacheConfig cacheConfig = getCacheConfig(keyValueSize, settings, removalListener); + ICache.Factory mockDiskCacheFactory = mock(MockDiskCache.MockDiskCacheFactory.class); + ICache mockDiskCache = mock(ICache.class); + when(mockDiskCacheFactory.create(any(), any(), any())).thenReturn(mockDiskCache); + doThrow(new RuntimeException("Test")).when(mockDiskCache).put(any(), any()); + + TieredSpilloverCache tieredSpilloverCache = getTieredSpilloverCache( + onHeapCacheFactory, + mockDiskCacheFactory, + cacheConfig, + null, + removalListener + ); + + String response = ""; + response = tieredSpilloverCache.computeIfAbsent(getICacheKey("test"), new LoadAwareCacheLoader<>() { + @Override + public boolean isLoaded() { + return false; + } + + @Override + public String load(ICacheKey key) { + return "test"; + } + }); + ImmutableCacheStats diskStats = getStatsSnapshotForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_DISK); + + assertEquals(0, diskStats.getSizeInBytes()); + assertEquals(1, removalListener.evictionsMetric.count()); + assertEquals("test", response); + assertEquals(0, tieredSpilloverCache.completableFutureMap.size()); + } + + public void testComputeIfAbsentConcurrentlyWithLoaderReturningNull() throws Exception { + LoadAwareCacheLoader, String> loadAwareCacheLoader = new LoadAwareCacheLoader<>() { + boolean isLoaded = false; + + @Override + public boolean isLoaded() { + return isLoaded; + } + + @Override + public String load(ICacheKey key) { + return null; + } + }; + verifyComputeIfAbsentThrowsException(NullPointerException.class, loadAwareCacheLoader, "Loader returned a null value"); } public void testConcurrencyForEvictionFlowFromOnHeapToDiskTier() throws Exception { @@ -1408,6 +1626,26 @@ public boolean isLoaded() { }; } + private TieredSpilloverCache getTieredSpilloverCache( + ICache.Factory onHeapCacheFactory, + ICache.Factory mockDiskCacheFactory, + CacheConfig cacheConfig, + List> policies, + RemovalListener, String> removalListener + ) { + TieredSpilloverCache.Builder builder = new TieredSpilloverCache.Builder().setCacheType( + CacheType.INDICES_REQUEST_CACHE + ) + .setRemovalListener(removalListener) + .setOnHeapCacheFactory(onHeapCacheFactory) + .setDiskCacheFactory(mockDiskCacheFactory) + .setCacheConfig(cacheConfig); + if (policies != null) { + builder.addPolicies(policies); + } + return builder.build(); + } + private TieredSpilloverCache initializeTieredSpilloverCache( int keyValueSize, int diskCacheSize, @@ -1450,17 +1688,34 @@ private TieredSpilloverCache intializeTieredSpilloverCache( .build(); ICache.Factory mockDiskCacheFactory = new MockDiskCache.MockDiskCacheFactory(diskDeliberateDelay, diskCacheSize, false); - TieredSpilloverCache.Builder builder = new TieredSpilloverCache.Builder().setCacheType( - CacheType.INDICES_REQUEST_CACHE - ) + return getTieredSpilloverCache(onHeapCacheFactory, mockDiskCacheFactory, cacheConfig, policies, removalListener); + } + + private CacheConfig getCacheConfig( + int keyValueSize, + Settings settings, + RemovalListener, String> removalListener + ) { + return new CacheConfig.Builder().setKeyType(String.class) + .setKeyType(String.class) + .setWeigher((k, v) -> keyValueSize) + .setSettings(settings) + .setDimensionNames(dimensionNames) .setRemovalListener(removalListener) - .setOnHeapCacheFactory(onHeapCacheFactory) - .setDiskCacheFactory(mockDiskCacheFactory) - .setCacheConfig(cacheConfig); - if (policies != null) { - builder.addPolicies(policies); - } - return builder.build(); + .setKeySerializer(new StringSerializer()) + .setValueSerializer(new StringSerializer()) + .setSettings( + Settings.builder() + .put( + CacheSettings.getConcreteStoreNameSettingForCacheType(CacheType.INDICES_REQUEST_CACHE).getKey(), + TieredSpilloverCache.TieredSpilloverCacheFactory.TIERED_SPILLOVER_CACHE_NAME + ) + .put(FeatureFlags.PLUGGABLE_CACHE, "true") + .put(settings) + .build() + ) + .setClusterSettings(clusterSettings) + .build(); } // Helper functions for extracting tier aggregated stats. @@ -1501,6 +1756,66 @@ private ImmutableCacheStats getStatsSnapshotForTier(TieredSpilloverCache t return snapshot; } + private void verifyComputeIfAbsentThrowsException( + Class expectedException, + LoadAwareCacheLoader, String> loader, + String expectedExceptionMessage + ) throws InterruptedException { + int onHeapCacheSize = randomIntBetween(100, 300); + int diskCacheSize = randomIntBetween(200, 400); + int keyValueSize = 50; + + MockCacheRemovalListener removalListener = new MockCacheRemovalListener<>(); + Settings settings = Settings.builder() + .put( + OpenSearchOnHeapCacheSettings.getSettingListForCacheType(CacheType.INDICES_REQUEST_CACHE) + .get(MAXIMUM_SIZE_IN_BYTES_KEY) + .getKey(), + onHeapCacheSize * keyValueSize + "b" + ) + .build(); + + TieredSpilloverCache tieredSpilloverCache = initializeTieredSpilloverCache( + keyValueSize, + diskCacheSize, + removalListener, + settings, + 0 + ); + + int numberOfSameKeys = randomIntBetween(10, onHeapCacheSize - 1); + ICacheKey key = getICacheKey(UUID.randomUUID().toString()); + String value = UUID.randomUUID().toString(); + AtomicInteger exceptionCount = new AtomicInteger(); + + Thread[] threads = new Thread[numberOfSameKeys]; + Phaser phaser = new Phaser(numberOfSameKeys + 1); + CountDownLatch countDownLatch = new CountDownLatch(numberOfSameKeys); // To wait for all threads to finish. + + for (int i = 0; i < numberOfSameKeys; i++) { + threads[i] = new Thread(() -> { + try { + phaser.arriveAndAwaitAdvance(); + tieredSpilloverCache.computeIfAbsent(key, loader); + } catch (Exception e) { + exceptionCount.incrementAndGet(); + assertEquals(ExecutionException.class, e.getClass()); + assertEquals(expectedException, e.getCause().getClass()); + assertEquals(expectedExceptionMessage, e.getCause().getMessage()); + } finally { + countDownLatch.countDown(); + } + }); + threads[i].start(); + } + phaser.arriveAndAwaitAdvance(); + countDownLatch.await(); // Wait for rest of tasks to be cancelled. + + // Verify exception count was equal to number of requests + assertEquals(numberOfSameKeys, exceptionCount.get()); + assertEquals(0, tieredSpilloverCache.completableFutureMap.size()); + } + private ImmutableCacheStats getTotalStatsSnapshot(TieredSpilloverCache tsc) throws IOException { ImmutableCacheStatsHolder cacheStats = tsc.stats(new String[0]); return cacheStats.getStatsForDimensionValues(List.of()); From badf851c4883c8ecd4bf47fbad3ca0d3a608f613 Mon Sep 17 00:00:00 2001 From: kkewwei Date: Wed, 26 Jun 2024 03:12:43 +0800 Subject: [PATCH 04/11] Fix Flaky Test ClusterRerouteIT.testDelayWithALargeAmountOfShards (#14510) Signed-off-by: kkewwei kkewwei@163.com Signed-off-by: kkewwei kkewwei@163.com Signed-off-by: kkewwei --- .../cluster/allocation/ClusterRerouteIT.java | 3 ++- .../opensearch/test/OpenSearchIntegTestCase.java | 16 +++++++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/cluster/allocation/ClusterRerouteIT.java b/server/src/internalClusterTest/java/org/opensearch/cluster/allocation/ClusterRerouteIT.java index dbcb030d8a4f7..f4b5f112f5785 100644 --- a/server/src/internalClusterTest/java/org/opensearch/cluster/allocation/ClusterRerouteIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/cluster/allocation/ClusterRerouteIT.java @@ -273,7 +273,8 @@ public void testDelayWithALargeAmountOfShards() throws Exception { internalCluster().stopRandomNode(InternalTestCluster.nameFilter(node_1)); // This might run slowly on older hardware - ensureGreen(TimeValue.timeValueMinutes(2)); + // In some case, the shards will be rebalanced back and forth, it seems like a very low probability bug. + ensureGreen(TimeValue.timeValueMinutes(2), false); } private void rerouteWithAllocateLocalGateway(Settings commonSettings) throws Exception { diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java index 71ab56c98312a..ca5ddf21710af 100644 --- a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java @@ -864,6 +864,10 @@ public ClusterHealthStatus ensureGreen(TimeValue timeout, String... indices) { return ensureColor(ClusterHealthStatus.GREEN, timeout, false, indices); } + public ClusterHealthStatus ensureGreen(TimeValue timeout, boolean waitForNoRelocatingShards, String... indices) { + return ensureColor(ClusterHealthStatus.GREEN, timeout, waitForNoRelocatingShards, false, indices); + } + /** * Ensures the cluster has a yellow state via the cluster health API. */ @@ -891,6 +895,16 @@ private ClusterHealthStatus ensureColor( TimeValue timeout, boolean waitForNoInitializingShards, String... indices + ) { + return ensureColor(clusterHealthStatus, timeout, true, waitForNoInitializingShards, indices); + } + + private ClusterHealthStatus ensureColor( + ClusterHealthStatus clusterHealthStatus, + TimeValue timeout, + boolean waitForNoRelocatingShards, + boolean waitForNoInitializingShards, + String... indices ) { String color = clusterHealthStatus.name().toLowerCase(Locale.ROOT); String method = "ensure" + Strings.capitalize(color); @@ -899,7 +913,7 @@ private ClusterHealthStatus ensureColor( .timeout(timeout) .waitForStatus(clusterHealthStatus) .waitForEvents(Priority.LANGUID) - .waitForNoRelocatingShards(true) + .waitForNoRelocatingShards(waitForNoRelocatingShards) .waitForNoInitializingShards(waitForNoInitializingShards) // We currently often use ensureGreen or ensureYellow to check whether the cluster is back in a good state after shutting down // a node. If the node that is stopped is the cluster-manager node, another node will become cluster-manager and publish a From d320f36ca01ff68f344b54a91a75ef3390824eb7 Mon Sep 17 00:00:00 2001 From: bowenlan-amzn Date: Tue, 25 Jun 2024 12:35:35 -0700 Subject: [PATCH 05/11] Add doc for debugging rest tests (#14491) * add doc for debugging rest tests Signed-off-by: bowenlan-amzn * Update TESTING.md Co-authored-by: Marc Handalian Signed-off-by: bowenlan-amzn * Address comment Signed-off-by: bowenlan-amzn --------- Signed-off-by: bowenlan-amzn Co-authored-by: Marc Handalian --- TESTING.md | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/TESTING.md b/TESTING.md index e8416f61be7e1..de7ab3eefe2f8 100644 --- a/TESTING.md +++ b/TESTING.md @@ -17,6 +17,8 @@ OpenSearch uses [jUnit](https://junit.org/junit5/) for testing, it also uses ran - [Miscellaneous](#miscellaneous) - [Running verification tasks](#running-verification-tasks) - [Testing the REST layer](#testing-the-rest-layer) + - [Running REST Tests Against An External Cluster](#running-rest-tests-against-an-external-cluster) + - [Debugging REST Tests](#debugging-rest-tests) - [Testing packaging](#testing-packaging) - [Testing packaging on Windows](#testing-packaging-on-windows) - [Testing VMs are disposable](#testing-vms-are-disposable) @@ -272,7 +274,18 @@ yamlRestTest’s and javaRestTest’s are easy to identify, since they are found If in doubt about which command to use, simply run <gradle path>:check -Note that the REST tests, like all the integration tests, can be run against an external cluster by specifying the `tests.cluster` property, which if present needs to contain a comma separated list of nodes to connect to (e.g. localhost:9300). +## Running REST Tests Against An External Cluster + +Note that the REST tests, like all the integration tests, can be run against an external cluster by specifying the following properties `tests.cluster`, `tests.rest.cluster`, `tests.clustername`. Use a comma separated list of node properties for the multi-node cluster. + +For example : + + ./gradlew :rest-api-spec:yamlRestTest \ + -Dtests.cluster=localhost:9200 -Dtests.rest.cluster=localhost:9200 -Dtests.clustername=opensearch + +## Debugging REST Tests + +You can launch a local OpenSearch cluster in debug mode following [Launching and debugging from an IDE](#launching-and-debugging-from-an-ide), and run your REST tests against that following [Running REST Tests Against An External Cluster](#running-rest-tests-against-an-external-cluster). # Testing packaging From 0eb39aec6796d9a576e51b186b1cb2474f16f70a Mon Sep 17 00:00:00 2001 From: Peter Alfonsi Date: Tue, 25 Jun 2024 13:26:54 -0700 Subject: [PATCH 06/11] Fix flaky DefaultCacheStatsHolderTests (#14462) Signed-off-by: Peter Alfonsi Co-authored-by: Peter Alfonsi --- .../stats/DefaultCacheStatsHolderTests.java | 75 +++++++++++-------- 1 file changed, 42 insertions(+), 33 deletions(-) diff --git a/server/src/test/java/org/opensearch/common/cache/stats/DefaultCacheStatsHolderTests.java b/server/src/test/java/org/opensearch/common/cache/stats/DefaultCacheStatsHolderTests.java index c6e8252ddf806..8a59dd9d2d105 100644 --- a/server/src/test/java/org/opensearch/common/cache/stats/DefaultCacheStatsHolderTests.java +++ b/server/src/test/java/org/opensearch/common/cache/stats/DefaultCacheStatsHolderTests.java @@ -127,49 +127,58 @@ public void testCount() throws Exception { } public void testConcurrentRemoval() throws Exception { - List dimensionNames = List.of("dim1", "dim2"); + List dimensionNames = List.of("A", "B"); DefaultCacheStatsHolder cacheStatsHolder = new DefaultCacheStatsHolder(dimensionNames, storeName); // Create stats for the following dimension sets - List> populatedStats = List.of(List.of("A1", "B1"), List.of("A2", "B2"), List.of("A2", "B3")); + List> populatedStats = new ArrayList<>(); + int numAValues = 10; + int numBValues = 2; + for (int indexA = 0; indexA < numAValues; indexA++) { + for (int indexB = 0; indexB < numBValues; indexB++) { + populatedStats.add(List.of("A" + indexA, "B" + indexB)); + } + } for (List dims : populatedStats) { cacheStatsHolder.incrementHits(dims); } - // Remove (A2, B2) and (A1, B1), before re-adding (A2, B2). At the end we should have stats for (A2, B2) but not (A1, B1). - - Thread[] threads = new Thread[3]; - CountDownLatch countDownLatch = new CountDownLatch(3); - threads[0] = new Thread(() -> { - cacheStatsHolder.removeDimensions(List.of("A2", "B2")); - countDownLatch.countDown(); - }); - threads[1] = new Thread(() -> { - cacheStatsHolder.removeDimensions(List.of("A1", "B1")); - countDownLatch.countDown(); - }); - threads[2] = new Thread(() -> { - cacheStatsHolder.incrementMisses(List.of("A2", "B2")); - cacheStatsHolder.incrementMisses(List.of("A2", "B3")); - countDownLatch.countDown(); - }); + // Remove a subset of the dimensions concurrently. + // Remove both (A0, B0), and (A0, B1), so we expect the intermediate node for A0 to be null afterwards. + // For all the others, remove only the B0 value. Then we expect the intermediate nodes for A1 through A9 to be present + // and reflect only the stats for their B1 child. + + Thread[] threads = new Thread[numAValues + 1]; + for (int i = 0; i < numAValues; i++) { + int finalI = i; + threads[i] = new Thread(() -> { cacheStatsHolder.removeDimensions(List.of("A" + finalI, "B0")); }); + } + threads[numAValues] = new Thread(() -> { cacheStatsHolder.removeDimensions(List.of("A0", "B1")); }); for (Thread thread : threads) { thread.start(); - // Add short sleep to ensure threads start their functions in order (so that incrementing doesn't happen before removal) - Thread.sleep(1); } - countDownLatch.await(); - assertNull(getNode(List.of("A1", "B1"), cacheStatsHolder.getStatsRoot())); - assertNull(getNode(List.of("A1"), cacheStatsHolder.getStatsRoot())); - assertNotNull(getNode(List.of("A2", "B2"), cacheStatsHolder.getStatsRoot())); - assertEquals( - new ImmutableCacheStats(0, 1, 0, 0, 0), - getNode(List.of("A2", "B2"), cacheStatsHolder.getStatsRoot()).getImmutableStats() - ); - assertEquals( - new ImmutableCacheStats(1, 1, 0, 0, 0), - getNode(List.of("A2", "B3"), cacheStatsHolder.getStatsRoot()).getImmutableStats() - ); + for (Thread thread : threads) { + thread.join(); + } + + // intermediate node for A0 should be null + assertNull(getNode(List.of("A0"), cacheStatsHolder.getStatsRoot())); + + // leaf nodes for all B0 values should be null since they were removed + for (int indexA = 0; indexA < numAValues; indexA++) { + assertNull(getNode(List.of("A" + indexA, "B0"), cacheStatsHolder.getStatsRoot())); + } + + // leaf nodes for all B1 values, except (A0, B1), should not be null as they weren't removed, + // and the intermediate nodes A1 through A9 shouldn't be null as they have remaining children + for (int indexA = 1; indexA < numAValues; indexA++) { + DefaultCacheStatsHolder.Node b1LeafNode = getNode(List.of("A" + indexA, "B1"), cacheStatsHolder.getStatsRoot()); + assertNotNull(b1LeafNode); + assertEquals(new ImmutableCacheStats(1, 0, 0, 0, 0), b1LeafNode.getImmutableStats()); + DefaultCacheStatsHolder.Node intermediateLevelNode = getNode(List.of("A" + indexA), cacheStatsHolder.getStatsRoot()); + assertNotNull(intermediateLevelNode); + assertEquals(b1LeafNode.getImmutableStats(), intermediateLevelNode.getImmutableStats()); + } } /** From 8edfffb1064d61c4badad6de22bff66ea1df88cd Mon Sep 17 00:00:00 2001 From: Bharathwaj G Date: Tue, 25 Jun 2024 16:39:53 +0530 Subject: [PATCH 07/11] Startreemapping (#24) * Star tree mapping changes with feature flag Signed-off-by: Bharathwaj G * Fixes and tests Signed-off-by: Bharathwaj G * addressing review comments and adding validations, integ tests Signed-off-by: Bharathwaj G * addressing review comments Signed-off-by: Bharathwaj G --------- Signed-off-by: Bharathwaj G --- distribution/src/config/opensearch.yml | 4 + .../metadata/MetadataCreateIndexService.java | 5 + .../metadata/MetadataMappingService.java | 12 +- .../common/settings/ClusterSettings.java | 6 +- .../common/settings/FeatureFlagSettings.java | 3 +- .../common/settings/IndexScopedSettings.java | 10 + .../opensearch/common/util/FeatureFlags.java | 10 +- .../org/opensearch/index/IndexModule.java | 10 +- .../org/opensearch/index/IndexService.java | 11 +- .../CompositeIndexSettings.java | 55 ++ .../CompositeIndexValidator.java | 120 +++++ .../datacube/DateDimension.java | 99 ++++ .../compositeindex/datacube/Dimension.java | 58 +++ .../index/compositeindex/datacube/Metric.java | 65 +++ .../compositeindex/datacube/MetricStat.java | 44 ++ .../compositeindex/datacube/package-info.java | 11 + .../datacube/startree/StarTreeField.java | 94 ++++ .../startree/StarTreeFieldConfiguration.java | 108 ++++ .../startree/StarTreeIndexSettings.java | 114 +++++ .../datacube/startree/package-info.java | 11 + .../index/compositeindex/package-info.java | 13 + .../mapper/CompositeDataCubeFieldType.java | 54 ++ .../mapper/CompositeMappedFieldType.java | 75 +++ .../org/opensearch/index/mapper/Mapper.java | 5 + .../index/mapper/MapperService.java | 17 + .../opensearch/index/mapper/ObjectMapper.java | 106 +++- .../index/mapper/RootObjectMapper.java | 15 +- .../index/mapper/StarTreeMapper.java | 430 ++++++++++++++++ .../org/opensearch/indices/IndicesModule.java | 2 + .../opensearch/indices/IndicesService.java | 17 +- .../main/java/org/opensearch/node/Node.java | 5 +- .../startree/StarTreeMappingIntegTests.java | 364 ++++++++++++++ .../index/mapper/StarTreeMapperTests.java | 469 ++++++++++++++++++ .../index/mapper/MapperTestCase.java | 2 +- .../aggregations/AggregatorTestCase.java | 2 + 35 files changed, 2411 insertions(+), 15 deletions(-) create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexSettings.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexValidator.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/DateDimension.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/Dimension.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/Metric.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/MetricStat.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/package-info.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeField.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeFieldConfiguration.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeIndexSettings.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/package-info.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/package-info.java create mode 100644 server/src/main/java/org/opensearch/index/mapper/CompositeDataCubeFieldType.java create mode 100644 server/src/main/java/org/opensearch/index/mapper/CompositeMappedFieldType.java create mode 100644 server/src/main/java/org/opensearch/index/mapper/StarTreeMapper.java create mode 100644 server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeMappingIntegTests.java create mode 100644 server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java diff --git a/distribution/src/config/opensearch.yml b/distribution/src/config/opensearch.yml index 10bab9b3fce92..4115601f62ada 100644 --- a/distribution/src/config/opensearch.yml +++ b/distribution/src/config/opensearch.yml @@ -125,3 +125,7 @@ ${path.logs} # Gates the functionality of enabling Opensearch to use pluggable caches with respective store names via setting. # #opensearch.experimental.feature.pluggable.caching.enabled: false +# +# Gates the functionality of star tree index, which improves the performance of search aggregations. +# +#opensearch.experimental.feature.composite_index.star_tree.enabled: true diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java index 16edec112f123..7973745ce84b3 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java @@ -85,6 +85,7 @@ import org.opensearch.index.IndexNotFoundException; import org.opensearch.index.IndexService; import org.opensearch.index.IndexSettings; +import org.opensearch.index.compositeindex.CompositeIndexValidator; import org.opensearch.index.mapper.DocumentMapper; import org.opensearch.index.mapper.MapperService; import org.opensearch.index.mapper.MapperService.MergeReason; @@ -1318,6 +1319,10 @@ private static void updateIndexMappingsAndBuildSortOrder( } } + if (mapperService.isCompositeIndexPresent()) { + CompositeIndexValidator.validate(mapperService, indexService.getCompositeIndexSettings(), indexService.getIndexSettings()); + } + if (sourceMetadata == null) { // now that the mapping is merged we can validate the index sort. // we cannot validate for index shrinking since the mapping is empty diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataMappingService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataMappingService.java index 1406287149e8d..43894db86c512 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataMappingService.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataMappingService.java @@ -55,6 +55,7 @@ import org.opensearch.core.common.Strings; import org.opensearch.core.index.Index; import org.opensearch.index.IndexService; +import org.opensearch.index.compositeindex.CompositeIndexValidator; import org.opensearch.index.mapper.DocumentMapper; import org.opensearch.index.mapper.MapperService; import org.opensearch.index.mapper.MapperService.MergeReason; @@ -282,6 +283,7 @@ private ClusterState applyRequest( // first, simulate: just call merge and ignore the result existingMapper.merge(newMapper.mapping(), MergeReason.MAPPING_UPDATE); } + } Metadata.Builder builder = Metadata.builder(metadata); boolean updated = false; @@ -291,7 +293,7 @@ private ClusterState applyRequest( // we use the exact same indexService and metadata we used to validate above here to actually apply the update final Index index = indexMetadata.getIndex(); final MapperService mapperService = indexMapperServices.get(index); - + boolean isCompositeFieldPresent = !mapperService.getCompositeFieldTypes().isEmpty(); CompressedXContent existingSource = null; DocumentMapper existingMapper = mapperService.documentMapper(); if (existingMapper != null) { @@ -302,6 +304,14 @@ private ClusterState applyRequest( mappingUpdateSource, MergeReason.MAPPING_UPDATE ); + + CompositeIndexValidator.validate( + mapperService, + indicesService.getCompositeIndexSettings(), + mapperService.getIndexSettings(), + isCompositeFieldPresent + ); + CompressedXContent updatedSource = mergedMapper.mappingSource(); if (existingSource != null) { diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index 233a8d732d178..5dcf23ae52294 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -115,6 +115,7 @@ import org.opensearch.index.ShardIndexingPressureMemoryManager; import org.opensearch.index.ShardIndexingPressureSettings; import org.opensearch.index.ShardIndexingPressureStore; +import org.opensearch.index.compositeindex.CompositeIndexSettings; import org.opensearch.index.remote.RemoteStorePressureSettings; import org.opensearch.index.remote.RemoteStoreStatsTrackerFactory; import org.opensearch.index.store.remote.filecache.FileCacheSettings; @@ -754,7 +755,10 @@ public void apply(Settings value, Settings current, Settings previous) { RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_HASH_ALGORITHM_SETTING, RemoteStoreSettings.CLUSTER_REMOTE_MAX_TRANSLOG_READERS, RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_METADATA, - SearchService.CLUSTER_ALLOW_DERIVED_FIELD_SETTING + SearchService.CLUSTER_ALLOW_DERIVED_FIELD_SETTING, + + // Composite index settings + CompositeIndexSettings.STAR_TREE_INDEX_ENABLED_SETTING ) ) ); diff --git a/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java b/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java index 238df1bd90113..b6166f5d3cce1 100644 --- a/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java @@ -37,6 +37,7 @@ protected FeatureFlagSettings( FeatureFlags.TIERED_REMOTE_INDEX_SETTING, FeatureFlags.REMOTE_STORE_MIGRATION_EXPERIMENTAL_SETTING, FeatureFlags.PLUGGABLE_CACHE_SETTING, - FeatureFlags.REMOTE_PUBLICATION_EXPERIMENTAL_SETTING + FeatureFlags.REMOTE_PUBLICATION_EXPERIMENTAL_SETTING, + FeatureFlags.STAR_TREE_INDEX_SETTING ); } diff --git a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java index 1488f5d30b4ba..ca2c4dab6102b 100644 --- a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java @@ -52,6 +52,7 @@ import org.opensearch.index.SearchSlowLog; import org.opensearch.index.TieredMergePolicyProvider; import org.opensearch.index.cache.bitset.BitsetFilterCache; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeIndexSettings; import org.opensearch.index.engine.EngineConfig; import org.opensearch.index.fielddata.IndexFieldDataService; import org.opensearch.index.mapper.FieldMapper; @@ -239,6 +240,15 @@ public final class IndexScopedSettings extends AbstractScopedSettings { // Settings for concurrent segment search IndexSettings.INDEX_CONCURRENT_SEGMENT_SEARCH_SETTING, IndexSettings.ALLOW_DERIVED_FIELDS, + + // Settings for star tree index + StarTreeIndexSettings.STAR_TREE_DEFAULT_MAX_LEAF_DOCS, + StarTreeIndexSettings.STAR_TREE_MAX_DIMENSIONS_SETTING, + StarTreeIndexSettings.STAR_TREE_MAX_FIELDS_SETTING, + StarTreeIndexSettings.DEFAULT_METRICS_LIST, + StarTreeIndexSettings.DEFAULT_DATE_INTERVALS, + StarTreeIndexSettings.STAR_TREE_MAX_DATE_INTERVALS_SETTING, + // validate that built-in similarities don't get redefined Setting.groupSetting("index.similarity.", (s) -> { Map groups = s.getAsGroups(); diff --git a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java index 6c6e2f2d600f0..ceb2559a0e16c 100644 --- a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java +++ b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java @@ -100,6 +100,13 @@ public class FeatureFlags { Property.NodeScope ); + /** + * Gates the functionality of star tree index, which improves the performance of search + * aggregations. + */ + public static final String STAR_TREE_INDEX = "opensearch.experimental.feature.composite_index.star_tree.enabled"; + public static final Setting STAR_TREE_INDEX_SETTING = Setting.boolSetting(STAR_TREE_INDEX, false, Property.NodeScope); + private static final List> ALL_FEATURE_FLAG_SETTINGS = List.of( REMOTE_STORE_MIGRATION_EXPERIMENTAL_SETTING, EXTENSIONS_SETTING, @@ -108,7 +115,8 @@ public class FeatureFlags { DATETIME_FORMATTER_CACHING_SETTING, TIERED_REMOTE_INDEX_SETTING, PLUGGABLE_CACHE_SETTING, - REMOTE_PUBLICATION_EXPERIMENTAL_SETTING + REMOTE_PUBLICATION_EXPERIMENTAL_SETTING, + STAR_TREE_INDEX_SETTING ); /** * Should store the settings from opensearch.yml. diff --git a/server/src/main/java/org/opensearch/index/IndexModule.java b/server/src/main/java/org/opensearch/index/IndexModule.java index 4c494a6b35153..09b904394ee09 100644 --- a/server/src/main/java/org/opensearch/index/IndexModule.java +++ b/server/src/main/java/org/opensearch/index/IndexModule.java @@ -66,6 +66,7 @@ import org.opensearch.index.cache.query.DisabledQueryCache; import org.opensearch.index.cache.query.IndexQueryCache; import org.opensearch.index.cache.query.QueryCache; +import org.opensearch.index.compositeindex.CompositeIndexSettings; import org.opensearch.index.engine.Engine; import org.opensearch.index.engine.EngineConfigFactory; import org.opensearch.index.engine.EngineFactory; @@ -311,6 +312,7 @@ public Iterator> settings() { private final BooleanSupplier allowExpensiveQueries; private final Map recoveryStateFactories; private final FileCache fileCache; + private final CompositeIndexSettings compositeIndexSettings; /** * Construct the index module for the index with the specified index settings. The index module contains extension points for plugins @@ -330,7 +332,8 @@ public IndexModule( final BooleanSupplier allowExpensiveQueries, final IndexNameExpressionResolver expressionResolver, final Map recoveryStateFactories, - final FileCache fileCache + final FileCache fileCache, + final CompositeIndexSettings compositeIndexSettings ) { this.indexSettings = indexSettings; this.analysisRegistry = analysisRegistry; @@ -343,6 +346,7 @@ public IndexModule( this.expressionResolver = expressionResolver; this.recoveryStateFactories = recoveryStateFactories; this.fileCache = fileCache; + this.compositeIndexSettings = compositeIndexSettings; } public IndexModule( @@ -364,6 +368,7 @@ public IndexModule( allowExpensiveQueries, expressionResolver, recoveryStateFactories, + null, null ); } @@ -739,7 +744,8 @@ public IndexService newIndexService( clusterDefaultRefreshIntervalSupplier, recoverySettings, remoteStoreSettings, - fileCache + fileCache, + compositeIndexSettings ); success = true; return indexService; diff --git a/server/src/main/java/org/opensearch/index/IndexService.java b/server/src/main/java/org/opensearch/index/IndexService.java index a7849bcf80474..1c0db0095bb98 100644 --- a/server/src/main/java/org/opensearch/index/IndexService.java +++ b/server/src/main/java/org/opensearch/index/IndexService.java @@ -73,6 +73,7 @@ import org.opensearch.index.cache.IndexCache; import org.opensearch.index.cache.bitset.BitsetFilterCache; import org.opensearch.index.cache.query.QueryCache; +import org.opensearch.index.compositeindex.CompositeIndexSettings; import org.opensearch.index.engine.Engine; import org.opensearch.index.engine.EngineConfigFactory; import org.opensearch.index.engine.EngineFactory; @@ -192,6 +193,7 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust private final RecoverySettings recoverySettings; private final RemoteStoreSettings remoteStoreSettings; private final FileCache fileCache; + private final CompositeIndexSettings compositeIndexSettings; public IndexService( IndexSettings indexSettings, @@ -228,7 +230,8 @@ public IndexService( Supplier clusterDefaultRefreshIntervalSupplier, RecoverySettings recoverySettings, RemoteStoreSettings remoteStoreSettings, - FileCache fileCache + FileCache fileCache, + CompositeIndexSettings compositeIndexSettings ) { super(indexSettings); this.allowExpensiveQueries = allowExpensiveQueries; @@ -306,6 +309,7 @@ public IndexService( this.translogFactorySupplier = translogFactorySupplier; this.recoverySettings = recoverySettings; this.remoteStoreSettings = remoteStoreSettings; + this.compositeIndexSettings = compositeIndexSettings; this.fileCache = fileCache; updateFsyncTaskIfNecessary(); } @@ -381,6 +385,7 @@ public IndexService( clusterDefaultRefreshIntervalSupplier, recoverySettings, remoteStoreSettings, + null, null ); } @@ -1110,6 +1115,10 @@ private void rescheduleRefreshTasks() { } } + public CompositeIndexSettings getCompositeIndexSettings() { + return compositeIndexSettings; + } + /** * Shard Store Deleter Interface * diff --git a/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexSettings.java b/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexSettings.java new file mode 100644 index 0000000000000..014dd22426a10 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexSettings.java @@ -0,0 +1,55 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex; + +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Setting; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; + +/** + * Cluster level settings for composite indices + * + * @opensearch.experimental + */ +@ExperimentalApi +public class CompositeIndexSettings { + public static final Setting STAR_TREE_INDEX_ENABLED_SETTING = Setting.boolSetting( + "indices.composite_index.star_tree.enabled", + false, + value -> { + if (FeatureFlags.isEnabled(FeatureFlags.STAR_TREE_INDEX_SETTING) == false && value == true) { + throw new IllegalArgumentException( + "star tree index is under an experimental feature and can be activated only by enabling " + + FeatureFlags.STAR_TREE_INDEX_SETTING.getKey() + + " feature flag in the JVM options" + ); + } + }, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + + private volatile boolean starTreeIndexCreationEnabled; + + public CompositeIndexSettings(Settings settings, ClusterSettings clusterSettings) { + this.starTreeIndexCreationEnabled = STAR_TREE_INDEX_ENABLED_SETTING.get(settings); + clusterSettings.addSettingsUpdateConsumer(STAR_TREE_INDEX_ENABLED_SETTING, this::starTreeIndexCreationEnabled); + + } + + private void starTreeIndexCreationEnabled(boolean value) { + this.starTreeIndexCreationEnabled = value; + } + + public boolean isStarTreeIndexCreationEnabled() { + return starTreeIndexCreationEnabled; + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexValidator.java b/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexValidator.java new file mode 100644 index 0000000000000..9547d2ac9cfce --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexValidator.java @@ -0,0 +1,120 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex; + +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.index.IndexSettings; +import org.opensearch.index.compositeindex.datacube.Dimension; +import org.opensearch.index.compositeindex.datacube.Metric; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeIndexSettings; +import org.opensearch.index.mapper.CompositeMappedFieldType; +import org.opensearch.index.mapper.MappedFieldType; +import org.opensearch.index.mapper.MapperService; +import org.opensearch.index.mapper.StarTreeMapper; + +import java.util.Locale; +import java.util.Set; + +/** + * Validation for composite indices + * + * @opensearch.experimental + */ +@ExperimentalApi +public class CompositeIndexValidator { + + public static void validate(MapperService mapperService, CompositeIndexSettings compositeIndexSettings, IndexSettings indexSettings) { + validateStarTreeMappings(mapperService, compositeIndexSettings, indexSettings); + } + + public static void validate( + MapperService mapperService, + CompositeIndexSettings compositeIndexSettings, + IndexSettings indexSettings, + boolean isCompositeFieldPresent + ) { + if (!isCompositeFieldPresent && mapperService.isCompositeIndexPresent()) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "Composite fields must be specified during index creation, addition of new composite fields during update is not supported" + ) + ); + } + validateStarTreeMappings(mapperService, compositeIndexSettings, indexSettings); + } + + private static void validateStarTreeMappings( + MapperService mapperService, + CompositeIndexSettings compositeIndexSettings, + IndexSettings indexSettings + ) { + Set compositeFieldTypes = mapperService.getCompositeFieldTypes(); + if (compositeFieldTypes.size() > StarTreeIndexSettings.STAR_TREE_MAX_FIELDS_SETTING.get(indexSettings.getSettings())) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "Index cannot have more than [%s] star tree fields", + StarTreeIndexSettings.STAR_TREE_MAX_FIELDS_SETTING.get(indexSettings.getSettings()) + ) + ); + } + for (CompositeMappedFieldType compositeFieldType : compositeFieldTypes) { + if (!(compositeFieldType instanceof StarTreeMapper.StarTreeFieldType)) { + continue; + } + if (!compositeIndexSettings.isStarTreeIndexCreationEnabled()) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "star tree index cannot be created, enable it using [%s] setting", + CompositeIndexSettings.STAR_TREE_INDEX_ENABLED_SETTING.getKey() + ) + ); + } + StarTreeMapper.StarTreeFieldType dataCubeFieldType = (StarTreeMapper.StarTreeFieldType) compositeFieldType; + for (Dimension dim : dataCubeFieldType.getDimensions()) { + MappedFieldType ft = mapperService.fieldType(dim.getField()); + if (ft == null) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "unknown dimension field [%s] as part of star tree field", dim.getField()) + ); + } + if (ft.isAggregatable() == false) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "Aggregations not supported for the dimension field [%s] with field type [%s] as part of star tree field", + dim.getField(), + ft.typeName() + ) + ); + } + } + for (Metric metric : dataCubeFieldType.getMetrics()) { + MappedFieldType ft = mapperService.fieldType(metric.getField()); + if (ft == null) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "unknown metric field [%s] as part of star tree field", metric.getField()) + ); + } + if (ft.isAggregatable() == false) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "Aggregations not supported for the metrics field [%s] with field type [%s] as part of star tree field", + metric.getField(), + ft.typeName() + ) + ); + } + } + } + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/DateDimension.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/DateDimension.java new file mode 100644 index 0000000000000..14fcfc56f6354 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/DateDimension.java @@ -0,0 +1,99 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube; + +import org.opensearch.common.Rounding; +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.common.xcontent.support.XContentMapValues; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeIndexSettings; +import org.opensearch.index.mapper.Mapper; +import org.opensearch.index.mapper.StarTreeMapper; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Date dimension class + * + * @opensearch.experimental + */ +@ExperimentalApi +public class DateDimension extends Dimension { + private final List calendarIntervals; + public static final String CALENDAR_INTERVALS = "calendar_intervals"; + public static final String DATE = "date"; + + public DateDimension(String name, Map dimensionMap, Mapper.TypeParser.ParserContext c) { + super(name); + List intervalStrings = XContentMapValues.extractRawValues(CALENDAR_INTERVALS, dimensionMap) + .stream() + .map(Object::toString) + .collect(Collectors.toList()); + if (intervalStrings == null || intervalStrings.isEmpty()) { + this.calendarIntervals = StarTreeIndexSettings.DEFAULT_DATE_INTERVALS.get(c.getSettings()); + } else { + if (intervalStrings.size() > StarTreeIndexSettings.STAR_TREE_MAX_DATE_INTERVALS_SETTING.get(c.getSettings())) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "At most [%s] calendar intervals are allowed in dimension [%s]", + StarTreeIndexSettings.STAR_TREE_MAX_DATE_INTERVALS_SETTING.get(c.getSettings()), + name + ) + ); + } + Set calendarIntervals = new LinkedHashSet<>(); + for (String interval : intervalStrings) { + calendarIntervals.add(StarTreeIndexSettings.getTimeUnit(interval)); + } + this.calendarIntervals = new ArrayList<>(calendarIntervals); + } + dimensionMap.remove(CALENDAR_INTERVALS); + } + + public List getIntervals() { + return calendarIntervals; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(StarTreeMapper.NAME, this.getField()); + builder.field(StarTreeMapper.TYPE, DATE); + builder.startArray(CALENDAR_INTERVALS); + for (Rounding.DateTimeUnit interval : calendarIntervals) { + builder.value(interval.shortName()); + } + builder.endArray(); + builder.endObject(); + return builder; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + if (!super.equals(o)) return false; + DateDimension that = (DateDimension) o; + return Objects.equals(calendarIntervals, that.calendarIntervals); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), calendarIntervals); + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/Dimension.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/Dimension.java new file mode 100644 index 0000000000000..cc3ce3b88e757 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/Dimension.java @@ -0,0 +1,58 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube; + +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.index.mapper.StarTreeMapper; + +import java.io.IOException; +import java.util.Objects; + +/** + * Composite index dimension base class + * + * @opensearch.experimental + */ +@ExperimentalApi +public class Dimension implements ToXContent { + public static final String NUMERIC = "numeric"; + private final String field; + + public Dimension(String field) { + this.field = field; + } + + public String getField() { + return field; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(StarTreeMapper.NAME, field); + builder.field(StarTreeMapper.TYPE, NUMERIC); + builder.endObject(); + return builder; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Dimension dimension = (Dimension) o; + return Objects.equals(field, dimension.field); + } + + @Override + public int hashCode() { + return Objects.hash(field); + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/Metric.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/Metric.java new file mode 100644 index 0000000000000..9accb0201170a --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/Metric.java @@ -0,0 +1,65 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube; + +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.List; +import java.util.Objects; + +/** + * Holds details of metrics field as part of composite field + */ +@ExperimentalApi +public class Metric implements ToXContent { + private final String field; + private final List metrics; + + public Metric(String field, List metrics) { + this.field = field; + this.metrics = metrics; + } + + public String getField() { + return field; + } + + public List getMetrics() { + return metrics; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field("name", field); + builder.startArray("stats"); + for (MetricStat metricType : metrics) { + builder.value(metricType.getTypeName()); + } + builder.endArray(); + builder.endObject(); + return builder; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Metric metric = (Metric) o; + return Objects.equals(field, metric.field) && Objects.equals(metrics, metric.metrics); + } + + @Override + public int hashCode() { + return Objects.hash(field, metrics); + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/MetricStat.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/MetricStat.java new file mode 100644 index 0000000000000..fbde296b15f7e --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/MetricStat.java @@ -0,0 +1,44 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube; + +import org.opensearch.common.annotation.ExperimentalApi; + +/** + * Supported metric types for composite index + * + * @opensearch.experimental + */ +@ExperimentalApi +public enum MetricStat { + COUNT("count"), + AVG("avg"), + SUM("sum"), + MIN("min"), + MAX("max"); + + private final String typeName; + + MetricStat(String typeName) { + this.typeName = typeName; + } + + public String getTypeName() { + return typeName; + } + + public static MetricStat fromTypeName(String typeName) { + for (MetricStat metric : MetricStat.values()) { + if (metric.getTypeName().equalsIgnoreCase(typeName)) { + return metric; + } + } + throw new IllegalArgumentException("Invalid metric stat: " + typeName); + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/package-info.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/package-info.java new file mode 100644 index 0000000000000..320876ea937bf --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/package-info.java @@ -0,0 +1,11 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +/** + * Core classes for handling data cube indices such as star tree index. + */ +package org.opensearch.index.compositeindex.datacube; diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeField.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeField.java new file mode 100644 index 0000000000000..922ddcbea4fe2 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeField.java @@ -0,0 +1,94 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube.startree; + +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.index.compositeindex.datacube.Dimension; +import org.opensearch.index.compositeindex.datacube.Metric; + +import java.io.IOException; +import java.util.List; +import java.util.Objects; + +/** + * Star tree field which contains dimensions, metrics and specs + * + * @opensearch.experimental + */ +@ExperimentalApi +public class StarTreeField implements ToXContent { + private final String name; + private final List dimensionsOrder; + private final List metrics; + private final StarTreeFieldConfiguration starTreeConfig; + + public StarTreeField(String name, List dimensions, List metrics, StarTreeFieldConfiguration starTreeConfig) { + this.name = name; + this.dimensionsOrder = dimensions; + this.metrics = metrics; + this.starTreeConfig = starTreeConfig; + } + + public String getName() { + return name; + } + + public List getDimensionsOrder() { + return dimensionsOrder; + } + + public List getMetrics() { + return metrics; + } + + public StarTreeFieldConfiguration getStarTreeConfig() { + return starTreeConfig; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field("name", name); + if (dimensionsOrder != null && !dimensionsOrder.isEmpty()) { + builder.startArray("ordered_dimensions"); + for (Dimension dimension : dimensionsOrder) { + dimension.toXContent(builder, params); + } + builder.endArray(); + } + if (metrics != null && !metrics.isEmpty()) { + builder.startArray("metrics"); + for (Metric metric : metrics) { + metric.toXContent(builder, params); + } + builder.endArray(); + } + starTreeConfig.toXContent(builder, params); + builder.endObject(); + return builder; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + StarTreeField that = (StarTreeField) o; + return Objects.equals(name, that.name) + && Objects.equals(dimensionsOrder, that.dimensionsOrder) + && Objects.equals(metrics, that.metrics) + && Objects.equals(starTreeConfig, that.starTreeConfig); + } + + @Override + public int hashCode() { + return Objects.hash(name, dimensionsOrder, metrics, starTreeConfig); + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeFieldConfiguration.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeFieldConfiguration.java new file mode 100644 index 0000000000000..5dd066b34f108 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeFieldConfiguration.java @@ -0,0 +1,108 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube.startree; + +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Locale; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Star tree index specific configuration + * + * @opensearch.experimental + */ +@ExperimentalApi +public class StarTreeFieldConfiguration implements ToXContent { + + private final AtomicInteger maxLeafDocs = new AtomicInteger(); + private final Set skipStarNodeCreationInDims; + private final StarTreeBuildMode buildMode; + + public StarTreeFieldConfiguration(int maxLeafDocs, Set skipStarNodeCreationInDims, StarTreeBuildMode buildMode) { + this.maxLeafDocs.set(maxLeafDocs); + this.skipStarNodeCreationInDims = skipStarNodeCreationInDims; + this.buildMode = buildMode; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.field("max_leaf_docs", maxLeafDocs.get()); + builder.field("build_mode", buildMode.getTypeName()); + builder.startArray("skip_star_node_creation_for_dimensions"); + for (String dim : skipStarNodeCreationInDims) { + builder.value(dim); + } + builder.endArray(); + return builder; + } + + /** + * Star tree build mode using which sorting and aggregations are performed during index creation. + * + * @opensearch.experimental + */ + @ExperimentalApi + public enum StarTreeBuildMode { + // TODO : remove onheap support unless this proves useful + ON_HEAP("onheap"), + OFF_HEAP("offheap"); + + private final String typeName; + + StarTreeBuildMode(String typeName) { + this.typeName = typeName; + } + + public String getTypeName() { + return typeName; + } + + public static StarTreeBuildMode fromTypeName(String typeName) { + for (StarTreeBuildMode starTreeBuildMode : StarTreeBuildMode.values()) { + if (starTreeBuildMode.getTypeName().equalsIgnoreCase(typeName)) { + return starTreeBuildMode; + } + } + throw new IllegalArgumentException(String.format(Locale.ROOT, "Invalid star tree build mode: [%s] ", typeName)); + } + } + + public int maxLeafDocs() { + return maxLeafDocs.get(); + } + + public StarTreeBuildMode getBuildMode() { + return buildMode; + } + + public Set getSkipStarNodeCreationInDims() { + return skipStarNodeCreationInDims; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + StarTreeFieldConfiguration that = (StarTreeFieldConfiguration) o; + return Objects.equals(maxLeafDocs.get(), that.maxLeafDocs.get()) + && Objects.equals(skipStarNodeCreationInDims, that.skipStarNodeCreationInDims) + && buildMode == that.buildMode; + } + + @Override + public int hashCode() { + return Objects.hash(maxLeafDocs.get(), skipStarNodeCreationInDims, buildMode); + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeIndexSettings.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeIndexSettings.java new file mode 100644 index 0000000000000..191d4912d06a8 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeIndexSettings.java @@ -0,0 +1,114 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube.startree; + +import org.opensearch.common.Rounding; +import org.opensearch.common.settings.Setting; +import org.opensearch.index.compositeindex.datacube.MetricStat; +import org.opensearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder; + +import java.util.Arrays; +import java.util.List; + +/** + * Index settings for star tree fields. The settings are final as right now + * there is no support for update of star tree mapping. + * + * @opensearch.experimental + */ +public class StarTreeIndexSettings { + /** + * This setting determines the max number of star tree fields that can be part of composite index mapping. For each + * star tree field, we will generate associated star tree index. + */ + public static final Setting STAR_TREE_MAX_FIELDS_SETTING = Setting.intSetting( + "index.composite_index.star_tree.max_fields", + 1, + 1, + 1, + Setting.Property.IndexScope, + Setting.Property.Final + ); + + /** + * This setting determines the max number of dimensions that can be part of star tree index field. Number of + * dimensions and associated cardinality has direct effect of star tree index size and query performance. + */ + public static final Setting STAR_TREE_MAX_DIMENSIONS_SETTING = Setting.intSetting( + "index.composite_index.star_tree.field.max_dimensions", + 10, + 2, + 10, + Setting.Property.IndexScope, + Setting.Property.Final + ); + + /** + * This setting determines the max number of date intervals that can be part of star tree date field. + */ + public static final Setting STAR_TREE_MAX_DATE_INTERVALS_SETTING = Setting.intSetting( + "index.composite_index.star_tree.field.max_date_intervals", + 3, + 1, + 3, + Setting.Property.IndexScope, + Setting.Property.Final + ); + + /** + * This setting configures the default "maxLeafDocs" setting of star tree. This affects both query performance and + * star tree index size. Lesser the leaves, better the query latency but higher storage size and vice versa + *

+ * We can remove this later or change it to an enum based constant setting. + * + * @opensearch.experimental + */ + public static final Setting STAR_TREE_DEFAULT_MAX_LEAF_DOCS = Setting.intSetting( + "index.composite_index.star_tree.default.max_leaf_docs", + 10000, + 1, + Setting.Property.IndexScope, + Setting.Property.Final + ); + + /** + * Default intervals for date dimension as part of star tree fields + */ + public static final Setting> DEFAULT_DATE_INTERVALS = Setting.listSetting( + "index.composite_index.star_tree.field.default.date_intervals", + Arrays.asList(Rounding.DateTimeUnit.MINUTES_OF_HOUR.shortName(), Rounding.DateTimeUnit.HOUR_OF_DAY.shortName()), + StarTreeIndexSettings::getTimeUnit, + Setting.Property.IndexScope, + Setting.Property.Final + ); + + /** + * Default metrics for metrics as part of star tree fields + */ + public static final Setting> DEFAULT_METRICS_LIST = Setting.listSetting( + "index.composite_index.star_tree.field.default.metrics", + Arrays.asList( + MetricStat.AVG.toString(), + MetricStat.COUNT.toString(), + MetricStat.SUM.toString(), + MetricStat.MAX.toString(), + MetricStat.MIN.toString() + ), + MetricStat::fromTypeName, + Setting.Property.IndexScope, + Setting.Property.Final + ); + + public static Rounding.DateTimeUnit getTimeUnit(String expression) { + if (!DateHistogramAggregationBuilder.DATE_FIELD_UNITS.containsKey(expression)) { + throw new IllegalArgumentException("unknown calendar intervals specified in star tree index mapping"); + } + return DateHistogramAggregationBuilder.DATE_FIELD_UNITS.get(expression); + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/package-info.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/package-info.java new file mode 100644 index 0000000000000..4f4e670478e2f --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/package-info.java @@ -0,0 +1,11 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +/** + * Core classes for handling star tree index. + */ +package org.opensearch.index.compositeindex.datacube.startree; diff --git a/server/src/main/java/org/opensearch/index/compositeindex/package-info.java b/server/src/main/java/org/opensearch/index/compositeindex/package-info.java new file mode 100644 index 0000000000000..59f18efec26b1 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/package-info.java @@ -0,0 +1,13 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * Core classes for handling composite indices. + * @opensearch.experimental + */ +package org.opensearch.index.compositeindex; diff --git a/server/src/main/java/org/opensearch/index/mapper/CompositeDataCubeFieldType.java b/server/src/main/java/org/opensearch/index/mapper/CompositeDataCubeFieldType.java new file mode 100644 index 0000000000000..b01555260e760 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/mapper/CompositeDataCubeFieldType.java @@ -0,0 +1,54 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.mapper; + +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.index.compositeindex.datacube.Dimension; +import org.opensearch.index.compositeindex.datacube.Metric; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * Base class for multi field data cube fields + * + * @opensearch.experimental + */ +@ExperimentalApi +public abstract class CompositeDataCubeFieldType extends CompositeMappedFieldType { + private final List dimensions; + private final List metrics; + + public CompositeDataCubeFieldType(String name, List dims, List metrics, CompositeFieldType type) { + super(name, getFields(dims, metrics), type); + this.dimensions = dims; + this.metrics = metrics; + } + + private static List getFields(List dims, List metrics) { + Set fields = new HashSet<>(); + for (Dimension dim : dims) { + fields.add(dim.getField()); + } + for (Metric metric : metrics) { + fields.add(metric.getField()); + } + return new ArrayList<>(fields); + } + + public List getDimensions() { + return dimensions; + } + + public List getMetrics() { + return metrics; + } +} diff --git a/server/src/main/java/org/opensearch/index/mapper/CompositeMappedFieldType.java b/server/src/main/java/org/opensearch/index/mapper/CompositeMappedFieldType.java new file mode 100644 index 0000000000000..f52ce29a86dd2 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/mapper/CompositeMappedFieldType.java @@ -0,0 +1,75 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.mapper; + +import org.opensearch.common.annotation.ExperimentalApi; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +/** + * Base class for composite field types + * + * @opensearch.experimental + */ +@ExperimentalApi +public abstract class CompositeMappedFieldType extends MappedFieldType { + private final List fields; + private final CompositeFieldType type; + + public CompositeMappedFieldType( + String name, + boolean isIndexed, + boolean isStored, + boolean hasDocValues, + TextSearchInfo textSearchInfo, + Map meta, + List fields, + CompositeFieldType type + ) { + super(name, isIndexed, isStored, hasDocValues, textSearchInfo, meta); + this.fields = fields; + this.type = type; + } + + public CompositeMappedFieldType(String name, List fields, CompositeFieldType type) { + this(name, false, false, false, TextSearchInfo.NONE, Collections.emptyMap(), fields, type); + } + + /** + * Supported composite field types + */ + public enum CompositeFieldType { + STAR_TREE("star_tree"); + + private final String name; + + CompositeFieldType(String name) { + this.name = name; + } + + public String getName() { + return name; + } + + public static CompositeFieldType fromName(String name) { + for (CompositeFieldType metric : CompositeFieldType.values()) { + if (metric.getName().equalsIgnoreCase(name)) { + return metric; + } + } + throw new IllegalArgumentException("Invalid metric stat: " + name); + } + } + + public List fields() { + return fields; + } +} diff --git a/server/src/main/java/org/opensearch/index/mapper/Mapper.java b/server/src/main/java/org/opensearch/index/mapper/Mapper.java index bd5d3f15c0706..46a5050d4fc18 100644 --- a/server/src/main/java/org/opensearch/index/mapper/Mapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/Mapper.java @@ -253,6 +253,11 @@ public boolean isWithinMultiField() { } Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException; + + default Mapper.Builder parse(String name, Map node, ParserContext parserContext, ObjectMapper.Builder objBuilder) + throws MapperParsingException { + throw new UnsupportedOperationException("should not be invoked"); + } } private final String simpleName; diff --git a/server/src/main/java/org/opensearch/index/mapper/MapperService.java b/server/src/main/java/org/opensearch/index/mapper/MapperService.java index a1f3894c9f14c..c2e7411a3b47a 100644 --- a/server/src/main/java/org/opensearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/opensearch/index/mapper/MapperService.java @@ -650,6 +650,23 @@ public Iterable fieldTypes() { return this.mapper == null ? Collections.emptySet() : this.mapper.fieldTypes(); } + public boolean isCompositeIndexPresent() { + return this.mapper != null && !getCompositeFieldTypes().isEmpty(); + } + + public Set getCompositeFieldTypes() { + Set compositeMappedFieldTypes = new HashSet<>(); + if (this.mapper == null) { + return Collections.emptySet(); + } + for (MappedFieldType type : this.mapper.fieldTypes()) { + if (type instanceof CompositeMappedFieldType) { + compositeMappedFieldTypes.add((CompositeMappedFieldType) type); + } + } + return compositeMappedFieldTypes; + } + public ObjectMapper getObjectMapper(String name) { return this.mapper == null ? null : this.mapper.objectMappers().get(name); } diff --git a/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java b/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java index 92ffdb60e6cde..be3adfe8b2c4e 100644 --- a/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java @@ -42,9 +42,11 @@ import org.opensearch.common.collect.CopyOnWriteHashMap; import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.support.XContentMapValues; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeIndexSettings; import org.opensearch.index.mapper.MapperService.MergeReason; import java.io.IOException; @@ -176,6 +178,7 @@ public void setIncludeInRoot(boolean value) { * @opensearch.internal */ @SuppressWarnings("rawtypes") + @PublicApi(since = "1.0.0") public static class Builder extends Mapper.Builder { protected Explicit enabled = new Explicit<>(true, false); @@ -262,14 +265,25 @@ public static class TypeParser implements Mapper.TypeParser { public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { ObjectMapper.Builder builder = new Builder(name); parseNested(name, node, builder, parserContext); + Object compositeField = null; for (Iterator> iterator = node.entrySet().iterator(); iterator.hasNext();) { Map.Entry entry = iterator.next(); String fieldName = entry.getKey(); Object fieldNode = entry.getValue(); - if (parseObjectOrDocumentTypeProperties(fieldName, fieldNode, parserContext, builder)) { + if (fieldName.equals("composite")) { + compositeField = fieldNode; iterator.remove(); + } else { + if (parseObjectOrDocumentTypeProperties(fieldName, fieldNode, parserContext, builder)) { + iterator.remove(); + } } } + // Important : Composite field is made up of 2 or more source fields of the index, so this must be called + // after parsing all other properties + if (compositeField != null) { + parseCompositeField(builder, (Map) compositeField, parserContext); + } return builder; } @@ -407,6 +421,96 @@ protected static void parseDerived(ObjectMapper.Builder objBuilder, Map compositeNode, + ParserContext parserContext + ) { + if (!FeatureFlags.isEnabled(FeatureFlags.STAR_TREE_INDEX_SETTING)) { + throw new IllegalArgumentException( + "star tree index is under an experimental feature and can be activated only by enabling " + + FeatureFlags.STAR_TREE_INDEX_SETTING.getKey() + + " feature flag in the JVM options" + ); + } + Iterator> iterator = compositeNode.entrySet().iterator(); + if (compositeNode.size() > StarTreeIndexSettings.STAR_TREE_MAX_FIELDS_SETTING.get(parserContext.getSettings())) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "Composite fields cannot have more than [%s] fields", + StarTreeIndexSettings.STAR_TREE_MAX_FIELDS_SETTING.get(parserContext.getSettings()) + ) + ); + } + while (iterator.hasNext()) { + Map.Entry entry = iterator.next(); + String fieldName = entry.getKey(); + // Should accept empty arrays, as a work around for when the + // user can't provide an empty Map. (PHP for example) + boolean isEmptyList = entry.getValue() instanceof List && ((List) entry.getValue()).isEmpty(); + if (entry.getValue() instanceof Map) { + @SuppressWarnings("unchecked") + Map propNode = (Map) entry.getValue(); + String type; + Object typeNode = propNode.get("type"); + if (typeNode != null) { + type = typeNode.toString(); + } else { + // lets see if we can derive this... + throw new MapperParsingException("No type specified for field [" + fieldName + "]"); + } + Mapper.TypeParser typeParser = getSupportedCompositeTypeParser(type, parserContext); + if (typeParser == null) { + throw new MapperParsingException("No handler for type [" + type + "] declared on field [" + fieldName + "]"); + } + String[] fieldNameParts = fieldName.split("\\."); + // field name is just ".", which is invalid + if (fieldNameParts.length < 1) { + throw new MapperParsingException("Invalid field name " + fieldName); + } + String realFieldName = fieldNameParts[fieldNameParts.length - 1]; + Mapper.Builder fieldBuilder = typeParser.parse(realFieldName, propNode, parserContext, objBuilder); + for (int i = fieldNameParts.length - 2; i >= 0; --i) { + ObjectMapper.Builder intermediate = new ObjectMapper.Builder<>(fieldNameParts[i]); + intermediate.add(fieldBuilder); + fieldBuilder = intermediate; + } + objBuilder.add(fieldBuilder); + propNode.remove("type"); + DocumentMapperParser.checkNoRemainingFields(fieldName, propNode, parserContext.indexVersionCreated()); + iterator.remove(); + } else if (isEmptyList) { + iterator.remove(); + } else { + throw new MapperParsingException( + "Expected map for property [fields] on field [" + fieldName + "] but got a " + fieldName.getClass() + ); + } + } + + DocumentMapperParser.checkNoRemainingFields( + compositeNode, + parserContext.indexVersionCreated(), + "DocType mapping definition has unsupported parameters: " + ); + } + + private static Mapper.TypeParser getSupportedCompositeTypeParser(String type, ParserContext parserContext) { + switch (type) { + case StarTreeMapper.CONTENT_TYPE: + return parserContext.typeParser(type); + default: + throw new IllegalArgumentException( + String.format(Locale.ROOT, "Type [%s] isn't supported in composite field context.", type) + ); + } + } + protected static void parseProperties(ObjectMapper.Builder objBuilder, Map propsNode, ParserContext parserContext) { Iterator> iterator = propsNode.entrySet().iterator(); while (iterator.hasNext()) { diff --git a/server/src/main/java/org/opensearch/index/mapper/RootObjectMapper.java b/server/src/main/java/org/opensearch/index/mapper/RootObjectMapper.java index 9504e6eafc046..e06e5be4633f9 100644 --- a/server/src/main/java/org/opensearch/index/mapper/RootObjectMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/RootObjectMapper.java @@ -177,15 +177,26 @@ public Mapper.Builder parse(String name, Map node, ParserContext RootObjectMapper.Builder builder = new Builder(name); Iterator> iterator = node.entrySet().iterator(); + Object compositeField = null; while (iterator.hasNext()) { Map.Entry entry = iterator.next(); String fieldName = entry.getKey(); Object fieldNode = entry.getValue(); - if (parseObjectOrDocumentTypeProperties(fieldName, fieldNode, parserContext, builder) - || processField(builder, fieldName, fieldNode, parserContext)) { + if (fieldName.equals("composite")) { + compositeField = fieldNode; iterator.remove(); + } else { + if (parseObjectOrDocumentTypeProperties(fieldName, fieldNode, parserContext, builder) + || processField(builder, fieldName, fieldNode, parserContext)) { + iterator.remove(); + } } } + // Important : Composite field is made up of 2 or more source properties of the index, so this must be called + // after parsing all other properties + if (compositeField != null) { + parseCompositeField(builder, (Map) compositeField, parserContext); + } return builder; } diff --git a/server/src/main/java/org/opensearch/index/mapper/StarTreeMapper.java b/server/src/main/java/org/opensearch/index/mapper/StarTreeMapper.java new file mode 100644 index 0000000000000..fbd3fa6b8b1d3 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/mapper/StarTreeMapper.java @@ -0,0 +1,430 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.mapper; + +import org.apache.lucene.search.Query; +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.common.xcontent.support.XContentMapValues; +import org.opensearch.index.compositeindex.datacube.DateDimension; +import org.opensearch.index.compositeindex.datacube.Dimension; +import org.opensearch.index.compositeindex.datacube.Metric; +import org.opensearch.index.compositeindex.datacube.MetricStat; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeIndexSettings; +import org.opensearch.index.query.QueryShardContext; +import org.opensearch.search.lookup.SearchLookup; + +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * A field mapper for star tree fields + * + * @opensearch.experimental + */ +@ExperimentalApi +public class StarTreeMapper extends ParametrizedFieldMapper { + public static final String CONTENT_TYPE = "star_tree"; + public static final String CONFIG = "config"; + public static final String MAX_LEAF_DOCS = "max_leaf_docs"; + public static final String SKIP_STAR_NODE_IN_DIMS = "skip_star_node_creation_for_dimensions"; + public static final String BUILD_MODE = "build_mode"; + public static final String ORDERED_DIMENSIONS = "ordered_dimensions"; + public static final String METRICS = "metrics"; + public static final String NAME = "name"; + public static final String TYPE = "type"; + public static final String STATS = "stats"; + + @Override + public ParametrizedFieldMapper.Builder getMergeBuilder() { + return new Builder(simpleName(), objBuilder).init(this); + + } + + /** + * Builder for the star tree field mapper + * + * @opensearch.internal + */ + public static class Builder extends ParametrizedFieldMapper.Builder { + private ObjectMapper.Builder objbuilder; + private static final Set> ALLOWED_DIMENSION_MAPPER_BUILDERS = Set.of( + NumberFieldMapper.Builder.class, + DateFieldMapper.Builder.class + ); + private static final Set> ALLOWED_METRIC_MAPPER_BUILDERS = Set.of(NumberFieldMapper.Builder.class); + + @SuppressWarnings("unchecked") + private final Parameter config = new Parameter<>(CONFIG, false, () -> null, (name, context, nodeObj) -> { + if (nodeObj instanceof Map) { + Map paramMap = (Map) nodeObj; + int maxLeafDocs = XContentMapValues.nodeIntegerValue( + paramMap.get(MAX_LEAF_DOCS), + StarTreeIndexSettings.STAR_TREE_DEFAULT_MAX_LEAF_DOCS.get(context.getSettings()) + ); + if (maxLeafDocs < 1) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "%s [%s] must be greater than 0", MAX_LEAF_DOCS, maxLeafDocs) + ); + } + paramMap.remove(MAX_LEAF_DOCS); + Set skipStarInDims = new LinkedHashSet<>( + List.of(XContentMapValues.nodeStringArrayValue(paramMap.getOrDefault(SKIP_STAR_NODE_IN_DIMS, new ArrayList()))) + ); + paramMap.remove(SKIP_STAR_NODE_IN_DIMS); + StarTreeFieldConfiguration.StarTreeBuildMode buildMode = StarTreeFieldConfiguration.StarTreeBuildMode.fromTypeName( + XContentMapValues.nodeStringValue( + paramMap.get(BUILD_MODE), + StarTreeFieldConfiguration.StarTreeBuildMode.OFF_HEAP.getTypeName() + ) + ); + paramMap.remove(BUILD_MODE); + List dimensions = buildDimensions(name, paramMap, context); + paramMap.remove(ORDERED_DIMENSIONS); + List metrics = buildMetrics(name, paramMap, context); + paramMap.remove(METRICS); + if (paramMap.containsKey(NAME)) { + paramMap.remove(NAME); + } + for (String dim : skipStarInDims) { + if (dimensions.stream().filter(d -> d.getField().equals(dim)).findAny().isEmpty()) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "[%s] in skip_star_node_creation_for_dimensions should be part of ordered_dimensions", + dim + ) + ); + } + } + StarTreeFieldConfiguration spec = new StarTreeFieldConfiguration(maxLeafDocs, skipStarInDims, buildMode); + DocumentMapperParser.checkNoRemainingFields( + paramMap, + context.indexVersionCreated(), + "Star tree mapping definition has unsupported parameters: " + ); + return new StarTreeField(this.name, dimensions, metrics, spec); + + } else { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "unable to parse config for star tree field [%s]", this.name) + ); + } + }, m -> toType(m).starTreeField); + + /** + * Build dimensions from mapping + */ + @SuppressWarnings("unchecked") + private List buildDimensions(String fieldName, Map map, Mapper.TypeParser.ParserContext context) { + Object dims = XContentMapValues.extractValue("ordered_dimensions", map); + if (dims == null) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "ordered_dimensions is required for star tree field [%s]", fieldName) + ); + } + List dimensions = new LinkedList<>(); + if (dims instanceof List) { + List dimList = (List) dims; + if (dimList.size() > context.getSettings().getAsInt(StarTreeIndexSettings.STAR_TREE_MAX_DIMENSIONS_SETTING.getKey(), 10)) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "ordered_dimensions cannot have more than %s dimensions for star tree field [%s]", + context.getSettings().getAsInt(StarTreeIndexSettings.STAR_TREE_MAX_DIMENSIONS_SETTING.getKey(), 10), + fieldName + ) + ); + } + if (dimList.size() < 2) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "Atleast two dimensions are required to build star tree index field [%s]", fieldName) + ); + } + for (Object dim : dimList) { + dimensions.add(getDimension(fieldName, dim, context)); + } + } else { + throw new MapperParsingException( + String.format(Locale.ROOT, "unable to parse ordered_dimensions for star tree field [%s]", fieldName) + ); + } + return dimensions; + } + + /** + * Get dimension based on mapping + */ + @SuppressWarnings("unchecked") + private Dimension getDimension(String fieldName, Object dimensionMapping, Mapper.TypeParser.ParserContext context) { + Dimension dimension; + Map dimensionMap = (Map) dimensionMapping; + String name = (String) XContentMapValues.extractValue(NAME, dimensionMap); + dimensionMap.remove(NAME); + if (this.objbuilder == null || this.objbuilder.mappersBuilders == null) { + String type = (String) XContentMapValues.extractValue(TYPE, dimensionMap); + dimensionMap.remove(TYPE); + if (type == null) { + throw new MapperParsingException( + String.format(Locale.ROOT, "unable to parse ordered_dimensions for star tree field [%s]", fieldName) + ); + } + if (type.equals(DateDimension.DATE)) { + dimension = new DateDimension(name, dimensionMap, context); + } else if (type.equals(Dimension.NUMERIC)) { + dimension = new Dimension(name); + } else { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "unsupported field type associated with dimension [%s] as part of star tree field [%s]", + name, + fieldName + ) + ); + } + return dimension; + } else { + Optional dimBuilder = findMapperBuilderByName(name, this.objbuilder.mappersBuilders); + if (dimBuilder.isEmpty()) { + throw new IllegalArgumentException(String.format(Locale.ROOT, "unknown dimension field [%s]", name)); + } + if (!isBuilderAllowedForDimension(dimBuilder.get())) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "unsupported field type associated with dimension [%s] as part of star tree field [%s]", + name, + fieldName + ) + ); + } + if (dimBuilder.get() instanceof DateFieldMapper.Builder) { + dimension = new DateDimension(name, dimensionMap, context); + } + // Numeric dimension - default + else if (dimBuilder.get() instanceof NumberFieldMapper.Builder) { + dimension = new Dimension(name); + } else { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "unsupported field type associated with star tree dimension [%s]", name) + ); + } + } + DocumentMapperParser.checkNoRemainingFields( + dimensionMap, + context.indexVersionCreated(), + "Star tree mapping definition has unsupported parameters: " + ); + return dimension; + } + + /** + * Build metrics from mapping + */ + @SuppressWarnings("unchecked") + private List buildMetrics(String fieldName, Map map, Mapper.TypeParser.ParserContext context) { + List metrics = new LinkedList<>(); + Object metricsFromInput = XContentMapValues.extractValue(METRICS, map); + if (metricsFromInput == null) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "metrics section is required for star tree field [%s]", fieldName) + ); + } + if (metricsFromInput instanceof List) { + List metricsList = (List) metricsFromInput; + for (Object metric : metricsList) { + Map metricMap = (Map) metric; + String name = (String) XContentMapValues.extractValue(NAME, metricMap); + metricMap.remove(NAME); + if (objbuilder == null || objbuilder.mappersBuilders == null) { + metrics.add(getMetric(name, metricMap, context)); + } else { + Optional meticBuilder = findMapperBuilderByName(name, this.objbuilder.mappersBuilders); + if (meticBuilder.isEmpty()) { + throw new IllegalArgumentException(String.format(Locale.ROOT, "unknown metric field [%s]", name)); + } + if (!isBuilderAllowedForMetric(meticBuilder.get())) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "non-numeric field type is associated with star tree metric [%s]", this.name) + ); + } + metrics.add(getMetric(name, metricMap, context)); + DocumentMapperParser.checkNoRemainingFields( + metricMap, + context.indexVersionCreated(), + "Star tree mapping definition has unsupported parameters: " + ); + } + } + } else { + throw new MapperParsingException(String.format(Locale.ROOT, "unable to parse metrics for star tree field [%s]", this.name)); + } + + return metrics; + } + + @SuppressWarnings("unchecked") + private Metric getMetric(String name, Map metric, Mapper.TypeParser.ParserContext context) { + List metricTypes; + List metricStrings = XContentMapValues.extractRawValues(STATS, metric) + .stream() + .map(Object::toString) + .collect(Collectors.toList()); + metric.remove(STATS); + if (metricStrings.isEmpty()) { + metricTypes = new ArrayList<>(StarTreeIndexSettings.DEFAULT_METRICS_LIST.get(context.getSettings())); + } else { + Set metricSet = new LinkedHashSet<>(); + for (String metricString : metricStrings) { + metricSet.add(MetricStat.fromTypeName(metricString)); + } + metricTypes = new ArrayList<>(metricSet); + } + return new Metric(name, metricTypes); + } + + @Override + protected List> getParameters() { + return List.of(config); + } + + private static boolean isBuilderAllowedForDimension(Mapper.Builder builder) { + return ALLOWED_DIMENSION_MAPPER_BUILDERS.stream().anyMatch(allowedType -> allowedType.isInstance(builder)); + } + + private static boolean isBuilderAllowedForMetric(Mapper.Builder builder) { + return ALLOWED_METRIC_MAPPER_BUILDERS.stream().anyMatch(allowedType -> allowedType.isInstance(builder)); + } + + private Optional findMapperBuilderByName(String field, List mappersBuilders) { + return mappersBuilders.stream().filter(builder -> builder.name().equals(field)).findFirst(); + } + + public Builder(String name, ObjectMapper.Builder objBuilder) { + super(name); + this.objbuilder = objBuilder; + } + + @Override + public ParametrizedFieldMapper build(BuilderContext context) { + StarTreeFieldType type = new StarTreeFieldType(name, this.config.get()); + return new StarTreeMapper(name, type, this, objbuilder); + } + } + + private static StarTreeMapper toType(FieldMapper in) { + return (StarTreeMapper) in; + } + + /** + * Concrete parse for star tree type + * + * @opensearch.internal + */ + public static class TypeParser implements Mapper.TypeParser { + + /** + * default constructor of VectorFieldMapper.TypeParser + */ + public TypeParser() {} + + @Override + public Mapper.Builder parse(String name, Map node, ParserContext context) throws MapperParsingException { + Builder builder = new StarTreeMapper.Builder(name, null); + builder.parse(name, context, node); + return builder; + } + + @Override + public Mapper.Builder parse(String name, Map node, ParserContext context, ObjectMapper.Builder objBuilder) + throws MapperParsingException { + Builder builder = new StarTreeMapper.Builder(name, objBuilder); + builder.parse(name, context, node); + return builder; + } + } + + private final StarTreeField starTreeField; + + private final ObjectMapper.Builder objBuilder; + + protected StarTreeMapper(String simpleName, StarTreeFieldType type, Builder builder, ObjectMapper.Builder objbuilder) { + super(simpleName, type, MultiFields.empty(), CopyTo.empty()); + this.starTreeField = builder.config.get(); + this.objBuilder = objbuilder; + } + + @Override + public StarTreeFieldType fieldType() { + return (StarTreeFieldType) super.fieldType(); + } + + @Override + protected String contentType() { + return CONTENT_TYPE; + } + + @Override + protected void parseCreateField(ParseContext context) { + throw new MapperParsingException( + String.format( + Locale.ROOT, + "Field [%s] is a star tree field and cannot be added inside a document. Use the index API request parameters.", + name() + ) + ); + } + + /** + * Star tree mapped field type containing dimensions, metrics, star tree specs + * + * @opensearch.experimental + */ + @ExperimentalApi + public static final class StarTreeFieldType extends CompositeDataCubeFieldType { + + private final StarTreeFieldConfiguration starTreeConfig; + + public StarTreeFieldType(String name, StarTreeField starTreeField) { + super(name, starTreeField.getDimensionsOrder(), starTreeField.getMetrics(), CompositeFieldType.STAR_TREE); + this.starTreeConfig = starTreeField.getStarTreeConfig(); + } + + public StarTreeFieldConfiguration getStarTreeConfig() { + return starTreeConfig; + } + + @Override + public ValueFetcher valueFetcher(QueryShardContext context, SearchLookup searchLookup, String format) { + // TODO : evaluate later + throw new UnsupportedOperationException("Cannot fetch values for star tree field [" + name() + "]."); + } + + @Override + public String typeName() { + return CONTENT_TYPE; + } + + @Override + public Query termQuery(Object value, QueryShardContext context) { + // TODO : evaluate later + throw new UnsupportedOperationException("Cannot perform terms query on star tree field [" + name() + "]."); + } + } + +} diff --git a/server/src/main/java/org/opensearch/indices/IndicesModule.java b/server/src/main/java/org/opensearch/indices/IndicesModule.java index 033b163bb0d67..f7e52ce9fc1ae 100644 --- a/server/src/main/java/org/opensearch/indices/IndicesModule.java +++ b/server/src/main/java/org/opensearch/indices/IndicesModule.java @@ -70,6 +70,7 @@ import org.opensearch.index.mapper.RoutingFieldMapper; import org.opensearch.index.mapper.SeqNoFieldMapper; import org.opensearch.index.mapper.SourceFieldMapper; +import org.opensearch.index.mapper.StarTreeMapper; import org.opensearch.index.mapper.TextFieldMapper; import org.opensearch.index.mapper.VersionFieldMapper; import org.opensearch.index.mapper.WildcardFieldMapper; @@ -174,6 +175,7 @@ public static Map getMappers(List mappe mappers.put(ConstantKeywordFieldMapper.CONTENT_TYPE, new ConstantKeywordFieldMapper.TypeParser()); mappers.put(DerivedFieldMapper.CONTENT_TYPE, DerivedFieldMapper.PARSER); mappers.put(WildcardFieldMapper.CONTENT_TYPE, WildcardFieldMapper.PARSER); + mappers.put(StarTreeMapper.CONTENT_TYPE, new StarTreeMapper.TypeParser()); for (MapperPlugin mapperPlugin : mapperPlugins) { for (Map.Entry entry : mapperPlugin.getMappers().entrySet()) { diff --git a/server/src/main/java/org/opensearch/indices/IndicesService.java b/server/src/main/java/org/opensearch/indices/IndicesService.java index a7d879fc06981..902ca95643625 100644 --- a/server/src/main/java/org/opensearch/indices/IndicesService.java +++ b/server/src/main/java/org/opensearch/indices/IndicesService.java @@ -106,6 +106,7 @@ import org.opensearch.index.IndexSettings; import org.opensearch.index.analysis.AnalysisRegistry; import org.opensearch.index.cache.request.ShardRequestCache; +import org.opensearch.index.compositeindex.CompositeIndexSettings; import org.opensearch.index.engine.CommitStats; import org.opensearch.index.engine.EngineConfig; import org.opensearch.index.engine.EngineConfigFactory; @@ -356,6 +357,7 @@ public class IndicesService extends AbstractLifecycleComponent private volatile TimeValue clusterDefaultRefreshInterval; private final SearchRequestStats searchRequestStats; private final FileCache fileCache; + private final CompositeIndexSettings compositeIndexSettings; @Override protected void doStart() { @@ -391,7 +393,8 @@ public IndicesService( RecoverySettings recoverySettings, CacheService cacheService, RemoteStoreSettings remoteStoreSettings, - FileCache fileCache + FileCache fileCache, + CompositeIndexSettings compositeIndexSettings ) { this.settings = settings; this.threadPool = threadPool; @@ -498,6 +501,7 @@ protected void closeInternal() { .addSettingsUpdateConsumer(CLUSTER_DEFAULT_INDEX_REFRESH_INTERVAL_SETTING, this::onRefreshIntervalUpdate); this.recoverySettings = recoverySettings; this.remoteStoreSettings = remoteStoreSettings; + this.compositeIndexSettings = compositeIndexSettings; this.fileCache = fileCache; } @@ -558,6 +562,7 @@ public IndicesService( recoverySettings, cacheService, remoteStoreSettings, + null, null ); } @@ -939,7 +944,8 @@ private synchronized IndexService createIndexService( () -> allowExpensiveQueries, indexNameExpressionResolver, recoveryStateFactories, - fileCache + fileCache, + compositeIndexSettings ); for (IndexingOperationListener operationListener : indexingOperationListeners) { indexModule.addIndexOperationListener(operationListener); @@ -1030,7 +1036,8 @@ public synchronized MapperService createIndexMapperService(IndexMetadata indexMe () -> allowExpensiveQueries, indexNameExpressionResolver, recoveryStateFactories, - fileCache + fileCache, + compositeIndexSettings ); pluginsService.onIndexModule(indexModule); return indexModule.newIndexMapperService(xContentRegistry, mapperRegistry, scriptService); @@ -2098,4 +2105,8 @@ private TimeValue getClusterDefaultRefreshInterval() { public RemoteStoreSettings getRemoteStoreSettings() { return this.remoteStoreSettings; } + + public CompositeIndexSettings getCompositeIndexSettings() { + return this.compositeIndexSettings; + } } diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index a91dce4ece126..b7c57e04b30bc 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -147,6 +147,7 @@ import org.opensearch.index.IndexingPressureService; import org.opensearch.index.SegmentReplicationStatsTracker; import org.opensearch.index.analysis.AnalysisRegistry; +import org.opensearch.index.compositeindex.CompositeIndexSettings; import org.opensearch.index.engine.EngineFactory; import org.opensearch.index.recovery.RemoteStoreRestoreService; import org.opensearch.index.remote.RemoteIndexPathUploader; @@ -830,6 +831,7 @@ protected Node( final RecoverySettings recoverySettings = new RecoverySettings(settings, settingsModule.getClusterSettings()); final RemoteStoreSettings remoteStoreSettings = new RemoteStoreSettings(settings, settingsModule.getClusterSettings()); + final CompositeIndexSettings compositeIndexSettings = new CompositeIndexSettings(settings, settingsModule.getClusterSettings()); final IndexStorePlugin.DirectoryFactory remoteDirectoryFactory = new RemoteSegmentStoreDirectoryFactory( repositoriesServiceReference::get, @@ -870,7 +872,8 @@ protected Node( recoverySettings, cacheService, remoteStoreSettings, - fileCache + fileCache, + compositeIndexSettings ); final IngestService ingestService = new IngestService( diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeMappingIntegTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeMappingIntegTests.java new file mode 100644 index 0000000000000..0097574a19b85 --- /dev/null +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeMappingIntegTests.java @@ -0,0 +1,364 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube.startree; + +import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.common.Rounding; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.core.index.Index; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.index.IndexService; +import org.opensearch.index.compositeindex.CompositeIndexSettings; +import org.opensearch.index.compositeindex.datacube.DateDimension; +import org.opensearch.index.compositeindex.datacube.MetricStat; +import org.opensearch.index.mapper.CompositeMappedFieldType; +import org.opensearch.index.mapper.MapperParsingException; +import org.opensearch.index.mapper.StarTreeMapper; +import org.opensearch.indices.IndicesService; +import org.opensearch.test.OpenSearchIntegTestCase; +import org.junit.After; +import org.junit.Before; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Set; + +import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; + +public class StarTreeMappingIntegTests extends OpenSearchIntegTestCase { + private static final String TEST_INDEX = "test"; + + private static XContentBuilder createMinimalTestMapping(boolean invalidDim, boolean invalidMetric, boolean keywordDim) { + try { + return jsonBuilder().startObject() + .startObject("composite") + .startObject("startree-1") + .field("type", "star_tree") + .startObject("config") + .startArray("ordered_dimensions") + .startObject() + .field("name", "timestamp") + .endObject() + .startObject() + .field("name", getDim(invalidDim, keywordDim)) + .endObject() + .endArray() + .startArray("metrics") + .startObject() + .field("name", getDim(invalidMetric, false)) + .endObject() + .endArray() + .endObject() + .endObject() + .endObject() + .startObject("properties") + .startObject("timestamp") + .field("type", "date") + .endObject() + .startObject("numeric_dv") + .field("type", "integer") + .field("doc_values", true) + .endObject() + .startObject("numeric") + .field("type", "integer") + .field("doc_values", false) + .endObject() + .startObject("keyword_dv") + .field("type", "keyword") + .field("doc_values", true) + .endObject() + .startObject("keyword") + .field("type", "keyword") + .field("doc_values", false) + .endObject() + .endObject() + .endObject(); + } catch (IOException e) { + throw new IllegalStateException(e); + } + } + + private static XContentBuilder createTestMappingWithoutStarTree(boolean invalidDim, boolean invalidMetric, boolean keywordDim) { + try { + return jsonBuilder().startObject() + .startObject("properties") + .startObject("timestamp") + .field("type", "date") + .endObject() + .startObject("numeric_dv") + .field("type", "integer") + .field("doc_values", true) + .endObject() + .startObject("numeric") + .field("type", "integer") + .field("doc_values", false) + .endObject() + .startObject("keyword_dv") + .field("type", "keyword") + .field("doc_values", true) + .endObject() + .startObject("keyword") + .field("type", "keyword") + .field("doc_values", false) + .endObject() + .endObject() + .endObject(); + } catch (IOException e) { + throw new IllegalStateException(e); + } + } + + private static XContentBuilder createUpdateTestMapping(boolean changeDim, boolean sameStarTree) { + try { + return jsonBuilder().startObject() + .startObject("composite") + .startObject(sameStarTree ? "startree-1" : "startree-2") + .field("type", "star_tree") + .startObject("config") + .startArray("ordered_dimensions") + .startObject() + .field("name", "timestamp") + .endObject() + .startObject() + .field("name", changeDim ? "numeric_new" : getDim(false, false)) + .endObject() + .endArray() + .startArray("metrics") + .startObject() + .field("name", getDim(false, false)) + .endObject() + .endArray() + .endObject() + .endObject() + .endObject() + .startObject("properties") + .startObject("timestamp") + .field("type", "date") + .endObject() + .startObject("numeric_dv") + .field("type", "integer") + .field("doc_values", true) + .endObject() + .startObject("numeric") + .field("type", "integer") + .field("doc_values", false) + .endObject() + .startObject("numeric_new") + .field("type", "integer") + .field("doc_values", true) + .endObject() + .startObject("keyword_dv") + .field("type", "keyword") + .field("doc_values", true) + .endObject() + .startObject("keyword") + .field("type", "keyword") + .field("doc_values", false) + .endObject() + .endObject() + .endObject(); + } catch (IOException e) { + throw new IllegalStateException(e); + } + } + + private static String getDim(boolean hasDocValues, boolean isKeyword) { + if (hasDocValues) { + return "numeric"; + } else if (isKeyword) { + return "keyword"; + } + return "numeric_dv"; + } + + @Override + protected Settings featureFlagSettings() { + return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.STAR_TREE_INDEX, "true").build(); + } + + @Before + public final void setupNodeSettings() { + Settings request = Settings.builder().put(CompositeIndexSettings.STAR_TREE_INDEX_ENABLED_SETTING.getKey(), true).build(); + assertAcked(client().admin().cluster().prepareUpdateSettings().setPersistentSettings(request).get()); + } + + public void testValidCompositeIndex() { + prepareCreate(TEST_INDEX).setMapping(createMinimalTestMapping(false, false, false)).get(); + Iterable dataNodeInstances = internalCluster().getDataNodeInstances(IndicesService.class); + for (IndicesService service : dataNodeInstances) { + final Index index = resolveIndex("test"); + if (service.hasIndex(index)) { + IndexService indexService = service.indexService(index); + Set fts = indexService.mapperService().getCompositeFieldTypes(); + + for (CompositeMappedFieldType ft : fts) { + assertTrue(ft instanceof StarTreeMapper.StarTreeFieldType); + StarTreeMapper.StarTreeFieldType starTreeFieldType = (StarTreeMapper.StarTreeFieldType) ft; + assertEquals("timestamp", starTreeFieldType.getDimensions().get(0).getField()); + assertTrue(starTreeFieldType.getDimensions().get(0) instanceof DateDimension); + DateDimension dateDim = (DateDimension) starTreeFieldType.getDimensions().get(0); + List expectedTimeUnits = Arrays.asList( + Rounding.DateTimeUnit.MINUTES_OF_HOUR, + Rounding.DateTimeUnit.HOUR_OF_DAY + ); + assertEquals(expectedTimeUnits, dateDim.getIntervals()); + assertEquals("numeric_dv", starTreeFieldType.getDimensions().get(1).getField()); + assertEquals("numeric_dv", starTreeFieldType.getMetrics().get(0).getField()); + List expectedMetrics = Arrays.asList( + MetricStat.AVG, + MetricStat.COUNT, + MetricStat.SUM, + MetricStat.MAX, + MetricStat.MIN + ); + assertEquals(expectedMetrics, starTreeFieldType.getMetrics().get(0).getMetrics()); + assertEquals(10000, starTreeFieldType.getStarTreeConfig().maxLeafDocs()); + assertEquals( + StarTreeFieldConfiguration.StarTreeBuildMode.OFF_HEAP, + starTreeFieldType.getStarTreeConfig().getBuildMode() + ); + assertEquals(Collections.emptySet(), starTreeFieldType.getStarTreeConfig().getSkipStarNodeCreationInDims()); + } + } + } + } + + public void testUpdateIndexWithAdditionOfStarTree() { + prepareCreate(TEST_INDEX).setMapping(createMinimalTestMapping(false, false, false)).get(); + + IllegalArgumentException ex = expectThrows( + IllegalArgumentException.class, + () -> client().admin().indices().preparePutMapping(TEST_INDEX).setSource(createUpdateTestMapping(false, false)).get() + ); + assertEquals("Index cannot have more than [1] star tree fields", ex.getMessage()); + } + + public void testUpdateIndexWithNewerStarTree() { + prepareCreate(TEST_INDEX).setMapping(createTestMappingWithoutStarTree(false, false, false)).get(); + + IllegalArgumentException ex = expectThrows( + IllegalArgumentException.class, + () -> client().admin().indices().preparePutMapping(TEST_INDEX).setSource(createUpdateTestMapping(false, false)).get() + ); + assertEquals( + "Composite fields must be specified during index creation, addition of new composite fields during update is not supported", + ex.getMessage() + ); + } + + public void testUpdateIndexWhenMappingIsDifferent() { + prepareCreate(TEST_INDEX).setMapping(createMinimalTestMapping(false, false, false)).get(); + + // update some field in the mapping + IllegalArgumentException ex = expectThrows( + IllegalArgumentException.class, + () -> client().admin().indices().preparePutMapping(TEST_INDEX).setSource(createUpdateTestMapping(true, true)).get() + ); + assertTrue(ex.getMessage().contains("Cannot update parameter [config] from")); + } + + public void testUpdateIndexWhenMappingIsSame() { + prepareCreate(TEST_INDEX).setMapping(createMinimalTestMapping(false, false, false)).get(); + + // update some field in the mapping + AcknowledgedResponse putMappingResponse = client().admin() + .indices() + .preparePutMapping(TEST_INDEX) + .setSource(createMinimalTestMapping(false, false, false)) + .get(); + assertAcked(putMappingResponse); + + Iterable dataNodeInstances = internalCluster().getDataNodeInstances(IndicesService.class); + for (IndicesService service : dataNodeInstances) { + final Index index = resolveIndex("test"); + if (service.hasIndex(index)) { + IndexService indexService = service.indexService(index); + Set fts = indexService.mapperService().getCompositeFieldTypes(); + + for (CompositeMappedFieldType ft : fts) { + assertTrue(ft instanceof StarTreeMapper.StarTreeFieldType); + StarTreeMapper.StarTreeFieldType starTreeFieldType = (StarTreeMapper.StarTreeFieldType) ft; + assertEquals("timestamp", starTreeFieldType.getDimensions().get(0).getField()); + assertTrue(starTreeFieldType.getDimensions().get(0) instanceof DateDimension); + DateDimension dateDim = (DateDimension) starTreeFieldType.getDimensions().get(0); + List expectedTimeUnits = Arrays.asList( + Rounding.DateTimeUnit.MINUTES_OF_HOUR, + Rounding.DateTimeUnit.HOUR_OF_DAY + ); + assertEquals(expectedTimeUnits, dateDim.getIntervals()); + assertEquals("numeric_dv", starTreeFieldType.getDimensions().get(1).getField()); + assertEquals("numeric_dv", starTreeFieldType.getMetrics().get(0).getField()); + List expectedMetrics = Arrays.asList( + MetricStat.AVG, + MetricStat.COUNT, + MetricStat.SUM, + MetricStat.MAX, + MetricStat.MIN + ); + assertEquals(expectedMetrics, starTreeFieldType.getMetrics().get(0).getMetrics()); + assertEquals(10000, starTreeFieldType.getStarTreeConfig().maxLeafDocs()); + assertEquals( + StarTreeFieldConfiguration.StarTreeBuildMode.OFF_HEAP, + starTreeFieldType.getStarTreeConfig().getBuildMode() + ); + assertEquals(Collections.emptySet(), starTreeFieldType.getStarTreeConfig().getSkipStarNodeCreationInDims()); + } + } + } + } + + public void testInvalidDimCompositeIndex() { + IllegalArgumentException ex = expectThrows( + IllegalArgumentException.class, + () -> prepareCreate(TEST_INDEX).setMapping(createMinimalTestMapping(true, false, false)).get() + ); + assertEquals( + "Aggregations not supported for the dimension field [numeric] with field type [integer] as part of star tree field", + ex.getMessage() + ); + } + + public void testUnsupportedDim() { + MapperParsingException ex = expectThrows( + MapperParsingException.class, + () -> prepareCreate(TEST_INDEX).setMapping(createMinimalTestMapping(false, false, true)).get() + ); + assertEquals( + "Failed to parse mapping [_doc]: unsupported field type associated with dimension [keyword] as part of star tree field [startree-1]", + ex.getMessage() + ); + } + + public void testInvalidMetric() { + IllegalArgumentException ex = expectThrows( + IllegalArgumentException.class, + () -> prepareCreate(TEST_INDEX).setMapping(createMinimalTestMapping(false, true, false)).get() + ); + assertEquals( + "Aggregations not supported for the metrics field [numeric] with field type [integer] as part of star tree field", + ex.getMessage() + ); + } + + @After + public final void cleanupNodeSettings() { + assertAcked( + client().admin() + .cluster() + .prepareUpdateSettings() + .setPersistentSettings(Settings.builder().putNull("*")) + .setTransientSettings(Settings.builder().putNull("*")) + ); + } + +} diff --git a/server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java new file mode 100644 index 0000000000000..e2fef4c360e19 --- /dev/null +++ b/server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java @@ -0,0 +1,469 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.mapper; + +import org.opensearch.common.CheckedConsumer; +import org.opensearch.common.Rounding; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.index.compositeindex.datacube.DateDimension; +import org.opensearch.index.compositeindex.datacube.MetricStat; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; +import org.junit.After; +import org.junit.Before; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.containsString; + +/** + * Tests for {@link StarTreeMapper}. + */ +public class StarTreeMapperTests extends MapperTestCase { + + @Before + public void setup() { + FeatureFlags.initializeFeatureFlags(Settings.builder().put(FeatureFlags.STAR_TREE_INDEX, true).build()); + } + + @After + public void teardown() { + FeatureFlags.initializeFeatureFlags(Settings.EMPTY); + } + + public void testValidStarTree() throws IOException { + MapperService mapperService = createMapperService(getExpandedMapping("status", "size")); + Set compositeFieldTypes = mapperService.getCompositeFieldTypes(); + for (CompositeMappedFieldType type : compositeFieldTypes) { + StarTreeMapper.StarTreeFieldType starTreeFieldType = (StarTreeMapper.StarTreeFieldType) type; + assertEquals("@timestamp", starTreeFieldType.getDimensions().get(0).getField()); + assertTrue(starTreeFieldType.getDimensions().get(0) instanceof DateDimension); + DateDimension dateDim = (DateDimension) starTreeFieldType.getDimensions().get(0); + List expectedTimeUnits = Arrays.asList( + Rounding.DateTimeUnit.DAY_OF_MONTH, + Rounding.DateTimeUnit.MONTH_OF_YEAR + ); + assertEquals(expectedTimeUnits, dateDim.getIntervals()); + assertEquals("status", starTreeFieldType.getDimensions().get(1).getField()); + assertEquals("size", starTreeFieldType.getMetrics().get(0).getField()); + List expectedMetrics = Arrays.asList(MetricStat.SUM, MetricStat.AVG); + assertEquals(expectedMetrics, starTreeFieldType.getMetrics().get(0).getMetrics()); + assertEquals(100, starTreeFieldType.getStarTreeConfig().maxLeafDocs()); + assertEquals(StarTreeFieldConfiguration.StarTreeBuildMode.ON_HEAP, starTreeFieldType.getStarTreeConfig().getBuildMode()); + assertEquals( + new HashSet<>(Arrays.asList("@timestamp", "status")), + starTreeFieldType.getStarTreeConfig().getSkipStarNodeCreationInDims() + ); + } + } + + public void testValidStarTreeDefaults() throws IOException { + MapperService mapperService = createMapperService(getMinMapping()); + Set compositeFieldTypes = mapperService.getCompositeFieldTypes(); + for (CompositeMappedFieldType type : compositeFieldTypes) { + StarTreeMapper.StarTreeFieldType starTreeFieldType = (StarTreeMapper.StarTreeFieldType) type; + assertEquals("@timestamp", starTreeFieldType.getDimensions().get(0).getField()); + assertTrue(starTreeFieldType.getDimensions().get(0) instanceof DateDimension); + DateDimension dateDim = (DateDimension) starTreeFieldType.getDimensions().get(0); + List expectedTimeUnits = Arrays.asList( + Rounding.DateTimeUnit.MINUTES_OF_HOUR, + Rounding.DateTimeUnit.HOUR_OF_DAY + ); + assertEquals(expectedTimeUnits, dateDim.getIntervals()); + assertEquals("status", starTreeFieldType.getDimensions().get(1).getField()); + assertEquals("status", starTreeFieldType.getMetrics().get(0).getField()); + List expectedMetrics = Arrays.asList( + MetricStat.AVG, + MetricStat.COUNT, + MetricStat.SUM, + MetricStat.MAX, + MetricStat.MIN + ); + assertEquals(expectedMetrics, starTreeFieldType.getMetrics().get(0).getMetrics()); + assertEquals(10000, starTreeFieldType.getStarTreeConfig().maxLeafDocs()); + assertEquals(StarTreeFieldConfiguration.StarTreeBuildMode.OFF_HEAP, starTreeFieldType.getStarTreeConfig().getBuildMode()); + assertEquals(Collections.emptySet(), starTreeFieldType.getStarTreeConfig().getSkipStarNodeCreationInDims()); + } + } + + public void testInvalidDim() { + MapperParsingException ex = expectThrows( + MapperParsingException.class, + () -> createMapperService(getExpandedMapping("invalid", "size")) + ); + assertEquals("Failed to parse mapping [_doc]: unknown dimension field [invalid]", ex.getMessage()); + } + + public void testInvalidMetric() { + MapperParsingException ex = expectThrows( + MapperParsingException.class, + () -> createMapperService(getExpandedMapping("status", "invalid")) + ); + assertEquals("Failed to parse mapping [_doc]: unknown metric field [invalid]", ex.getMessage()); + } + + public void testNoMetrics() { + MapperParsingException ex = expectThrows( + MapperParsingException.class, + () -> createMapperService(getMinMapping(false, true, false, false)) + ); + assertThat( + ex.getMessage(), + containsString("Failed to parse mapping [_doc]: metrics section is required for star tree field [startree]") + ); + } + + public void testInvalidParam() { + MapperParsingException ex = expectThrows( + MapperParsingException.class, + () -> createMapperService(getInvalidMapping(false, false, false, false, true)) + ); + assertEquals( + "Failed to parse mapping [_doc]: Star tree mapping definition has unsupported parameters: [invalid : {invalid=invalid}]", + ex.getMessage() + ); + } + + public void testNoDims() { + MapperParsingException ex = expectThrows( + MapperParsingException.class, + () -> createMapperService(getMinMapping(true, false, false, false)) + ); + assertThat( + ex.getMessage(), + containsString("Failed to parse mapping [_doc]: ordered_dimensions is required for star tree field [startree]") + ); + } + + public void testMissingDims() { + MapperParsingException ex = expectThrows( + MapperParsingException.class, + () -> createMapperService(getMinMapping(false, false, true, false)) + ); + assertThat(ex.getMessage(), containsString("Failed to parse mapping [_doc]: unknown dimension field [@timestamp]")); + } + + public void testMissingMetrics() { + MapperParsingException ex = expectThrows( + MapperParsingException.class, + () -> createMapperService(getMinMapping(false, false, false, true)) + ); + assertThat(ex.getMessage(), containsString("Failed to parse mapping [_doc]: unknown metric field [metric_field]")); + } + + public void testInvalidMetricType() { + MapperParsingException ex = expectThrows( + MapperParsingException.class, + () -> createMapperService(getInvalidMapping(false, false, false, true)) + ); + assertEquals( + "Failed to parse mapping [_doc]: non-numeric field type is associated with star tree metric [startree]", + ex.getMessage() + ); + } + + public void testInvalidDimType() { + MapperParsingException ex = expectThrows( + MapperParsingException.class, + () -> createMapperService(getInvalidMapping(false, false, true, false)) + ); + assertEquals( + "Failed to parse mapping [_doc]: unsupported field type associated with dimension [@timestamp] as part of star tree field [startree]", + ex.getMessage() + ); + } + + public void testInvalidSkipDim() { + MapperParsingException ex = expectThrows( + MapperParsingException.class, + () -> createMapperService(getInvalidMapping(false, true, false, false)) + ); + assertEquals( + "Failed to parse mapping [_doc]: [invalid] in skip_star_node_creation_for_dimensions should be part of ordered_dimensions", + ex.getMessage() + ); + } + + public void testInvalidSingleDim() { + MapperParsingException ex = expectThrows( + MapperParsingException.class, + () -> createMapperService(getInvalidMapping(true, false, false, false)) + ); + assertEquals( + "Failed to parse mapping [_doc]: Atleast two dimensions are required to build star tree index field [startree]", + ex.getMessage() + ); + } + + private XContentBuilder getExpandedMapping(String dim, String metric) throws IOException { + return topMapping(b -> { + b.startObject("composite"); + b.startObject("startree"); + b.field("type", "star_tree"); + b.startObject("config"); + b.field("build_mode", "onheap"); + b.field("max_leaf_docs", 100); + b.startArray("skip_star_node_creation_for_dimensions"); + { + b.value("@timestamp"); + b.value("status"); + } + b.endArray(); + b.startArray("ordered_dimensions"); + b.startObject(); + b.field("name", "@timestamp"); + b.startArray("calendar_intervals"); + b.value("day"); + b.value("month"); + b.endArray(); + b.endObject(); + b.startObject(); + b.field("name", dim); + b.endObject(); + b.endArray(); + b.startArray("metrics"); + b.startObject(); + b.field("name", metric); + b.startArray("stats"); + b.value("sum"); + b.value("avg"); + b.endArray(); + b.endObject(); + b.endArray(); + b.endObject(); + b.endObject(); + b.endObject(); + b.startObject("properties"); + b.startObject("@timestamp"); + b.field("type", "date"); + b.endObject(); + b.startObject("status"); + b.field("type", "integer"); + b.endObject(); + b.startObject("size"); + b.field("type", "integer"); + b.endObject(); + b.endObject(); + }); + } + + private XContentBuilder getMinMapping() throws IOException { + return getMinMapping(false, false, false, false); + } + + private XContentBuilder getMinMapping(boolean isEmptyDims, boolean isEmptyMetrics, boolean missingDim, boolean missingMetric) + throws IOException { + return topMapping(b -> { + b.startObject("composite"); + b.startObject("startree"); + b.field("type", "star_tree"); + b.startObject("config"); + if (!isEmptyDims) { + b.startArray("ordered_dimensions"); + b.startObject(); + b.field("name", "@timestamp"); + b.endObject(); + b.startObject(); + b.field("name", "status"); + b.endObject(); + b.endArray(); + } + if (!isEmptyMetrics) { + b.startArray("metrics"); + b.startObject(); + b.field("name", "status"); + b.endObject(); + b.startObject(); + b.field("name", "metric_field"); + b.endObject(); + b.endArray(); + } + b.endObject(); + b.endObject(); + b.endObject(); + b.startObject("properties"); + if (!missingDim) { + b.startObject("@timestamp"); + b.field("type", "date"); + b.endObject(); + } + b.startObject("status"); + b.field("type", "integer"); + b.endObject(); + if (!missingMetric) { + b.startObject("metric_field"); + b.field("type", "integer"); + b.endObject(); + } + b.endObject(); + }); + } + + private XContentBuilder getInvalidMapping( + boolean singleDim, + boolean invalidSkipDims, + boolean invalidDimType, + boolean invalidMetricType, + boolean invalidParam + ) throws IOException { + return topMapping(b -> { + b.startObject("composite"); + b.startObject("startree"); + b.field("type", "star_tree"); + b.startObject("config"); + + b.startArray("skip_star_node_creation_for_dimensions"); + { + if (invalidSkipDims) { + b.value("invalid"); + } + b.value("status"); + } + b.endArray(); + if (invalidParam) { + b.startObject("invalid"); + b.field("invalid", "invalid"); + b.endObject(); + } + b.startArray("ordered_dimensions"); + if (!singleDim) { + b.startObject(); + b.field("name", "@timestamp"); + b.endObject(); + } + b.startObject(); + b.field("name", "status"); + b.endObject(); + b.endArray(); + b.startArray("metrics"); + b.startObject(); + b.field("name", "status"); + b.endObject(); + b.startObject(); + b.field("name", "metric_field"); + b.endObject(); + b.endArray(); + b.endObject(); + b.endObject(); + b.endObject(); + b.startObject("properties"); + b.startObject("@timestamp"); + if (!invalidDimType) { + b.field("type", "date"); + } else { + b.field("type", "keyword"); + } + b.endObject(); + + b.startObject("status"); + b.field("type", "integer"); + b.endObject(); + b.startObject("metric_field"); + if (invalidMetricType) { + b.field("type", "date"); + } else { + b.field("type", "integer"); + } + b.endObject(); + b.endObject(); + }); + } + + private XContentBuilder getInvalidMapping(boolean singleDim, boolean invalidSkipDims, boolean invalidDimType, boolean invalidMetricType) + throws IOException { + return getInvalidMapping(singleDim, invalidSkipDims, invalidDimType, invalidMetricType, false); + } + + protected boolean supportsOrIgnoresBoost() { + return false; + } + + protected boolean supportsMeta() { + return false; + } + + @Override + protected void assertExistsQuery(MapperService mapperService) {} + + // Overriding fieldMapping to make it create composite mappings by default. + // This way, the parent tests are checking the right behavior for this Mapper. + @Override + protected final XContentBuilder fieldMapping(CheckedConsumer buildField) throws IOException { + return topMapping(b -> { + b.startObject("composite"); + b.startObject("startree"); + buildField.accept(b); + b.endObject(); + b.endObject(); + b.startObject("properties"); + b.startObject("size"); + b.field("type", "integer"); + b.endObject(); + b.startObject("status"); + b.field("type", "integer"); + b.endObject(); + b.endObject(); + }); + } + + @Override + public void testEmptyName() { + MapperParsingException e = expectThrows(MapperParsingException.class, () -> createMapperService(topMapping(b -> { + b.startObject("composite"); + b.startObject(""); + minimalMapping(b); + b.endObject(); + b.endObject(); + b.startObject("properties"); + b.startObject("size"); + b.field("type", "integer"); + b.endObject(); + b.startObject("status"); + b.field("type", "integer"); + b.endObject(); + b.endObject(); + }))); + assertThat(e.getMessage(), containsString("name cannot be empty string")); + assertParseMinimalWarnings(); + } + + @Override + protected void minimalMapping(XContentBuilder b) throws IOException { + b.field("type", "star_tree"); + b.startObject("config"); + b.startArray("ordered_dimensions"); + b.startObject(); + b.field("name", "size"); + b.endObject(); + b.startObject(); + b.field("name", "status"); + b.endObject(); + b.endArray(); + b.startArray("metrics"); + b.startObject(); + b.field("name", "status"); + b.endObject(); + b.endArray(); + b.endObject(); + } + + @Override + protected void writeFieldValue(XContentBuilder builder) throws IOException {} + + @Override + protected void registerParameters(ParameterChecker checker) throws IOException { + + } +} diff --git a/test/framework/src/main/java/org/opensearch/index/mapper/MapperTestCase.java b/test/framework/src/main/java/org/opensearch/index/mapper/MapperTestCase.java index dc5954907a4fa..01a4005255f29 100644 --- a/test/framework/src/main/java/org/opensearch/index/mapper/MapperTestCase.java +++ b/test/framework/src/main/java/org/opensearch/index/mapper/MapperTestCase.java @@ -174,7 +174,7 @@ protected static void assertNoDocValuesField(ParseContext.Document doc, String f } } - public final void testEmptyName() { + public void testEmptyName() { MapperParsingException e = expectThrows(MapperParsingException.class, () -> createMapperService(mapping(b -> { b.startObject(""); minimalMapping(b); diff --git a/test/framework/src/main/java/org/opensearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/opensearch/search/aggregations/AggregatorTestCase.java index 28323a94af721..544fb100a17bf 100644 --- a/test/framework/src/main/java/org/opensearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/opensearch/search/aggregations/AggregatorTestCase.java @@ -115,6 +115,7 @@ import org.opensearch.index.mapper.ObjectMapper.Nested; import org.opensearch.index.mapper.RangeFieldMapper; import org.opensearch.index.mapper.RangeType; +import org.opensearch.index.mapper.StarTreeMapper; import org.opensearch.index.mapper.TextFieldMapper; import org.opensearch.index.query.QueryShardContext; import org.opensearch.index.shard.IndexShard; @@ -201,6 +202,7 @@ public abstract class AggregatorTestCase extends OpenSearchTestCase { denylist.add(CompletionFieldMapper.CONTENT_TYPE); // TODO support completion denylist.add(FieldAliasMapper.CONTENT_TYPE); // TODO support alias denylist.add(DerivedFieldMapper.CONTENT_TYPE); // TODO support derived fields + denylist.add(StarTreeMapper.CONTENT_TYPE); // TODO evaluate support for star tree fields TYPE_TEST_DENYLIST = denylist; } From 72bc12cacf3d3de1da4c1d15ced0346069bd0e00 Mon Sep 17 00:00:00 2001 From: Sarthak Aggarwal Date: Sun, 23 Jun 2024 20:49:27 +0530 Subject: [PATCH 08/11] OnHeap Star Tree Implementation Signed-off-by: Sarthak Aggarwal --- .../index/BaseSingleStarTreeBuilder.java | 660 ++++++++++++++++++ .../lucene/index/StarTreeDocValuesWriter.java | 37 + .../common/inject/util/Modules.java | 2 +- .../compositeindex/datacube/MetricStat.java | 3 +- .../aggregators/MetricStatFieldPair.java | 115 +++ .../aggregators/SumValueAggregator.java | 82 +++ .../startree/aggregators/ValueAggregator.java | 64 ++ .../aggregators/ValueAggregatorFactory.java | 51 ++ .../numerictype/StarTreeNumericType.java | 45 ++ .../StarTreeNumericTypeConverters.java | 31 + .../aggregators/numerictype/package-info.java | 13 + .../startree/aggregators/package-info.java | 13 + .../builder/DocValuesIteratorFactory.java | 38 + .../builder/MultipleTreesBuilder.java | 110 +++ .../builder/OnHeapSingleTreeBuilder.java | 204 ++++++ .../startree/builder/SingleTreeBuilder.java | 33 + .../StarTreeDocValuesIteratorFactory.java | 54 ++ .../startree/builder/package-info.java | 13 + .../datacube/startree/data/DataType.java | 67 ++ .../startree/data/StarTreeDocument.java | 29 + .../datacube/startree/node/package-info.java | 13 + .../startree/utils/StarTreeBuilderUtils.java | 131 ++++ .../datacube/startree/utils/package-info.java | 13 + .../aggregators/MetricStatFieldPairTests.java | 69 ++ .../aggregators/SumValueAggregatorTests.java | 64 ++ .../ValueAggregatorFactoryTests.java | 43 ++ .../BaseSingleStarTreeBuilderTests.java | 213 ++++++ .../builder/OnHeapSingleTreeBuilderTests.java | 264 +++++++ .../StarTreeValuesIteratorFactoryTests.java | 113 +++ 29 files changed, 2585 insertions(+), 2 deletions(-) create mode 100644 server/src/main/java/org/apache/lucene/index/BaseSingleStarTreeBuilder.java create mode 100644 server/src/main/java/org/apache/lucene/index/StarTreeDocValuesWriter.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricStatFieldPair.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregator.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregator.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactory.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericType.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericTypeConverters.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/package-info.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/package-info.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/DocValuesIteratorFactory.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/MultipleTreesBuilder.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapSingleTreeBuilder.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/SingleTreeBuilder.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocValuesIteratorFactory.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/package-info.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/data/DataType.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/data/StarTreeDocument.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/package-info.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeBuilderUtils.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/package-info.java create mode 100644 server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricStatFieldPairTests.java create mode 100644 server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregatorTests.java create mode 100644 server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactoryTests.java create mode 100644 server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseSingleStarTreeBuilderTests.java create mode 100644 server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapSingleTreeBuilderTests.java create mode 100644 server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeValuesIteratorFactoryTests.java diff --git a/server/src/main/java/org/apache/lucene/index/BaseSingleStarTreeBuilder.java b/server/src/main/java/org/apache/lucene/index/BaseSingleStarTreeBuilder.java new file mode 100644 index 0000000000000..7b0957b1328de --- /dev/null +++ b/server/src/main/java/org/apache/lucene/index/BaseSingleStarTreeBuilder.java @@ -0,0 +1,660 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.apache.lucene.index; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.lucene.codecs.DocValuesConsumer; +import org.apache.lucene.codecs.DocValuesProducer; +import org.apache.lucene.search.DocIdSetIterator; +import org.opensearch.index.compositeindex.datacube.DateDimension; +import org.opensearch.index.compositeindex.datacube.Dimension; +import org.opensearch.index.compositeindex.datacube.Metric; +import org.opensearch.index.compositeindex.datacube.MetricStat; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; +import org.opensearch.index.compositeindex.datacube.startree.aggregators.MetricStatFieldPair; +import org.opensearch.index.compositeindex.datacube.startree.aggregators.ValueAggregator; +import org.opensearch.index.compositeindex.datacube.startree.aggregators.ValueAggregatorFactory; +import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; +import org.opensearch.index.compositeindex.datacube.startree.builder.SingleTreeBuilder; +import org.opensearch.index.compositeindex.datacube.startree.builder.StarTreeDocValuesIteratorFactory; +import org.opensearch.index.compositeindex.datacube.startree.data.StarTreeDocument; +import org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeBuilderUtils; +import org.opensearch.index.fielddata.IndexNumericFieldData; +import org.opensearch.index.mapper.Mapper; +import org.opensearch.index.mapper.MapperService; +import org.opensearch.index.mapper.NumberFieldMapper; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Base class for star-tree builder + */ +public abstract class BaseSingleStarTreeBuilder implements SingleTreeBuilder { + + // TODO: STAR_TREE_CODEC will be moved to CodecService once the Star Tree Codec is defined + public static final String STAR_TREE_CODEC = "startreecodec"; + + private static final Logger logger = LogManager.getLogger(BaseSingleStarTreeBuilder.class); + + public static final int STAR_IN_DOC_VALUES_INDEX = -1; + + protected final String[] dimensionsSplitOrder; + protected final Set skipStarNodeCreationForDimensions; + protected final String[] metrics; + + protected final int numMetrics; + protected final int numDimensions; + protected int numDocs; + protected int totalDocs; + protected int numNodes; + protected final int maxLeafDocuments; + + protected final StarTreeBuilderUtils.TreeNode rootNode = getNewNode(); + + // TODO: This will be initialized with OnHeap / OffHeap Implementations (Commented it's occurrences for now) + // private IndexOutput indexOutput; + protected DocIdSetIterator[] dimensionReaders; + protected DocIdSetIterator[] metricReaders; + + protected ValueAggregator[] valueAggregators; + protected IndexNumericFieldData.NumericType[] numericTypes; + protected DocValuesConsumer docValuesConsumer; + protected DocValuesProducer docValuesProducer; + + private final StarTreeDocValuesIteratorFactory starTreeDocValuesIteratorFactory; + private final StarTreeField starTreeField; + private final StarTreeFieldConfiguration starTreeFieldSpec; + private final List metricStatFieldPairs; + private final MapperService mapperService; + + /** + * Constructor for base star-tree builder + * + * @param starTreeField holds the configuration for the star tree + * @param docValuesProducer helps return the doc values iterator for each type based on field name + * @param docValuesConsumer to consume the new aggregated metrics during flush + * @param state stores the segment state + * @param mapperService helps to find the original type of the field + */ + protected BaseSingleStarTreeBuilder( + StarTreeField starTreeField, + DocValuesProducer docValuesProducer, + DocValuesConsumer docValuesConsumer, + SegmentWriteState state, + MapperService mapperService + ) throws IOException { + + logger.info("Building in base star tree builder"); + + // String docFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, "stttree"); + // logger.info("Star tree file name : {}", docFileName); + + // indexOutput = state.directory.createOutput(docFileName, state.context); + // CodecUtil.writeIndexHeader(indexOutput, STAR_TREE_CODEC, 0, state.segmentInfo.getId(), state.segmentSuffix); + this.mapperService = mapperService; + this.starTreeField = starTreeField; + this.starTreeFieldSpec = starTreeField.getStarTreeConfig(); + this.docValuesConsumer = docValuesConsumer; + this.docValuesProducer = docValuesProducer; + this.starTreeDocValuesIteratorFactory = new StarTreeDocValuesIteratorFactory(); + + List dimensionsSplitOrder = starTreeField.getDimensionsOrder(); + this.numDimensions = dimensionsSplitOrder.size(); + this.dimensionsSplitOrder = new String[numDimensions]; + + this.skipStarNodeCreationForDimensions = new HashSet<>(); + this.totalDocs = state.segmentInfo.maxDoc(); + this.dimensionReaders = new DocIdSetIterator[numDimensions]; + Set skipStarNodeCreationForDimensions = this.starTreeFieldSpec.getSkipStarNodeCreationInDims(); + + for (int i = 0; i < numDimensions; i++) { + String dimension = dimensionsSplitOrder.get(i).getField(); + this.dimensionsSplitOrder[i] = dimension; + if (skipStarNodeCreationForDimensions.contains(dimensionsSplitOrder.get(i).getField())) { + this.skipStarNodeCreationForDimensions.add(i); + } + FieldInfo dimensionFieldInfos = state.fieldInfos.fieldInfo(dimension); + DocValuesType dimensionDocValuesType = state.fieldInfos.fieldInfo(dimension).getDocValuesType(); + dimensionReaders[i] = starTreeDocValuesIteratorFactory.createIterator( + dimensionDocValuesType, + dimensionFieldInfos, + docValuesProducer + ); + } + + this.metricStatFieldPairs = generateMetricStatFieldPairs(); + this.numMetrics = metricStatFieldPairs.size(); + this.metrics = new String[numMetrics]; + this.valueAggregators = new ValueAggregator[numMetrics]; + this.numericTypes = new IndexNumericFieldData.NumericType[numMetrics]; + this.metricReaders = new DocIdSetIterator[numMetrics]; + + int index = 0; + for (MetricStatFieldPair metricStatFieldPair : metricStatFieldPairs) { + metrics[index] = metricStatFieldPair.toFieldName(); + valueAggregators[index] = ValueAggregatorFactory.getValueAggregator(metricStatFieldPair.getMetricStat()); + + Mapper fieldMapper = mapperService.documentMapper().mappers().getMapper(metrics[index]); + if (fieldMapper instanceof NumberFieldMapper) { + numericTypes[index] = ((NumberFieldMapper) fieldMapper).fieldType().numericType(); + } else { + numericTypes[index] = IndexNumericFieldData.NumericType.DOUBLE; + } + // Ignore the column for COUNT aggregation function + if (valueAggregators[index].getAggregationType() != MetricStat.COUNT) { + String metricName = metricStatFieldPair.getField(); + FieldInfo metricFieldInfos = state.fieldInfos.fieldInfo(metricName); + DocValuesType metricDocValuesType = state.fieldInfos.fieldInfo(metricName).getDocValuesType(); + metricReaders[index] = starTreeDocValuesIteratorFactory.createIterator( + metricDocValuesType, + metricFieldInfos, + docValuesProducer + ); + } + index++; + } + this.maxLeafDocuments = starTreeFieldSpec.maxLeafDocs(); + } + + /** + * Generates the MetricStatFieldPairs for all the metrics on a field + * + * @return list of metric stat mapped with respective fields + */ + public List generateMetricStatFieldPairs() { + List metricStatFieldPairs = new ArrayList<>(); + for (Metric metric : this.starTreeField.getMetrics()) { + for (MetricStat metricType : metric.getMetrics()) { + MetricStatFieldPair metricStatFieldPair = new MetricStatFieldPair(metricType, metric.getField()); + metricStatFieldPairs.add(metricStatFieldPair); + } + } + return metricStatFieldPairs; + } + + /** + * Appends a star-tree document to the star-tree. + * + * @param starTreeDocument star tree document to be appended + */ + public abstract void appendStarTreeDocument(StarTreeDocument starTreeDocument) throws IOException; + + /** + * Returns the star-tree document of the given document id in the star-tree. + * + * @param docId Document dd + * @return Star tree document + */ + public abstract StarTreeDocument getStarTreeDocument(int docId) throws IOException; + + /** + * Returns the star-tree document of the given document id in the star-tree. + * + * @return Star tree document + */ + public abstract List getStarTreeDocuments() throws IOException; + + /** + * Returns the dimension value of the given document and dimension id in the star-tree. + * + * @param docId Document Id + * @param dimensionId Dimension Id + * @return Dimension value + */ + public abstract long getDimensionValue(int docId, int dimensionId) throws IOException; + + /** + * Sorts and aggregates the star-tree Document in the segment, and returns a star-tree document iterator for all the + * aggregated star-tree document. + * + *

This method reads star-tree document from segment and generates the initial star-tree document for the star-tree. + * + * @param numDocs Number of documents in the segment + * @return Iterator for the aggregated star-tree document + */ + public abstract Iterator processSegmentStarTreeDocuments(int numDocs) throws IOException; + + /** + * Generates aggregated star-tree document for star-node. + * + *

This method will do the following steps: + * + *

    + *
  • Creates a temporary buffer for the given range of documents + *
  • Replaces the value for the given dimension Id to {@code STAR} + *
  • Sorts the starTreeDocument inside the temporary buffer + *
  • Aggregates the starTreeDocument with same dimensions + *
  • Returns an iterator for the aggregated starTreeDocument + *
+ * + * @param startDocId Start document id in the star-tree + * @param endDocId End document id (exclusive) in the star-tree + * @param dimensionId Dimension id of the star-node + * @return Iterator for the aggregated starTreeDocument + */ + public abstract Iterator generateStarTreeForStarNode(int startDocId, int endDocId, int dimensionId) + throws IOException; + + /** + * Returns the next segment star-tree document + */ + protected StarTreeDocument getNextSegmentStarTreeDocument() throws IOException { + long[] dimensions = getNextSegmentStarTreeDocumentDimensions(); + Object[] metrics = getNextSegmentStarTreeDocumentMetrics(); + return new StarTreeDocument(dimensions, metrics); + } + + /** + * Returns the next segment star-tree document for the dimensions + * + * @return dimension values for each of the star-tree dimension + * @throws IOException when we are unable to iterate to the next doc + */ + long[] getNextSegmentStarTreeDocumentDimensions() throws IOException { + long[] dimensions = new long[numDimensions]; + for (int i = 0; i < numDimensions; i++) { + try { + dimensionReaders[i].nextDoc(); + } catch (IOException e) { + logger.error("unable to iterate to next doc", e); + } + + if (starTreeField.getDimensionsOrder().get(i) instanceof DateDimension) { + dimensions[i] = handleDateDimension( + dimensionsSplitOrder[i], + starTreeDocValuesIteratorFactory.getNextValue(dimensionReaders[i]) + ); + } else { + dimensions[i] = starTreeDocValuesIteratorFactory.getNextValue(dimensionReaders[i]); + } + } + return dimensions; + } + + /** + * Returns the next segment star-tree document for the metrics + * + * @return metric values for each of the star-tree metric + * @throws IOException when we are unable to iterate to the next doc + */ + private Object[] getNextSegmentStarTreeDocumentMetrics() throws IOException { + Object[] metrics = new Object[numMetrics]; + for (int i = 0; i < numMetrics; i++) { + // Ignore the column for COUNT aggregation function + if (metricReaders[i] != null) { + try { + metricReaders[i].nextDoc(); + } catch (IOException e) { + // TODO : handle null values in columns + logger.error("unable to iterate to next doc", e); + } + metrics[i] = starTreeDocValuesIteratorFactory.getNextValue(metricReaders[i]); + } + } + return metrics; + } + + /** + * Merges a segment star-tree document (raw) into the aggregated star-tree document. + * + *

Will create a new aggregated star-tree document if the current one is {@code null}. + * + * @param aggregatedStarTreeDocument Aggregated star-tree document + * @param segmentStarTreeDocument Segment star-tree document + * @return Merged starTreeDocument + */ + protected StarTreeDocument aggregateSegmentStarTreeDocument( + StarTreeDocument aggregatedStarTreeDocument, + StarTreeDocument segmentStarTreeDocument + ) { + // TODO: HANDLE KEYWORDS LATER! + if (aggregatedStarTreeDocument == null) { + long[] dimensions = Arrays.copyOf(segmentStarTreeDocument.dimensions, numDimensions); + Object[] metrics = new Object[numMetrics]; + for (int i = 0; i < numMetrics; i++) { + try { + StarTreeNumericType numericType = StarTreeNumericType.fromNumericType(numericTypes[i]); + metrics[i] = valueAggregators[i].getInitialAggregatedValue((Long) segmentStarTreeDocument.metrics[i], numericType); + } catch (IllegalArgumentException | NullPointerException e) { + logger.error("Cannot parse initial aggregated value", e); + throw new IllegalArgumentException( + "Cannot parse initial aggregated value [" + segmentStarTreeDocument.metrics[i] + "]" + ); + } + } + return new StarTreeDocument(dimensions, metrics); + } else { + for (int i = 0; i < numMetrics; i++) { + try { + StarTreeNumericType numericType = StarTreeNumericType.fromNumericType(numericTypes[i]); + aggregatedStarTreeDocument.metrics[i] = valueAggregators[i].applySegmentRawValue( + aggregatedStarTreeDocument.metrics[i], + (Long) segmentStarTreeDocument.metrics[i], + numericType + ); + } catch (IllegalArgumentException | NullPointerException e) { + logger.error("Cannot apply segment raw value", e); + throw new IllegalArgumentException("Cannot aggregate on segment value [" + segmentStarTreeDocument.metrics[i] + "]"); + } + } + return aggregatedStarTreeDocument; + } + } + + /** + * Merges a star-tree document (aggregated) into the aggregated document. + * + *

Will create a new aggregated starTreeDocument if the current one is {@code null}. + * + * @param aggregatedStarTreeDocument Aggregated star-tree document + * @param starTreeStarTreeDocument Star-tree document + * @return Merged star-tree document + */ + public StarTreeDocument aggregateStarTreeDocument( + StarTreeDocument aggregatedStarTreeDocument, + StarTreeDocument starTreeStarTreeDocument + ) { + // aggregate the documents + if (aggregatedStarTreeDocument == null) { + long[] dimensions = Arrays.copyOf(starTreeStarTreeDocument.dimensions, numDimensions); + Object[] metrics = new Object[numMetrics]; + for (int i = 0; i < numMetrics; i++) { + try { + metrics[i] = valueAggregators[i].cloneAggregatedValue(starTreeStarTreeDocument.metrics[i]); + } catch (IllegalArgumentException | NullPointerException e) { + logger.error("Cannot clone aggregated value", e); + throw new IllegalArgumentException("Cannot clone aggregated value [" + starTreeStarTreeDocument.metrics[i] + "]"); + } + } + return new StarTreeDocument(dimensions, metrics); + } else { + for (int i = 0; i < numMetrics; i++) { + try { + aggregatedStarTreeDocument.metrics[i] = valueAggregators[i].applyAggregatedValue( + starTreeStarTreeDocument.metrics[i], + aggregatedStarTreeDocument.metrics[i] + ); + } catch (IllegalArgumentException | NullPointerException e) { + logger.error("Cannot apply aggregated value", e); + throw new IllegalArgumentException("Cannot apply aggregated value [" + starTreeStarTreeDocument.metrics[i] + "]"); + } + } + return aggregatedStarTreeDocument; + } + } + + // TODO: This will be taken care in off heap implementation for merges + // public abstract void build(List starTreeValues) throws IOException; + + public void build() throws IOException { + long startTime = System.currentTimeMillis(); + logger.info("Tree of Aggregations build is a go with config {}", starTreeField); + + Iterator starTreeDocumentIterator = processSegmentStarTreeDocuments(totalDocs); + logger.info("Sorting and aggregating star-tree in ms : {}", (System.currentTimeMillis() - startTime)); + build(starTreeDocumentIterator); + logger.info("Finished Building TOA in ms : {}", (System.currentTimeMillis() - startTime)); + } + + /** + * Builds the star tree using Star-Tree Document + * + * @param starTreeDocumentIterator contains the sorted and aggregated documents + * @throws IOException when we are unable to build star-tree + */ + public void build(Iterator starTreeDocumentIterator) throws IOException { + int numSegmentStarTreeDocument = totalDocs; + + while (starTreeDocumentIterator.hasNext()) { + appendToStarTree(starTreeDocumentIterator.next()); + } + int numStarTreeDocument = numDocs; + logger.info("Generated star tree docs : [{}] from segment docs : [{}]", numStarTreeDocument, numSegmentStarTreeDocument); + + if (numDocs == 0) { + // TODO: Uncomment when segment codec is ready + // StarTreeBuilderUtils.serializeTree(indexOutput, rootNode, dimensionsSplitOrder, numNodes); + return; + } + + constructStarTree(rootNode, 0, numDocs); + int numStarTreeDocumentUnderStarNode = numDocs - numStarTreeDocument; + logger.info( + "Finished constructing star-tree, got [ {} ] tree nodes and [ {} ] starTreeDocument under star-node", + numNodes, + numStarTreeDocumentUnderStarNode + ); + + createAggregatedDocs(rootNode); + int numAggregatedStarTreeDocument = numDocs - numStarTreeDocument - numStarTreeDocumentUnderStarNode; + logger.info("Finished creating aggregated documents : {}", numAggregatedStarTreeDocument); + + // Create doc values indices in disk + // TODO: Uncomment when segment codec is ready + // createSortedDocValuesIndices(docValuesConsumer); + + // Serialize and save in disk + // TODO: Uncomment when segment codec is ready + // StarTreeBuilderUtils.serializeTree(indexOutput, rootNode, dimensionsSplitOrder, numNodes); + + // TODO: Write star tree metadata for off heap implementation + + } + + /** + * Appends a starTreeDocument to star tree + * + * @param starTreeDocument star-tree document + * @throws IOException throws an exception if we are unable to append the doc + */ + private void appendToStarTree(StarTreeDocument starTreeDocument) throws IOException { + appendStarTreeDocument(starTreeDocument); + numDocs++; + } + + /** + * Returns a new node + * + * @return return new star-tree node + */ + private StarTreeBuilderUtils.TreeNode getNewNode() { + numNodes++; + return new StarTreeBuilderUtils.TreeNode(); + } + + /** + * Implements the algorithm to construct a star-tree based on star-tree documents + * + * @param node star-tree node + * @param startDocId start document id + * @param endDocId end document id + * @throws IOException throws an exception if we are unable to construct the tree + */ + private void constructStarTree(StarTreeBuilderUtils.TreeNode node, int startDocId, int endDocId) throws IOException { + + int childDimensionId = node.dimensionId + 1; + if (childDimensionId == numDimensions) { + return; + } + + // Construct all non-star children nodes + node.childDimensionId = childDimensionId; + Map children = constructNonStarNodes(startDocId, endDocId, childDimensionId); + node.children = children; + + // Construct star-node if required + if (!skipStarNodeCreationForDimensions.contains(childDimensionId) && children.size() > 1) { + children.put((long) StarTreeBuilderUtils.ALL, constructStarNode(startDocId, endDocId, childDimensionId)); + } + + // Further split on child nodes if required + for (StarTreeBuilderUtils.TreeNode child : children.values()) { + if (child.endDocId - child.startDocId > maxLeafDocuments) { + constructStarTree(child, child.startDocId, child.endDocId); + } + } + } + + /** + * Constructs non star tree nodes + * + * @param startDocId start document id + * @param endDocId end document id + * @param dimensionId id of the dimension in the star tree + * @return root node with non-star nodes constructed + * @throws IOException throws an exception if we are unable to construct non-star nodes + */ + private Map constructNonStarNodes(int startDocId, int endDocId, int dimensionId) + throws IOException { + Map nodes = new HashMap<>(); + int nodeStartDocId = startDocId; + long nodeDimensionValue = getDimensionValue(startDocId, dimensionId); + for (int i = startDocId + 1; i < endDocId; i++) { + long dimensionValue = getDimensionValue(i, dimensionId); + if (dimensionValue != nodeDimensionValue) { + StarTreeBuilderUtils.TreeNode child = getNewNode(); + child.dimensionId = dimensionId; + child.dimensionValue = nodeDimensionValue; + child.startDocId = nodeStartDocId; + child.endDocId = i; + nodes.put(nodeDimensionValue, child); + + nodeStartDocId = i; + nodeDimensionValue = dimensionValue; + } + } + StarTreeBuilderUtils.TreeNode lastNode = getNewNode(); + lastNode.dimensionId = dimensionId; + lastNode.dimensionValue = nodeDimensionValue; + lastNode.startDocId = nodeStartDocId; + lastNode.endDocId = endDocId; + nodes.put(nodeDimensionValue, lastNode); + return nodes; + } + + /** + * Constructs star tree nodes + * + * @param startDocId start document id + * @param endDocId end document id + * @param dimensionId id of the dimension in the star tree + * @return root node with star nodes constructed + * @throws IOException throws an exception if we are unable to construct non-star nodes + */ + private StarTreeBuilderUtils.TreeNode constructStarNode(int startDocId, int endDocId, int dimensionId) throws IOException { + StarTreeBuilderUtils.TreeNode starNode = getNewNode(); + starNode.dimensionId = dimensionId; + starNode.dimensionValue = StarTreeBuilderUtils.ALL; + starNode.startDocId = numDocs; + Iterator starTreeDocumentIterator = generateStarTreeForStarNode(startDocId, endDocId, dimensionId); + while (starTreeDocumentIterator.hasNext()) { + appendToStarTree(starTreeDocumentIterator.next()); + } + starNode.endDocId = numDocs; + return starNode; + } + + /** + * Returns aggregated star-tree document + * + * @param node star-tree node + * @return aggregated star-tree documents + * @throws IOException throws an exception upon failing to create new aggregated docs based on star tree + */ + private StarTreeDocument createAggregatedDocs(StarTreeBuilderUtils.TreeNode node) throws IOException { + StarTreeDocument aggregatedStarTreeDocument = null; + if (node.children == null) { + // For leaf node + + if (node.startDocId == node.endDocId - 1) { + // If it has only one document, use it as the aggregated document + aggregatedStarTreeDocument = getStarTreeDocument(node.startDocId); + node.aggregatedDocId = node.startDocId; + } else { + // If it has multiple documents, aggregate all of them + for (int i = node.startDocId; i < node.endDocId; i++) { + aggregatedStarTreeDocument = aggregateStarTreeDocument(aggregatedStarTreeDocument, getStarTreeDocument(i)); + } + assert aggregatedStarTreeDocument != null; + for (int i = node.dimensionId + 1; i < numDimensions; i++) { + aggregatedStarTreeDocument.dimensions[i] = STAR_IN_DOC_VALUES_INDEX; + } + node.aggregatedDocId = numDocs; + appendToStarTree(aggregatedStarTreeDocument); + } + } else { + // For non-leaf node + if (node.children.containsKey((long) StarTreeBuilderUtils.ALL)) { + // If it has star child, use the star child aggregated document directly + for (StarTreeBuilderUtils.TreeNode child : node.children.values()) { + if (child.dimensionValue == StarTreeBuilderUtils.ALL) { + aggregatedStarTreeDocument = createAggregatedDocs(child); + node.aggregatedDocId = child.aggregatedDocId; + } else { + createAggregatedDocs(child); + } + } + } else { + // If no star child exists, aggregate all aggregated documents from non-star children + for (StarTreeBuilderUtils.TreeNode child : node.children.values()) { + aggregatedStarTreeDocument = aggregateStarTreeDocument(aggregatedStarTreeDocument, createAggregatedDocs(child)); + } + assert aggregatedStarTreeDocument != null; + for (int i = node.dimensionId + 1; i < numDimensions; i++) { + aggregatedStarTreeDocument.dimensions[i] = STAR_IN_DOC_VALUES_INDEX; + } + node.aggregatedDocId = numDocs; + appendToStarTree(aggregatedStarTreeDocument); + } + } + return aggregatedStarTreeDocument; + } + + /** + * Handles the dimension of date time field type + * + * @param fieldName name of the field + * @param val value of the field + * @return returns the converted dimension of the field to a particular granularity + */ + private long handleDateDimension(final String fieldName, final long val) { + // TODO: handle timestamp granularity + return val; + } + + public void close() throws IOException { + // boolean success = false; + // try { + // if (indexOutput != null) { + // indexOutput.writeInt(-1); + // CodecUtil.writeFooter(indexOutput); // write checksum + // } + // success = true; + // } catch (Exception e) { + // throw new RuntimeException(e); + // } finally { + // if (success) { + // IOUtils.close(indexOutput); + // } else { + // IOUtils.closeWhileHandlingException(indexOutput); + // } + // indexOutput = null; + // } + } + +} diff --git a/server/src/main/java/org/apache/lucene/index/StarTreeDocValuesWriter.java b/server/src/main/java/org/apache/lucene/index/StarTreeDocValuesWriter.java new file mode 100644 index 0000000000000..e8d96226da2ee --- /dev/null +++ b/server/src/main/java/org/apache/lucene/index/StarTreeDocValuesWriter.java @@ -0,0 +1,37 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.apache.lucene.index; + +/** + * A wrapper for {@link DocValuesWriter} that contains the {@link DocValuesType} of the doc + */ +public class StarTreeDocValuesWriter { + + private final DocValuesType docValuesType; + private final DocValuesWriter docValuesWriter; + + public StarTreeDocValuesWriter(DocValuesType docValuesType, DocValuesWriter docValuesWriter) { + this.docValuesType = docValuesType; + this.docValuesWriter = docValuesWriter; + } + + /** + * Get the doc values type + */ + public DocValuesType getDocValuesType() { + return docValuesType; + } + + /** + * Get the doc values writer + */ + public DocValuesWriter getDocValuesWriter() { + return docValuesWriter; + } +} diff --git a/server/src/main/java/org/opensearch/common/inject/util/Modules.java b/server/src/main/java/org/opensearch/common/inject/util/Modules.java index ae37cb3d29a68..b5a5a83ac3af9 100644 --- a/server/src/main/java/org/opensearch/common/inject/util/Modules.java +++ b/server/src/main/java/org/opensearch/common/inject/util/Modules.java @@ -202,7 +202,7 @@ public Void visit(Binding binding) { if (!overriddenKeys.remove(binding.getKey())) { super.visit(binding); - // Record when a scope instance is used in a binding + // record when a scope instance is used in a binding Scope scope = getScopeInstanceOrNull(binding); if (scope != null) { scopeInstancesInUse.put(scope, binding.getSource()); diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/MetricStat.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/MetricStat.java index fbde296b15f7e..a5a6d07ad9bbc 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/MetricStat.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/MetricStat.java @@ -21,7 +21,8 @@ public enum MetricStat { AVG("avg"), SUM("sum"), MIN("min"), - MAX("max"); + MAX("max"), + UNSUPPORTED("unsupported"); private final String typeName; diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricStatFieldPair.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricStatFieldPair.java new file mode 100644 index 0000000000000..3fb041373ff51 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricStatFieldPair.java @@ -0,0 +1,115 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.index.compositeindex.datacube.startree.aggregators; + +import org.opensearch.index.compositeindex.datacube.MetricStat; + +import java.util.Comparator; + +/** + * Builds aggregation function and doc values field pair to support various aggregations + * @opensearch.experimental + */ +public class MetricStatFieldPair implements Comparable { + + public static final String DELIMITER = "__"; + public static final String STAR = "*"; + public static final MetricStatFieldPair COUNT_STAR = new MetricStatFieldPair(MetricStat.COUNT, STAR); + + private final MetricStat metricStat; + private final String field; + + /** + * Constructor for MetricStatFieldPair + */ + public MetricStatFieldPair(MetricStat metricStat, String field) { + this.metricStat = metricStat; + if (metricStat == MetricStat.COUNT) { + this.field = STAR; + } else { + this.field = field; + } + } + + /** + * @return Metric Type + */ + public MetricStat getMetricStat() { + return metricStat; + } + + /** + * @return field Name + */ + public String getField() { + return field; + } + + /** + * @return field name with metric type and field + */ + public String toFieldName() { + return toFieldName(metricStat, field); + } + + /** + * Builds field name with metric type and field + */ + public static String toFieldName(MetricStat metricType, String field) { + return metricType.getTypeName() + DELIMITER + field; + } + + /** + * Builds MetricStatFieldPair from field name + */ + public static MetricStatFieldPair fromFieldName(String fieldName) { + String[] parts = fieldName.split(DELIMITER, 2); + return fromMetricAndFieldName(parts[0], parts[1]); + } + + /** + * Builds MetricStatFieldPair from metric and field name + */ + private static MetricStatFieldPair fromMetricAndFieldName(String metricName, String fieldName) { + MetricStat metricType = MetricStat.fromTypeName(metricName); + if (metricType == MetricStat.COUNT) { + return COUNT_STAR; + } else { + return new MetricStatFieldPair(metricType, fieldName); + } + } + + @Override + public int hashCode() { + return 31 * metricStat.hashCode() + field.hashCode(); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj instanceof MetricStatFieldPair) { + MetricStatFieldPair anotherPair = (MetricStatFieldPair) obj; + return metricStat == anotherPair.metricStat && field.equals(anotherPair.field); + } + return false; + } + + @Override + public String toString() { + return toFieldName(); + } + + @Override + public int compareTo(MetricStatFieldPair other) { + return Comparator.comparing((MetricStatFieldPair o) -> o.field) + .thenComparing((MetricStatFieldPair o) -> o.metricStat) + .compare(this, other); + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregator.java new file mode 100644 index 0000000000000..a3a14f9bbb11a --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregator.java @@ -0,0 +1,82 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.index.compositeindex.datacube.startree.aggregators; + +import org.apache.lucene.util.NumericUtils; +import org.opensearch.index.compositeindex.datacube.MetricStat; +import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; +import org.opensearch.index.compositeindex.datacube.startree.data.DataType; +import org.opensearch.search.aggregations.metrics.CompensatedSum; + +/** + * Sum value aggregator for star tree + * + * @opensearch.internal + */ +public class SumValueAggregator implements ValueAggregator { + public static final DataType AGGREGATED_VALUE_TYPE = DataType.DOUBLE; + + @Override + public MetricStat getAggregationType() { + return MetricStat.SUM; + } + + @Override + public DataType getAggregatedValueType() { + return AGGREGATED_VALUE_TYPE; + } + + @Override + public Double getInitialAggregatedValue(Long rawValue, StarTreeNumericType starTreeNumericType) { + return starTreeNumericType.getDoubleValue(rawValue); + } + + @Override + public Double applySegmentRawValue(Double value, Long rawValue, StarTreeNumericType starTreeNumericType) { + CompensatedSum kahanSummation = new CompensatedSum(0, 0); + kahanSummation.add(value); + kahanSummation.add(starTreeNumericType.getDoubleValue(rawValue)); + return kahanSummation.value(); + } + + @Override + public Double applyAggregatedValue(Double value, Double aggregatedValue) { + CompensatedSum kahanSummation = new CompensatedSum(0, 0); + kahanSummation.add(value); + kahanSummation.add(aggregatedValue); + return kahanSummation.value(); + } + + @Override + public Double cloneAggregatedValue(Double value) { + return value; + } + + @Override + public int getMaxAggregatedValueByteSize() { + return Double.BYTES; + } + + @Override + public Long convertAggregationTypeToSortableLongValue(Double value) { + try { + return NumericUtils.doubleToSortableLong(value); + } catch (IllegalArgumentException | NullPointerException | IllegalStateException e) { + throw new IllegalArgumentException("Cannot convert " + value + " to sortable long", e); + } + } + + @Override + public Double convertSortableLongToAggregatedTypeValue(Long value, StarTreeNumericType type) { + try { + return type.getDoubleValue(value); + } catch (IllegalArgumentException | NullPointerException | IllegalStateException e) { + throw new IllegalArgumentException("Cannot convert " + value + " to sortable aggregation type", e); + } + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregator.java new file mode 100644 index 0000000000000..b9fa3d5b69a3f --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregator.java @@ -0,0 +1,64 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.index.compositeindex.datacube.startree.aggregators; + +import org.opensearch.index.compositeindex.datacube.MetricStat; +import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; +import org.opensearch.index.compositeindex.datacube.startree.data.DataType; + +/** + * A value aggregator that pre-aggregates on the input values for a specific type of aggregation. + * @opensearch.experimental + */ +public interface ValueAggregator { + + /** + * Returns the type of the aggregation. + */ + MetricStat getAggregationType(); + + /** + * Returns the data type of the aggregated value. + */ + DataType getAggregatedValueType(); + + /** + * Returns the initial aggregated value. + */ + A getInitialAggregatedValue(Long rawValue, StarTreeNumericType starTreeNumericType); + + /** + * Applies a raw value to the current aggregated value. + */ + A applySegmentRawValue(A value, Long rawValue, StarTreeNumericType starTreeNumericType); + + /** + * Applies an aggregated value to the current aggregated value. + */ + A applyAggregatedValue(A value, A aggregatedValue); + + /** + * Clones an aggregated value. + */ + A cloneAggregatedValue(A value); + + /** + * Returns the maximum size in bytes of the aggregated values seen so far. + */ + int getMaxAggregatedValueByteSize(); + + /** + * Converts an aggregated value into a Long type. + */ + Long convertAggregationTypeToSortableLongValue(A value); + + /** + * Converts an aggregated value from a Long type. + */ + A convertSortableLongToAggregatedTypeValue(Long rawValue, StarTreeNumericType type); +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactory.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactory.java new file mode 100644 index 0000000000000..523b3fbec2e7c --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactory.java @@ -0,0 +1,51 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.index.compositeindex.datacube.startree.aggregators; + +import org.opensearch.index.compositeindex.datacube.MetricStat; +import org.opensearch.index.compositeindex.datacube.startree.data.DataType; + +/** + * Value aggregator factory for a given aggregation type + * @opensearch.experimental + */ +public class ValueAggregatorFactory { + private ValueAggregatorFactory() {} + + /** + * Returns a new instance of value aggregator for the given aggregation type. + * + * @param aggregationType Aggregation type + * @return Value aggregator + */ + public static ValueAggregator getValueAggregator(MetricStat aggregationType) { + switch (aggregationType) { + // other metric types (count, min, max, avg) will be supported in the future + case SUM: + return new SumValueAggregator(); + default: + throw new IllegalStateException("Unsupported aggregation type: " + aggregationType); + } + } + + /** + * Returns the data type of the aggregated value for the given aggregation type. + * + * @param aggregationType Aggregation type + * @return Data type of the aggregated value + */ + public static DataType getAggregatedValueType(MetricStat aggregationType) { + switch (aggregationType) { + // other metric types (count, min, max, avg) will be supported in the future + case SUM: + return SumValueAggregator.AGGREGATED_VALUE_TYPE; + default: + throw new IllegalStateException("Unsupported aggregation type: " + aggregationType); + } + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericType.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericType.java new file mode 100644 index 0000000000000..ed41445ac5cdf --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericType.java @@ -0,0 +1,45 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype; + +import org.opensearch.index.fielddata.IndexNumericFieldData; + +import java.util.function.Function; + +public enum StarTreeNumericType { + HALF_FLOAT(IndexNumericFieldData.NumericType.HALF_FLOAT, StarTreeNumericTypeConverters::halfFloatPointToDouble), + FLOAT(IndexNumericFieldData.NumericType.FLOAT, StarTreeNumericTypeConverters::floatPointToDouble), + LONG(IndexNumericFieldData.NumericType.LONG, StarTreeNumericTypeConverters::longToDouble), + DOUBLE(IndexNumericFieldData.NumericType.DOUBLE, StarTreeNumericTypeConverters::sortableLongtoDouble); + + final IndexNumericFieldData.NumericType numericType; + final Function converter; + + StarTreeNumericType(IndexNumericFieldData.NumericType numericType, Function converter) { + this.numericType = numericType; + this.converter = converter; + } + + public double getDoubleValue(long rawValue) { + return this.converter.apply(rawValue); + } + + public static StarTreeNumericType fromNumericType(IndexNumericFieldData.NumericType numericType) { + switch (numericType) { + case HALF_FLOAT: + return StarTreeNumericType.HALF_FLOAT; + case FLOAT: + return StarTreeNumericType.FLOAT; + case LONG: + return StarTreeNumericType.LONG; + default: + return StarTreeNumericType.DOUBLE; + } + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericTypeConverters.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericTypeConverters.java new file mode 100644 index 0000000000000..59a2c3418e931 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericTypeConverters.java @@ -0,0 +1,31 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype; + +import org.apache.lucene.sandbox.document.HalfFloatPoint; +import org.apache.lucene.util.NumericUtils; + +public class StarTreeNumericTypeConverters { + + public static double halfFloatPointToDouble(Long value) { + return HalfFloatPoint.sortableShortToHalfFloat((short) value.longValue()); + } + + public static double floatPointToDouble(Long value) { + return NumericUtils.sortableIntToFloat((int) value.longValue()); + } + + public static double longToDouble(Long value) { + return (double) value; + } + + public static Double sortableLongtoDouble(Long value) { + return NumericUtils.sortableLongToDouble(value); + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/package-info.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/package-info.java new file mode 100644 index 0000000000000..b9a9bb8f1427d --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/package-info.java @@ -0,0 +1,13 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * Numeric Types for Composite Index Star Tree + * @opensearch.experimental + */ +package org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype; diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/package-info.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/package-info.java new file mode 100644 index 0000000000000..27565ffded2cf --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/package-info.java @@ -0,0 +1,13 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * Aggregators for Composite Index Star Tree + * @opensearch.experimental + */ +package org.opensearch.index.compositeindex.datacube.startree.aggregators; diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/DocValuesIteratorFactory.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/DocValuesIteratorFactory.java new file mode 100644 index 0000000000000..66302f84f1449 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/DocValuesIteratorFactory.java @@ -0,0 +1,38 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube.startree.builder; + +import org.apache.lucene.codecs.DocValuesProducer; +import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.search.DocIdSetIterator; + +import java.io.IOException; + +/** + * An interface to support iterators for various doc values types. + * @opensearch.experimental + */ +public interface DocValuesIteratorFactory { + + /** + * Creates an iterator for the given doc values type and field using the doc values producer + */ + DocIdSetIterator createIterator(DocValuesType type, FieldInfo field, DocValuesProducer producer) throws IOException; + + /** + * Returns the next value for the given iterator + */ + long getNextValue(DocIdSetIterator iterator) throws IOException; + + /** + * Returns the doc id for the next document from the given iterator + */ + int nextDoc(DocIdSetIterator iterator) throws IOException; +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/MultipleTreesBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/MultipleTreesBuilder.java new file mode 100644 index 0000000000000..ad3d33474840f --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/MultipleTreesBuilder.java @@ -0,0 +1,110 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube.startree.builder; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.lucene.codecs.DocValuesConsumer; +import org.apache.lucene.codecs.DocValuesProducer; +import org.apache.lucene.index.SegmentWriteState; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; +import org.opensearch.index.mapper.MapperService; + +import java.io.Closeable; +import java.io.IOException; +import java.util.List; +import java.util.Locale; + +public class MultipleTreesBuilder implements Closeable { + + private static final Logger logger = LogManager.getLogger(MultipleTreesBuilder.class); + + private final List starTreeFields; + private final StarTreeFieldConfiguration.StarTreeBuildMode buildMode; + private final DocValuesConsumer docValuesConsumer; + private final SegmentWriteState state; + private final DocValuesProducer docValuesProducer; + private final MapperService mapperService; + + public MultipleTreesBuilder( + List starTreeFields, + StarTreeFieldConfiguration.StarTreeBuildMode buildMode, + DocValuesProducer docValuesProducer, + DocValuesConsumer docValuesConsumer, + SegmentWriteState segmentWriteState, + MapperService mapperService + ) { + this.starTreeFields = starTreeFields; + if (starTreeFields == null || starTreeFields.isEmpty()) { + throw new IllegalArgumentException("Must provide star-tree builder configs"); + } + this.buildMode = buildMode; + this.docValuesProducer = docValuesProducer; + this.docValuesConsumer = docValuesConsumer; + this.state = segmentWriteState; + this.mapperService = mapperService; + } + + /** + * Builds the star-trees. + */ + public void build() throws Exception { + long startTime = System.currentTimeMillis(); + int numStarTrees = starTreeFields.size(); + logger.info("Starting building {} star-trees with configs: {} using {} builder", numStarTrees, starTreeFields, buildMode); + + // Build all star-trees + for (int i = 0; i < numStarTrees; i++) { + StarTreeField starTreeField = starTreeFields.get(i); + try ( + SingleTreeBuilder singleTreeBuilder = getSingleTreeBuilder( + starTreeField, + buildMode, + docValuesProducer, + docValuesConsumer, + state, + mapperService + ) + ) { + singleTreeBuilder.build(); + } + } + logger.info( + "Took {} ms to building {} star-trees with configs: {} using {} builder", + System.currentTimeMillis() - startTime, + numStarTrees, + starTreeFields, + buildMode + ); + } + + @Override + public void close() throws IOException { + + } + + private static SingleTreeBuilder getSingleTreeBuilder( + StarTreeField starTreeField, + StarTreeFieldConfiguration.StarTreeBuildMode buildMode, + DocValuesProducer docValuesProducer, + DocValuesConsumer docValuesConsumer, + SegmentWriteState state, + MapperService mapperService + ) throws IOException { + switch (buildMode) { + case ON_HEAP: + return new OnHeapSingleTreeBuilder(starTreeField, docValuesProducer, docValuesConsumer, state, mapperService); + default: + throw new IllegalArgumentException( + String.format(Locale.ROOT, "No star tree implementation is available for [%s] build mode", buildMode) + ); + } + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapSingleTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapSingleTreeBuilder.java new file mode 100644 index 0000000000000..6baaedb1d0710 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapSingleTreeBuilder.java @@ -0,0 +1,204 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.index.compositeindex.datacube.startree.builder; + +import org.apache.lucene.codecs.DocValuesConsumer; +import org.apache.lucene.codecs.DocValuesProducer; +import org.apache.lucene.index.BaseSingleStarTreeBuilder; +import org.apache.lucene.index.SegmentWriteState; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; +import org.opensearch.index.compositeindex.datacube.startree.data.StarTreeDocument; +import org.opensearch.index.mapper.MapperService; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; + +/** + * On heap single tree builder + */ +public class OnHeapSingleTreeBuilder extends BaseSingleStarTreeBuilder { + + private final List starTreeDocuments = new ArrayList<>(); + + /** + * Constructor for OnHeapSingleTreeBuilder + * + * @param starTreeField star-tree field + * @param docValuesProducer document values producer + * @param docValuesConsumer document values consumer + * @param segmentWriteState segment write state + * @param mapperService + * @throws IOException throws an exception we are unable to construct an onheap star-tree + */ + public OnHeapSingleTreeBuilder( + StarTreeField starTreeField, + DocValuesProducer docValuesProducer, + DocValuesConsumer docValuesConsumer, + SegmentWriteState segmentWriteState, + MapperService mapperService + ) throws IOException { + super(starTreeField, docValuesProducer, docValuesConsumer, segmentWriteState, mapperService); + } + + @Override + public void appendStarTreeDocument(StarTreeDocument starTreeDocument) throws IOException { + starTreeDocuments.add(starTreeDocument); + } + + @Override + public StarTreeDocument getStarTreeDocument(int docId) throws IOException { + return starTreeDocuments.get(docId); + } + + @Override + public List getStarTreeDocuments() throws IOException { + return starTreeDocuments; + } + + // TODO: should this be just long? + @Override + public long getDimensionValue(int docId, int dimensionId) throws IOException { + return starTreeDocuments.get(docId).dimensions[dimensionId]; + } + + // Handles star-tree rebuilds during merges :) + // @Override + // public void build(List starTreeValues) throws IOException { + // TODO: This will be handled during off heap implementation for merges + // } + + @Override + public Iterator processSegmentStarTreeDocuments(int numDocs) throws IOException { + StarTreeDocument[] starTreeDocuments = new StarTreeDocument[numDocs]; + for (int i = 0; i < numDocs; i++) { + starTreeDocuments[i] = getNextSegmentStarTreeDocument(); + } + return processStarTreeDocuments(starTreeDocuments); + } + + /** + * Sort, aggregates and merges the star-tree documents + * @param starTreeDocuments star-tree documents + * @return iterator for star-tree documents + * @throws IOException throws when unable to sort, merge and aggregate star-tree documents + */ + public Iterator processStarTreeDocuments(StarTreeDocument[] starTreeDocuments) throws IOException { + + // sort the documents + Arrays.sort(starTreeDocuments, (o1, o2) -> { + for (int i = 0; i < numDimensions; i++) { + if (o1.dimensions[i] != o2.dimensions[i]) { + return Math.toIntExact(o1.dimensions[i] - o2.dimensions[i]); + } + } + return 0; + }); + + // merge the documents + return mergeStarTreeDocuments(starTreeDocuments); + } + + /** + * Merges the star-tree documents + * @param starTreeDocuments star-tree documents + * @return iterator to aggregate star-tree documents + */ + private Iterator mergeStarTreeDocuments(StarTreeDocument[] starTreeDocuments) { + return new Iterator<>() { + boolean hasNext = true; + StarTreeDocument currentStarTreeDocument = starTreeDocuments[0]; + int docId = 1; + + @Override + public boolean hasNext() { + return hasNext; + } + + @Override + public StarTreeDocument next() { + // aggregate as we move on to the next doc + StarTreeDocument next = aggregateStarTreeDocument(null, currentStarTreeDocument); + while (docId < starTreeDocuments.length) { + StarTreeDocument starTreeDocument = starTreeDocuments[docId++]; + if (!Arrays.equals(starTreeDocument.dimensions, next.dimensions)) { + currentStarTreeDocument = starTreeDocument; + return next; + } else { + next = aggregateStarTreeDocument(next, starTreeDocument); + } + } + hasNext = false; + return next; + } + }; + } + + /** + * Generates a star-tree for a given star-node + * @param startDocId Start document id in the star-tree + * @param endDocId End document id (exclusive) in the star-tree + * @param dimensionId Dimension id of the star-node + * @return iterator for star-tree documents of star-node + * @throws IOException throws when unable to generate star-tree for star-node + */ + @Override + public Iterator generateStarTreeForStarNode(int startDocId, int endDocId, int dimensionId) throws IOException { + int numDocs = endDocId - startDocId; + StarTreeDocument[] starTreeDocuments = new StarTreeDocument[numDocs]; + for (int i = 0; i < numDocs; i++) { + starTreeDocuments[i] = getStarTreeDocument(startDocId + i); + } + Arrays.sort(starTreeDocuments, (o1, o2) -> { + for (int i = dimensionId + 1; i < numDimensions; i++) { + if (o1.dimensions[i] != o2.dimensions[i]) { + return Math.toIntExact(o1.dimensions[i] - o2.dimensions[i]); + } + } + return 0; + }); + return new Iterator() { + boolean hasNext = true; + StarTreeDocument currentStarTreeDocument = starTreeDocuments[0]; + int docId = 1; + + private boolean hasSameDimensions(StarTreeDocument starTreeDocument1, StarTreeDocument starTreeDocument2) { + for (int i = dimensionId + 1; i < numDimensions; i++) { + if (starTreeDocument1.dimensions[i] != starTreeDocument2.dimensions[i]) { + return false; + } + } + return true; + } + + @Override + public boolean hasNext() { + return hasNext; + } + + @Override + public StarTreeDocument next() { + StarTreeDocument next = aggregateStarTreeDocument(null, currentStarTreeDocument); + next.dimensions[dimensionId] = STAR_IN_DOC_VALUES_INDEX; + while (docId < numDocs) { + StarTreeDocument starTreeDocument = starTreeDocuments[docId++]; + if (!hasSameDimensions(starTreeDocument, currentStarTreeDocument)) { + currentStarTreeDocument = starTreeDocument; + return next; + } else { + next = aggregateStarTreeDocument(next, starTreeDocument); + } + } + hasNext = false; + return next; + } + }; + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/SingleTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/SingleTreeBuilder.java new file mode 100644 index 0000000000000..3b26f4c5e04c3 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/SingleTreeBuilder.java @@ -0,0 +1,33 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube.startree.builder; + +import java.io.Closeable; +import java.io.IOException; + +/** + * A star-tree builder that builds a single star-tree. + * @opensearch.experimental + */ +public interface SingleTreeBuilder extends Closeable { + + /** + * Builds the star tree based on star-tree field + * @throws IOException when we are unable to build star-tree + */ + void build() throws Exception; + + /** + * Builds the star tree using star-tree document values during segment merges + * @param starTreeValues star-tree document values + * @throws IOException when we are unable to build star-tree + */ + // void build(List starTreeValues) throws IOException; + +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocValuesIteratorFactory.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocValuesIteratorFactory.java new file mode 100644 index 0000000000000..0105ff42045de --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocValuesIteratorFactory.java @@ -0,0 +1,54 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube.startree.builder; + +import org.apache.lucene.codecs.DocValuesProducer; +import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.search.DocIdSetIterator; + +import java.io.IOException; + +/** + * A factory class to return respective doc values iterator based on the doc volues type. + * @opensearch.experimental + */ +public class StarTreeDocValuesIteratorFactory implements DocValuesIteratorFactory { + + @Override + public DocIdSetIterator createIterator(DocValuesType type, FieldInfo field, DocValuesProducer producer) throws IOException { + switch (type) { + case SORTED_SET: + return producer.getSortedSet(field); + case SORTED_NUMERIC: + return producer.getSortedNumeric(field); + default: + throw new IllegalArgumentException("Unsupported DocValuesType: " + type); + } + } + + @Override + public long getNextValue(DocIdSetIterator iterator) throws IOException { + if (iterator instanceof SortedSetDocValues) { + return ((SortedSetDocValues) iterator).nextOrd(); + } else if (iterator instanceof SortedNumericDocValues) { + return ((SortedNumericDocValues) iterator).nextValue(); + } else { + throw new IllegalArgumentException("Unsupported Iterator: " + iterator.toString()); + } + } + + @Override + public int nextDoc(DocIdSetIterator iterator) throws IOException { + return iterator.nextDoc(); + } + +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/package-info.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/package-info.java new file mode 100644 index 0000000000000..80eed545ef8a5 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/package-info.java @@ -0,0 +1,13 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * Builders for Composite Index Star Tree + * @opensearch.experimental + */ +package org.opensearch.index.compositeindex.datacube.startree.builder; diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/data/DataType.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/data/DataType.java new file mode 100644 index 0000000000000..3c737c44b7219 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/data/DataType.java @@ -0,0 +1,67 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.index.compositeindex.datacube.startree.data; + +/** + * Data type of doc values + * @opensearch.internal + */ +public enum DataType { + INT(Integer.BYTES, true), + LONG(Long.BYTES, true), + FLOAT(Float.BYTES, true), + DOUBLE(Double.BYTES, true); + + private final int size; + private final boolean numeric; + + DataType(int size, boolean numeric) { + this.size = size; + this.numeric = numeric; + } + + /** + * Returns the number of bytes needed to store the data type. + */ + public int size() { + if (size >= 0) { + return size; + } + throw new IllegalStateException("Cannot get number of bytes for: " + this); + } + + /** + * Returns {@code true} if the data type is numeric (INT, LONG, FLOAT, DOUBLE, BIG_DECIMAL), + * {@code false} otherwise. + */ + public boolean isNumeric() { + return numeric; + } + + /** + * Converts the given string value to the data type. Returns byte[] for BYTES. + */ + public Object convert(String value) { + try { + switch (this) { + case INT: + return Integer.valueOf(value); + case LONG: + return Long.valueOf(value); + case FLOAT: + return Float.valueOf(value); + case DOUBLE: + return Double.valueOf(value); + default: + throw new IllegalStateException(); + } + } catch (Exception e) { + throw new IllegalArgumentException(e); + } + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/data/StarTreeDocument.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/data/StarTreeDocument.java new file mode 100644 index 0000000000000..cb4253e21c141 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/data/StarTreeDocument.java @@ -0,0 +1,29 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube.startree.data; + +import java.util.Arrays; + +/** + * Star tree document + */ +public class StarTreeDocument { + public final long[] dimensions; + public final Object[] metrics; + + public StarTreeDocument(long[] dimensions, Object[] metrics) { + this.dimensions = dimensions; + this.metrics = metrics; + } + + @Override + public String toString() { + return Arrays.toString(dimensions) + " | " + Arrays.toString(metrics); + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/package-info.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/package-info.java new file mode 100644 index 0000000000000..3af5020b76da2 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/package-info.java @@ -0,0 +1,13 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * Node for Composite Index Star Tree + * @opensearch.experimental + */ +package org.opensearch.index.compositeindex.datacube.startree.node; diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeBuilderUtils.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeBuilderUtils.java new file mode 100644 index 0000000000000..9fa980277f27b --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeBuilderUtils.java @@ -0,0 +1,131 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.index.compositeindex.datacube.startree.utils; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.lucene.store.IndexOutput; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Queue; + +import static java.nio.charset.StandardCharsets.UTF_8; + +/** + * Util class for building star tree + * @opensearch.experimental + */ +public class StarTreeBuilderUtils { + + private static final Logger logger = LogManager.getLogger(StarTreeBuilderUtils.class); + + // TODO: To be moved to off heap star tree implementation + public static final int NUM_INT_SERIALIZABLE_FIELDS = 6; + public static final int NUM_LONG_SERIALIZABLE_FIELDS = 1; + public static final long SERIALIZABLE_SIZE_IN_BYTES = (Integer.BYTES * NUM_INT_SERIALIZABLE_FIELDS) + (Long.BYTES + * NUM_LONG_SERIALIZABLE_FIELDS); + + private StarTreeBuilderUtils() {} + + public static final int ALL = -1; + public static final long MAGIC_MARKER = 0xBADDA55B00DAD00DL; + + /** Tree node representation */ + public static class TreeNode { + public int dimensionId = ALL; + public long dimensionValue = ALL; + public int startDocId = ALL; + public int endDocId = ALL; + public int aggregatedDocId = ALL; + public int childDimensionId = ALL; + public Map children; + } + + /** Serializes the tree */ + public static void serializeTree(IndexOutput indexOutput, TreeNode rootNode, String[] dimensions, int numNodes) throws IOException { + int headerSizeInBytes = computeHeaderByteSize(dimensions); + long totalSizeInBytes = headerSizeInBytes + (long) numNodes * SERIALIZABLE_SIZE_IN_BYTES; + + logger.info("Star tree size in bytes : {}", totalSizeInBytes); + + writeHeader(indexOutput, headerSizeInBytes, dimensions, numNodes); + writeNodes(indexOutput, rootNode); + } + + /** Computes the size of the header for the tree */ + static int computeHeaderByteSize(String[] dimensions) { + // Magic marker (8), version (4), size of header (4) and number of dimensions (4) + int headerSizeInBytes = 20; + + for (String dimension : dimensions) { + headerSizeInBytes += Integer.BYTES; // For dimension index + headerSizeInBytes += Integer.BYTES; // For length of dimension name + headerSizeInBytes += dimension.getBytes(UTF_8).length; // For dimension name + } + + headerSizeInBytes += Integer.BYTES; // For number of nodes. + return headerSizeInBytes; + } + + /** Writes the header of the tree */ + static void writeHeader(IndexOutput output, int headerSizeInBytes, String[] dimensions, int numNodes) throws IOException { + output.writeLong(MAGIC_MARKER); + output.writeInt(1); + output.writeInt(headerSizeInBytes); + output.writeInt(dimensions.length); + for (int i = 0; i < dimensions.length; i++) { + output.writeInt(i); + output.writeString(dimensions[i]); + } + output.writeInt(numNodes); + } + + /** Writes the nodes of the tree */ + static void writeNodes(IndexOutput output, TreeNode rootNode) throws IOException { + Queue queue = new LinkedList<>(); + queue.add(rootNode); + + int currentNodeId = 0; + while (!queue.isEmpty()) { + TreeNode node = queue.remove(); + + if (node.children == null) { + writeNode(output, node, ALL, ALL); + } else { + // Sort all children nodes based on dimension value + List sortedChildren = new ArrayList<>(node.children.values()); + sortedChildren.sort(Comparator.comparingLong(o -> o.dimensionValue)); + + int firstChildId = currentNodeId + queue.size() + 1; + int lastChildId = firstChildId + sortedChildren.size() - 1; + writeNode(output, node, firstChildId, lastChildId); + + queue.addAll(sortedChildren); + } + + currentNodeId++; + } + } + + /** Writes a node of the tree */ + private static void writeNode(IndexOutput output, TreeNode node, int firstChildId, int lastChildId) throws IOException { + output.writeInt(node.dimensionId); + output.writeLong(node.dimensionValue); + output.writeInt(node.startDocId); + output.writeInt(node.endDocId); + output.writeInt(node.aggregatedDocId); + output.writeInt(firstChildId); + output.writeInt(lastChildId); + } + +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/package-info.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/package-info.java new file mode 100644 index 0000000000000..92930de98970d --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/package-info.java @@ -0,0 +1,13 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * Utility to support Composite Index Star Tree + * @opensearch.experimental + */ +package org.opensearch.index.compositeindex.datacube.startree.utils; diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricStatFieldPairTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricStatFieldPairTests.java new file mode 100644 index 0000000000000..fd026f49c0140 --- /dev/null +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricStatFieldPairTests.java @@ -0,0 +1,69 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube.startree.aggregators; + +import org.opensearch.index.compositeindex.datacube.MetricStat; +import org.opensearch.test.OpenSearchTestCase; + +public class MetricStatFieldPairTests extends OpenSearchTestCase { + + public void testConstructor() { + MetricStatFieldPair pair = new MetricStatFieldPair(MetricStat.SUM, "column1"); + assertEquals(MetricStat.SUM, pair.getMetricStat()); + assertEquals("column1", pair.getField()); + } + + public void testCountStarConstructor() { + MetricStatFieldPair pair = new MetricStatFieldPair(MetricStat.COUNT, "anything"); + assertEquals(MetricStat.COUNT, pair.getMetricStat()); + assertEquals("*", pair.getField()); + } + + public void testToFieldName() { + MetricStatFieldPair pair = new MetricStatFieldPair(MetricStat.AVG, "column2"); + assertEquals("avg__column2", pair.toFieldName()); + } + + public void testFromFieldName() { + MetricStatFieldPair pair = MetricStatFieldPair.fromFieldName("max__column3"); + assertEquals(MetricStat.MAX, pair.getMetricStat()); + assertEquals("column3", pair.getField()); + } + + public void testCountStarFromFieldName() { + MetricStatFieldPair pair = MetricStatFieldPair.fromFieldName("count__*"); + assertEquals(MetricStat.COUNT, pair.getMetricStat()); + assertEquals("*", pair.getField()); + assertSame(MetricStatFieldPair.COUNT_STAR, pair); + } + + public void testEquals() { + MetricStatFieldPair pair1 = new MetricStatFieldPair(MetricStat.SUM, "column1"); + MetricStatFieldPair pair2 = new MetricStatFieldPair(MetricStat.SUM, "column1"); + assertEquals(pair1, pair2); + assertNotEquals(pair1, new MetricStatFieldPair(MetricStat.AVG, "column1")); + assertNotEquals(pair1, new MetricStatFieldPair(MetricStat.SUM, "column2")); + } + + public void testHashCode() { + MetricStatFieldPair pair1 = new MetricStatFieldPair(MetricStat.SUM, "column1"); + MetricStatFieldPair pair2 = new MetricStatFieldPair(MetricStat.SUM, "column1"); + assertEquals(pair1.hashCode(), pair2.hashCode()); + } + + public void testCompareTo() { + MetricStatFieldPair pair1 = new MetricStatFieldPair(MetricStat.SUM, "column1"); + MetricStatFieldPair pair2 = new MetricStatFieldPair(MetricStat.SUM, "column2"); + MetricStatFieldPair pair3 = new MetricStatFieldPair(MetricStat.AVG, "column1"); + assertTrue(pair1.compareTo(pair2) < 0); + assertTrue(pair2.compareTo(pair1) > 0); + assertTrue(pair1.compareTo(pair3) > 0); + assertTrue(pair3.compareTo(pair1) < 0); + } +} diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregatorTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregatorTests.java new file mode 100644 index 0000000000000..58709934c894e --- /dev/null +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregatorTests.java @@ -0,0 +1,64 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube.startree.aggregators; + +import org.apache.lucene.util.NumericUtils; +import org.opensearch.index.compositeindex.datacube.MetricStat; +import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; +import org.opensearch.test.OpenSearchTestCase; + +public class SumValueAggregatorTests extends OpenSearchTestCase { + + private final SumValueAggregator aggregator = new SumValueAggregator(); + + public void testGetAggregationType() { + assertEquals(MetricStat.SUM.getTypeName(), aggregator.getAggregationType().getTypeName()); + } + + public void testGetAggregatedValueType() { + assertEquals(SumValueAggregator.AGGREGATED_VALUE_TYPE, aggregator.getAggregatedValueType()); + } + + public void testGetInitialAggregatedValue() { + assertEquals(1.0, aggregator.getInitialAggregatedValue(1L, StarTreeNumericType.LONG), 0.0); + assertThrows(NullPointerException.class, () -> aggregator.getInitialAggregatedValue(null, StarTreeNumericType.DOUBLE)); + } + + public void testApplySegmentRawValue() { + assertEquals(5.0, aggregator.applySegmentRawValue(2.0, 3L, StarTreeNumericType.LONG), 0.0); + assertThrows(NullPointerException.class, () -> aggregator.applySegmentRawValue(3.14, null, StarTreeNumericType.DOUBLE)); + } + + public void testApplyAggregatedValue() { + assertEquals(5.0, aggregator.applyAggregatedValue(2.0, 3.0), 0.0); + assertEquals(7.28, aggregator.applyAggregatedValue(3.14, 4.14), 0.0000001); + } + + public void testCloneAggregatedValue() { + assertEquals(3.14, aggregator.cloneAggregatedValue(3.14), 0.0); + } + + public void testGetMaxAggregatedValueByteSize() { + assertEquals(Double.BYTES, aggregator.getMaxAggregatedValueByteSize()); + } + + public void testConvertAggregationTypeToSortableLongValue() { + SumValueAggregator aggregator = new SumValueAggregator(); + assertEquals(NumericUtils.doubleToSortableLong(3.14), aggregator.convertAggregationTypeToSortableLongValue(3.14), 0.0); + } + + public void testConvertSortableLongToAggregatedTypeValue() { + SumValueAggregator aggregator = new SumValueAggregator(); + assertEquals( + NumericUtils.sortableLongToDouble(3L), + aggregator.convertSortableLongToAggregatedTypeValue(3L, StarTreeNumericType.DOUBLE), + 0.0 + ); + } +} diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactoryTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactoryTests.java new file mode 100644 index 0000000000000..f80783dacb643 --- /dev/null +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactoryTests.java @@ -0,0 +1,43 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube.startree.aggregators; + +import org.opensearch.index.compositeindex.datacube.MetricStat; +import org.opensearch.index.compositeindex.datacube.startree.data.DataType; +import org.opensearch.test.OpenSearchTestCase; + +public class ValueAggregatorFactoryTests extends OpenSearchTestCase { + + public void testGetValueAggregatorForSumType() { + ValueAggregator aggregator = ValueAggregatorFactory.getValueAggregator(MetricStat.SUM); + assertNotNull(aggregator); + assertEquals(SumValueAggregator.class, aggregator.getClass()); + } + + public void testGetValueAggregatorForUnsupportedType() { + IllegalStateException exception = expectThrows( + IllegalStateException.class, + () -> ValueAggregatorFactory.getValueAggregator(MetricStat.UNSUPPORTED) + ); + assertEquals("Unsupported aggregation type: UNSUPPORTED", exception.getMessage()); + } + + public void testGetAggregatedValueTypeForSumType() { + DataType dataType = ValueAggregatorFactory.getAggregatedValueType(MetricStat.SUM); + assertEquals(SumValueAggregator.AGGREGATED_VALUE_TYPE, dataType); + } + + public void testGetAggregatedValueTypeForUnsupportedType() { + IllegalStateException exception = expectThrows( + IllegalStateException.class, + () -> ValueAggregatorFactory.getAggregatedValueType(MetricStat.UNSUPPORTED) + ); + assertEquals("Unsupported aggregation type: UNSUPPORTED", exception.getMessage()); + } +} diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseSingleStarTreeBuilderTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseSingleStarTreeBuilderTests.java new file mode 100644 index 0000000000000..82be99faa8bd8 --- /dev/null +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseSingleStarTreeBuilderTests.java @@ -0,0 +1,213 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube.startree.builder; + +import org.apache.lucene.codecs.DocValuesConsumer; +import org.apache.lucene.codecs.DocValuesProducer; +import org.apache.lucene.codecs.lucene99.Lucene99Codec; +import org.apache.lucene.index.BaseSingleStarTreeBuilder; +import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.VectorEncoding; +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.InfoStream; +import org.apache.lucene.util.Version; +import org.opensearch.common.settings.Settings; +import org.opensearch.index.compositeindex.datacube.Dimension; +import org.opensearch.index.compositeindex.datacube.Metric; +import org.opensearch.index.compositeindex.datacube.MetricStat; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; +import org.opensearch.index.compositeindex.datacube.startree.aggregators.MetricStatFieldPair; +import org.opensearch.index.compositeindex.datacube.startree.data.StarTreeDocument; +import org.opensearch.index.mapper.ContentPath; +import org.opensearch.index.mapper.DocumentMapper; +import org.opensearch.index.mapper.Mapper; +import org.opensearch.index.mapper.MapperService; +import org.opensearch.index.mapper.MappingLookup; +import org.opensearch.index.mapper.NumberFieldMapper; +import org.opensearch.test.OpenSearchTestCase; +import org.junit.BeforeClass; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Set; +import java.util.UUID; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class BaseSingleStarTreeBuilderTests extends OpenSearchTestCase { + + private static BaseSingleStarTreeBuilder builder; + private static MapperService mapperService; + private static List dimensionsOrder; + private static List fields = List.of( + "field1", + "field2", + "field3", + "field4", + "field5", + "field6", + "field7", + "field8", + "field9", + "field10" + ); + private static List metrics; + private static Directory directory; + private static FieldInfo[] fieldsInfo; + + @BeforeClass + public static void setup() throws IOException { + + dimensionsOrder = List.of(new Dimension("field1"), new Dimension("field3"), new Dimension("field5"), new Dimension("field8")); + metrics = List.of(new Metric("field2", List.of(MetricStat.SUM)), new Metric("field4", List.of(MetricStat.SUM))); + + StarTreeField compositeField = new StarTreeField( + "test", + dimensionsOrder, + metrics, + new StarTreeFieldConfiguration(1, Set.of("field8"), StarTreeFieldConfiguration.StarTreeBuildMode.ON_HEAP) + ); + DocValuesConsumer docValuesConsumer = mock(DocValuesConsumer.class); + DocValuesProducer docValuesProducer = mock(DocValuesProducer.class); + directory = newFSDirectory(createTempDir()); + SegmentInfo segmentInfo = new SegmentInfo( + directory, + Version.LATEST, + Version.LUCENE_9_11_0, + "test_segment", + 5, + false, + false, + new Lucene99Codec(), + new HashMap<>(), + UUID.randomUUID().toString().substring(0, 16).getBytes(StandardCharsets.UTF_8), + new HashMap<>(), + null + ); + + fieldsInfo = new FieldInfo[fields.size()]; + + for (int i = 0; i < fieldsInfo.length; i++) { + fieldsInfo[i] = new FieldInfo( + fields.get(i), + i, + false, + false, + true, + IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS, + DocValuesType.SORTED_NUMERIC, + -1, + Collections.emptyMap(), + 0, + 0, + 0, + 0, + VectorEncoding.FLOAT32, + VectorSimilarityFunction.EUCLIDEAN, + false, + false + ); + } + FieldInfos fieldInfos = new FieldInfos(fieldsInfo); + final SegmentWriteState state = new SegmentWriteState( + InfoStream.getDefault(), + segmentInfo.dir, + segmentInfo, + fieldInfos, + null, + newIOContext(random()) + ); + + mapperService = mock(MapperService.class); + DocumentMapper documentMapper = mock(DocumentMapper.class); + when(mapperService.documentMapper()).thenReturn(documentMapper); + Settings settings = Settings.builder().put(settings(org.opensearch.Version.CURRENT).build()).build(); + NumberFieldMapper numberFieldMapper1 = new NumberFieldMapper.Builder("field2", NumberFieldMapper.NumberType.DOUBLE, false, true) + .build(new Mapper.BuilderContext(settings, new ContentPath())); + NumberFieldMapper numberFieldMapper2 = new NumberFieldMapper.Builder("field4", NumberFieldMapper.NumberType.DOUBLE, false, true) + .build(new Mapper.BuilderContext(settings, new ContentPath())); + MappingLookup fieldMappers = new MappingLookup( + Set.of(numberFieldMapper1, numberFieldMapper2), + Collections.emptyList(), + Collections.emptyList(), + 0, + null + ); + when(documentMapper.mappers()).thenReturn(fieldMappers); + + builder = new BaseSingleStarTreeBuilder(compositeField, docValuesProducer, docValuesConsumer, state, mapperService) { + @Override + public void appendStarTreeDocument(StarTreeDocument starTreeDocument) throws IOException {} + + @Override + public StarTreeDocument getStarTreeDocument(int docId) throws IOException { + return null; + } + + @Override + public List getStarTreeDocuments() throws IOException { + return List.of(); + } + + @Override + public long getDimensionValue(int docId, int dimensionId) throws IOException { + return 0; + } + + @Override + public Iterator processSegmentStarTreeDocuments(int numDocs) throws IOException { + return null; + } + + @Override + public Iterator generateStarTreeForStarNode(int startDocId, int endDocId, int dimensionId) + throws IOException { + return null; + } + }; + } + + public void test_generateMetricStatFieldPairs() throws IOException { + List metricStatFieldPairs = builder.generateMetricStatFieldPairs(); + List expectedMetricStatFieldPairs = List.of( + new MetricStatFieldPair(MetricStat.SUM, "field2"), + new MetricStatFieldPair(MetricStat.SUM, "field4") + ); + assertEquals(metricStatFieldPairs, expectedMetricStatFieldPairs); + } + + public void test_aggregateStarTreeDocument() { + StarTreeDocument starTreeDocument1 = new StarTreeDocument(new long[] { 1, 3, 5, 8 }, new Double[] { 4.0, 8.0 }); + StarTreeDocument starTreeDocument2 = new StarTreeDocument(new long[] { 1, 3, 5, 8 }, new Double[] { 10.0, 6.0 }); + + StarTreeDocument expectedeMergedStarTreeDocument = new StarTreeDocument(new long[] { 1, 3, 5, 8 }, new Double[] { 14.0, 14.0 }); + StarTreeDocument mergedStarTreeDocument = builder.aggregateStarTreeDocument(starTreeDocument1, starTreeDocument2); + + assertEquals(mergedStarTreeDocument.metrics[0], expectedeMergedStarTreeDocument.metrics[0]); + assertEquals(mergedStarTreeDocument.metrics[1], expectedeMergedStarTreeDocument.metrics[1]); + } + + @Override + public void tearDown() throws Exception { + super.tearDown(); + directory.close(); + } +} diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapSingleTreeBuilderTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapSingleTreeBuilderTests.java new file mode 100644 index 0000000000000..e81c836c9d723 --- /dev/null +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapSingleTreeBuilderTests.java @@ -0,0 +1,264 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube.startree.builder; + +import org.apache.lucene.codecs.DocValuesConsumer; +import org.apache.lucene.codecs.DocValuesProducer; +import org.apache.lucene.codecs.lucene99.Lucene99Codec; +import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.VectorEncoding; +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.InfoStream; +import org.apache.lucene.util.Version; +import org.opensearch.common.settings.Settings; +import org.opensearch.index.compositeindex.datacube.Dimension; +import org.opensearch.index.compositeindex.datacube.Metric; +import org.opensearch.index.compositeindex.datacube.MetricStat; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; +import org.opensearch.index.compositeindex.datacube.startree.data.StarTreeDocument; +import org.opensearch.index.mapper.ContentPath; +import org.opensearch.index.mapper.DocumentMapper; +import org.opensearch.index.mapper.Mapper; +import org.opensearch.index.mapper.MapperService; +import org.opensearch.index.mapper.MappingLookup; +import org.opensearch.index.mapper.NumberFieldMapper; +import org.opensearch.test.OpenSearchTestCase; +import org.junit.BeforeClass; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Set; +import java.util.UUID; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class OnHeapSingleTreeBuilderTests extends OpenSearchTestCase { + + private static OnHeapSingleTreeBuilder builder; + private static MapperService mapperService; + private static List dimensionsOrder; + private static List fields = List.of( + "field1", + "field2", + "field3", + "field4", + "field5", + "field6", + "field7", + "field8", + "field9", + "field10" + ); + private static List metrics; + private static Directory directory; + private static FieldInfo[] fieldsInfo; + + @BeforeClass + public static void setup() throws IOException { + dimensionsOrder = List.of(new Dimension("field1"), new Dimension("field3"), new Dimension("field5"), new Dimension("field8")); + metrics = List.of(new Metric("field2", List.of(MetricStat.SUM)), new Metric("field4", List.of(MetricStat.SUM))); + + DocValuesConsumer docValuesConsumer = mock(DocValuesConsumer.class); + DocValuesProducer docValuesProducer = mock(DocValuesProducer.class); + + StarTreeField compositeField = new StarTreeField( + "test", + dimensionsOrder, + metrics, + new StarTreeFieldConfiguration(1, Set.of("field8"), StarTreeFieldConfiguration.StarTreeBuildMode.ON_HEAP) + ); + directory = newFSDirectory(createTempDir()); + SegmentInfo segmentInfo = new SegmentInfo( + directory, + Version.LATEST, + Version.LUCENE_9_11_0, + "test_segment", + 5, + false, + false, + new Lucene99Codec(), + new HashMap<>(), + UUID.randomUUID().toString().substring(0, 16).getBytes(StandardCharsets.UTF_8), + new HashMap<>(), + null + ); + + fieldsInfo = new FieldInfo[fields.size()]; + + for (int i = 0; i < fieldsInfo.length; i++) { + fieldsInfo[i] = new FieldInfo( + fields.get(i), + i, + false, + false, + true, + IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS, + DocValuesType.SORTED_NUMERIC, + -1, + Collections.emptyMap(), + 0, + 0, + 0, + 0, + VectorEncoding.FLOAT32, + VectorSimilarityFunction.EUCLIDEAN, + false, + false + ); + } + FieldInfos fieldInfos = new FieldInfos(fieldsInfo); + final SegmentWriteState writeState = new SegmentWriteState( + InfoStream.getDefault(), + segmentInfo.dir, + segmentInfo, + fieldInfos, + null, + newIOContext(random()) + ); + + mapperService = mock(MapperService.class); + DocumentMapper documentMapper = mock(DocumentMapper.class); + when(mapperService.documentMapper()).thenReturn(documentMapper); + Settings settings = Settings.builder().put(settings(org.opensearch.Version.CURRENT).build()).build(); + NumberFieldMapper numberFieldMapper1 = new NumberFieldMapper.Builder("field2", NumberFieldMapper.NumberType.DOUBLE, false, true) + .build(new Mapper.BuilderContext(settings, new ContentPath())); + NumberFieldMapper numberFieldMapper2 = new NumberFieldMapper.Builder("field4", NumberFieldMapper.NumberType.DOUBLE, false, true) + .build(new Mapper.BuilderContext(settings, new ContentPath())); + MappingLookup fieldMappers = new MappingLookup( + Set.of(numberFieldMapper1, numberFieldMapper2), + Collections.emptyList(), + Collections.emptyList(), + 0, + null + ); + when(documentMapper.mappers()).thenReturn(fieldMappers); + builder = new OnHeapSingleTreeBuilder(compositeField, docValuesProducer, docValuesConsumer, writeState, mapperService); + } + + public void test_processStarTreeDocuments() throws IOException { + + int noOfStarTreeDocuments = 5; + StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; + + starTreeDocuments[0] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 12.0, 10.0 }); + starTreeDocuments[1] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 10.0, 6.0 }); + starTreeDocuments[2] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 14.0, 12.0 }); + starTreeDocuments[3] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 9.0, 4.0 }); + starTreeDocuments[4] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 11.0, 16.0 }); + + List inorderStarTreeDocuments = List.of( + new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 21.0, 14.0 }), + new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 35.0, 34.0 }) + ); + Iterator expectedStarTreeDocumentIterator = inorderStarTreeDocuments.iterator(); + Iterator starTreeDocumentIterator = builder.processStarTreeDocuments(starTreeDocuments); + int numOfAggregatedDocuments = 0; + while (starTreeDocumentIterator.hasNext() && expectedStarTreeDocumentIterator.hasNext()) { + StarTreeDocument resultStarTreeDocument = starTreeDocumentIterator.next(); + StarTreeDocument expectedStarTreeDocument = expectedStarTreeDocumentIterator.next(); + + assertEquals(expectedStarTreeDocument.dimensions[0], resultStarTreeDocument.dimensions[0]); + assertEquals(expectedStarTreeDocument.dimensions[1], resultStarTreeDocument.dimensions[1]); + assertEquals(expectedStarTreeDocument.dimensions[2], resultStarTreeDocument.dimensions[2]); + assertEquals(expectedStarTreeDocument.dimensions[3], resultStarTreeDocument.dimensions[3]); + assertEquals(expectedStarTreeDocument.metrics[0], resultStarTreeDocument.metrics[0]); + assertEquals(expectedStarTreeDocument.metrics[1], resultStarTreeDocument.metrics[1]); + + numOfAggregatedDocuments++; + } + + assertEquals(inorderStarTreeDocuments.size(), numOfAggregatedDocuments); + + } + + public void test_processStarTreeDocuments_nullDocument() throws IOException { + + int noOfStarTreeDocuments = 5; + StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; + + starTreeDocuments[0] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 12.0, 10.0 }); + starTreeDocuments[1] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 10.0, 6.0 }); + starTreeDocuments[2] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 14.0, 12.0 }); + starTreeDocuments[3] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 9.0, 4.0 }); + starTreeDocuments[4] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 11.0, null }); + StarTreeDocument expectedStarTreeDocument = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 21.0, 14.0 }); + Iterator starTreeDocumentIterator = builder.processStarTreeDocuments(starTreeDocuments); + + StarTreeDocument resultStarTreeDocument = starTreeDocumentIterator.next(); + assertEquals(expectedStarTreeDocument.dimensions[0], resultStarTreeDocument.dimensions[0]); + assertEquals(expectedStarTreeDocument.dimensions[1], resultStarTreeDocument.dimensions[1]); + assertEquals(expectedStarTreeDocument.dimensions[2], resultStarTreeDocument.dimensions[2]); + assertEquals(expectedStarTreeDocument.dimensions[3], resultStarTreeDocument.dimensions[3]); + assertEquals(expectedStarTreeDocument.metrics[0], resultStarTreeDocument.metrics[0]); + assertEquals(expectedStarTreeDocument.metrics[1], resultStarTreeDocument.metrics[1]); + + assertThrows("Cannot apply aggregated value [null]", IllegalArgumentException.class, starTreeDocumentIterator::next); + + } + + public void test_build() throws IOException { + + int noOfStarTreeDocuments = 5; + StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; + + starTreeDocuments[0] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 12.0, 10.0 }); + starTreeDocuments[1] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 10.0, 6.0 }); + starTreeDocuments[2] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 14.0, 12.0 }); + starTreeDocuments[3] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 9.0, 4.0 }); + starTreeDocuments[4] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 11.0, 16.0 }); + + Iterator starTreeDocumentIterator = builder.processStarTreeDocuments(starTreeDocuments); + builder.build(starTreeDocumentIterator); + + List resultStarTreeDocuments = builder.getStarTreeDocuments(); + assertEquals(8, resultStarTreeDocuments.size()); + + List expectedStarTreeDocuments = List.of( + new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 21.0, 14.0 }), + new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 35.0, 34.0 }), + new StarTreeDocument(new long[] { -1, 4, 2, 1 }, new Double[] { 35.0, 34.0 }), + new StarTreeDocument(new long[] { -1, 4, 3, 4 }, new Double[] { 21.0, 14.0 }), + new StarTreeDocument(new long[] { -1, 4, -1, 1 }, new Double[] { 35.0, 34.0 }), + new StarTreeDocument(new long[] { -1, 4, -1, 4 }, new Double[] { 21.0, 14.0 }), + new StarTreeDocument(new long[] { -1, 4, -1, -1 }, new Double[] { 56.0, 48.0 }), + new StarTreeDocument(new long[] { -1, -1, -1, -1 }, new Double[] { 56.0, 48.0 }) + ); + Iterator expectedStarTreeDocumentIterator = expectedStarTreeDocuments.iterator(); + Iterator resultStarTreeDocumentIterator = resultStarTreeDocuments.iterator(); + while (resultStarTreeDocumentIterator.hasNext() && expectedStarTreeDocumentIterator.hasNext()) { + StarTreeDocument resultStarTreeDocument = resultStarTreeDocumentIterator.next(); + StarTreeDocument expectedStarTreeDocument = expectedStarTreeDocumentIterator.next(); + + assertEquals(expectedStarTreeDocument.dimensions[0], resultStarTreeDocument.dimensions[0]); + assertEquals(expectedStarTreeDocument.dimensions[1], resultStarTreeDocument.dimensions[1]); + assertEquals(expectedStarTreeDocument.dimensions[2], resultStarTreeDocument.dimensions[2]); + assertEquals(expectedStarTreeDocument.dimensions[3], resultStarTreeDocument.dimensions[3]); + assertEquals(expectedStarTreeDocument.metrics[0], resultStarTreeDocument.metrics[0]); + assertEquals(expectedStarTreeDocument.metrics[1], resultStarTreeDocument.metrics[1]); + } + } + + @Override + public void tearDown() throws Exception { + super.tearDown(); + directory.close(); + } +} diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeValuesIteratorFactoryTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeValuesIteratorFactoryTests.java new file mode 100644 index 0000000000000..7f6978dbf2720 --- /dev/null +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeValuesIteratorFactoryTests.java @@ -0,0 +1,113 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube.startree.builder; + +import org.apache.lucene.codecs.DocValuesProducer; +import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.index.VectorEncoding; +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.search.DocIdSetIterator; +import org.opensearch.test.OpenSearchTestCase; +import org.junit.BeforeClass; + +import java.io.IOException; +import java.util.Collections; + +import org.mockito.Mockito; + +import static org.mockito.Mockito.when; + +public class StarTreeValuesIteratorFactoryTests extends OpenSearchTestCase { + + private static StarTreeDocValuesIteratorFactory factory; + private static FieldInfo mockFieldInfo; + + @BeforeClass + public static void setup() { + factory = new StarTreeDocValuesIteratorFactory(); + mockFieldInfo = new FieldInfo( + "field", + 1, + false, + false, + true, + IndexOptions.NONE, + DocValuesType.NONE, + -1, + Collections.emptyMap(), + 0, + 0, + 0, + 0, + VectorEncoding.FLOAT32, + VectorSimilarityFunction.EUCLIDEAN, + false, + false + ); + } + + public void testCreateIterator_SortedSet() throws IOException { + DocValuesProducer producer = Mockito.mock(DocValuesProducer.class); + SortedSetDocValues iterator = Mockito.mock(SortedSetDocValues.class); + when(producer.getSortedSet(mockFieldInfo)).thenReturn(iterator); + DocIdSetIterator result = factory.createIterator(DocValuesType.SORTED_SET, mockFieldInfo, producer); + assertEquals(iterator.getClass(), result.getClass()); + } + + public void testCreateIterator_SortedNumeric() throws IOException { + DocValuesProducer producer = Mockito.mock(DocValuesProducer.class); + SortedNumericDocValues iterator = Mockito.mock(SortedNumericDocValues.class); + when(producer.getSortedNumeric(mockFieldInfo)).thenReturn(iterator); + DocIdSetIterator result = factory.createIterator(DocValuesType.SORTED_NUMERIC, mockFieldInfo, producer); + assertEquals(iterator.getClass(), result.getClass()); + } + + public void testCreateIterator_UnsupportedType() { + DocValuesProducer producer = Mockito.mock(DocValuesProducer.class); + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> { + factory.createIterator(DocValuesType.BINARY, mockFieldInfo, producer); + }); + assertEquals("Unsupported DocValuesType: BINARY", exception.getMessage()); + } + + public void testGetNextValue_SortedSet() throws IOException { + SortedSetDocValues iterator = Mockito.mock(SortedSetDocValues.class); + when(iterator.nextOrd()).thenReturn(42L); + + long result = factory.getNextValue(iterator); + assertEquals(42L, result); + } + + public void testGetNextValue_SortedNumeric() throws IOException { + SortedNumericDocValues iterator = Mockito.mock(SortedNumericDocValues.class); + when(iterator.nextValue()).thenReturn(123L); + + long result = factory.getNextValue(iterator); + assertEquals(123L, result); + } + + public void testGetNextValue_UnsupportedIterator() { + DocIdSetIterator iterator = Mockito.mock(DocIdSetIterator.class); + + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> { factory.getNextValue(iterator); }); + assertEquals("Unsupported Iterator: " + iterator.toString(), exception.getMessage()); + } + + public void testNextDoc() throws IOException { + DocIdSetIterator iterator = Mockito.mock(DocIdSetIterator.class); + when(iterator.nextDoc()).thenReturn(5); + + int result = factory.nextDoc(iterator); + assertEquals(5, result); + } +} From d3b302bbd9aae4b3854f9c0a6051a604490a971d Mon Sep 17 00:00:00 2001 From: Sarthak Aggarwal Date: Mon, 24 Jun 2024 01:01:10 +0530 Subject: [PATCH 09/11] addressed nits Signed-off-by: Sarthak Aggarwal --- ...eBuilder.java => BaseStarTreeBuilder.java} | 188 +++++++----------- .../common/inject/util/Modules.java | 3 +- .../aggregators/SumValueAggregator.java | 2 +- .../startree/aggregators/ValueAggregator.java | 2 +- .../numerictype/StarTreeNumericType.java | 4 +- ...ory.java => DocValuesIteratorAdapter.java} | 4 +- .../builder/OnHeapSingleTreeBuilder.java | 24 +-- .../startree/builder/SingleTreeBuilder.java | 8 - ... => StarTreeDocValuesIteratorAdapter.java} | 4 +- ...reesBuilder.java => StarTreesBuilder.java} | 10 +- .../startree/utils/StarTreeBuilderUtils.java | 101 +--------- .../aggregators/SumValueAggregatorTests.java | 4 +- ...sts.java => BaseStarTreeBuilderTests.java} | 14 +- .../StarTreeValuesIteratorFactoryTests.java | 10 +- 14 files changed, 117 insertions(+), 261 deletions(-) rename server/src/main/java/org/apache/lucene/index/{BaseSingleStarTreeBuilder.java => BaseStarTreeBuilder.java} (78%) rename server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/{DocValuesIteratorFactory.java => DocValuesIteratorAdapter.java} (85%) rename server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/{StarTreeDocValuesIteratorFactory.java => StarTreeDocValuesIteratorAdapter.java} (88%) rename server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/{MultipleTreesBuilder.java => StarTreesBuilder.java} (91%) rename server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/{BaseSingleStarTreeBuilderTests.java => BaseStarTreeBuilderTests.java} (93%) diff --git a/server/src/main/java/org/apache/lucene/index/BaseSingleStarTreeBuilder.java b/server/src/main/java/org/apache/lucene/index/BaseStarTreeBuilder.java similarity index 78% rename from server/src/main/java/org/apache/lucene/index/BaseSingleStarTreeBuilder.java rename to server/src/main/java/org/apache/lucene/index/BaseStarTreeBuilder.java index 7b0957b1328de..f2f1f35d7c23f 100644 --- a/server/src/main/java/org/apache/lucene/index/BaseSingleStarTreeBuilder.java +++ b/server/src/main/java/org/apache/lucene/index/BaseStarTreeBuilder.java @@ -23,7 +23,7 @@ import org.opensearch.index.compositeindex.datacube.startree.aggregators.ValueAggregatorFactory; import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; import org.opensearch.index.compositeindex.datacube.startree.builder.SingleTreeBuilder; -import org.opensearch.index.compositeindex.datacube.startree.builder.StarTreeDocValuesIteratorFactory; +import org.opensearch.index.compositeindex.datacube.startree.builder.StarTreeDocValuesIteratorAdapter; import org.opensearch.index.compositeindex.datacube.startree.data.StarTreeDocument; import org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeBuilderUtils; import org.opensearch.index.fielddata.IndexNumericFieldData; @@ -44,12 +44,9 @@ /** * Base class for star-tree builder */ -public abstract class BaseSingleStarTreeBuilder implements SingleTreeBuilder { +public abstract class BaseStarTreeBuilder implements SingleTreeBuilder { - // TODO: STAR_TREE_CODEC will be moved to CodecService once the Star Tree Codec is defined - public static final String STAR_TREE_CODEC = "startreecodec"; - - private static final Logger logger = LogManager.getLogger(BaseSingleStarTreeBuilder.class); + private static final Logger logger = LogManager.getLogger(BaseStarTreeBuilder.class); public static final int STAR_IN_DOC_VALUES_INDEX = -1; @@ -59,15 +56,13 @@ public abstract class BaseSingleStarTreeBuilder implements SingleTreeBuilder { protected final int numMetrics; protected final int numDimensions; - protected int numDocs; - protected int totalDocs; - protected int numNodes; + protected int numStarTreeDocs; + protected int totalSegmentDocs; + protected int numStarTreeNodes; protected final int maxLeafDocuments; protected final StarTreeBuilderUtils.TreeNode rootNode = getNewNode(); - // TODO: This will be initialized with OnHeap / OffHeap Implementations (Commented it's occurrences for now) - // private IndexOutput indexOutput; protected DocIdSetIterator[] dimensionReaders; protected DocIdSetIterator[] metricReaders; @@ -76,7 +71,7 @@ public abstract class BaseSingleStarTreeBuilder implements SingleTreeBuilder { protected DocValuesConsumer docValuesConsumer; protected DocValuesProducer docValuesProducer; - private final StarTreeDocValuesIteratorFactory starTreeDocValuesIteratorFactory; + private final StarTreeDocValuesIteratorAdapter starTreeDocValuesIteratorFactory; private final StarTreeField starTreeField; private final StarTreeFieldConfiguration starTreeFieldSpec; private final List metricStatFieldPairs; @@ -91,7 +86,7 @@ public abstract class BaseSingleStarTreeBuilder implements SingleTreeBuilder { * @param state stores the segment state * @param mapperService helps to find the original type of the field */ - protected BaseSingleStarTreeBuilder( + protected BaseStarTreeBuilder( StarTreeField starTreeField, DocValuesProducer docValuesProducer, DocValuesConsumer docValuesConsumer, @@ -99,26 +94,21 @@ protected BaseSingleStarTreeBuilder( MapperService mapperService ) throws IOException { - logger.info("Building in base star tree builder"); - - // String docFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, "stttree"); - // logger.info("Star tree file name : {}", docFileName); + logger.debug("Building in base star tree builder"); - // indexOutput = state.directory.createOutput(docFileName, state.context); - // CodecUtil.writeIndexHeader(indexOutput, STAR_TREE_CODEC, 0, state.segmentInfo.getId(), state.segmentSuffix); this.mapperService = mapperService; this.starTreeField = starTreeField; this.starTreeFieldSpec = starTreeField.getStarTreeConfig(); this.docValuesConsumer = docValuesConsumer; this.docValuesProducer = docValuesProducer; - this.starTreeDocValuesIteratorFactory = new StarTreeDocValuesIteratorFactory(); + this.starTreeDocValuesIteratorFactory = new StarTreeDocValuesIteratorAdapter(); List dimensionsSplitOrder = starTreeField.getDimensionsOrder(); this.numDimensions = dimensionsSplitOrder.size(); this.dimensionsSplitOrder = new String[numDimensions]; this.skipStarNodeCreationForDimensions = new HashSet<>(); - this.totalDocs = state.segmentInfo.maxDoc(); + this.totalSegmentDocs = state.segmentInfo.maxDoc(); this.dimensionReaders = new DocIdSetIterator[numDimensions]; Set skipStarNodeCreationForDimensions = this.starTreeFieldSpec.getSkipStarNodeCreationInDims(); @@ -130,7 +120,7 @@ protected BaseSingleStarTreeBuilder( } FieldInfo dimensionFieldInfos = state.fieldInfos.fieldInfo(dimension); DocValuesType dimensionDocValuesType = state.fieldInfos.fieldInfo(dimension).getDocValuesType(); - dimensionReaders[i] = starTreeDocValuesIteratorFactory.createIterator( + dimensionReaders[i] = starTreeDocValuesIteratorFactory.getDocValuesIterator( dimensionDocValuesType, dimensionFieldInfos, docValuesProducer @@ -160,7 +150,7 @@ protected BaseSingleStarTreeBuilder( String metricName = metricStatFieldPair.getField(); FieldInfo metricFieldInfos = state.fieldInfos.fieldInfo(metricName); DocValuesType metricDocValuesType = state.fieldInfos.fieldInfo(metricName).getDocValuesType(); - metricReaders[index] = starTreeDocValuesIteratorFactory.createIterator( + metricReaders[index] = starTreeDocValuesIteratorFactory.getDocValuesIterator( metricDocValuesType, metricFieldInfos, docValuesProducer @@ -237,9 +227,9 @@ public List generateMetricStatFieldPairs() { *

* * @param startDocId Start document id in the star-tree @@ -247,15 +237,15 @@ public List generateMetricStatFieldPairs() { * @param dimensionId Dimension id of the star-node * @return Iterator for the aggregated starTreeDocument */ - public abstract Iterator generateStarTreeForStarNode(int startDocId, int endDocId, int dimensionId) + public abstract Iterator generateStarTreeDocumentsForStarNode(int startDocId, int endDocId, int dimensionId) throws IOException; /** - * Returns the next segment star-tree document + * Returns the segment star-tree document */ - protected StarTreeDocument getNextSegmentStarTreeDocument() throws IOException { - long[] dimensions = getNextSegmentStarTreeDocumentDimensions(); - Object[] metrics = getNextSegmentStarTreeDocumentMetrics(); + protected StarTreeDocument getSegmentStarTreeDocument() throws IOException { + long[] dimensions = getStarTreeDimensionsFromSegment(); + Object[] metrics = getStarTreeMetricsFromSegment(); return new StarTreeDocument(dimensions, metrics); } @@ -265,7 +255,7 @@ protected StarTreeDocument getNextSegmentStarTreeDocument() throws IOException { * @return dimension values for each of the star-tree dimension * @throws IOException when we are unable to iterate to the next doc */ - long[] getNextSegmentStarTreeDocumentDimensions() throws IOException { + long[] getStarTreeDimensionsFromSegment() throws IOException { long[] dimensions = new long[numDimensions]; for (int i = 0; i < numDimensions; i++) { try { @@ -292,7 +282,7 @@ long[] getNextSegmentStarTreeDocumentDimensions() throws IOException { * @return metric values for each of the star-tree metric * @throws IOException when we are unable to iterate to the next doc */ - private Object[] getNextSegmentStarTreeDocumentMetrics() throws IOException { + private Object[] getStarTreeMetricsFromSegment() throws IOException { Object[] metrics = new Object[numMetrics]; for (int i = 0; i < numMetrics; i++) { // Ignore the column for COUNT aggregation function @@ -314,26 +304,26 @@ private Object[] getNextSegmentStarTreeDocumentMetrics() throws IOException { * *

Will create a new aggregated star-tree document if the current one is {@code null}. * - * @param aggregatedStarTreeDocument Aggregated star-tree document - * @param segmentStarTreeDocument Segment star-tree document + * @param aggregatedSegmentDocument Aggregated star-tree document + * @param segmentDocument Segment star-tree document * @return Merged starTreeDocument */ - protected StarTreeDocument aggregateSegmentStarTreeDocument( - StarTreeDocument aggregatedStarTreeDocument, - StarTreeDocument segmentStarTreeDocument + protected StarTreeDocument aggregateSegmentDocuments( + StarTreeDocument aggregatedSegmentDocument, + StarTreeDocument segmentDocument ) { // TODO: HANDLE KEYWORDS LATER! - if (aggregatedStarTreeDocument == null) { - long[] dimensions = Arrays.copyOf(segmentStarTreeDocument.dimensions, numDimensions); + if (aggregatedSegmentDocument == null) { + long[] dimensions = Arrays.copyOf(segmentDocument.dimensions, numDimensions); Object[] metrics = new Object[numMetrics]; for (int i = 0; i < numMetrics; i++) { try { StarTreeNumericType numericType = StarTreeNumericType.fromNumericType(numericTypes[i]); - metrics[i] = valueAggregators[i].getInitialAggregatedValue((Long) segmentStarTreeDocument.metrics[i], numericType); + metrics[i] = valueAggregators[i].getInitialAggregatedValue((Long) segmentDocument.metrics[i], numericType); } catch (IllegalArgumentException | NullPointerException e) { logger.error("Cannot parse initial aggregated value", e); throw new IllegalArgumentException( - "Cannot parse initial aggregated value [" + segmentStarTreeDocument.metrics[i] + "]" + "Cannot parse initial aggregated value [" + segmentDocument.metrics[i] + "]" ); } } @@ -342,17 +332,17 @@ protected StarTreeDocument aggregateSegmentStarTreeDocument( for (int i = 0; i < numMetrics; i++) { try { StarTreeNumericType numericType = StarTreeNumericType.fromNumericType(numericTypes[i]); - aggregatedStarTreeDocument.metrics[i] = valueAggregators[i].applySegmentRawValue( - aggregatedStarTreeDocument.metrics[i], - (Long) segmentStarTreeDocument.metrics[i], + aggregatedSegmentDocument.metrics[i] = valueAggregators[i].applySegmentRawValue( + aggregatedSegmentDocument.metrics[i], + (Long) segmentDocument.metrics[i], numericType ); } catch (IllegalArgumentException | NullPointerException e) { logger.error("Cannot apply segment raw value", e); - throw new IllegalArgumentException("Cannot aggregate on segment value [" + segmentStarTreeDocument.metrics[i] + "]"); + throw new IllegalArgumentException("Cannot aggregate on segment value [" + segmentDocument.metrics[i] + "]"); } } - return aggregatedStarTreeDocument; + return aggregatedSegmentDocument; } } @@ -361,54 +351,51 @@ protected StarTreeDocument aggregateSegmentStarTreeDocument( * *

Will create a new aggregated starTreeDocument if the current one is {@code null}. * - * @param aggregatedStarTreeDocument Aggregated star-tree document - * @param starTreeStarTreeDocument Star-tree document + * @param aggregatedDocument Aggregated star-tree document + * @param starTreeDocument Star-tree document * @return Merged star-tree document */ - public StarTreeDocument aggregateStarTreeDocument( - StarTreeDocument aggregatedStarTreeDocument, - StarTreeDocument starTreeStarTreeDocument + public StarTreeDocument aggregateDocuments( + StarTreeDocument aggregatedDocument, + StarTreeDocument starTreeDocument ) { // aggregate the documents - if (aggregatedStarTreeDocument == null) { - long[] dimensions = Arrays.copyOf(starTreeStarTreeDocument.dimensions, numDimensions); + if (aggregatedDocument == null) { + long[] dimensions = Arrays.copyOf(starTreeDocument.dimensions, numDimensions); Object[] metrics = new Object[numMetrics]; for (int i = 0; i < numMetrics; i++) { try { - metrics[i] = valueAggregators[i].cloneAggregatedValue(starTreeStarTreeDocument.metrics[i]); + metrics[i] = valueAggregators[i].getAggregatedValue(starTreeDocument.metrics[i]); } catch (IllegalArgumentException | NullPointerException e) { logger.error("Cannot clone aggregated value", e); - throw new IllegalArgumentException("Cannot clone aggregated value [" + starTreeStarTreeDocument.metrics[i] + "]"); + throw new IllegalArgumentException("Cannot clone aggregated value [" + starTreeDocument.metrics[i] + "]"); } } return new StarTreeDocument(dimensions, metrics); } else { for (int i = 0; i < numMetrics; i++) { try { - aggregatedStarTreeDocument.metrics[i] = valueAggregators[i].applyAggregatedValue( - starTreeStarTreeDocument.metrics[i], - aggregatedStarTreeDocument.metrics[i] + aggregatedDocument.metrics[i] = valueAggregators[i].applyAggregatedValue( + starTreeDocument.metrics[i], + aggregatedDocument.metrics[i] ); } catch (IllegalArgumentException | NullPointerException e) { logger.error("Cannot apply aggregated value", e); - throw new IllegalArgumentException("Cannot apply aggregated value [" + starTreeStarTreeDocument.metrics[i] + "]"); + throw new IllegalArgumentException("Cannot apply aggregated value [" + starTreeDocument.metrics[i] + "]"); } } - return aggregatedStarTreeDocument; + return aggregatedDocument; } } - // TODO: This will be taken care in off heap implementation for merges - // public abstract void build(List starTreeValues) throws IOException; - public void build() throws IOException { long startTime = System.currentTimeMillis(); - logger.info("Tree of Aggregations build is a go with config {}", starTreeField); + logger.debug("Tree of Aggregations build is a go with config {}", starTreeField); - Iterator starTreeDocumentIterator = processSegmentStarTreeDocuments(totalDocs); - logger.info("Sorting and aggregating star-tree in ms : {}", (System.currentTimeMillis() - startTime)); + Iterator starTreeDocumentIterator = processSegmentStarTreeDocuments(totalSegmentDocs); + logger.debug("Sorting and aggregating star-tree in ms : {}", (System.currentTimeMillis() - startTime)); build(starTreeDocumentIterator); - logger.info("Finished Building TOA in ms : {}", (System.currentTimeMillis() - startTime)); + logger.debug("Finished Building star-tree in ms : {}", (System.currentTimeMillis() - startTime)); } /** @@ -418,41 +405,36 @@ public void build() throws IOException { * @throws IOException when we are unable to build star-tree */ public void build(Iterator starTreeDocumentIterator) throws IOException { - int numSegmentStarTreeDocument = totalDocs; + int numSegmentStarTreeDocument = totalSegmentDocs; while (starTreeDocumentIterator.hasNext()) { appendToStarTree(starTreeDocumentIterator.next()); } - int numStarTreeDocument = numDocs; - logger.info("Generated star tree docs : [{}] from segment docs : [{}]", numStarTreeDocument, numSegmentStarTreeDocument); + int numStarTreeDocument = numStarTreeDocs; + logger.debug("Generated star tree docs : [{}] from segment docs : [{}]", numStarTreeDocument, numSegmentStarTreeDocument); - if (numDocs == 0) { + if (numStarTreeDocs == 0) { // TODO: Uncomment when segment codec is ready // StarTreeBuilderUtils.serializeTree(indexOutput, rootNode, dimensionsSplitOrder, numNodes); return; } - constructStarTree(rootNode, 0, numDocs); - int numStarTreeDocumentUnderStarNode = numDocs - numStarTreeDocument; - logger.info( + constructStarTree(rootNode, 0, numStarTreeDocs); + int numStarTreeDocumentUnderStarNode = numStarTreeDocs - numStarTreeDocument; + logger.debug( "Finished constructing star-tree, got [ {} ] tree nodes and [ {} ] starTreeDocument under star-node", - numNodes, + numStarTreeNodes, numStarTreeDocumentUnderStarNode ); createAggregatedDocs(rootNode); - int numAggregatedStarTreeDocument = numDocs - numStarTreeDocument - numStarTreeDocumentUnderStarNode; - logger.info("Finished creating aggregated documents : {}", numAggregatedStarTreeDocument); + int numAggregatedStarTreeDocument = numStarTreeDocs - numStarTreeDocument - numStarTreeDocumentUnderStarNode; + logger.debug("Finished creating aggregated documents : {}", numAggregatedStarTreeDocument); + // TODO: When StarTree Codec is ready // Create doc values indices in disk - // TODO: Uncomment when segment codec is ready - // createSortedDocValuesIndices(docValuesConsumer); - // Serialize and save in disk - // TODO: Uncomment when segment codec is ready - // StarTreeBuilderUtils.serializeTree(indexOutput, rootNode, dimensionsSplitOrder, numNodes); - - // TODO: Write star tree metadata for off heap implementation + // Write star tree metadata for off heap implementation } @@ -464,7 +446,7 @@ public void build(Iterator starTreeDocumentIterator) throws IO */ private void appendToStarTree(StarTreeDocument starTreeDocument) throws IOException { appendStarTreeDocument(starTreeDocument); - numDocs++; + numStarTreeDocs++; } /** @@ -473,7 +455,7 @@ private void appendToStarTree(StarTreeDocument starTreeDocument) throws IOExcept * @return return new star-tree node */ private StarTreeBuilderUtils.TreeNode getNewNode() { - numNodes++; + numStarTreeNodes++; return new StarTreeBuilderUtils.TreeNode(); } @@ -560,12 +542,12 @@ private StarTreeBuilderUtils.TreeNode constructStarNode(int startDocId, int endD StarTreeBuilderUtils.TreeNode starNode = getNewNode(); starNode.dimensionId = dimensionId; starNode.dimensionValue = StarTreeBuilderUtils.ALL; - starNode.startDocId = numDocs; - Iterator starTreeDocumentIterator = generateStarTreeForStarNode(startDocId, endDocId, dimensionId); + starNode.startDocId = numStarTreeDocs; + Iterator starTreeDocumentIterator = generateStarTreeDocumentsForStarNode(startDocId, endDocId, dimensionId); while (starTreeDocumentIterator.hasNext()) { appendToStarTree(starTreeDocumentIterator.next()); } - starNode.endDocId = numDocs; + starNode.endDocId = numStarTreeDocs; return starNode; } @@ -588,13 +570,13 @@ private StarTreeDocument createAggregatedDocs(StarTreeBuilderUtils.TreeNode node } else { // If it has multiple documents, aggregate all of them for (int i = node.startDocId; i < node.endDocId; i++) { - aggregatedStarTreeDocument = aggregateStarTreeDocument(aggregatedStarTreeDocument, getStarTreeDocument(i)); + aggregatedStarTreeDocument = aggregateDocuments(aggregatedStarTreeDocument, getStarTreeDocument(i)); } assert aggregatedStarTreeDocument != null; for (int i = node.dimensionId + 1; i < numDimensions; i++) { aggregatedStarTreeDocument.dimensions[i] = STAR_IN_DOC_VALUES_INDEX; } - node.aggregatedDocId = numDocs; + node.aggregatedDocId = numStarTreeDocs; appendToStarTree(aggregatedStarTreeDocument); } } else { @@ -612,13 +594,13 @@ private StarTreeDocument createAggregatedDocs(StarTreeBuilderUtils.TreeNode node } else { // If no star child exists, aggregate all aggregated documents from non-star children for (StarTreeBuilderUtils.TreeNode child : node.children.values()) { - aggregatedStarTreeDocument = aggregateStarTreeDocument(aggregatedStarTreeDocument, createAggregatedDocs(child)); + aggregatedStarTreeDocument = aggregateDocuments(aggregatedStarTreeDocument, createAggregatedDocs(child)); } assert aggregatedStarTreeDocument != null; for (int i = node.dimensionId + 1; i < numDimensions; i++) { aggregatedStarTreeDocument.dimensions[i] = STAR_IN_DOC_VALUES_INDEX; } - node.aggregatedDocId = numDocs; + node.aggregatedDocId = numStarTreeDocs; appendToStarTree(aggregatedStarTreeDocument); } } @@ -638,23 +620,7 @@ private long handleDateDimension(final String fieldName, final long val) { } public void close() throws IOException { - // boolean success = false; - // try { - // if (indexOutput != null) { - // indexOutput.writeInt(-1); - // CodecUtil.writeFooter(indexOutput); // write checksum - // } - // success = true; - // } catch (Exception e) { - // throw new RuntimeException(e); - // } finally { - // if (success) { - // IOUtils.close(indexOutput); - // } else { - // IOUtils.closeWhileHandlingException(indexOutput); - // } - // indexOutput = null; - // } + } } diff --git a/server/src/main/java/org/opensearch/common/inject/util/Modules.java b/server/src/main/java/org/opensearch/common/inject/util/Modules.java index b5a5a83ac3af9..71eafc182c573 100644 --- a/server/src/main/java/org/opensearch/common/inject/util/Modules.java +++ b/server/src/main/java/org/opensearch/common/inject/util/Modules.java @@ -202,13 +202,12 @@ public Void visit(Binding binding) { if (!overriddenKeys.remove(binding.getKey())) { super.visit(binding); - // record when a scope instance is used in a binding + // Record when a scope instance is used in a binding Scope scope = getScopeInstanceOrNull(binding); if (scope != null) { scopeInstancesInUse.put(scope, binding.getSource()); } } - return null; } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregator.java index a3a14f9bbb11a..36caf48ef0fb7 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregator.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregator.java @@ -53,7 +53,7 @@ public Double applyAggregatedValue(Double value, Double aggregatedValue) { } @Override - public Double cloneAggregatedValue(Double value) { + public Double getAggregatedValue(Double value) { return value; } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregator.java index b9fa3d5b69a3f..c139a18eef91b 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregator.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregator.java @@ -45,7 +45,7 @@ public interface ValueAggregator { /** * Clones an aggregated value. */ - A cloneAggregatedValue(A value); + A getAggregatedValue(A value); /** * Returns the maximum size in bytes of the aggregated values seen so far. diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericType.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericType.java index ed41445ac5cdf..cd7ec84015848 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericType.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericType.java @@ -38,8 +38,10 @@ public static StarTreeNumericType fromNumericType(IndexNumericFieldData.NumericT return StarTreeNumericType.FLOAT; case LONG: return StarTreeNumericType.LONG; - default: + case DOUBLE: return StarTreeNumericType.DOUBLE; + default: + throw new UnsupportedOperationException("Unknown numeric type [" + numericType + "]"); } } } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/DocValuesIteratorFactory.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/DocValuesIteratorAdapter.java similarity index 85% rename from server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/DocValuesIteratorFactory.java rename to server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/DocValuesIteratorAdapter.java index 66302f84f1449..6dc54e7139e9f 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/DocValuesIteratorFactory.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/DocValuesIteratorAdapter.java @@ -19,12 +19,12 @@ * An interface to support iterators for various doc values types. * @opensearch.experimental */ -public interface DocValuesIteratorFactory { +public interface DocValuesIteratorAdapter { /** * Creates an iterator for the given doc values type and field using the doc values producer */ - DocIdSetIterator createIterator(DocValuesType type, FieldInfo field, DocValuesProducer producer) throws IOException; + DocIdSetIterator getDocValuesIterator(DocValuesType type, FieldInfo field, DocValuesProducer producer) throws IOException; /** * Returns the next value for the given iterator diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapSingleTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapSingleTreeBuilder.java index 6baaedb1d0710..26e217d91dbc3 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapSingleTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapSingleTreeBuilder.java @@ -9,7 +9,7 @@ import org.apache.lucene.codecs.DocValuesConsumer; import org.apache.lucene.codecs.DocValuesProducer; -import org.apache.lucene.index.BaseSingleStarTreeBuilder; +import org.apache.lucene.index.BaseStarTreeBuilder; import org.apache.lucene.index.SegmentWriteState; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; import org.opensearch.index.compositeindex.datacube.startree.data.StarTreeDocument; @@ -24,7 +24,7 @@ /** * On heap single tree builder */ -public class OnHeapSingleTreeBuilder extends BaseSingleStarTreeBuilder { +public class OnHeapSingleTreeBuilder extends BaseStarTreeBuilder { private final List starTreeDocuments = new ArrayList<>(); @@ -69,17 +69,11 @@ public long getDimensionValue(int docId, int dimensionId) throws IOException { return starTreeDocuments.get(docId).dimensions[dimensionId]; } - // Handles star-tree rebuilds during merges :) - // @Override - // public void build(List starTreeValues) throws IOException { - // TODO: This will be handled during off heap implementation for merges - // } - @Override public Iterator processSegmentStarTreeDocuments(int numDocs) throws IOException { StarTreeDocument[] starTreeDocuments = new StarTreeDocument[numDocs]; for (int i = 0; i < numDocs; i++) { - starTreeDocuments[i] = getNextSegmentStarTreeDocument(); + starTreeDocuments[i] = getSegmentStarTreeDocument(); } return processStarTreeDocuments(starTreeDocuments); } @@ -96,7 +90,7 @@ public Iterator processStarTreeDocuments(StarTreeDocument[] st Arrays.sort(starTreeDocuments, (o1, o2) -> { for (int i = 0; i < numDimensions; i++) { if (o1.dimensions[i] != o2.dimensions[i]) { - return Math.toIntExact(o1.dimensions[i] - o2.dimensions[i]); + return Long.compare(o1.dimensions[i], o2.dimensions[i]); } } return 0; @@ -125,14 +119,14 @@ public boolean hasNext() { @Override public StarTreeDocument next() { // aggregate as we move on to the next doc - StarTreeDocument next = aggregateStarTreeDocument(null, currentStarTreeDocument); + StarTreeDocument next = aggregateDocuments(null, currentStarTreeDocument); while (docId < starTreeDocuments.length) { StarTreeDocument starTreeDocument = starTreeDocuments[docId++]; if (!Arrays.equals(starTreeDocument.dimensions, next.dimensions)) { currentStarTreeDocument = starTreeDocument; return next; } else { - next = aggregateStarTreeDocument(next, starTreeDocument); + next = aggregateDocuments(next, starTreeDocument); } } hasNext = false; @@ -150,7 +144,7 @@ public StarTreeDocument next() { * @throws IOException throws when unable to generate star-tree for star-node */ @Override - public Iterator generateStarTreeForStarNode(int startDocId, int endDocId, int dimensionId) throws IOException { + public Iterator generateStarTreeDocumentsForStarNode(int startDocId, int endDocId, int dimensionId) throws IOException { int numDocs = endDocId - startDocId; StarTreeDocument[] starTreeDocuments = new StarTreeDocument[numDocs]; for (int i = 0; i < numDocs; i++) { @@ -185,7 +179,7 @@ public boolean hasNext() { @Override public StarTreeDocument next() { - StarTreeDocument next = aggregateStarTreeDocument(null, currentStarTreeDocument); + StarTreeDocument next = aggregateDocuments(null, currentStarTreeDocument); next.dimensions[dimensionId] = STAR_IN_DOC_VALUES_INDEX; while (docId < numDocs) { StarTreeDocument starTreeDocument = starTreeDocuments[docId++]; @@ -193,7 +187,7 @@ public StarTreeDocument next() { currentStarTreeDocument = starTreeDocument; return next; } else { - next = aggregateStarTreeDocument(next, starTreeDocument); + next = aggregateDocuments(next, starTreeDocument); } } hasNext = false; diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/SingleTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/SingleTreeBuilder.java index 3b26f4c5e04c3..548a5261a78a3 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/SingleTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/SingleTreeBuilder.java @@ -22,12 +22,4 @@ public interface SingleTreeBuilder extends Closeable { * @throws IOException when we are unable to build star-tree */ void build() throws Exception; - - /** - * Builds the star tree using star-tree document values during segment merges - * @param starTreeValues star-tree document values - * @throws IOException when we are unable to build star-tree - */ - // void build(List starTreeValues) throws IOException; - } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocValuesIteratorFactory.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocValuesIteratorAdapter.java similarity index 88% rename from server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocValuesIteratorFactory.java rename to server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocValuesIteratorAdapter.java index 0105ff42045de..a48b944b4e6c7 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocValuesIteratorFactory.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocValuesIteratorAdapter.java @@ -21,10 +21,10 @@ * A factory class to return respective doc values iterator based on the doc volues type. * @opensearch.experimental */ -public class StarTreeDocValuesIteratorFactory implements DocValuesIteratorFactory { +public class StarTreeDocValuesIteratorAdapter implements DocValuesIteratorAdapter { @Override - public DocIdSetIterator createIterator(DocValuesType type, FieldInfo field, DocValuesProducer producer) throws IOException { + public DocIdSetIterator getDocValuesIterator(DocValuesType type, FieldInfo field, DocValuesProducer producer) throws IOException { switch (type) { case SORTED_SET: return producer.getSortedSet(field); diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/MultipleTreesBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilder.java similarity index 91% rename from server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/MultipleTreesBuilder.java rename to server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilder.java index ad3d33474840f..e8ec930c4a3a0 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/MultipleTreesBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilder.java @@ -22,9 +22,9 @@ import java.util.List; import java.util.Locale; -public class MultipleTreesBuilder implements Closeable { +public class StarTreesBuilder implements Closeable { - private static final Logger logger = LogManager.getLogger(MultipleTreesBuilder.class); + private static final Logger logger = LogManager.getLogger(StarTreesBuilder.class); private final List starTreeFields; private final StarTreeFieldConfiguration.StarTreeBuildMode buildMode; @@ -33,7 +33,7 @@ public class MultipleTreesBuilder implements Closeable { private final DocValuesProducer docValuesProducer; private final MapperService mapperService; - public MultipleTreesBuilder( + public StarTreesBuilder( List starTreeFields, StarTreeFieldConfiguration.StarTreeBuildMode buildMode, DocValuesProducer docValuesProducer, @@ -58,7 +58,7 @@ public MultipleTreesBuilder( public void build() throws Exception { long startTime = System.currentTimeMillis(); int numStarTrees = starTreeFields.size(); - logger.info("Starting building {} star-trees with configs: {} using {} builder", numStarTrees, starTreeFields, buildMode); + logger.debug("Starting building {} star-trees with configs: {} using {} builder", numStarTrees, starTreeFields, buildMode); // Build all star-trees for (int i = 0; i < numStarTrees; i++) { @@ -76,7 +76,7 @@ public void build() throws Exception { singleTreeBuilder.build(); } } - logger.info( + logger.debug( "Took {} ms to building {} star-trees with configs: {} using {} builder", System.currentTimeMillis() - startTime, numStarTrees, diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeBuilderUtils.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeBuilderUtils.java index 9fa980277f27b..e1841b3c2ff68 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeBuilderUtils.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeBuilderUtils.java @@ -7,19 +7,7 @@ */ package org.opensearch.index.compositeindex.datacube.startree.utils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.lucene.store.IndexOutput; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.LinkedList; -import java.util.List; import java.util.Map; -import java.util.Queue; - -import static java.nio.charset.StandardCharsets.UTF_8; /** * Util class for building star tree @@ -27,18 +15,10 @@ */ public class StarTreeBuilderUtils { - private static final Logger logger = LogManager.getLogger(StarTreeBuilderUtils.class); - - // TODO: To be moved to off heap star tree implementation - public static final int NUM_INT_SERIALIZABLE_FIELDS = 6; - public static final int NUM_LONG_SERIALIZABLE_FIELDS = 1; - public static final long SERIALIZABLE_SIZE_IN_BYTES = (Integer.BYTES * NUM_INT_SERIALIZABLE_FIELDS) + (Long.BYTES - * NUM_LONG_SERIALIZABLE_FIELDS); - - private StarTreeBuilderUtils() {} + private StarTreeBuilderUtils() { + } public static final int ALL = -1; - public static final long MAGIC_MARKER = 0xBADDA55B00DAD00DL; /** Tree node representation */ public static class TreeNode { @@ -51,81 +31,4 @@ public static class TreeNode { public Map children; } - /** Serializes the tree */ - public static void serializeTree(IndexOutput indexOutput, TreeNode rootNode, String[] dimensions, int numNodes) throws IOException { - int headerSizeInBytes = computeHeaderByteSize(dimensions); - long totalSizeInBytes = headerSizeInBytes + (long) numNodes * SERIALIZABLE_SIZE_IN_BYTES; - - logger.info("Star tree size in bytes : {}", totalSizeInBytes); - - writeHeader(indexOutput, headerSizeInBytes, dimensions, numNodes); - writeNodes(indexOutput, rootNode); - } - - /** Computes the size of the header for the tree */ - static int computeHeaderByteSize(String[] dimensions) { - // Magic marker (8), version (4), size of header (4) and number of dimensions (4) - int headerSizeInBytes = 20; - - for (String dimension : dimensions) { - headerSizeInBytes += Integer.BYTES; // For dimension index - headerSizeInBytes += Integer.BYTES; // For length of dimension name - headerSizeInBytes += dimension.getBytes(UTF_8).length; // For dimension name - } - - headerSizeInBytes += Integer.BYTES; // For number of nodes. - return headerSizeInBytes; - } - - /** Writes the header of the tree */ - static void writeHeader(IndexOutput output, int headerSizeInBytes, String[] dimensions, int numNodes) throws IOException { - output.writeLong(MAGIC_MARKER); - output.writeInt(1); - output.writeInt(headerSizeInBytes); - output.writeInt(dimensions.length); - for (int i = 0; i < dimensions.length; i++) { - output.writeInt(i); - output.writeString(dimensions[i]); - } - output.writeInt(numNodes); - } - - /** Writes the nodes of the tree */ - static void writeNodes(IndexOutput output, TreeNode rootNode) throws IOException { - Queue queue = new LinkedList<>(); - queue.add(rootNode); - - int currentNodeId = 0; - while (!queue.isEmpty()) { - TreeNode node = queue.remove(); - - if (node.children == null) { - writeNode(output, node, ALL, ALL); - } else { - // Sort all children nodes based on dimension value - List sortedChildren = new ArrayList<>(node.children.values()); - sortedChildren.sort(Comparator.comparingLong(o -> o.dimensionValue)); - - int firstChildId = currentNodeId + queue.size() + 1; - int lastChildId = firstChildId + sortedChildren.size() - 1; - writeNode(output, node, firstChildId, lastChildId); - - queue.addAll(sortedChildren); - } - - currentNodeId++; - } - } - - /** Writes a node of the tree */ - private static void writeNode(IndexOutput output, TreeNode node, int firstChildId, int lastChildId) throws IOException { - output.writeInt(node.dimensionId); - output.writeLong(node.dimensionValue); - output.writeInt(node.startDocId); - output.writeInt(node.endDocId); - output.writeInt(node.aggregatedDocId); - output.writeInt(firstChildId); - output.writeInt(lastChildId); - } - } diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregatorTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregatorTests.java index 58709934c894e..f30a9c82b1fd3 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregatorTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregatorTests.java @@ -40,8 +40,8 @@ public void testApplyAggregatedValue() { assertEquals(7.28, aggregator.applyAggregatedValue(3.14, 4.14), 0.0000001); } - public void testCloneAggregatedValue() { - assertEquals(3.14, aggregator.cloneAggregatedValue(3.14), 0.0); + public void testGetAggregatedValue() { + assertEquals(3.14, aggregator.getAggregatedValue(3.14), 0.0); } public void testGetMaxAggregatedValueByteSize() { diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseSingleStarTreeBuilderTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java similarity index 93% rename from server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseSingleStarTreeBuilderTests.java rename to server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java index 82be99faa8bd8..753297a8004ce 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseSingleStarTreeBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java @@ -11,7 +11,7 @@ import org.apache.lucene.codecs.DocValuesConsumer; import org.apache.lucene.codecs.DocValuesProducer; import org.apache.lucene.codecs.lucene99.Lucene99Codec; -import org.apache.lucene.index.BaseSingleStarTreeBuilder; +import org.apache.lucene.index.BaseStarTreeBuilder; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; @@ -52,9 +52,9 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -public class BaseSingleStarTreeBuilderTests extends OpenSearchTestCase { +public class BaseStarTreeBuilderTests extends OpenSearchTestCase { - private static BaseSingleStarTreeBuilder builder; + private static BaseStarTreeBuilder builder; private static MapperService mapperService; private static List dimensionsOrder; private static List fields = List.of( @@ -153,7 +153,7 @@ public static void setup() throws IOException { ); when(documentMapper.mappers()).thenReturn(fieldMappers); - builder = new BaseSingleStarTreeBuilder(compositeField, docValuesProducer, docValuesConsumer, state, mapperService) { + builder = new BaseStarTreeBuilder(compositeField, docValuesProducer, docValuesConsumer, state, mapperService) { @Override public void appendStarTreeDocument(StarTreeDocument starTreeDocument) throws IOException {} @@ -178,7 +178,7 @@ public Iterator processSegmentStarTreeDocuments(int numDocs) t } @Override - public Iterator generateStarTreeForStarNode(int startDocId, int endDocId, int dimensionId) + public Iterator generateStarTreeDocumentsForStarNode(int startDocId, int endDocId, int dimensionId) throws IOException { return null; } @@ -194,12 +194,12 @@ public void test_generateMetricStatFieldPairs() throws IOException { assertEquals(metricStatFieldPairs, expectedMetricStatFieldPairs); } - public void test_aggregateStarTreeDocument() { + public void test_aggregateDocuments() { StarTreeDocument starTreeDocument1 = new StarTreeDocument(new long[] { 1, 3, 5, 8 }, new Double[] { 4.0, 8.0 }); StarTreeDocument starTreeDocument2 = new StarTreeDocument(new long[] { 1, 3, 5, 8 }, new Double[] { 10.0, 6.0 }); StarTreeDocument expectedeMergedStarTreeDocument = new StarTreeDocument(new long[] { 1, 3, 5, 8 }, new Double[] { 14.0, 14.0 }); - StarTreeDocument mergedStarTreeDocument = builder.aggregateStarTreeDocument(starTreeDocument1, starTreeDocument2); + StarTreeDocument mergedStarTreeDocument = builder.aggregateDocuments(starTreeDocument1, starTreeDocument2); assertEquals(mergedStarTreeDocument.metrics[0], expectedeMergedStarTreeDocument.metrics[0]); assertEquals(mergedStarTreeDocument.metrics[1], expectedeMergedStarTreeDocument.metrics[1]); diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeValuesIteratorFactoryTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeValuesIteratorFactoryTests.java index 7f6978dbf2720..f64da30eb591d 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeValuesIteratorFactoryTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeValuesIteratorFactoryTests.java @@ -29,12 +29,12 @@ public class StarTreeValuesIteratorFactoryTests extends OpenSearchTestCase { - private static StarTreeDocValuesIteratorFactory factory; + private static StarTreeDocValuesIteratorAdapter factory; private static FieldInfo mockFieldInfo; @BeforeClass public static void setup() { - factory = new StarTreeDocValuesIteratorFactory(); + factory = new StarTreeDocValuesIteratorAdapter(); mockFieldInfo = new FieldInfo( "field", 1, @@ -60,7 +60,7 @@ public void testCreateIterator_SortedSet() throws IOException { DocValuesProducer producer = Mockito.mock(DocValuesProducer.class); SortedSetDocValues iterator = Mockito.mock(SortedSetDocValues.class); when(producer.getSortedSet(mockFieldInfo)).thenReturn(iterator); - DocIdSetIterator result = factory.createIterator(DocValuesType.SORTED_SET, mockFieldInfo, producer); + DocIdSetIterator result = factory.getDocValuesIterator(DocValuesType.SORTED_SET, mockFieldInfo, producer); assertEquals(iterator.getClass(), result.getClass()); } @@ -68,14 +68,14 @@ public void testCreateIterator_SortedNumeric() throws IOException { DocValuesProducer producer = Mockito.mock(DocValuesProducer.class); SortedNumericDocValues iterator = Mockito.mock(SortedNumericDocValues.class); when(producer.getSortedNumeric(mockFieldInfo)).thenReturn(iterator); - DocIdSetIterator result = factory.createIterator(DocValuesType.SORTED_NUMERIC, mockFieldInfo, producer); + DocIdSetIterator result = factory.getDocValuesIterator(DocValuesType.SORTED_NUMERIC, mockFieldInfo, producer); assertEquals(iterator.getClass(), result.getClass()); } public void testCreateIterator_UnsupportedType() { DocValuesProducer producer = Mockito.mock(DocValuesProducer.class); IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> { - factory.createIterator(DocValuesType.BINARY, mockFieldInfo, producer); + factory.getDocValuesIterator(DocValuesType.BINARY, mockFieldInfo, producer); }); assertEquals("Unsupported DocValuesType: BINARY", exception.getMessage()); } From d7ddd36458d05a4b2341f99770bb2c7790af927f Mon Sep 17 00:00:00 2001 From: Sarthak Aggarwal Date: Tue, 25 Jun 2024 20:57:18 +0530 Subject: [PATCH 10/11] addressed major nits Signed-off-by: Sarthak Aggarwal --- .../lucene/index/BaseStarTreeBuilder.java | 314 ++++++++------- .../lucene/index/StarTreeDocValuesWriter.java | 37 -- .../compositeindex/datacube/MetricStat.java | 3 +- .../startree/{data => }/StarTreeDocument.java | 6 +- .../aggregators/CountValueAggregator.java | 74 ++++ .../MetricAggregationDescriptor.java | 144 +++++++ .../aggregators/MetricStatFieldPair.java | 115 ------ .../aggregators/SumValueAggregator.java | 22 +- .../startree/aggregators/ValueAggregator.java | 14 +- .../aggregators/ValueAggregatorFactory.java | 11 +- .../numerictype/StarTreeNumericType.java | 4 + .../StarTreeNumericTypeConverters.java | 6 + .../builder/DocValuesIteratorAdapter.java | 38 -- .../builder/OnHeapSingleTreeBuilder.java | 33 +- .../startree/builder/SingleTreeBuilder.java | 3 + .../StarTreeDocValuesIteratorAdapter.java | 19 +- .../startree/builder/StarTreesBuilder.java | 20 +- .../datacube/startree/data/DataType.java | 67 ---- .../datacube/startree/node/package-info.java | 13 - .../startree/utils/StarTreeBuilderUtils.java | 9 +- .../MetricAggregationDescriptorTests.java | 110 +++++ .../aggregators/MetricStatFieldPairTests.java | 69 ---- .../aggregators/SumValueAggregatorTests.java | 17 +- .../ValueAggregatorFactoryTests.java | 24 +- .../builder/BaseStarTreeBuilderTests.java | 37 +- .../builder/OnHeapSingleTreeBuilderTests.java | 378 ++++++++++++++++-- .../StarTreeValuesIteratorFactoryTests.java | 6 +- 27 files changed, 958 insertions(+), 635 deletions(-) delete mode 100644 server/src/main/java/org/apache/lucene/index/StarTreeDocValuesWriter.java rename server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/{data => }/StarTreeDocument.java (79%) create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregator.java create mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregationDescriptor.java delete mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricStatFieldPair.java delete mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/DocValuesIteratorAdapter.java delete mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/data/DataType.java delete mode 100644 server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/package-info.java create mode 100644 server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregationDescriptorTests.java delete mode 100644 server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricStatFieldPairTests.java diff --git a/server/src/main/java/org/apache/lucene/index/BaseStarTreeBuilder.java b/server/src/main/java/org/apache/lucene/index/BaseStarTreeBuilder.java index f2f1f35d7c23f..425101678ba58 100644 --- a/server/src/main/java/org/apache/lucene/index/BaseStarTreeBuilder.java +++ b/server/src/main/java/org/apache/lucene/index/BaseStarTreeBuilder.java @@ -16,15 +16,14 @@ import org.opensearch.index.compositeindex.datacube.Dimension; import org.opensearch.index.compositeindex.datacube.Metric; import org.opensearch.index.compositeindex.datacube.MetricStat; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeDocument; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; -import org.opensearch.index.compositeindex.datacube.startree.aggregators.MetricStatFieldPair; +import org.opensearch.index.compositeindex.datacube.startree.aggregators.MetricAggregationDescriptor; import org.opensearch.index.compositeindex.datacube.startree.aggregators.ValueAggregator; -import org.opensearch.index.compositeindex.datacube.startree.aggregators.ValueAggregatorFactory; import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; import org.opensearch.index.compositeindex.datacube.startree.builder.SingleTreeBuilder; import org.opensearch.index.compositeindex.datacube.startree.builder.StarTreeDocValuesIteratorAdapter; -import org.opensearch.index.compositeindex.datacube.startree.data.StarTreeDocument; import org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeBuilderUtils; import org.opensearch.index.fielddata.IndexNumericFieldData; import org.opensearch.index.mapper.Mapper; @@ -43,6 +42,7 @@ /** * Base class for star-tree builder + * @opensearch.experimental */ public abstract class BaseStarTreeBuilder implements SingleTreeBuilder { @@ -50,10 +50,9 @@ public abstract class BaseStarTreeBuilder implements SingleTreeBuilder { public static final int STAR_IN_DOC_VALUES_INDEX = -1; - protected final String[] dimensionsSplitOrder; protected final Set skipStarNodeCreationForDimensions; - protected final String[] metrics; + protected final List metricAggregationDescriptors; protected final int numMetrics; protected final int numDimensions; protected int numStarTreeDocs; @@ -64,31 +63,25 @@ public abstract class BaseStarTreeBuilder implements SingleTreeBuilder { protected final StarTreeBuilderUtils.TreeNode rootNode = getNewNode(); protected DocIdSetIterator[] dimensionReaders; - protected DocIdSetIterator[] metricReaders; - protected ValueAggregator[] valueAggregators; - protected IndexNumericFieldData.NumericType[] numericTypes; + protected Map fieldProducerMap; protected DocValuesConsumer docValuesConsumer; - protected DocValuesProducer docValuesProducer; private final StarTreeDocValuesIteratorAdapter starTreeDocValuesIteratorFactory; private final StarTreeField starTreeField; - private final StarTreeFieldConfiguration starTreeFieldSpec; - private final List metricStatFieldPairs; - private final MapperService mapperService; /** * Constructor for base star-tree builder * * @param starTreeField holds the configuration for the star tree - * @param docValuesProducer helps return the doc values iterator for each type based on field name + * @param fieldProducerMap helps return the doc values iterator for each type based on field name * @param docValuesConsumer to consume the new aggregated metrics during flush - * @param state stores the segment state + * @param state stores the segment write state * @param mapperService helps to find the original type of the field */ protected BaseStarTreeBuilder( StarTreeField starTreeField, - DocValuesProducer docValuesProducer, + Map fieldProducerMap, DocValuesConsumer docValuesConsumer, SegmentWriteState state, MapperService mapperService @@ -96,25 +89,22 @@ protected BaseStarTreeBuilder( logger.debug("Building in base star tree builder"); - this.mapperService = mapperService; this.starTreeField = starTreeField; - this.starTreeFieldSpec = starTreeField.getStarTreeConfig(); + StarTreeFieldConfiguration starTreeFieldSpec = starTreeField.getStarTreeConfig(); this.docValuesConsumer = docValuesConsumer; - this.docValuesProducer = docValuesProducer; + this.fieldProducerMap = fieldProducerMap; this.starTreeDocValuesIteratorFactory = new StarTreeDocValuesIteratorAdapter(); List dimensionsSplitOrder = starTreeField.getDimensionsOrder(); this.numDimensions = dimensionsSplitOrder.size(); - this.dimensionsSplitOrder = new String[numDimensions]; this.skipStarNodeCreationForDimensions = new HashSet<>(); this.totalSegmentDocs = state.segmentInfo.maxDoc(); this.dimensionReaders = new DocIdSetIterator[numDimensions]; - Set skipStarNodeCreationForDimensions = this.starTreeFieldSpec.getSkipStarNodeCreationInDims(); + Set skipStarNodeCreationForDimensions = starTreeFieldSpec.getSkipStarNodeCreationInDims(); for (int i = 0; i < numDimensions; i++) { String dimension = dimensionsSplitOrder.get(i).getField(); - this.dimensionsSplitOrder[i] = dimension; if (skipStarNodeCreationForDimensions.contains(dimensionsSplitOrder.get(i).getField())) { this.skipStarNodeCreationForDimensions.add(i); } @@ -123,41 +113,12 @@ protected BaseStarTreeBuilder( dimensionReaders[i] = starTreeDocValuesIteratorFactory.getDocValuesIterator( dimensionDocValuesType, dimensionFieldInfos, - docValuesProducer + fieldProducerMap.get(dimensionFieldInfos.name) ); } - this.metricStatFieldPairs = generateMetricStatFieldPairs(); - this.numMetrics = metricStatFieldPairs.size(); - this.metrics = new String[numMetrics]; - this.valueAggregators = new ValueAggregator[numMetrics]; - this.numericTypes = new IndexNumericFieldData.NumericType[numMetrics]; - this.metricReaders = new DocIdSetIterator[numMetrics]; - - int index = 0; - for (MetricStatFieldPair metricStatFieldPair : metricStatFieldPairs) { - metrics[index] = metricStatFieldPair.toFieldName(); - valueAggregators[index] = ValueAggregatorFactory.getValueAggregator(metricStatFieldPair.getMetricStat()); - - Mapper fieldMapper = mapperService.documentMapper().mappers().getMapper(metrics[index]); - if (fieldMapper instanceof NumberFieldMapper) { - numericTypes[index] = ((NumberFieldMapper) fieldMapper).fieldType().numericType(); - } else { - numericTypes[index] = IndexNumericFieldData.NumericType.DOUBLE; - } - // Ignore the column for COUNT aggregation function - if (valueAggregators[index].getAggregationType() != MetricStat.COUNT) { - String metricName = metricStatFieldPair.getField(); - FieldInfo metricFieldInfos = state.fieldInfos.fieldInfo(metricName); - DocValuesType metricDocValuesType = state.fieldInfos.fieldInfo(metricName).getDocValuesType(); - metricReaders[index] = starTreeDocValuesIteratorFactory.getDocValuesIterator( - metricDocValuesType, - metricFieldInfos, - docValuesProducer - ); - } - index++; - } + this.metricAggregationDescriptors = generateMetricStatFieldPairs(mapperService, state); + this.numMetrics = metricAggregationDescriptors.size(); this.maxLeafDocuments = starTreeFieldSpec.maxLeafDocs(); } @@ -166,82 +127,98 @@ protected BaseStarTreeBuilder( * * @return list of metric stat mapped with respective fields */ - public List generateMetricStatFieldPairs() { - List metricStatFieldPairs = new ArrayList<>(); + public List generateMetricStatFieldPairs(MapperService mapperService, SegmentWriteState state) + throws IOException { + List metricAggregationDescriptors = new ArrayList<>(); + IndexNumericFieldData.NumericType numericType; + DocIdSetIterator metricStatReader = null; for (Metric metric : this.starTreeField.getMetrics()) { for (MetricStat metricType : metric.getMetrics()) { - MetricStatFieldPair metricStatFieldPair = new MetricStatFieldPair(metricType, metric.getField()); - metricStatFieldPairs.add(metricStatFieldPair); + Mapper fieldMapper = mapperService.documentMapper().mappers().getMapper(metric.getField()); + if (fieldMapper instanceof NumberFieldMapper) { + numericType = ((NumberFieldMapper) fieldMapper).fieldType().numericType(); + } else { + logger.error("metric mapper is not of type number field mapper"); + throw new IllegalStateException("metric mapper is not of type number field mapper"); + } + // Ignore the column for COUNT aggregation function + if (metricType != MetricStat.COUNT) { + FieldInfo metricFieldInfos = state.fieldInfos.fieldInfo(metric.getField()); + DocValuesType metricDocValuesType = metricFieldInfos.getDocValuesType(); + metricStatReader = starTreeDocValuesIteratorFactory.getDocValuesIterator( + metricDocValuesType, + metricFieldInfos, + fieldProducerMap.get(metricFieldInfos.name) + ); + } + MetricAggregationDescriptor metricAggregationDescriptor = new MetricAggregationDescriptor( + metricType, + metric.getField(), + numericType, + metricStatReader + ); + metricAggregationDescriptors.add(metricAggregationDescriptor); } } - return metricStatFieldPairs; + return metricAggregationDescriptors; } /** - * Appends a star-tree document to the star-tree. + * Adds a document to the star-tree. * - * @param starTreeDocument star tree document to be appended + * @param starTreeDocument star tree document to be added + * @throws IOException if an I/O error occurs while adding the document */ public abstract void appendStarTreeDocument(StarTreeDocument starTreeDocument) throws IOException; /** - * Returns the star-tree document of the given document id in the star-tree. + * Returns the document of the given document id in the star-tree. * - * @param docId Document dd - * @return Star tree document + * @param docId document id + * @return star tree document + * @throws IOException if an I/O error occurs while fetching the star-tree document */ public abstract StarTreeDocument getStarTreeDocument(int docId) throws IOException; /** - * Returns the star-tree document of the given document id in the star-tree. + * Retrieves the list of star-tree documents in the star-tree. * - * @return Star tree document + * @return Star tree documents */ public abstract List getStarTreeDocuments() throws IOException; /** - * Returns the dimension value of the given document and dimension id in the star-tree. + * Returns the value of the dimension for the given dimension id and document in the star-tree. * - * @param docId Document Id - * @param dimensionId Dimension Id - * @return Dimension value + * @param docId document id + * @param dimensionId dimension id + * @return dimension value */ public abstract long getDimensionValue(int docId, int dimensionId) throws IOException; /** - * Sorts and aggregates the star-tree Document in the segment, and returns a star-tree document iterator for all the + * Sorts and aggregates the star-tree document in the segment, and returns a star-tree document iterator for all the * aggregated star-tree document. * - *

This method reads star-tree document from segment and generates the initial star-tree document for the star-tree. - * - * @param numDocs Number of documents in the segment + * @param numDocs number of documents in the given segment * @return Iterator for the aggregated star-tree document */ - public abstract Iterator processSegmentStarTreeDocuments(int numDocs) throws IOException; + public abstract Iterator sortMergeAndAggregateStarTreeDocument(int numDocs) throws IOException; /** * Generates aggregated star-tree document for star-node. * - *

This method will do the following steps: - * - *

- * - * @param startDocId Start document id in the star-tree - * @param endDocId End document id (exclusive) in the star-tree - * @param dimensionId Dimension id of the star-node - * @return Iterator for the aggregated starTreeDocument + * @param startDocId start document id (inclusive) in the star-tree + * @param endDocId end document id (exclusive) in the star-tree + * @param dimensionId dimension id of the star-node + * @return Iterator for the aggregated star-tree documents */ public abstract Iterator generateStarTreeDocumentsForStarNode(int startDocId, int endDocId, int dimensionId) throws IOException; /** - * Returns the segment star-tree document + * Returns the star-tree document from the segment + * @throws IOException when we are unable to build a star tree document from the segment */ protected StarTreeDocument getSegmentStarTreeDocument() throws IOException { long[] dimensions = getStarTreeDimensionsFromSegment(); @@ -250,10 +227,10 @@ protected StarTreeDocument getSegmentStarTreeDocument() throws IOException { } /** - * Returns the next segment star-tree document for the dimensions + * Returns the dimension values for the next document from the segment * * @return dimension values for each of the star-tree dimension - * @throws IOException when we are unable to iterate to the next doc + * @throws IOException when we are unable to iterate to the next doc for the given dimension readers */ long[] getStarTreeDimensionsFromSegment() throws IOException { long[] dimensions = new long[numDimensions]; @@ -262,84 +239,100 @@ long[] getStarTreeDimensionsFromSegment() throws IOException { dimensionReaders[i].nextDoc(); } catch (IOException e) { logger.error("unable to iterate to next doc", e); + } catch (NullPointerException e) { + logger.error("dimension does not have an associated reader", e); + throw new IllegalStateException("dimension should have a valid associated reader", e); + } catch (Exception e) { + logger.error("unable to read the dimension values from the segment", e); + throw new IllegalStateException("unable to read the dimension values from the segment", e); } if (starTreeField.getDimensionsOrder().get(i) instanceof DateDimension) { dimensions[i] = handleDateDimension( - dimensionsSplitOrder[i], - starTreeDocValuesIteratorFactory.getNextValue(dimensionReaders[i]) + starTreeField.getDimensionsOrder().get(i).getField(), + starTreeDocValuesIteratorFactory.getNextOrd(dimensionReaders[i]) ); } else { - dimensions[i] = starTreeDocValuesIteratorFactory.getNextValue(dimensionReaders[i]); + dimensions[i] = starTreeDocValuesIteratorFactory.getNextOrd(dimensionReaders[i]); } } return dimensions; } /** - * Returns the next segment star-tree document for the metrics + * Returns the metric values for the next document from the segment * * @return metric values for each of the star-tree metric - * @throws IOException when we are unable to iterate to the next doc + * @throws IOException when we are unable to iterate to the next doc for the given metric readers */ private Object[] getStarTreeMetricsFromSegment() throws IOException { Object[] metrics = new Object[numMetrics]; for (int i = 0; i < numMetrics; i++) { // Ignore the column for COUNT aggregation function - if (metricReaders[i] != null) { + DocIdSetIterator metricStatReader = metricAggregationDescriptors.get(i).getMetricStatReader(); + if (metricStatReader != null) { try { - metricReaders[i].nextDoc(); + metricStatReader.nextDoc(); } catch (IOException e) { - // TODO : handle null values in columns logger.error("unable to iterate to next doc", e); + } catch (NullPointerException e) { + logger.error("metric does not have an associated reader", e); + throw new IllegalStateException("metric should have a valid associated reader", e); + } catch (Exception e) { + logger.error("unable to read the metric values from the segment", e); + throw new IllegalStateException("unable to read the metric values from the segment", e); } - metrics[i] = starTreeDocValuesIteratorFactory.getNextValue(metricReaders[i]); + metrics[i] = starTreeDocValuesIteratorFactory.getNextOrd(metricStatReader); } } return metrics; } /** - * Merges a segment star-tree document (raw) into the aggregated star-tree document. + * Merges a star-tree document from the segment into an aggregated star-tree document. + * A new aggregated star-tree document is created if the aggregated segment document is null. * - *

Will create a new aggregated star-tree document if the current one is {@code null}. - * - * @param aggregatedSegmentDocument Aggregated star-tree document - * @param segmentDocument Segment star-tree document - * @return Merged starTreeDocument + * @param aggregatedSegmentDocument aggregated star-tree document + * @param segmentDocument segment star-tree document + * @return merged star-tree document */ - protected StarTreeDocument aggregateSegmentDocuments( + protected StarTreeDocument reduceSegmentStarTreeDocuments( StarTreeDocument aggregatedSegmentDocument, StarTreeDocument segmentDocument ) { - // TODO: HANDLE KEYWORDS LATER! if (aggregatedSegmentDocument == null) { long[] dimensions = Arrays.copyOf(segmentDocument.dimensions, numDimensions); Object[] metrics = new Object[numMetrics]; for (int i = 0; i < numMetrics; i++) { try { - StarTreeNumericType numericType = StarTreeNumericType.fromNumericType(numericTypes[i]); - metrics[i] = valueAggregators[i].getInitialAggregatedValue((Long) segmentDocument.metrics[i], numericType); - } catch (IllegalArgumentException | NullPointerException e) { - logger.error("Cannot parse initial aggregated value", e); - throw new IllegalArgumentException( - "Cannot parse initial aggregated value [" + segmentDocument.metrics[i] + "]" - ); + ValueAggregator metricValueAggregator = metricAggregationDescriptors.get(i).getValueAggregators(); + StarTreeNumericType starTreeNumericType = metricAggregationDescriptors.get(i).getStarTreeNumericType(); + metrics[i] = metricValueAggregator.getInitialAggregatedValue((Long) segmentDocument.metrics[i], starTreeNumericType); + } catch (IllegalArgumentException | NullPointerException | IllegalStateException e) { + logger.error("Cannot parse initial segment doc value", e); + throw new IllegalStateException("Cannot parse initial segment doc value [" + segmentDocument.metrics[i] + "]"); + } catch (Exception e) { + logger.error("Cannot parse initial segment doc value", e); + throw new RuntimeException("Cannot parse initial segment doc value [" + segmentDocument.metrics[i] + "]"); } } return new StarTreeDocument(dimensions, metrics); } else { for (int i = 0; i < numMetrics; i++) { try { - StarTreeNumericType numericType = StarTreeNumericType.fromNumericType(numericTypes[i]); - aggregatedSegmentDocument.metrics[i] = valueAggregators[i].applySegmentRawValue( + ValueAggregator metricValueAggregator = metricAggregationDescriptors.get(i).getValueAggregators(); + StarTreeNumericType starTreeNumericType = metricAggregationDescriptors.get(i).getStarTreeNumericType(); + aggregatedSegmentDocument.metrics[i] = metricValueAggregator.applySegmentRawValue( aggregatedSegmentDocument.metrics[i], (Long) segmentDocument.metrics[i], - numericType + starTreeNumericType ); - } catch (IllegalArgumentException | NullPointerException e) { - logger.error("Cannot apply segment raw value", e); - throw new IllegalArgumentException("Cannot aggregate on segment value [" + segmentDocument.metrics[i] + "]"); + } catch (IllegalArgumentException | NullPointerException | IllegalStateException e) { + logger.error("Cannot apply segment doc value for aggregation", e); + throw new IllegalStateException("Cannot apply segment doc value for aggregation [" + segmentDocument.metrics[i] + "]"); + } catch (Exception e) { + logger.error("Cannot apply segment doc value for aggregation", e); + throw new RuntimeException("Cannot apply segment doc value for aggregation [" + segmentDocument.metrics[i] + "]"); } } return aggregatedSegmentDocument; @@ -347,52 +340,62 @@ protected StarTreeDocument aggregateSegmentDocuments( } /** - * Merges a star-tree document (aggregated) into the aggregated document. - * - *

Will create a new aggregated starTreeDocument if the current one is {@code null}. + * Merges a star-tree document into an aggregated star-tree document. + * A new aggregated star-tree document is created if the aggregated document is null. * - * @param aggregatedDocument Aggregated star-tree document - * @param starTreeDocument Star-tree document - * @return Merged star-tree document + * @param aggregatedDocument aggregated star-tree document + * @param starTreeDocument segment star-tree document + * @return merged star-tree document */ - public StarTreeDocument aggregateDocuments( - StarTreeDocument aggregatedDocument, - StarTreeDocument starTreeDocument - ) { + public StarTreeDocument reduceStarTreeDocuments(StarTreeDocument aggregatedDocument, StarTreeDocument starTreeDocument) { // aggregate the documents if (aggregatedDocument == null) { long[] dimensions = Arrays.copyOf(starTreeDocument.dimensions, numDimensions); Object[] metrics = new Object[numMetrics]; for (int i = 0; i < numMetrics; i++) { try { - metrics[i] = valueAggregators[i].getAggregatedValue(starTreeDocument.metrics[i]); + metrics[i] = metricAggregationDescriptors.get(i).getValueAggregators().getAggregatedValue(starTreeDocument.metrics[i]); } catch (IllegalArgumentException | NullPointerException e) { - logger.error("Cannot clone aggregated value", e); - throw new IllegalArgumentException("Cannot clone aggregated value [" + starTreeDocument.metrics[i] + "]"); + logger.error("Cannot get aggregated value", e); + throw new IllegalArgumentException("Cannot get aggregated value[" + starTreeDocument.metrics[i] + "]"); + } catch (Exception e) { + logger.error("Cannot get value for aggregation", e); + throw new RuntimeException("Cannot get value for aggregation[" + starTreeDocument.metrics[i] + "]"); } } return new StarTreeDocument(dimensions, metrics); } else { for (int i = 0; i < numMetrics; i++) { try { - aggregatedDocument.metrics[i] = valueAggregators[i].applyAggregatedValue( - starTreeDocument.metrics[i], - aggregatedDocument.metrics[i] - ); + aggregatedDocument.metrics[i] = metricAggregationDescriptors.get(i) + .getValueAggregators() + .applyAggregatedValue(starTreeDocument.metrics[i], aggregatedDocument.metrics[i]); } catch (IllegalArgumentException | NullPointerException e) { - logger.error("Cannot apply aggregated value", e); - throw new IllegalArgumentException("Cannot apply aggregated value [" + starTreeDocument.metrics[i] + "]"); + logger.error("Cannot apply value to aggregated document for aggregation", e); + throw new IllegalArgumentException( + "Cannot apply value to aggregated document for aggregation[" + starTreeDocument.metrics[i] + "]" + ); + } catch (Exception e) { + logger.error("Cannot apply value to aggregated document for aggregation", e); + throw new RuntimeException( + "Cannot apply value to aggregated document for aggregation [" + starTreeDocument.metrics[i] + "]" + ); } } return aggregatedDocument; } } + /** + * Builds the star tree using total segment documents + * + * @throws IOException when we are unable to build star-tree + */ public void build() throws IOException { long startTime = System.currentTimeMillis(); - logger.debug("Tree of Aggregations build is a go with config {}", starTreeField); + logger.debug("Star-tree build is a go with star tree field{}", starTreeField); - Iterator starTreeDocumentIterator = processSegmentStarTreeDocuments(totalSegmentDocs); + Iterator starTreeDocumentIterator = sortMergeAndAggregateStarTreeDocument(totalSegmentDocs); logger.debug("Sorting and aggregating star-tree in ms : {}", (System.currentTimeMillis() - startTime)); build(starTreeDocumentIterator); logger.debug("Finished Building star-tree in ms : {}", (System.currentTimeMillis() - startTime)); @@ -439,10 +442,10 @@ public void build(Iterator starTreeDocumentIterator) throws IO } /** - * Appends a starTreeDocument to star tree + * Adds a document to star-tree * * @param starTreeDocument star-tree document - * @throws IOException throws an exception if we are unable to append the doc + * @throws IOException throws an exception if we are unable to add the doc */ private void appendToStarTree(StarTreeDocument starTreeDocument) throws IOException { appendStarTreeDocument(starTreeDocument); @@ -450,7 +453,7 @@ private void appendToStarTree(StarTreeDocument starTreeDocument) throws IOExcept } /** - * Returns a new node + * Returns a new star-tree node * * @return return new star-tree node */ @@ -460,11 +463,11 @@ private StarTreeBuilderUtils.TreeNode getNewNode() { } /** - * Implements the algorithm to construct a star-tree based on star-tree documents + * Implements the algorithm to construct a star-tree * - * @param node star-tree node - * @param startDocId start document id - * @param endDocId end document id + * @param node star-tree node + * @param startDocId start document id + * @param endDocId end document id * @throws IOException throws an exception if we are unable to construct the tree */ private void constructStarTree(StarTreeBuilderUtils.TreeNode node, int startDocId, int endDocId) throws IOException { @@ -542,6 +545,7 @@ private StarTreeBuilderUtils.TreeNode constructStarNode(int startDocId, int endD StarTreeBuilderUtils.TreeNode starNode = getNewNode(); starNode.dimensionId = dimensionId; starNode.dimensionValue = StarTreeBuilderUtils.ALL; + starNode.isStarNode = true; starNode.startDocId = numStarTreeDocs; Iterator starTreeDocumentIterator = generateStarTreeDocumentsForStarNode(startDocId, endDocId, dimensionId); while (starTreeDocumentIterator.hasNext()) { @@ -570,9 +574,12 @@ private StarTreeDocument createAggregatedDocs(StarTreeBuilderUtils.TreeNode node } else { // If it has multiple documents, aggregate all of them for (int i = node.startDocId; i < node.endDocId; i++) { - aggregatedStarTreeDocument = aggregateDocuments(aggregatedStarTreeDocument, getStarTreeDocument(i)); + aggregatedStarTreeDocument = reduceStarTreeDocuments(aggregatedStarTreeDocument, getStarTreeDocument(i)); } assert aggregatedStarTreeDocument != null; + if (null == aggregatedStarTreeDocument) { + throw new IllegalStateException("aggregated star-tree document is null after reducing the documents"); + } for (int i = node.dimensionId + 1; i < numDimensions; i++) { aggregatedStarTreeDocument.dimensions[i] = STAR_IN_DOC_VALUES_INDEX; } @@ -584,7 +591,7 @@ private StarTreeDocument createAggregatedDocs(StarTreeBuilderUtils.TreeNode node if (node.children.containsKey((long) StarTreeBuilderUtils.ALL)) { // If it has star child, use the star child aggregated document directly for (StarTreeBuilderUtils.TreeNode child : node.children.values()) { - if (child.dimensionValue == StarTreeBuilderUtils.ALL) { + if (child.isStarNode) { aggregatedStarTreeDocument = createAggregatedDocs(child); node.aggregatedDocId = child.aggregatedDocId; } else { @@ -594,9 +601,12 @@ private StarTreeDocument createAggregatedDocs(StarTreeBuilderUtils.TreeNode node } else { // If no star child exists, aggregate all aggregated documents from non-star children for (StarTreeBuilderUtils.TreeNode child : node.children.values()) { - aggregatedStarTreeDocument = aggregateDocuments(aggregatedStarTreeDocument, createAggregatedDocs(child)); + aggregatedStarTreeDocument = reduceStarTreeDocuments(aggregatedStarTreeDocument, createAggregatedDocs(child)); } assert aggregatedStarTreeDocument != null; + if (null == aggregatedStarTreeDocument) { + throw new IllegalStateException("aggregated star-tree document is null after reducing the documents"); + } for (int i = node.dimensionId + 1; i < numDimensions; i++) { aggregatedStarTreeDocument.dimensions[i] = STAR_IN_DOC_VALUES_INDEX; } diff --git a/server/src/main/java/org/apache/lucene/index/StarTreeDocValuesWriter.java b/server/src/main/java/org/apache/lucene/index/StarTreeDocValuesWriter.java deleted file mode 100644 index e8d96226da2ee..0000000000000 --- a/server/src/main/java/org/apache/lucene/index/StarTreeDocValuesWriter.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.apache.lucene.index; - -/** - * A wrapper for {@link DocValuesWriter} that contains the {@link DocValuesType} of the doc - */ -public class StarTreeDocValuesWriter { - - private final DocValuesType docValuesType; - private final DocValuesWriter docValuesWriter; - - public StarTreeDocValuesWriter(DocValuesType docValuesType, DocValuesWriter docValuesWriter) { - this.docValuesType = docValuesType; - this.docValuesWriter = docValuesWriter; - } - - /** - * Get the doc values type - */ - public DocValuesType getDocValuesType() { - return docValuesType; - } - - /** - * Get the doc values writer - */ - public DocValuesWriter getDocValuesWriter() { - return docValuesWriter; - } -} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/MetricStat.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/MetricStat.java index a5a6d07ad9bbc..fbde296b15f7e 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/MetricStat.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/MetricStat.java @@ -21,8 +21,7 @@ public enum MetricStat { AVG("avg"), SUM("sum"), MIN("min"), - MAX("max"), - UNSUPPORTED("unsupported"); + MAX("max"); private final String typeName; diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/data/StarTreeDocument.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeDocument.java similarity index 79% rename from server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/data/StarTreeDocument.java rename to server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeDocument.java index cb4253e21c141..3d537be1c3227 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/data/StarTreeDocument.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeDocument.java @@ -6,13 +6,17 @@ * compatible open source license. */ -package org.opensearch.index.compositeindex.datacube.startree.data; +package org.opensearch.index.compositeindex.datacube.startree; + +import org.opensearch.common.annotation.ExperimentalApi; import java.util.Arrays; /** * Star tree document + * @opensearch.experimental */ +@ExperimentalApi public class StarTreeDocument { public final long[] dimensions; public final Object[] metrics; diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregator.java new file mode 100644 index 0000000000000..b4837809b9706 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregator.java @@ -0,0 +1,74 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.index.compositeindex.datacube.startree.aggregators; + +import org.apache.lucene.util.NumericUtils; +import org.opensearch.index.compositeindex.datacube.MetricStat; +import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; + +/** + * Count value aggregator for star tree + * + * @opensearch.experimental + */ +public class CountValueAggregator implements ValueAggregator { + public static final StarTreeNumericType STAR_TREE_NUMERIC_TYPE = StarTreeNumericType.DOUBLE; + + @Override + public MetricStat getAggregationType() { + return MetricStat.COUNT; + } + + @Override + public StarTreeNumericType getStarTreeNumericType() { + return STAR_TREE_NUMERIC_TYPE; + } + + @Override + public Double getInitialAggregatedValue(Long segmentDocValue, StarTreeNumericType starTreeNumericType) { + return 1.0; + } + + @Override + public Double applySegmentRawValue(Double value, Long segmentDocValue, StarTreeNumericType starTreeNumericType) { + return value + 1; + } + + @Override + public Double applyAggregatedValue(Double value, Double aggregatedValue) { + return value + aggregatedValue; + } + + @Override + public Double getAggregatedValue(Double value) { + return value; + } + + @Override + public int getMaxAggregatedValueByteSize() { + return Long.BYTES; + } + + @Override + public Long toLongValue(Double value) { + try { + return NumericUtils.doubleToSortableLong(value); + } catch (IllegalArgumentException | NullPointerException | IllegalStateException e) { + throw new IllegalArgumentException("Cannot convert " + value + " to sortable long", e); + } + } + + @Override + public Double toStarTreeNumericTypeValue(Long value, StarTreeNumericType type) { + try { + return type.getDoubleValue(value); + } catch (IllegalArgumentException | NullPointerException | IllegalStateException e) { + throw new IllegalArgumentException("Cannot convert " + value + " to sortable aggregation type", e); + } + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregationDescriptor.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregationDescriptor.java new file mode 100644 index 0000000000000..f5e1055b6ef24 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregationDescriptor.java @@ -0,0 +1,144 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.index.compositeindex.datacube.startree.aggregators; + +import org.apache.lucene.search.DocIdSetIterator; +import org.opensearch.index.compositeindex.datacube.MetricStat; +import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; +import org.opensearch.index.fielddata.IndexNumericFieldData; + +import java.util.Comparator; + +/** + * Builds aggregation function and doc values field pair to support various aggregations + * @opensearch.experimental + */ +public class MetricAggregationDescriptor implements Comparable { + + public static final String DELIMITER = "__"; + public static final String STAR = "*"; + public static final MetricAggregationDescriptor COUNT_STAR = new MetricAggregationDescriptor( + MetricStat.COUNT, + STAR, + IndexNumericFieldData.NumericType.DOUBLE, + null + ); + + private final String metricStatName; + private final MetricStat metricStat; + private final String field; + private final ValueAggregator valueAggregators; + private final StarTreeNumericType starTreeNumericType; + private final DocIdSetIterator metricStatReader; + + /** + * Constructor for MetricAggregationDescriptor + */ + public MetricAggregationDescriptor( + MetricStat metricStat, + String field, + IndexNumericFieldData.NumericType numericType, + DocIdSetIterator metricStatReader + ) { + this.metricStat = metricStat; + this.valueAggregators = ValueAggregatorFactory.getValueAggregator(metricStat); + this.starTreeNumericType = StarTreeNumericType.fromNumericType(numericType); + this.metricStatReader = metricStatReader; + if (metricStat == MetricStat.COUNT) { + this.field = STAR; + } else { + this.field = field; + } + this.metricStatName = toFieldName(); + } + + /** + * @return metric type + */ + public MetricStat getMetricStat() { + return metricStat; + } + + /** + * @return field Name + */ + public String getField() { + return field; + } + + /** + * @return the metric stat name + */ + public String getMetricStatName() { + return metricStatName; + } + + /** + * @return aggregator for the field value + */ + public ValueAggregator getValueAggregators() { + return valueAggregators; + } + + /** + * @return star tree numeric type + */ + public StarTreeNumericType getStarTreeNumericType() { + return starTreeNumericType; + } + + /** + * @return metric value reader iterator + */ + public DocIdSetIterator getMetricStatReader() { + return metricStatReader; + } + + /** + * @return field name with metric type and field + */ + public String toFieldName() { + return toFieldName(metricStat, field); + } + + /** + * Builds field name with metric type and field + */ + public static String toFieldName(MetricStat metricType, String field) { + return metricType.getTypeName() + DELIMITER + field; + } + + @Override + public int hashCode() { + return 31 * metricStat.hashCode() + field.hashCode(); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj instanceof MetricAggregationDescriptor) { + MetricAggregationDescriptor anotherPair = (MetricAggregationDescriptor) obj; + return metricStat == anotherPair.metricStat && field.equals(anotherPair.field); + } + return false; + } + + @Override + public String toString() { + return toFieldName(); + } + + @Override + public int compareTo(MetricAggregationDescriptor other) { + return Comparator.comparing((MetricAggregationDescriptor o) -> o.field) + .thenComparing((MetricAggregationDescriptor o) -> o.metricStat) + .compare(this, other); + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricStatFieldPair.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricStatFieldPair.java deleted file mode 100644 index 3fb041373ff51..0000000000000 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricStatFieldPair.java +++ /dev/null @@ -1,115 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ -package org.opensearch.index.compositeindex.datacube.startree.aggregators; - -import org.opensearch.index.compositeindex.datacube.MetricStat; - -import java.util.Comparator; - -/** - * Builds aggregation function and doc values field pair to support various aggregations - * @opensearch.experimental - */ -public class MetricStatFieldPair implements Comparable { - - public static final String DELIMITER = "__"; - public static final String STAR = "*"; - public static final MetricStatFieldPair COUNT_STAR = new MetricStatFieldPair(MetricStat.COUNT, STAR); - - private final MetricStat metricStat; - private final String field; - - /** - * Constructor for MetricStatFieldPair - */ - public MetricStatFieldPair(MetricStat metricStat, String field) { - this.metricStat = metricStat; - if (metricStat == MetricStat.COUNT) { - this.field = STAR; - } else { - this.field = field; - } - } - - /** - * @return Metric Type - */ - public MetricStat getMetricStat() { - return metricStat; - } - - /** - * @return field Name - */ - public String getField() { - return field; - } - - /** - * @return field name with metric type and field - */ - public String toFieldName() { - return toFieldName(metricStat, field); - } - - /** - * Builds field name with metric type and field - */ - public static String toFieldName(MetricStat metricType, String field) { - return metricType.getTypeName() + DELIMITER + field; - } - - /** - * Builds MetricStatFieldPair from field name - */ - public static MetricStatFieldPair fromFieldName(String fieldName) { - String[] parts = fieldName.split(DELIMITER, 2); - return fromMetricAndFieldName(parts[0], parts[1]); - } - - /** - * Builds MetricStatFieldPair from metric and field name - */ - private static MetricStatFieldPair fromMetricAndFieldName(String metricName, String fieldName) { - MetricStat metricType = MetricStat.fromTypeName(metricName); - if (metricType == MetricStat.COUNT) { - return COUNT_STAR; - } else { - return new MetricStatFieldPair(metricType, fieldName); - } - } - - @Override - public int hashCode() { - return 31 * metricStat.hashCode() + field.hashCode(); - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj instanceof MetricStatFieldPair) { - MetricStatFieldPair anotherPair = (MetricStatFieldPair) obj; - return metricStat == anotherPair.metricStat && field.equals(anotherPair.field); - } - return false; - } - - @Override - public String toString() { - return toFieldName(); - } - - @Override - public int compareTo(MetricStatFieldPair other) { - return Comparator.comparing((MetricStatFieldPair o) -> o.field) - .thenComparing((MetricStatFieldPair o) -> o.metricStat) - .compare(this, other); - } -} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregator.java index 36caf48ef0fb7..d9e55caae8784 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregator.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregator.java @@ -10,16 +10,16 @@ import org.apache.lucene.util.NumericUtils; import org.opensearch.index.compositeindex.datacube.MetricStat; import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; -import org.opensearch.index.compositeindex.datacube.startree.data.DataType; import org.opensearch.search.aggregations.metrics.CompensatedSum; /** * Sum value aggregator for star tree * - * @opensearch.internal + * @opensearch.experimental */ public class SumValueAggregator implements ValueAggregator { - public static final DataType AGGREGATED_VALUE_TYPE = DataType.DOUBLE; + + public static final StarTreeNumericType STAR_TREE_NUMERIC_TYPE = StarTreeNumericType.DOUBLE; @Override public MetricStat getAggregationType() { @@ -27,20 +27,20 @@ public MetricStat getAggregationType() { } @Override - public DataType getAggregatedValueType() { - return AGGREGATED_VALUE_TYPE; + public StarTreeNumericType getStarTreeNumericType() { + return STAR_TREE_NUMERIC_TYPE; } @Override - public Double getInitialAggregatedValue(Long rawValue, StarTreeNumericType starTreeNumericType) { - return starTreeNumericType.getDoubleValue(rawValue); + public Double getInitialAggregatedValue(Long segmentDocValue, StarTreeNumericType starTreeNumericType) { + return starTreeNumericType.getDoubleValue(segmentDocValue); } @Override - public Double applySegmentRawValue(Double value, Long rawValue, StarTreeNumericType starTreeNumericType) { + public Double applySegmentRawValue(Double value, Long segmentDocValue, StarTreeNumericType starTreeNumericType) { CompensatedSum kahanSummation = new CompensatedSum(0, 0); kahanSummation.add(value); - kahanSummation.add(starTreeNumericType.getDoubleValue(rawValue)); + kahanSummation.add(starTreeNumericType.getDoubleValue(segmentDocValue)); return kahanSummation.value(); } @@ -63,7 +63,7 @@ public int getMaxAggregatedValueByteSize() { } @Override - public Long convertAggregationTypeToSortableLongValue(Double value) { + public Long toLongValue(Double value) { try { return NumericUtils.doubleToSortableLong(value); } catch (IllegalArgumentException | NullPointerException | IllegalStateException e) { @@ -72,7 +72,7 @@ public Long convertAggregationTypeToSortableLongValue(Double value) { } @Override - public Double convertSortableLongToAggregatedTypeValue(Long value, StarTreeNumericType type) { + public Double toStarTreeNumericTypeValue(Long value, StarTreeNumericType type) { try { return type.getDoubleValue(value); } catch (IllegalArgumentException | NullPointerException | IllegalStateException e) { diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregator.java index c139a18eef91b..397f497d0105e 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregator.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregator.java @@ -9,10 +9,10 @@ import org.opensearch.index.compositeindex.datacube.MetricStat; import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; -import org.opensearch.index.compositeindex.datacube.startree.data.DataType; /** * A value aggregator that pre-aggregates on the input values for a specific type of aggregation. + * * @opensearch.experimental */ public interface ValueAggregator { @@ -25,17 +25,17 @@ public interface ValueAggregator { /** * Returns the data type of the aggregated value. */ - DataType getAggregatedValueType(); + StarTreeNumericType getStarTreeNumericType(); /** * Returns the initial aggregated value. */ - A getInitialAggregatedValue(Long rawValue, StarTreeNumericType starTreeNumericType); + A getInitialAggregatedValue(Long segmentDocValue, StarTreeNumericType starTreeNumericType); /** - * Applies a raw value to the current aggregated value. + * Applies a segment doc value to the current aggregated value. */ - A applySegmentRawValue(A value, Long rawValue, StarTreeNumericType starTreeNumericType); + A applySegmentRawValue(A value, Long segmentDocValue, StarTreeNumericType starTreeNumericType); /** * Applies an aggregated value to the current aggregated value. @@ -55,10 +55,10 @@ public interface ValueAggregator { /** * Converts an aggregated value into a Long type. */ - Long convertAggregationTypeToSortableLongValue(A value); + Long toLongValue(A value); /** * Converts an aggregated value from a Long type. */ - A convertSortableLongToAggregatedTypeValue(Long rawValue, StarTreeNumericType type); + A toStarTreeNumericTypeValue(Long rawValue, StarTreeNumericType type); } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactory.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactory.java index 523b3fbec2e7c..2bde2f16f91f8 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactory.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactory.java @@ -8,10 +8,11 @@ package org.opensearch.index.compositeindex.datacube.startree.aggregators; import org.opensearch.index.compositeindex.datacube.MetricStat; -import org.opensearch.index.compositeindex.datacube.startree.data.DataType; +import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; /** * Value aggregator factory for a given aggregation type + * * @opensearch.experimental */ public class ValueAggregatorFactory { @@ -28,6 +29,8 @@ public static ValueAggregator getValueAggregator(MetricStat aggregationType) { // other metric types (count, min, max, avg) will be supported in the future case SUM: return new SumValueAggregator(); + case COUNT: + return new CountValueAggregator(); default: throw new IllegalStateException("Unsupported aggregation type: " + aggregationType); } @@ -39,11 +42,13 @@ public static ValueAggregator getValueAggregator(MetricStat aggregationType) { * @param aggregationType Aggregation type * @return Data type of the aggregated value */ - public static DataType getAggregatedValueType(MetricStat aggregationType) { + public static StarTreeNumericType getAggregatedValueType(MetricStat aggregationType) { switch (aggregationType) { // other metric types (count, min, max, avg) will be supported in the future case SUM: - return SumValueAggregator.AGGREGATED_VALUE_TYPE; + return SumValueAggregator.STAR_TREE_NUMERIC_TYPE; + case COUNT: + return CountValueAggregator.STAR_TREE_NUMERIC_TYPE; default: throw new IllegalStateException("Unsupported aggregation type: " + aggregationType); } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericType.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericType.java index cd7ec84015848..28bfd82c69fdf 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericType.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericType.java @@ -12,6 +12,10 @@ import java.util.function.Function; +/** + * Enum to map Star Tree Numeric Types to Lucene's Numeric Type + * @opensearch.experimental + */ public enum StarTreeNumericType { HALF_FLOAT(IndexNumericFieldData.NumericType.HALF_FLOAT, StarTreeNumericTypeConverters::halfFloatPointToDouble), FLOAT(IndexNumericFieldData.NumericType.FLOAT, StarTreeNumericTypeConverters::floatPointToDouble), diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericTypeConverters.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericTypeConverters.java index 59a2c3418e931..f3fdd0d1162c3 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericTypeConverters.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericTypeConverters.java @@ -10,7 +10,13 @@ import org.apache.lucene.sandbox.document.HalfFloatPoint; import org.apache.lucene.util.NumericUtils; +import org.opensearch.common.annotation.ExperimentalApi; +/** + * Numeric converters used during aggregations of metric values + * @opensearch.experimental + */ +@ExperimentalApi public class StarTreeNumericTypeConverters { public static double halfFloatPointToDouble(Long value) { diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/DocValuesIteratorAdapter.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/DocValuesIteratorAdapter.java deleted file mode 100644 index 6dc54e7139e9f..0000000000000 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/DocValuesIteratorAdapter.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.index.compositeindex.datacube.startree.builder; - -import org.apache.lucene.codecs.DocValuesProducer; -import org.apache.lucene.index.DocValuesType; -import org.apache.lucene.index.FieldInfo; -import org.apache.lucene.search.DocIdSetIterator; - -import java.io.IOException; - -/** - * An interface to support iterators for various doc values types. - * @opensearch.experimental - */ -public interface DocValuesIteratorAdapter { - - /** - * Creates an iterator for the given doc values type and field using the doc values producer - */ - DocIdSetIterator getDocValuesIterator(DocValuesType type, FieldInfo field, DocValuesProducer producer) throws IOException; - - /** - * Returns the next value for the given iterator - */ - long getNextValue(DocIdSetIterator iterator) throws IOException; - - /** - * Returns the doc id for the next document from the given iterator - */ - int nextDoc(DocIdSetIterator iterator) throws IOException; -} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapSingleTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapSingleTreeBuilder.java index 26e217d91dbc3..0289a56ce5b5f 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapSingleTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapSingleTreeBuilder.java @@ -11,8 +11,9 @@ import org.apache.lucene.codecs.DocValuesProducer; import org.apache.lucene.index.BaseStarTreeBuilder; import org.apache.lucene.index.SegmentWriteState; +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeDocument; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; -import org.opensearch.index.compositeindex.datacube.startree.data.StarTreeDocument; import org.opensearch.index.mapper.MapperService; import java.io.IOException; @@ -20,10 +21,13 @@ import java.util.Arrays; import java.util.Iterator; import java.util.List; +import java.util.Map; /** * On heap single tree builder + * @opensearch.experimental */ +@ExperimentalApi public class OnHeapSingleTreeBuilder extends BaseStarTreeBuilder { private final List starTreeDocuments = new ArrayList<>(); @@ -32,20 +36,20 @@ public class OnHeapSingleTreeBuilder extends BaseStarTreeBuilder { * Constructor for OnHeapSingleTreeBuilder * * @param starTreeField star-tree field - * @param docValuesProducer document values producer + * @param fieldProducerMap helps with document values producer for a particular field * @param docValuesConsumer document values consumer * @param segmentWriteState segment write state - * @param mapperService + * @param mapperService helps with the numeric type of field * @throws IOException throws an exception we are unable to construct an onheap star-tree */ public OnHeapSingleTreeBuilder( StarTreeField starTreeField, - DocValuesProducer docValuesProducer, + Map fieldProducerMap, DocValuesConsumer docValuesConsumer, SegmentWriteState segmentWriteState, MapperService mapperService ) throws IOException { - super(starTreeField, docValuesProducer, docValuesConsumer, segmentWriteState, mapperService); + super(starTreeField, fieldProducerMap, docValuesConsumer, segmentWriteState, mapperService); } @Override @@ -70,12 +74,12 @@ public long getDimensionValue(int docId, int dimensionId) throws IOException { } @Override - public Iterator processSegmentStarTreeDocuments(int numDocs) throws IOException { + public Iterator sortMergeAndAggregateStarTreeDocument(int numDocs) throws IOException { StarTreeDocument[] starTreeDocuments = new StarTreeDocument[numDocs]; for (int i = 0; i < numDocs; i++) { starTreeDocuments[i] = getSegmentStarTreeDocument(); } - return processStarTreeDocuments(starTreeDocuments); + return sortMergeAndAggregateStarTreeDocument(starTreeDocuments); } /** @@ -84,7 +88,7 @@ public Iterator processSegmentStarTreeDocuments(int numDocs) t * @return iterator for star-tree documents * @throws IOException throws when unable to sort, merge and aggregate star-tree documents */ - public Iterator processStarTreeDocuments(StarTreeDocument[] starTreeDocuments) throws IOException { + public Iterator sortMergeAndAggregateStarTreeDocument(StarTreeDocument[] starTreeDocuments) throws IOException { // sort the documents Arrays.sort(starTreeDocuments, (o1, o2) -> { @@ -119,14 +123,14 @@ public boolean hasNext() { @Override public StarTreeDocument next() { // aggregate as we move on to the next doc - StarTreeDocument next = aggregateDocuments(null, currentStarTreeDocument); + StarTreeDocument next = reduceSegmentStarTreeDocuments(null, currentStarTreeDocument); while (docId < starTreeDocuments.length) { StarTreeDocument starTreeDocument = starTreeDocuments[docId++]; if (!Arrays.equals(starTreeDocument.dimensions, next.dimensions)) { currentStarTreeDocument = starTreeDocument; return next; } else { - next = aggregateDocuments(next, starTreeDocument); + next = reduceSegmentStarTreeDocuments(next, starTreeDocument); } } hasNext = false; @@ -144,7 +148,8 @@ public StarTreeDocument next() { * @throws IOException throws when unable to generate star-tree for star-node */ @Override - public Iterator generateStarTreeDocumentsForStarNode(int startDocId, int endDocId, int dimensionId) throws IOException { + public Iterator generateStarTreeDocumentsForStarNode(int startDocId, int endDocId, int dimensionId) + throws IOException { int numDocs = endDocId - startDocId; StarTreeDocument[] starTreeDocuments = new StarTreeDocument[numDocs]; for (int i = 0; i < numDocs; i++) { @@ -153,7 +158,7 @@ public Iterator generateStarTreeDocumentsForStarNode(int start Arrays.sort(starTreeDocuments, (o1, o2) -> { for (int i = dimensionId + 1; i < numDimensions; i++) { if (o1.dimensions[i] != o2.dimensions[i]) { - return Math.toIntExact(o1.dimensions[i] - o2.dimensions[i]); + return Long.compare(o1.dimensions[i], o2.dimensions[i]); } } return 0; @@ -179,7 +184,7 @@ public boolean hasNext() { @Override public StarTreeDocument next() { - StarTreeDocument next = aggregateDocuments(null, currentStarTreeDocument); + StarTreeDocument next = reduceStarTreeDocuments(null, currentStarTreeDocument); next.dimensions[dimensionId] = STAR_IN_DOC_VALUES_INDEX; while (docId < numDocs) { StarTreeDocument starTreeDocument = starTreeDocuments[docId++]; @@ -187,7 +192,7 @@ public StarTreeDocument next() { currentStarTreeDocument = starTreeDocument; return next; } else { - next = aggregateDocuments(next, starTreeDocument); + next = reduceStarTreeDocuments(next, starTreeDocument); } } hasNext = false; diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/SingleTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/SingleTreeBuilder.java index 548a5261a78a3..44866caba938b 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/SingleTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/SingleTreeBuilder.java @@ -8,6 +8,8 @@ package org.opensearch.index.compositeindex.datacube.startree.builder; +import org.opensearch.common.annotation.ExperimentalApi; + import java.io.Closeable; import java.io.IOException; @@ -15,6 +17,7 @@ * A star-tree builder that builds a single star-tree. * @opensearch.experimental */ +@ExperimentalApi public interface SingleTreeBuilder extends Closeable { /** diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocValuesIteratorAdapter.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocValuesIteratorAdapter.java index a48b944b4e6c7..e4cf2a45e131b 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocValuesIteratorAdapter.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocValuesIteratorAdapter.java @@ -14,16 +14,21 @@ import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.search.DocIdSetIterator; +import org.opensearch.common.annotation.ExperimentalApi; import java.io.IOException; /** * A factory class to return respective doc values iterator based on the doc volues type. + * * @opensearch.experimental */ -public class StarTreeDocValuesIteratorAdapter implements DocValuesIteratorAdapter { +@ExperimentalApi +public class StarTreeDocValuesIteratorAdapter { - @Override + /** + * Creates an iterator for the given doc values type and field using the doc values producer + */ public DocIdSetIterator getDocValuesIterator(DocValuesType type, FieldInfo field, DocValuesProducer producer) throws IOException { switch (type) { case SORTED_SET: @@ -35,8 +40,10 @@ public DocIdSetIterator getDocValuesIterator(DocValuesType type, FieldInfo field } } - @Override - public long getNextValue(DocIdSetIterator iterator) throws IOException { + /** + * Returns the next ordinal for the given iterator + */ + public long getNextOrd(DocIdSetIterator iterator) throws IOException { if (iterator instanceof SortedSetDocValues) { return ((SortedSetDocValues) iterator).nextOrd(); } else if (iterator instanceof SortedNumericDocValues) { @@ -46,7 +53,9 @@ public long getNextValue(DocIdSetIterator iterator) throws IOException { } } - @Override + /** + * Returns the doc id for the next document from the given iterator + */ public int nextDoc(DocIdSetIterator iterator) throws IOException { return iterator.nextDoc(); } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilder.java index e8ec930c4a3a0..4d110609462a5 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilder.java @@ -13,6 +13,7 @@ import org.apache.lucene.codecs.DocValuesConsumer; import org.apache.lucene.codecs.DocValuesProducer; import org.apache.lucene.index.SegmentWriteState; +import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; import org.opensearch.index.mapper.MapperService; @@ -21,7 +22,14 @@ import java.io.IOException; import java.util.List; import java.util.Locale; +import java.util.Map; +/** + * Builder to construct star tree based on multiple star tree configs + * + * @opensearch.experimental + */ +@ExperimentalApi public class StarTreesBuilder implements Closeable { private static final Logger logger = LogManager.getLogger(StarTreesBuilder.class); @@ -30,13 +38,13 @@ public class StarTreesBuilder implements Closeable { private final StarTreeFieldConfiguration.StarTreeBuildMode buildMode; private final DocValuesConsumer docValuesConsumer; private final SegmentWriteState state; - private final DocValuesProducer docValuesProducer; + private final Map fieldProducerMap; private final MapperService mapperService; public StarTreesBuilder( List starTreeFields, StarTreeFieldConfiguration.StarTreeBuildMode buildMode, - DocValuesProducer docValuesProducer, + Map fieldProducerMap, DocValuesConsumer docValuesConsumer, SegmentWriteState segmentWriteState, MapperService mapperService @@ -46,7 +54,7 @@ public StarTreesBuilder( throw new IllegalArgumentException("Must provide star-tree builder configs"); } this.buildMode = buildMode; - this.docValuesProducer = docValuesProducer; + this.fieldProducerMap = fieldProducerMap; this.docValuesConsumer = docValuesConsumer; this.state = segmentWriteState; this.mapperService = mapperService; @@ -67,7 +75,7 @@ public void build() throws Exception { SingleTreeBuilder singleTreeBuilder = getSingleTreeBuilder( starTreeField, buildMode, - docValuesProducer, + fieldProducerMap, docValuesConsumer, state, mapperService @@ -93,14 +101,14 @@ public void close() throws IOException { private static SingleTreeBuilder getSingleTreeBuilder( StarTreeField starTreeField, StarTreeFieldConfiguration.StarTreeBuildMode buildMode, - DocValuesProducer docValuesProducer, + Map fieldProducerMap, DocValuesConsumer docValuesConsumer, SegmentWriteState state, MapperService mapperService ) throws IOException { switch (buildMode) { case ON_HEAP: - return new OnHeapSingleTreeBuilder(starTreeField, docValuesProducer, docValuesConsumer, state, mapperService); + return new OnHeapSingleTreeBuilder(starTreeField, fieldProducerMap, docValuesConsumer, state, mapperService); default: throw new IllegalArgumentException( String.format(Locale.ROOT, "No star tree implementation is available for [%s] build mode", buildMode) diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/data/DataType.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/data/DataType.java deleted file mode 100644 index 3c737c44b7219..0000000000000 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/data/DataType.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ -package org.opensearch.index.compositeindex.datacube.startree.data; - -/** - * Data type of doc values - * @opensearch.internal - */ -public enum DataType { - INT(Integer.BYTES, true), - LONG(Long.BYTES, true), - FLOAT(Float.BYTES, true), - DOUBLE(Double.BYTES, true); - - private final int size; - private final boolean numeric; - - DataType(int size, boolean numeric) { - this.size = size; - this.numeric = numeric; - } - - /** - * Returns the number of bytes needed to store the data type. - */ - public int size() { - if (size >= 0) { - return size; - } - throw new IllegalStateException("Cannot get number of bytes for: " + this); - } - - /** - * Returns {@code true} if the data type is numeric (INT, LONG, FLOAT, DOUBLE, BIG_DECIMAL), - * {@code false} otherwise. - */ - public boolean isNumeric() { - return numeric; - } - - /** - * Converts the given string value to the data type. Returns byte[] for BYTES. - */ - public Object convert(String value) { - try { - switch (this) { - case INT: - return Integer.valueOf(value); - case LONG: - return Long.valueOf(value); - case FLOAT: - return Float.valueOf(value); - case DOUBLE: - return Double.valueOf(value); - default: - throw new IllegalStateException(); - } - } catch (Exception e) { - throw new IllegalArgumentException(e); - } - } -} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/package-info.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/package-info.java deleted file mode 100644 index 3af5020b76da2..0000000000000 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/package-info.java +++ /dev/null @@ -1,13 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -/** - * Node for Composite Index Star Tree - * @opensearch.experimental - */ -package org.opensearch.index.compositeindex.datacube.startree.node; diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeBuilderUtils.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeBuilderUtils.java index e1841b3c2ff68..656196c7ffe0f 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeBuilderUtils.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeBuilderUtils.java @@ -7,27 +7,30 @@ */ package org.opensearch.index.compositeindex.datacube.startree.utils; +import org.opensearch.common.annotation.ExperimentalApi; + import java.util.Map; /** * Util class for building star tree * @opensearch.experimental */ +@ExperimentalApi public class StarTreeBuilderUtils { - private StarTreeBuilderUtils() { - } + private StarTreeBuilderUtils() {} public static final int ALL = -1; /** Tree node representation */ public static class TreeNode { public int dimensionId = ALL; - public long dimensionValue = ALL; public int startDocId = ALL; public int endDocId = ALL; public int aggregatedDocId = ALL; public int childDimensionId = ALL; + public long dimensionValue = ALL; + public boolean isStarNode = false; public Map children; } diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregationDescriptorTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregationDescriptorTests.java new file mode 100644 index 0000000000000..98b79141df9eb --- /dev/null +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregationDescriptorTests.java @@ -0,0 +1,110 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube.startree.aggregators; + +import org.opensearch.index.compositeindex.datacube.MetricStat; +import org.opensearch.index.fielddata.IndexNumericFieldData; +import org.opensearch.test.OpenSearchTestCase; + +public class MetricAggregationDescriptorTests extends OpenSearchTestCase { + + public void testConstructor() { + MetricAggregationDescriptor pair = new MetricAggregationDescriptor( + MetricStat.SUM, + "column1", + IndexNumericFieldData.NumericType.DOUBLE, + null + ); + assertEquals(MetricStat.SUM, pair.getMetricStat()); + assertEquals("column1", pair.getField()); + } + + public void testCountStarConstructor() { + MetricAggregationDescriptor pair = new MetricAggregationDescriptor( + MetricStat.COUNT, + "anything", + IndexNumericFieldData.NumericType.DOUBLE, + null + ); + assertEquals(MetricStat.COUNT, pair.getMetricStat()); + assertEquals("*", pair.getField()); + } + + public void testToFieldName() { + MetricAggregationDescriptor pair = new MetricAggregationDescriptor( + MetricStat.SUM, + "column2", + IndexNumericFieldData.NumericType.DOUBLE, + null + ); + assertEquals("sum__column2", pair.toFieldName()); + } + + public void testEquals() { + MetricAggregationDescriptor pair1 = new MetricAggregationDescriptor( + MetricStat.SUM, + "column1", + IndexNumericFieldData.NumericType.DOUBLE, + null + ); + MetricAggregationDescriptor pair2 = new MetricAggregationDescriptor( + MetricStat.SUM, + "column1", + IndexNumericFieldData.NumericType.DOUBLE, + null + ); + assertEquals(pair1, pair2); + assertNotEquals( + pair1, + new MetricAggregationDescriptor(MetricStat.COUNT, "column1", IndexNumericFieldData.NumericType.DOUBLE, null) + ); + assertNotEquals(pair1, new MetricAggregationDescriptor(MetricStat.SUM, "column2", IndexNumericFieldData.NumericType.DOUBLE, null)); + } + + public void testHashCode() { + MetricAggregationDescriptor pair1 = new MetricAggregationDescriptor( + MetricStat.SUM, + "column1", + IndexNumericFieldData.NumericType.DOUBLE, + null + ); + MetricAggregationDescriptor pair2 = new MetricAggregationDescriptor( + MetricStat.SUM, + "column1", + IndexNumericFieldData.NumericType.DOUBLE, + null + ); + assertEquals(pair1.hashCode(), pair2.hashCode()); + } + + public void testCompareTo() { + MetricAggregationDescriptor pair1 = new MetricAggregationDescriptor( + MetricStat.SUM, + "column1", + IndexNumericFieldData.NumericType.DOUBLE, + null + ); + MetricAggregationDescriptor pair2 = new MetricAggregationDescriptor( + MetricStat.SUM, + "column2", + IndexNumericFieldData.NumericType.DOUBLE, + null + ); + MetricAggregationDescriptor pair3 = new MetricAggregationDescriptor( + MetricStat.COUNT, + "column1", + IndexNumericFieldData.NumericType.DOUBLE, + null + ); + assertTrue(pair1.compareTo(pair2) < 0); + assertTrue(pair2.compareTo(pair1) > 0); + assertTrue(pair1.compareTo(pair3) > 0); + assertTrue(pair3.compareTo(pair1) < 0); + } +} diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricStatFieldPairTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricStatFieldPairTests.java deleted file mode 100644 index fd026f49c0140..0000000000000 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricStatFieldPairTests.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.index.compositeindex.datacube.startree.aggregators; - -import org.opensearch.index.compositeindex.datacube.MetricStat; -import org.opensearch.test.OpenSearchTestCase; - -public class MetricStatFieldPairTests extends OpenSearchTestCase { - - public void testConstructor() { - MetricStatFieldPair pair = new MetricStatFieldPair(MetricStat.SUM, "column1"); - assertEquals(MetricStat.SUM, pair.getMetricStat()); - assertEquals("column1", pair.getField()); - } - - public void testCountStarConstructor() { - MetricStatFieldPair pair = new MetricStatFieldPair(MetricStat.COUNT, "anything"); - assertEquals(MetricStat.COUNT, pair.getMetricStat()); - assertEquals("*", pair.getField()); - } - - public void testToFieldName() { - MetricStatFieldPair pair = new MetricStatFieldPair(MetricStat.AVG, "column2"); - assertEquals("avg__column2", pair.toFieldName()); - } - - public void testFromFieldName() { - MetricStatFieldPair pair = MetricStatFieldPair.fromFieldName("max__column3"); - assertEquals(MetricStat.MAX, pair.getMetricStat()); - assertEquals("column3", pair.getField()); - } - - public void testCountStarFromFieldName() { - MetricStatFieldPair pair = MetricStatFieldPair.fromFieldName("count__*"); - assertEquals(MetricStat.COUNT, pair.getMetricStat()); - assertEquals("*", pair.getField()); - assertSame(MetricStatFieldPair.COUNT_STAR, pair); - } - - public void testEquals() { - MetricStatFieldPair pair1 = new MetricStatFieldPair(MetricStat.SUM, "column1"); - MetricStatFieldPair pair2 = new MetricStatFieldPair(MetricStat.SUM, "column1"); - assertEquals(pair1, pair2); - assertNotEquals(pair1, new MetricStatFieldPair(MetricStat.AVG, "column1")); - assertNotEquals(pair1, new MetricStatFieldPair(MetricStat.SUM, "column2")); - } - - public void testHashCode() { - MetricStatFieldPair pair1 = new MetricStatFieldPair(MetricStat.SUM, "column1"); - MetricStatFieldPair pair2 = new MetricStatFieldPair(MetricStat.SUM, "column1"); - assertEquals(pair1.hashCode(), pair2.hashCode()); - } - - public void testCompareTo() { - MetricStatFieldPair pair1 = new MetricStatFieldPair(MetricStat.SUM, "column1"); - MetricStatFieldPair pair2 = new MetricStatFieldPair(MetricStat.SUM, "column2"); - MetricStatFieldPair pair3 = new MetricStatFieldPair(MetricStat.AVG, "column1"); - assertTrue(pair1.compareTo(pair2) < 0); - assertTrue(pair2.compareTo(pair1) > 0); - assertTrue(pair1.compareTo(pair3) > 0); - assertTrue(pair3.compareTo(pair1) < 0); - } -} diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregatorTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregatorTests.java index f30a9c82b1fd3..287deeead9d9d 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregatorTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregatorTests.java @@ -21,8 +21,8 @@ public void testGetAggregationType() { assertEquals(MetricStat.SUM.getTypeName(), aggregator.getAggregationType().getTypeName()); } - public void testGetAggregatedValueType() { - assertEquals(SumValueAggregator.AGGREGATED_VALUE_TYPE, aggregator.getAggregatedValueType()); + public void testGetStarTreeNumericType() { + assertEquals(SumValueAggregator.STAR_TREE_NUMERIC_TYPE, aggregator.getStarTreeNumericType()); } public void testGetInitialAggregatedValue() { @@ -37,7 +37,6 @@ public void testApplySegmentRawValue() { public void testApplyAggregatedValue() { assertEquals(5.0, aggregator.applyAggregatedValue(2.0, 3.0), 0.0); - assertEquals(7.28, aggregator.applyAggregatedValue(3.14, 4.14), 0.0000001); } public void testGetAggregatedValue() { @@ -48,17 +47,13 @@ public void testGetMaxAggregatedValueByteSize() { assertEquals(Double.BYTES, aggregator.getMaxAggregatedValueByteSize()); } - public void testConvertAggregationTypeToSortableLongValue() { + public void testToLongValue() { SumValueAggregator aggregator = new SumValueAggregator(); - assertEquals(NumericUtils.doubleToSortableLong(3.14), aggregator.convertAggregationTypeToSortableLongValue(3.14), 0.0); + assertEquals(NumericUtils.doubleToSortableLong(3.14), aggregator.toLongValue(3.14), 0.0); } - public void testConvertSortableLongToAggregatedTypeValue() { + public void testToStarTreeNumericTypeValue() { SumValueAggregator aggregator = new SumValueAggregator(); - assertEquals( - NumericUtils.sortableLongToDouble(3L), - aggregator.convertSortableLongToAggregatedTypeValue(3L, StarTreeNumericType.DOUBLE), - 0.0 - ); + assertEquals(NumericUtils.sortableLongToDouble(3L), aggregator.toStarTreeNumericTypeValue(3L, StarTreeNumericType.DOUBLE), 0.0); } } diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactoryTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactoryTests.java index f80783dacb643..21e2d1304f085 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactoryTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactoryTests.java @@ -9,7 +9,7 @@ package org.opensearch.index.compositeindex.datacube.startree.aggregators; import org.opensearch.index.compositeindex.datacube.MetricStat; -import org.opensearch.index.compositeindex.datacube.startree.data.DataType; +import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; import org.opensearch.test.OpenSearchTestCase; public class ValueAggregatorFactoryTests extends OpenSearchTestCase { @@ -20,24 +20,8 @@ public void testGetValueAggregatorForSumType() { assertEquals(SumValueAggregator.class, aggregator.getClass()); } - public void testGetValueAggregatorForUnsupportedType() { - IllegalStateException exception = expectThrows( - IllegalStateException.class, - () -> ValueAggregatorFactory.getValueAggregator(MetricStat.UNSUPPORTED) - ); - assertEquals("Unsupported aggregation type: UNSUPPORTED", exception.getMessage()); - } - - public void testGetAggregatedValueTypeForSumType() { - DataType dataType = ValueAggregatorFactory.getAggregatedValueType(MetricStat.SUM); - assertEquals(SumValueAggregator.AGGREGATED_VALUE_TYPE, dataType); - } - - public void testGetAggregatedValueTypeForUnsupportedType() { - IllegalStateException exception = expectThrows( - IllegalStateException.class, - () -> ValueAggregatorFactory.getAggregatedValueType(MetricStat.UNSUPPORTED) - ); - assertEquals("Unsupported aggregation type: UNSUPPORTED", exception.getMessage()); + public void testGetStarTreeNumericTypeForSumType() { + StarTreeNumericType starTreeNumericType = ValueAggregatorFactory.getAggregatedValueType(MetricStat.SUM); + assertEquals(SumValueAggregator.STAR_TREE_NUMERIC_TYPE, starTreeNumericType); } } diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java index 753297a8004ce..7a13ab7701912 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java @@ -27,10 +27,11 @@ import org.opensearch.index.compositeindex.datacube.Dimension; import org.opensearch.index.compositeindex.datacube.Metric; import org.opensearch.index.compositeindex.datacube.MetricStat; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeDocument; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; -import org.opensearch.index.compositeindex.datacube.startree.aggregators.MetricStatFieldPair; -import org.opensearch.index.compositeindex.datacube.startree.data.StarTreeDocument; +import org.opensearch.index.compositeindex.datacube.startree.aggregators.MetricAggregationDescriptor; +import org.opensearch.index.fielddata.IndexNumericFieldData; import org.opensearch.index.mapper.ContentPath; import org.opensearch.index.mapper.DocumentMapper; import org.opensearch.index.mapper.Mapper; @@ -46,6 +47,7 @@ import java.util.HashMap; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.UUID; @@ -72,6 +74,7 @@ public class BaseStarTreeBuilderTests extends OpenSearchTestCase { private static List metrics; private static Directory directory; private static FieldInfo[] fieldsInfo; + private static SegmentWriteState state; @BeforeClass public static void setup() throws IOException { @@ -104,7 +107,7 @@ public static void setup() throws IOException { ); fieldsInfo = new FieldInfo[fields.size()]; - + Map fieldProducerMap = new HashMap<>(); for (int i = 0; i < fieldsInfo.length; i++) { fieldsInfo[i] = new FieldInfo( fields.get(i), @@ -125,16 +128,10 @@ public static void setup() throws IOException { false, false ); + fieldProducerMap.put(fields.get(i), docValuesProducer); } FieldInfos fieldInfos = new FieldInfos(fieldsInfo); - final SegmentWriteState state = new SegmentWriteState( - InfoStream.getDefault(), - segmentInfo.dir, - segmentInfo, - fieldInfos, - null, - newIOContext(random()) - ); + state = new SegmentWriteState(InfoStream.getDefault(), segmentInfo.dir, segmentInfo, fieldInfos, null, newIOContext(random())); mapperService = mock(MapperService.class); DocumentMapper documentMapper = mock(DocumentMapper.class); @@ -153,7 +150,7 @@ public static void setup() throws IOException { ); when(documentMapper.mappers()).thenReturn(fieldMappers); - builder = new BaseStarTreeBuilder(compositeField, docValuesProducer, docValuesConsumer, state, mapperService) { + builder = new BaseStarTreeBuilder(compositeField, fieldProducerMap, docValuesConsumer, state, mapperService) { @Override public void appendStarTreeDocument(StarTreeDocument starTreeDocument) throws IOException {} @@ -173,7 +170,7 @@ public long getDimensionValue(int docId, int dimensionId) throws IOException { } @Override - public Iterator processSegmentStarTreeDocuments(int numDocs) throws IOException { + public Iterator sortMergeAndAggregateStarTreeDocument(int numDocs) throws IOException { return null; } @@ -186,20 +183,20 @@ public Iterator generateStarTreeDocumentsForStarNode(int start } public void test_generateMetricStatFieldPairs() throws IOException { - List metricStatFieldPairs = builder.generateMetricStatFieldPairs(); - List expectedMetricStatFieldPairs = List.of( - new MetricStatFieldPair(MetricStat.SUM, "field2"), - new MetricStatFieldPair(MetricStat.SUM, "field4") + List metricAggregationDescriptors = builder.generateMetricStatFieldPairs(mapperService, state); + List expectedMetricAggregationDescriptors = List.of( + new MetricAggregationDescriptor(MetricStat.SUM, "field2", IndexNumericFieldData.NumericType.DOUBLE, null), + new MetricAggregationDescriptor(MetricStat.SUM, "field4", IndexNumericFieldData.NumericType.DOUBLE, null) ); - assertEquals(metricStatFieldPairs, expectedMetricStatFieldPairs); + assertEquals(metricAggregationDescriptors, expectedMetricAggregationDescriptors); } - public void test_aggregateDocuments() { + public void test_reduceStarTreeDocuments() { StarTreeDocument starTreeDocument1 = new StarTreeDocument(new long[] { 1, 3, 5, 8 }, new Double[] { 4.0, 8.0 }); StarTreeDocument starTreeDocument2 = new StarTreeDocument(new long[] { 1, 3, 5, 8 }, new Double[] { 10.0, 6.0 }); StarTreeDocument expectedeMergedStarTreeDocument = new StarTreeDocument(new long[] { 1, 3, 5, 8 }, new Double[] { 14.0, 14.0 }); - StarTreeDocument mergedStarTreeDocument = builder.aggregateDocuments(starTreeDocument1, starTreeDocument2); + StarTreeDocument mergedStarTreeDocument = builder.reduceStarTreeDocuments(starTreeDocument1, starTreeDocument2); assertEquals(mergedStarTreeDocument.metrics[0], expectedeMergedStarTreeDocument.metrics[0]); assertEquals(mergedStarTreeDocument.metrics[1], expectedeMergedStarTreeDocument.metrics[1]); diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapSingleTreeBuilderTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapSingleTreeBuilderTests.java index e81c836c9d723..73955080c36ae 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapSingleTreeBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapSingleTreeBuilderTests.java @@ -19,16 +19,18 @@ import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.index.VectorEncoding; import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.sandbox.document.HalfFloatPoint; import org.apache.lucene.store.Directory; import org.apache.lucene.util.InfoStream; +import org.apache.lucene.util.NumericUtils; import org.apache.lucene.util.Version; import org.opensearch.common.settings.Settings; import org.opensearch.index.compositeindex.datacube.Dimension; import org.opensearch.index.compositeindex.datacube.Metric; import org.opensearch.index.compositeindex.datacube.MetricStat; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeDocument; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; -import org.opensearch.index.compositeindex.datacube.startree.data.StarTreeDocument; import org.opensearch.index.mapper.ContentPath; import org.opensearch.index.mapper.DocumentMapper; import org.opensearch.index.mapper.Mapper; @@ -36,7 +38,7 @@ import org.opensearch.index.mapper.MappingLookup; import org.opensearch.index.mapper.NumberFieldMapper; import org.opensearch.test.OpenSearchTestCase; -import org.junit.BeforeClass; +import org.junit.Before; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -44,6 +46,7 @@ import java.util.HashMap; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.UUID; @@ -52,10 +55,10 @@ public class OnHeapSingleTreeBuilderTests extends OpenSearchTestCase { - private static OnHeapSingleTreeBuilder builder; - private static MapperService mapperService; - private static List dimensionsOrder; - private static List fields = List.of( + private OnHeapSingleTreeBuilder builder; + private MapperService mapperService; + private List dimensionsOrder; + private List fields = List.of( "field1", "field2", "field3", @@ -67,19 +70,23 @@ public class OnHeapSingleTreeBuilderTests extends OpenSearchTestCase { "field9", "field10" ); - private static List metrics; - private static Directory directory; - private static FieldInfo[] fieldsInfo; - - @BeforeClass - public static void setup() throws IOException { + private List metrics; + private Directory directory; + private FieldInfo[] fieldsInfo; + private StarTreeField compositeField; + private Map fieldProducerMap; + private DocValuesConsumer docValuesConsumer; + private SegmentWriteState writeState; + + @Before + public void setup() throws IOException { dimensionsOrder = List.of(new Dimension("field1"), new Dimension("field3"), new Dimension("field5"), new Dimension("field8")); metrics = List.of(new Metric("field2", List.of(MetricStat.SUM)), new Metric("field4", List.of(MetricStat.SUM))); - DocValuesConsumer docValuesConsumer = mock(DocValuesConsumer.class); + docValuesConsumer = mock(DocValuesConsumer.class); DocValuesProducer docValuesProducer = mock(DocValuesProducer.class); - StarTreeField compositeField = new StarTreeField( + compositeField = new StarTreeField( "test", dimensionsOrder, metrics, @@ -102,7 +109,7 @@ public static void setup() throws IOException { ); fieldsInfo = new FieldInfo[fields.size()]; - + fieldProducerMap = new HashMap<>(); for (int i = 0; i < fieldsInfo.length; i++) { fieldsInfo[i] = new FieldInfo( fields.get(i), @@ -123,16 +130,10 @@ public static void setup() throws IOException { false, false ); + fieldProducerMap.put(fields.get(i), docValuesProducer); } FieldInfos fieldInfos = new FieldInfos(fieldsInfo); - final SegmentWriteState writeState = new SegmentWriteState( - InfoStream.getDefault(), - segmentInfo.dir, - segmentInfo, - fieldInfos, - null, - newIOContext(random()) - ); + writeState = new SegmentWriteState(InfoStream.getDefault(), segmentInfo.dir, segmentInfo, fieldInfos, null, newIOContext(random())); mapperService = mock(MapperService.class); DocumentMapper documentMapper = mock(DocumentMapper.class); @@ -150,10 +151,10 @@ public static void setup() throws IOException { null ); when(documentMapper.mappers()).thenReturn(fieldMappers); - builder = new OnHeapSingleTreeBuilder(compositeField, docValuesProducer, docValuesConsumer, writeState, mapperService); + builder = new OnHeapSingleTreeBuilder(compositeField, fieldProducerMap, docValuesConsumer, writeState, mapperService); } - public void test_processStarTreeDocuments() throws IOException { + public void test_sortMergeAndAggregateStarTreeDocument() throws IOException { int noOfStarTreeDocuments = 5; StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; @@ -169,10 +170,20 @@ public void test_processStarTreeDocuments() throws IOException { new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 35.0, 34.0 }) ); Iterator expectedStarTreeDocumentIterator = inorderStarTreeDocuments.iterator(); - Iterator starTreeDocumentIterator = builder.processStarTreeDocuments(starTreeDocuments); + + StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; + for (int i = 0; i < noOfStarTreeDocuments; i++) { + long metric1 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[0]); + long metric2 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[1]); + segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2 }); + } + + Iterator segmentStarTreeDocumentIterator = builder.sortMergeAndAggregateStarTreeDocument( + segmentStarTreeDocuments + ); int numOfAggregatedDocuments = 0; - while (starTreeDocumentIterator.hasNext() && expectedStarTreeDocumentIterator.hasNext()) { - StarTreeDocument resultStarTreeDocument = starTreeDocumentIterator.next(); + while (segmentStarTreeDocumentIterator.hasNext() && expectedStarTreeDocumentIterator.hasNext()) { + StarTreeDocument resultStarTreeDocument = segmentStarTreeDocumentIterator.next(); StarTreeDocument expectedStarTreeDocument = expectedStarTreeDocumentIterator.next(); assertEquals(expectedStarTreeDocument.dimensions[0], resultStarTreeDocument.dimensions[0]); @@ -189,7 +200,7 @@ public void test_processStarTreeDocuments() throws IOException { } - public void test_processStarTreeDocuments_nullDocument() throws IOException { + public void test_sortMergeAndAggregateStarTreeDocument_nullMetric() throws IOException { int noOfStarTreeDocuments = 5; StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; @@ -200,9 +211,21 @@ public void test_processStarTreeDocuments_nullDocument() throws IOException { starTreeDocuments[3] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 9.0, 4.0 }); starTreeDocuments[4] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 11.0, null }); StarTreeDocument expectedStarTreeDocument = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 21.0, 14.0 }); - Iterator starTreeDocumentIterator = builder.processStarTreeDocuments(starTreeDocuments); - StarTreeDocument resultStarTreeDocument = starTreeDocumentIterator.next(); + StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; + for (int i = 0; i < noOfStarTreeDocuments; i++) { + Long metric1 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[0]); + Long metric2 = starTreeDocuments[i].metrics[1] != null + ? NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[1]) + : null; + segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Object[] { metric1, metric2 }); + } + + Iterator segmentStarTreeDocumentIterator = builder.sortMergeAndAggregateStarTreeDocument( + segmentStarTreeDocuments + ); + + StarTreeDocument resultStarTreeDocument = segmentStarTreeDocumentIterator.next(); assertEquals(expectedStarTreeDocument.dimensions[0], resultStarTreeDocument.dimensions[0]); assertEquals(expectedStarTreeDocument.dimensions[1], resultStarTreeDocument.dimensions[1]); assertEquals(expectedStarTreeDocument.dimensions[2], resultStarTreeDocument.dimensions[2]); @@ -210,27 +233,270 @@ public void test_processStarTreeDocuments_nullDocument() throws IOException { assertEquals(expectedStarTreeDocument.metrics[0], resultStarTreeDocument.metrics[0]); assertEquals(expectedStarTreeDocument.metrics[1], resultStarTreeDocument.metrics[1]); - assertThrows("Cannot apply aggregated value [null]", IllegalArgumentException.class, starTreeDocumentIterator::next); + assertThrows( + "Null metric should have resulted in IllegalStateException", + IllegalStateException.class, + segmentStarTreeDocumentIterator::next + ); } - public void test_build() throws IOException { + public void test_sortMergeAndAggregateStarTreeDocument_longMaxAndLongMinDimensions() throws IOException { int noOfStarTreeDocuments = 5; StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; - starTreeDocuments[0] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 12.0, 10.0 }); + starTreeDocuments[0] = new StarTreeDocument(new long[] { Long.MIN_VALUE, 4, 3, 4 }, new Double[] { 12.0, 10.0 }); + starTreeDocuments[1] = new StarTreeDocument(new long[] { 3, 4, 2, Long.MAX_VALUE }, new Double[] { 10.0, 6.0 }); + starTreeDocuments[2] = new StarTreeDocument(new long[] { 3, 4, 2, Long.MAX_VALUE }, new Double[] { 14.0, 12.0 }); + starTreeDocuments[3] = new StarTreeDocument(new long[] { Long.MIN_VALUE, 4, 3, 4 }, new Double[] { 9.0, 4.0 }); + starTreeDocuments[4] = new StarTreeDocument(new long[] { 3, 4, 2, Long.MAX_VALUE }, new Double[] { 11.0, 16.0 }); + + List inorderStarTreeDocuments = List.of( + new StarTreeDocument(new long[] { Long.MIN_VALUE, 4, 3, 4 }, new Double[] { 21.0, 14.0 }), + new StarTreeDocument(new long[] { 3, 4, 2, Long.MAX_VALUE }, new Double[] { 35.0, 34.0 }) + ); + Iterator expectedStarTreeDocumentIterator = inorderStarTreeDocuments.iterator(); + + StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; + for (int i = 0; i < noOfStarTreeDocuments; i++) { + long metric1 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[0]); + long metric2 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[1]); + segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2 }); + } + + Iterator segmentStarTreeDocumentIterator = builder.sortMergeAndAggregateStarTreeDocument( + segmentStarTreeDocuments + ); + int numOfAggregatedDocuments = 0; + while (segmentStarTreeDocumentIterator.hasNext() && expectedStarTreeDocumentIterator.hasNext()) { + StarTreeDocument resultStarTreeDocument = segmentStarTreeDocumentIterator.next(); + StarTreeDocument expectedStarTreeDocument = expectedStarTreeDocumentIterator.next(); + + assertEquals(expectedStarTreeDocument.dimensions[0], resultStarTreeDocument.dimensions[0]); + assertEquals(expectedStarTreeDocument.dimensions[1], resultStarTreeDocument.dimensions[1]); + assertEquals(expectedStarTreeDocument.dimensions[2], resultStarTreeDocument.dimensions[2]); + assertEquals(expectedStarTreeDocument.dimensions[3], resultStarTreeDocument.dimensions[3]); + assertEquals(expectedStarTreeDocument.metrics[0], resultStarTreeDocument.metrics[0]); + assertEquals(expectedStarTreeDocument.metrics[1], resultStarTreeDocument.metrics[1]); + + numOfAggregatedDocuments++; + } + + assertEquals(inorderStarTreeDocuments.size(), numOfAggregatedDocuments); + + } + + public void test_build_DoubleMaxAndDoubleMinMetrics() throws IOException { + + int noOfStarTreeDocuments = 5; + StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; + + starTreeDocuments[0] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { Double.MAX_VALUE, 10.0 }); starTreeDocuments[1] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 10.0, 6.0 }); - starTreeDocuments[2] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 14.0, 12.0 }); + starTreeDocuments[2] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 14.0, Double.MIN_VALUE }); starTreeDocuments[3] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 9.0, 4.0 }); starTreeDocuments[4] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 11.0, 16.0 }); - Iterator starTreeDocumentIterator = builder.processStarTreeDocuments(starTreeDocuments); - builder.build(starTreeDocumentIterator); + List inorderStarTreeDocuments = List.of( + new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { Double.MAX_VALUE + 9, 14.0 }), + new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 35.0, Double.MIN_VALUE + 22 }) + ); + Iterator expectedStarTreeDocumentIterator = inorderStarTreeDocuments.iterator(); + + StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; + for (int i = 0; i < noOfStarTreeDocuments; i++) { + long metric1 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[0]); + long metric2 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[1]); + segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2 }); + } + + Iterator segmentStarTreeDocumentIterator = builder.sortMergeAndAggregateStarTreeDocument( + segmentStarTreeDocuments + ); + int numOfAggregatedDocuments = 0; + while (segmentStarTreeDocumentIterator.hasNext() && expectedStarTreeDocumentIterator.hasNext()) { + StarTreeDocument resultStarTreeDocument = segmentStarTreeDocumentIterator.next(); + StarTreeDocument expectedStarTreeDocument = expectedStarTreeDocumentIterator.next(); + + assertEquals(expectedStarTreeDocument.dimensions[0], resultStarTreeDocument.dimensions[0]); + assertEquals(expectedStarTreeDocument.dimensions[1], resultStarTreeDocument.dimensions[1]); + assertEquals(expectedStarTreeDocument.dimensions[2], resultStarTreeDocument.dimensions[2]); + assertEquals(expectedStarTreeDocument.dimensions[3], resultStarTreeDocument.dimensions[3]); + assertEquals(expectedStarTreeDocument.metrics[0], resultStarTreeDocument.metrics[0]); + assertEquals(expectedStarTreeDocument.metrics[1], resultStarTreeDocument.metrics[1]); + + numOfAggregatedDocuments++; + } + + assertEquals(inorderStarTreeDocuments.size(), numOfAggregatedDocuments); + + } + + public void test_build_halfFloatMetrics() throws IOException { + + mapperService = mock(MapperService.class); + DocumentMapper documentMapper = mock(DocumentMapper.class); + when(mapperService.documentMapper()).thenReturn(documentMapper); + Settings settings = Settings.builder().put(settings(org.opensearch.Version.CURRENT).build()).build(); + NumberFieldMapper numberFieldMapper1 = new NumberFieldMapper.Builder("field2", NumberFieldMapper.NumberType.HALF_FLOAT, false, true) + .build(new Mapper.BuilderContext(settings, new ContentPath())); + NumberFieldMapper numberFieldMapper2 = new NumberFieldMapper.Builder("field4", NumberFieldMapper.NumberType.HALF_FLOAT, false, true) + .build(new Mapper.BuilderContext(settings, new ContentPath())); + MappingLookup fieldMappers = new MappingLookup( + Set.of(numberFieldMapper1, numberFieldMapper2), + Collections.emptyList(), + Collections.emptyList(), + 0, + null + ); + when(documentMapper.mappers()).thenReturn(fieldMappers); + builder = new OnHeapSingleTreeBuilder(compositeField, fieldProducerMap, docValuesConsumer, writeState, mapperService); + + int noOfStarTreeDocuments = 5; + StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; + + starTreeDocuments[0] = new StarTreeDocument( + new long[] { 2, 4, 3, 4 }, + new HalfFloatPoint[] { new HalfFloatPoint("hf1", 12), new HalfFloatPoint("hf6", 10) } + ); + starTreeDocuments[1] = new StarTreeDocument( + new long[] { 3, 4, 2, 1 }, + new HalfFloatPoint[] { new HalfFloatPoint("hf2", 10), new HalfFloatPoint("hf7", 6) } + ); + starTreeDocuments[2] = new StarTreeDocument( + new long[] { 3, 4, 2, 1 }, + new HalfFloatPoint[] { new HalfFloatPoint("hf3", 14), new HalfFloatPoint("hf8", 12) } + ); + starTreeDocuments[3] = new StarTreeDocument( + new long[] { 2, 4, 3, 4 }, + new HalfFloatPoint[] { new HalfFloatPoint("hf4", 9), new HalfFloatPoint("hf9", 4) } + ); + starTreeDocuments[4] = new StarTreeDocument( + new long[] { 3, 4, 2, 1 }, + new HalfFloatPoint[] { new HalfFloatPoint("hf5", 11), new HalfFloatPoint("hf10", 16) } + ); + + StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; + for (int i = 0; i < noOfStarTreeDocuments; i++) { + long metric1 = HalfFloatPoint.halfFloatToSortableShort( + ((HalfFloatPoint) starTreeDocuments[i].metrics[0]).numericValue().floatValue() + ); + long metric2 = HalfFloatPoint.halfFloatToSortableShort( + ((HalfFloatPoint) starTreeDocuments[i].metrics[1]).numericValue().floatValue() + ); + segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2 }); + } + + Iterator segmentStarTreeDocumentIterator = builder.sortMergeAndAggregateStarTreeDocument( + segmentStarTreeDocuments + ); + builder.build(segmentStarTreeDocumentIterator); List resultStarTreeDocuments = builder.getStarTreeDocuments(); assertEquals(8, resultStarTreeDocuments.size()); + Iterator expectedStarTreeDocumentIterator = getExpectedStarTreeDocumentIterator(); + assertStarTreeDocuments(resultStarTreeDocuments, expectedStarTreeDocumentIterator); + } + + public void test_build_floatMetrics() throws IOException { + + mapperService = mock(MapperService.class); + DocumentMapper documentMapper = mock(DocumentMapper.class); + when(mapperService.documentMapper()).thenReturn(documentMapper); + Settings settings = Settings.builder().put(settings(org.opensearch.Version.CURRENT).build()).build(); + NumberFieldMapper numberFieldMapper1 = new NumberFieldMapper.Builder("field2", NumberFieldMapper.NumberType.FLOAT, false, true) + .build(new Mapper.BuilderContext(settings, new ContentPath())); + NumberFieldMapper numberFieldMapper2 = new NumberFieldMapper.Builder("field4", NumberFieldMapper.NumberType.FLOAT, false, true) + .build(new Mapper.BuilderContext(settings, new ContentPath())); + MappingLookup fieldMappers = new MappingLookup( + Set.of(numberFieldMapper1, numberFieldMapper2), + Collections.emptyList(), + Collections.emptyList(), + 0, + null + ); + when(documentMapper.mappers()).thenReturn(fieldMappers); + builder = new OnHeapSingleTreeBuilder(compositeField, fieldProducerMap, docValuesConsumer, writeState, mapperService); + + int noOfStarTreeDocuments = 5; + StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; + + starTreeDocuments[0] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Float[] { 12.0F, 10.0F }); + starTreeDocuments[1] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Float[] { 10.0F, 6.0F }); + starTreeDocuments[2] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Float[] { 14.0F, 12.0F }); + starTreeDocuments[3] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Float[] { 9.0F, 4.0F }); + starTreeDocuments[4] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Float[] { 11.0F, 16.0F }); + + StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; + for (int i = 0; i < noOfStarTreeDocuments; i++) { + long metric1 = NumericUtils.floatToSortableInt((Float) starTreeDocuments[i].metrics[0]); + long metric2 = NumericUtils.floatToSortableInt((Float) starTreeDocuments[i].metrics[1]); + segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2 }); + } + + Iterator segmentStarTreeDocumentIterator = builder.sortMergeAndAggregateStarTreeDocument( + segmentStarTreeDocuments + ); + builder.build(segmentStarTreeDocumentIterator); + + List resultStarTreeDocuments = builder.getStarTreeDocuments(); + assertEquals(8, resultStarTreeDocuments.size()); + + Iterator expectedStarTreeDocumentIterator = getExpectedStarTreeDocumentIterator(); + assertStarTreeDocuments(resultStarTreeDocuments, expectedStarTreeDocumentIterator); + } + + public void test_build_longMetrics() throws IOException { + + mapperService = mock(MapperService.class); + DocumentMapper documentMapper = mock(DocumentMapper.class); + when(mapperService.documentMapper()).thenReturn(documentMapper); + Settings settings = Settings.builder().put(settings(org.opensearch.Version.CURRENT).build()).build(); + NumberFieldMapper numberFieldMapper1 = new NumberFieldMapper.Builder("field2", NumberFieldMapper.NumberType.LONG, false, true) + .build(new Mapper.BuilderContext(settings, new ContentPath())); + NumberFieldMapper numberFieldMapper2 = new NumberFieldMapper.Builder("field4", NumberFieldMapper.NumberType.LONG, false, true) + .build(new Mapper.BuilderContext(settings, new ContentPath())); + MappingLookup fieldMappers = new MappingLookup( + Set.of(numberFieldMapper1, numberFieldMapper2), + Collections.emptyList(), + Collections.emptyList(), + 0, + null + ); + when(documentMapper.mappers()).thenReturn(fieldMappers); + builder = new OnHeapSingleTreeBuilder(compositeField, fieldProducerMap, docValuesConsumer, writeState, mapperService); + + int noOfStarTreeDocuments = 5; + StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; + + starTreeDocuments[0] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Long[] { 12L, 10L }); + starTreeDocuments[1] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Long[] { 10L, 6L }); + starTreeDocuments[2] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Long[] { 14L, 12L }); + starTreeDocuments[3] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Long[] { 9L, 4L }); + starTreeDocuments[4] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Long[] { 11L, 16L }); + + StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; + for (int i = 0; i < noOfStarTreeDocuments; i++) { + long metric1 = (Long) starTreeDocuments[i].metrics[0]; + long metric2 = (Long) starTreeDocuments[i].metrics[1]; + segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2 }); + } + + Iterator segmentStarTreeDocumentIterator = builder.sortMergeAndAggregateStarTreeDocument( + segmentStarTreeDocuments + ); + builder.build(segmentStarTreeDocumentIterator); + + List resultStarTreeDocuments = builder.getStarTreeDocuments(); + assertEquals(8, resultStarTreeDocuments.size()); + + Iterator expectedStarTreeDocumentIterator = getExpectedStarTreeDocumentIterator(); + assertStarTreeDocuments(resultStarTreeDocuments, expectedStarTreeDocumentIterator); + } + + private static Iterator getExpectedStarTreeDocumentIterator() { List expectedStarTreeDocuments = List.of( new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 21.0, 14.0 }), new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 35.0, 34.0 }), @@ -241,7 +507,43 @@ public void test_build() throws IOException { new StarTreeDocument(new long[] { -1, 4, -1, -1 }, new Double[] { 56.0, 48.0 }), new StarTreeDocument(new long[] { -1, -1, -1, -1 }, new Double[] { 56.0, 48.0 }) ); - Iterator expectedStarTreeDocumentIterator = expectedStarTreeDocuments.iterator(); + return expectedStarTreeDocuments.iterator(); + } + + public void test_build() throws IOException { + + int noOfStarTreeDocuments = 5; + StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; + + starTreeDocuments[0] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 12.0, 10.0 }); + starTreeDocuments[1] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 10.0, 6.0 }); + starTreeDocuments[2] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 14.0, 12.0 }); + starTreeDocuments[3] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 9.0, 4.0 }); + starTreeDocuments[4] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 11.0, 16.0 }); + + StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; + for (int i = 0; i < noOfStarTreeDocuments; i++) { + long metric1 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[0]); + long metric2 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[1]); + segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2 }); + } + + Iterator segmentStarTreeDocumentIterator = builder.sortMergeAndAggregateStarTreeDocument( + segmentStarTreeDocuments + ); + builder.build(segmentStarTreeDocumentIterator); + + List resultStarTreeDocuments = builder.getStarTreeDocuments(); + assertEquals(8, resultStarTreeDocuments.size()); + + Iterator expectedStarTreeDocumentIterator = getExpectedStarTreeDocumentIterator(); + assertStarTreeDocuments(resultStarTreeDocuments, expectedStarTreeDocumentIterator); + } + + private static void assertStarTreeDocuments( + List resultStarTreeDocuments, + Iterator expectedStarTreeDocumentIterator + ) { Iterator resultStarTreeDocumentIterator = resultStarTreeDocuments.iterator(); while (resultStarTreeDocumentIterator.hasNext() && expectedStarTreeDocumentIterator.hasNext()) { StarTreeDocument resultStarTreeDocument = resultStarTreeDocumentIterator.next(); diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeValuesIteratorFactoryTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeValuesIteratorFactoryTests.java index f64da30eb591d..f5d53afb8f774 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeValuesIteratorFactoryTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeValuesIteratorFactoryTests.java @@ -84,7 +84,7 @@ public void testGetNextValue_SortedSet() throws IOException { SortedSetDocValues iterator = Mockito.mock(SortedSetDocValues.class); when(iterator.nextOrd()).thenReturn(42L); - long result = factory.getNextValue(iterator); + long result = factory.getNextOrd(iterator); assertEquals(42L, result); } @@ -92,14 +92,14 @@ public void testGetNextValue_SortedNumeric() throws IOException { SortedNumericDocValues iterator = Mockito.mock(SortedNumericDocValues.class); when(iterator.nextValue()).thenReturn(123L); - long result = factory.getNextValue(iterator); + long result = factory.getNextOrd(iterator); assertEquals(123L, result); } public void testGetNextValue_UnsupportedIterator() { DocIdSetIterator iterator = Mockito.mock(DocIdSetIterator.class); - IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> { factory.getNextValue(iterator); }); + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> { factory.getNextOrd(iterator); }); assertEquals("Unsupported Iterator: " + iterator.toString(), exception.getMessage()); } From d90486f35af46852f5c06b2d8828b3be54544687 Mon Sep 17 00:00:00 2001 From: Sarthak Aggarwal Date: Thu, 27 Jun 2024 00:43:55 +0530 Subject: [PATCH 11/11] includes Count Aggregator Signed-off-by: Sarthak Aggarwal --- .../lucene/index/BaseStarTreeBuilder.java | 195 +++++++++--------- .../common/inject/util/Modules.java | 1 + .../aggregators/CountValueAggregator.java | 22 +- ...criptor.java => MetricAggregatorInfo.java} | 58 ++---- .../aggregators/SumValueAggregator.java | 22 +- .../startree/aggregators/ValueAggregator.java | 10 +- .../aggregators/ValueAggregatorFactory.java | 4 +- .../numerictype/StarTreeNumericType.java | 11 +- .../StarTreeNumericTypeConverters.java | 14 ++ ...uilder.java => OnHeapStarTreeBuilder.java} | 13 +- ...eTreeBuilder.java => StarTreeBuilder.java} | 4 +- .../StarTreeDocValuesIteratorAdapter.java | 14 +- .../startree/builder/StarTreesBuilder.java | 33 +-- .../startree/utils/StarTreeBuilderUtils.java | 37 +++- .../CountValueAggregatorTests.java | 56 +++++ ...ts.java => MetricAggregatorInfoTests.java} | 43 ++-- .../aggregators/SumValueAggregatorTests.java | 30 +-- .../ValueAggregatorFactoryTests.java | 4 +- .../builder/BaseStarTreeBuilderTests.java | 21 +- ...s.java => OnHeapStarTreeBuilderTests.java} | 177 +++++++++------- .../StarTreeValuesIteratorFactoryTests.java | 21 +- 21 files changed, 448 insertions(+), 342 deletions(-) rename server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/{MetricAggregationDescriptor.java => MetricAggregatorInfo.java} (63%) rename server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/{OnHeapSingleTreeBuilder.java => OnHeapStarTreeBuilder.java} (93%) rename server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/{SingleTreeBuilder.java => StarTreeBuilder.java} (87%) create mode 100644 server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregatorTests.java rename server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/{MetricAggregationDescriptorTests.java => MetricAggregatorInfoTests.java} (63%) rename server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/{OnHeapSingleTreeBuilderTests.java => OnHeapStarTreeBuilderTests.java} (79%) diff --git a/server/src/main/java/org/apache/lucene/index/BaseStarTreeBuilder.java b/server/src/main/java/org/apache/lucene/index/BaseStarTreeBuilder.java index 425101678ba58..1d851603369a3 100644 --- a/server/src/main/java/org/apache/lucene/index/BaseStarTreeBuilder.java +++ b/server/src/main/java/org/apache/lucene/index/BaseStarTreeBuilder.java @@ -9,21 +9,20 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.apache.lucene.codecs.DocValuesConsumer; import org.apache.lucene.codecs.DocValuesProducer; import org.apache.lucene.search.DocIdSetIterator; -import org.opensearch.index.compositeindex.datacube.DateDimension; import org.opensearch.index.compositeindex.datacube.Dimension; import org.opensearch.index.compositeindex.datacube.Metric; import org.opensearch.index.compositeindex.datacube.MetricStat; import org.opensearch.index.compositeindex.datacube.startree.StarTreeDocument; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; -import org.opensearch.index.compositeindex.datacube.startree.aggregators.MetricAggregationDescriptor; +import org.opensearch.index.compositeindex.datacube.startree.aggregators.MetricAggregatorInfo; import org.opensearch.index.compositeindex.datacube.startree.aggregators.ValueAggregator; import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; -import org.opensearch.index.compositeindex.datacube.startree.builder.SingleTreeBuilder; +import org.opensearch.index.compositeindex.datacube.startree.builder.StarTreeBuilder; import org.opensearch.index.compositeindex.datacube.startree.builder.StarTreeDocValuesIteratorAdapter; +import org.opensearch.index.compositeindex.datacube.startree.builder.StarTreesBuilder; import org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeBuilderUtils; import org.opensearch.index.fielddata.IndexNumericFieldData; import org.opensearch.index.mapper.Mapper; @@ -41,10 +40,12 @@ import java.util.Set; /** - * Base class for star-tree builder + * Builder for star tree. Defines the algorithm to construct star-tree + * See {@link StarTreesBuilder} for information around the construction of star-trees based on star-tree fields + * * @opensearch.experimental */ -public abstract class BaseStarTreeBuilder implements SingleTreeBuilder { +public abstract class BaseStarTreeBuilder implements StarTreeBuilder { private static final Logger logger = LogManager.getLogger(BaseStarTreeBuilder.class); @@ -52,7 +53,7 @@ public abstract class BaseStarTreeBuilder implements SingleTreeBuilder { protected final Set skipStarNodeCreationForDimensions; - protected final List metricAggregationDescriptors; + protected final List metricAggregatorInfos; protected final int numMetrics; protected final int numDimensions; protected int numStarTreeDocs; @@ -65,24 +66,21 @@ public abstract class BaseStarTreeBuilder implements SingleTreeBuilder { protected DocIdSetIterator[] dimensionReaders; protected Map fieldProducerMap; - protected DocValuesConsumer docValuesConsumer; private final StarTreeDocValuesIteratorAdapter starTreeDocValuesIteratorFactory; private final StarTreeField starTreeField; /** - * Constructor for base star-tree builder + * Reads all the configuration related to dimensions and metrics, builds a star-tree based on the different construction parameters. * - * @param starTreeField holds the configuration for the star tree - * @param fieldProducerMap helps return the doc values iterator for each type based on field name - * @param docValuesConsumer to consume the new aggregated metrics during flush - * @param state stores the segment write state - * @param mapperService helps to find the original type of the field + * @param starTreeField holds the configuration for the star tree + * @param fieldProducerMap helps return the doc values iterator for each type based on field name + * @param state stores the segment write state + * @param mapperService helps to find the original type of the field */ protected BaseStarTreeBuilder( StarTreeField starTreeField, Map fieldProducerMap, - DocValuesConsumer docValuesConsumer, SegmentWriteState state, MapperService mapperService ) throws IOException { @@ -91,7 +89,6 @@ protected BaseStarTreeBuilder( this.starTreeField = starTreeField; StarTreeFieldConfiguration starTreeFieldSpec = starTreeField.getStarTreeConfig(); - this.docValuesConsumer = docValuesConsumer; this.fieldProducerMap = fieldProducerMap; this.starTreeDocValuesIteratorFactory = new StarTreeDocValuesIteratorAdapter(); @@ -117,50 +114,53 @@ protected BaseStarTreeBuilder( ); } - this.metricAggregationDescriptors = generateMetricStatFieldPairs(mapperService, state); - this.numMetrics = metricAggregationDescriptors.size(); + this.metricAggregatorInfos = generateMetricAggregatorInfos(mapperService, state); + this.numMetrics = metricAggregatorInfos.size(); this.maxLeafDocuments = starTreeFieldSpec.maxLeafDocs(); } /** - * Generates the MetricStatFieldPairs for all the metrics on a field + * Generates the configuration required to perform aggregation for all the metrics on a field * - * @return list of metric stat mapped with respective fields + * @return list of MetricAggregatorInfo */ - public List generateMetricStatFieldPairs(MapperService mapperService, SegmentWriteState state) + public List generateMetricAggregatorInfos(MapperService mapperService, SegmentWriteState state) throws IOException { - List metricAggregationDescriptors = new ArrayList<>(); - IndexNumericFieldData.NumericType numericType; - DocIdSetIterator metricStatReader = null; + List metricAggregatorInfos = new ArrayList<>(); for (Metric metric : this.starTreeField.getMetrics()) { for (MetricStat metricType : metric.getMetrics()) { + IndexNumericFieldData.NumericType numericType; + DocIdSetIterator metricStatReader = null; Mapper fieldMapper = mapperService.documentMapper().mappers().getMapper(metric.getField()); if (fieldMapper instanceof NumberFieldMapper) { numericType = ((NumberFieldMapper) fieldMapper).fieldType().numericType(); } else { - logger.error("metric mapper is not of type number field mapper"); - throw new IllegalStateException("metric mapper is not of type number field mapper"); + logger.error("unsupported mapper type"); + throw new IllegalStateException("unsupported mapper type"); } - // Ignore the column for COUNT aggregation function + + FieldInfo metricFieldInfos = state.fieldInfos.fieldInfo(metric.getField()); + DocValuesType metricDocValuesType = metricFieldInfos.getDocValuesType(); if (metricType != MetricStat.COUNT) { - FieldInfo metricFieldInfos = state.fieldInfos.fieldInfo(metric.getField()); - DocValuesType metricDocValuesType = metricFieldInfos.getDocValuesType(); + // Need not initialize the metric reader for COUNT metric type metricStatReader = starTreeDocValuesIteratorFactory.getDocValuesIterator( metricDocValuesType, metricFieldInfos, fieldProducerMap.get(metricFieldInfos.name) ); } - MetricAggregationDescriptor metricAggregationDescriptor = new MetricAggregationDescriptor( + + MetricAggregatorInfo metricAggregatorInfo = new MetricAggregatorInfo( metricType, metric.getField(), + starTreeField.getName(), numericType, metricStatReader ); - metricAggregationDescriptors.add(metricAggregationDescriptor); + metricAggregatorInfos.add(metricAggregatorInfo); } } - return metricAggregationDescriptors; + return metricAggregatorInfos; } /** @@ -185,7 +185,7 @@ public List generateMetricStatFieldPairs(MapperServ * * @return Star tree documents */ - public abstract List getStarTreeDocuments() throws IOException; + public abstract List getStarTreeDocuments(); /** * Returns the value of the dimension for the given dimension id and document in the star-tree. @@ -206,7 +206,7 @@ public List generateMetricStatFieldPairs(MapperServ public abstract Iterator sortMergeAndAggregateStarTreeDocument(int numDocs) throws IOException; /** - * Generates aggregated star-tree document for star-node. + * Generates aggregated star-tree documents for star-node. * * @param startDocId start document id (inclusive) in the star-tree * @param endDocId end document id (exclusive) in the star-tree @@ -218,6 +218,7 @@ public abstract Iterator generateStarTreeDocumentsForStarNode( /** * Returns the star-tree document from the segment + * * @throws IOException when we are unable to build a star tree document from the segment */ protected StarTreeDocument getSegmentStarTreeDocument() throws IOException { @@ -235,25 +236,18 @@ protected StarTreeDocument getSegmentStarTreeDocument() throws IOException { long[] getStarTreeDimensionsFromSegment() throws IOException { long[] dimensions = new long[numDimensions]; for (int i = 0; i < numDimensions; i++) { - try { - dimensionReaders[i].nextDoc(); - } catch (IOException e) { - logger.error("unable to iterate to next doc", e); - } catch (NullPointerException e) { - logger.error("dimension does not have an associated reader", e); - throw new IllegalStateException("dimension should have a valid associated reader", e); - } catch (Exception e) { - logger.error("unable to read the dimension values from the segment", e); - throw new IllegalStateException("unable to read the dimension values from the segment", e); - } + if (dimensionReaders[i] != null) { + try { + dimensionReaders[i].nextDoc(); + } catch (IOException e) { + logger.error("unable to iterate to next doc", e); + throw e; + } catch (Exception e) { + logger.error("unable to read the dimension values from the segment", e); + throw new IllegalStateException("unable to read the dimension values from the segment", e); + } - if (starTreeField.getDimensionsOrder().get(i) instanceof DateDimension) { - dimensions[i] = handleDateDimension( - starTreeField.getDimensionsOrder().get(i).getField(), - starTreeDocValuesIteratorFactory.getNextOrd(dimensionReaders[i]) - ); - } else { - dimensions[i] = starTreeDocValuesIteratorFactory.getNextOrd(dimensionReaders[i]); + dimensions[i] = starTreeDocValuesIteratorFactory.getNextValue(dimensionReaders[i]); } } return dimensions; @@ -268,21 +262,18 @@ long[] getStarTreeDimensionsFromSegment() throws IOException { private Object[] getStarTreeMetricsFromSegment() throws IOException { Object[] metrics = new Object[numMetrics]; for (int i = 0; i < numMetrics; i++) { - // Ignore the column for COUNT aggregation function - DocIdSetIterator metricStatReader = metricAggregationDescriptors.get(i).getMetricStatReader(); + DocIdSetIterator metricStatReader = metricAggregatorInfos.get(i).getMetricStatReader(); if (metricStatReader != null) { try { metricStatReader.nextDoc(); } catch (IOException e) { logger.error("unable to iterate to next doc", e); - } catch (NullPointerException e) { - logger.error("metric does not have an associated reader", e); - throw new IllegalStateException("metric should have a valid associated reader", e); + throw e; } catch (Exception e) { logger.error("unable to read the metric values from the segment", e); throw new IllegalStateException("unable to read the metric values from the segment", e); } - metrics[i] = starTreeDocValuesIteratorFactory.getNextOrd(metricStatReader); + metrics[i] = starTreeDocValuesIteratorFactory.getNextValue(metricStatReader); } } return metrics; @@ -305,40 +296,62 @@ protected StarTreeDocument reduceSegmentStarTreeDocuments( Object[] metrics = new Object[numMetrics]; for (int i = 0; i < numMetrics; i++) { try { - ValueAggregator metricValueAggregator = metricAggregationDescriptors.get(i).getValueAggregators(); - StarTreeNumericType starTreeNumericType = metricAggregationDescriptors.get(i).getStarTreeNumericType(); - metrics[i] = metricValueAggregator.getInitialAggregatedValue((Long) segmentDocument.metrics[i], starTreeNumericType); - } catch (IllegalArgumentException | NullPointerException | IllegalStateException e) { - logger.error("Cannot parse initial segment doc value", e); - throw new IllegalStateException("Cannot parse initial segment doc value [" + segmentDocument.metrics[i] + "]"); + ValueAggregator metricValueAggregator = metricAggregatorInfos.get(i).getValueAggregators(); + StarTreeNumericType starTreeNumericType = metricAggregatorInfos.get(i).getAggregatedValueType(); + metrics[i] = metricValueAggregator.getInitialAggregatedValueForSegmentDocValue( + getLong(segmentDocument.metrics[i]), + starTreeNumericType + ); } catch (Exception e) { logger.error("Cannot parse initial segment doc value", e); - throw new RuntimeException("Cannot parse initial segment doc value [" + segmentDocument.metrics[i] + "]"); + throw new IllegalStateException("Cannot parse initial segment doc value [" + segmentDocument.metrics[i] + "]"); } } return new StarTreeDocument(dimensions, metrics); } else { for (int i = 0; i < numMetrics; i++) { try { - ValueAggregator metricValueAggregator = metricAggregationDescriptors.get(i).getValueAggregators(); - StarTreeNumericType starTreeNumericType = metricAggregationDescriptors.get(i).getStarTreeNumericType(); - aggregatedSegmentDocument.metrics[i] = metricValueAggregator.applySegmentRawValue( + ValueAggregator metricValueAggregator = metricAggregatorInfos.get(i).getValueAggregators(); + StarTreeNumericType starTreeNumericType = metricAggregatorInfos.get(i).getAggregatedValueType(); + aggregatedSegmentDocument.metrics[i] = metricValueAggregator.mergeAggregatedValueAndSegmentValue( aggregatedSegmentDocument.metrics[i], - (Long) segmentDocument.metrics[i], + getLong(segmentDocument.metrics[i]), starTreeNumericType ); - } catch (IllegalArgumentException | NullPointerException | IllegalStateException e) { - logger.error("Cannot apply segment doc value for aggregation", e); - throw new IllegalStateException("Cannot apply segment doc value for aggregation [" + segmentDocument.metrics[i] + "]"); } catch (Exception e) { logger.error("Cannot apply segment doc value for aggregation", e); - throw new RuntimeException("Cannot apply segment doc value for aggregation [" + segmentDocument.metrics[i] + "]"); + throw new IllegalStateException("Cannot apply segment doc value for aggregation [" + segmentDocument.metrics[i] + "]"); } } return aggregatedSegmentDocument; } } + /** + * Safely converts the metric value of object type to long. + * + * @param metric value of the metric + * @return converted metric value to long + */ + private static long getLong(Object metric) { + + Long metricValue = null; + try { + if (metric instanceof Long) { + metricValue = (long) metric; + } else if (metric != null) { + metricValue = Long.valueOf(String.valueOf(metric)); + } + } catch (Exception e) { + throw new IllegalStateException("unable to cast segment metric", e); + } + + if (metricValue == null) { + throw new IllegalStateException("unable to cast segment metric"); + } + return metricValue; + } + /** * Merges a star-tree document into an aggregated star-tree document. * A new aggregated star-tree document is created if the aggregated document is null. @@ -354,30 +367,22 @@ public StarTreeDocument reduceStarTreeDocuments(StarTreeDocument aggregatedDocum Object[] metrics = new Object[numMetrics]; for (int i = 0; i < numMetrics; i++) { try { - metrics[i] = metricAggregationDescriptors.get(i).getValueAggregators().getAggregatedValue(starTreeDocument.metrics[i]); - } catch (IllegalArgumentException | NullPointerException e) { - logger.error("Cannot get aggregated value", e); - throw new IllegalArgumentException("Cannot get aggregated value[" + starTreeDocument.metrics[i] + "]"); + metrics[i] = metricAggregatorInfos.get(i).getValueAggregators().getInitialAggregatedValue(starTreeDocument.metrics[i]); } catch (Exception e) { logger.error("Cannot get value for aggregation", e); - throw new RuntimeException("Cannot get value for aggregation[" + starTreeDocument.metrics[i] + "]"); + throw new IllegalStateException("Cannot get value for aggregation[" + starTreeDocument.metrics[i] + "]"); } } return new StarTreeDocument(dimensions, metrics); } else { for (int i = 0; i < numMetrics; i++) { try { - aggregatedDocument.metrics[i] = metricAggregationDescriptors.get(i) + aggregatedDocument.metrics[i] = metricAggregatorInfos.get(i) .getValueAggregators() - .applyAggregatedValue(starTreeDocument.metrics[i], aggregatedDocument.metrics[i]); - } catch (IllegalArgumentException | NullPointerException e) { - logger.error("Cannot apply value to aggregated document for aggregation", e); - throw new IllegalArgumentException( - "Cannot apply value to aggregated document for aggregation[" + starTreeDocument.metrics[i] + "]" - ); + .mergeAggregatedValues(starTreeDocument.metrics[i], aggregatedDocument.metrics[i]); } catch (Exception e) { logger.error("Cannot apply value to aggregated document for aggregation", e); - throw new RuntimeException( + throw new IllegalStateException( "Cannot apply value to aggregated document for aggregation [" + starTreeDocument.metrics[i] + "]" ); } @@ -393,7 +398,7 @@ public StarTreeDocument reduceStarTreeDocuments(StarTreeDocument aggregatedDocum */ public void build() throws IOException { long startTime = System.currentTimeMillis(); - logger.debug("Star-tree build is a go with star tree field{}", starTreeField); + logger.debug("Star-tree build is a go with star tree field {}", starTreeField.getName()); Iterator starTreeDocumentIterator = sortMergeAndAggregateStarTreeDocument(totalSegmentDocs); logger.debug("Sorting and aggregating star-tree in ms : {}", (System.currentTimeMillis() - startTime)); @@ -465,9 +470,9 @@ private StarTreeBuilderUtils.TreeNode getNewNode() { /** * Implements the algorithm to construct a star-tree * - * @param node star-tree node - * @param startDocId start document id - * @param endDocId end document id + * @param node star-tree node + * @param startDocId start document id + * @param endDocId end document id * @throws IOException throws an exception if we are unable to construct the tree */ private void constructStarTree(StarTreeBuilderUtils.TreeNode node, int startDocId, int endDocId) throws IOException { @@ -498,8 +503,8 @@ private void constructStarTree(StarTreeBuilderUtils.TreeNode node, int startDocI /** * Constructs non star tree nodes * - * @param startDocId start document id - * @param endDocId end document id + * @param startDocId start document id (inclusive) + * @param endDocId end document id (exclusive) * @param dimensionId id of the dimension in the star tree * @return root node with non-star nodes constructed * @throws IOException throws an exception if we are unable to construct non-star nodes @@ -535,8 +540,8 @@ private Map constructNonStarNodes(int start /** * Constructs star tree nodes * - * @param startDocId start document id - * @param endDocId end document id + * @param startDocId start document id (inclusive) + * @param endDocId end document id (exclusive) * @param dimensionId id of the dimension in the star tree * @return root node with star nodes constructed * @throws IOException throws an exception if we are unable to construct non-star nodes @@ -576,7 +581,6 @@ private StarTreeDocument createAggregatedDocs(StarTreeBuilderUtils.TreeNode node for (int i = node.startDocId; i < node.endDocId; i++) { aggregatedStarTreeDocument = reduceStarTreeDocuments(aggregatedStarTreeDocument, getStarTreeDocument(i)); } - assert aggregatedStarTreeDocument != null; if (null == aggregatedStarTreeDocument) { throw new IllegalStateException("aggregated star-tree document is null after reducing the documents"); } @@ -603,7 +607,6 @@ private StarTreeDocument createAggregatedDocs(StarTreeBuilderUtils.TreeNode node for (StarTreeBuilderUtils.TreeNode child : node.children.values()) { aggregatedStarTreeDocument = reduceStarTreeDocuments(aggregatedStarTreeDocument, createAggregatedDocs(child)); } - assert aggregatedStarTreeDocument != null; if (null == aggregatedStarTreeDocument) { throw new IllegalStateException("aggregated star-tree document is null after reducing the documents"); } diff --git a/server/src/main/java/org/opensearch/common/inject/util/Modules.java b/server/src/main/java/org/opensearch/common/inject/util/Modules.java index 71eafc182c573..ae37cb3d29a68 100644 --- a/server/src/main/java/org/opensearch/common/inject/util/Modules.java +++ b/server/src/main/java/org/opensearch/common/inject/util/Modules.java @@ -208,6 +208,7 @@ public Void visit(Binding binding) { scopeInstancesInUse.put(scope, binding.getSource()); } } + return null; } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregator.java index b4837809b9706..193c66cfa033c 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregator.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregator.java @@ -17,7 +17,7 @@ * @opensearch.experimental */ public class CountValueAggregator implements ValueAggregator { - public static final StarTreeNumericType STAR_TREE_NUMERIC_TYPE = StarTreeNumericType.DOUBLE; + public static final StarTreeNumericType VALUE_AGGREGATOR_TYPE = StarTreeNumericType.DOUBLE; @Override public MetricStat getAggregationType() { @@ -25,27 +25,27 @@ public MetricStat getAggregationType() { } @Override - public StarTreeNumericType getStarTreeNumericType() { - return STAR_TREE_NUMERIC_TYPE; + public StarTreeNumericType getAggregatedValueType() { + return VALUE_AGGREGATOR_TYPE; } @Override - public Double getInitialAggregatedValue(Long segmentDocValue, StarTreeNumericType starTreeNumericType) { + public Double getInitialAggregatedValueForSegmentDocValue(Long segmentDocValue, StarTreeNumericType starTreeNumericType) { return 1.0; } @Override - public Double applySegmentRawValue(Double value, Long segmentDocValue, StarTreeNumericType starTreeNumericType) { + public Double mergeAggregatedValueAndSegmentValue(Double value, Long segmentDocValue, StarTreeNumericType starTreeNumericType) { return value + 1; } @Override - public Double applyAggregatedValue(Double value, Double aggregatedValue) { + public Double mergeAggregatedValues(Double value, Double aggregatedValue) { return value + aggregatedValue; } @Override - public Double getAggregatedValue(Double value) { + public Double getInitialAggregatedValue(Double value) { return value; } @@ -58,8 +58,8 @@ public int getMaxAggregatedValueByteSize() { public Long toLongValue(Double value) { try { return NumericUtils.doubleToSortableLong(value); - } catch (IllegalArgumentException | NullPointerException | IllegalStateException e) { - throw new IllegalArgumentException("Cannot convert " + value + " to sortable long", e); + } catch (Exception e) { + throw new IllegalStateException("Cannot convert " + value + " to sortable long", e); } } @@ -67,8 +67,8 @@ public Long toLongValue(Double value) { public Double toStarTreeNumericTypeValue(Long value, StarTreeNumericType type) { try { return type.getDoubleValue(value); - } catch (IllegalArgumentException | NullPointerException | IllegalStateException e) { - throw new IllegalArgumentException("Cannot convert " + value + " to sortable aggregation type", e); + } catch (Exception e) { + throw new IllegalStateException("Cannot convert " + value + " to sortable aggregation type", e); } } } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregationDescriptor.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregatorInfo.java similarity index 63% rename from server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregationDescriptor.java rename to server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregatorInfo.java index f5e1055b6ef24..03e3de0831ed6 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregationDescriptor.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregatorInfo.java @@ -18,18 +18,11 @@ * Builds aggregation function and doc values field pair to support various aggregations * @opensearch.experimental */ -public class MetricAggregationDescriptor implements Comparable { - - public static final String DELIMITER = "__"; - public static final String STAR = "*"; - public static final MetricAggregationDescriptor COUNT_STAR = new MetricAggregationDescriptor( - MetricStat.COUNT, - STAR, - IndexNumericFieldData.NumericType.DOUBLE, - null - ); - - private final String metricStatName; +public class MetricAggregatorInfo implements Comparable { + + public static final String DELIMITER = "_"; + private final String metric; + private final String starFieldName; private final MetricStat metricStat; private final String field; private final ValueAggregator valueAggregators; @@ -37,11 +30,12 @@ public class MetricAggregationDescriptor implements Comparable o.field) - .thenComparing((MetricAggregationDescriptor o) -> o.metricStat) + public int compareTo(MetricAggregatorInfo other) { + return Comparator.comparing((MetricAggregatorInfo o) -> o.field) + .thenComparing((MetricAggregatorInfo o) -> o.metricStat) .compare(this, other); } } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregator.java index d9e55caae8784..982860d16bfa3 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregator.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregator.java @@ -19,7 +19,7 @@ */ public class SumValueAggregator implements ValueAggregator { - public static final StarTreeNumericType STAR_TREE_NUMERIC_TYPE = StarTreeNumericType.DOUBLE; + public static final StarTreeNumericType VALUE_AGGREGATOR_TYPE = StarTreeNumericType.DOUBLE; @Override public MetricStat getAggregationType() { @@ -27,17 +27,17 @@ public MetricStat getAggregationType() { } @Override - public StarTreeNumericType getStarTreeNumericType() { - return STAR_TREE_NUMERIC_TYPE; + public StarTreeNumericType getAggregatedValueType() { + return VALUE_AGGREGATOR_TYPE; } @Override - public Double getInitialAggregatedValue(Long segmentDocValue, StarTreeNumericType starTreeNumericType) { + public Double getInitialAggregatedValueForSegmentDocValue(Long segmentDocValue, StarTreeNumericType starTreeNumericType) { return starTreeNumericType.getDoubleValue(segmentDocValue); } @Override - public Double applySegmentRawValue(Double value, Long segmentDocValue, StarTreeNumericType starTreeNumericType) { + public Double mergeAggregatedValueAndSegmentValue(Double value, Long segmentDocValue, StarTreeNumericType starTreeNumericType) { CompensatedSum kahanSummation = new CompensatedSum(0, 0); kahanSummation.add(value); kahanSummation.add(starTreeNumericType.getDoubleValue(segmentDocValue)); @@ -45,7 +45,7 @@ public Double applySegmentRawValue(Double value, Long segmentDocValue, StarTreeN } @Override - public Double applyAggregatedValue(Double value, Double aggregatedValue) { + public Double mergeAggregatedValues(Double value, Double aggregatedValue) { CompensatedSum kahanSummation = new CompensatedSum(0, 0); kahanSummation.add(value); kahanSummation.add(aggregatedValue); @@ -53,7 +53,7 @@ public Double applyAggregatedValue(Double value, Double aggregatedValue) { } @Override - public Double getAggregatedValue(Double value) { + public Double getInitialAggregatedValue(Double value) { return value; } @@ -66,8 +66,8 @@ public int getMaxAggregatedValueByteSize() { public Long toLongValue(Double value) { try { return NumericUtils.doubleToSortableLong(value); - } catch (IllegalArgumentException | NullPointerException | IllegalStateException e) { - throw new IllegalArgumentException("Cannot convert " + value + " to sortable long", e); + } catch (Exception e) { + throw new IllegalStateException("Cannot convert " + value + " to sortable long", e); } } @@ -75,8 +75,8 @@ public Long toLongValue(Double value) { public Double toStarTreeNumericTypeValue(Long value, StarTreeNumericType type) { try { return type.getDoubleValue(value); - } catch (IllegalArgumentException | NullPointerException | IllegalStateException e) { - throw new IllegalArgumentException("Cannot convert " + value + " to sortable aggregation type", e); + } catch (Exception e) { + throw new IllegalStateException("Cannot convert " + value + " to sortable aggregation type", e); } } } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregator.java index 397f497d0105e..3dd1f85845c17 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregator.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregator.java @@ -25,27 +25,27 @@ public interface ValueAggregator { /** * Returns the data type of the aggregated value. */ - StarTreeNumericType getStarTreeNumericType(); + StarTreeNumericType getAggregatedValueType(); /** * Returns the initial aggregated value. */ - A getInitialAggregatedValue(Long segmentDocValue, StarTreeNumericType starTreeNumericType); + A getInitialAggregatedValueForSegmentDocValue(Long segmentDocValue, StarTreeNumericType starTreeNumericType); /** * Applies a segment doc value to the current aggregated value. */ - A applySegmentRawValue(A value, Long segmentDocValue, StarTreeNumericType starTreeNumericType); + A mergeAggregatedValueAndSegmentValue(A value, Long segmentDocValue, StarTreeNumericType starTreeNumericType); /** * Applies an aggregated value to the current aggregated value. */ - A applyAggregatedValue(A value, A aggregatedValue); + A mergeAggregatedValues(A value, A aggregatedValue); /** * Clones an aggregated value. */ - A getAggregatedValue(A value); + A getInitialAggregatedValue(A value); /** * Returns the maximum size in bytes of the aggregated values seen so far. diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactory.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactory.java index 2bde2f16f91f8..4ee0b0b5b13f8 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactory.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactory.java @@ -46,9 +46,9 @@ public static StarTreeNumericType getAggregatedValueType(MetricStat aggregationT switch (aggregationType) { // other metric types (count, min, max, avg) will be supported in the future case SUM: - return SumValueAggregator.STAR_TREE_NUMERIC_TYPE; + return SumValueAggregator.VALUE_AGGREGATOR_TYPE; case COUNT: - return CountValueAggregator.STAR_TREE_NUMERIC_TYPE; + return CountValueAggregator.VALUE_AGGREGATOR_TYPE; default: throw new IllegalStateException("Unsupported aggregation type: " + aggregationType); } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericType.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericType.java index 28bfd82c69fdf..f09fb1567cb26 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericType.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericType.java @@ -20,7 +20,10 @@ public enum StarTreeNumericType { HALF_FLOAT(IndexNumericFieldData.NumericType.HALF_FLOAT, StarTreeNumericTypeConverters::halfFloatPointToDouble), FLOAT(IndexNumericFieldData.NumericType.FLOAT, StarTreeNumericTypeConverters::floatPointToDouble), LONG(IndexNumericFieldData.NumericType.LONG, StarTreeNumericTypeConverters::longToDouble), - DOUBLE(IndexNumericFieldData.NumericType.DOUBLE, StarTreeNumericTypeConverters::sortableLongtoDouble); + DOUBLE(IndexNumericFieldData.NumericType.DOUBLE, StarTreeNumericTypeConverters::sortableLongtoDouble), + INT(IndexNumericFieldData.NumericType.INT, StarTreeNumericTypeConverters::intToDouble), + SHORT(IndexNumericFieldData.NumericType.SHORT, StarTreeNumericTypeConverters::shortToDouble), + UNSIGNED_LONG(IndexNumericFieldData.NumericType.UNSIGNED_LONG, StarTreeNumericTypeConverters::unsignedlongToDouble); final IndexNumericFieldData.NumericType numericType; final Function converter; @@ -44,6 +47,12 @@ public static StarTreeNumericType fromNumericType(IndexNumericFieldData.NumericT return StarTreeNumericType.LONG; case DOUBLE: return StarTreeNumericType.DOUBLE; + case INT: + return StarTreeNumericType.INT; + case SHORT: + return StarTreeNumericType.SHORT; + case UNSIGNED_LONG: + return StarTreeNumericType.UNSIGNED_LONG; default: throw new UnsupportedOperationException("Unknown numeric type [" + numericType + "]"); } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericTypeConverters.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericTypeConverters.java index f3fdd0d1162c3..b840d9436a26a 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericTypeConverters.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/numerictype/StarTreeNumericTypeConverters.java @@ -10,6 +10,7 @@ import org.apache.lucene.sandbox.document.HalfFloatPoint; import org.apache.lucene.util.NumericUtils; +import org.opensearch.common.Numbers; import org.opensearch.common.annotation.ExperimentalApi; /** @@ -31,7 +32,20 @@ public static double longToDouble(Long value) { return (double) value; } + public static double intToDouble(Long value) { + return (double) value; + } + + public static double shortToDouble(Long value) { + return (double) value; + } + public static Double sortableLongtoDouble(Long value) { return NumericUtils.sortableLongToDouble(value); } + + public static double unsignedlongToDouble(Long value) { + return Numbers.unsignedLongToDouble(value); + } + } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapSingleTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java similarity index 93% rename from server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapSingleTreeBuilder.java rename to server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java index 0289a56ce5b5f..7e00509a74ac0 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapSingleTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java @@ -7,7 +7,6 @@ */ package org.opensearch.index.compositeindex.datacube.startree.builder; -import org.apache.lucene.codecs.DocValuesConsumer; import org.apache.lucene.codecs.DocValuesProducer; import org.apache.lucene.index.BaseStarTreeBuilder; import org.apache.lucene.index.SegmentWriteState; @@ -28,28 +27,26 @@ * @opensearch.experimental */ @ExperimentalApi -public class OnHeapSingleTreeBuilder extends BaseStarTreeBuilder { +public class OnHeapStarTreeBuilder extends BaseStarTreeBuilder { private final List starTreeDocuments = new ArrayList<>(); /** - * Constructor for OnHeapSingleTreeBuilder + * Constructor for OnHeapStarTreeBuilder * * @param starTreeField star-tree field * @param fieldProducerMap helps with document values producer for a particular field - * @param docValuesConsumer document values consumer * @param segmentWriteState segment write state * @param mapperService helps with the numeric type of field * @throws IOException throws an exception we are unable to construct an onheap star-tree */ - public OnHeapSingleTreeBuilder( + public OnHeapStarTreeBuilder( StarTreeField starTreeField, Map fieldProducerMap, - DocValuesConsumer docValuesConsumer, SegmentWriteState segmentWriteState, MapperService mapperService ) throws IOException { - super(starTreeField, fieldProducerMap, docValuesConsumer, segmentWriteState, mapperService); + super(starTreeField, fieldProducerMap, segmentWriteState, mapperService); } @Override @@ -63,7 +60,7 @@ public StarTreeDocument getStarTreeDocument(int docId) throws IOException { } @Override - public List getStarTreeDocuments() throws IOException { + public List getStarTreeDocuments() { return starTreeDocuments; } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/SingleTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilder.java similarity index 87% rename from server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/SingleTreeBuilder.java rename to server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilder.java index 44866caba938b..ef542a1c848f2 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/SingleTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilder.java @@ -18,11 +18,11 @@ * @opensearch.experimental */ @ExperimentalApi -public interface SingleTreeBuilder extends Closeable { +public interface StarTreeBuilder extends Closeable { /** * Builds the star tree based on star-tree field * @throws IOException when we are unable to build star-tree */ - void build() throws Exception; + void build() throws IOException; } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocValuesIteratorAdapter.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocValuesIteratorAdapter.java index e4cf2a45e131b..7c035ed6f21be 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocValuesIteratorAdapter.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocValuesIteratorAdapter.java @@ -12,7 +12,6 @@ import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.SortedNumericDocValues; -import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.search.DocIdSetIterator; import org.opensearch.common.annotation.ExperimentalApi; @@ -31,8 +30,6 @@ public class StarTreeDocValuesIteratorAdapter { */ public DocIdSetIterator getDocValuesIterator(DocValuesType type, FieldInfo field, DocValuesProducer producer) throws IOException { switch (type) { - case SORTED_SET: - return producer.getSortedSet(field); case SORTED_NUMERIC: return producer.getSortedNumeric(field); default: @@ -41,12 +38,10 @@ public DocIdSetIterator getDocValuesIterator(DocValuesType type, FieldInfo field } /** - * Returns the next ordinal for the given iterator + * Returns the next value for the given iterator */ - public long getNextOrd(DocIdSetIterator iterator) throws IOException { - if (iterator instanceof SortedSetDocValues) { - return ((SortedSetDocValues) iterator).nextOrd(); - } else if (iterator instanceof SortedNumericDocValues) { + public long getNextValue(DocIdSetIterator iterator) throws IOException { + if (iterator instanceof SortedNumericDocValues) { return ((SortedNumericDocValues) iterator).nextValue(); } else { throw new IllegalArgumentException("Unsupported Iterator: " + iterator.toString()); @@ -54,7 +49,8 @@ public long getNextOrd(DocIdSetIterator iterator) throws IOException { } /** - * Returns the doc id for the next document from the given iterator + * Moves to the next doc in the iterator + * Returns the doc id for the next document from the given iterator */ public int nextDoc(DocIdSetIterator iterator) throws IOException { return iterator.nextDoc(); diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilder.java index 4d110609462a5..7b6d774698447 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilder.java @@ -10,7 +10,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.apache.lucene.codecs.DocValuesConsumer; import org.apache.lucene.codecs.DocValuesProducer; import org.apache.lucene.index.SegmentWriteState; import org.opensearch.common.annotation.ExperimentalApi; @@ -25,7 +24,7 @@ import java.util.Map; /** - * Builder to construct star tree based on multiple star tree configs + * Builder to construct star-trees based on multiple star-tree fields. * * @opensearch.experimental */ @@ -36,7 +35,6 @@ public class StarTreesBuilder implements Closeable { private final List starTreeFields; private final StarTreeFieldConfiguration.StarTreeBuildMode buildMode; - private final DocValuesConsumer docValuesConsumer; private final SegmentWriteState state; private final Map fieldProducerMap; private final MapperService mapperService; @@ -45,17 +43,15 @@ public StarTreesBuilder( List starTreeFields, StarTreeFieldConfiguration.StarTreeBuildMode buildMode, Map fieldProducerMap, - DocValuesConsumer docValuesConsumer, SegmentWriteState segmentWriteState, MapperService mapperService ) { this.starTreeFields = starTreeFields; if (starTreeFields == null || starTreeFields.isEmpty()) { - throw new IllegalArgumentException("Must provide star-tree builder configs"); + throw new IllegalArgumentException("Must provide star-tree field to build star trees"); } this.buildMode = buildMode; this.fieldProducerMap = fieldProducerMap; - this.docValuesConsumer = docValuesConsumer; this.state = segmentWriteState; this.mapperService = mapperService; } @@ -63,32 +59,22 @@ public StarTreesBuilder( /** * Builds the star-trees. */ - public void build() throws Exception { + public void build() throws IOException { long startTime = System.currentTimeMillis(); int numStarTrees = starTreeFields.size(); - logger.debug("Starting building {} star-trees with configs: {} using {} builder", numStarTrees, starTreeFields, buildMode); + logger.debug("Starting building {} star-trees with star-tree fields using {} builder", numStarTrees, buildMode); // Build all star-trees for (int i = 0; i < numStarTrees; i++) { StarTreeField starTreeField = starTreeFields.get(i); - try ( - SingleTreeBuilder singleTreeBuilder = getSingleTreeBuilder( - starTreeField, - buildMode, - fieldProducerMap, - docValuesConsumer, - state, - mapperService - ) - ) { - singleTreeBuilder.build(); + try (StarTreeBuilder starTreeBuilder = getSingleTreeBuilder(starTreeField, buildMode, fieldProducerMap, state, mapperService)) { + starTreeBuilder.build(); } } logger.debug( - "Took {} ms to building {} star-trees with configs: {} using {} builder", + "Took {} ms to building {} star-trees with star-tree fields using {} builder", System.currentTimeMillis() - startTime, numStarTrees, - starTreeFields, buildMode ); } @@ -98,17 +84,16 @@ public void close() throws IOException { } - private static SingleTreeBuilder getSingleTreeBuilder( + private static StarTreeBuilder getSingleTreeBuilder( StarTreeField starTreeField, StarTreeFieldConfiguration.StarTreeBuildMode buildMode, Map fieldProducerMap, - DocValuesConsumer docValuesConsumer, SegmentWriteState state, MapperService mapperService ) throws IOException { switch (buildMode) { case ON_HEAP: - return new OnHeapSingleTreeBuilder(starTreeField, fieldProducerMap, docValuesConsumer, state, mapperService); + return new OnHeapStarTreeBuilder(starTreeField, fieldProducerMap, state, mapperService); default: throw new IllegalArgumentException( String.format(Locale.ROOT, "No star tree implementation is available for [%s] build mode", buildMode) diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeBuilderUtils.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeBuilderUtils.java index 656196c7ffe0f..96e6681d40a76 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeBuilderUtils.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeBuilderUtils.java @@ -22,15 +22,50 @@ private StarTreeBuilderUtils() {} public static final int ALL = -1; - /** Tree node representation */ + /** + * Represents a node in a tree data structure, specifically designed for a star-tree implementation. + * A star-tree node will represent both star and non-star nodes. + */ public static class TreeNode { + + /** + * The dimension id for the dimension (field) associated with this star-tree node. + */ public int dimensionId = ALL; + + /** + * The starting document id (inclusive) associated with this star-tree node. + */ public int startDocId = ALL; + + /** + * The ending document id (exclusive) associated with this star-tree node. + */ public int endDocId = ALL; + + /** + * The aggregated document id associated with this star-tree node. + */ public int aggregatedDocId = ALL; + + /** + * The child dimension identifier associated with this star-tree node. + */ public int childDimensionId = ALL; + + /** + * The value of the dimension associated with this star-tree node. + */ public long dimensionValue = ALL; + + /** + * A flag indicating whether this node is a star node (a node that represents an aggregation of all dimensions). + */ public boolean isStarNode = false; + + /** + * A map containing the child nodes of this star-tree node, keyed by their dimension id. + */ public Map children; } diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregatorTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregatorTests.java new file mode 100644 index 0000000000000..e7d659ee177f3 --- /dev/null +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregatorTests.java @@ -0,0 +1,56 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube.startree.aggregators; + +import org.apache.lucene.util.NumericUtils; +import org.opensearch.index.compositeindex.datacube.MetricStat; +import org.opensearch.index.compositeindex.datacube.startree.aggregators.numerictype.StarTreeNumericType; +import org.opensearch.test.OpenSearchTestCase; + +public class CountValueAggregatorTests extends OpenSearchTestCase { + private final CountValueAggregator aggregator = new CountValueAggregator(); + + public void testGetAggregationType() { + assertEquals(MetricStat.COUNT.getTypeName(), aggregator.getAggregationType().getTypeName()); + } + + public void testGetAggregatedValueType() { + assertEquals(CountValueAggregator.VALUE_AGGREGATOR_TYPE, aggregator.getAggregatedValueType()); + } + + public void testGetInitialAggregatedValueForSegmentDocValue() { + assertEquals(1.0, aggregator.getInitialAggregatedValueForSegmentDocValue(randomLong(), StarTreeNumericType.LONG), 0.0); + } + + public void testMergeAggregatedValueAndSegmentValue() { + assertEquals(3.0, aggregator.mergeAggregatedValueAndSegmentValue(2.0, 3L, StarTreeNumericType.LONG), 0.0); + } + + public void testMergeAggregatedValues() { + assertEquals(5.0, aggregator.mergeAggregatedValues(2.0, 3.0), 0.0); + } + + public void testGetInitialAggregatedValue() { + assertEquals(3.0, aggregator.getInitialAggregatedValue(3.0), 0.0); + } + + public void testGetMaxAggregatedValueByteSize() { + assertEquals(Double.BYTES, aggregator.getMaxAggregatedValueByteSize()); + } + + public void testToLongValue() { + SumValueAggregator aggregator = new SumValueAggregator(); + assertEquals(NumericUtils.doubleToSortableLong(3.0), aggregator.toLongValue(3.0), 0.0); + } + + public void testToStarTreeNumericTypeValue() { + SumValueAggregator aggregator = new SumValueAggregator(); + assertEquals(NumericUtils.sortableLongToDouble(3L), aggregator.toStarTreeNumericTypeValue(3L, StarTreeNumericType.DOUBLE), 0.0); + } +} diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregationDescriptorTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregatorInfoTests.java similarity index 63% rename from server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregationDescriptorTests.java rename to server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregatorInfoTests.java index 98b79141df9eb..d08f637a3f0a9 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregationDescriptorTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/MetricAggregatorInfoTests.java @@ -12,12 +12,13 @@ import org.opensearch.index.fielddata.IndexNumericFieldData; import org.opensearch.test.OpenSearchTestCase; -public class MetricAggregationDescriptorTests extends OpenSearchTestCase { +public class MetricAggregatorInfoTests extends OpenSearchTestCase { public void testConstructor() { - MetricAggregationDescriptor pair = new MetricAggregationDescriptor( + MetricAggregatorInfo pair = new MetricAggregatorInfo( MetricStat.SUM, "column1", + "star_tree_field", IndexNumericFieldData.NumericType.DOUBLE, null ); @@ -26,57 +27,66 @@ public void testConstructor() { } public void testCountStarConstructor() { - MetricAggregationDescriptor pair = new MetricAggregationDescriptor( + MetricAggregatorInfo pair = new MetricAggregatorInfo( MetricStat.COUNT, "anything", + "star_tree_field", IndexNumericFieldData.NumericType.DOUBLE, null ); assertEquals(MetricStat.COUNT, pair.getMetricStat()); - assertEquals("*", pair.getField()); + assertEquals("anything", pair.getField()); } public void testToFieldName() { - MetricAggregationDescriptor pair = new MetricAggregationDescriptor( + MetricAggregatorInfo pair = new MetricAggregatorInfo( MetricStat.SUM, "column2", + "star_tree_field", IndexNumericFieldData.NumericType.DOUBLE, null ); - assertEquals("sum__column2", pair.toFieldName()); + assertEquals("star_tree_field_column2_sum", pair.toFieldName()); } public void testEquals() { - MetricAggregationDescriptor pair1 = new MetricAggregationDescriptor( + MetricAggregatorInfo pair1 = new MetricAggregatorInfo( MetricStat.SUM, "column1", + "star_tree_field", IndexNumericFieldData.NumericType.DOUBLE, null ); - MetricAggregationDescriptor pair2 = new MetricAggregationDescriptor( + MetricAggregatorInfo pair2 = new MetricAggregatorInfo( MetricStat.SUM, "column1", + "star_tree_field", IndexNumericFieldData.NumericType.DOUBLE, null ); assertEquals(pair1, pair2); assertNotEquals( pair1, - new MetricAggregationDescriptor(MetricStat.COUNT, "column1", IndexNumericFieldData.NumericType.DOUBLE, null) + new MetricAggregatorInfo(MetricStat.COUNT, "column1", "star_tree_field", IndexNumericFieldData.NumericType.DOUBLE, null) + ); + assertNotEquals( + pair1, + new MetricAggregatorInfo(MetricStat.SUM, "column2", "star_tree_field", IndexNumericFieldData.NumericType.DOUBLE, null) ); - assertNotEquals(pair1, new MetricAggregationDescriptor(MetricStat.SUM, "column2", IndexNumericFieldData.NumericType.DOUBLE, null)); } public void testHashCode() { - MetricAggregationDescriptor pair1 = new MetricAggregationDescriptor( + MetricAggregatorInfo pair1 = new MetricAggregatorInfo( MetricStat.SUM, "column1", + "star_tree_field", IndexNumericFieldData.NumericType.DOUBLE, null ); - MetricAggregationDescriptor pair2 = new MetricAggregationDescriptor( + MetricAggregatorInfo pair2 = new MetricAggregatorInfo( MetricStat.SUM, "column1", + "star_tree_field", IndexNumericFieldData.NumericType.DOUBLE, null ); @@ -84,21 +94,24 @@ public void testHashCode() { } public void testCompareTo() { - MetricAggregationDescriptor pair1 = new MetricAggregationDescriptor( + MetricAggregatorInfo pair1 = new MetricAggregatorInfo( MetricStat.SUM, "column1", + "star_tree_field", IndexNumericFieldData.NumericType.DOUBLE, null ); - MetricAggregationDescriptor pair2 = new MetricAggregationDescriptor( + MetricAggregatorInfo pair2 = new MetricAggregatorInfo( MetricStat.SUM, "column2", + "star_tree_field", IndexNumericFieldData.NumericType.DOUBLE, null ); - MetricAggregationDescriptor pair3 = new MetricAggregationDescriptor( + MetricAggregatorInfo pair3 = new MetricAggregatorInfo( MetricStat.COUNT, "column1", + "star_tree_field", IndexNumericFieldData.NumericType.DOUBLE, null ); diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregatorTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregatorTests.java index 287deeead9d9d..99af5a7b153ee 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregatorTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregatorTests.java @@ -21,26 +21,32 @@ public void testGetAggregationType() { assertEquals(MetricStat.SUM.getTypeName(), aggregator.getAggregationType().getTypeName()); } - public void testGetStarTreeNumericType() { - assertEquals(SumValueAggregator.STAR_TREE_NUMERIC_TYPE, aggregator.getStarTreeNumericType()); + public void testGetAggregatedValueType() { + assertEquals(SumValueAggregator.VALUE_AGGREGATOR_TYPE, aggregator.getAggregatedValueType()); } - public void testGetInitialAggregatedValue() { - assertEquals(1.0, aggregator.getInitialAggregatedValue(1L, StarTreeNumericType.LONG), 0.0); - assertThrows(NullPointerException.class, () -> aggregator.getInitialAggregatedValue(null, StarTreeNumericType.DOUBLE)); + public void testGetInitialAggregatedValueForSegmentDocValue() { + assertEquals(1.0, aggregator.getInitialAggregatedValueForSegmentDocValue(1L, StarTreeNumericType.LONG), 0.0); + assertThrows( + NullPointerException.class, + () -> aggregator.getInitialAggregatedValueForSegmentDocValue(null, StarTreeNumericType.DOUBLE) + ); } - public void testApplySegmentRawValue() { - assertEquals(5.0, aggregator.applySegmentRawValue(2.0, 3L, StarTreeNumericType.LONG), 0.0); - assertThrows(NullPointerException.class, () -> aggregator.applySegmentRawValue(3.14, null, StarTreeNumericType.DOUBLE)); + public void testMergeAggregatedValueAndSegmentValue() { + assertEquals(5.0, aggregator.mergeAggregatedValueAndSegmentValue(2.0, 3L, StarTreeNumericType.LONG), 0.0); + assertThrows( + NullPointerException.class, + () -> aggregator.mergeAggregatedValueAndSegmentValue(3.14, null, StarTreeNumericType.DOUBLE) + ); } - public void testApplyAggregatedValue() { - assertEquals(5.0, aggregator.applyAggregatedValue(2.0, 3.0), 0.0); + public void testMergeAggregatedValues() { + assertEquals(5.0, aggregator.mergeAggregatedValues(2.0, 3.0), 0.0); } - public void testGetAggregatedValue() { - assertEquals(3.14, aggregator.getAggregatedValue(3.14), 0.0); + public void testGetInitialAggregatedValue() { + assertEquals(3.14, aggregator.getInitialAggregatedValue(3.14), 0.0); } public void testGetMaxAggregatedValueByteSize() { diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactoryTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactoryTests.java index 21e2d1304f085..ce61ab839cc61 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactoryTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregatorFactoryTests.java @@ -20,8 +20,8 @@ public void testGetValueAggregatorForSumType() { assertEquals(SumValueAggregator.class, aggregator.getClass()); } - public void testGetStarTreeNumericTypeForSumType() { + public void testGetAggregatedValueTypeForSumType() { StarTreeNumericType starTreeNumericType = ValueAggregatorFactory.getAggregatedValueType(MetricStat.SUM); - assertEquals(SumValueAggregator.STAR_TREE_NUMERIC_TYPE, starTreeNumericType); + assertEquals(SumValueAggregator.VALUE_AGGREGATOR_TYPE, starTreeNumericType); } } diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java index 7a13ab7701912..eba396ffa175a 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java @@ -30,7 +30,7 @@ import org.opensearch.index.compositeindex.datacube.startree.StarTreeDocument; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; -import org.opensearch.index.compositeindex.datacube.startree.aggregators.MetricAggregationDescriptor; +import org.opensearch.index.compositeindex.datacube.startree.aggregators.MetricAggregatorInfo; import org.opensearch.index.fielddata.IndexNumericFieldData; import org.opensearch.index.mapper.ContentPath; import org.opensearch.index.mapper.DocumentMapper; @@ -75,6 +75,7 @@ public class BaseStarTreeBuilderTests extends OpenSearchTestCase { private static Directory directory; private static FieldInfo[] fieldsInfo; private static SegmentWriteState state; + private static StarTreeField starTreeField; @BeforeClass public static void setup() throws IOException { @@ -82,7 +83,7 @@ public static void setup() throws IOException { dimensionsOrder = List.of(new Dimension("field1"), new Dimension("field3"), new Dimension("field5"), new Dimension("field8")); metrics = List.of(new Metric("field2", List.of(MetricStat.SUM)), new Metric("field4", List.of(MetricStat.SUM))); - StarTreeField compositeField = new StarTreeField( + starTreeField = new StarTreeField( "test", dimensionsOrder, metrics, @@ -150,7 +151,7 @@ public static void setup() throws IOException { ); when(documentMapper.mappers()).thenReturn(fieldMappers); - builder = new BaseStarTreeBuilder(compositeField, fieldProducerMap, docValuesConsumer, state, mapperService) { + builder = new BaseStarTreeBuilder(starTreeField, fieldProducerMap, state, mapperService) { @Override public void appendStarTreeDocument(StarTreeDocument starTreeDocument) throws IOException {} @@ -160,7 +161,7 @@ public StarTreeDocument getStarTreeDocument(int docId) throws IOException { } @Override - public List getStarTreeDocuments() throws IOException { + public List getStarTreeDocuments() { return List.of(); } @@ -182,13 +183,13 @@ public Iterator generateStarTreeDocumentsForStarNode(int start }; } - public void test_generateMetricStatFieldPairs() throws IOException { - List metricAggregationDescriptors = builder.generateMetricStatFieldPairs(mapperService, state); - List expectedMetricAggregationDescriptors = List.of( - new MetricAggregationDescriptor(MetricStat.SUM, "field2", IndexNumericFieldData.NumericType.DOUBLE, null), - new MetricAggregationDescriptor(MetricStat.SUM, "field4", IndexNumericFieldData.NumericType.DOUBLE, null) + public void test_generateMetricAggregatorInfos() throws IOException { + List metricAggregatorInfos = builder.generateMetricAggregatorInfos(mapperService, state); + List expectedMetricAggregatorInfos = List.of( + new MetricAggregatorInfo(MetricStat.SUM, "field2", starTreeField.getName(), IndexNumericFieldData.NumericType.DOUBLE, null), + new MetricAggregatorInfo(MetricStat.SUM, "field4", starTreeField.getName(), IndexNumericFieldData.NumericType.DOUBLE, null) ); - assertEquals(metricAggregationDescriptors, expectedMetricAggregationDescriptors); + assertEquals(metricAggregatorInfos, expectedMetricAggregatorInfos); } public void test_reduceStarTreeDocuments() { diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapSingleTreeBuilderTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilderTests.java similarity index 79% rename from server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapSingleTreeBuilderTests.java rename to server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilderTests.java index 73955080c36ae..e6277dfa7ec5a 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapSingleTreeBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilderTests.java @@ -8,7 +8,6 @@ package org.opensearch.index.compositeindex.datacube.startree.builder; -import org.apache.lucene.codecs.DocValuesConsumer; import org.apache.lucene.codecs.DocValuesProducer; import org.apache.lucene.codecs.lucene99.Lucene99Codec; import org.apache.lucene.index.DocValuesType; @@ -53,9 +52,9 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -public class OnHeapSingleTreeBuilderTests extends OpenSearchTestCase { +public class OnHeapStarTreeBuilderTests extends OpenSearchTestCase { - private OnHeapSingleTreeBuilder builder; + private OnHeapStarTreeBuilder builder; private MapperService mapperService; private List dimensionsOrder; private List fields = List.of( @@ -75,15 +74,17 @@ public class OnHeapSingleTreeBuilderTests extends OpenSearchTestCase { private FieldInfo[] fieldsInfo; private StarTreeField compositeField; private Map fieldProducerMap; - private DocValuesConsumer docValuesConsumer; private SegmentWriteState writeState; @Before public void setup() throws IOException { dimensionsOrder = List.of(new Dimension("field1"), new Dimension("field3"), new Dimension("field5"), new Dimension("field8")); - metrics = List.of(new Metric("field2", List.of(MetricStat.SUM)), new Metric("field4", List.of(MetricStat.SUM))); + metrics = List.of( + new Metric("field2", List.of(MetricStat.SUM)), + new Metric("field4", List.of(MetricStat.SUM)), + new Metric("field6", List.of(MetricStat.COUNT)) + ); - docValuesConsumer = mock(DocValuesConsumer.class); DocValuesProducer docValuesProducer = mock(DocValuesProducer.class); compositeField = new StarTreeField( @@ -143,15 +144,17 @@ public void setup() throws IOException { .build(new Mapper.BuilderContext(settings, new ContentPath())); NumberFieldMapper numberFieldMapper2 = new NumberFieldMapper.Builder("field4", NumberFieldMapper.NumberType.DOUBLE, false, true) .build(new Mapper.BuilderContext(settings, new ContentPath())); + NumberFieldMapper numberFieldMapper3 = new NumberFieldMapper.Builder("field6", NumberFieldMapper.NumberType.DOUBLE, false, true) + .build(new Mapper.BuilderContext(settings, new ContentPath())); MappingLookup fieldMappers = new MappingLookup( - Set.of(numberFieldMapper1, numberFieldMapper2), + Set.of(numberFieldMapper1, numberFieldMapper2, numberFieldMapper3), Collections.emptyList(), Collections.emptyList(), 0, null ); when(documentMapper.mappers()).thenReturn(fieldMappers); - builder = new OnHeapSingleTreeBuilder(compositeField, fieldProducerMap, docValuesConsumer, writeState, mapperService); + builder = new OnHeapStarTreeBuilder(compositeField, fieldProducerMap, writeState, mapperService); } public void test_sortMergeAndAggregateStarTreeDocument() throws IOException { @@ -159,15 +162,15 @@ public void test_sortMergeAndAggregateStarTreeDocument() throws IOException { int noOfStarTreeDocuments = 5; StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; - starTreeDocuments[0] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 12.0, 10.0 }); - starTreeDocuments[1] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 10.0, 6.0 }); - starTreeDocuments[2] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 14.0, 12.0 }); - starTreeDocuments[3] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 9.0, 4.0 }); - starTreeDocuments[4] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 11.0, 16.0 }); + starTreeDocuments[0] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 12.0, 10.0, randomDouble() }); + starTreeDocuments[1] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 10.0, 6.0, randomDouble() }); + starTreeDocuments[2] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 14.0, 12.0, randomDouble() }); + starTreeDocuments[3] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 9.0, 4.0, randomDouble() }); + starTreeDocuments[4] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 11.0, 16.0, randomDouble() }); List inorderStarTreeDocuments = List.of( - new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 21.0, 14.0 }), - new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 35.0, 34.0 }) + new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 21.0, 14.0, 2.0 }), + new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 35.0, 34.0, 3.0 }) ); Iterator expectedStarTreeDocumentIterator = inorderStarTreeDocuments.iterator(); @@ -175,7 +178,8 @@ public void test_sortMergeAndAggregateStarTreeDocument() throws IOException { for (int i = 0; i < noOfStarTreeDocuments; i++) { long metric1 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[0]); long metric2 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[1]); - segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2 }); + long metric3 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[2]); + segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2, metric3 }); } Iterator segmentStarTreeDocumentIterator = builder.sortMergeAndAggregateStarTreeDocument( @@ -192,6 +196,7 @@ public void test_sortMergeAndAggregateStarTreeDocument() throws IOException { assertEquals(expectedStarTreeDocument.dimensions[3], resultStarTreeDocument.dimensions[3]); assertEquals(expectedStarTreeDocument.metrics[0], resultStarTreeDocument.metrics[0]); assertEquals(expectedStarTreeDocument.metrics[1], resultStarTreeDocument.metrics[1]); + assertEquals(expectedStarTreeDocument.metrics[2], resultStarTreeDocument.metrics[2]); numOfAggregatedDocuments++; } @@ -205,12 +210,12 @@ public void test_sortMergeAndAggregateStarTreeDocument_nullMetric() throws IOExc int noOfStarTreeDocuments = 5; StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; - starTreeDocuments[0] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 12.0, 10.0 }); - starTreeDocuments[1] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 10.0, 6.0 }); - starTreeDocuments[2] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 14.0, 12.0 }); - starTreeDocuments[3] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 9.0, 4.0 }); - starTreeDocuments[4] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 11.0, null }); - StarTreeDocument expectedStarTreeDocument = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 21.0, 14.0 }); + starTreeDocuments[0] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 12.0, 10.0, randomDouble() }); + starTreeDocuments[1] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 10.0, 6.0, randomDouble() }); + starTreeDocuments[2] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 14.0, 12.0, randomDouble() }); + starTreeDocuments[3] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 9.0, 4.0, randomDouble() }); + starTreeDocuments[4] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 11.0, null, randomDouble() }); + StarTreeDocument expectedStarTreeDocument = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 21.0, 14.0, 2.0 }); StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; for (int i = 0; i < noOfStarTreeDocuments; i++) { @@ -218,7 +223,8 @@ public void test_sortMergeAndAggregateStarTreeDocument_nullMetric() throws IOExc Long metric2 = starTreeDocuments[i].metrics[1] != null ? NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[1]) : null; - segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Object[] { metric1, metric2 }); + Long metric3 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[2]); + segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Object[] { metric1, metric2, metric3 }); } Iterator segmentStarTreeDocumentIterator = builder.sortMergeAndAggregateStarTreeDocument( @@ -246,15 +252,15 @@ public void test_sortMergeAndAggregateStarTreeDocument_longMaxAndLongMinDimensio int noOfStarTreeDocuments = 5; StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; - starTreeDocuments[0] = new StarTreeDocument(new long[] { Long.MIN_VALUE, 4, 3, 4 }, new Double[] { 12.0, 10.0 }); - starTreeDocuments[1] = new StarTreeDocument(new long[] { 3, 4, 2, Long.MAX_VALUE }, new Double[] { 10.0, 6.0 }); - starTreeDocuments[2] = new StarTreeDocument(new long[] { 3, 4, 2, Long.MAX_VALUE }, new Double[] { 14.0, 12.0 }); - starTreeDocuments[3] = new StarTreeDocument(new long[] { Long.MIN_VALUE, 4, 3, 4 }, new Double[] { 9.0, 4.0 }); - starTreeDocuments[4] = new StarTreeDocument(new long[] { 3, 4, 2, Long.MAX_VALUE }, new Double[] { 11.0, 16.0 }); + starTreeDocuments[0] = new StarTreeDocument(new long[] { Long.MIN_VALUE, 4, 3, 4 }, new Double[] { 12.0, 10.0, randomDouble() }); + starTreeDocuments[1] = new StarTreeDocument(new long[] { 3, 4, 2, Long.MAX_VALUE }, new Double[] { 10.0, 6.0, randomDouble() }); + starTreeDocuments[2] = new StarTreeDocument(new long[] { 3, 4, 2, Long.MAX_VALUE }, new Double[] { 14.0, 12.0, randomDouble() }); + starTreeDocuments[3] = new StarTreeDocument(new long[] { Long.MIN_VALUE, 4, 3, 4 }, new Double[] { 9.0, 4.0, randomDouble() }); + starTreeDocuments[4] = new StarTreeDocument(new long[] { 3, 4, 2, Long.MAX_VALUE }, new Double[] { 11.0, 16.0, randomDouble() }); List inorderStarTreeDocuments = List.of( - new StarTreeDocument(new long[] { Long.MIN_VALUE, 4, 3, 4 }, new Double[] { 21.0, 14.0 }), - new StarTreeDocument(new long[] { 3, 4, 2, Long.MAX_VALUE }, new Double[] { 35.0, 34.0 }) + new StarTreeDocument(new long[] { Long.MIN_VALUE, 4, 3, 4 }, new Double[] { 21.0, 14.0, 2.0 }), + new StarTreeDocument(new long[] { 3, 4, 2, Long.MAX_VALUE }, new Double[] { 35.0, 34.0, 3.0 }) ); Iterator expectedStarTreeDocumentIterator = inorderStarTreeDocuments.iterator(); @@ -262,7 +268,8 @@ public void test_sortMergeAndAggregateStarTreeDocument_longMaxAndLongMinDimensio for (int i = 0; i < noOfStarTreeDocuments; i++) { long metric1 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[0]); long metric2 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[1]); - segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2 }); + long metric3 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[2]); + segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2, metric3 }); } Iterator segmentStarTreeDocumentIterator = builder.sortMergeAndAggregateStarTreeDocument( @@ -279,6 +286,7 @@ public void test_sortMergeAndAggregateStarTreeDocument_longMaxAndLongMinDimensio assertEquals(expectedStarTreeDocument.dimensions[3], resultStarTreeDocument.dimensions[3]); assertEquals(expectedStarTreeDocument.metrics[0], resultStarTreeDocument.metrics[0]); assertEquals(expectedStarTreeDocument.metrics[1], resultStarTreeDocument.metrics[1]); + assertEquals(expectedStarTreeDocument.metrics[2], resultStarTreeDocument.metrics[2]); numOfAggregatedDocuments++; } @@ -292,15 +300,15 @@ public void test_build_DoubleMaxAndDoubleMinMetrics() throws IOException { int noOfStarTreeDocuments = 5; StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; - starTreeDocuments[0] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { Double.MAX_VALUE, 10.0 }); - starTreeDocuments[1] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 10.0, 6.0 }); - starTreeDocuments[2] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 14.0, Double.MIN_VALUE }); - starTreeDocuments[3] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 9.0, 4.0 }); - starTreeDocuments[4] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 11.0, 16.0 }); + starTreeDocuments[0] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { Double.MAX_VALUE, 10.0, randomDouble() }); + starTreeDocuments[1] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 10.0, 6.0, randomDouble() }); + starTreeDocuments[2] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 14.0, Double.MIN_VALUE, randomDouble() }); + starTreeDocuments[3] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 9.0, 4.0, randomDouble() }); + starTreeDocuments[4] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 11.0, 16.0, randomDouble() }); List inorderStarTreeDocuments = List.of( - new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { Double.MAX_VALUE + 9, 14.0 }), - new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 35.0, Double.MIN_VALUE + 22 }) + new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { Double.MAX_VALUE + 9, 14.0, 2.0 }), + new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 35.0, Double.MIN_VALUE + 22, 3.0 }) ); Iterator expectedStarTreeDocumentIterator = inorderStarTreeDocuments.iterator(); @@ -308,7 +316,8 @@ public void test_build_DoubleMaxAndDoubleMinMetrics() throws IOException { for (int i = 0; i < noOfStarTreeDocuments; i++) { long metric1 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[0]); long metric2 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[1]); - segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2 }); + long metric3 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[2]); + segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2, metric3 }); } Iterator segmentStarTreeDocumentIterator = builder.sortMergeAndAggregateStarTreeDocument( @@ -325,6 +334,7 @@ public void test_build_DoubleMaxAndDoubleMinMetrics() throws IOException { assertEquals(expectedStarTreeDocument.dimensions[3], resultStarTreeDocument.dimensions[3]); assertEquals(expectedStarTreeDocument.metrics[0], resultStarTreeDocument.metrics[0]); assertEquals(expectedStarTreeDocument.metrics[1], resultStarTreeDocument.metrics[1]); + assertEquals(expectedStarTreeDocument.metrics[2], resultStarTreeDocument.metrics[2]); numOfAggregatedDocuments++; } @@ -343,38 +353,40 @@ public void test_build_halfFloatMetrics() throws IOException { .build(new Mapper.BuilderContext(settings, new ContentPath())); NumberFieldMapper numberFieldMapper2 = new NumberFieldMapper.Builder("field4", NumberFieldMapper.NumberType.HALF_FLOAT, false, true) .build(new Mapper.BuilderContext(settings, new ContentPath())); + NumberFieldMapper numberFieldMapper3 = new NumberFieldMapper.Builder("field6", NumberFieldMapper.NumberType.HALF_FLOAT, false, true) + .build(new Mapper.BuilderContext(settings, new ContentPath())); MappingLookup fieldMappers = new MappingLookup( - Set.of(numberFieldMapper1, numberFieldMapper2), + Set.of(numberFieldMapper1, numberFieldMapper2, numberFieldMapper3), Collections.emptyList(), Collections.emptyList(), 0, null ); when(documentMapper.mappers()).thenReturn(fieldMappers); - builder = new OnHeapSingleTreeBuilder(compositeField, fieldProducerMap, docValuesConsumer, writeState, mapperService); + builder = new OnHeapStarTreeBuilder(compositeField, fieldProducerMap, writeState, mapperService); int noOfStarTreeDocuments = 5; StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; starTreeDocuments[0] = new StarTreeDocument( new long[] { 2, 4, 3, 4 }, - new HalfFloatPoint[] { new HalfFloatPoint("hf1", 12), new HalfFloatPoint("hf6", 10) } + new HalfFloatPoint[] { new HalfFloatPoint("hf1", 12), new HalfFloatPoint("hf6", 10), new HalfFloatPoint("field6", 10) } ); starTreeDocuments[1] = new StarTreeDocument( new long[] { 3, 4, 2, 1 }, - new HalfFloatPoint[] { new HalfFloatPoint("hf2", 10), new HalfFloatPoint("hf7", 6) } + new HalfFloatPoint[] { new HalfFloatPoint("hf2", 10), new HalfFloatPoint("hf7", 6), new HalfFloatPoint("field6", 10) } ); starTreeDocuments[2] = new StarTreeDocument( new long[] { 3, 4, 2, 1 }, - new HalfFloatPoint[] { new HalfFloatPoint("hf3", 14), new HalfFloatPoint("hf8", 12) } + new HalfFloatPoint[] { new HalfFloatPoint("hf3", 14), new HalfFloatPoint("hf8", 12), new HalfFloatPoint("field6", 10) } ); starTreeDocuments[3] = new StarTreeDocument( new long[] { 2, 4, 3, 4 }, - new HalfFloatPoint[] { new HalfFloatPoint("hf4", 9), new HalfFloatPoint("hf9", 4) } + new HalfFloatPoint[] { new HalfFloatPoint("hf4", 9), new HalfFloatPoint("hf9", 4), new HalfFloatPoint("field6", 10) } ); starTreeDocuments[4] = new StarTreeDocument( new long[] { 3, 4, 2, 1 }, - new HalfFloatPoint[] { new HalfFloatPoint("hf5", 11), new HalfFloatPoint("hf10", 16) } + new HalfFloatPoint[] { new HalfFloatPoint("hf5", 11), new HalfFloatPoint("hf10", 16), new HalfFloatPoint("field6", 10) } ); StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; @@ -385,7 +397,10 @@ public void test_build_halfFloatMetrics() throws IOException { long metric2 = HalfFloatPoint.halfFloatToSortableShort( ((HalfFloatPoint) starTreeDocuments[i].metrics[1]).numericValue().floatValue() ); - segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2 }); + long metric3 = HalfFloatPoint.halfFloatToSortableShort( + ((HalfFloatPoint) starTreeDocuments[i].metrics[2]).numericValue().floatValue() + ); + segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2, metric3 }); } Iterator segmentStarTreeDocumentIterator = builder.sortMergeAndAggregateStarTreeDocument( @@ -410,30 +425,33 @@ public void test_build_floatMetrics() throws IOException { .build(new Mapper.BuilderContext(settings, new ContentPath())); NumberFieldMapper numberFieldMapper2 = new NumberFieldMapper.Builder("field4", NumberFieldMapper.NumberType.FLOAT, false, true) .build(new Mapper.BuilderContext(settings, new ContentPath())); + NumberFieldMapper numberFieldMapper3 = new NumberFieldMapper.Builder("field6", NumberFieldMapper.NumberType.FLOAT, false, true) + .build(new Mapper.BuilderContext(settings, new ContentPath())); MappingLookup fieldMappers = new MappingLookup( - Set.of(numberFieldMapper1, numberFieldMapper2), + Set.of(numberFieldMapper1, numberFieldMapper2, numberFieldMapper3), Collections.emptyList(), Collections.emptyList(), 0, null ); when(documentMapper.mappers()).thenReturn(fieldMappers); - builder = new OnHeapSingleTreeBuilder(compositeField, fieldProducerMap, docValuesConsumer, writeState, mapperService); + builder = new OnHeapStarTreeBuilder(compositeField, fieldProducerMap, writeState, mapperService); int noOfStarTreeDocuments = 5; StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; - starTreeDocuments[0] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Float[] { 12.0F, 10.0F }); - starTreeDocuments[1] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Float[] { 10.0F, 6.0F }); - starTreeDocuments[2] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Float[] { 14.0F, 12.0F }); - starTreeDocuments[3] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Float[] { 9.0F, 4.0F }); - starTreeDocuments[4] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Float[] { 11.0F, 16.0F }); + starTreeDocuments[0] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Float[] { 12.0F, 10.0F, randomFloat() }); + starTreeDocuments[1] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Float[] { 10.0F, 6.0F, randomFloat() }); + starTreeDocuments[2] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Float[] { 14.0F, 12.0F, randomFloat() }); + starTreeDocuments[3] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Float[] { 9.0F, 4.0F, randomFloat() }); + starTreeDocuments[4] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Float[] { 11.0F, 16.0F, randomFloat() }); StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; for (int i = 0; i < noOfStarTreeDocuments; i++) { long metric1 = NumericUtils.floatToSortableInt((Float) starTreeDocuments[i].metrics[0]); long metric2 = NumericUtils.floatToSortableInt((Float) starTreeDocuments[i].metrics[1]); - segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2 }); + long metric3 = NumericUtils.floatToSortableInt((Float) starTreeDocuments[i].metrics[2]); + segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2, metric3 }); } Iterator segmentStarTreeDocumentIterator = builder.sortMergeAndAggregateStarTreeDocument( @@ -458,30 +476,33 @@ public void test_build_longMetrics() throws IOException { .build(new Mapper.BuilderContext(settings, new ContentPath())); NumberFieldMapper numberFieldMapper2 = new NumberFieldMapper.Builder("field4", NumberFieldMapper.NumberType.LONG, false, true) .build(new Mapper.BuilderContext(settings, new ContentPath())); + NumberFieldMapper numberFieldMapper3 = new NumberFieldMapper.Builder("field6", NumberFieldMapper.NumberType.LONG, false, true) + .build(new Mapper.BuilderContext(settings, new ContentPath())); MappingLookup fieldMappers = new MappingLookup( - Set.of(numberFieldMapper1, numberFieldMapper2), + Set.of(numberFieldMapper1, numberFieldMapper2, numberFieldMapper3), Collections.emptyList(), Collections.emptyList(), 0, null ); when(documentMapper.mappers()).thenReturn(fieldMappers); - builder = new OnHeapSingleTreeBuilder(compositeField, fieldProducerMap, docValuesConsumer, writeState, mapperService); + builder = new OnHeapStarTreeBuilder(compositeField, fieldProducerMap, writeState, mapperService); int noOfStarTreeDocuments = 5; StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; - starTreeDocuments[0] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Long[] { 12L, 10L }); - starTreeDocuments[1] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Long[] { 10L, 6L }); - starTreeDocuments[2] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Long[] { 14L, 12L }); - starTreeDocuments[3] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Long[] { 9L, 4L }); - starTreeDocuments[4] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Long[] { 11L, 16L }); + starTreeDocuments[0] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Long[] { 12L, 10L, randomLong() }); + starTreeDocuments[1] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Long[] { 10L, 6L, randomLong() }); + starTreeDocuments[2] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Long[] { 14L, 12L, randomLong() }); + starTreeDocuments[3] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Long[] { 9L, 4L, randomLong() }); + starTreeDocuments[4] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Long[] { 11L, 16L, randomLong() }); StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; for (int i = 0; i < noOfStarTreeDocuments; i++) { long metric1 = (Long) starTreeDocuments[i].metrics[0]; long metric2 = (Long) starTreeDocuments[i].metrics[1]; - segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2 }); + long metric3 = (Long) starTreeDocuments[i].metrics[2]; + segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2, metric3 }); } Iterator segmentStarTreeDocumentIterator = builder.sortMergeAndAggregateStarTreeDocument( @@ -498,14 +519,14 @@ public void test_build_longMetrics() throws IOException { private static Iterator getExpectedStarTreeDocumentIterator() { List expectedStarTreeDocuments = List.of( - new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 21.0, 14.0 }), - new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 35.0, 34.0 }), - new StarTreeDocument(new long[] { -1, 4, 2, 1 }, new Double[] { 35.0, 34.0 }), - new StarTreeDocument(new long[] { -1, 4, 3, 4 }, new Double[] { 21.0, 14.0 }), - new StarTreeDocument(new long[] { -1, 4, -1, 1 }, new Double[] { 35.0, 34.0 }), - new StarTreeDocument(new long[] { -1, 4, -1, 4 }, new Double[] { 21.0, 14.0 }), - new StarTreeDocument(new long[] { -1, 4, -1, -1 }, new Double[] { 56.0, 48.0 }), - new StarTreeDocument(new long[] { -1, -1, -1, -1 }, new Double[] { 56.0, 48.0 }) + new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 21.0, 14.0, 2.0 }), + new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 35.0, 34.0, 3.0 }), + new StarTreeDocument(new long[] { -1, 4, 2, 1 }, new Double[] { 35.0, 34.0, 3.0 }), + new StarTreeDocument(new long[] { -1, 4, 3, 4 }, new Double[] { 21.0, 14.0, 2.0 }), + new StarTreeDocument(new long[] { -1, 4, -1, 1 }, new Double[] { 35.0, 34.0, 3.0 }), + new StarTreeDocument(new long[] { -1, 4, -1, 4 }, new Double[] { 21.0, 14.0, 2.0 }), + new StarTreeDocument(new long[] { -1, 4, -1, -1 }, new Double[] { 56.0, 48.0, 5.0 }), + new StarTreeDocument(new long[] { -1, -1, -1, -1 }, new Double[] { 56.0, 48.0, 5.0 }) ); return expectedStarTreeDocuments.iterator(); } @@ -515,17 +536,18 @@ public void test_build() throws IOException { int noOfStarTreeDocuments = 5; StarTreeDocument[] starTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; - starTreeDocuments[0] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 12.0, 10.0 }); - starTreeDocuments[1] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 10.0, 6.0 }); - starTreeDocuments[2] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 14.0, 12.0 }); - starTreeDocuments[3] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 9.0, 4.0 }); - starTreeDocuments[4] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 11.0, 16.0 }); + starTreeDocuments[0] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 12.0, 10.0, randomDouble() }); + starTreeDocuments[1] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 10.0, 6.0, randomDouble() }); + starTreeDocuments[2] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 14.0, 12.0, randomDouble() }); + starTreeDocuments[3] = new StarTreeDocument(new long[] { 2, 4, 3, 4 }, new Double[] { 9.0, 4.0, randomDouble() }); + starTreeDocuments[4] = new StarTreeDocument(new long[] { 3, 4, 2, 1 }, new Double[] { 11.0, 16.0, randomDouble() }); StarTreeDocument[] segmentStarTreeDocuments = new StarTreeDocument[noOfStarTreeDocuments]; for (int i = 0; i < noOfStarTreeDocuments; i++) { long metric1 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[0]); long metric2 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[1]); - segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2 }); + long metric3 = NumericUtils.doubleToSortableLong((Double) starTreeDocuments[i].metrics[2]); + segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, metric2, metric3 }); } Iterator segmentStarTreeDocumentIterator = builder.sortMergeAndAggregateStarTreeDocument( @@ -555,6 +577,7 @@ private static void assertStarTreeDocuments( assertEquals(expectedStarTreeDocument.dimensions[3], resultStarTreeDocument.dimensions[3]); assertEquals(expectedStarTreeDocument.metrics[0], resultStarTreeDocument.metrics[0]); assertEquals(expectedStarTreeDocument.metrics[1], resultStarTreeDocument.metrics[1]); + assertEquals(expectedStarTreeDocument.metrics[2], resultStarTreeDocument.metrics[2]); } } diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeValuesIteratorFactoryTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeValuesIteratorFactoryTests.java index f5d53afb8f774..59cadfa737786 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeValuesIteratorFactoryTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeValuesIteratorFactoryTests.java @@ -13,7 +13,6 @@ import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.SortedNumericDocValues; -import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.index.VectorEncoding; import org.apache.lucene.index.VectorSimilarityFunction; import org.apache.lucene.search.DocIdSetIterator; @@ -56,14 +55,6 @@ public static void setup() { ); } - public void testCreateIterator_SortedSet() throws IOException { - DocValuesProducer producer = Mockito.mock(DocValuesProducer.class); - SortedSetDocValues iterator = Mockito.mock(SortedSetDocValues.class); - when(producer.getSortedSet(mockFieldInfo)).thenReturn(iterator); - DocIdSetIterator result = factory.getDocValuesIterator(DocValuesType.SORTED_SET, mockFieldInfo, producer); - assertEquals(iterator.getClass(), result.getClass()); - } - public void testCreateIterator_SortedNumeric() throws IOException { DocValuesProducer producer = Mockito.mock(DocValuesProducer.class); SortedNumericDocValues iterator = Mockito.mock(SortedNumericDocValues.class); @@ -80,26 +71,18 @@ public void testCreateIterator_UnsupportedType() { assertEquals("Unsupported DocValuesType: BINARY", exception.getMessage()); } - public void testGetNextValue_SortedSet() throws IOException { - SortedSetDocValues iterator = Mockito.mock(SortedSetDocValues.class); - when(iterator.nextOrd()).thenReturn(42L); - - long result = factory.getNextOrd(iterator); - assertEquals(42L, result); - } - public void testGetNextValue_SortedNumeric() throws IOException { SortedNumericDocValues iterator = Mockito.mock(SortedNumericDocValues.class); when(iterator.nextValue()).thenReturn(123L); - long result = factory.getNextOrd(iterator); + long result = factory.getNextValue(iterator); assertEquals(123L, result); } public void testGetNextValue_UnsupportedIterator() { DocIdSetIterator iterator = Mockito.mock(DocIdSetIterator.class); - IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> { factory.getNextOrd(iterator); }); + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> { factory.getNextValue(iterator); }); assertEquals("Unsupported Iterator: " + iterator.toString(), exception.getMessage()); }