-
Notifications
You must be signed in to change notification settings - Fork 135
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Adds unit tests that map to each codec component. Did not add tests for merging and codec utils. This can be undertaken later. Adds a utils folder for sharing testing functionality between codec tests. Cleans up a few minor details around codec source code. Signed-off-by: John Mazanec <[email protected]>
- Loading branch information
1 parent
1cc9f24
commit c5ba1dc
Showing
13 changed files
with
1,112 additions
and
119 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
69 changes: 69 additions & 0 deletions
69
src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80BinaryDocValuesTests.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,69 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package org.opensearch.knn.index.codec.KNN80Codec; | ||
|
|
||
| import com.google.common.collect.ImmutableList; | ||
| import org.apache.lucene.index.BinaryDocValues; | ||
| import org.apache.lucene.index.DocIDMerger; | ||
| import org.apache.lucene.index.MergeState; | ||
| import org.opensearch.knn.KNNTestCase; | ||
| import org.opensearch.knn.index.codec.KNNCodecTestUtil; | ||
| import org.opensearch.knn.index.codec.util.BinaryDocValuesSub; | ||
|
|
||
| import java.io.IOException; | ||
|
|
||
| public class KNN80BinaryDocValuesTests extends KNNTestCase { | ||
|
|
||
| public void testDocId() { | ||
| KNN80BinaryDocValues knn80BinaryDocValues = new KNN80BinaryDocValues(null); | ||
| assertEquals(-1, knn80BinaryDocValues.docID()); | ||
| } | ||
|
|
||
| public void testNextDoc() throws IOException { | ||
| final int expectedDoc = 12; | ||
|
|
||
| BinaryDocValuesSub sub = new BinaryDocValuesSub(new MergeState.DocMap() { | ||
| @Override | ||
| public int get(int docID) { | ||
| return expectedDoc; | ||
| } | ||
| }, new KNNCodecTestUtil.ConstantVectorBinaryDocValues(10, 128, 1.0f)); | ||
|
|
||
| DocIDMerger<BinaryDocValuesSub> docIDMerger = DocIDMerger.of(ImmutableList.of(sub), false); | ||
| KNN80BinaryDocValues knn80BinaryDocValues = new KNN80BinaryDocValues(docIDMerger); | ||
| assertEquals(expectedDoc, knn80BinaryDocValues.nextDoc()); | ||
| } | ||
|
|
||
| public void testAdvance() { | ||
| KNN80BinaryDocValues knn80BinaryDocValues = new KNN80BinaryDocValues(null); | ||
| expectThrows(UnsupportedOperationException.class, () -> knn80BinaryDocValues.advance(0)); | ||
| } | ||
|
|
||
| public void testAdvanceExact() { | ||
| KNN80BinaryDocValues knn80BinaryDocValues = new KNN80BinaryDocValues(null); | ||
| expectThrows(UnsupportedOperationException.class, () -> knn80BinaryDocValues.advanceExact(0)); | ||
| } | ||
|
|
||
| public void testCost() { | ||
| KNN80BinaryDocValues knn80BinaryDocValues = new KNN80BinaryDocValues(null); | ||
| expectThrows(UnsupportedOperationException.class, knn80BinaryDocValues::cost); | ||
| } | ||
|
|
||
| public void testBinaryValue() throws IOException { | ||
| BinaryDocValues binaryDocValues = new KNNCodecTestUtil.ConstantVectorBinaryDocValues(10, 128, 1.0f); | ||
| BinaryDocValuesSub sub = new BinaryDocValuesSub(new MergeState.DocMap() { | ||
| @Override | ||
| public int get(int docID) { | ||
| return docID; | ||
| } | ||
| }, binaryDocValues); | ||
|
|
||
| DocIDMerger<BinaryDocValuesSub> docIDMerger = DocIDMerger.of(ImmutableList.of(sub), false); | ||
| KNN80BinaryDocValues knn80BinaryDocValues = new KNN80BinaryDocValues(docIDMerger); | ||
| knn80BinaryDocValues.nextDoc(); | ||
| assertEquals(binaryDocValues.binaryValue(), knn80BinaryDocValues.binaryValue()); | ||
| } | ||
| } |
96 changes: 96 additions & 0 deletions
96
src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80CompoundFormatTests.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,96 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package org.opensearch.knn.index.codec.KNN80Codec; | ||
|
|
||
| import org.apache.lucene.codecs.Codec; | ||
| import org.apache.lucene.codecs.CompoundDirectory; | ||
| import org.apache.lucene.codecs.CompoundFormat; | ||
| import org.apache.lucene.index.SegmentInfo; | ||
| import org.apache.lucene.store.Directory; | ||
| import org.apache.lucene.store.IOContext; | ||
| import org.apache.lucene.store.IndexOutput; | ||
| import org.junit.AfterClass; | ||
| import org.junit.BeforeClass; | ||
| import org.opensearch.common.util.set.Sets; | ||
| import org.opensearch.knn.KNNTestCase; | ||
| import org.opensearch.knn.index.codec.KNN87Codec.KNN87Codec; | ||
| import org.opensearch.knn.index.codec.KNNCodecTestUtil; | ||
| import org.opensearch.knn.index.util.KNNEngine; | ||
|
|
||
| import java.io.IOException; | ||
| import java.util.Arrays; | ||
| import java.util.Set; | ||
|
|
||
| import static org.mockito.Mockito.doNothing; | ||
| import static org.mockito.Mockito.mock; | ||
| import static org.mockito.Mockito.when; | ||
|
|
||
| public class KNN80CompoundFormatTests extends KNNTestCase { | ||
|
|
||
|
|
||
| private static Directory directory; | ||
| private static Codec codec; | ||
|
|
||
| @BeforeClass | ||
| public static void setStaticVariables() { | ||
| directory = newFSDirectory(createTempDir()); | ||
| codec = new KNN87Codec(); | ||
| } | ||
|
|
||
| @AfterClass | ||
| public static void closeStaticVariables() throws IOException { | ||
| directory.close(); | ||
| } | ||
|
|
||
|
|
||
| public void testGetCompoundReader() throws IOException { | ||
| CompoundDirectory dir = mock(CompoundDirectory.class); | ||
| CompoundFormat delegate = mock(CompoundFormat.class); | ||
| when(delegate.getCompoundReader(null, null, null)).thenReturn(dir); | ||
| KNN80CompoundFormat knn80CompoundFormat = new KNN80CompoundFormat(delegate); | ||
| assertEquals(dir, knn80CompoundFormat.getCompoundReader(null, null, null)); | ||
| } | ||
|
|
||
| public void testWrite() throws IOException { | ||
| // Check that all normal engine files correctly get set to compound extension files after write | ||
| String segmentName = "_test"; | ||
|
|
||
| Set<String> segmentFiles = Sets.newHashSet( | ||
| String.format("%s_nmslib1%s", segmentName, KNNEngine.NMSLIB.getExtension()), | ||
| String.format("%s_nmslib2%s", segmentName, KNNEngine.NMSLIB.getExtension()), | ||
| String.format("%s_nmslib3%s", segmentName, KNNEngine.NMSLIB.getExtension()), | ||
| String.format("%s_faiss1%s", segmentName, KNNEngine.FAISS.getExtension()), | ||
| String.format("%s_faiss2%s", segmentName, KNNEngine.FAISS.getExtension()), | ||
| String.format("%s_faiss3%s", segmentName, KNNEngine.FAISS.getExtension()) | ||
| ); | ||
|
|
||
| SegmentInfo segmentInfo = KNNCodecTestUtil.SegmentInfoBuilder.builder(directory, segmentName, | ||
| segmentFiles.size(), codec).build(); | ||
|
|
||
| for (String name : segmentFiles) { | ||
| IndexOutput indexOutput = directory.createOutput(name, IOContext.DEFAULT); | ||
| indexOutput.close(); | ||
| } | ||
| segmentInfo.setFiles(segmentFiles); | ||
|
|
||
| CompoundFormat delegate = mock(CompoundFormat.class); | ||
| doNothing().when(delegate).write(directory, segmentInfo, IOContext.DEFAULT); | ||
|
|
||
| KNN80CompoundFormat knn80CompoundFormat = new KNN80CompoundFormat(delegate); | ||
| knn80CompoundFormat.write(directory, segmentInfo, IOContext.DEFAULT); | ||
|
|
||
| assertTrue(segmentInfo.files().isEmpty()); | ||
|
|
||
| Arrays.stream(directory.listAll()).forEach(filename -> { | ||
| try { | ||
| directory.deleteFile(filename); | ||
| } catch (IOException e) { | ||
| fail(String.format("Failed to delete: %s", filename)); | ||
| } | ||
| }); | ||
| } | ||
|
|
||
| } |
Oops, something went wrong.