Avoid duplicate values in MapperTestCase#testFetchMany (#71068)

The test currently generates a list of random values and checks whether
retrieval of these values via doc values is equivallent to fetching them with a
value fetcher from source. If the random value array contains a duplicate value,
we will only get one back via doc values, but fetching from source will return
both, which is a case we should probably avoid in this test.

Closes #71053

server/src/test/java/org/elasticsearch/index/mapper/BinaryFieldMapperTests.java

@@ -131,4 +131,9 @@ protected Object generateRandomInputValue(MappedFieldType ft) {
     protected void randomFetchTestFieldConfig(XContentBuilder b) throws IOException {
         b.field("type", "binary").field("doc_values", true); // enable doc_values so the test is happy
     }
+
+    @Override
+    protected boolean dedupAfterFetch() {
+        return true;
+    }
 }

server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java

@@ -208,4 +208,9 @@ public void testNullValue() throws IOException {
     protected String generateRandomInputValue(MappedFieldType ft) {
         return NetworkAddress.format(randomIp(randomBoolean()));
     }
+
+    @Override
+    protected boolean dedupAfterFetch() {
+        return true;
+    }
 }

server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java

@@ -476,4 +476,9 @@ protected Object generateRandomInputValue(MappedFieldType ft) {
                 throw new IllegalStateException();
         }
     }
+
+    @Override
+    protected boolean dedupAfterFetch() {
+        return true;
+    }
 }

test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java

@@ -47,6 +47,7 @@
 import java.util.function.BiFunction;
 import java.util.function.Consumer;
 import java.util.function.Supplier;
+import java.util.stream.Collectors;
 
 import static java.util.stream.Collectors.toList;
 import static org.hamcrest.Matchers.anyOf;
@@ -487,7 +488,6 @@ public final void testFetch() throws IOException {
      * any unique and interesting failure case. See the tests for
      * {@link DateFieldMapper} for some examples.
      */
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/71053")
     public final void testFetchMany() throws IOException {
         MapperService mapperService = randomFetchTestMapper();
         try {
@@ -573,6 +573,10 @@ protected void assertFetch(MapperService mapperService, String field, Object val
                 }
                 return o;
             }).collect(toList());
+
+            if (dedupAfterFetch()) {
+                fromNative = fromNative.stream().distinct().collect(Collectors.toList());
+            }
             /*
              * Doc values sort according to something appropriate to the field
              * and the native fetchers usually don't sort. We're ok with this
@@ -582,6 +586,14 @@ protected void assertFetch(MapperService mapperService, String field, Object val
         });
     }
 
+    /**
+     * A few field types (e.g. keyword fields) don't allow duplicate values, so in those cases we need to de-dup our expected values.
+     * Field types where this is the case should overwrite this. The default is to not de-duplicate though.
+     */
+    protected boolean dedupAfterFetch() {
+        return false;
+    }
+
     /**
      * @return whether or not this field type supports access to its values from a SearchLookup
      */

x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java

@@ -150,4 +150,9 @@ private String randomPrerelease() {
         }
         return randomFrom("alpha", "beta", "") + randomVersionNumber();
     }
+
+    @Override
+    protected boolean dedupAfterFetch() {
+        return true;
+    }
 }

x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java

@@ -565,7 +565,7 @@ public void testWildcardAcceleration() throws IOException, ParseException {
         }
 
     }
-    
+
     public void testQueryCachingEquality() throws IOException, ParseException {
         String pattern = "A*b*B?a";
         // Case sensitivity matters when it comes to caching
@@ -742,7 +742,7 @@ void testExpectedAccelerationQuery(String regex, Query combinedQuery, String exp
         Query expectedAccelerationQuery = qsp.parse(expectedAccelerationQueryString);
         testExpectedAccelerationQuery(regex, combinedQuery, expectedAccelerationQuery);
     }
-    
+
     private Query unwrapAnyConstantScore(Query q) {
         if (q instanceof ConstantScoreQuery) {
             ConstantScoreQuery csq = (ConstantScoreQuery) q;
@@ -1022,4 +1022,9 @@ private String getRandomWildcardPattern() {
     protected String generateRandomInputValue(MappedFieldType ft) {
         return randomAlphaOfLengthBetween(1, 100);
     }
+
+    @Override
+    protected boolean dedupAfterFetch() {
+        return true;
+    }
 }