fixes neo4j-contrib#864: Adds text similarity/distance methods and do…

…uble metaphone text encoding. Added Levenshtein Distance and Similarity code and test Added Hamming Distance code and test Added Jaro-Winkler Distance code and test Added Double Metaphone text encoding and test Added Apache commons-text dependency
graphgrid · Jul 19, 2018 · 806dac3 · 806dac3
1 parent 2e1b263
commit 806dac3
Show file tree

Hide file tree

Showing 5 changed files with 143 additions and 66 deletions.
diff --git a/src/main/java/apoc/algo/Similarity.java b/src/main/java/apoc/algo/Similarity.java
@@ -1,8 +1,5 @@
 package apoc.algo;
 
-import org.apache.commons.text.similarity.HammingDistance;
-import org.apache.commons.text.similarity.JaroWinklerDistance;
-import org.apache.commons.text.similarity.LevenshteinDistance;
 import org.neo4j.graphdb.GraphDatabaseService;
 import org.neo4j.procedure.Context;
 import org.neo4j.procedure.Description;
@@ -67,52 +64,4 @@ public double euclideanDistance(@Name("vector1") List<Number> vector1, @Name("ve
     public double euclideanSimilarity(@Name("vector1") List<Number> vector1, @Name("vector2") List<Number> vector2) {
         return 1.0d / (1 + euclideanDistance(vector1, vector2));
     }
-
-    @UserFunction
-    @Description("apoc.algo.levenshteinDistance(lhs, rhs) return the Levenshtein distance of two texts")
-    public double levenshteinDistance( @Name("lhs") String lhs, @Name("rhs") String rhs ) {
-        if ( lhs == null || rhs == null )
-        {
-            return 0.0;
-        }
-
-        LevenshteinDistance ld = new LevenshteinDistance();
-        return ld.apply( lhs, rhs );
-    }
-
-    @UserFunction
-    @Description("apoc.algo.levenshteinSimilarity(lhs, rhs) return the similarity of two texts based on Levenshtein distance")
-    public double levenshteinSimilarity( @Name("lhs") String lhs, @Name("rhs") String rhs ) {
-        if ( lhs == null || rhs == null )
-        {
-            return 0.0;
-        }
-
-        double editDistance = levenshteinDistance( lhs, rhs );
-        return 1 - (editDistance / Math.max( lhs.length(), rhs.length() ));
-    }
-
-    @UserFunction
-    @Description("apoc.algo.hammingDistance(lhs, rhs) return the Hamming distance of two texts")
-    public double hammingDistance( @Name("lhs") String lhs, @Name("rhs") String rhs ) {
-        if ( lhs == null || rhs == null )
-        {
-            return 0.0;
-        }
-
-        HammingDistance hd = new HammingDistance();
-        return hd.apply( lhs, rhs );
-    }
-
-    @UserFunction
-    @Description("apoc.algo.jaroWinklerDistance(lhs, rhs) return the Jaro-Winkler distance of two texts")
-    public double jaroWinklerDistance( @Name("lhs") String lhs, @Name("rhs") String rhs ) {
-        if ( lhs == null || rhs == null )
-        {
-            return 0.0;
-        }
-
-        JaroWinklerDistance jwd = new JaroWinklerDistance();
-        return jwd.apply( lhs, rhs );
-    }
 }
diff --git a/src/main/java/apoc/text/Phonetic.java b/src/main/java/apoc/text/Phonetic.java
@@ -1,5 +1,8 @@
 package apoc.text;
 
+import org.apache.commons.codec.language.DoubleMetaphone;
+import org.neo4j.procedure.Description;
+import apoc.result.LongResult;
 import apoc.result.StringResult;
 import org.apache.commons.codec.EncoderException;
 import org.apache.commons.codec.language.DoubleMetaphone;
@@ -49,12 +52,11 @@ public Stream<StringResult> doubleMetaphone(final @Name("value") Object value)
         Stream<Object> stream = value instanceof Iterable ? StreamSupport.stream(((Iterable) value).spliterator(), false) : Stream.of(value);
 
         return stream.map(str -> (str == null || str.toString().isEmpty()) ? StringResult.EMPTY :
-                new StringResult(Stream.of(str.toString().trim().split("\\W+"))
-                        .map(DOUBLE_METAPHONE::doubleMetaphone).reduce("", (a, s) -> a + s)));
+                                 new StringResult(Stream.of(str.toString().trim().split("\\W+"))
+                                         .map(DOUBLE_METAPHONE::doubleMetaphone).reduce("", (a, s) -> a + s)));
     }
 
-    public static class PhoneticResult
-    {
+    public static class PhoneticResult {
         public final String phonetic1, phonetic2;
         public final long delta;
 

diff --git a/src/main/java/apoc/text/Strings.java b/src/main/java/apoc/text/Strings.java
@@ -17,6 +17,9 @@
 import java.util.stream.Stream;
 
 import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.text.similarity.HammingDistance;
+import org.apache.commons.text.similarity.JaroWinklerDistance;
+import org.apache.commons.text.similarity.LevenshteinDistance;
 
 import static java.lang.Math.toIntExact;
 import static java.util.Arrays.asList;
@@ -27,6 +30,10 @@
  */
 public class Strings {
 
+    private final static JaroWinklerDistance jaroWinklerDistance = new JaroWinklerDistance();
+    private final static HammingDistance hammingDistance = new HammingDistance();
+    private final static LevenshteinDistance levenshteinDistance = new LevenshteinDistance();
+
     @UserFunction
     @Description("apoc.text.replace(text, regex, replacement) - replace each substring of the given string that matches the given regular expression with the given replacement.")
     public String replace(final @Name("text") String text, final @Name("regex") String regex, final @Name("replacement") String replacement) {
@@ -105,7 +112,7 @@ public Long distance(final @Name("text1") String text1, @Name("text2")final Stri
         if (text1 == null || text2 == null) {
             return null;
         }
-        return (long) StringUtils.getLevenshteinDistance(text1, text2);
+        return Integer.toUnsignedLong( levenshteinDistance.apply( text1, text2 ) );
     }
 
     @UserFunction
@@ -372,4 +379,46 @@ public String base64Decode(@Name("text") String text) {
         byte[] decoded = Base64.getDecoder().decode(text.getBytes());
         return new String(decoded);
     }
+
+    @UserFunction
+    @Description( "apoc.text.levenshteinDistance(lhs, rhs) return the Levenshtein distance of two texts" )
+    public Long levenshteinDistance( @Name( "lhs" ) String lhs, @Name( "rhs" ) String rhs )
+    {
+        return distance( lhs, rhs );
+    }
+
+    @UserFunction
+    @Description( "apoc.text.levenshteinSimilarity(lhs, rhs) return the similarity of two texts based on Levenshtein distance" )
+    public Double levenshteinSimilarity( @Name( "lhs" ) String lhs, @Name( "rhs" ) String rhs )
+    {
+        if ( lhs == null || rhs == null )
+        {
+            return null;
+        }
+
+        double editDistance = levenshteinDistance( lhs, rhs );
+        return 1 - (editDistance / Math.max( lhs.length(), rhs.length() ));
+    }
+
+    @UserFunction
+    @Description( "apoc.text.hammingDistance(lhs, rhs) return the Hamming distance of two texts" )
+    public Long hammingDistance( @Name( "lhs" ) String lhs, @Name( "rhs" ) String rhs )
+    {
+        if ( lhs == null || rhs == null )
+        {
+            return null;
+        }
+        return Integer.toUnsignedLong( hammingDistance.apply( lhs, rhs ) );
+    }
+
+    @UserFunction
+    @Description( "apoc.text.jaroWinklerDistance(lhs, rhs) return the Jaro-Winkler distance of two texts" )
+    public Double jaroWinklerDistance( @Name( "lhs" ) String lhs, @Name( "rhs" ) String rhs )
+    {
+        if ( lhs == null || rhs == null )
+        {
+            return null;
+        }
+        return jaroWinklerDistance.apply( lhs, rhs );
+    }
 }
diff --git a/src/test/java/apoc/text/PhoneticTest.java b/src/test/java/apoc/text/PhoneticTest.java
@@ -63,6 +63,13 @@ public void shouldComputeSoundexEncodingOfManyWordsEvenIfTheStringContainsSomeEx
         );
     }
 
+    @Test
+    public void shouldComputeWordSoundexEncodingOfNull() {
+        testCall(db, "CALL apoc.text.phonetic(NULL)", (row) ->
+                assertThat(row.get("value"), equalTo(null))
+        );
+    }
+
     @Test
     public void shouldComputeSoundexDifference() {
         testCall(db, "CALL apoc.text.phoneticDelta('Hello Mr Rabbit', 'Hello Mr Ribbit')", (row) ->
@@ -71,29 +78,29 @@ public void shouldComputeSoundexDifference() {
     }
 
     @Test
-    public void shoudlComputeDoubleMetaphone() {
-        testCall(db, "CALL apoc.text.doubleMetaphone('Apoc')", (row) ->
-                assertThat(row.get("value"), equalTo("APK"))
+    public void shouldComputeDoubleMetaphone() {
+        testCall(db, "CALL apoc.text.doubleMetaphone('Apoc') YIELD value RETURN value",
+                (row) -> assertEquals("APK", row.get("value"))
         );
     }
 
     @Test
-    public void shoudlComputeDoubleMetaphoneOfNull() {
-        testCall(db, "CALL apoc.text.doubleMetaphone(NULL)", (row) ->
-                assertThat(row.get("value"), equalTo(null))
+    public void shouldComputeDoubleMetaphoneOfNull() {
+        testCall(db, "CALL apoc.text.doubleMetaphone(NULL) YIELD value RETURN value",
+                (row) -> assertEquals(null, row.get("value"))
         );
     }
 
     @Test
-    public void shoudlComputeDoubleMetaphoneForTheEmptyString() {
-        testCall(db, "CALL apoc.text.doubleMetaphone('')", (row) ->
-                assertThat(row.get("value"), equalTo(null))
+    public void shouldComputeDoubleMetaphoneForTheEmptyString() {
+        testCall(db, "CALL apoc.text.doubleMetaphone('') YIELD value RETURN value",
+                (row) -> assertThat(row.get("value"), equalTo(null))
         );
     }
 
     @Test
     public void shouldComputeDoubleMetaphoneOfManyWords() {
-        testCall(db, "CALL apoc.text.doubleMetaphone('Hello, dear User!')", (row) ->
+        testCall(db, "CALL apoc.text.doubleMetaphone('Hello, dear User!') YIELD value RETURN value", (row) ->
                 assertThat(row.get("value"), equalTo("HLTRASR"))
         );
     }

diff --git a/src/test/java/apoc/text/StringsTest.java b/src/test/java/apoc/text/StringsTest.java
@@ -489,4 +489,74 @@ public void testSorensenDiceSimilarityWithTurkishLocale() {
                  map("text1",text1,"text2", text2, "languageTag", languageTag),
                  row -> assertEquals(0.5, row.get("value")));
     }
+
+    @Test
+    public void testLevenshteinDistance() {
+        testCall(db, "RETURN apoc.text.levenshteinDistance('cat','cut') AS value",
+                row -> assertEquals(1L, row.get("value"))
+        );
+    }
+
+    @Test
+    public void testLevenshteinDistanceOfNull() {
+        testCall(db, "RETURN apoc.text.levenshteinDistance(null,'cut') AS value",
+                row -> assertEquals(null, row.get("value"))
+        );
+    }
+
+    @Test
+    public void testLevenshteinDistanceForTheEmptyString() {
+        testCall(db, "RETURN apoc.text.levenshteinDistance('','cut') AS value",
+                row -> assertEquals(3L, row.get("value"))
+        );
+    }
+
+    @Test
+    public void testLevenshteinSimilarity() {
+        testCall(db, "RETURN apoc.text.levenshteinSimilarity('cold','cool') AS value",
+                row -> assertEquals(0.5, row.get("value"))
+        );
+    }
+
+    @Test
+    public void testLevenshteinSimilarityOfNull() {
+        testCall(db, "RETURN apoc.text.levenshteinSimilarity(null,'cool') AS value",
+                row -> assertEquals(null, row.get("value"))
+        );
+    }
+
+    @Test
+    public void testHammingDistance() {
+        testCall(db, "RETURN apoc.text.hammingDistance('cat','cut') AS value",
+                row -> assertEquals(1L, row.get("value"))
+        );
+    }
+
+    @Test
+    public void testHammingDistanceOfNull() {
+        testCall(db, "RETURN apoc.text.hammingDistance(null,'cut') AS value",
+                row -> assertEquals(null, row.get("value"))
+        );
+    }
+
+    @Test
+    public void testHighJaroWinklerDistanceForSimilarNames() {
+        testCall(db, "RETURN apoc.text.jaroWinklerDistance('Sherry','Shelly') AS value",
+                row -> assertEquals("0.84444", String.format( "%.5f", (double)row.get("value")))
+        );
+    }
+
+    @Test
+    public void testLowJaroWinklerDistanceForNotSimilarNames() {
+        testCall(db, "RETURN apoc.text.jaroWinklerDistance('Douglas','Shelly') AS value",
+                row -> assertEquals("0.43651", String.format( "%.5f", (double)row.get("value")))
+        );
+    }
+
+    @Test
+    public void testJaroWinklerDistanceOfNull() {
+        testCall(db, "RETURN apoc.text.jaroWinklerDistance(null,'Shelly') AS value",
+                row -> assertEquals(null, row.get("value"))
+        );
+    }
 }