Optimizations

.gitignore

@@ -3,3 +3,5 @@
 .flattened-pom.xml
 .idea/**
 rlp_01.iml
+
+src/test/**/internal/

src/main/java/com/teragrep/blf_01/Entanglement.java

@@ -48,25 +48,23 @@
 
 import java.util.ArrayList;
 import java.util.ListIterator;
-import java.util.concurrent.ExecutionException;
 
 public class Entanglement {
 
-    public final ArrayList<Token> tokens;
-
-    public Entanglement(ArrayList<Token> tokens) {
-        this.tokens = tokens;
+    private final ArrayList<Token> endWindowScanTokens;
+    private final ArrayList<Token> allTokens;
+    private final ArrayList<Token> forwardScanTokens;
+    private final ArrayList<Token> backwardsScanTokens;
+    public Entanglement() {
+        this.endWindowScanTokens = new ArrayList<>(512);
+        this.allTokens = new ArrayList<>(512);
+        this.forwardScanTokens = new ArrayList<>(512);
+        this.backwardsScanTokens = new ArrayList<>(512);
     }
 
-    public ArrayList<Token> entangle() {
-        ArrayList<Token> rv;
-        try {
-            rv = startWindowScan(tokens);
-        } catch (ExecutionException | InterruptedException e) {
-            throw new RuntimeException(e);
-        }
+    public ArrayList<Token> entangle(ArrayList<Token> tokens) {
         //System.out.println("entangling> " + tokens + " results into " + rv);
-        return rv;
+        return startWindowScan(tokens);
 
     }
 
@@ -76,49 +74,44 @@ public ArrayList<Token> entangle() {
      * starting from the largest subList and going to smaller ones
      * and processes reverse order ones within
      */
-    private ArrayList<Token> startWindowScan(ArrayList<Token> tokenList) throws ExecutionException, InterruptedException {
-        ArrayList<Token> resultTokens = new ArrayList<>();
-
+    private ArrayList<Token> startWindowScan(ArrayList<Token> tokenList)  {
+        allTokens.clear();
         for (int i = 0; i < tokenList.size(); i++) {
             ListIterator<Token> forwardIterator = tokenList.listIterator(i);
             // +++++ task
-            ArrayList<Token> windowTokens = new ArrayList<>();
+            forwardScanTokens.clear();
             while (forwardIterator.hasNext()) {
-                windowTokens.add(forwardIterator.next());
+                forwardScanTokens.add(forwardIterator.next());
             }
             // +++++ subtask endWindowScan
-            try {
-                resultTokens.addAll(endWindowScan(windowTokens));
-            } catch (ExecutionException | InterruptedException e) {
-                throw new RuntimeException(e);
-            }
+            allTokens.addAll(endWindowScan(forwardScanTokens));
             // -----
-            Token concatenated = new Token(new ConcatenatedToken(windowTokens).concatenate());
-            resultTokens.add(concatenated);
+            Token concatenated = new Token(new ConcatenatedToken(forwardScanTokens).concatenate());
+            allTokens.add(concatenated);
             // -----
         }
 
-        return resultTokens;
+        return allTokens;
     }
 
     /**
      * Iterates Token list in reverse order,
      * starting from the largest subList and going to smaller ones
      */
-    private ArrayList<Token> endWindowScan(ArrayList<Token> tokenList) throws ExecutionException, InterruptedException {
-        ArrayList<Token> resultTokens = new ArrayList<>();
+    private ArrayList<Token> endWindowScan(ArrayList<Token> tokenList) {
+        endWindowScanTokens.clear();
 
         for (int i = tokenList.size() - 1; i > 0; i--) {
             ListIterator<Token> backwardIterator = tokenList.listIterator(i);
             // +++++ task
-            ArrayList<Token> windowTokens = new ArrayList<>();
+            backwardsScanTokens.clear();
             while (backwardIterator.hasPrevious()) {
-                windowTokens.add(0, backwardIterator.previous());
+                backwardsScanTokens.add(0, backwardIterator.previous());
             }
-            resultTokens.add(new Token(new ConcatenatedToken(windowTokens).concatenate()));
+            endWindowScanTokens.add(new Token(new ConcatenatedToken(backwardsScanTokens).concatenate()));
             // ----- task
         }
 
-        return resultTokens;
+        return endWindowScanTokens;
     }
 }

src/main/java/com/teragrep/blf_01/MajorDelimiters.java

@@ -52,7 +52,7 @@
 public class MajorDelimiters implements Delimiters {
 
     private final HashMap<ByteBuffer, Delimiter> delimiterSet;
-    MajorDelimiters() {
+    public MajorDelimiters() {
         this.delimiterSet = new HashMap<>();
 
         delimiterSet.putAll(new Delimiter("\t").asMap());

src/main/java/com/teragrep/blf_01/MinorDelimiters.java

@@ -53,7 +53,7 @@ public class MinorDelimiters implements Delimiters {
 
     private final HashMap<ByteBuffer, Delimiter> delimiterSet;
 
-    MinorDelimiters() {
+    public MinorDelimiters() {
         this.delimiterSet = new HashMap<>();
 
         delimiterSet.putAll(new Delimiter("#").asMap());

src/main/java/com/teragrep/blf_01/Stream.java

@@ -50,16 +50,27 @@
 import java.io.UncheckedIOException;
 import java.util.function.Supplier;
 
-final class Stream implements Supplier<Byte> {
+public final class Stream implements Supplier<Byte> {
 
-    private final InputStream inputStream;
+    private InputStream inputStream;
 
-    private final byte[] buffer = new byte[8];
+    private final byte[] buffer = new byte[256*1024];
     private int pointer = -1;
     private int bytesInBuffer = -1;
     private byte b;
 
-    Stream(InputStream inputStream) {
+    public Stream() {
+        this.inputStream = new InputStream() {
+            @Override
+            public int read() {
+                return 0;
+            }
+        };
+    }
+
+    public void setInputStream(InputStream inputStream) {
+        this.pointer = -1;
+        this.bytesInBuffer = -1;
         this.inputStream = inputStream;
     }
 

src/main/java/com/teragrep/blf_01/TokenScan.java

@@ -55,8 +55,10 @@ public class TokenScan {
 
     private final ByteBuffer windowBuffer;
     private final Delimiter stubDelimiter;
+    private final ArrayList<Token> tokens;
+    private ByteBuffer partialToken;
 
-    TokenScan(Delimiters delimiters) {
+    public TokenScan(Delimiters delimiters) {
         this.delimiters = delimiters;
 
         // create windowBuffer with size of longest delimiter
@@ -69,13 +71,14 @@ public class TokenScan {
         }
         this.windowBuffer = ByteBuffer.allocateDirect(size);
         this.stubDelimiter = new Delimiter();
+        this.tokens = new ArrayList<>();
+        this.partialToken = ByteBuffer.allocateDirect(256);
     }
 
     public ArrayList<Token> findBy(Stream stream) {
         windowBuffer.clear();
-        ArrayList<Token> tokens = new ArrayList<>();
-
-        ByteBuffer partialToken = ByteBuffer.allocateDirect(256);
+        tokens.clear();
+        partialToken.clear();
         while (fillWindowBufferFrom(stream) || windowBuffer.position() != windowBuffer.limit()) {
             windowBuffer.flip();
 

src/test/java/com/teragrep/blf_01/EntanglementTest.java

@@ -61,7 +61,8 @@ public void subTokenSplitTest() {
         String testString = "b.c.d";
         ByteArrayInputStream bais = new ByteArrayInputStream(testString.getBytes(StandardCharsets.UTF_8));
 
-        Stream stream = new Stream(bais);
+        Stream stream = new Stream();
+        stream.setInputStream(bais);
 
         TokenScan tokenScan = new TokenScan(new MinorDelimiters());
 
@@ -75,8 +76,8 @@ public void subTokenSplitTest() {
         Assertions.assertEquals(tokens.get(3), new Token("."));
         Assertions.assertEquals(tokens.get(4), new Token("d"));
 
-        Entanglement entanglement = new Entanglement(tokens);
-        ArrayList<Token> subTokens = entanglement.entangle();
+        Entanglement entanglement = new Entanglement();
+        ArrayList<Token> subTokens = entanglement.entangle(tokens);
 
         Assertions.assertEquals(15, subTokens.size());
 

src/test/java/com/teragrep/blf_01/PerformanceTest.java

@@ -50,107 +50,146 @@
 
 import java.io.ByteArrayInputStream;
 import java.io.FileInputStream;
-import java.io.FileNotFoundException;
+import java.io.IOException;
 import java.time.Instant;
 import java.time.temporal.ChronoUnit;
 import java.util.ArrayList;
 
 public class PerformanceTest {
 
     @Test
-    public void testAll() throws FileNotFoundException {
+    public void testAll() throws IOException {
 
         Instant start = Instant.now();
         TokenScan majorTokenScan = new TokenScan(new MajorDelimiters());
 
         FileInputStream bais = new FileInputStream("src/test/resources/base64.txt");
-        Stream stream = new Stream(bais);
+        Stream stream = new Stream();
+        stream.setInputStream(bais);
 
         ArrayList<Token> majorTokens = majorTokenScan.findBy(stream);
 
         ArrayList<Token> allTokens = new ArrayList<>(majorTokens);
         Delimiters minorDelimiters = new MinorDelimiters();
+
+        TokenScan minorTokenScan = new TokenScan(minorDelimiters);
+        Entanglement entanglement = new Entanglement();
         for (Token token : majorTokens) {
             ByteArrayInputStream tokenBais = new ByteArrayInputStream(token.bytes);
 
-            Stream tokenStream = new Stream(tokenBais);
-
-            TokenScan minorTokenScan = new TokenScan(minorDelimiters);
+            stream.setInputStream(tokenBais);
 
-            ArrayList<Token> minorTokens = minorTokenScan.findBy(tokenStream);
+            ArrayList<Token> minorTokens = minorTokenScan.findBy(stream);
 
-            Entanglement entanglement = new Entanglement(minorTokens);
 
-            allTokens.addAll(entanglement.entangle());
+            allTokens.addAll(entanglement.entangle(minorTokens));
         }
         Instant end = Instant.now();
         float duration = (float) ChronoUnit.MILLIS.between(start, end)/1000;
         System.out.println("Time taken: " + duration + " seconds");
         System.out.println("Tokens: " + allTokens.size() + " (" + allTokens.size()/duration + "/s)");
+        System.out.println("Data size: " + bais.getChannel().size() + " (" + bais.getChannel().size()/duration + "/s)");
     }
 
     @Test
-    public void testAllBig() throws FileNotFoundException {
+    public void testAllBig() throws IOException {
 
         Instant start = Instant.now();
         TokenScan majorTokenScan = new TokenScan(new MajorDelimiters());
 
         FileInputStream bais = new FileInputStream("src/test/resources/base64-8m.txt");
-        Stream stream = new Stream(bais);
+        Stream stream = new Stream();
+        stream.setInputStream(bais);
 
         ArrayList<Token> majorTokens = majorTokenScan.findBy(stream);
 
         ArrayList<Token> allTokens = new ArrayList<>(majorTokens);
         Delimiters minorDelimiters = new MinorDelimiters();
+        TokenScan minorTokenScan = new TokenScan(minorDelimiters);
+        Entanglement entanglement = new Entanglement();
         for (Token token : majorTokens) {
             ByteArrayInputStream tokenBais = new ByteArrayInputStream(token.bytes);
 
-            Stream tokenStream = new Stream(tokenBais);
+            stream.setInputStream(tokenBais);
 
-            TokenScan minorTokenScan = new TokenScan(minorDelimiters);
 
-            ArrayList<Token> minorTokens = minorTokenScan.findBy(tokenStream);
+            ArrayList<Token> minorTokens = minorTokenScan.findBy(stream);
 
-            Entanglement entanglement = new Entanglement(minorTokens);
 
-            allTokens.addAll(entanglement.entangle());
+            allTokens.addAll(entanglement.entangle(minorTokens));
         }
         Instant end = Instant.now();
         float duration = (float) ChronoUnit.MILLIS.between(start, end)/1000;
         System.out.println("Time taken: " + duration + " seconds");
         System.out.println("Tokens: " + allTokens.size() + " (" + allTokens.size()/duration + "/s)");
+        System.out.println("Data size: " + bais.getChannel().size() + " (" + bais.getChannel().size()/duration + "/s)");
     }
 
     @Test
-    public void testSmall() {
+    public void testSmallDelimiters() {
 
         Instant start = Instant.now();
         String input = new String(new char[64]).replace("\0", "#");
         TokenScan majorTokenScan = new TokenScan(new MajorDelimiters());
 
-        Stream stream = new Stream(new ByteArrayInputStream(input.getBytes()));
+        Stream stream = new Stream();
+        stream.setInputStream(new ByteArrayInputStream(input.getBytes()));
 
         ArrayList<Token> majorTokens = majorTokenScan.findBy(stream);
 
         ArrayList<Token> allTokens = new ArrayList<>(majorTokens);
+        Entanglement entanglement = new Entanglement();
 
         for (Token token : majorTokens) {
             ByteArrayInputStream tokenBais = new ByteArrayInputStream(token.bytes);
 
-            Stream tokenStream = new Stream(tokenBais);
+            stream.setInputStream(tokenBais);
 
             TokenScan minorTokenScan = new TokenScan(new MinorDelimiters());
 
-            ArrayList<Token> minorTokens = minorTokenScan.findBy(tokenStream);
+            ArrayList<Token> minorTokens = minorTokenScan.findBy(stream);
+
+
+            ArrayList<Token> tokenized = entanglement.entangle(minorTokens);
+            allTokens.addAll(tokenized);
+        }
+        Instant end = Instant.now();
+        float duration = (float) ChronoUnit.MICROS.between(start, end)/1_000_000;
+        System.out.println("Time taken: " + duration + " seconds");
+        System.out.println("Tokens: " + allTokens.size() + " (" + allTokens.size()/duration + "/s)");
+        System.out.println("Data size: " + input.length() + " (" + input.length()/duration + "/s)");
+    }
+
+    @Test
+    public void testSmallCharacters() {
+
+        String input = new String(new char[128*1024]).replace("\0", "X");
+        Instant start = Instant.now();
+        TokenScan majorTokenScan = new TokenScan(new MajorDelimiters());
+
+        Stream stream = new Stream();
+        stream.setInputStream(new ByteArrayInputStream(input.getBytes()));
+
+        ArrayList<Token> majorTokens = majorTokenScan.findBy(stream);
+
+        ArrayList<Token> allTokens = new ArrayList<>(majorTokens);
+        Entanglement entanglement = new Entanglement();
+        for (Token token : majorTokens) {
+            ByteArrayInputStream tokenBais = new ByteArrayInputStream(token.bytes);
+
+            stream.setInputStream(tokenBais);
+
+            TokenScan minorTokenScan = new TokenScan(new MinorDelimiters());
 
-            Entanglement entanglement = new Entanglement(minorTokens);
+            ArrayList<Token> minorTokens = minorTokenScan.findBy(stream);
 
-            ArrayList<Token> tokenized = entanglement.entangle();
+            ArrayList<Token> tokenized = entanglement.entangle(minorTokens);
             allTokens.addAll(tokenized);
         }
         Instant end = Instant.now();
         float duration = (float) ChronoUnit.MICROS.between(start, end)/1_000_000;
         System.out.println("Time taken: " + duration + " seconds");
         System.out.println("Tokens: " + allTokens.size() + " (" + allTokens.size()/duration + "/s)");
+        System.out.println("Data size: " + input.length() + " (" + input.length()/duration + "/s)");
     }
 }