Merge pull request #3 from StrongestNumber9/refactor-3

Fixes extendBuffer
kortemik · Oct 9, 2023 · 85e9e1a · 85e9e1a
2 parents b992507 + 873257b
commit 85e9e1a
Show file tree

Hide file tree

Showing 2 changed files with 35 additions and 3 deletions.
diff --git a/src/main/java/com/teragrep/blf_01/TokenScan.java b/src/main/java/com/teragrep/blf_01/TokenScan.java
@@ -161,9 +161,8 @@ private boolean fillWindowBufferFrom(Stream stream) {
     // +++++ PartialToken stuff
     private ByteBuffer extendBuffer(ByteBuffer byteBuffer, int size) {
         ByteBuffer newBuffer = ByteBuffer.allocateDirect(byteBuffer.capacity() + size);
-        ByteBuffer originalBuffer = byteBuffer.slice();
-        originalBuffer.flip();
-        newBuffer.put(originalBuffer);
+        byteBuffer.flip();
+        newBuffer.put(byteBuffer);
         return newBuffer;
     }
     // -----

diff --git a/src/test/java/com/teragrep/blf_01/PerformanceTest.java b/src/test/java/com/teragrep/blf_01/PerformanceTest.java
@@ -120,4 +120,37 @@ public void testAllBig() throws FileNotFoundException {
         System.out.println("Time taken: " + duration + " seconds");
         System.out.println("Tokens: " + allTokens.size() + " (" + allTokens.size()/duration + "/s)");
     }
+
+    @Test
+    public void testSmall() {
+
+        Instant start = Instant.now();
+        String input = new String(new char[64]).replace("\0", "#");
+        TokenScan majorTokenScan = new TokenScan(new MajorDelimiters());
+
+        Stream stream = new Stream(new ByteArrayInputStream(input.getBytes()));
+
+        ArrayList<Token> majorTokens = majorTokenScan.findBy(stream);
+
+        ArrayList<Token> allTokens = new ArrayList<>(majorTokens);
+
+        for (Token token : majorTokens) {
+            ByteArrayInputStream tokenBais = new ByteArrayInputStream(token.bytes);
+
+            Stream tokenStream = new Stream(tokenBais);
+
+            TokenScan minorTokenScan = new TokenScan(new MinorDelimiters());
+
+            ArrayList<Token> minorTokens = minorTokenScan.findBy(tokenStream);
+
+            Entanglement entanglement = new Entanglement(minorTokens);
+
+            ArrayList<Token> tokenized = entanglement.entangle();
+            allTokens.addAll(tokenized);
+        }
+        Instant end = Instant.now();
+        float duration = (float) ChronoUnit.MILLIS.between(start, end)/1000;
+        System.out.println("Time taken: " + duration + " seconds");
+        System.out.println("Tokens: " + allTokens.size() + " (" + allTokens.size()/duration + "/s)");
+    }
 }