From bd131955ffa2ef064aa7b9543a99808041dfcf5d Mon Sep 17 00:00:00 2001 From: "Taro L. Saito" Date: Wed, 11 Dec 2013 18:03:36 +0900 Subject: [PATCH 01/13] snapshot version --- version.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.sbt b/version.sbt index 7e69a72..f476830 100755 --- a/version.sbt +++ b/version.sbt @@ -1,3 +1,3 @@ -version in ThisBuild := "0.2" +version in ThisBuild := "0.2-SNAPSHOT" From 203486ecf397e24de447cc1136e857d43847872b Mon Sep 17 00:00:00 2001 From: "Taro L. Saito" Date: Wed, 11 Dec 2013 18:09:01 +0900 Subject: [PATCH 02/13] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 735f915..204c049 100644 --- a/README.md +++ b/README.md @@ -137,7 +137,7 @@ l2.free l2(0) // The result of accessing released LArray is undefined ``` -For more examples, see [xerial/larray/example/LArrayExample.scala](larray-scala/src/main/scala/xerial/larray/example/LArrayExample.scala) +For more examples, see [xerial/larray/example/LArrayExample.scala](larray/src/main/scala/xerial/larray/example/LArrayExample.scala) ## Usage (Java) @@ -165,7 +165,7 @@ int e0 = l.apply(0L); // Get l[0L] // release l.free(); ``` -For more examples, see [xerial/larray/example/LArrayJavaExample.scala](larray-scala/src/main/scala/xerial/larray/example/LArrayJavaExample.java) +For more examples, see [xerial/larray/example/LArrayJavaExample.scala](larray/src/main/scala/xerial/larray/example/LArrayJavaExample.java) ## Scaladoc From 7052c7322d91fa22a37fa8b35ee300f7331b10be Mon Sep 17 00:00:00 2001 From: "Taro L. Saito" Date: Wed, 11 Dec 2013 18:10:51 +0900 Subject: [PATCH 03/13] add LBuffer example --- .../xerial/larray/example/LArrayJavaExample.java | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/larray/src/main/scala/xerial/larray/example/LArrayJavaExample.java b/larray/src/main/scala/xerial/larray/example/LArrayJavaExample.java index e5859ee..54fbde8 100644 --- a/larray/src/main/scala/xerial/larray/example/LArrayJavaExample.java +++ b/larray/src/main/scala/xerial/larray/example/LArrayJavaExample.java @@ -16,7 +16,11 @@ package xerial.larray.example; import scala.runtime.AbstractFunction1; -import xerial.larray.*; +import xerial.larray.LArray; +import xerial.larray.LIntArray; +import xerial.larray.LIntArrayBuilder; +import xerial.larray.LIterator; +import xerial.larray.buffer.LBuffer; import xerial.larray.japi.LArrayJ; import java.io.File; @@ -83,6 +87,15 @@ public Object apply(Object v1) { // Release the memory contents l.free(); + + // Using LBuffer + LBuffer lbuf = new LBuffer(1000); + lbuf.putInt(0, 10); + int ten = lbuf.getInt(0); + lbuf.address(); // memory address + lbuf.release(); // deallocate the memory + + System.out.println("done."); } } From bce7d91d3446d2aa09040d931f828f92aa6cf616 Mon Sep 17 00:00:00 2001 From: "Taro L. Saito" Date: Wed, 11 Dec 2013 18:32:37 +0900 Subject: [PATCH 04/13] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 204c049..cf4383f 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ LArray === -A library for managing large off-heap arrays that can hold more than 2G (2^31) entries in Java and Scala. +A library for managing large off-heap arrays that can hold more than 2G (2^31) entries in Java and Scala. Notablly LArray is *disposable* by calling `LArray.free` or let GC automatically release the memory. LArray also has `mmap` (memory-mapped file) support for more than 2GB of files. ## Features * LArray can create arrays with more than 2G(2^31) entries. From 55e28fd4b839740fb7df1703a05058b484efe074 Mon Sep 17 00:00:00 2001 From: "Taro L. Saito" Date: Wed, 11 Dec 2013 18:45:23 +0900 Subject: [PATCH 05/13] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index cf4383f..a333b00 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ LArray === -A library for managing large off-heap arrays that can hold more than 2G (2^31) entries in Java and Scala. Notablly LArray is *disposable* by calling `LArray.free` or let GC automatically release the memory. LArray also has `mmap` (memory-mapped file) support for more than 2GB of files. +A library for managing large off-heap arrays that can hold more than 2G (2^31) entries in Java and Scala. Notablly LArray is *disposable* by calling `LArray.free` or you can let GC automatically release the memory. LArray also has `mmap` (memory-mapped file) support for a file whose size is more than 2GB. ## Features * LArray can create arrays with more than 2G(2^31) entries. From 649cfe6c1205824757543dd0c68ed35180c31cc3 Mon Sep 17 00:00:00 2001 From: "Taro L. Saito" Date: Wed, 11 Dec 2013 18:46:47 +0900 Subject: [PATCH 06/13] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a333b00..178f648 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ LArray === -A library for managing large off-heap arrays that can hold more than 2G (2^31) entries in Java and Scala. Notablly LArray is *disposable* by calling `LArray.free` or you can let GC automatically release the memory. LArray also has `mmap` (memory-mapped file) support for a file whose size is more than 2GB. +A library for managing large off-heap arrays that can hold more than 2G (2^31) entries in Java and Scala. Notablly LArray is *disposable* by calling `LArray.free` or you can let GC automatically release the memory. LArray also can be used to create an `mmap` (memory-mapped file) whose size is more than 2GB. ## Features * LArray can create arrays with more than 2G(2^31) entries. From 83c4e6105b2a93fd3225dc784ae8af550eba32be Mon Sep 17 00:00:00 2001 From: "Taro L. Saito" Date: Wed, 11 Dec 2013 22:41:05 +0900 Subject: [PATCH 07/13] Fix javadoc locale. Removed slf4j --- .../xerial/larray/example/LArrayJavaExample.java | 2 +- project/Build.scala | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/larray/src/main/scala/xerial/larray/example/LArrayJavaExample.java b/larray/src/main/scala/xerial/larray/example/LArrayJavaExample.java index 54fbde8..0ea9ec7 100644 --- a/larray/src/main/scala/xerial/larray/example/LArrayJavaExample.java +++ b/larray/src/main/scala/xerial/larray/example/LArrayJavaExample.java @@ -94,7 +94,7 @@ public Object apply(Object v1) { int ten = lbuf.getInt(0); lbuf.address(); // memory address lbuf.release(); // deallocate the memory - + System.out.println("done."); } diff --git a/project/Build.scala b/project/Build.scala index 99a924f..62a788b 100755 --- a/project/Build.scala +++ b/project/Build.scala @@ -24,7 +24,7 @@ object Build extends sbt.Build { private val SCALA_VERSION = "2.10.3" val buildSettings = Defaults.defaultSettings ++ Seq( - organization := "org.xerial", + organization := "org.xerial.larray", organizationName := "xerial.org", organizationHomepage := Some(new URL("http://xerial.org")), publishMavenStyle := true, @@ -41,6 +41,7 @@ object Build extends sbt.Build { parallelExecution in Test := false, javacOptions in Compile ++= Seq("-Xlint:unchecked"), javacOptions in (Compile, doc) <<= (baseDirectory, version) map { (bd, v) => Seq( + "-locale", "en_US", "-sourcepath", bd.getAbsolutePath, "-doctitle", s"LArray ${v} API" )}, @@ -132,7 +133,6 @@ object Build extends sbt.Build { snappy % "test", junit, "org.iq80.snappy" % "snappy" % "0.3" % "test", - "com.novocode" % "junit-interface" % "0.10-M2" % "test", "org.scalatest" % "scalatest_2.10" % "2.0.M5b" % "test", "org.scalacheck" % "scalacheck_2.10" % "1.10.0" % "test", @@ -150,9 +150,9 @@ object Build extends sbt.Build { autoScalaLibrary := false, libraryDependencies ++= Seq( "org.scalatest" % "scalatest_2.10" % "2.0.M5b" % "test", - "org.xerial" % "xerial-core" % "3.2.2" % "test", - slf4j, - slf4jSimple % "test" + "org.xerial.java" % "xerial-core" % "2.1" +// slf4j, +// slf4jSimple % "test" ) ) ) @@ -163,6 +163,7 @@ object Build extends sbt.Build { settings = buildSettings ++ Seq( description := "LArray mmap implementation", + autoScalaLibrary := false, libraryDependencies ++= Seq( snappy % "test", junit From 95c9b102c9c822702ee623c072ba8e1f77d9569f Mon Sep 17 00:00:00 2001 From: "Taro L. Saito" Date: Wed, 11 Dec 2013 22:44:31 +0900 Subject: [PATCH 08/13] Use xerial logger instead of slf4j --- .../java/xerial/larray/buffer/OffHeapMemoryAllocator.java | 6 +++--- project/Build.scala | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/larray-buffer/src/main/java/xerial/larray/buffer/OffHeapMemoryAllocator.java b/larray-buffer/src/main/java/xerial/larray/buffer/OffHeapMemoryAllocator.java index fb94e56..8b000bb 100644 --- a/larray-buffer/src/main/java/xerial/larray/buffer/OffHeapMemoryAllocator.java +++ b/larray-buffer/src/main/java/xerial/larray/buffer/OffHeapMemoryAllocator.java @@ -1,7 +1,7 @@ package xerial.larray.buffer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; + +import org.xerial.util.log.Logger; import java.lang.ref.ReferenceQueue; import java.util.Map; @@ -100,7 +100,7 @@ public Memory toMemory() { */ public class OffHeapMemoryAllocator implements MemoryAllocator { - private Logger logger = LoggerFactory.getLogger(OffHeapMemoryAllocator.class); + private Logger logger = Logger.getLogger(OffHeapMemoryAllocator.class); // Table from address -> MemoryReference private Map allocatedMemoryReferences = new ConcurrentHashMap(); diff --git a/project/Build.scala b/project/Build.scala index 62a788b..457dedd 100755 --- a/project/Build.scala +++ b/project/Build.scala @@ -150,7 +150,8 @@ object Build extends sbt.Build { autoScalaLibrary := false, libraryDependencies ++= Seq( "org.scalatest" % "scalatest_2.10" % "2.0.M5b" % "test", - "org.xerial.java" % "xerial-core" % "2.1" + "org.xerial.java" % "xerial-core" % "2.1", + "org.xerial" % "xerial-core" % "3.2.2" % "test" // slf4j, // slf4jSimple % "test" ) From 22709dfb55a94efdabe11c0653838497797e5393 Mon Sep 17 00:00:00 2001 From: "Taro L. Saito" Date: Wed, 11 Dec 2013 22:50:47 +0900 Subject: [PATCH 09/13] Fix reported deallocated memory size --- .../xerial/larray/buffer/OffHeapMemoryAllocator.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/larray-buffer/src/main/java/xerial/larray/buffer/OffHeapMemoryAllocator.java b/larray-buffer/src/main/java/xerial/larray/buffer/OffHeapMemoryAllocator.java index 8b000bb..d7770cb 100644 --- a/larray-buffer/src/main/java/xerial/larray/buffer/OffHeapMemoryAllocator.java +++ b/larray-buffer/src/main/java/xerial/larray/buffer/OffHeapMemoryAllocator.java @@ -102,11 +102,15 @@ public class OffHeapMemoryAllocator implements MemoryAllocator { private Logger logger = Logger.getLogger(OffHeapMemoryAllocator.class); + // Table from address -> MemoryReference private Map allocatedMemoryReferences = new ConcurrentHashMap(); private ReferenceQueue queue = new ReferenceQueue(); { + // Enable ANSI Color + logger.enableColor(true); + // Start OffHeapMemory collector that releases the allocated memory when the corresponding Memory object is collected by GC. Thread collector = new Thread(new Runnable() { @Override @@ -183,10 +187,9 @@ public void release(Memory m) { synchronized(this) { long address = m.headerAddress(); if(allocatedMemoryReferences.containsKey(address)) { - long size = m.size(); if(logger.isTraceEnabled()) - logger.trace(String.format("Released memory address:%x, size:%,d", address, size)); - totalAllocatedSize.getAndAdd(-size); + logger.trace(String.format("Released memory address:%x, size:%,d", address, m.dataSize())); + totalAllocatedSize.getAndAdd(-m.size()); allocatedMemoryReferences.remove(address); m.release(); } From a5d3eb4a58d1f64134aea8e2349164d0cdb52a48 Mon Sep 17 00:00:00 2001 From: "Taro L. Saito" Date: Wed, 11 Dec 2013 22:52:16 +0900 Subject: [PATCH 10/13] 0.2.1-SNAPSHOT --- version.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.sbt b/version.sbt index f476830..d1c5964 100755 --- a/version.sbt +++ b/version.sbt @@ -1,3 +1,3 @@ -version in ThisBuild := "0.2-SNAPSHOT" +version in ThisBuild := "0.2.1-SNAPSHOT" From 403ef92f0124ecbdb1ab0f0f79bdf332792784df Mon Sep 17 00:00:00 2001 From: "Taro L. Saito" Date: Wed, 11 Dec 2013 23:27:37 +0900 Subject: [PATCH 11/13] Enhance javadoc comments --- .../xerial/larray/buffer/BufferConfig.java | 9 -- ...cator.java => DefaultMemoryCollector.java} | 100 +---------------- .../java/xerial/larray/buffer/LBuffer.java | 2 +- .../java/xerial/larray/buffer/LBufferAPI.java | 103 +++++++++++++++++- .../xerial/larray/buffer/LBufferConfig.java | 10 ++ .../xerial/larray/buffer/MemoryAllocator.java | 29 ----- .../xerial/larray/buffer/MemoryCollector.java | 25 +++++ .../xerial/larray/buffer/OffHeapMemory.java | 84 ++++++++++++++ .../larray/buffer/OffHeapMemoryReference.java | 28 +++++ .../xerial/larray/buffer/WrappedLBuffer.java | 28 +++++ ...orTest.scala => MemoryCollectorTest.scala} | 4 +- .../larray/buffer/WrappedLBufferTest.scala | 34 ++++++ .../java/xerial/larray/mmap/MMapBuffer.java | 4 +- .../main/java/xerial/larray/japi/LArrayJ.java | 5 +- .../src/main/scala/xerial/larray/LArray.scala | 32 +++--- .../xerial/larray/MappedLByteArray.scala | 4 +- .../scala/xerial/larray/UInt32Array.scala | 6 +- .../main/scala/xerial/larray/package.scala | 4 +- .../test/scala/xerial/larray/LArrayTest.scala | 2 +- 19 files changed, 345 insertions(+), 168 deletions(-) delete mode 100644 larray-buffer/src/main/java/xerial/larray/buffer/BufferConfig.java rename larray-buffer/src/main/java/xerial/larray/buffer/{OffHeapMemoryAllocator.java => DefaultMemoryCollector.java} (54%) create mode 100644 larray-buffer/src/main/java/xerial/larray/buffer/LBufferConfig.java delete mode 100644 larray-buffer/src/main/java/xerial/larray/buffer/MemoryAllocator.java create mode 100644 larray-buffer/src/main/java/xerial/larray/buffer/MemoryCollector.java create mode 100644 larray-buffer/src/main/java/xerial/larray/buffer/OffHeapMemory.java create mode 100644 larray-buffer/src/main/java/xerial/larray/buffer/OffHeapMemoryReference.java create mode 100644 larray-buffer/src/main/java/xerial/larray/buffer/WrappedLBuffer.java rename larray-buffer/src/test/scala/xerial/larray/buffer/{MemoryAllocatorTest.scala => MemoryCollectorTest.scala} (95%) create mode 100644 larray-buffer/src/test/scala/xerial/larray/buffer/WrappedLBufferTest.scala diff --git a/larray-buffer/src/main/java/xerial/larray/buffer/BufferConfig.java b/larray-buffer/src/main/java/xerial/larray/buffer/BufferConfig.java deleted file mode 100644 index d510989..0000000 --- a/larray-buffer/src/main/java/xerial/larray/buffer/BufferConfig.java +++ /dev/null @@ -1,9 +0,0 @@ -package xerial.larray.buffer; - -/** - * @author Taro L. Saito - */ -public class BufferConfig { - - public static MemoryAllocator allocator = new OffHeapMemoryAllocator(); -} diff --git a/larray-buffer/src/main/java/xerial/larray/buffer/OffHeapMemoryAllocator.java b/larray-buffer/src/main/java/xerial/larray/buffer/DefaultMemoryCollector.java similarity index 54% rename from larray-buffer/src/main/java/xerial/larray/buffer/OffHeapMemoryAllocator.java rename to larray-buffer/src/main/java/xerial/larray/buffer/DefaultMemoryCollector.java index d7770cb..7390bbc 100644 --- a/larray-buffer/src/main/java/xerial/larray/buffer/OffHeapMemoryAllocator.java +++ b/larray-buffer/src/main/java/xerial/larray/buffer/DefaultMemoryCollector.java @@ -8,99 +8,18 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicLong; -import static xerial.larray.buffer.UnsafeUtil.unsafe; -/** - * Stores |(memory size:long)| data ... | - */ -class OffHeapMemory implements Memory { - - private final long _data; - - public static long HEADER_SIZE = 8L; - - /** - * Create an empty memory - */ - public OffHeapMemory() { - this._data = 0L; - } - - public OffHeapMemory(long address) { - if(address != 0L) - this._data = address + HEADER_SIZE; - else - this._data = 0L; - } - - public OffHeapMemory(long address, long size) { - if(address != 0L) { - this._data = address + HEADER_SIZE; - unsafe.putLong(address, size); - } - else { - this._data = 0L; - } - } - - public long headerAddress() { - return _data - HEADER_SIZE; - } - public long size() { - return (_data == 0) ? 0L : unsafe.getLong(headerAddress()) + HEADER_SIZE; - } - - public long address() { - return _data; - } - - public long dataSize() { - return (_data == 0) ? 0L : unsafe.getLong(headerAddress()); - } - - public MemoryReference toRef(ReferenceQueue queue) { - return new OffHeapMemoryReference(this, queue); - } - - public void release() { - if(_data != 0) - UnsafeUtil.unsafe.freeMemory(headerAddress()); - } -} - -class OffHeapMemoryReference extends MemoryReference { - - /** - * Create a phantom reference - * @param m the allocated memory - * @param queue the reference queue to which GCed reference of the Memory will be inserted - */ - public OffHeapMemoryReference(Memory m, ReferenceQueue queue) { - super(m, queue); - } - - public Memory toMemory() { - if(address != 0) - return new OffHeapMemory(address); - else - return new OffHeapMemory(); - } - - public String name() { return "off-heap"; } - -} - /** - * Allocating off-heap memory + * A default implementation of MemoryCollector that releases allocated memories in a background thread. * * @author Taro L. Saito */ -public class OffHeapMemoryAllocator implements MemoryAllocator { +public class DefaultMemoryCollector implements MemoryCollector { - private Logger logger = Logger.getLogger(OffHeapMemoryAllocator.class); + private Logger logger = Logger.getLogger(DefaultMemoryCollector.class); // Table from address -> MemoryReference @@ -140,19 +59,6 @@ public void run() { */ public long allocatedSize() { return totalAllocatedSize.get(); } - public Memory allocate(long size) { - if(size == 0L) - return new OffHeapMemory(); - - // Allocate memory of the given size + HEADER space - long memorySize = size + OffHeapMemory.HEADER_SIZE; - long address = unsafe.allocateMemory(memorySize); - if(logger.isTraceEnabled()) - logger.trace(String.format("Allocated memory address:%x, size:%,d", address, size)); - Memory m = new OffHeapMemory(address, size); - register(m); - return m; - } public void register(Memory m) { // Register a memory reference that will be collected upon GC diff --git a/larray-buffer/src/main/java/xerial/larray/buffer/LBuffer.java b/larray-buffer/src/main/java/xerial/larray/buffer/LBuffer.java index 24aa1c1..ec5a98d 100644 --- a/larray-buffer/src/main/java/xerial/larray/buffer/LBuffer.java +++ b/larray-buffer/src/main/java/xerial/larray/buffer/LBuffer.java @@ -14,7 +14,7 @@ public class LBuffer extends LBufferAPI { * @param size byte size of the array */ public LBuffer(long size) { - super(BufferConfig.allocator.allocate(size)); + super(OffHeapMemory.allocate(size)); } diff --git a/larray-buffer/src/main/java/xerial/larray/buffer/LBufferAPI.java b/larray-buffer/src/main/java/xerial/larray/buffer/LBufferAPI.java index 0e72ecb..da51d71 100644 --- a/larray-buffer/src/main/java/xerial/larray/buffer/LBufferAPI.java +++ b/larray-buffer/src/main/java/xerial/larray/buffer/LBufferAPI.java @@ -21,6 +21,7 @@ public class LBufferAPI { public LBufferAPI() { } + public LBufferAPI(Memory m) { this.m = m; } @@ -71,7 +72,7 @@ public void update(long offset, byte value) { * getXXX and putXXX methods becomes undefined. */ public void release() { - BufferConfig.allocator.release(m); + LBufferConfig.allocator.release(m); m = null; } @@ -85,14 +86,27 @@ public long address() { return m.address(); } + /** + * Size of this buffer + * @return + */ public long size() { return m.dataSize(); } + /** + * Clear the buffer by filling with zeros + */ public void clear() { fill(0, size(), (byte) 0); } + /** + * Fill the buffer of the specified range with a given value + * @param offset + * @param length + * @param value + */ public void fill(long offset, long length, byte value) { unsafe.setMemory(address() + offset, length, value); } @@ -210,6 +224,13 @@ public void putDouble(long offset, double value) { } + /** + * Copy the contents of this buffer begginning from the srcOffset to a destination byte array + * @param srcOffset + * @param destArray + * @param destOffset + * @param size + */ public void copyTo(int srcOffset, byte[] destArray, int destOffset, int size) { int cursor = destOffset; for (ByteBuffer bb : toDirectByteBuffers(srcOffset, size)) { @@ -221,10 +242,23 @@ public void copyTo(int srcOffset, byte[] destArray, int destOffset, int size) { } } - public void copyTo(long srcOffset, LBuffer dest, long destOffset, long size) { + /** + * Copy the contents of this buffer to the destination LBuffer + * @param srcOffset + * @param dest + * @param destOffset + * @param size + */ + public void copyTo(long srcOffset, LBufferAPI dest, long destOffset, long size) { unsafe.copyMemory(address() + srcOffset, dest.address() + destOffset, size); } + /** + * Extract a slice [from, to) of this buffer. This methods creates a copy of the specified region. + * @param from + * @param to + * @return + */ public LBuffer slice(long from, long to) { if(from > to) throw new IllegalArgumentException(String.format("invalid range %,d to %,d", from, to)); @@ -235,6 +269,24 @@ public LBuffer slice(long from, long to) { return b; } + /** + * Create a view of the range [from, to) of this buffer. Unlike slice(from, to), the generated view + * is a reference to this buffer. + * @param from + * @param to + * @return + */ + public WrappedLBuffer view(long from, long to) { + if(from > to) + throw new IllegalArgumentException(String.format("invalid range %,d to %,d", from, to)); + + return new WrappedLBuffer(m, from, to - from); + } + + /** + * Convert this buffer to a java array. + * @return + */ public byte[] toArray() { if (size() > Integer.MAX_VALUE) throw new IllegalStateException("Cannot create byte array of more than 2GB"); @@ -247,10 +299,21 @@ public byte[] toArray() { return b; } + /** + * Write the buffer contents to the given file channel. This method just + * calls channel.write(this.toDirectByteBuffers()); + * @param channel + * @throws IOException + */ public void writeTo(FileChannel channel) throws IOException { channel.write(toDirectByteBuffers()); } + /** + * Dump the buffer contents to a file + * @param file + * @throws IOException + */ public void writeTo(File file) throws IOException { FileChannel channel = new FileOutputStream(file).getChannel(); try { @@ -260,10 +323,24 @@ public void writeTo(File file) throws IOException { } } + /** + * Read the given source byte array, then overwrite the buffer contents + * @param src + * @param destOffset + * @return + */ public int readFrom(byte[] src, long destOffset) { return readFrom(src, 0, destOffset, src.length); } + /** + * Read the given source byte arrey, then overwrite the buffer contents + * @param src + * @param srcOffset + * @param destOffset + * @param length + * @return + */ public int readFrom(byte[] src, int srcOffset, long destOffset, int length) { int readLen = (int) Math.min(src.length - srcOffset, Math.min(size() - destOffset, length)); ByteBuffer b = toDirectByteBuffer(destOffset, readLen); @@ -273,6 +350,12 @@ public int readFrom(byte[] src, int srcOffset, long destOffset, int length) { } + /** + * Create an LBuffer from a given file. + * @param file + * @return + * @throws IOException + */ public static LBuffer loadFrom(File file) throws IOException { FileChannel fin = new FileInputStream(file).getChannel(); long fileSize = fin.size(); @@ -288,10 +371,20 @@ public static LBuffer loadFrom(File file) throws IOException { } + /** + * Gives an sequence of ByteBuffers. Writing to these ByteBuffers modifies the contents of this LBuffer. + * @return + */ public ByteBuffer[] toDirectByteBuffers() { return toDirectByteBuffers(0, size()); } + /** + * Gives an sequence of ByteBuffers of a specified range. Writing to these ByteBuffers modifies the contents of this LBuffer. + * @param offset + * @param size + * @return + */ public ByteBuffer[] toDirectByteBuffers(long offset, long size) { long pos = offset; long blockSize = Integer.MAX_VALUE; @@ -308,6 +401,12 @@ public ByteBuffer[] toDirectByteBuffers(long offset, long size) { } + /** + * Gives a ByteBuffer view of the specified range. Writing to the returned ByteBuffer modifies the contenets of this LByteBuffer + * @param offset + * @param size + * @return + */ public ByteBuffer toDirectByteBuffer(long offset, int size) { return UnsafeUtil.newDirectByteBuffer(address() + offset, size); } diff --git a/larray-buffer/src/main/java/xerial/larray/buffer/LBufferConfig.java b/larray-buffer/src/main/java/xerial/larray/buffer/LBufferConfig.java new file mode 100644 index 0000000..533ac9f --- /dev/null +++ b/larray-buffer/src/main/java/xerial/larray/buffer/LBufferConfig.java @@ -0,0 +1,10 @@ +package xerial.larray.buffer; + +/** + * Holding the default memory allocator + * @author Taro L. Saito + */ +public class LBufferConfig { + + public static MemoryCollector allocator = new DefaultMemoryCollector(); +} diff --git a/larray-buffer/src/main/java/xerial/larray/buffer/MemoryAllocator.java b/larray-buffer/src/main/java/xerial/larray/buffer/MemoryAllocator.java deleted file mode 100644 index 472d0dd..0000000 --- a/larray-buffer/src/main/java/xerial/larray/buffer/MemoryAllocator.java +++ /dev/null @@ -1,29 +0,0 @@ -package xerial.larray.buffer; - -/** - * Memory allocator interface - * @author Taro L. Saito - */ -public interface MemoryAllocator { - - /** - * Allocate a memory of the specified byte length. The allocated memory must be released via `release` - * as in malloc() in C/C++. - * @param size byte length of the memory - * @return allocated memory information - */ - Memory allocate(long size); - - void register(Memory m); - - /** - * Release the memory allocated by allocate(size). - */ - void release(Memory m); - - /** - * Release the memory allocated by allocate(size). - */ - void release(MemoryReference ref); - -} diff --git a/larray-buffer/src/main/java/xerial/larray/buffer/MemoryCollector.java b/larray-buffer/src/main/java/xerial/larray/buffer/MemoryCollector.java new file mode 100644 index 0000000..3054f34 --- /dev/null +++ b/larray-buffer/src/main/java/xerial/larray/buffer/MemoryCollector.java @@ -0,0 +1,25 @@ +package xerial.larray.buffer; + +/** + * Memory collector interface + * @author Taro L. Saito + */ +public interface MemoryCollector { + + /** + * Register a memory + * @param m + */ + void register(Memory m); + + /** + * Release a memory + */ + void release(Memory m); + + /** + * Release a memory, referenced by ref + */ + void release(MemoryReference ref); + +} diff --git a/larray-buffer/src/main/java/xerial/larray/buffer/OffHeapMemory.java b/larray-buffer/src/main/java/xerial/larray/buffer/OffHeapMemory.java new file mode 100644 index 0000000..46bd786 --- /dev/null +++ b/larray-buffer/src/main/java/xerial/larray/buffer/OffHeapMemory.java @@ -0,0 +1,84 @@ +package xerial.larray.buffer; + +import java.lang.ref.ReferenceQueue; + +import static xerial.larray.buffer.UnsafeUtil.unsafe; + +/** + * Stores |(memory size:long)| data ... | + */ +public class OffHeapMemory implements Memory { + + private final long _data; + + public static long HEADER_SIZE = 8L; + + /** + * Create an empty memory + */ + public OffHeapMemory() { + this._data = 0L; + } + + public OffHeapMemory(long address) { + if(address != 0L) + this._data = address + HEADER_SIZE; + else + this._data = 0L; + } + + public OffHeapMemory(long address, long size) { + if(address != 0L) { + this._data = address + HEADER_SIZE; + unsafe.putLong(address, size); + } + else { + this._data = 0L; + } + } + + /** + * Allocate a memory of the specified byte length. The allocated memory must be released via `release` + * as in malloc() in C/C++. + * @param size byte length of the memory + * @return allocated memory information + */ + public static Memory allocate(long size) { + if(size == 0L) + return new OffHeapMemory(); + + // Allocate memory of the given size + HEADER space + long memorySize = size + OffHeapMemory.HEADER_SIZE; + long address = unsafe.allocateMemory(memorySize); + Memory m = new OffHeapMemory(address, size); + LBufferConfig.allocator.register(m); + return m; + } + + + public long headerAddress() { + return _data - HEADER_SIZE; + } + public long size() { + return (_data == 0) ? 0L : unsafe.getLong(headerAddress()) + HEADER_SIZE; + } + + public long address() { + return _data; + } + + public long dataSize() { + return (_data == 0) ? 0L : unsafe.getLong(headerAddress()); + } + + public MemoryReference toRef(ReferenceQueue queue) { + return new OffHeapMemoryReference(this, queue); + } + + public void release() { + if(_data != 0) + UnsafeUtil.unsafe.freeMemory(headerAddress()); + } +} + + diff --git a/larray-buffer/src/main/java/xerial/larray/buffer/OffHeapMemoryReference.java b/larray-buffer/src/main/java/xerial/larray/buffer/OffHeapMemoryReference.java new file mode 100644 index 0000000..8900e0c --- /dev/null +++ b/larray-buffer/src/main/java/xerial/larray/buffer/OffHeapMemoryReference.java @@ -0,0 +1,28 @@ +package xerial.larray.buffer; + +import java.lang.ref.ReferenceQueue; + +/** + * @author Taro L. Saito + */ +public class OffHeapMemoryReference extends MemoryReference { + + /** + * Create a phantom reference + * @param m the allocated memory + * @param queue the reference queue to which GCed reference of the Memory will be inserted + */ + public OffHeapMemoryReference(Memory m, ReferenceQueue queue) { + super(m, queue); + } + + public Memory toMemory() { + if(address != 0) + return new OffHeapMemory(address); + else + return new OffHeapMemory(); + } + + public String name() { return "off-heap"; } + +} diff --git a/larray-buffer/src/main/java/xerial/larray/buffer/WrappedLBuffer.java b/larray-buffer/src/main/java/xerial/larray/buffer/WrappedLBuffer.java new file mode 100644 index 0000000..58ba96b --- /dev/null +++ b/larray-buffer/src/main/java/xerial/larray/buffer/WrappedLBuffer.java @@ -0,0 +1,28 @@ +package xerial.larray.buffer; + +/** + * A subrange of memory + * + * @author Taro L. Saito + */ +public class WrappedLBuffer extends LBufferAPI { + + private final long offset; + private final long size; + + public WrappedLBuffer(Memory m, long offset, long size) { + super(m); + this.offset = offset; + this.size = size; + } + + @Override + public long address() { + return m.address() + offset; + } + + @Override + public long size() { + return size; + } +} diff --git a/larray-buffer/src/test/scala/xerial/larray/buffer/MemoryAllocatorTest.scala b/larray-buffer/src/test/scala/xerial/larray/buffer/MemoryCollectorTest.scala similarity index 95% rename from larray-buffer/src/test/scala/xerial/larray/buffer/MemoryAllocatorTest.scala rename to larray-buffer/src/test/scala/xerial/larray/buffer/MemoryCollectorTest.scala index a743258..d6fc84a 100644 --- a/larray-buffer/src/test/scala/xerial/larray/buffer/MemoryAllocatorTest.scala +++ b/larray-buffer/src/test/scala/xerial/larray/buffer/MemoryCollectorTest.scala @@ -15,7 +15,7 @@ *--------------------------------------------------------------------------*/ //-------------------------------------- // -// MemoryAllocatorTest.scala +// MemoryCollectorTest.scala // Since: 2013/03/22 12:02 // //-------------------------------------- @@ -28,7 +28,7 @@ import xerial.larray.LArraySpec /** * @author Taro L. Saito */ -class MemoryAllocatorTest extends LArraySpec { +class MemoryCollectorTest extends LArraySpec { "ConcurrentMemoryAllocator" should { "perform better than the default heap allocator" in { diff --git a/larray-buffer/src/test/scala/xerial/larray/buffer/WrappedLBufferTest.scala b/larray-buffer/src/test/scala/xerial/larray/buffer/WrappedLBufferTest.scala new file mode 100644 index 0000000..5cd8093 --- /dev/null +++ b/larray-buffer/src/test/scala/xerial/larray/buffer/WrappedLBufferTest.scala @@ -0,0 +1,34 @@ +//-------------------------------------- +// +// WrappedLBufferTest.scala +// Since: 2013/12/11 23:07 +// +//-------------------------------------- + +package xerial.larray.buffer + +import xerial.larray.LArraySpec + +/** + * @author Taro L. Saito + */ +class WrappedLBufferTest extends LArraySpec { + + "WrappedLBuffer" should { + + "be a subrange of LBuffer" in { + val l = new LBuffer(10) + for(i <- 0 until l.size().toInt) { + l(i) = (10 - i).toByte + } + + debug(l.toArray.mkString(", ")) + val v = l.view(3, 8) + + debug(v.toArray.mkString(", ")) + v.size() shouldBe 8 - 3 + v.toArray.zipWithIndex.forall{case (a, i) => a == l(i+3)} + } + + } +} \ No newline at end of file diff --git a/larray-mmap/src/main/java/xerial/larray/mmap/MMapBuffer.java b/larray-mmap/src/main/java/xerial/larray/mmap/MMapBuffer.java index 477ae44..b8f5834 100644 --- a/larray-mmap/src/main/java/xerial/larray/mmap/MMapBuffer.java +++ b/larray-mmap/src/main/java/xerial/larray/mmap/MMapBuffer.java @@ -1,7 +1,7 @@ package xerial.larray.mmap; import sun.misc.SharedSecrets; -import xerial.larray.buffer.BufferConfig; +import xerial.larray.buffer.LBufferConfig; import xerial.larray.buffer.LBufferAPI; import xerial.larray.buffer.UnsafeUtil; import xerial.larray.impl.LArrayNative; @@ -101,7 +101,7 @@ public MMapBuffer(File f, long offset, long size, MMapMode mode) throws IOExcept } this.m = new MMapMemory(rawAddr, mapSize); - BufferConfig.allocator.register(m); + LBufferConfig.allocator.register(m); this.address = rawAddr + pagePosition; } diff --git a/larray/src/main/java/xerial/larray/japi/LArrayJ.java b/larray/src/main/java/xerial/larray/japi/LArrayJ.java index 35e518b..6d2ee42 100644 --- a/larray/src/main/java/xerial/larray/japi/LArrayJ.java +++ b/larray/src/main/java/xerial/larray/japi/LArrayJ.java @@ -17,7 +17,8 @@ import scala.reflect.ClassTag$; import xerial.larray.*; -import xerial.larray.buffer.BufferConfig; +import xerial.larray.buffer.LBufferConfig; +import xerial.larray.buffer.MemoryCollector; import xerial.larray.mmap.MMapMode; import java.io.File; @@ -28,7 +29,7 @@ */ public class LArrayJ { - static xerial.larray.buffer.MemoryAllocator defaultAllocator() { return BufferConfig.allocator; } + static MemoryCollector defaultAllocator() { return LBufferConfig.allocator; } public static MappedLByteArray mmap(File f, MMapMode mode) { return new MappedLByteArray(f, 0L, f.length(), mode, defaultAllocator()); diff --git a/larray/src/main/scala/xerial/larray/LArray.scala b/larray/src/main/scala/xerial/larray/LArray.scala index 2cd6419..601c641 100644 --- a/larray/src/main/scala/xerial/larray/LArray.scala +++ b/larray/src/main/scala/xerial/larray/LArray.scala @@ -28,7 +28,7 @@ import java.nio.{ByteOrder, ByteBuffer} import java.nio.channels.{FileChannel, WritableByteChannel} import sun.nio.ch.DirectBuffer import java.io.{FileInputStream, FileOutputStream, File} -import xerial.larray.buffer.{Memory, MemoryAllocator} +import xerial.larray.buffer.{Memory, MemoryCollector} import xerial.larray.mmap.MMapMode @@ -743,7 +743,7 @@ private[larray] trait UnsafeArray[T] extends RawByteArray[T] with Logger { private[larray] def m: Memory def address = m.address - private[larray] def alloc :MemoryAllocator + private[larray] def alloc :MemoryCollector /** * Release the memory of LArray. After calling this method, the results of calling the behavior of the other methods becomes undefined or might cause JVM crash. @@ -754,12 +754,12 @@ private[larray] trait UnsafeArray[T] extends RawByteArray[T] with Logger { } -class LCharArray(val size: Long, private[larray] val m: Memory)(implicit val alloc: MemoryAllocator) +class LCharArray(val size: Long, private[larray] val m: Memory)(implicit val alloc: MemoryCollector) extends LArray[Char] with UnsafeArray[Char] { protected[this] def newBuilder = new LCharArrayBuilder - def this(size: Long)(implicit alloc: MemoryAllocator) = this(size, alloc.allocate(size << 1))(alloc) + def this(size: Long)(implicit alloc: MemoryCollector) = this(size, alloc.allocate(size << 1))(alloc) import UnsafeUtil.unsafe @@ -788,12 +788,12 @@ class LCharArray(val size: Long, private[larray] val m: Memory)(implicit val all * @param m allocated memory * @param alloc memory allocator */ -class LIntArray(val size: Long, private[larray] val m: Memory)(implicit val alloc: MemoryAllocator) +class LIntArray(val size: Long, private[larray] val m: Memory)(implicit val alloc: MemoryCollector) extends LArray[Int] with UnsafeArray[Int] { protected[this] def newBuilder = new LIntArrayBuilder - def this(size: Long)(implicit alloc: MemoryAllocator) = this(size, alloc.allocate(size << 2))(alloc) + def this(size: Long)(implicit alloc: MemoryCollector) = this(size, alloc.allocate(size << 2))(alloc) import UnsafeUtil.unsafe @@ -822,10 +822,10 @@ class LIntArray(val size: Long, private[larray] val m: Memory)(implicit val allo * @param m allocated memory * @param alloc memory allocator */ -class LLongArray(val size: Long, private[larray] val m: Memory)(implicit val alloc: MemoryAllocator) +class LLongArray(val size: Long, private[larray] val m: Memory)(implicit val alloc: MemoryCollector) extends LArray[Long] with UnsafeArray[Long] { - def this(size: Long)(implicit alloc: MemoryAllocator) = this(size, alloc.allocate(size << 3))(alloc) + def this(size: Long)(implicit alloc: MemoryCollector) = this(size, alloc.allocate(size << 3))(alloc) protected[this] def newBuilder = new LLongArrayBuilder @@ -854,12 +854,12 @@ class LLongArray(val size: Long, private[larray] val m: Memory)(implicit val all * @param m allocated memory * @param alloc memory allocator */ -class LByteArray(val size: Long, private[larray] val m: Memory)(implicit val alloc: MemoryAllocator) +class LByteArray(val size: Long, private[larray] val m: Memory)(implicit val alloc: MemoryCollector) extends LArray[Byte] with UnsafeArray[Byte] { self => - def this(size: Long)(implicit alloc: MemoryAllocator) = this(size, alloc.allocate(size))(alloc) + def this(size: Long)(implicit alloc: MemoryCollector) = this(size, alloc.allocate(size))(alloc) protected[this] def newBuilder = new LByteArrayBuilder @@ -943,10 +943,10 @@ class LByteArray(val size: Long, private[larray] val m: Memory)(implicit val all } -class LDoubleArray(val size: Long, private[larray] val m: Memory)(implicit val alloc: MemoryAllocator) +class LDoubleArray(val size: Long, private[larray] val m: Memory)(implicit val alloc: MemoryCollector) extends LArray[Double] with UnsafeArray[Double] { - def this(size: Long)(implicit alloc: MemoryAllocator) = this(size, alloc.allocate(size << 3))(alloc) + def this(size: Long)(implicit alloc: MemoryCollector) = this(size, alloc.allocate(size << 3))(alloc) private[larray] def elementByteSize = 8 @@ -968,10 +968,10 @@ class LDoubleArray(val size: Long, private[larray] val m: Memory)(implicit val a def view(from: Long, to: Long) = new LArrayView.LDoubleArrayView(this, from, to - from) } -class LFloatArray(val size: Long, private[larray] val m: Memory)(implicit val alloc: MemoryAllocator) +class LFloatArray(val size: Long, private[larray] val m: Memory)(implicit val alloc: MemoryCollector) extends LArray[Float] with UnsafeArray[Float] { - def this(size: Long)(implicit alloc: MemoryAllocator) = this(size, alloc.allocate(size << 2))(alloc) + def this(size: Long)(implicit alloc: MemoryCollector) = this(size, alloc.allocate(size << 2))(alloc) private[larray] def elementByteSize = 4 @@ -992,10 +992,10 @@ class LFloatArray(val size: Long, private[larray] val m: Memory)(implicit val al def view(from: Long, to: Long) = new LArrayView.LFloatArrayView(this, from, to - from) } -class LShortArray(val size: Long, private[larray] val m: Memory)(implicit val alloc: MemoryAllocator) +class LShortArray(val size: Long, private[larray] val m: Memory)(implicit val alloc: MemoryCollector) extends LArray[Short] with UnsafeArray[Short] { - def this(size: Long)(implicit alloc: MemoryAllocator) = this(size, alloc.allocate(size << 1))(alloc) + def this(size: Long)(implicit alloc: MemoryCollector) = this(size, alloc.allocate(size << 1))(alloc) private[larray] def elementByteSize = 2 diff --git a/larray/src/main/scala/xerial/larray/MappedLByteArray.scala b/larray/src/main/scala/xerial/larray/MappedLByteArray.scala index 985d8c4..c9cf164 100644 --- a/larray/src/main/scala/xerial/larray/MappedLByteArray.scala +++ b/larray/src/main/scala/xerial/larray/MappedLByteArray.scala @@ -25,7 +25,7 @@ package xerial.larray import java.io.{IOException, FileDescriptor, RandomAccessFile, File} import java.lang.reflect.InvocationTargetException import xerial.larray._ -import xerial.larray.buffer.{Memory, MemoryAllocator} +import xerial.larray.buffer.{Memory, MemoryCollector} import xerial.larray.mmap.{MMapMemory, MMapMode, MMapBuffer} import sun.misc.SharedSecrets import sun.awt.OSInfo @@ -36,7 +36,7 @@ import sun.awt.OSInfo * Memory-mapped LByteArray * @author Taro L. Saito */ -class MappedLByteArray(f:File, offset:Long = 0, val size:Long = -1, mode:MMapMode=MMapMode.READ_WRITE)(implicit alloc:MemoryAllocator) extends RawByteArray[Byte] { +class MappedLByteArray(f:File, offset:Long = 0, val size:Long = -1, mode:MMapMode=MMapMode.READ_WRITE)(implicit alloc:MemoryCollector) extends RawByteArray[Byte] { import UnsafeUtil.unsafe import java.{lang=>jl} diff --git a/larray/src/main/scala/xerial/larray/UInt32Array.scala b/larray/src/main/scala/xerial/larray/UInt32Array.scala index 5192488..af59b5f 100644 --- a/larray/src/main/scala/xerial/larray/UInt32Array.scala +++ b/larray/src/main/scala/xerial/larray/UInt32Array.scala @@ -22,7 +22,7 @@ package xerial.larray -import xerial.larray.buffer.{MemoryAllocator, Memory} +import xerial.larray.buffer.{MemoryCollector, Memory} object UInt32Array { @@ -67,8 +67,8 @@ private[larray] class UInt32ArrayView(base:UInt32Array, offset:Long, val size:Lo * * @author Taro L. Saito */ -class UInt32Array(val size: Long, private[larray] val m:Memory)(implicit val alloc: MemoryAllocator) extends LArray[Long] with UnsafeArray[Long] { self => - def this(size:Long)(implicit alloc: MemoryAllocator) = this(size, alloc.allocate(size << 2))(alloc) +class UInt32Array(val size: Long, private[larray] val m:Memory)(implicit val alloc: MemoryCollector) extends LArray[Long] with UnsafeArray[Long] { self => + def this(size:Long)(implicit alloc: MemoryCollector) = this(size, alloc.allocate(size << 2))(alloc) import UnsafeUtil.unsafe diff --git a/larray/src/main/scala/xerial/larray/package.scala b/larray/src/main/scala/xerial/larray/package.scala index 2febfa0..893f4a8 100644 --- a/larray/src/main/scala/xerial/larray/package.scala +++ b/larray/src/main/scala/xerial/larray/package.scala @@ -16,7 +16,7 @@ package xerial import reflect.ClassTag -import xerial.larray.buffer.BufferConfig +import xerial.larray.buffer.LBufferConfig /** * == LArray == @@ -48,7 +48,7 @@ import xerial.larray.buffer.BufferConfig */ package object larray { - implicit def defaultAllocator : xerial.larray.buffer.MemoryAllocator = BufferConfig.allocator + implicit def defaultAllocator : xerial.larray.buffer.MemoryCollector = LBufferConfig.allocator implicit class ConvertArrayToLArray[A : ClassTag](arr:Array[A]) { diff --git a/larray/src/test/scala/xerial/larray/LArrayTest.scala b/larray/src/test/scala/xerial/larray/LArrayTest.scala index 28476a8..e5e1dec 100644 --- a/larray/src/test/scala/xerial/larray/LArrayTest.scala +++ b/larray/src/test/scala/xerial/larray/LArrayTest.scala @@ -33,7 +33,7 @@ class LArrayTest extends LArraySpec { val G: Long = 1024L * 1024L * 1024L override def afterEach { - // MemoryAllocator.default.releaseAll + // MemoryCollector.default.releaseAll // System.gc() } From 8fdb4bbfef14c91c7c05122ceb6518387238a5c1 Mon Sep 17 00:00:00 2001 From: "Taro L. Saito" Date: Wed, 11 Dec 2013 23:40:46 +0900 Subject: [PATCH 12/13] MemoryCollector -> MemoryAllocator --- ...ector.java => DefaultMemoryAllocator.java} | 25 +++++++++++--- .../java/xerial/larray/buffer/LBuffer.java | 2 +- .../xerial/larray/buffer/LBufferConfig.java | 2 +- .../xerial/larray/buffer/MemoryAllocator.java | 33 +++++++++++++++++++ .../xerial/larray/buffer/MemoryCollector.java | 25 -------------- .../xerial/larray/buffer/OffHeapMemory.java | 17 ---------- ...orTest.scala => MemoryAllocatorTest.scala} | 4 +-- .../main/java/xerial/larray/japi/LArrayJ.java | 4 +-- .../src/main/scala/xerial/larray/LArray.scala | 32 +++++++++--------- .../xerial/larray/MappedLByteArray.scala | 4 +-- .../scala/xerial/larray/UInt32Array.scala | 6 ++-- .../main/scala/xerial/larray/package.scala | 2 +- .../test/scala/xerial/larray/LArrayTest.scala | 2 +- 13 files changed, 82 insertions(+), 76 deletions(-) rename larray-buffer/src/main/java/xerial/larray/buffer/{DefaultMemoryCollector.java => DefaultMemoryAllocator.java} (77%) create mode 100644 larray-buffer/src/main/java/xerial/larray/buffer/MemoryAllocator.java delete mode 100644 larray-buffer/src/main/java/xerial/larray/buffer/MemoryCollector.java rename larray-buffer/src/test/scala/xerial/larray/buffer/{MemoryCollectorTest.scala => MemoryAllocatorTest.scala} (95%) diff --git a/larray-buffer/src/main/java/xerial/larray/buffer/DefaultMemoryCollector.java b/larray-buffer/src/main/java/xerial/larray/buffer/DefaultMemoryAllocator.java similarity index 77% rename from larray-buffer/src/main/java/xerial/larray/buffer/DefaultMemoryCollector.java rename to larray-buffer/src/main/java/xerial/larray/buffer/DefaultMemoryAllocator.java index 7390bbc..a6d3662 100644 --- a/larray-buffer/src/main/java/xerial/larray/buffer/DefaultMemoryCollector.java +++ b/larray-buffer/src/main/java/xerial/larray/buffer/DefaultMemoryAllocator.java @@ -10,16 +10,14 @@ - - /** - * A default implementation of MemoryCollector that releases allocated memories in a background thread. + * A default implementation of MemoryAllocator that allocates off-heap memory and releases allocated memories in a background thread. * * @author Taro L. Saito */ -public class DefaultMemoryCollector implements MemoryCollector { +public class DefaultMemoryAllocator implements MemoryAllocator { - private Logger logger = Logger.getLogger(DefaultMemoryCollector.class); + private Logger logger = Logger.getLogger(DefaultMemoryAllocator.class); // Table from address -> MemoryReference @@ -59,6 +57,23 @@ public void run() { */ public long allocatedSize() { return totalAllocatedSize.get(); } + /** + * Allocate a memory of the specified byte length. The allocated memory must be released via `release` + * as in malloc() in C/C++. + * @param size byte length of the memory + * @return allocated memory information + */ + public Memory allocate(long size) { + if(size == 0L) + return new OffHeapMemory(); + + // Allocate memory of the given size + HEADER space + long memorySize = size + OffHeapMemory.HEADER_SIZE; + long address = UnsafeUtil.unsafe.allocateMemory(memorySize); + Memory m = new OffHeapMemory(address, size); + register(m); + return m; + } public void register(Memory m) { // Register a memory reference that will be collected upon GC diff --git a/larray-buffer/src/main/java/xerial/larray/buffer/LBuffer.java b/larray-buffer/src/main/java/xerial/larray/buffer/LBuffer.java index ec5a98d..f2c310d 100644 --- a/larray-buffer/src/main/java/xerial/larray/buffer/LBuffer.java +++ b/larray-buffer/src/main/java/xerial/larray/buffer/LBuffer.java @@ -14,7 +14,7 @@ public class LBuffer extends LBufferAPI { * @param size byte size of the array */ public LBuffer(long size) { - super(OffHeapMemory.allocate(size)); + super(LBufferConfig.allocator.allocate(size)); } diff --git a/larray-buffer/src/main/java/xerial/larray/buffer/LBufferConfig.java b/larray-buffer/src/main/java/xerial/larray/buffer/LBufferConfig.java index 533ac9f..2ee7441 100644 --- a/larray-buffer/src/main/java/xerial/larray/buffer/LBufferConfig.java +++ b/larray-buffer/src/main/java/xerial/larray/buffer/LBufferConfig.java @@ -6,5 +6,5 @@ */ public class LBufferConfig { - public static MemoryCollector allocator = new DefaultMemoryCollector(); + public static MemoryAllocator allocator = new DefaultMemoryAllocator(); } diff --git a/larray-buffer/src/main/java/xerial/larray/buffer/MemoryAllocator.java b/larray-buffer/src/main/java/xerial/larray/buffer/MemoryAllocator.java new file mode 100644 index 0000000..0b4ddd2 --- /dev/null +++ b/larray-buffer/src/main/java/xerial/larray/buffer/MemoryAllocator.java @@ -0,0 +1,33 @@ +package xerial.larray.buffer; + +/** + * Memory allocator interface + * @author Taro L. Saito + */ +public interface MemoryAllocator { + + /** + * Allocate a memory of the specified byte length. The allocated memory must be released via `release` + * as in malloc() in C/C++. + * @param size byte length of the memory + * @return allocated memory information + */ + Memory allocate(long size); + + /** + * Register a memory + * @param m + */ + void register(Memory m); + + /** + * Release a memory + */ + void release(Memory m); + + /** + * Release a memory, referenced by ref + */ + void release(MemoryReference ref); + +} diff --git a/larray-buffer/src/main/java/xerial/larray/buffer/MemoryCollector.java b/larray-buffer/src/main/java/xerial/larray/buffer/MemoryCollector.java deleted file mode 100644 index 3054f34..0000000 --- a/larray-buffer/src/main/java/xerial/larray/buffer/MemoryCollector.java +++ /dev/null @@ -1,25 +0,0 @@ -package xerial.larray.buffer; - -/** - * Memory collector interface - * @author Taro L. Saito - */ -public interface MemoryCollector { - - /** - * Register a memory - * @param m - */ - void register(Memory m); - - /** - * Release a memory - */ - void release(Memory m); - - /** - * Release a memory, referenced by ref - */ - void release(MemoryReference ref); - -} diff --git a/larray-buffer/src/main/java/xerial/larray/buffer/OffHeapMemory.java b/larray-buffer/src/main/java/xerial/larray/buffer/OffHeapMemory.java index 46bd786..b9bad84 100644 --- a/larray-buffer/src/main/java/xerial/larray/buffer/OffHeapMemory.java +++ b/larray-buffer/src/main/java/xerial/larray/buffer/OffHeapMemory.java @@ -37,23 +37,6 @@ public OffHeapMemory(long address, long size) { } } - /** - * Allocate a memory of the specified byte length. The allocated memory must be released via `release` - * as in malloc() in C/C++. - * @param size byte length of the memory - * @return allocated memory information - */ - public static Memory allocate(long size) { - if(size == 0L) - return new OffHeapMemory(); - - // Allocate memory of the given size + HEADER space - long memorySize = size + OffHeapMemory.HEADER_SIZE; - long address = unsafe.allocateMemory(memorySize); - Memory m = new OffHeapMemory(address, size); - LBufferConfig.allocator.register(m); - return m; - } public long headerAddress() { diff --git a/larray-buffer/src/test/scala/xerial/larray/buffer/MemoryCollectorTest.scala b/larray-buffer/src/test/scala/xerial/larray/buffer/MemoryAllocatorTest.scala similarity index 95% rename from larray-buffer/src/test/scala/xerial/larray/buffer/MemoryCollectorTest.scala rename to larray-buffer/src/test/scala/xerial/larray/buffer/MemoryAllocatorTest.scala index d6fc84a..a743258 100644 --- a/larray-buffer/src/test/scala/xerial/larray/buffer/MemoryCollectorTest.scala +++ b/larray-buffer/src/test/scala/xerial/larray/buffer/MemoryAllocatorTest.scala @@ -15,7 +15,7 @@ *--------------------------------------------------------------------------*/ //-------------------------------------- // -// MemoryCollectorTest.scala +// MemoryAllocatorTest.scala // Since: 2013/03/22 12:02 // //-------------------------------------- @@ -28,7 +28,7 @@ import xerial.larray.LArraySpec /** * @author Taro L. Saito */ -class MemoryCollectorTest extends LArraySpec { +class MemoryAllocatorTest extends LArraySpec { "ConcurrentMemoryAllocator" should { "perform better than the default heap allocator" in { diff --git a/larray/src/main/java/xerial/larray/japi/LArrayJ.java b/larray/src/main/java/xerial/larray/japi/LArrayJ.java index 6d2ee42..87f40ab 100644 --- a/larray/src/main/java/xerial/larray/japi/LArrayJ.java +++ b/larray/src/main/java/xerial/larray/japi/LArrayJ.java @@ -18,7 +18,7 @@ import scala.reflect.ClassTag$; import xerial.larray.*; import xerial.larray.buffer.LBufferConfig; -import xerial.larray.buffer.MemoryCollector; +import xerial.larray.buffer.MemoryAllocator; import xerial.larray.mmap.MMapMode; import java.io.File; @@ -29,7 +29,7 @@ */ public class LArrayJ { - static MemoryCollector defaultAllocator() { return LBufferConfig.allocator; } + static MemoryAllocator defaultAllocator() { return LBufferConfig.allocator; } public static MappedLByteArray mmap(File f, MMapMode mode) { return new MappedLByteArray(f, 0L, f.length(), mode, defaultAllocator()); diff --git a/larray/src/main/scala/xerial/larray/LArray.scala b/larray/src/main/scala/xerial/larray/LArray.scala index 601c641..2cd6419 100644 --- a/larray/src/main/scala/xerial/larray/LArray.scala +++ b/larray/src/main/scala/xerial/larray/LArray.scala @@ -28,7 +28,7 @@ import java.nio.{ByteOrder, ByteBuffer} import java.nio.channels.{FileChannel, WritableByteChannel} import sun.nio.ch.DirectBuffer import java.io.{FileInputStream, FileOutputStream, File} -import xerial.larray.buffer.{Memory, MemoryCollector} +import xerial.larray.buffer.{Memory, MemoryAllocator} import xerial.larray.mmap.MMapMode @@ -743,7 +743,7 @@ private[larray] trait UnsafeArray[T] extends RawByteArray[T] with Logger { private[larray] def m: Memory def address = m.address - private[larray] def alloc :MemoryCollector + private[larray] def alloc :MemoryAllocator /** * Release the memory of LArray. After calling this method, the results of calling the behavior of the other methods becomes undefined or might cause JVM crash. @@ -754,12 +754,12 @@ private[larray] trait UnsafeArray[T] extends RawByteArray[T] with Logger { } -class LCharArray(val size: Long, private[larray] val m: Memory)(implicit val alloc: MemoryCollector) +class LCharArray(val size: Long, private[larray] val m: Memory)(implicit val alloc: MemoryAllocator) extends LArray[Char] with UnsafeArray[Char] { protected[this] def newBuilder = new LCharArrayBuilder - def this(size: Long)(implicit alloc: MemoryCollector) = this(size, alloc.allocate(size << 1))(alloc) + def this(size: Long)(implicit alloc: MemoryAllocator) = this(size, alloc.allocate(size << 1))(alloc) import UnsafeUtil.unsafe @@ -788,12 +788,12 @@ class LCharArray(val size: Long, private[larray] val m: Memory)(implicit val all * @param m allocated memory * @param alloc memory allocator */ -class LIntArray(val size: Long, private[larray] val m: Memory)(implicit val alloc: MemoryCollector) +class LIntArray(val size: Long, private[larray] val m: Memory)(implicit val alloc: MemoryAllocator) extends LArray[Int] with UnsafeArray[Int] { protected[this] def newBuilder = new LIntArrayBuilder - def this(size: Long)(implicit alloc: MemoryCollector) = this(size, alloc.allocate(size << 2))(alloc) + def this(size: Long)(implicit alloc: MemoryAllocator) = this(size, alloc.allocate(size << 2))(alloc) import UnsafeUtil.unsafe @@ -822,10 +822,10 @@ class LIntArray(val size: Long, private[larray] val m: Memory)(implicit val allo * @param m allocated memory * @param alloc memory allocator */ -class LLongArray(val size: Long, private[larray] val m: Memory)(implicit val alloc: MemoryCollector) +class LLongArray(val size: Long, private[larray] val m: Memory)(implicit val alloc: MemoryAllocator) extends LArray[Long] with UnsafeArray[Long] { - def this(size: Long)(implicit alloc: MemoryCollector) = this(size, alloc.allocate(size << 3))(alloc) + def this(size: Long)(implicit alloc: MemoryAllocator) = this(size, alloc.allocate(size << 3))(alloc) protected[this] def newBuilder = new LLongArrayBuilder @@ -854,12 +854,12 @@ class LLongArray(val size: Long, private[larray] val m: Memory)(implicit val all * @param m allocated memory * @param alloc memory allocator */ -class LByteArray(val size: Long, private[larray] val m: Memory)(implicit val alloc: MemoryCollector) +class LByteArray(val size: Long, private[larray] val m: Memory)(implicit val alloc: MemoryAllocator) extends LArray[Byte] with UnsafeArray[Byte] { self => - def this(size: Long)(implicit alloc: MemoryCollector) = this(size, alloc.allocate(size))(alloc) + def this(size: Long)(implicit alloc: MemoryAllocator) = this(size, alloc.allocate(size))(alloc) protected[this] def newBuilder = new LByteArrayBuilder @@ -943,10 +943,10 @@ class LByteArray(val size: Long, private[larray] val m: Memory)(implicit val all } -class LDoubleArray(val size: Long, private[larray] val m: Memory)(implicit val alloc: MemoryCollector) +class LDoubleArray(val size: Long, private[larray] val m: Memory)(implicit val alloc: MemoryAllocator) extends LArray[Double] with UnsafeArray[Double] { - def this(size: Long)(implicit alloc: MemoryCollector) = this(size, alloc.allocate(size << 3))(alloc) + def this(size: Long)(implicit alloc: MemoryAllocator) = this(size, alloc.allocate(size << 3))(alloc) private[larray] def elementByteSize = 8 @@ -968,10 +968,10 @@ class LDoubleArray(val size: Long, private[larray] val m: Memory)(implicit val a def view(from: Long, to: Long) = new LArrayView.LDoubleArrayView(this, from, to - from) } -class LFloatArray(val size: Long, private[larray] val m: Memory)(implicit val alloc: MemoryCollector) +class LFloatArray(val size: Long, private[larray] val m: Memory)(implicit val alloc: MemoryAllocator) extends LArray[Float] with UnsafeArray[Float] { - def this(size: Long)(implicit alloc: MemoryCollector) = this(size, alloc.allocate(size << 2))(alloc) + def this(size: Long)(implicit alloc: MemoryAllocator) = this(size, alloc.allocate(size << 2))(alloc) private[larray] def elementByteSize = 4 @@ -992,10 +992,10 @@ class LFloatArray(val size: Long, private[larray] val m: Memory)(implicit val al def view(from: Long, to: Long) = new LArrayView.LFloatArrayView(this, from, to - from) } -class LShortArray(val size: Long, private[larray] val m: Memory)(implicit val alloc: MemoryCollector) +class LShortArray(val size: Long, private[larray] val m: Memory)(implicit val alloc: MemoryAllocator) extends LArray[Short] with UnsafeArray[Short] { - def this(size: Long)(implicit alloc: MemoryCollector) = this(size, alloc.allocate(size << 1))(alloc) + def this(size: Long)(implicit alloc: MemoryAllocator) = this(size, alloc.allocate(size << 1))(alloc) private[larray] def elementByteSize = 2 diff --git a/larray/src/main/scala/xerial/larray/MappedLByteArray.scala b/larray/src/main/scala/xerial/larray/MappedLByteArray.scala index c9cf164..985d8c4 100644 --- a/larray/src/main/scala/xerial/larray/MappedLByteArray.scala +++ b/larray/src/main/scala/xerial/larray/MappedLByteArray.scala @@ -25,7 +25,7 @@ package xerial.larray import java.io.{IOException, FileDescriptor, RandomAccessFile, File} import java.lang.reflect.InvocationTargetException import xerial.larray._ -import xerial.larray.buffer.{Memory, MemoryCollector} +import xerial.larray.buffer.{Memory, MemoryAllocator} import xerial.larray.mmap.{MMapMemory, MMapMode, MMapBuffer} import sun.misc.SharedSecrets import sun.awt.OSInfo @@ -36,7 +36,7 @@ import sun.awt.OSInfo * Memory-mapped LByteArray * @author Taro L. Saito */ -class MappedLByteArray(f:File, offset:Long = 0, val size:Long = -1, mode:MMapMode=MMapMode.READ_WRITE)(implicit alloc:MemoryCollector) extends RawByteArray[Byte] { +class MappedLByteArray(f:File, offset:Long = 0, val size:Long = -1, mode:MMapMode=MMapMode.READ_WRITE)(implicit alloc:MemoryAllocator) extends RawByteArray[Byte] { import UnsafeUtil.unsafe import java.{lang=>jl} diff --git a/larray/src/main/scala/xerial/larray/UInt32Array.scala b/larray/src/main/scala/xerial/larray/UInt32Array.scala index af59b5f..5192488 100644 --- a/larray/src/main/scala/xerial/larray/UInt32Array.scala +++ b/larray/src/main/scala/xerial/larray/UInt32Array.scala @@ -22,7 +22,7 @@ package xerial.larray -import xerial.larray.buffer.{MemoryCollector, Memory} +import xerial.larray.buffer.{MemoryAllocator, Memory} object UInt32Array { @@ -67,8 +67,8 @@ private[larray] class UInt32ArrayView(base:UInt32Array, offset:Long, val size:Lo * * @author Taro L. Saito */ -class UInt32Array(val size: Long, private[larray] val m:Memory)(implicit val alloc: MemoryCollector) extends LArray[Long] with UnsafeArray[Long] { self => - def this(size:Long)(implicit alloc: MemoryCollector) = this(size, alloc.allocate(size << 2))(alloc) +class UInt32Array(val size: Long, private[larray] val m:Memory)(implicit val alloc: MemoryAllocator) extends LArray[Long] with UnsafeArray[Long] { self => + def this(size:Long)(implicit alloc: MemoryAllocator) = this(size, alloc.allocate(size << 2))(alloc) import UnsafeUtil.unsafe diff --git a/larray/src/main/scala/xerial/larray/package.scala b/larray/src/main/scala/xerial/larray/package.scala index 893f4a8..f58d60c 100644 --- a/larray/src/main/scala/xerial/larray/package.scala +++ b/larray/src/main/scala/xerial/larray/package.scala @@ -48,7 +48,7 @@ import xerial.larray.buffer.LBufferConfig */ package object larray { - implicit def defaultAllocator : xerial.larray.buffer.MemoryCollector = LBufferConfig.allocator + implicit def defaultAllocator : xerial.larray.buffer.MemoryAllocator = LBufferConfig.allocator implicit class ConvertArrayToLArray[A : ClassTag](arr:Array[A]) { diff --git a/larray/src/test/scala/xerial/larray/LArrayTest.scala b/larray/src/test/scala/xerial/larray/LArrayTest.scala index e5e1dec..28476a8 100644 --- a/larray/src/test/scala/xerial/larray/LArrayTest.scala +++ b/larray/src/test/scala/xerial/larray/LArrayTest.scala @@ -33,7 +33,7 @@ class LArrayTest extends LArraySpec { val G: Long = 1024L * 1024L * 1024L override def afterEach { - // MemoryCollector.default.releaseAll + // MemoryAllocator.default.releaseAll // System.gc() } From 0ca48eb7c7061c2755e4ea2f9faf8b5559ca6daf Mon Sep 17 00:00:00 2001 From: "Taro L. Saito" Date: Thu, 12 Dec 2013 00:05:44 +0900 Subject: [PATCH 13/13] Release note for 0.2.1 --- README.md | 13 +++++++------ version.sbt | 2 +- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 178f648..7085d0c 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ LArray === -A library for managing large off-heap arrays that can hold more than 2G (2^31) entries in Java and Scala. Notablly LArray is *disposable* by calling `LArray.free` or you can let GC automatically release the memory. LArray also can be used to create an `mmap` (memory-mapped file) whose size is more than 2GB. +A library for managing large off-heap arrays that can hold more than 2G (2^31) entries in Java and Scala. Notably LArray is *disposable* by calling `LArray.free` or you can let GC automatically release the memory. LArray also can be used to create an `mmap` (memory-mapped file) whose size is more than 2GB. ## Features * LArray can create arrays with more than 2G(2^31) entries. @@ -94,6 +94,7 @@ A standard JVM, (e.g. Oracle JVM (standard JVM, HotSpotVM) or OpenJDK) must be u ## History + * November 11, 2013 version 0.2.1 - Use orgnization name `org.xerial.larray`. Add LBuffer.view. * November 11, 2013 version 0.2 - Extracted pure-java modules (larray-buffer.jar and larray-mmap.jar) from larray.jar (for Scala). * August 28, 2013 version 0.1.2 - improved memory layout * August 28, 2013 version 0.1.1 (for Scala 2.10.2) @@ -105,7 +106,7 @@ A standard JVM, (e.g. Oracle JVM (standard JVM, HotSpotVM) or OpenJDK) must be u Add the following sbt dependency to your project settings: ```scala -libraryDependencies += "org.xerial" % "larray" % "0.2" +libraryDependencies += "org.xerial.larray" % "larray" % "0.2.1" ``` * Using snapshot versions: @@ -113,7 +114,7 @@ libraryDependencies += "org.xerial" % "larray" % "0.2" ```scala resolvers += "Sonatype shapshot repo" at "https://oss.sonatype.org/content/repositories/snapshots/" -libraryDependencies += "org.xerial" % "larray" % "0.2-SNAPSHOT" +libraryDependencies += "org.xerial.larray" % "larray" % "0.2.2-SNAPSHOT" ``` ### Example @@ -144,9 +145,9 @@ For more examples, see [xerial/larray/example/LArrayExample.scala](larray/src/ma Add the following dependency to your pom.xml (Maven): ```xml - org.xerial + org.xerial.larray larray - 0.2 + 0.2.1 ``` @@ -169,7 +170,7 @@ For more examples, see [xerial/larray/example/LArrayJavaExample.scala](larray/sr ## Scaladoc - * [LArray Scala API](https://oss.sonatype.org/service/local/repositories/releases/archive/org/xerial/larray/0.2/larray-0.2-javadoc.jar/!/index.html#xerial.larray.package) + * [LArray Scala API](https://oss.sonatype.org/service/local/repositories/releases/archive/org/xerial/larray/larray/0.2.1/larray-0.2.1-javadoc.jar/!/index.html#xerial.larray.package) ## For developers diff --git a/version.sbt b/version.sbt index d1c5964..43b1414 100755 --- a/version.sbt +++ b/version.sbt @@ -1,3 +1,3 @@ -version in ThisBuild := "0.2.1-SNAPSHOT" +version in ThisBuild := "0.2.1"