From 11797d8d179a7f745cbcb62ab3546e3d04b211e4 Mon Sep 17 00:00:00 2001 From: Lars-Magnus Skog Date: Thu, 6 Dec 2018 03:35:04 +0100 Subject: [PATCH 01/42] Implement leveldown function (test/leveldown-test.js pass) --- binding.gyp | 11 ++--------- napi/leveldown.cc | 14 ++++++++++++++ package.json | 5 +++-- 3 files changed, 19 insertions(+), 11 deletions(-) create mode 100644 napi/leveldown.cc diff --git a/binding.gyp b/binding.gyp index 525a1c5f..081fe687 100644 --- a/binding.gyp +++ b/binding.gyp @@ -39,17 +39,10 @@ "<(module_root_dir)/deps/leveldb/leveldb.gyp:leveldb" ], "include_dirs" : [ - " +#include + +// TODO add to napi-macros.h +#define NAPI_RETURN_UNDEFINED() \ + return 0; + +NAPI_METHOD(leveldown) { + NAPI_RETURN_UNDEFINED(); +} + +NAPI_INIT() { + NAPI_EXPORT_FUNCTION(leveldown); +} diff --git a/package.json b/package.json index 30668526..6cb50de4 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,8 @@ "abstract-leveldown": "~6.0.0", "bindings": "~1.3.0", "fast-future": "~1.0.2", - "nan": "~2.11.0", + "napi-macros": "^1.8.1", + "node-gyp-build": "^3.5.1", "prebuild-install": "^5.0.0" }, "devDependencies": { @@ -43,7 +44,7 @@ "verify-travis-appveyor": "^3.0.0" }, "scripts": { - "install": "prebuild-install || node-gyp rebuild", + "install": "node-gyp-build", "test": "standard && verify-travis-appveyor && nyc tape test/*-test.js && prebuild-ci", "coverage": "nyc report --reporter=text-lcov | coveralls", "rebuild": "prebuild --compile", From 71e782349ac5c263eeafcf64406fb35d0bf38d35 Mon Sep 17 00:00:00 2001 From: Lars-Magnus Skog Date: Fri, 7 Dec 2018 00:53:11 +0100 Subject: [PATCH 02/42] Implement open() --- leveldown.js | 11 +- napi/leveldown.cc | 249 +++++++++++++++++++++++++++++++++++++++++++++- napi/napi.h | 10 ++ package.json | 1 + test/index.js | 14 +++ 5 files changed, 277 insertions(+), 8 deletions(-) create mode 100644 napi/napi.h create mode 100644 test/index.js diff --git a/leveldown.js b/leveldown.js index a7f72041..e416f0bf 100644 --- a/leveldown.js +++ b/leveldown.js @@ -1,6 +1,6 @@ const util = require('util') const AbstractLevelDOWN = require('abstract-leveldown').AbstractLevelDOWN -const binding = require('bindings')('leveldown').leveldown +const binding = require('node-gyp-build')(__dirname) const ChainedBatch = require('./chained-batch') const Iterator = require('./iterator') @@ -16,13 +16,18 @@ function LevelDOWN (location) { AbstractLevelDOWN.call(this) this.location = location - this.binding = binding(location) + this.dbContext = binding.leveldown() } util.inherits(LevelDOWN, AbstractLevelDOWN) LevelDOWN.prototype._open = function (options, callback) { - this.binding.open(options, callback) + binding.open( + this.dbContext, + this.location, + options, + callback + ) } LevelDOWN.prototype._close = function (callback) { diff --git a/napi/leveldown.cc b/napi/leveldown.cc index 8461213f..5a786aad 100644 --- a/napi/leveldown.cc +++ b/napi/leveldown.cc @@ -1,14 +1,253 @@ -#include -#include +#include "napi.h" +#include +#include +#include -// TODO add to napi-macros.h -#define NAPI_RETURN_UNDEFINED() \ - return 0; +/** + * Owns the LevelDB storage together with cache and filter + * policy. + */ +struct DbContext { + DbContext (napi_env env) + : env_(env), db_(NULL), blockCache_(NULL), filterPolicy_(NULL) {} + ~DbContext () { + if (db_ != NULL) { + delete db_; + db_ = NULL; + } + } + + napi_env env_; + leveldb::DB* db_; + leveldb::Cache* blockCache_; + const leveldb::FilterPolicy* filterPolicy_; +}; + +/** + * Runs when a DbContext is garbage collected. + */ +static void FinalizeDbContext(napi_env env, void* data, void* hint) { + if (data) { + delete (DbContext*)data; + } +} + +/** + * Returns a context object for a database. + * E.g. `var dbContext = leveldown()` + */ NAPI_METHOD(leveldown) { + DbContext* dbContext = new DbContext(env); + + napi_value result; + NAPI_STATUS_THROWS(napi_create_external(env, dbContext, + FinalizeDbContext, + NULL, &result)); + return result; +} + +// TODO BooleanProperty() and Uint32Property() can be refactored + +/** + * Returns a boolean property 'key' from 'obj'. + * Returns 'DEFAULT' if the property doesn't exist. + */ +static bool BooleanProperty(napi_env env, + napi_value obj, + const char* key, + bool DEFAULT) +{ + bool exists = false; + napi_value _key; + napi_create_string_utf8(env, key, strlen(key), &_key); + napi_has_property(env, obj, _key, &exists); + + if (exists) { + napi_value value; + napi_get_property(env, obj, _key, &value); + bool result; + napi_get_value_bool(env, value, &result); + return result; + } + + return DEFAULT; +} + +/** + * Returns a uint32 property 'key' from 'obj' + * Returns 'DEFAULT' if the property doesn't exist. + */ +static uint32_t Uint32Property(napi_env env, + napi_value obj, + const char* key, + uint32_t DEFAULT) +{ + bool exists = false; + napi_value _key; + napi_create_string_utf8(env, key, strlen(key), &_key); + napi_has_property(env, obj, _key, &exists); + + if (exists) { + napi_value value; + napi_get_property(env, obj, _key, &value); + uint32_t result; + napi_get_value_uint32(env, value, &result); + return result; + } + + return DEFAULT; +} + +/** + * Worker class for opening the database + */ +struct OpenWorker { + OpenWorker(napi_env env, + DbContext* dbContext, + char* location, + bool createIfMissing, + bool errorIfExists, + bool compression, + uint32_t writeBufferSize, + uint32_t blockSize, + uint32_t maxOpenFiles, + uint32_t blockRestartInterval, + uint32_t maxFileSize, + napi_value callback) + : env_(env), dbContext_(dbContext), location_(location) { + options_.block_cache = dbContext->blockCache_; + options_.filter_policy = dbContext->filterPolicy_; + options_.create_if_missing = createIfMissing; + options_.error_if_exists = errorIfExists; + options_.compression = compression + ? leveldb::kSnappyCompression + : leveldb::kNoCompression; + options_.write_buffer_size = writeBufferSize; + options_.block_size = blockSize; + options_.max_open_files = maxOpenFiles; + options_.block_restart_interval = blockRestartInterval; + options_.max_file_size = maxFileSize; + + // Create reference to callback with ref count set to one. + // TODO move to base class constructor + napi_create_reference(env_, callback, 1, &callbackRef_); + napi_value asyncResourceName; + napi_create_string_utf8(env_, "leveldown::open", + NAPI_AUTO_LENGTH, + &asyncResourceName); + napi_create_async_work(env_, callback, + asyncResourceName, + OpenWorker::Execute, + OpenWorker::Complete, + this, + &asyncWork_); + } + + ~OpenWorker() { + free(location_); + // TODO move to base class destructor + napi_delete_reference(env_, callbackRef_); + napi_delete_async_work(env_, asyncWork_); + } + + // TODO move to base class + void Queue() { + napi_queue_async_work(env_, asyncWork_); + } + + static void Execute(napi_env env, void* data) { + OpenWorker* self = (OpenWorker*)data; + DbContext* dbContext = self->dbContext_; + self->status_ = leveldb::DB::Open(self->options_, + self->location_, + &dbContext->db_); + } + + static void Complete(napi_env env, napi_status status, void* data) { + OpenWorker* self = (OpenWorker*)data; + + // TODO most of the things below can be moved to a + // base class, all operations either calling back with + // NULL or an error have identical logic. + + const int argc = 1; + napi_value argv[argc]; + + napi_value global; + napi_get_global(env, &global); + napi_value callback; + napi_get_reference_value(env, self->callbackRef_, &callback); + + if (self->status_.ok()) { + napi_get_null(env, &argv[0]); + } else { + const char* str = self->status_.ToString().c_str(); + napi_value msg; + napi_create_string_utf8(env, str, strlen(str), &msg); + napi_create_error(env, NULL, msg, &argv[0]); + } + + napi_call_function(env, global, callback, argc, argv, NULL); + + delete self; + } + + // TODO move to base class + napi_env env_; + napi_ref callbackRef_; + napi_async_work asyncWork_; + DbContext* dbContext_; + leveldb::Status status_; + + leveldb::Options options_; + char* location_; +}; + +/** + * Opens a database. + */ +NAPI_METHOD(open) { + NAPI_ARGV(4); + NAPI_DB_CONTEXT(); + NAPI_ARGV_UTF8_MALLOC(location, 1); + + // Options object and properties. + napi_value options = argv[2]; + bool createIfMissing = BooleanProperty(env, options, "createIfMissing", true); + bool errorIfExists = BooleanProperty(env, options, "errorIfExists", false); + bool compression = BooleanProperty(env, options, "compression", true); + + uint32_t cacheSize = Uint32Property(env, options, "cacheSize", 8 << 20); + uint32_t writeBufferSize = Uint32Property(env, options , "writeBufferSize" , 4 << 20); + uint32_t blockSize = Uint32Property(env, options, "blockSize", 4096); + uint32_t maxOpenFiles = Uint32Property(env, options, "maxOpenFiles", 1000); + uint32_t blockRestartInterval = Uint32Property(env, options, + "blockRestartInterval", 16); + uint32_t maxFileSize = Uint32Property(env, options, "maxFileSize", 2 << 20); + + // TODO clean these up in close() + dbContext->blockCache_ = leveldb::NewLRUCache(cacheSize); + dbContext->filterPolicy_ = leveldb::NewBloomFilterPolicy(10); + + napi_value callback = argv[3]; + OpenWorker* worker = new OpenWorker(env, + dbContext, + location, + createIfMissing, + errorIfExists, + compression, + writeBufferSize, + blockSize, + maxOpenFiles, + blockRestartInterval, + maxFileSize, + callback); + worker->Queue(); NAPI_RETURN_UNDEFINED(); } NAPI_INIT() { NAPI_EXPORT_FUNCTION(leveldown); + NAPI_EXPORT_FUNCTION(open); } diff --git a/napi/napi.h b/napi/napi.h new file mode 100644 index 00000000..321cfccc --- /dev/null +++ b/napi/napi.h @@ -0,0 +1,10 @@ +#include +#include + +#define NAPI_DB_CONTEXT() \ + DbContext* dbContext = NULL; \ + NAPI_STATUS_THROWS(napi_get_value_external(env, argv[0], (void**)&dbContext)); + +// TODO move to napi-macros.h +#define NAPI_RETURN_UNDEFINED() \ + return 0; diff --git a/package.json b/package.json index 6cb50de4..d50dc8f8 100644 --- a/package.json +++ b/package.json @@ -46,6 +46,7 @@ "scripts": { "install": "node-gyp-build", "test": "standard && verify-travis-appveyor && nyc tape test/*-test.js && prebuild-ci", + "test2": "node test", "coverage": "nyc report --reporter=text-lcov | coveralls", "rebuild": "prebuild --compile", "prebuild": "prebuild --all --strip --verbose", diff --git a/test/index.js b/test/index.js new file mode 100644 index 00000000..cece63c4 --- /dev/null +++ b/test/index.js @@ -0,0 +1,14 @@ +require('./leveldown-test') +require('./abstract-leveldown-test') +// require('./approximate-size-test') +// require('./cleanup-hanging-iterators-test') +// require('./compact-range-test') +// require('./compression-test') +// require('./destroy-test') +// require('./getproperty-test') +// require('./iterator-gc-test') +// require('./iterator-recursion-test') +// require('./iterator-test') +// require('./port-libuv-fix-test') +// require('./repair-test') +// require('./segfault-test') From 9f03290229f4d0cac75a8e2a00f6173febc14ece Mon Sep 17 00:00:00 2001 From: Lars-Magnus Skog Date: Fri, 7 Dec 2018 03:22:11 +0100 Subject: [PATCH 03/42] Implement put() (not yet handling buffers) --- leveldown.js | 8 +- napi/leveldown.cc | 182 +++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 185 insertions(+), 5 deletions(-) diff --git a/leveldown.js b/leveldown.js index e416f0bf..f0aba05e 100644 --- a/leveldown.js +++ b/leveldown.js @@ -43,7 +43,13 @@ LevelDOWN.prototype._serializeValue = function (value) { } LevelDOWN.prototype._put = function (key, value, options, callback) { - this.binding.put(key, value, options, callback) + binding.put( + this.dbContext, + key, + value, + options, + callback + ) } LevelDOWN.prototype._get = function (key, options, callback) { diff --git a/napi/leveldown.cc b/napi/leveldown.cc index 5a786aad..2d430e0f 100644 --- a/napi/leveldown.cc +++ b/napi/leveldown.cc @@ -56,8 +56,7 @@ NAPI_METHOD(leveldown) { static bool BooleanProperty(napi_env env, napi_value obj, const char* key, - bool DEFAULT) -{ + bool DEFAULT) { bool exists = false; napi_value _key; napi_create_string_utf8(env, key, strlen(key), &_key); @@ -81,8 +80,7 @@ static bool BooleanProperty(napi_env env, static uint32_t Uint32Property(napi_env env, napi_value obj, const char* key, - uint32_t DEFAULT) -{ + uint32_t DEFAULT) { bool exists = false; napi_value _key; napi_create_string_utf8(env, key, strlen(key), &_key); @@ -247,7 +245,183 @@ NAPI_METHOD(open) { NAPI_RETURN_UNDEFINED(); } +/** + * Returns true if 'value' is a buffer otherwise false. + */ +static bool isBuffer(napi_env env, napi_value value) { + bool isBuffer; + napi_is_buffer(env, value, &isBuffer); + return isBuffer; +} + +/** + * Convert a napi_value to a leveldb::Slice. + */ +static leveldb::Slice ToSlice(napi_env env, napi_value from) { + // size_t to ## Sz_; + // char* to ## Ch_; + // if (from->IsNull() || from->IsUndefined()) { + // to ## Sz_ = 0; + // to ## Ch_ = 0; + // } else if (!from->ToObject().IsEmpty() + // && node::Buffer::HasInstance(from->ToObject())) { + // to ## Sz_ = node::Buffer::Length(from->ToObject()); + // to ## Ch_ = node::Buffer::Data(from->ToObject()); + // } else { + // v8::Local to ## Str = from->ToString(); + // to ## Sz_ = to ## Str->Utf8Length(); + // to ## Ch_ = new char[to ## Sz_]; + // to ## Str->WriteUtf8(to ## Ch_, -1, NULL, v8::String::NO_NULL_TERMINATION); + // } + // leveldb::Slice to(to ## Ch_, to ## Sz_); + size_t toLength = 0; + char* toChar = 0; + + napi_valuetype type; + napi_typeof(env, from, &type); + + if (type == napi_null) { + printf("FROM is null\n"); + } else if (type == napi_undefined) { + printf("FROM is undefined\n"); + } + + if (type != napi_null && type != napi_undefined) { + if (type == napi_string) { + printf("FROM is a string\n"); + size_t size = 0; + napi_get_value_string_utf8(env, from, NULL, 0, &size); + if (size > 0) { + toChar = (char*)malloc((size + 1) * sizeof(char)); + napi_get_value_string_utf8(env, from, toChar, size + 1, &toLength); + toChar[toLength] = '\0'; + } + } else if (isBuffer(env, from)) { + printf("FROM is a buffer\n"); + // TODO what to do with buffers? we could either always + // copy them, or use an out parameter to store the fact + // that we did allocate and only clean in that case + } + } + + return leveldb::Slice(toChar, toLength); +} + +/** + * Worker class for putting key/value to the database + */ +struct PutWorker { + PutWorker(napi_env env, + DbContext* dbContext, + napi_value key, + napi_value value, + bool sync, + napi_value callback) + : env_(env), dbContext_(dbContext), + key_(ToSlice(env, key)), + value_(ToSlice(env, value)) { + options_.sync = sync; + // Create reference to callback with ref count set to one. + // TODO move to base class constructor + napi_create_reference(env_, callback, 1, &callbackRef_); + napi_value asyncResourceName; + napi_create_string_utf8(env_, "leveldown::put", + NAPI_AUTO_LENGTH, + &asyncResourceName); + napi_create_async_work(env_, callback, + asyncResourceName, + PutWorker::Execute, + PutWorker::Complete, + this, + &asyncWork_); + } + + ~PutWorker() { + // TODO move to base class destructor + napi_delete_reference(env_, callbackRef_); + napi_delete_async_work(env_, asyncWork_); + + // TODO clean up key_ and value_ if they aren't empty? + // See DisposeStringOrBufferFromSlice() + } + + // TODO move to base class + void Queue() { + napi_queue_async_work(env_, asyncWork_); + } + + static void Execute(napi_env env, void* data) { + PutWorker* self = (PutWorker*)data; + DbContext* dbContext = self->dbContext_; + self->status_ = dbContext->db_->Put(self->options_, + self->key_, + self->value_); + } + + static void Complete(napi_env env, napi_status status, void* data) { + PutWorker* self = (PutWorker*)data; + + // TODO most of the things below can be moved to a + // base class, all operations either calling back with + // NULL or an error have identical logic. + + const int argc = 1; + napi_value argv[argc]; + + napi_value global; + napi_get_global(env, &global); + napi_value callback; + napi_get_reference_value(env, self->callbackRef_, &callback); + + if (self->status_.ok()) { + napi_get_null(env, &argv[0]); + } else { + const char* str = self->status_.ToString().c_str(); + napi_value msg; + napi_create_string_utf8(env, str, strlen(str), &msg); + napi_create_error(env, NULL, msg, &argv[0]); + } + + napi_call_function(env, global, callback, argc, argv, NULL); + + delete self; + } + + // TODO move to base class + napi_env env_; + napi_ref callbackRef_; + napi_async_work asyncWork_; + DbContext* dbContext_; + leveldb::Status status_; + + leveldb::WriteOptions options_; + leveldb::Slice key_; + leveldb::Slice value_; +}; + +/** + * Puts a key and a value to the database. + */ +NAPI_METHOD(put) { + NAPI_ARGV(5); + NAPI_DB_CONTEXT(); + + napi_value key = argv[1]; + napi_value value = argv[2]; + bool sync = BooleanProperty(env, argv[3], "sync", false); + napi_value callback = argv[4]; + PutWorker* worker = new PutWorker(env, + dbContext, + key, + value, + sync, + callback); + worker->Queue(); + NAPI_RETURN_UNDEFINED(); +} + NAPI_INIT() { NAPI_EXPORT_FUNCTION(leveldown); NAPI_EXPORT_FUNCTION(open); + NAPI_EXPORT_FUNCTION(put); } From 1fba29c962860aca86ab67732f70e97c0b71b5c1 Mon Sep 17 00:00:00 2001 From: Lars-Magnus Skog Date: Fri, 7 Dec 2018 03:53:46 +0100 Subject: [PATCH 04/42] Refactor common logic from OpenWorker and PutWorker into BaseWorker --- napi/leveldown.cc | 232 +++++++++++++++++++--------------------------- 1 file changed, 97 insertions(+), 135 deletions(-) diff --git a/napi/leveldown.cc b/napi/leveldown.cc index 2d430e0f..0d1b1acf 100644 --- a/napi/leveldown.cc +++ b/napi/leveldown.cc @@ -98,105 +98,130 @@ static uint32_t Uint32Property(napi_env env, } /** - * Worker class for opening the database + * Base worker class. */ -struct OpenWorker { - OpenWorker(napi_env env, - DbContext* dbContext, - char* location, - bool createIfMissing, - bool errorIfExists, - bool compression, - uint32_t writeBufferSize, - uint32_t blockSize, - uint32_t maxOpenFiles, - uint32_t blockRestartInterval, - uint32_t maxFileSize, - napi_value callback) - : env_(env), dbContext_(dbContext), location_(location) { - options_.block_cache = dbContext->blockCache_; - options_.filter_policy = dbContext->filterPolicy_; - options_.create_if_missing = createIfMissing; - options_.error_if_exists = errorIfExists; - options_.compression = compression - ? leveldb::kSnappyCompression - : leveldb::kNoCompression; - options_.write_buffer_size = writeBufferSize; - options_.block_size = blockSize; - options_.max_open_files = maxOpenFiles; - options_.block_restart_interval = blockRestartInterval; - options_.max_file_size = maxFileSize; - - // Create reference to callback with ref count set to one. - // TODO move to base class constructor +struct BaseWorker { + BaseWorker(napi_env env, + DbContext* dbContext, + napi_value callback, + const char* resourceName) + : env_(env), dbContext_(dbContext) { napi_create_reference(env_, callback, 1, &callbackRef_); napi_value asyncResourceName; - napi_create_string_utf8(env_, "leveldown::open", + napi_create_string_utf8(env_, resourceName, NAPI_AUTO_LENGTH, &asyncResourceName); napi_create_async_work(env_, callback, asyncResourceName, - OpenWorker::Execute, - OpenWorker::Complete, + BaseWorker::Execute, + BaseWorker::Complete, this, &asyncWork_); } - ~OpenWorker() { - free(location_); - // TODO move to base class destructor + virtual ~BaseWorker() { napi_delete_reference(env_, callbackRef_); napi_delete_async_work(env_, asyncWork_); } - // TODO move to base class - void Queue() { - napi_queue_async_work(env_, asyncWork_); - } - + /** + * Calls virtual DoExecute(). + */ static void Execute(napi_env env, void* data) { - OpenWorker* self = (OpenWorker*)data; - DbContext* dbContext = self->dbContext_; - self->status_ = leveldb::DB::Open(self->options_, - self->location_, - &dbContext->db_); + BaseWorker* self = (BaseWorker*)data; + self->DoExecute(); } - static void Complete(napi_env env, napi_status status, void* data) { - OpenWorker* self = (OpenWorker*)data; + /** + * MUST be overriden. + */ + virtual void DoExecute() = 0; - // TODO most of the things below can be moved to a - // base class, all operations either calling back with - // NULL or an error have identical logic. + /** + * Calls DoComplete() and kills the worker. + */ + static void Complete(napi_env env, napi_status status, void* data) { + BaseWorker* self = (BaseWorker*)data; + self->DoComplete(); + delete self; + } + /** + * Default handling when work is complete. + * - Calls back with NULL if no error + * - Calls back with error if status is an error + */ + virtual void DoComplete() { const int argc = 1; napi_value argv[argc]; napi_value global; - napi_get_global(env, &global); + napi_get_global(env_, &global); napi_value callback; - napi_get_reference_value(env, self->callbackRef_, &callback); + napi_get_reference_value(env_, callbackRef_, &callback); - if (self->status_.ok()) { - napi_get_null(env, &argv[0]); + if (status_.ok()) { + napi_get_null(env_, &argv[0]); } else { - const char* str = self->status_.ToString().c_str(); + const char* str = status_.ToString().c_str(); napi_value msg; - napi_create_string_utf8(env, str, strlen(str), &msg); - napi_create_error(env, NULL, msg, &argv[0]); + napi_create_string_utf8(env_, str, strlen(str), &msg); + napi_create_error(env_, NULL, msg, &argv[0]); } - napi_call_function(env, global, callback, argc, argv, NULL); + napi_call_function(env_, global, callback, argc, argv, NULL); + } - delete self; + void Queue() { + napi_queue_async_work(env_, asyncWork_); } - // TODO move to base class napi_env env_; napi_ref callbackRef_; napi_async_work asyncWork_; DbContext* dbContext_; leveldb::Status status_; +}; + +/** + * Worker class for opening the database + */ +struct OpenWorker : public BaseWorker { + OpenWorker(napi_env env, + DbContext* dbContext, + napi_value callback, + char* location, + bool createIfMissing, + bool errorIfExists, + bool compression, + uint32_t writeBufferSize, + uint32_t blockSize, + uint32_t maxOpenFiles, + uint32_t blockRestartInterval, + uint32_t maxFileSize) + : BaseWorker(env, dbContext, callback, "leveldown::open"), + location_(location) { + options_.block_cache = dbContext->blockCache_; + options_.filter_policy = dbContext->filterPolicy_; + options_.create_if_missing = createIfMissing; + options_.error_if_exists = errorIfExists; + options_.compression = compression + ? leveldb::kSnappyCompression + : leveldb::kNoCompression; + options_.write_buffer_size = writeBufferSize; + options_.block_size = blockSize; + options_.max_open_files = maxOpenFiles; + options_.block_restart_interval = blockRestartInterval; + options_.max_file_size = maxFileSize; + } + + virtual ~OpenWorker() { + free(location_); + } + + virtual void DoExecute() { + status_ = leveldb::DB::Open(options_, location_, &dbContext_->db_); + } leveldb::Options options_; char* location_; @@ -231,6 +256,7 @@ NAPI_METHOD(open) { napi_value callback = argv[3]; OpenWorker* worker = new OpenWorker(env, dbContext, + callback, location, createIfMissing, errorIfExists, @@ -239,8 +265,7 @@ NAPI_METHOD(open) { blockSize, maxOpenFiles, blockRestartInterval, - maxFileSize, - callback); + maxFileSize); worker->Queue(); NAPI_RETURN_UNDEFINED(); } @@ -310,90 +335,27 @@ static leveldb::Slice ToSlice(napi_env env, napi_value from) { /** * Worker class for putting key/value to the database */ -struct PutWorker { +struct PutWorker : public BaseWorker { PutWorker(napi_env env, DbContext* dbContext, + napi_value callback, napi_value key, napi_value value, - bool sync, - napi_value callback) - : env_(env), dbContext_(dbContext), - key_(ToSlice(env, key)), - value_(ToSlice(env, value)) { + bool sync) + : BaseWorker(env, dbContext, callback, "leveldown::put"), + key_(ToSlice(env, key)), value_(ToSlice(env, value)) { options_.sync = sync; - // Create reference to callback with ref count set to one. - // TODO move to base class constructor - napi_create_reference(env_, callback, 1, &callbackRef_); - napi_value asyncResourceName; - napi_create_string_utf8(env_, "leveldown::put", - NAPI_AUTO_LENGTH, - &asyncResourceName); - napi_create_async_work(env_, callback, - asyncResourceName, - PutWorker::Execute, - PutWorker::Complete, - this, - &asyncWork_); } ~PutWorker() { - // TODO move to base class destructor - napi_delete_reference(env_, callbackRef_); - napi_delete_async_work(env_, asyncWork_); - // TODO clean up key_ and value_ if they aren't empty? // See DisposeStringOrBufferFromSlice() } - // TODO move to base class - void Queue() { - napi_queue_async_work(env_, asyncWork_); + virtual void DoExecute() { + status_ = dbContext_->db_->Put(options_, key_, value_); } - static void Execute(napi_env env, void* data) { - PutWorker* self = (PutWorker*)data; - DbContext* dbContext = self->dbContext_; - self->status_ = dbContext->db_->Put(self->options_, - self->key_, - self->value_); - } - - static void Complete(napi_env env, napi_status status, void* data) { - PutWorker* self = (PutWorker*)data; - - // TODO most of the things below can be moved to a - // base class, all operations either calling back with - // NULL or an error have identical logic. - - const int argc = 1; - napi_value argv[argc]; - - napi_value global; - napi_get_global(env, &global); - napi_value callback; - napi_get_reference_value(env, self->callbackRef_, &callback); - - if (self->status_.ok()) { - napi_get_null(env, &argv[0]); - } else { - const char* str = self->status_.ToString().c_str(); - napi_value msg; - napi_create_string_utf8(env, str, strlen(str), &msg); - napi_create_error(env, NULL, msg, &argv[0]); - } - - napi_call_function(env, global, callback, argc, argv, NULL); - - delete self; - } - - // TODO move to base class - napi_env env_; - napi_ref callbackRef_; - napi_async_work asyncWork_; - DbContext* dbContext_; - leveldb::Status status_; - leveldb::WriteOptions options_; leveldb::Slice key_; leveldb::Slice value_; @@ -412,10 +374,10 @@ NAPI_METHOD(put) { napi_value callback = argv[4]; PutWorker* worker = new PutWorker(env, dbContext, + callback, key, value, - sync, - callback); + sync); worker->Queue(); NAPI_RETURN_UNDEFINED(); } From e37cda04f05a6a3c4f76b360096f869e2ad61b02 Mon Sep 17 00:00:00 2001 From: Lars-Magnus Skog Date: Fri, 7 Dec 2018 04:12:13 +0100 Subject: [PATCH 05/42] Rename binding functions --- leveldown.js | 10 +++++----- napi/leveldown.cc | 12 ++++++------ 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/leveldown.js b/leveldown.js index f0aba05e..8dbd674f 100644 --- a/leveldown.js +++ b/leveldown.js @@ -16,14 +16,14 @@ function LevelDOWN (location) { AbstractLevelDOWN.call(this) this.location = location - this.dbContext = binding.leveldown() + this.context = binding.db() } util.inherits(LevelDOWN, AbstractLevelDOWN) LevelDOWN.prototype._open = function (options, callback) { - binding.open( - this.dbContext, + binding.db_open( + this.context, this.location, options, callback @@ -43,8 +43,8 @@ LevelDOWN.prototype._serializeValue = function (value) { } LevelDOWN.prototype._put = function (key, value, options, callback) { - binding.put( - this.dbContext, + binding.db_put( + this.context, key, value, options, diff --git a/napi/leveldown.cc b/napi/leveldown.cc index 0d1b1acf..e67fc3e6 100644 --- a/napi/leveldown.cc +++ b/napi/leveldown.cc @@ -37,7 +37,7 @@ static void FinalizeDbContext(napi_env env, void* data, void* hint) { * Returns a context object for a database. * E.g. `var dbContext = leveldown()` */ -NAPI_METHOD(leveldown) { +NAPI_METHOD(db) { DbContext* dbContext = new DbContext(env); napi_value result; @@ -230,7 +230,7 @@ struct OpenWorker : public BaseWorker { /** * Opens a database. */ -NAPI_METHOD(open) { +NAPI_METHOD(db_open) { NAPI_ARGV(4); NAPI_DB_CONTEXT(); NAPI_ARGV_UTF8_MALLOC(location, 1); @@ -364,7 +364,7 @@ struct PutWorker : public BaseWorker { /** * Puts a key and a value to the database. */ -NAPI_METHOD(put) { +NAPI_METHOD(db_put) { NAPI_ARGV(5); NAPI_DB_CONTEXT(); @@ -383,7 +383,7 @@ NAPI_METHOD(put) { } NAPI_INIT() { - NAPI_EXPORT_FUNCTION(leveldown); - NAPI_EXPORT_FUNCTION(open); - NAPI_EXPORT_FUNCTION(put); + NAPI_EXPORT_FUNCTION(db); + NAPI_EXPORT_FUNCTION(db_open); + NAPI_EXPORT_FUNCTION(db_put); } From 085ef08597412ab31410802d6650f067f4fa830e Mon Sep 17 00:00:00 2001 From: Lars-Magnus Skog Date: Fri, 7 Dec 2018 04:39:15 +0100 Subject: [PATCH 06/42] Stub out iterator methods --- binding.js | 1 + iterator.js | 9 +++++---- leveldown.js | 2 +- napi/leveldown.cc | 49 +++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 56 insertions(+), 5 deletions(-) create mode 100644 binding.js diff --git a/binding.js b/binding.js new file mode 100644 index 00000000..ffd93865 --- /dev/null +++ b/binding.js @@ -0,0 +1 @@ +module.exports = require('node-gyp-build')(__dirname) diff --git a/iterator.js b/iterator.js index 1e99a819..c5f423ee 100644 --- a/iterator.js +++ b/iterator.js @@ -1,11 +1,12 @@ const util = require('util') const AbstractIterator = require('abstract-leveldown').AbstractIterator const fastFuture = require('fast-future') +const binding = require('./binding') function Iterator (db, options) { AbstractIterator.call(this, db) - this.binding = db.binding.iterator(options) + this.context = binding.iterator(db.context, options) this.cache = null this.finished = false this.fastFuture = fastFuture() @@ -19,7 +20,7 @@ Iterator.prototype._seek = function (target) { } this.cache = null - this.binding.seek(target) + binding.iterator_seek(this.context, target) this.finished = false } @@ -40,7 +41,7 @@ Iterator.prototype._next = function (callback) { callback() }) } else { - this.binding.next(function (err, array, finished) { + binding.iterator_next(this.context, function (err, array, finished) { if (err) return callback(err) that.cache = array @@ -54,7 +55,7 @@ Iterator.prototype._next = function (callback) { Iterator.prototype._end = function (callback) { delete this.cache - this.binding.end(callback) + binding.iterator_end(this.context, callback) } module.exports = Iterator diff --git a/leveldown.js b/leveldown.js index 8dbd674f..547624e7 100644 --- a/leveldown.js +++ b/leveldown.js @@ -1,6 +1,6 @@ const util = require('util') const AbstractLevelDOWN = require('abstract-leveldown').AbstractLevelDOWN -const binding = require('node-gyp-build')(__dirname) +const binding = require('./binding') const ChainedBatch = require('./chained-batch') const Iterator = require('./iterator') diff --git a/napi/leveldown.cc b/napi/leveldown.cc index e67fc3e6..543f48ef 100644 --- a/napi/leveldown.cc +++ b/napi/leveldown.cc @@ -382,8 +382,57 @@ NAPI_METHOD(db_put) { NAPI_RETURN_UNDEFINED(); } +/** + * Creates an iterator. + */ +NAPI_METHOD(iterator) { + NAPI_ARGV(2); + NAPI_DB_CONTEXT(); + + // TODO assume options object is always passed in! it will + // make the code simpler + + // TODO return an external for IteratorContext + NAPI_RETURN_UNDEFINED(); +} + +/** + * Seeks an iterator. + */ +NAPI_METHOD(iterator_seek) { + NAPI_ARGV(2); + // NAPI_ITERATOR_CONTEXT(); + + NAPI_RETURN_UNDEFINED(); +} + +/** + * Moves an iterator to next element. + */ +NAPI_METHOD(iterator_next) { + NAPI_ARGV(2); + // NAPI_ITERATOR_CONTEXT(); + + NAPI_RETURN_UNDEFINED(); +} + +/** + * Ends an iterator. + */ +NAPI_METHOD(iterator_end) { + NAPI_ARGV(2); + // NAPI_ITERATOR_CONTEXT(); + + NAPI_RETURN_UNDEFINED(); +} + NAPI_INIT() { NAPI_EXPORT_FUNCTION(db); NAPI_EXPORT_FUNCTION(db_open); NAPI_EXPORT_FUNCTION(db_put); + + NAPI_EXPORT_FUNCTION(iterator); + NAPI_EXPORT_FUNCTION(iterator_seek); + NAPI_EXPORT_FUNCTION(iterator_next); + NAPI_EXPORT_FUNCTION(iterator_end); } From 3ede7b3d1cba70c262c553ffe71012e2a75aaf85 Mon Sep 17 00:00:00 2001 From: Lars-Magnus Skog Date: Fri, 7 Dec 2018 15:09:42 +0100 Subject: [PATCH 07/42] Lets adopt same naming as for nan case, DbContext -> Database --- napi/leveldown.cc | 49 +++++++++++++++++++++++------------------------ napi/napi.h | 4 ++-- 2 files changed, 26 insertions(+), 27 deletions(-) diff --git a/napi/leveldown.cc b/napi/leveldown.cc index 543f48ef..c236e9b5 100644 --- a/napi/leveldown.cc +++ b/napi/leveldown.cc @@ -7,11 +7,11 @@ * Owns the LevelDB storage together with cache and filter * policy. */ -struct DbContext { - DbContext (napi_env env) +struct Database { + Database (napi_env env) : env_(env), db_(NULL), blockCache_(NULL), filterPolicy_(NULL) {} - ~DbContext () { + ~Database () { if (db_ != NULL) { delete db_; db_ = NULL; @@ -25,24 +25,23 @@ struct DbContext { }; /** - * Runs when a DbContext is garbage collected. + * Runs when a Database is garbage collected. */ -static void FinalizeDbContext(napi_env env, void* data, void* hint) { +static void FinalizeDatabase(napi_env env, void* data, void* hint) { if (data) { - delete (DbContext*)data; + delete (Database*)data; } } /** * Returns a context object for a database. - * E.g. `var dbContext = leveldown()` */ NAPI_METHOD(db) { - DbContext* dbContext = new DbContext(env); + Database* database = new Database(env); napi_value result; - NAPI_STATUS_THROWS(napi_create_external(env, dbContext, - FinalizeDbContext, + NAPI_STATUS_THROWS(napi_create_external(env, database, + FinalizeDatabase, NULL, &result)); return result; } @@ -102,10 +101,10 @@ static uint32_t Uint32Property(napi_env env, */ struct BaseWorker { BaseWorker(napi_env env, - DbContext* dbContext, + Database* database, napi_value callback, const char* resourceName) - : env_(env), dbContext_(dbContext) { + : env_(env), database_(database) { napi_create_reference(env_, callback, 1, &callbackRef_); napi_value asyncResourceName; napi_create_string_utf8(env_, resourceName, @@ -179,7 +178,7 @@ struct BaseWorker { napi_env env_; napi_ref callbackRef_; napi_async_work asyncWork_; - DbContext* dbContext_; + Database* database_; leveldb::Status status_; }; @@ -188,7 +187,7 @@ struct BaseWorker { */ struct OpenWorker : public BaseWorker { OpenWorker(napi_env env, - DbContext* dbContext, + Database* database, napi_value callback, char* location, bool createIfMissing, @@ -199,10 +198,10 @@ struct OpenWorker : public BaseWorker { uint32_t maxOpenFiles, uint32_t blockRestartInterval, uint32_t maxFileSize) - : BaseWorker(env, dbContext, callback, "leveldown::open"), + : BaseWorker(env, database, callback, "leveldown::open"), location_(location) { - options_.block_cache = dbContext->blockCache_; - options_.filter_policy = dbContext->filterPolicy_; + options_.block_cache = database->blockCache_; + options_.filter_policy = database->filterPolicy_; options_.create_if_missing = createIfMissing; options_.error_if_exists = errorIfExists; options_.compression = compression @@ -220,7 +219,7 @@ struct OpenWorker : public BaseWorker { } virtual void DoExecute() { - status_ = leveldb::DB::Open(options_, location_, &dbContext_->db_); + status_ = leveldb::DB::Open(options_, location_, &database_->db_); } leveldb::Options options_; @@ -250,12 +249,12 @@ NAPI_METHOD(db_open) { uint32_t maxFileSize = Uint32Property(env, options, "maxFileSize", 2 << 20); // TODO clean these up in close() - dbContext->blockCache_ = leveldb::NewLRUCache(cacheSize); - dbContext->filterPolicy_ = leveldb::NewBloomFilterPolicy(10); + database->blockCache_ = leveldb::NewLRUCache(cacheSize); + database->filterPolicy_ = leveldb::NewBloomFilterPolicy(10); napi_value callback = argv[3]; OpenWorker* worker = new OpenWorker(env, - dbContext, + database, callback, location, createIfMissing, @@ -337,12 +336,12 @@ static leveldb::Slice ToSlice(napi_env env, napi_value from) { */ struct PutWorker : public BaseWorker { PutWorker(napi_env env, - DbContext* dbContext, + Database* database, napi_value callback, napi_value key, napi_value value, bool sync) - : BaseWorker(env, dbContext, callback, "leveldown::put"), + : BaseWorker(env, database, callback, "leveldown::put"), key_(ToSlice(env, key)), value_(ToSlice(env, value)) { options_.sync = sync; } @@ -353,7 +352,7 @@ struct PutWorker : public BaseWorker { } virtual void DoExecute() { - status_ = dbContext_->db_->Put(options_, key_, value_); + status_ = database_->db_->Put(options_, key_, value_); } leveldb::WriteOptions options_; @@ -373,7 +372,7 @@ NAPI_METHOD(db_put) { bool sync = BooleanProperty(env, argv[3], "sync", false); napi_value callback = argv[4]; PutWorker* worker = new PutWorker(env, - dbContext, + database, callback, key, value, diff --git a/napi/napi.h b/napi/napi.h index 321cfccc..ea1e1100 100644 --- a/napi/napi.h +++ b/napi/napi.h @@ -2,8 +2,8 @@ #include #define NAPI_DB_CONTEXT() \ - DbContext* dbContext = NULL; \ - NAPI_STATUS_THROWS(napi_get_value_external(env, argv[0], (void**)&dbContext)); + Database* database = NULL; \ + NAPI_STATUS_THROWS(napi_get_value_external(env, argv[0], (void**)&database)); // TODO move to napi-macros.h #define NAPI_RETURN_UNDEFINED() \ From 20f2aea52dec9d9f7eab1c77cf2dabbf7af78b9b Mon Sep 17 00:00:00 2001 From: Lars-Magnus Skog Date: Fri, 7 Dec 2018 15:15:51 +0100 Subject: [PATCH 08/42] Move leveldb specific calls to Database --- napi/leveldown.cc | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/napi/leveldown.cc b/napi/leveldown.cc index c236e9b5..383059db 100644 --- a/napi/leveldown.cc +++ b/napi/leveldown.cc @@ -18,6 +18,17 @@ struct Database { } } + leveldb::Status Open(const leveldb::Options& options, + const char* location) { + return leveldb::DB::Open(options, location, &db_); + } + + leveldb::Status Put(const leveldb::WriteOptions& options, + leveldb::Slice key, + leveldb::Slice value) { + return db_->Put(options, key, value); + } + napi_env env_; leveldb::DB* db_; leveldb::Cache* blockCache_; @@ -219,7 +230,7 @@ struct OpenWorker : public BaseWorker { } virtual void DoExecute() { - status_ = leveldb::DB::Open(options_, location_, &database_->db_); + status_ = database_->Open(options_, location_); } leveldb::Options options_; @@ -352,7 +363,7 @@ struct PutWorker : public BaseWorker { } virtual void DoExecute() { - status_ = database_->db_->Put(options_, key_, value_); + status_ = database_->Put(options_, key_, value_); } leveldb::WriteOptions options_; From 6889722fb25c8d8ac137a743abec3fe7a67037f3 Mon Sep 17 00:00:00 2001 From: Lars-Magnus Skog Date: Fri, 7 Dec 2018 22:05:41 +0100 Subject: [PATCH 09/42] Implement iterator() --- napi/leveldown.cc | 426 +++++++++++++++++++++++++++++++++++++++++----- src/database.h | 2 + 2 files changed, 386 insertions(+), 42 deletions(-) diff --git a/napi/leveldown.cc b/napi/leveldown.cc index 383059db..88c3388b 100644 --- a/napi/leveldown.cc +++ b/napi/leveldown.cc @@ -2,14 +2,23 @@ #include #include #include +#include /** - * Owns the LevelDB storage together with cache and filter - * policy. + * Forward declarations. + */ +struct Iterator; + +/** + * Owns the LevelDB storage together with cache and filter policy. */ struct Database { Database (napi_env env) - : env_(env), db_(NULL), blockCache_(NULL), filterPolicy_(NULL) {} + : env_(env), + db_(NULL), + blockCache_(NULL), + filterPolicy_(NULL), + currentIteratorId_(0) {} ~Database () { if (db_ != NULL) { @@ -29,10 +38,19 @@ struct Database { return db_->Put(options, key, value); } + const leveldb::Snapshot* NewSnapshot() { + return db_->GetSnapshot(); + } + napi_env env_; leveldb::DB* db_; leveldb::Cache* blockCache_; + // TODO figure out if we can use filterPolicy_ across + // several Open()/Close(), i.e. can we create it _one_ + // time in the constructor? const leveldb::FilterPolicy* filterPolicy_; + uint32_t currentIteratorId_; + std::map< uint32_t, Iterator * > iterators_; }; /** @@ -59,6 +77,32 @@ NAPI_METHOD(db) { // TODO BooleanProperty() and Uint32Property() can be refactored +/** + * Returns true if 'obj' has a property 'key' + */ +static bool HasProperty(napi_env env, napi_value obj, const char* key) { + bool has = false; + napi_value _key; + napi_create_string_utf8(env, key, strlen(key), &_key); + napi_has_property(env, obj, _key, &has); + return has; +} + +/** + * Returns a property in napi_value form + */ +static napi_value GetProperty(napi_env env, + napi_value obj, + const char* key) { + napi_value _key; + napi_create_string_utf8(env, key, strlen(key), &_key); + + napi_value value; + napi_get_property(env, obj, _key, &value); + + return value; +} + /** * Returns a boolean property 'key' from 'obj'. * Returns 'DEFAULT' if the property doesn't exist. @@ -67,14 +111,8 @@ static bool BooleanProperty(napi_env env, napi_value obj, const char* key, bool DEFAULT) { - bool exists = false; - napi_value _key; - napi_create_string_utf8(env, key, strlen(key), &_key); - napi_has_property(env, obj, _key, &exists); - - if (exists) { - napi_value value; - napi_get_property(env, obj, _key, &value); + if (HasProperty(env, obj, key)) { + napi_value value = GetProperty(env, obj, key); bool result; napi_get_value_bool(env, value, &result); return result; @@ -91,6 +129,24 @@ static uint32_t Uint32Property(napi_env env, napi_value obj, const char* key, uint32_t DEFAULT) { + if (HasProperty(env, obj, key)) { + napi_value value = GetProperty(env, obj, key); + uint32_t result; + napi_get_value_uint32(env, value, &result); + return result; + } + + return DEFAULT; +} + +/** + * Returns a uint32 property 'key' from 'obj' + * Returns 'DEFAULT' if the property doesn't exist. + */ +static int Int32Property(napi_env env, + napi_value obj, + const char* key, + int DEFAULT) { bool exists = false; napi_value _key; napi_create_string_utf8(env, key, strlen(key), &_key); @@ -99,8 +155,8 @@ static uint32_t Uint32Property(napi_env env, if (exists) { napi_value value; napi_get_property(env, obj, _key, &value); - uint32_t result; - napi_get_value_uint32(env, value, &result); + int result; + napi_get_value_int32(env, value, &result); return result; } @@ -243,6 +299,8 @@ struct OpenWorker : public BaseWorker { NAPI_METHOD(db_open) { NAPI_ARGV(4); NAPI_DB_CONTEXT(); + // TODO create a NAPI_ARGV_UTF8_NEW() macro that uses new instead of malloc + // so we have similar allocation/deallocation mechanisms everywhere NAPI_ARGV_UTF8_MALLOC(location, 1); // Options object and properties. @@ -280,10 +338,19 @@ NAPI_METHOD(db_open) { NAPI_RETURN_UNDEFINED(); } +/** + * Returns true if 'value' is a string otherwise false. + */ +static bool IsString(napi_env env, napi_value value) { + napi_valuetype type; + napi_typeof(env, value, &type); + return type == napi_string; +} + /** * Returns true if 'value' is a buffer otherwise false. */ -static bool isBuffer(napi_env env, napi_value value) { +static bool IsBuffer(napi_env env, napi_value value) { bool isBuffer; napi_is_buffer(env, value, &isBuffer); return isBuffer; @@ -312,31 +379,21 @@ static leveldb::Slice ToSlice(napi_env env, napi_value from) { size_t toLength = 0; char* toChar = 0; - napi_valuetype type; - napi_typeof(env, from, &type); - - if (type == napi_null) { - printf("FROM is null\n"); - } else if (type == napi_undefined) { - printf("FROM is undefined\n"); - } - - if (type != napi_null && type != napi_undefined) { - if (type == napi_string) { - printf("FROM is a string\n"); - size_t size = 0; - napi_get_value_string_utf8(env, from, NULL, 0, &size); - if (size > 0) { - toChar = (char*)malloc((size + 1) * sizeof(char)); - napi_get_value_string_utf8(env, from, toChar, size + 1, &toLength); - toChar[toLength] = '\0'; - } - } else if (isBuffer(env, from)) { - printf("FROM is a buffer\n"); - // TODO what to do with buffers? we could either always - // copy them, or use an out parameter to store the fact - // that we did allocate and only clean in that case + if (IsString(env, from)) { + printf("FROM is a string\n"); + size_t size = 0; + napi_get_value_string_utf8(env, from, NULL, 0, &size); + if (size > 0) { + // TODO use new here + toChar = (char*)malloc((size + 1) * sizeof(char)); + napi_get_value_string_utf8(env, from, toChar, size + 1, &toLength); + toChar[toLength] = '\0'; } + } else if (IsBuffer(env, from)) { + printf("FROM is a buffer\n"); + // TODO what to do with buffers? we could either always + // copy them, or use an out parameter to store the fact + // that we did allocate and only clean in that case } return leveldb::Slice(toChar, toLength); @@ -392,6 +449,146 @@ NAPI_METHOD(db_put) { NAPI_RETURN_UNDEFINED(); } +/** + * Owns a leveldb iterator. + */ +struct Iterator { + Iterator(Database* database, + uint32_t id, + leveldb::Slice* start, + std::string* end, + bool reverse, + bool keys, + bool values, + int limit, + std::string* lt, + std::string* lte, + std::string* gt, + std::string* gte, + bool fillCache, + bool keyAsBuffer, + bool valueAsBuffer, + uint32_t highWaterMark) + : database_(database), + id_(id), + start_(start), + end_(end), + reverse_(reverse), + keys_(keys), + values_(values), + limit_(limit), + lt_(lt), + lte_(lte), + gt_(gt), + gte_(gte), + keyAsBuffer_(keyAsBuffer), + valueAsBuffer_(valueAsBuffer), + highWaterMark_(highWaterMark), + dbIterator_(NULL), + count_(0), + target_(NULL), + seeking_(false), + landed_(false), + nexting_(false), + ended_(false) { + // TODO need something different for endWorker_ + // endWorker_ = NULL; + // options = new leveldb::ReadOptions(); + options_.fill_cache = fillCache; + options_.snapshot = database->NewSnapshot(); + } + + ~Iterator() { + + } + + // TODO pull in all methods from src/iterator.cc and go + // through EACH of them and make sure that they are used + // and WHERE they are used + + // TODO can we get rid of pointers below? Generally it shouldn't + // be needed since this is a class and can handle the memory + // automatically (with member constructors/destructors) + // Keeping it the same for now. + + Database* database_; + uint32_t id_; + leveldb::Slice* start_; + std::string* end_; + bool reverse_; + bool keys_; + bool values_; + int limit_; + std::string* lt_; + std::string* lte_; + std::string* gt_; + std::string* gte_; + bool keyAsBuffer_; + bool valueAsBuffer_; + uint32_t highWaterMark_; + leveldb::Iterator* dbIterator_; + int count_; + leveldb::Slice* target_; + bool seeking_; + bool landed_; + bool nexting_; + bool ended_; + + leveldb::ReadOptions options_; + // TODO what is this used for and how can we do it otherwise? + // AsyncWorker* endWorker_; +}; + +/** + * Runs when an Iterator is garbage collected. + */ +static void FinalizeIterator(napi_env env, void* data, void* hint) { + if (data) { + // TODO this might be incorrect, at the moment mimicing the behavior + // of Database. We might need to review the life cycle of Iterator + // and if it's garbage collected it might be enough to unhook itself + // from the Database (it has a pointer to it and could do this from + // its destructor). + delete (Iterator*)data; + } +} + +/** + * Macro to copy memory from a buffer or string. + */ +#define LD_STRING_OR_BUFFER_TO_COPY(env, from, to) \ + char* to##Ch_ = 0; \ + size_t to##Sz_ = 0; \ + if (IsBuffer(env, from)) { \ + char* buf = 0; \ + napi_get_buffer_info(env, from, (void **)&buf, &to##Sz_); \ + to##Ch_ = new char[to##Sz_]; \ + memcpy(to##Ch_, buf, to##Sz_); \ + printf("LD_STRING_OR_BUFFER_TO_COPY BUFFER length: %d content: %s\n", to##Sz_, to##Ch_); \ + } else if (IsString(env, from)) { \ + napi_get_value_string_utf8(env, from, NULL, 0, &to##Sz_); \ + to##Ch_ = new char[to##Sz_ + 1]; \ + napi_get_value_string_utf8(env, from, to##Ch_, to##Sz_ + 1, &to##Sz_); \ + to##Ch_[to##Sz_] = '\0'; \ + printf("LD_STRING_OR_BUFFER_TO_COPY STRING length: %d content: %s\n", to##Sz_, to##Ch_); \ + } + +/** + * Returns length of string or buffer + */ +static size_t StringOrBufferLength(napi_env env, napi_value value) { + size_t size = 0; + + if (IsString(env, value)) { + napi_get_value_string_utf8(env, value, NULL, 0, &size); + } else if (IsBuffer(env, value)) { + char* buf; + napi_get_buffer_info(env, value, (void **)&buf, &size); + } + + return size; +} + /** * Creates an iterator. */ @@ -399,11 +596,156 @@ NAPI_METHOD(iterator) { NAPI_ARGV(2); NAPI_DB_CONTEXT(); - // TODO assume options object is always passed in! it will - // make the code simpler + napi_value options = argv[1]; + bool reverse = BooleanProperty(env, options, "reverse", false); + bool keys = BooleanProperty(env, options, "keys", true); + bool values = BooleanProperty(env, options, "values", true); + bool fillCache = BooleanProperty(env, options, "fillCache", false); + bool keyAsBuffer = BooleanProperty(env, options, "keyAsBuffer", true); + bool valueAsBuffer = BooleanProperty(env, options, "valueAsBuffer", true); + int limit = Int32Property(env, options, "limit", -1); + uint32_t highWaterMark = Uint32Property(env, options, "highWaterMark", + 16 * 1024); + + // TODO simplify and refactor the hideous code below + + leveldb::Slice* start = NULL; + char *startStr = NULL; + if (HasProperty(env, options, "start")) { + napi_value value = GetProperty(env, options, "start"); + if (IsString(env, value) || IsBuffer(env, value)) { + if (StringOrBufferLength(env, value) > 0) { + LD_STRING_OR_BUFFER_TO_COPY(env, value, _start); + start = new leveldb::Slice(_startCh_, _startSz_); + startStr = _startCh_; + } + } + } + + std::string* end = NULL; + if (HasProperty(env, options, "end")) { + napi_value value = GetProperty(env, options, "end"); + if (IsString(env, value) || IsBuffer(env, value)) { + if (StringOrBufferLength(env, value) > 0) { + LD_STRING_OR_BUFFER_TO_COPY(env, value, _end); + end = new std::string(_endCh_, _endSz_); + delete [] _endCh_; + } + } + } - // TODO return an external for IteratorContext - NAPI_RETURN_UNDEFINED(); + std::string* lt = NULL; + if (HasProperty(env, options, "lt")) { + napi_value value = GetProperty(env, options, "lt"); + if (IsString(env, value) || IsBuffer(env, value)) { + if (StringOrBufferLength(env, value) > 0) { + LD_STRING_OR_BUFFER_TO_COPY(env, value, _lt); + lt = new std::string(_ltCh_, _ltSz_); + delete [] _ltCh_; + if (reverse) { + if (startStr != NULL) { + delete [] startStr; + startStr = NULL; + } + if (start != NULL) { + delete start; + } + start = new leveldb::Slice(lt->data(), lt->size()); + } + } + } + } + + std::string* lte = NULL; + if (HasProperty(env, options, "lte")) { + napi_value value = GetProperty(env, options, "lte"); + if (IsString(env, value) || IsBuffer(env, value)) { + if (StringOrBufferLength(env, value) > 0) { + LD_STRING_OR_BUFFER_TO_COPY(env, value, _lte); + lte = new std::string(_lteCh_, _lteSz_); + delete [] _lteCh_; + if (reverse) { + if (startStr != NULL) { + delete [] startStr; + startStr = NULL; + } + if (start != NULL) { + delete start; + } + start = new leveldb::Slice(lte->data(), lte->size()); + } + } + } + } + + std::string* gt = NULL; + if (HasProperty(env, options, "gt")) { + napi_value value = GetProperty(env, options, "gt"); + if (IsString(env, value) || IsBuffer(env, value)) { + if (StringOrBufferLength(env, value) > 0) { + LD_STRING_OR_BUFFER_TO_COPY(env, value, _gt); + gt = new std::string(_gtCh_, _gtSz_); + delete [] _gtCh_; + if (!reverse) { + if (startStr != NULL) { + delete [] startStr; + startStr = NULL; + } + if (start != NULL) { + delete start; + } + start = new leveldb::Slice(gt->data(), gt->size()); + } + } + } + } + + std::string* gte = NULL; + if (HasProperty(env, options, "gte")) { + napi_value value = GetProperty(env, options, "gte"); + if (IsString(env, value) || IsBuffer(env, value)) { + if (StringOrBufferLength(env, value) > 0) { + LD_STRING_OR_BUFFER_TO_COPY(env, value, _gte); + gte = new std::string(_gteCh_, _gteSz_); + delete [] _gteCh_; + if (!reverse) { + if (startStr != NULL) { + delete [] startStr; + startStr = NULL; + } + if (start != NULL) { + delete start; + } + start = new leveldb::Slice(gte->data(), gte->size()); + } + } + } + } + + uint32_t id = database->currentIteratorId_++; + Iterator* iterator = new Iterator(database, + id, + start, + end, + reverse, + keys, + values, + limit, + lt, + lte, + gt, + gte, + fillCache, + keyAsBuffer, + valueAsBuffer, + highWaterMark); + database->iterators_[id] = iterator; + + napi_value result; + NAPI_STATUS_THROWS(napi_create_external(env, iterator, + FinalizeIterator, + NULL, &result)); + return result; } /** diff --git a/src/database.h b/src/database.h index ef1b4fae..6bbc05db 100644 --- a/src/database.h +++ b/src/database.h @@ -78,12 +78,14 @@ class Database : public Nan::ObjectWrap { Nan::Utf8String* location; leveldb::DB* db; uint32_t currentIteratorId; + // What is this for? void(*pendingCloseWorker); leveldb::Cache* blockCache; const leveldb::FilterPolicy* filterPolicy; std::map< uint32_t, leveldown::Iterator * > iterators; + // What are these for? static void WriteDoing(uv_work_t *req); static void WriteAfter(uv_work_t *req); From 182a58bf72b5a874cd85a5e7893df7ec20a02247 Mon Sep 17 00:00:00 2001 From: Lars-Magnus Skog Date: Fri, 7 Dec 2018 22:21:16 +0100 Subject: [PATCH 10/42] Reuse LD_STRING_OR_BUFFER_TO_COPY() macro in ToSlice() function The ToSlice() function corresponds to the old LD_STRING_OR_BUFFER_TO_SLICE() macro with the exception that we _always_ copy the buffer/string. I've chosen this approach for the following reasons: * We use the _same_ code for copying buffers/strings for iterators (e.g. for lt, lte, gt, gte etc) * The code for handling cleanup of leveldb::Slice() should always cleanup the underlying buffer (since we always copy) * For simplicity. At the moment of rewriting to napi it's a lot to keep in the head to move forward. Later on we should benchmark this and tweak/optimize if needed. Maybe it has a huge impact on performance, but until we know this for sure, I think we can simplify with good conscience. The difference between old LD_STRING_OR_BUFFER_TO_COPY --- napi/leveldown.cc | 79 +++++++++++++---------------------------------- 1 file changed, 22 insertions(+), 57 deletions(-) diff --git a/napi/leveldown.cc b/napi/leveldown.cc index 88c3388b..74a088b6 100644 --- a/napi/leveldown.cc +++ b/napi/leveldown.cc @@ -357,46 +357,31 @@ static bool IsBuffer(napi_env env, napi_value value) { } /** - * Convert a napi_value to a leveldb::Slice. + * Macro to copy memory from a buffer or string. */ -static leveldb::Slice ToSlice(napi_env env, napi_value from) { - // size_t to ## Sz_; - // char* to ## Ch_; - // if (from->IsNull() || from->IsUndefined()) { - // to ## Sz_ = 0; - // to ## Ch_ = 0; - // } else if (!from->ToObject().IsEmpty() - // && node::Buffer::HasInstance(from->ToObject())) { - // to ## Sz_ = node::Buffer::Length(from->ToObject()); - // to ## Ch_ = node::Buffer::Data(from->ToObject()); - // } else { - // v8::Local to ## Str = from->ToString(); - // to ## Sz_ = to ## Str->Utf8Length(); - // to ## Ch_ = new char[to ## Sz_]; - // to ## Str->WriteUtf8(to ## Ch_, -1, NULL, v8::String::NO_NULL_TERMINATION); - // } - // leveldb::Slice to(to ## Ch_, to ## Sz_); - size_t toLength = 0; - char* toChar = 0; - - if (IsString(env, from)) { - printf("FROM is a string\n"); - size_t size = 0; - napi_get_value_string_utf8(env, from, NULL, 0, &size); - if (size > 0) { - // TODO use new here - toChar = (char*)malloc((size + 1) * sizeof(char)); - napi_get_value_string_utf8(env, from, toChar, size + 1, &toLength); - toChar[toLength] = '\0'; - } - } else if (IsBuffer(env, from)) { - printf("FROM is a buffer\n"); - // TODO what to do with buffers? we could either always - // copy them, or use an out parameter to store the fact - // that we did allocate and only clean in that case +#define LD_STRING_OR_BUFFER_TO_COPY(env, from, to) \ + char* to##Ch_ = 0; \ + size_t to##Sz_ = 0; \ + if (IsString(env, from)) { \ + napi_get_value_string_utf8(env, from, NULL, 0, &to##Sz_); \ + to##Ch_ = new char[to##Sz_ + 1]; \ + napi_get_value_string_utf8(env, from, to##Ch_, to##Sz_ + 1, &to##Sz_); \ + to##Ch_[to##Sz_] = '\0'; \ + printf("LD_STRING_OR_BUFFER_TO_COPY STRING length: %d content: %s\n", to##Sz_, to##Ch_); \ + } else if (IsBuffer(env, from)) { \ + char* buf = 0; \ + napi_get_buffer_info(env, from, (void **)&buf, &to##Sz_); \ + to##Ch_ = new char[to##Sz_]; \ + memcpy(to##Ch_, buf, to##Sz_); \ + printf("LD_STRING_OR_BUFFER_TO_COPY BUFFER length: %d content: %s\n", to##Sz_, to##Ch_); \ } - return leveldb::Slice(toChar, toLength); +/** + * Convert a napi_value to a leveldb::Slice. + */ +static leveldb::Slice ToSlice(napi_env env, napi_value from) { + LD_STRING_OR_BUFFER_TO_COPY(env, from, to); + return leveldb::Slice(toCh_, toSz_); } /** @@ -553,26 +538,6 @@ static void FinalizeIterator(napi_env env, void* data, void* hint) { } } -/** - * Macro to copy memory from a buffer or string. - */ -#define LD_STRING_OR_BUFFER_TO_COPY(env, from, to) \ - char* to##Ch_ = 0; \ - size_t to##Sz_ = 0; \ - if (IsBuffer(env, from)) { \ - char* buf = 0; \ - napi_get_buffer_info(env, from, (void **)&buf, &to##Sz_); \ - to##Ch_ = new char[to##Sz_]; \ - memcpy(to##Ch_, buf, to##Sz_); \ - printf("LD_STRING_OR_BUFFER_TO_COPY BUFFER length: %d content: %s\n", to##Sz_, to##Ch_); \ - } else if (IsString(env, from)) { \ - napi_get_value_string_utf8(env, from, NULL, 0, &to##Sz_); \ - to##Ch_ = new char[to##Sz_ + 1]; \ - napi_get_value_string_utf8(env, from, to##Ch_, to##Sz_ + 1, &to##Sz_); \ - to##Ch_[to##Sz_] = '\0'; \ - printf("LD_STRING_OR_BUFFER_TO_COPY STRING length: %d content: %s\n", to##Sz_, to##Ch_); \ - } - /** * Returns length of string or buffer */ From 043508a785c9839986d2727dc4751eaed9fa2cd8 Mon Sep 17 00:00:00 2001 From: Lars-Magnus Skog Date: Sat, 8 Dec 2018 02:12:44 +0100 Subject: [PATCH 11/42] Implement iterator.next() and iterator.end() --- napi/leveldown.cc | 717 ++++++++++++++++++++++++++++++++++------------ napi/napi.h | 4 + 2 files changed, 541 insertions(+), 180 deletions(-) diff --git a/napi/leveldown.cc b/napi/leveldown.cc index 74a088b6..22598337 100644 --- a/napi/leveldown.cc +++ b/napi/leveldown.cc @@ -3,11 +3,119 @@ #include #include #include +#include /** * Forward declarations. */ +struct Database; struct Iterator; +struct EndWorker; + +/** + * Base worker class. + */ +struct BaseWorker { + BaseWorker (napi_env env, + Database* database, + napi_value callback, + const char* resourceName) + : env_(env), database_(database) { + napi_create_reference(env_, callback, 1, &callbackRef_); + napi_value asyncResourceName; + napi_create_string_utf8(env_, resourceName, + NAPI_AUTO_LENGTH, + &asyncResourceName); + napi_create_async_work(env_, callback, + asyncResourceName, + BaseWorker::Execute, + BaseWorker::Complete, + this, + &asyncWork_); + } + + virtual ~BaseWorker () { + napi_delete_reference(env_, callbackRef_); + napi_delete_async_work(env_, asyncWork_); + } + + /** + * Calls virtual DoExecute(). + */ + static void Execute (napi_env env, void* data) { + BaseWorker* self = (BaseWorker*)data; + self->DoExecute(); + } + + /** + * MUST be overriden. + */ + virtual void DoExecute () = 0; + + /** + * Calls DoComplete() and kills the worker. + */ + static void Complete (napi_env env, napi_status status, void* data) { + BaseWorker* self = (BaseWorker*)data; + self->DoComplete(); + delete self; + } + + /** + * Checks status and calls appropriate handler. + * - Calls virtual HandleOKCallback() if status is ok + * - Calls back with error if status is an error + */ + void DoComplete () { + if (status_.ok()) { + return HandleOKCallback(); + } + + // TODO the global, callback, and calling the function code + // could be refactored with HandleOKCallback() + + napi_value global; + napi_get_global(env_, &global); + napi_value callback; + napi_get_reference_value(env_, callbackRef_, &callback); + + const char* str = status_.ToString().c_str(); + napi_value msg; + napi_create_string_utf8(env_, str, strlen(str), &msg); + + const int argc = 1; + napi_value argv[argc]; + napi_create_error(env_, NULL, msg, &argv[0]); + + napi_call_function(env_, global, callback, argc, argv, NULL); + } + + /** + * Default behavior is to call back with NULL. + */ + virtual void HandleOKCallback () { + napi_value global; + napi_get_global(env_, &global); + napi_value callback; + napi_get_reference_value(env_, callbackRef_, &callback); + + const int argc = 1; + napi_value argv[argc]; + napi_get_null(env_, &argv[0]); + + napi_call_function(env_, global, callback, argc, argv, NULL); + } + + void Queue () { + napi_queue_async_work(env_, asyncWork_); + } + + napi_env env_; + napi_ref callbackRef_; + napi_async_work asyncWork_; + Database* database_; + leveldb::Status status_; +}; /** * Owns the LevelDB storage together with cache and filter policy. @@ -18,7 +126,8 @@ struct Database { db_(NULL), blockCache_(NULL), filterPolicy_(NULL), - currentIteratorId_(0) {} + currentIteratorId_(0), + pendingCloseWorker_(NULL) {} ~Database () { if (db_ != NULL) { @@ -27,21 +136,43 @@ struct Database { } } - leveldb::Status Open(const leveldb::Options& options, - const char* location) { + leveldb::Status Open (const leveldb::Options& options, + const char* location) { return leveldb::DB::Open(options, location, &db_); } - leveldb::Status Put(const leveldb::WriteOptions& options, - leveldb::Slice key, - leveldb::Slice value) { + leveldb::Status Put (const leveldb::WriteOptions& options, + leveldb::Slice key, + leveldb::Slice value) { return db_->Put(options, key, value); } - const leveldb::Snapshot* NewSnapshot() { + const leveldb::Snapshot* NewSnapshot () { return db_->GetSnapshot(); } + leveldb::Iterator* NewIterator (leveldb::ReadOptions* options) { + return db_->NewIterator(*options); + } + + void ReleaseSnapshot (const leveldb::Snapshot* snapshot) { + return db_->ReleaseSnapshot(snapshot); + } + + void ReleaseIterator (uint32_t id) { + // called each time an Iterator is End()ed, in the main thread + // we have to remove our reference to it and if it's the last + // iterator we have to invoke a pending CloseWorker if there + // is one if there is a pending CloseWorker it means that + // we're waiting for iterators to end before we can close them + iterators_.erase(id); + if (iterators_.empty() && pendingCloseWorker_ != NULL) { + // TODO Test this! + pendingCloseWorker_->Queue(); + pendingCloseWorker_ = NULL; + } + } + napi_env env_; leveldb::DB* db_; leveldb::Cache* blockCache_; @@ -50,13 +181,18 @@ struct Database { // time in the constructor? const leveldb::FilterPolicy* filterPolicy_; uint32_t currentIteratorId_; + // TODO this worker should have a proper type, now + // it's some form of function pointer bastard, it should + // just be a CloseWorker object + //void (*pendingCloseWorker_); + BaseWorker *pendingCloseWorker_; std::map< uint32_t, Iterator * > iterators_; }; /** * Runs when a Database is garbage collected. */ -static void FinalizeDatabase(napi_env env, void* data, void* hint) { +static void FinalizeDatabase (napi_env env, void* data, void* hint) { if (data) { delete (Database*)data; } @@ -80,7 +216,9 @@ NAPI_METHOD(db) { /** * Returns true if 'obj' has a property 'key' */ -static bool HasProperty(napi_env env, napi_value obj, const char* key) { +static bool HasProperty (napi_env env, + napi_value obj, + const char* key) { bool has = false; napi_value _key; napi_create_string_utf8(env, key, strlen(key), &_key); @@ -91,9 +229,9 @@ static bool HasProperty(napi_env env, napi_value obj, const char* key) { /** * Returns a property in napi_value form */ -static napi_value GetProperty(napi_env env, - napi_value obj, - const char* key) { +static napi_value GetProperty (napi_env env, + napi_value obj, + const char* key) { napi_value _key; napi_create_string_utf8(env, key, strlen(key), &_key); @@ -107,10 +245,10 @@ static napi_value GetProperty(napi_env env, * Returns a boolean property 'key' from 'obj'. * Returns 'DEFAULT' if the property doesn't exist. */ -static bool BooleanProperty(napi_env env, - napi_value obj, - const char* key, - bool DEFAULT) { +static bool BooleanProperty (napi_env env, + napi_value obj, + const char* key, + bool DEFAULT) { if (HasProperty(env, obj, key)) { napi_value value = GetProperty(env, obj, key); bool result; @@ -125,10 +263,10 @@ static bool BooleanProperty(napi_env env, * Returns a uint32 property 'key' from 'obj' * Returns 'DEFAULT' if the property doesn't exist. */ -static uint32_t Uint32Property(napi_env env, - napi_value obj, - const char* key, - uint32_t DEFAULT) { +static uint32_t Uint32Property (napi_env env, + napi_value obj, + const char* key, + uint32_t DEFAULT) { if (HasProperty(env, obj, key)) { napi_value value = GetProperty(env, obj, key); uint32_t result; @@ -143,10 +281,10 @@ static uint32_t Uint32Property(napi_env env, * Returns a uint32 property 'key' from 'obj' * Returns 'DEFAULT' if the property doesn't exist. */ -static int Int32Property(napi_env env, - napi_value obj, - const char* key, - int DEFAULT) { +static int Int32Property (napi_env env, + napi_value obj, + const char* key, + int DEFAULT) { bool exists = false; napi_value _key; napi_create_string_utf8(env, key, strlen(key), &_key); @@ -163,108 +301,22 @@ static int Int32Property(napi_env env, return DEFAULT; } -/** - * Base worker class. - */ -struct BaseWorker { - BaseWorker(napi_env env, - Database* database, - napi_value callback, - const char* resourceName) - : env_(env), database_(database) { - napi_create_reference(env_, callback, 1, &callbackRef_); - napi_value asyncResourceName; - napi_create_string_utf8(env_, resourceName, - NAPI_AUTO_LENGTH, - &asyncResourceName); - napi_create_async_work(env_, callback, - asyncResourceName, - BaseWorker::Execute, - BaseWorker::Complete, - this, - &asyncWork_); - } - - virtual ~BaseWorker() { - napi_delete_reference(env_, callbackRef_); - napi_delete_async_work(env_, asyncWork_); - } - - /** - * Calls virtual DoExecute(). - */ - static void Execute(napi_env env, void* data) { - BaseWorker* self = (BaseWorker*)data; - self->DoExecute(); - } - - /** - * MUST be overriden. - */ - virtual void DoExecute() = 0; - - /** - * Calls DoComplete() and kills the worker. - */ - static void Complete(napi_env env, napi_status status, void* data) { - BaseWorker* self = (BaseWorker*)data; - self->DoComplete(); - delete self; - } - - /** - * Default handling when work is complete. - * - Calls back with NULL if no error - * - Calls back with error if status is an error - */ - virtual void DoComplete() { - const int argc = 1; - napi_value argv[argc]; - - napi_value global; - napi_get_global(env_, &global); - napi_value callback; - napi_get_reference_value(env_, callbackRef_, &callback); - - if (status_.ok()) { - napi_get_null(env_, &argv[0]); - } else { - const char* str = status_.ToString().c_str(); - napi_value msg; - napi_create_string_utf8(env_, str, strlen(str), &msg); - napi_create_error(env_, NULL, msg, &argv[0]); - } - - napi_call_function(env_, global, callback, argc, argv, NULL); - } - - void Queue() { - napi_queue_async_work(env_, asyncWork_); - } - - napi_env env_; - napi_ref callbackRef_; - napi_async_work asyncWork_; - Database* database_; - leveldb::Status status_; -}; - /** * Worker class for opening the database */ struct OpenWorker : public BaseWorker { - OpenWorker(napi_env env, - Database* database, - napi_value callback, - char* location, - bool createIfMissing, - bool errorIfExists, - bool compression, - uint32_t writeBufferSize, - uint32_t blockSize, - uint32_t maxOpenFiles, - uint32_t blockRestartInterval, - uint32_t maxFileSize) + OpenWorker (napi_env env, + Database* database, + napi_value callback, + char* location, + bool createIfMissing, + bool errorIfExists, + bool compression, + uint32_t writeBufferSize, + uint32_t blockSize, + uint32_t maxOpenFiles, + uint32_t blockRestartInterval, + uint32_t maxFileSize) : BaseWorker(env, database, callback, "leveldown::open"), location_(location) { options_.block_cache = database->blockCache_; @@ -281,11 +333,11 @@ struct OpenWorker : public BaseWorker { options_.max_file_size = maxFileSize; } - virtual ~OpenWorker() { + virtual ~OpenWorker () { free(location_); } - virtual void DoExecute() { + virtual void DoExecute () { status_ = database_->Open(options_, location_); } @@ -341,7 +393,7 @@ NAPI_METHOD(db_open) { /** * Returns true if 'value' is a string otherwise false. */ -static bool IsString(napi_env env, napi_value value) { +static bool IsString (napi_env env, napi_value value) { napi_valuetype type; napi_typeof(env, value, &type); return type == napi_string; @@ -350,7 +402,7 @@ static bool IsString(napi_env env, napi_value value) { /** * Returns true if 'value' is a buffer otherwise false. */ -static bool IsBuffer(napi_env env, napi_value value) { +static bool IsBuffer (napi_env env, napi_value value) { bool isBuffer; napi_is_buffer(env, value, &isBuffer); return isBuffer; @@ -367,19 +419,19 @@ static bool IsBuffer(napi_env env, napi_value value) { to##Ch_ = new char[to##Sz_ + 1]; \ napi_get_value_string_utf8(env, from, to##Ch_, to##Sz_ + 1, &to##Sz_); \ to##Ch_[to##Sz_] = '\0'; \ - printf("LD_STRING_OR_BUFFER_TO_COPY STRING length: %d content: %s\n", to##Sz_, to##Ch_); \ + printf("LD_STRING_OR_BUFFER_TO_COPY STRING length: %zu content: %s\n", to##Sz_, to##Ch_); \ } else if (IsBuffer(env, from)) { \ char* buf = 0; \ napi_get_buffer_info(env, from, (void **)&buf, &to##Sz_); \ to##Ch_ = new char[to##Sz_]; \ memcpy(to##Ch_, buf, to##Sz_); \ - printf("LD_STRING_OR_BUFFER_TO_COPY BUFFER length: %d content: %s\n", to##Sz_, to##Ch_); \ + printf("LD_STRING_OR_BUFFER_TO_COPY BUFFER length: %zu content: %s\n", to##Sz_, to##Ch_); \ } /** * Convert a napi_value to a leveldb::Slice. */ -static leveldb::Slice ToSlice(napi_env env, napi_value from) { +static leveldb::Slice ToSlice (napi_env env, napi_value from) { LD_STRING_OR_BUFFER_TO_COPY(env, from, to); return leveldb::Slice(toCh_, toSz_); } @@ -388,23 +440,23 @@ static leveldb::Slice ToSlice(napi_env env, napi_value from) { * Worker class for putting key/value to the database */ struct PutWorker : public BaseWorker { - PutWorker(napi_env env, - Database* database, - napi_value callback, - napi_value key, - napi_value value, - bool sync) + PutWorker (napi_env env, + Database* database, + napi_value callback, + napi_value key, + napi_value value, + bool sync) : BaseWorker(env, database, callback, "leveldown::put"), key_(ToSlice(env, key)), value_(ToSlice(env, value)) { options_.sync = sync; } - ~PutWorker() { + virtual ~PutWorker () { // TODO clean up key_ and value_ if they aren't empty? // See DisposeStringOrBufferFromSlice() } - virtual void DoExecute() { + virtual void DoExecute () { status_ = database_->Put(options_, key_, value_); } @@ -438,22 +490,22 @@ NAPI_METHOD(db_put) { * Owns a leveldb iterator. */ struct Iterator { - Iterator(Database* database, - uint32_t id, - leveldb::Slice* start, - std::string* end, - bool reverse, - bool keys, - bool values, - int limit, - std::string* lt, - std::string* lte, - std::string* gt, - std::string* gte, - bool fillCache, - bool keyAsBuffer, - bool valueAsBuffer, - uint32_t highWaterMark) + Iterator (Database* database, + uint32_t id, + leveldb::Slice* start, + std::string* end, + bool reverse, + bool keys, + bool values, + int limit, + std::string* lt, + std::string* lte, + std::string* gt, + std::string* gte, + bool fillCache, + bool keyAsBuffer, + bool valueAsBuffer, + uint32_t highWaterMark) : database_(database), id_(id), start_(start), @@ -475,26 +527,189 @@ struct Iterator { seeking_(false), landed_(false), nexting_(false), - ended_(false) { - // TODO need something different for endWorker_ - // endWorker_ = NULL; - // options = new leveldb::ReadOptions(); - options_.fill_cache = fillCache; - options_.snapshot = database->NewSnapshot(); + ended_(false), + endWorker_(NULL) { + options_ = new leveldb::ReadOptions(); + options_->fill_cache = fillCache; + options_->snapshot = database->NewSnapshot(); + } + + ~Iterator () { + ReleaseTarget(); + if (start_ != NULL) { + // Special case for `start` option: it won't be + // freed up by any of the delete calls below. + if (!((lt_ != NULL && reverse_) + || (lte_ != NULL && reverse_) + || (gt_ != NULL && !reverse_) + || (gte_ != NULL && !reverse_))) { + delete [] start_->data(); + } + delete start_; + } + if (end_ != NULL) { + delete end_; + } + if (lt_ != NULL) { + delete lt_; + } + if (gt_ != NULL) { + delete gt_; + } + if (lte_ != NULL) { + delete lte_; + } + if (gte_ != NULL) { + delete gte_; + } + } + + void ReleaseTarget () { + if (target_ != NULL) { + if (!target_->empty()) { + delete [] target_->data(); + } + delete target_; + target_ = NULL; + } + } + + void Release () { + database_->ReleaseIterator(id_); + } + + leveldb::Status IteratorStatus () { + return dbIterator_->status(); + } + + void IteratorEnd () { + delete dbIterator_; + dbIterator_ = NULL; + database_->ReleaseSnapshot(options_->snapshot); } - ~Iterator() { + bool GetIterator () { + if (dbIterator_ == NULL) { + dbIterator_ = database_->NewIterator(options_); + + if (start_ != NULL) { + dbIterator_->Seek(*start_); + + if (reverse_) { + if (!dbIterator_->Valid()) { + // if it's past the last key, step back + dbIterator_->SeekToLast(); + } else { + std::string keyStr = dbIterator_->key().ToString(); + + if (lt_ != NULL) { + // TODO make a compount if statement + if (lt_->compare(keyStr) <= 0) { + dbIterator_->Prev(); + } + } else if (lte_ != NULL) { + // TODO make a compount if statement + if (lte_->compare(keyStr) < 0) { + dbIterator_->Prev(); + } + } else if (start_ != NULL) { + // TODO make a compount if statement + if (start_->compare(keyStr)) { + dbIterator_->Prev(); + } + } + } + + if (dbIterator_->Valid() && lt_ != NULL) { + if (lt_->compare(dbIterator_->key().ToString()) <= 0) { + dbIterator_->Prev(); + } + } + } else { + // TODO this could be an else if + if (dbIterator_->Valid() && gt_ != NULL + && gt_->compare(dbIterator_->key().ToString()) == 0) { + dbIterator_->Next(); + } + } + } else if (reverse_) { + dbIterator_->SeekToLast(); + } else { + dbIterator_->SeekToFirst(); + } + + return true; + } + + return false; + } + + bool Read (std::string& key, std::string& value) { + // if it's not the first call, move to next item. + if (!GetIterator() && !seeking_) { + if (reverse_) { + dbIterator_->Prev(); + } + else { + dbIterator_->Next(); + } + } + + seeking_ = false; + + // now check if this is the end or not, if not then return the key & value + if (dbIterator_->Valid()) { + std::string keyStr = dbIterator_->key().ToString(); + const int isEnd = end_ == NULL ? 1 : end_->compare(keyStr); + + if ((limit_ < 0 || ++count_ <= limit_) + && (end_ == NULL + || (reverse_ && (isEnd <= 0)) + || (!reverse_ && (isEnd >= 0))) + && ( lt_ != NULL ? (lt_->compare(keyStr) > 0) + : lte_ != NULL ? (lte_->compare(keyStr) >= 0) + : true ) + && ( gt_ != NULL ? (gt_->compare(keyStr) < 0) + : gte_ != NULL ? (gte_->compare(keyStr) <= 0) + : true ) + ) { + if (keys_) { + key.assign(dbIterator_->key().data(), dbIterator_->key().size()); + } + if (values_) { + value.assign(dbIterator_->value().data(), dbIterator_->value().size()); + } + return true; + } + } + return false; } - // TODO pull in all methods from src/iterator.cc and go - // through EACH of them and make sure that they are used - // and WHERE they are used + bool IteratorNext (std::vector >& result) { + size_t size = 0; + while (true) { + std::string key, value; + bool ok = Read(key, value); - // TODO can we get rid of pointers below? Generally it shouldn't - // be needed since this is a class and can handle the memory - // automatically (with member constructors/destructors) - // Keeping it the same for now. + if (ok) { + result.push_back(std::make_pair(key, value)); + + if (!landed_) { + landed_ = true; + return true; + } + + size = size + key.size() + value.size(); + if (size > highWaterMark_) { + return true; + } + + } else { + return false; + } + } + } Database* database_; uint32_t id_; @@ -519,15 +734,14 @@ struct Iterator { bool nexting_; bool ended_; - leveldb::ReadOptions options_; - // TODO what is this used for and how can we do it otherwise? - // AsyncWorker* endWorker_; + leveldb::ReadOptions* options_; + EndWorker* endWorker_; }; /** * Runs when an Iterator is garbage collected. */ -static void FinalizeIterator(napi_env env, void* data, void* hint) { +static void FinalizeIterator (napi_env env, void* data, void* hint) { if (data) { // TODO this might be incorrect, at the moment mimicing the behavior // of Database. We might need to review the life cycle of Iterator @@ -541,7 +755,7 @@ static void FinalizeIterator(napi_env env, void* data, void* hint) { /** * Returns length of string or buffer */ -static size_t StringOrBufferLength(napi_env env, napi_value value) { +static size_t StringOrBufferLength (napi_env env, napi_value value) { size_t size = 0; if (IsString(env, value)) { @@ -718,27 +932,170 @@ NAPI_METHOD(iterator) { */ NAPI_METHOD(iterator_seek) { NAPI_ARGV(2); - // NAPI_ITERATOR_CONTEXT(); + NAPI_ITERATOR_CONTEXT(); NAPI_RETURN_UNDEFINED(); } /** - * Moves an iterator to next element. + * Worker class for ending an iterator */ -NAPI_METHOD(iterator_next) { +struct EndWorker : public BaseWorker { + EndWorker (napi_env env, + Iterator* iterator, + napi_value callback) + : BaseWorker(env, iterator->database_, callback, + "leveldown::iterator.end"), + iterator_(iterator) {} + + virtual ~EndWorker () {} + + virtual void DoExecute () { + iterator_->IteratorEnd(); + } + + virtual void HandleOKCallback () { + iterator_->Release(); + BaseWorker::HandleOKCallback(); + } + + Iterator* iterator_; +}; + +/** + * Ends an iterator. + */ +NAPI_METHOD(iterator_end) { NAPI_ARGV(2); - // NAPI_ITERATOR_CONTEXT(); + NAPI_ITERATOR_CONTEXT(); + + // TODO assuming callback is always provided + napi_value callback = argv[1]; + + if (!iterator->ended_) { + EndWorker* worker = new EndWorker(env, iterator, callback); + iterator->ended_ = true; + + if (iterator->nexting_) { + // waiting for a next() to return, queue the end + iterator->endWorker_ = worker; + } else { + worker->Queue(); + } + } NAPI_RETURN_UNDEFINED(); } +void CheckEndCallback (Iterator* iterator) { + iterator->ReleaseTarget(); + iterator->nexting_ = false; + if (iterator->endWorker_ != NULL) { + iterator->endWorker_->Queue(); + iterator->endWorker_ = NULL; + } +} + /** - * Ends an iterator. + * Worker class for nexting an iterator */ -NAPI_METHOD(iterator_end) { +struct NextWorker : public BaseWorker { + NextWorker (napi_env env, + Iterator* iterator, + napi_value callback, + void (*localCallback)(Iterator*)) + : BaseWorker(env, iterator->database_, callback, + "leveldown::iterator.next"), + iterator_(iterator), + localCallback_(localCallback) {} + + virtual ~NextWorker () {} + + virtual void DoExecute () { + ok_ = iterator_->IteratorNext(result_); + if (!ok_) { + status_ = iterator_->IteratorStatus(); + } + } + + virtual void HandleOKCallback () { + size_t arraySize = result_.size() * 2; + napi_value jsArray; + napi_create_array_with_length(env_, arraySize, &jsArray); + + for (size_t idx = 0; idx < result_.size(); ++idx) { + std::pair row = result_[idx]; + std::string key = row.first; + std::string value = row.second; + + napi_value returnKey; + if (iterator_->keyAsBuffer_) { + napi_create_buffer_copy(env_, key.size(), key.data(), NULL, &returnKey); + } else { + napi_create_string_utf8(env_, key.data(), key.size(), &returnKey); + } + + napi_value returnValue; + if (iterator_->valueAsBuffer_) { + napi_create_buffer_copy(env_, value.size(), value.data(), NULL, &returnValue); + } else { + napi_create_string_utf8(env_, value.data(), value.size(), &returnValue); + } + + // put the key & value in a descending order, so that they can be .pop:ed in javascript-land + napi_set_element(env_, jsArray, static_cast(arraySize - idx * 2 - 1), returnKey); + napi_set_element(env_, jsArray, static_cast(arraySize - idx * 2 - 2), returnValue); + } + + // clean up & handle the next/end state + localCallback_(iterator_); + + const int argc = 3; + napi_value argv[argc]; + + napi_get_null(env_, &argv[0]); + argv[1] = jsArray; + napi_get_boolean(env_, !ok_, &argv[2]); + + // TODO move to base class + napi_value global; + napi_get_global(env_, &global); + napi_value callback; + napi_get_reference_value(env_, callbackRef_, &callback); + + napi_call_function(env_, global, callback, argc, argv, NULL); + } + + Iterator* iterator_; + // TODO(cosmetic) create a typedef for this function pointer + void (*localCallback_)(Iterator*); + std::vector > result_; + bool ok_; +}; + +/** + * Moves an iterator to next element. + */ +NAPI_METHOD(iterator_next) { NAPI_ARGV(2); - // NAPI_ITERATOR_CONTEXT(); + NAPI_ITERATOR_CONTEXT(); + + // TODO assuming callback is always provided + napi_value callback = argv[1]; + + if (iterator->ended_) { + // TODO call back with error "iterator has ended" + //v8::Local argv[] = { Nan::Error("iterator has ended") }; + //LD_RUN_CALLBACK("leveldown:iterator.next", callback, 1, argv); + NAPI_RETURN_UNDEFINED(); + } + + NextWorker* worker = new NextWorker(env, + iterator, + callback, + CheckEndCallback); + iterator->nexting_ = true; + worker->Queue(); NAPI_RETURN_UNDEFINED(); } @@ -750,6 +1107,6 @@ NAPI_INIT() { NAPI_EXPORT_FUNCTION(iterator); NAPI_EXPORT_FUNCTION(iterator_seek); - NAPI_EXPORT_FUNCTION(iterator_next); NAPI_EXPORT_FUNCTION(iterator_end); + NAPI_EXPORT_FUNCTION(iterator_next); } diff --git a/napi/napi.h b/napi/napi.h index ea1e1100..bfa6e8e3 100644 --- a/napi/napi.h +++ b/napi/napi.h @@ -5,6 +5,10 @@ Database* database = NULL; \ NAPI_STATUS_THROWS(napi_get_value_external(env, argv[0], (void**)&database)); +#define NAPI_ITERATOR_CONTEXT() \ + Iterator* iterator = NULL; \ + NAPI_STATUS_THROWS(napi_get_value_external(env, argv[0], (void**)&iterator)); + // TODO move to napi-macros.h #define NAPI_RETURN_UNDEFINED() \ return 0; From 57b0be4bfcd82f7f598f5366c3148a5ac37fac5f Mon Sep 17 00:00:00 2001 From: Lars-Magnus Skog Date: Sat, 8 Dec 2018 02:28:19 +0100 Subject: [PATCH 12/42] Implement db_close() This concludes the factory-test.js in abstract-leveldown --- leveldown.js | 2 +- napi/leveldown.cc | 56 ++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 51 insertions(+), 7 deletions(-) diff --git a/leveldown.js b/leveldown.js index 547624e7..b1bc5580 100644 --- a/leveldown.js +++ b/leveldown.js @@ -31,7 +31,7 @@ LevelDOWN.prototype._open = function (options, callback) { } LevelDOWN.prototype._close = function (callback) { - this.binding.close(callback) + binding.db_close(this.context, callback) } LevelDOWN.prototype._serializeKey = function (key) { diff --git a/napi/leveldown.cc b/napi/leveldown.cc index 22598337..7e44aecd 100644 --- a/napi/leveldown.cc +++ b/napi/leveldown.cc @@ -141,6 +141,19 @@ struct Database { return leveldb::DB::Open(options, location, &db_); } + void CloseDatabase () { + delete db_; + db_ = NULL; + if (blockCache_) { + delete blockCache_; + blockCache_ = NULL; + } + if (filterPolicy_) { + delete filterPolicy_; + filterPolicy_ = NULL; + } + } + leveldb::Status Put (const leveldb::WriteOptions& options, leveldb::Slice key, leveldb::Slice value) { @@ -302,7 +315,7 @@ static int Int32Property (napi_env env, } /** - * Worker class for opening the database + * Worker class for opening a database */ struct OpenWorker : public BaseWorker { OpenWorker (napi_env env, @@ -317,7 +330,7 @@ struct OpenWorker : public BaseWorker { uint32_t maxOpenFiles, uint32_t blockRestartInterval, uint32_t maxFileSize) - : BaseWorker(env, database, callback, "leveldown::open"), + : BaseWorker(env, database, callback, "leveldown.db.open"), location_(location) { options_.block_cache = database->blockCache_; options_.filter_policy = database->filterPolicy_; @@ -346,7 +359,7 @@ struct OpenWorker : public BaseWorker { }; /** - * Opens a database. + * Open a database. */ NAPI_METHOD(db_open) { NAPI_ARGV(4); @@ -390,6 +403,36 @@ NAPI_METHOD(db_open) { NAPI_RETURN_UNDEFINED(); } +/** + * Worker class for closing a database + */ +struct CloseWorker : public BaseWorker { + CloseWorker (napi_env env, + Database* database, + napi_value callback) + : BaseWorker(env, database, callback, "leveldown.db.close") {} + + virtual ~CloseWorker () {} + + virtual void DoExecute () { + database_->CloseDatabase(); + } +}; + +/** + * Close a database. + */ +NAPI_METHOD(db_close) { + NAPI_ARGV(2); + NAPI_DB_CONTEXT(); + + napi_value callback = argv[1]; + CloseWorker* worker = new CloseWorker(env, database, callback); + worker->Queue(); + + NAPI_RETURN_UNDEFINED(); +} + /** * Returns true if 'value' is a string otherwise false. */ @@ -446,7 +489,7 @@ struct PutWorker : public BaseWorker { napi_value key, napi_value value, bool sync) - : BaseWorker(env, database, callback, "leveldown::put"), + : BaseWorker(env, database, callback, "leveldown.db.put"), key_(ToSlice(env, key)), value_(ToSlice(env, value)) { options_.sync = sync; } @@ -945,7 +988,7 @@ struct EndWorker : public BaseWorker { Iterator* iterator, napi_value callback) : BaseWorker(env, iterator->database_, callback, - "leveldown::iterator.end"), + "leveldown.iterator.end"), iterator_(iterator) {} virtual ~EndWorker () {} @@ -1005,7 +1048,7 @@ struct NextWorker : public BaseWorker { napi_value callback, void (*localCallback)(Iterator*)) : BaseWorker(env, iterator->database_, callback, - "leveldown::iterator.next"), + "leveldown.iterator.next"), iterator_(iterator), localCallback_(localCallback) {} @@ -1103,6 +1146,7 @@ NAPI_METHOD(iterator_next) { NAPI_INIT() { NAPI_EXPORT_FUNCTION(db); NAPI_EXPORT_FUNCTION(db_open); + NAPI_EXPORT_FUNCTION(db_close); NAPI_EXPORT_FUNCTION(db_put); NAPI_EXPORT_FUNCTION(iterator); From 5613fb5bdd734fff6e7b884d464607921df4b5af Mon Sep 17 00:00:00 2001 From: Lars-Magnus Skog Date: Sat, 8 Dec 2018 02:58:15 +0100 Subject: [PATCH 13/42] Fix error messages (needs to be copied) --- napi/leveldown.cc | 35 +++++++++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/napi/leveldown.cc b/napi/leveldown.cc index 7e44aecd..bc1ecd9b 100644 --- a/napi/leveldown.cc +++ b/napi/leveldown.cc @@ -20,7 +20,7 @@ struct BaseWorker { Database* database, napi_value callback, const char* resourceName) - : env_(env), database_(database) { + : env_(env), database_(database), errMsg_(NULL) { napi_create_reference(env_, callback, 1, &callbackRef_); napi_value asyncResourceName; napi_create_string_utf8(env_, resourceName, @@ -35,6 +35,7 @@ struct BaseWorker { } virtual ~BaseWorker () { + delete [] errMsg_; napi_delete_reference(env_, callbackRef_); napi_delete_async_work(env_, asyncWork_); } @@ -47,6 +48,26 @@ struct BaseWorker { self->DoExecute(); } + /** + * Store status. Note that error message has to be copied. + */ + void SetStatus (leveldb::Status status) { + status_ = status; + if (!status.ok()) { + SetErrorMessage(status.ToString().c_str()); + } + } + + /** + * Copied error message. + */ + void SetErrorMessage(const char *msg) { + delete [] errMsg_; + size_t size = strlen(msg) + 1; + errMsg_ = new char[size]; + memcpy(errMsg_, msg, size); + } + /** * MUST be overriden. */ @@ -79,9 +100,8 @@ struct BaseWorker { napi_value callback; napi_get_reference_value(env_, callbackRef_, &callback); - const char* str = status_.ToString().c_str(); napi_value msg; - napi_create_string_utf8(env_, str, strlen(str), &msg); + napi_create_string_utf8(env_, errMsg_, strlen(errMsg_), &msg); const int argc = 1; napi_value argv[argc]; @@ -114,7 +134,10 @@ struct BaseWorker { napi_ref callbackRef_; napi_async_work asyncWork_; Database* database_; + +private: leveldb::Status status_; + char *errMsg_; }; /** @@ -351,7 +374,7 @@ struct OpenWorker : public BaseWorker { } virtual void DoExecute () { - status_ = database_->Open(options_, location_); + SetStatus(database_->Open(options_, location_)); } leveldb::Options options_; @@ -500,7 +523,7 @@ struct PutWorker : public BaseWorker { } virtual void DoExecute () { - status_ = database_->Put(options_, key_, value_); + SetStatus(database_->Put(options_, key_, value_)); } leveldb::WriteOptions options_; @@ -1057,7 +1080,7 @@ struct NextWorker : public BaseWorker { virtual void DoExecute () { ok_ = iterator_->IteratorNext(result_); if (!ok_) { - status_ = iterator_->IteratorStatus(); + SetStatus(iterator_->IteratorStatus()); } } From 2acd4b6b54ecee82f253c9183a3a36e5a98a5da6 Mon Sep 17 00:00:00 2001 From: Lars-Magnus Skog Date: Sat, 8 Dec 2018 03:33:23 +0100 Subject: [PATCH 14/42] Implement db_get() --- leveldown.js | 2 +- napi/leveldown.cc | 105 +++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 95 insertions(+), 12 deletions(-) diff --git a/leveldown.js b/leveldown.js index b1bc5580..077a00a6 100644 --- a/leveldown.js +++ b/leveldown.js @@ -53,7 +53,7 @@ LevelDOWN.prototype._put = function (key, value, options, callback) { } LevelDOWN.prototype._get = function (key, options, callback) { - this.binding.get(key, options, callback) + binding.db_get(this.context, key, options, callback) } LevelDOWN.prototype._del = function (key, options, callback) { diff --git a/napi/leveldown.cc b/napi/leveldown.cc index bc1ecd9b..63a72396 100644 --- a/napi/leveldown.cc +++ b/napi/leveldown.cc @@ -21,17 +21,16 @@ struct BaseWorker { napi_value callback, const char* resourceName) : env_(env), database_(database), errMsg_(NULL) { - napi_create_reference(env_, callback, 1, &callbackRef_); + NAPI_STATUS_THROWS(napi_create_reference(env_, callback, 1, &callbackRef_)); napi_value asyncResourceName; - napi_create_string_utf8(env_, resourceName, - NAPI_AUTO_LENGTH, - &asyncResourceName); - napi_create_async_work(env_, callback, - asyncResourceName, - BaseWorker::Execute, - BaseWorker::Complete, - this, - &asyncWork_); + NAPI_STATUS_THROWS(napi_create_string_utf8(env_, resourceName, + NAPI_AUTO_LENGTH, + &asyncResourceName)); + NAPI_STATUS_THROWS(napi_create_async_work(env_, callback, + asyncResourceName, + BaseWorker::Execute, + BaseWorker::Complete, + this, &asyncWork_)); } virtual ~BaseWorker () { @@ -183,6 +182,12 @@ struct Database { return db_->Put(options, key, value); } + leveldb::Status Get (const leveldb::ReadOptions& options, + leveldb::Slice key, + std::string& value) { + return db_->Get(options, key, &value); + } + const leveldb::Snapshot* NewSnapshot () { return db_->GetSnapshot(); } @@ -532,7 +537,7 @@ struct PutWorker : public BaseWorker { }; /** - * Puts a key and a value to the database. + * Puts a key and a value to a database. */ NAPI_METHOD(db_put) { NAPI_ARGV(5); @@ -542,6 +547,7 @@ NAPI_METHOD(db_put) { napi_value value = argv[2]; bool sync = BooleanProperty(env, argv[3], "sync", false); napi_value callback = argv[4]; + PutWorker* worker = new PutWorker(env, database, callback, @@ -549,6 +555,82 @@ NAPI_METHOD(db_put) { value, sync); worker->Queue(); + + NAPI_RETURN_UNDEFINED(); +} + +/** + * Worker class for getting values from a database + */ +struct GetWorker : public BaseWorker { + GetWorker (napi_env env, + Database* database, + napi_value callback, + napi_value key, + bool asBuffer, + bool fillCache) + : BaseWorker(env, database, callback, "leveldown.db.get"), + key_(ToSlice(env, key)), + asBuffer_(asBuffer) { + options_.fill_cache = fillCache; + } + + virtual ~GetWorker () { + // TODO clean up key_ if not empty? + // See DisposeStringOrBufferFromSlice() + } + + virtual void DoExecute () { + SetStatus(database_->Get(options_, key_, value_)); + } + + virtual void HandleOKCallback() { + const int argc = 2; + napi_value argv[argc]; + napi_get_null(env_, &argv[0]); + + if (asBuffer_) { + napi_create_buffer_copy(env_, value_.size(), value_.data(), NULL, &argv[1]); + } else { + napi_create_string_utf8(env_, value_.data(), value_.size(), &argv[1]); + } + + // TODO move to base class + napi_value global; + napi_get_global(env_, &global); + napi_value callback; + napi_get_reference_value(env_, callbackRef_, &callback); + + napi_call_function(env_, global, callback, argc, argv, NULL); + } + + leveldb::ReadOptions options_; + leveldb::Slice key_; + std::string value_; + bool asBuffer_; +}; + +/** + * Gets a value from a database. + */ +NAPI_METHOD(db_get) { + NAPI_ARGV(4); + NAPI_DB_CONTEXT(); + + napi_value key = argv[1]; + napi_value options = argv[2]; + bool asBuffer = BooleanProperty(env, options, "asBuffer", true); + bool fillCache = BooleanProperty(env, options, "fillCache", true); + napi_value callback = argv[3]; + + GetWorker* worker = new GetWorker(env, + database, + callback, + key, + asBuffer, + fillCache); + worker->Queue(); + NAPI_RETURN_UNDEFINED(); } @@ -1171,6 +1253,7 @@ NAPI_INIT() { NAPI_EXPORT_FUNCTION(db_open); NAPI_EXPORT_FUNCTION(db_close); NAPI_EXPORT_FUNCTION(db_put); + NAPI_EXPORT_FUNCTION(db_get); NAPI_EXPORT_FUNCTION(iterator); NAPI_EXPORT_FUNCTION(iterator_seek); From 8de66184ebc688c42400a44f039e7a45f463d2ef Mon Sep 17 00:00:00 2001 From: Lars-Magnus Skog Date: Sat, 8 Dec 2018 03:46:24 +0100 Subject: [PATCH 15/42] Implement db_del() --- leveldown.js | 2 +- napi/leveldown.cc | 56 ++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 56 insertions(+), 2 deletions(-) diff --git a/leveldown.js b/leveldown.js index 077a00a6..0fa445f0 100644 --- a/leveldown.js +++ b/leveldown.js @@ -57,7 +57,7 @@ LevelDOWN.prototype._get = function (key, options, callback) { } LevelDOWN.prototype._del = function (key, options, callback) { - this.binding.del(key, options, callback) + binding.db_del(this.context, key, options, callback) } LevelDOWN.prototype._chainedBatch = function () { diff --git a/napi/leveldown.cc b/napi/leveldown.cc index 63a72396..3455bb8a 100644 --- a/napi/leveldown.cc +++ b/napi/leveldown.cc @@ -188,6 +188,11 @@ struct Database { return db_->Get(options, key, &value); } + leveldb::Status Del (const leveldb::WriteOptions& options, + leveldb::Slice key) { + return db_->Delete(options, key); + } + const leveldb::Snapshot* NewSnapshot () { return db_->GetSnapshot(); } @@ -560,7 +565,7 @@ NAPI_METHOD(db_put) { } /** - * Worker class for getting values from a database + * Worker class for getting a value from a database. */ struct GetWorker : public BaseWorker { GetWorker (napi_env env, @@ -634,6 +639,54 @@ NAPI_METHOD(db_get) { NAPI_RETURN_UNDEFINED(); } +/** + * Worker class for getting a value from a database. + */ +struct DelWorker : public BaseWorker { + DelWorker (napi_env env, + Database* database, + napi_value callback, + napi_value key, + bool sync) + : BaseWorker(env, database, callback, "leveldown.db.get"), + key_(ToSlice(env, key)) { + options_.sync = sync; + } + + virtual ~DelWorker () { + // TODO clean up key_ if not empty? + // See DisposeStringOrBufferFromSlice() + } + + virtual void DoExecute () { + SetStatus(database_->Del(options_, key_)); + } + + leveldb::WriteOptions options_; + leveldb::Slice key_; +}; + +/** + * Gets a value from a database. + */ +NAPI_METHOD(db_del) { + NAPI_ARGV(4); + NAPI_DB_CONTEXT(); + + napi_value key = argv[1]; + bool sync = BooleanProperty(env, argv[2], "sync", false); + napi_value callback = argv[3]; + + DelWorker* worker = new DelWorker(env, + database, + callback, + key, + sync); + worker->Queue(); + + NAPI_RETURN_UNDEFINED(); +} + /** * Owns a leveldb iterator. */ @@ -1254,6 +1307,7 @@ NAPI_INIT() { NAPI_EXPORT_FUNCTION(db_close); NAPI_EXPORT_FUNCTION(db_put); NAPI_EXPORT_FUNCTION(db_get); + NAPI_EXPORT_FUNCTION(db_del); NAPI_EXPORT_FUNCTION(iterator); NAPI_EXPORT_FUNCTION(iterator_seek); From 6f09aaf0140f8f57bc43f15b2e52a990d3375f1c Mon Sep 17 00:00:00 2001 From: Lars-Magnus Skog Date: Sat, 8 Dec 2018 05:37:30 +0100 Subject: [PATCH 16/42] Reorganize, macros at top, helpers etc --- napi/leveldown.cc | 355 ++++++++++++++++++++++------------------------ napi/napi.h | 14 -- 2 files changed, 171 insertions(+), 198 deletions(-) delete mode 100644 napi/napi.h diff --git a/napi/leveldown.cc b/napi/leveldown.cc index 3455bb8a..7c7397af 100644 --- a/napi/leveldown.cc +++ b/napi/leveldown.cc @@ -1,4 +1,5 @@ -#include "napi.h" +#include +#include #include #include #include @@ -13,7 +14,157 @@ struct Iterator; struct EndWorker; /** - * Base worker class. + * Macros. + */ + +#define NAPI_DB_CONTEXT() \ + Database* database = NULL; \ + NAPI_STATUS_THROWS(napi_get_value_external(env, argv[0], (void**)&database)); + +#define NAPI_ITERATOR_CONTEXT() \ + Iterator* iterator = NULL; \ + NAPI_STATUS_THROWS(napi_get_value_external(env, argv[0], (void**)&iterator)); + +#define NAPI_RETURN_UNDEFINED() \ + return 0; + +#define LD_STRING_OR_BUFFER_TO_COPY(env, from, to) \ + char* to##Ch_ = 0; \ + size_t to##Sz_ = 0; \ + if (IsString(env, from)) { \ + napi_get_value_string_utf8(env, from, NULL, 0, &to##Sz_); \ + to##Ch_ = new char[to##Sz_ + 1]; \ + napi_get_value_string_utf8(env, from, to##Ch_, to##Sz_ + 1, &to##Sz_); \ + to##Ch_[to##Sz_] = '\0'; \ + printf("LD_STRING_OR_BUFFER_TO_COPY STRING length: %zu content: %s\n", to##Sz_, to##Ch_); \ + } else if (IsBuffer(env, from)) { \ + char* buf = 0; \ + napi_get_buffer_info(env, from, (void **)&buf, &to##Sz_); \ + to##Ch_ = new char[to##Sz_]; \ + memcpy(to##Ch_, buf, to##Sz_); \ + printf("LD_STRING_OR_BUFFER_TO_COPY BUFFER length: %zu content: %s\n", to##Sz_, to##Ch_); \ + } + +/********************************************************************* + * Helpers. + ********************************************************************/ + +/** + * Returns true if 'obj' has a property 'key'. + */ +static bool HasProperty (napi_env env, napi_value obj, const char* key) { + bool has = false; + napi_value _key; + napi_create_string_utf8(env, key, strlen(key), &_key); + napi_has_property(env, obj, _key, &has); + return has; +} + +/** + * Returns a property in napi_value form. + */ +static napi_value GetProperty (napi_env env, napi_value obj, const char* key) { + napi_value _key; + napi_create_string_utf8(env, key, strlen(key), &_key); + + napi_value value; + napi_get_property(env, obj, _key, &value); + + return value; +} + +/** + * Returns a boolean property 'key' from 'obj'. + * Returns 'DEFAULT' if the property doesn't exist. + */ +static bool BooleanProperty (napi_env env, napi_value obj, const char* key, + bool DEFAULT) { + if (HasProperty(env, obj, key)) { + napi_value value = GetProperty(env, obj, key); + bool result; + napi_get_value_bool(env, value, &result); + return result; + } + + return DEFAULT; +} + +/** + * Returns a uint32 property 'key' from 'obj'. + * Returns 'DEFAULT' if the property doesn't exist. + */ +static uint32_t Uint32Property (napi_env env, napi_value obj, const char* key, + uint32_t DEFAULT) { + if (HasProperty(env, obj, key)) { + napi_value value = GetProperty(env, obj, key); + uint32_t result; + napi_get_value_uint32(env, value, &result); + return result; + } + + return DEFAULT; +} + +/** + * Returns a uint32 property 'key' from 'obj' + * Returns 'DEFAULT' if the property doesn't exist. + */ +static int Int32Property (napi_env env, napi_value obj, const char* key, + int DEFAULT) { + if (HasProperty(env, obj, key)) { + napi_value value = GetProperty(env, obj, key); + int result; + napi_get_value_int32(env, value, &result); + return result; + } + + return DEFAULT; +} + +/** + * Returns true if 'value' is a string. + */ +static bool IsString (napi_env env, napi_value value) { + napi_valuetype type; + napi_typeof(env, value, &type); + return type == napi_string; +} + +/** + * Returns true if 'value' is a buffer. + */ +static bool IsBuffer (napi_env env, napi_value value) { + bool isBuffer; + napi_is_buffer(env, value, &isBuffer); + return isBuffer; +} + +/** + * Convert a napi_value to a leveldb::Slice. + */ +static leveldb::Slice ToSlice (napi_env env, napi_value from) { + LD_STRING_OR_BUFFER_TO_COPY(env, from, to); + return leveldb::Slice(toCh_, toSz_); +} + +/** + * Returns length of string or buffer + */ +static size_t StringOrBufferLength (napi_env env, napi_value value) { + size_t size = 0; + + if (IsString(env, value)) { + napi_get_value_string_utf8(env, value, NULL, 0, &size); + } else if (IsBuffer(env, value)) { + char* buf; + napi_get_buffer_info(env, value, (void **)&buf, &size); + } + + return size; +} + +/** + * Base worker class. Handles the async work. */ struct BaseWorker { BaseWorker (napi_env env, @@ -39,17 +190,11 @@ struct BaseWorker { napi_delete_async_work(env_, asyncWork_); } - /** - * Calls virtual DoExecute(). - */ static void Execute (napi_env env, void* data) { BaseWorker* self = (BaseWorker*)data; self->DoExecute(); } - /** - * Store status. Note that error message has to be copied. - */ void SetStatus (leveldb::Status status) { status_ = status; if (!status.ok()) { @@ -57,9 +202,6 @@ struct BaseWorker { } } - /** - * Copied error message. - */ void SetErrorMessage(const char *msg) { delete [] errMsg_; size_t size = strlen(msg) + 1; @@ -67,25 +209,14 @@ struct BaseWorker { memcpy(errMsg_, msg, size); } - /** - * MUST be overriden. - */ virtual void DoExecute () = 0; - /** - * Calls DoComplete() and kills the worker. - */ static void Complete (napi_env env, napi_status status, void* data) { BaseWorker* self = (BaseWorker*)data; self->DoComplete(); delete self; } - /** - * Checks status and calls appropriate handler. - * - Calls virtual HandleOKCallback() if status is ok - * - Calls back with error if status is an error - */ void DoComplete () { if (status_.ok()) { return HandleOKCallback(); @@ -109,9 +240,6 @@ struct BaseWorker { napi_call_function(env_, global, callback, argc, argv, NULL); } - /** - * Default behavior is to call back with NULL. - */ virtual void HandleOKCallback () { napi_value global; napi_get_global(env_, &global); @@ -140,7 +268,7 @@ struct BaseWorker { }; /** - * Owns the LevelDB storage together with cache and filter policy. + * Owns the LevelDB storage, cache, filter policy and iterators. */ struct Database { Database (napi_env env) @@ -257,98 +385,8 @@ NAPI_METHOD(db) { return result; } -// TODO BooleanProperty() and Uint32Property() can be refactored - -/** - * Returns true if 'obj' has a property 'key' - */ -static bool HasProperty (napi_env env, - napi_value obj, - const char* key) { - bool has = false; - napi_value _key; - napi_create_string_utf8(env, key, strlen(key), &_key); - napi_has_property(env, obj, _key, &has); - return has; -} - -/** - * Returns a property in napi_value form - */ -static napi_value GetProperty (napi_env env, - napi_value obj, - const char* key) { - napi_value _key; - napi_create_string_utf8(env, key, strlen(key), &_key); - - napi_value value; - napi_get_property(env, obj, _key, &value); - - return value; -} - -/** - * Returns a boolean property 'key' from 'obj'. - * Returns 'DEFAULT' if the property doesn't exist. - */ -static bool BooleanProperty (napi_env env, - napi_value obj, - const char* key, - bool DEFAULT) { - if (HasProperty(env, obj, key)) { - napi_value value = GetProperty(env, obj, key); - bool result; - napi_get_value_bool(env, value, &result); - return result; - } - - return DEFAULT; -} - -/** - * Returns a uint32 property 'key' from 'obj' - * Returns 'DEFAULT' if the property doesn't exist. - */ -static uint32_t Uint32Property (napi_env env, - napi_value obj, - const char* key, - uint32_t DEFAULT) { - if (HasProperty(env, obj, key)) { - napi_value value = GetProperty(env, obj, key); - uint32_t result; - napi_get_value_uint32(env, value, &result); - return result; - } - - return DEFAULT; -} - -/** - * Returns a uint32 property 'key' from 'obj' - * Returns 'DEFAULT' if the property doesn't exist. - */ -static int Int32Property (napi_env env, - napi_value obj, - const char* key, - int DEFAULT) { - bool exists = false; - napi_value _key; - napi_create_string_utf8(env, key, strlen(key), &_key); - napi_has_property(env, obj, _key, &exists); - - if (exists) { - napi_value value; - napi_get_property(env, obj, _key, &value); - int result; - napi_get_value_int32(env, value, &result); - return result; - } - - return DEFAULT; -} - /** - * Worker class for opening a database + * Worker class for opening a database. */ struct OpenWorker : public BaseWorker { OpenWorker (napi_env env, @@ -415,7 +453,6 @@ NAPI_METHOD(db_open) { "blockRestartInterval", 16); uint32_t maxFileSize = Uint32Property(env, options, "maxFileSize", 2 << 20); - // TODO clean these up in close() database->blockCache_ = leveldb::NewLRUCache(cacheSize); database->filterPolicy_ = leveldb::NewBloomFilterPolicy(10); @@ -466,52 +503,6 @@ NAPI_METHOD(db_close) { NAPI_RETURN_UNDEFINED(); } -/** - * Returns true if 'value' is a string otherwise false. - */ -static bool IsString (napi_env env, napi_value value) { - napi_valuetype type; - napi_typeof(env, value, &type); - return type == napi_string; -} - -/** - * Returns true if 'value' is a buffer otherwise false. - */ -static bool IsBuffer (napi_env env, napi_value value) { - bool isBuffer; - napi_is_buffer(env, value, &isBuffer); - return isBuffer; -} - -/** - * Macro to copy memory from a buffer or string. - */ -#define LD_STRING_OR_BUFFER_TO_COPY(env, from, to) \ - char* to##Ch_ = 0; \ - size_t to##Sz_ = 0; \ - if (IsString(env, from)) { \ - napi_get_value_string_utf8(env, from, NULL, 0, &to##Sz_); \ - to##Ch_ = new char[to##Sz_ + 1]; \ - napi_get_value_string_utf8(env, from, to##Ch_, to##Sz_ + 1, &to##Sz_); \ - to##Ch_[to##Sz_] = '\0'; \ - printf("LD_STRING_OR_BUFFER_TO_COPY STRING length: %zu content: %s\n", to##Sz_, to##Ch_); \ - } else if (IsBuffer(env, from)) { \ - char* buf = 0; \ - napi_get_buffer_info(env, from, (void **)&buf, &to##Sz_); \ - to##Ch_ = new char[to##Sz_]; \ - memcpy(to##Ch_, buf, to##Sz_); \ - printf("LD_STRING_OR_BUFFER_TO_COPY BUFFER length: %zu content: %s\n", to##Sz_, to##Ch_); \ - } - -/** - * Convert a napi_value to a leveldb::Slice. - */ -static leveldb::Slice ToSlice (napi_env env, napi_value from) { - LD_STRING_OR_BUFFER_TO_COPY(env, from, to); - return leveldb::Slice(toCh_, toSz_); -} - /** * Worker class for putting key/value to the database */ @@ -954,23 +945,7 @@ static void FinalizeIterator (napi_env env, void* data, void* hint) { } /** - * Returns length of string or buffer - */ -static size_t StringOrBufferLength (napi_env env, napi_value value) { - size_t size = 0; - - if (IsString(env, value)) { - napi_get_value_string_utf8(env, value, NULL, 0, &size); - } else if (IsBuffer(env, value)) { - char* buf; - napi_get_buffer_info(env, value, (void **)&buf, &size); - } - - return size; -} - -/** - * Creates an iterator. + * Create an iterator. */ NAPI_METHOD(iterator) { NAPI_ARGV(2); @@ -1170,7 +1145,6 @@ NAPI_METHOD(iterator_end) { NAPI_ARGV(2); NAPI_ITERATOR_CONTEXT(); - // TODO assuming callback is always provided napi_value callback = argv[1]; if (!iterator->ended_) { @@ -1188,6 +1162,10 @@ NAPI_METHOD(iterator_end) { NAPI_RETURN_UNDEFINED(); } +/** + * TODO Move this to Iterator. There isn't any reason + * for this function being a separate function pointer. + */ void CheckEndCallback (Iterator* iterator) { iterator->ReleaseTarget(); iterator->nexting_ = false; @@ -1249,6 +1227,7 @@ struct NextWorker : public BaseWorker { } // clean up & handle the next/end state + // TODO this should just do iterator_->CheckEndCallback(); localCallback_(iterator_); const int argc = 3; @@ -1268,7 +1247,7 @@ struct NextWorker : public BaseWorker { } Iterator* iterator_; - // TODO(cosmetic) create a typedef for this function pointer + // TODO why do we need a function pointer for this? void (*localCallback_)(Iterator*); std::vector > result_; bool ok_; @@ -1281,7 +1260,6 @@ NAPI_METHOD(iterator_next) { NAPI_ARGV(2); NAPI_ITERATOR_CONTEXT(); - // TODO assuming callback is always provided napi_value callback = argv[1]; if (iterator->ended_) { @@ -1301,7 +1279,13 @@ NAPI_METHOD(iterator_next) { NAPI_RETURN_UNDEFINED(); } +/** + * All exported functions. + */ NAPI_INIT() { + /** + * Database* related functions. + */ NAPI_EXPORT_FUNCTION(db); NAPI_EXPORT_FUNCTION(db_open); NAPI_EXPORT_FUNCTION(db_close); @@ -1309,6 +1293,9 @@ NAPI_INIT() { NAPI_EXPORT_FUNCTION(db_get); NAPI_EXPORT_FUNCTION(db_del); + /** + * Iterator* related functions. + */ NAPI_EXPORT_FUNCTION(iterator); NAPI_EXPORT_FUNCTION(iterator_seek); NAPI_EXPORT_FUNCTION(iterator_end); diff --git a/napi/napi.h b/napi/napi.h deleted file mode 100644 index bfa6e8e3..00000000 --- a/napi/napi.h +++ /dev/null @@ -1,14 +0,0 @@ -#include -#include - -#define NAPI_DB_CONTEXT() \ - Database* database = NULL; \ - NAPI_STATUS_THROWS(napi_get_value_external(env, argv[0], (void**)&database)); - -#define NAPI_ITERATOR_CONTEXT() \ - Iterator* iterator = NULL; \ - NAPI_STATUS_THROWS(napi_get_value_external(env, argv[0], (void**)&iterator)); - -// TODO move to napi-macros.h -#define NAPI_RETURN_UNDEFINED() \ - return 0; From f7c6d93238b1ea77a2d14b4c44e094c5dac5f1d5 Mon Sep 17 00:00:00 2001 From: Lars-Magnus Skog Date: Sat, 8 Dec 2018 05:38:54 +0100 Subject: [PATCH 17/42] napi/leveldown.cc -> binding.cc --- napi/leveldown.cc => binding.cc | 0 binding.gyp | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename napi/leveldown.cc => binding.cc (100%) diff --git a/napi/leveldown.cc b/binding.cc similarity index 100% rename from napi/leveldown.cc rename to binding.cc diff --git a/binding.gyp b/binding.gyp index 081fe687..63c59975 100644 --- a/binding.gyp +++ b/binding.gyp @@ -42,7 +42,7 @@ " Date: Sat, 8 Dec 2018 05:48:35 +0100 Subject: [PATCH 18/42] Add TODO for db_close() (when open iterators) --- binding.cc | 42 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/binding.cc b/binding.cc index 7c7397af..7054e0a1 100644 --- a/binding.cc +++ b/binding.cc @@ -498,7 +498,47 @@ NAPI_METHOD(db_close) { napi_value callback = argv[1]; CloseWorker* worker = new CloseWorker(env, database, callback); - worker->Queue(); + + if (database->iterators_.empty()) { + worker->Queue(); + NAPI_RETURN_UNDEFINED(); + } + + // TODO fix me! + + /* + // yikes, we still have iterators open! naughty naughty. + // we have to queue up a CloseWorker and manually close each of them. + // the CloseWorker will be invoked once they are all cleaned up + database->pendingCloseWorker = worker; + + for ( + std::map< uint32_t, leveldown::Iterator * >::iterator it + = database->iterators.begin() + ; it != database->iterators.end() + ; ++it) { + + // for each iterator still open, first check if it's already in + // the process of ending (ended==true means an async End() is + // in progress), if not, then we call End() with an empty callback + // function and wait for it to hit ReleaseIterator() where our + // CloseWorker will be invoked + + leveldown::Iterator *iterator = it->second; + + if (!iterator->ended) { + v8::Local end = + v8::Local::Cast(iterator->handle()->Get( + Nan::New("end").ToLocalChecked())); + v8::Local argv[] = { + Nan::New(EmptyMethod)->GetFunction() // empty callback + }; + Nan::AsyncResource ar("leveldown:iterator.end"); + ar.runInAsyncScope(iterator->handle(), end, 1, argv); + } + } + + */ NAPI_RETURN_UNDEFINED(); } From 52ccc82112f87d2461b7ca0803807fec3f305a3d Mon Sep 17 00:00:00 2001 From: Lars-Magnus Skog Date: Sat, 8 Dec 2018 15:45:15 +0100 Subject: [PATCH 19/42] Implement batch_do() for immediate write operation --- binding.cc | 184 +++++++++++++++++++++++++++++++++++++++++++++------ leveldown.js | 2 +- 2 files changed, 164 insertions(+), 22 deletions(-) diff --git a/binding.cc b/binding.cc index 7054e0a1..91e1118f 100644 --- a/binding.cc +++ b/binding.cc @@ -1,8 +1,11 @@ #include #include + #include +#include #include #include + #include #include @@ -36,23 +39,51 @@ struct EndWorker; to##Ch_ = new char[to##Sz_ + 1]; \ napi_get_value_string_utf8(env, from, to##Ch_, to##Sz_ + 1, &to##Sz_); \ to##Ch_[to##Sz_] = '\0'; \ - printf("LD_STRING_OR_BUFFER_TO_COPY STRING length: %zu content: %s\n", to##Sz_, to##Ch_); \ + printf("-- LD_STRING_OR_BUFFER_TO_COPY STRING length: %zu content: %s\n", to##Sz_, to##Ch_); \ } else if (IsBuffer(env, from)) { \ char* buf = 0; \ napi_get_buffer_info(env, from, (void **)&buf, &to##Sz_); \ to##Ch_ = new char[to##Sz_]; \ memcpy(to##Ch_, buf, to##Sz_); \ - printf("LD_STRING_OR_BUFFER_TO_COPY BUFFER length: %zu content: %s\n", to##Sz_, to##Ch_); \ + printf("-- LD_STRING_OR_BUFFER_TO_COPY BUFFER length: %zu content: %s\n", to##Sz_, to##Ch_); \ } /********************************************************************* * Helpers. ********************************************************************/ +/** + * Returns true if 'value' is a string. + */ +static bool IsString (napi_env env, napi_value value) { + napi_valuetype type; + napi_typeof(env, value, &type); + return type == napi_string; +} + +/** + * Returns true if 'value' is a buffer. + */ +static bool IsBuffer (napi_env env, napi_value value) { + bool isBuffer; + napi_is_buffer(env, value, &isBuffer); + return isBuffer; +} + +/** + * Returns true if 'value' is an object. + */ +static bool IsObject (napi_env env, napi_value value) { + napi_valuetype type; + napi_typeof(env, value, &type); + return type == napi_object; +} + /** * Returns true if 'obj' has a property 'key'. */ static bool HasProperty (napi_env env, napi_value obj, const char* key) { + // TODO switch to use napi_has_named_property() (no need to make a napi_value) bool has = false; napi_value _key; napi_create_string_utf8(env, key, strlen(key), &_key); @@ -64,12 +95,11 @@ static bool HasProperty (napi_env env, napi_value obj, const char* key) { * Returns a property in napi_value form. */ static napi_value GetProperty (napi_env env, napi_value obj, const char* key) { + // TODO switch to use napi_get_named_property() (no need to make a napi_value) napi_value _key; napi_create_string_utf8(env, key, strlen(key), &_key); - napi_value value; napi_get_property(env, obj, _key, &value); - return value; } @@ -106,7 +136,7 @@ static uint32_t Uint32Property (napi_env env, napi_value obj, const char* key, } /** - * Returns a uint32 property 'key' from 'obj' + * Returns a uint32 property 'key' from 'obj'. * Returns 'DEFAULT' if the property doesn't exist. */ static int Int32Property (napi_env env, napi_value obj, const char* key, @@ -122,21 +152,27 @@ static int Int32Property (napi_env env, napi_value obj, const char* key, } /** - * Returns true if 'value' is a string. + * Returns a string property 'key' from 'obj'. + * Returns empty string if the property doesn't exist. */ -static bool IsString (napi_env env, napi_value value) { - napi_valuetype type; - napi_typeof(env, value, &type); - return type == napi_string; -} +static std::string StringProperty (napi_env env, napi_value obj, const char* key) { + if (HasProperty(env, obj, key)) { + napi_value value = GetProperty(env, obj, key); + if (IsString(env, value)) { + size_t size = 0; + napi_get_value_string_utf8(env, value, NULL, 0, &size); -/** - * Returns true if 'value' is a buffer. - */ -static bool IsBuffer (napi_env env, napi_value value) { - bool isBuffer; - napi_is_buffer(env, value, &isBuffer); - return isBuffer; + char* buf = new char[size + 1]; + napi_get_value_string_utf8(env, value, buf, size + 1, &size); + buf[size] = '\0'; + + std::string result = buf; + delete [] buf; + return result; + } + } + + return ""; } /** @@ -321,6 +357,11 @@ struct Database { return db_->Delete(options, key); } + leveldb::Status WriteBatch (const leveldb::WriteOptions& options, + leveldb::WriteBatch* batch) { + return db_->Write(options, batch); + } + const leveldb::Snapshot* NewSnapshot () { return db_->GetSnapshot(); } @@ -1216,7 +1257,7 @@ void CheckEndCallback (Iterator* iterator) { } /** - * Worker class for nexting an iterator + * Worker class for nexting an iterator. */ struct NextWorker : public BaseWorker { NextWorker (napi_env env, @@ -1319,12 +1360,108 @@ NAPI_METHOD(iterator_next) { NAPI_RETURN_UNDEFINED(); } +/** + * Worker class for batch write operation. + */ +struct BatchWorker : public BaseWorker { + BatchWorker (napi_env env, + Database* database, + napi_value callback, + leveldb::WriteBatch* batch, + bool sync) + : BaseWorker(env, database, callback, "leveldown.batch.do"), + batch_(batch) { + options_.sync = sync; + } + + virtual ~BatchWorker () { + delete batch_; + } + + virtual void DoExecute () { + SetStatus(database_->WriteBatch(options_, batch_)); + } + + leveldb::WriteOptions options_; + leveldb::WriteBatch* batch_; +}; + +/** + * Does a batch write operation on a database. + */ +NAPI_METHOD(batch_do) { + NAPI_ARGV(4); + NAPI_DB_CONTEXT(); + + napi_value array = argv[1]; + bool sync = BooleanProperty(env, argv[2], "sync", false); + napi_value callback = argv[3]; + + uint32_t length; + napi_get_array_length(env, array, &length); + + leveldb::WriteBatch* batch = new leveldb::WriteBatch(); + bool hasData = false; + + for (uint32_t i = 0; i < length; i++) { + napi_value element; + napi_get_element(env, array, i, &element); + + if (!IsObject(env, element)) continue; + + std::string type = StringProperty(env, element, "type"); + printf("-- Batch element type: %s\n", type.c_str()); + + if (type == "del") { + if (!HasProperty(env, element, "key")) continue; + leveldb::Slice key = ToSlice(env, GetProperty(env, element, "key")); + + batch->Delete(key); + if (!hasData) hasData = true; + + // TODO clean up + //DisposeStringOrBufferFromSlice(keyBuffer, key); + } else if (type == "put") { + if (!HasProperty(env, element, "key")) continue; + if (!HasProperty(env, element, "value")) continue; + + leveldb::Slice key = ToSlice(env, GetProperty(env, element, "key")); + leveldb::Slice value = ToSlice(env, GetProperty(env, element, "value")); + + batch->Put(key, value); + if (!hasData) hasData = true; + + // TODO clean up + //DisposeStringOrBufferFromSlice(keyBuffer, key); + //DisposeStringOrBufferFromSlice(valueBuffer, value); + } + } + + if (hasData) { + BatchWorker* worker = new BatchWorker(env, database, callback, batch, sync); + worker->Queue(); + } else { + delete batch; + // TODO refactor with other callback code + napi_value global; + napi_get_global(env, &global); + + const int argc = 1; + napi_value argv[argc]; + napi_get_null(env, &argv[0]); + + napi_call_function(env, global, callback, argc, argv, NULL); + } + + NAPI_RETURN_UNDEFINED(); +} + /** * All exported functions. */ NAPI_INIT() { /** - * Database* related functions. + * Database related functions. */ NAPI_EXPORT_FUNCTION(db); NAPI_EXPORT_FUNCTION(db_open); @@ -1334,10 +1471,15 @@ NAPI_INIT() { NAPI_EXPORT_FUNCTION(db_del); /** - * Iterator* related functions. + * Iterator related functions. */ NAPI_EXPORT_FUNCTION(iterator); NAPI_EXPORT_FUNCTION(iterator_seek); NAPI_EXPORT_FUNCTION(iterator_end); NAPI_EXPORT_FUNCTION(iterator_next); + + /** + * Batch related functions. + */ + NAPI_EXPORT_FUNCTION(batch_do); } diff --git a/leveldown.js b/leveldown.js index 0fa445f0..98a61efd 100644 --- a/leveldown.js +++ b/leveldown.js @@ -65,7 +65,7 @@ LevelDOWN.prototype._chainedBatch = function () { } LevelDOWN.prototype._batch = function (operations, options, callback) { - return this.binding.batch(operations, options, callback) + binding.batch_do(this.context, operations, options, callback) } LevelDOWN.prototype.approximateSize = function (start, end, callback) { From 79966afccaf3c59037870e516d354fa7dda4c2b4 Mon Sep 17 00:00:00 2001 From: Lars-Magnus Skog Date: Sat, 8 Dec 2018 16:48:40 +0100 Subject: [PATCH 20/42] Implement chained batch (+ rename some functions) --- binding.cc | 172 +++++++++++++++++++++++++++++++++++++++++++++-- chained-batch.js | 17 +++-- iterator.js | 2 +- leveldown.js | 2 +- 4 files changed, 180 insertions(+), 13 deletions(-) diff --git a/binding.cc b/binding.cc index 91e1118f..539a1c16 100644 --- a/binding.cc +++ b/binding.cc @@ -28,6 +28,10 @@ struct EndWorker; Iterator* iterator = NULL; \ NAPI_STATUS_THROWS(napi_get_value_external(env, argv[0], (void**)&iterator)); +#define NAPI_BATCH_CONTEXT() \ + Batch* batch = NULL; \ + NAPI_STATUS_THROWS(napi_get_value_external(env, argv[0], (void**)&batch)); + #define NAPI_RETURN_UNDEFINED() \ return 0; @@ -416,7 +420,7 @@ static void FinalizeDatabase (napi_env env, void* data, void* hint) { /** * Returns a context object for a database. */ -NAPI_METHOD(db) { +NAPI_METHOD(db_init) { Database* database = new Database(env); napi_value result; @@ -1028,7 +1032,7 @@ static void FinalizeIterator (napi_env env, void* data, void* hint) { /** * Create an iterator. */ -NAPI_METHOD(iterator) { +NAPI_METHOD(iterator_init) { NAPI_ARGV(2); NAPI_DB_CONTEXT(); @@ -1419,7 +1423,7 @@ NAPI_METHOD(batch_do) { batch->Delete(key); if (!hasData) hasData = true; - // TODO clean up + // TODO clean up slices //DisposeStringOrBufferFromSlice(keyBuffer, key); } else if (type == "put") { if (!HasProperty(env, element, "key")) continue; @@ -1431,7 +1435,7 @@ NAPI_METHOD(batch_do) { batch->Put(key, value); if (!hasData) hasData = true; - // TODO clean up + // TODO clean up slices //DisposeStringOrBufferFromSlice(keyBuffer, key); //DisposeStringOrBufferFromSlice(valueBuffer, value); } @@ -1456,6 +1460,157 @@ NAPI_METHOD(batch_do) { NAPI_RETURN_UNDEFINED(); } +/** + * Owns a WriteBatch. + */ +struct Batch { + Batch (Database* database) + : database_(database), + batch_(new leveldb::WriteBatch()), + hasData_(false) {} + + ~Batch () { + delete batch_; + } + + void Put (leveldb::Slice key, leveldb::Slice value) { + batch_->Put(key, value); + hasData_ = true; + } + + void Del (leveldb::Slice key) { + batch_->Delete(key); + hasData_ = true; + } + + void Clear () { + batch_->Clear(); + hasData_ = false; + } + + leveldb::Status Write (bool sync) { + leveldb::WriteOptions options; + options.sync = sync; + return database_->WriteBatch(options, batch_); + } + + Database* database_; + leveldb::WriteBatch* batch_; + bool hasData_; +}; + +/** + * Runs when a Batch is garbage collected. + */ +static void FinalizeBatch (napi_env env, void* data, void* hint) { + if (data) { + delete (Batch*)data; + } +} + +/** + * Return a batch object. + */ +NAPI_METHOD(batch_init) { + NAPI_ARGV(1); + NAPI_DB_CONTEXT(); + + Batch* batch = new Batch(database); + + napi_value result; + NAPI_STATUS_THROWS(napi_create_external(env, batch, + FinalizeBatch, + NULL, &result)); + return result; +} + +/** + * Adds a put instruction to a batch object. + */ +NAPI_METHOD(batch_put) { + NAPI_ARGV(3); + NAPI_BATCH_CONTEXT(); + + leveldb::Slice key = ToSlice(env, argv[1]); + leveldb::Slice value = ToSlice(env, argv[2]); + + batch->Put(key, value); + + // TODO clean up slices + //DisposeStringOrBufferFromSlice(keyBuffer, key); + //DisposeStringOrBufferFromSlice(valueBuffer, value); + + NAPI_RETURN_UNDEFINED(); +} + +/** + * Adds a delete instruction to a batch object. + */ +NAPI_METHOD(batch_del) { + NAPI_ARGV(2); + NAPI_BATCH_CONTEXT(); + + leveldb::Slice key = ToSlice(env, argv[1]); + + batch->Del(key); + + // TODO clean up slice + //DisposeStringOrBufferFromSlice(keyBuffer, key); + + NAPI_RETURN_UNDEFINED(); +} + +/** + * Clears a batch object. + */ +NAPI_METHOD(batch_clear) { + NAPI_ARGV(1); + NAPI_BATCH_CONTEXT(); + + batch->Clear(); + + NAPI_RETURN_UNDEFINED(); +} + +/** + * Worker class for batch write operation. + */ +struct BatchWriteWorker : public BaseWorker { + BatchWriteWorker (napi_env env, + Batch* batch, + napi_value callback, + bool sync) + : BaseWorker(env, batch->database_, callback, "leveldown.batch.write"), + batch_(batch), + sync_(sync) {} + + virtual ~BatchWriteWorker () {} + + virtual void DoExecute () { + SetStatus(batch_->Write(sync_)); + } + + Batch* batch_; + bool sync_; +}; + +/** + * Writes a batch object. + */ +NAPI_METHOD(batch_write) { + NAPI_ARGV(3); + NAPI_BATCH_CONTEXT(); + + napi_value options = argv[1]; + bool sync = BooleanProperty(env, options, "sync", false); + napi_value callback = argv[2]; + + BatchWriteWorker* worker = new BatchWriteWorker(env, batch, callback, sync); + worker->Queue(); + + NAPI_RETURN_UNDEFINED(); +} + /** * All exported functions. */ @@ -1463,7 +1618,7 @@ NAPI_INIT() { /** * Database related functions. */ - NAPI_EXPORT_FUNCTION(db); + NAPI_EXPORT_FUNCTION(db_init); NAPI_EXPORT_FUNCTION(db_open); NAPI_EXPORT_FUNCTION(db_close); NAPI_EXPORT_FUNCTION(db_put); @@ -1473,7 +1628,7 @@ NAPI_INIT() { /** * Iterator related functions. */ - NAPI_EXPORT_FUNCTION(iterator); + NAPI_EXPORT_FUNCTION(iterator_init); NAPI_EXPORT_FUNCTION(iterator_seek); NAPI_EXPORT_FUNCTION(iterator_end); NAPI_EXPORT_FUNCTION(iterator_next); @@ -1482,4 +1637,9 @@ NAPI_INIT() { * Batch related functions. */ NAPI_EXPORT_FUNCTION(batch_do); + NAPI_EXPORT_FUNCTION(batch_init); + NAPI_EXPORT_FUNCTION(batch_put); + NAPI_EXPORT_FUNCTION(batch_del); + NAPI_EXPORT_FUNCTION(batch_clear); + NAPI_EXPORT_FUNCTION(batch_write); } diff --git a/chained-batch.js b/chained-batch.js index f0f14cfe..c6195b64 100644 --- a/chained-batch.js +++ b/chained-batch.js @@ -1,25 +1,32 @@ const util = require('util') const AbstractChainedBatch = require('abstract-leveldown').AbstractChainedBatch +const binding = require('./binding') function ChainedBatch (db) { AbstractChainedBatch.call(this, db) - this.binding = db.binding.batch() + this.context = binding.batch_init(db.context) } ChainedBatch.prototype._put = function (key, value) { - this.binding.put(key, value) + binding.batch_put(this.context, key, value) } ChainedBatch.prototype._del = function (key) { - this.binding.del(key) + binding.batch_del(this.context, key) } ChainedBatch.prototype._clear = function () { - this.binding.clear() + binding.batch_clear(this.context) } ChainedBatch.prototype._write = function (options, callback) { - this.binding.write(callback) + // TODO (ensure docs covers the following) + // Note that we're passing in options here, which we didn't do before. We + // must do this so we can use the `sync` property, which we didn't handle before + // since we never passed in an object at time of creation (bug) (the previous c++ + // used to assume we did this from the js side from the ChainedBatch() + // constructor). + binding.batch_write(this.context, options, callback) } util.inherits(ChainedBatch, AbstractChainedBatch) diff --git a/iterator.js b/iterator.js index c5f423ee..72e47e74 100644 --- a/iterator.js +++ b/iterator.js @@ -6,7 +6,7 @@ const binding = require('./binding') function Iterator (db, options) { AbstractIterator.call(this, db) - this.context = binding.iterator(db.context, options) + this.context = binding.iterator_init(db.context, options) this.cache = null this.finished = false this.fastFuture = fastFuture() diff --git a/leveldown.js b/leveldown.js index 98a61efd..4018b26e 100644 --- a/leveldown.js +++ b/leveldown.js @@ -16,7 +16,7 @@ function LevelDOWN (location) { AbstractLevelDOWN.call(this) this.location = location - this.context = binding.db() + this.context = binding.db_init() } util.inherits(LevelDOWN, AbstractLevelDOWN) From 60b79828fd4de520a9b3096e17799948916e1b82 Mon Sep 17 00:00:00 2001 From: Lars-Magnus Skog Date: Sat, 8 Dec 2018 17:08:03 +0100 Subject: [PATCH 21/42] Implement the rest of iterator_seek() This concludes the full abstract-leveldown test suite --- binding.cc | 78 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/binding.cc b/binding.cc index 539a1c16..15c13514 100644 --- a/binding.cc +++ b/binding.cc @@ -963,6 +963,38 @@ struct Iterator { return false; } + bool OutOfRange (leveldb::Slice* target) { + if (lt_ != NULL) { + if (target->compare(*lt_) >= 0) + return true; + } else if (lte_ != NULL) { + if (target->compare(*lte_) > 0) + return true; + } else if (start_ != NULL && reverse_) { + if (target->compare(*start_) > 0) + return true; + } + + if (end_ != NULL) { + int d = target->compare(*end_); + if (reverse_ ? d < 0 : d > 0) + return true; + } + + if (gt_ != NULL) { + if (target->compare(*gt_) <= 0) + return true; + } else if (gte_ != NULL) { + if (target->compare(*gte_) < 0) + return true; + } else if (start_ != NULL && !reverse_) { + if (target->compare(*start_) < 0) + return true; + } + + return false; + } + bool IteratorNext (std::vector >& result) { size_t size = 0; while (true) { @@ -1195,6 +1227,52 @@ NAPI_METHOD(iterator_seek) { NAPI_ARGV(2); NAPI_ITERATOR_CONTEXT(); + iterator->ReleaseTarget(); + iterator->target_ = new leveldb::Slice(ToSlice(env, argv[1])); + iterator->GetIterator(); + + leveldb::Iterator* dbIterator = iterator->dbIterator_; + dbIterator->Seek(*iterator->target_); + + iterator->seeking_ = true; + iterator->landed_ = false; + + if (iterator->OutOfRange(iterator->target_)) { + if (iterator->reverse_) { + dbIterator->SeekToFirst(); + dbIterator->Prev(); + } else { + dbIterator->SeekToLast(); + dbIterator->Next(); + } + } + else { + if (dbIterator->Valid()) { + int cmp = dbIterator->key().compare(*iterator->target_); + if (cmp > 0 && iterator->reverse_) { + dbIterator->Prev(); + } else if (cmp < 0 && !iterator->reverse_) { + dbIterator->Next(); + } + } else { + if (iterator->reverse_) { + dbIterator->SeekToLast(); + } else { + dbIterator->SeekToFirst(); + } + if (dbIterator->Valid()) { + int cmp = dbIterator->key().compare(*iterator->target_); + if (cmp > 0 && iterator->reverse_) { + dbIterator->SeekToFirst(); + dbIterator->Prev(); + } else if (cmp < 0 && !iterator->reverse_) { + dbIterator->SeekToLast(); + dbIterator->Next(); + } + } + } + } + NAPI_RETURN_UNDEFINED(); } From 8aba0c3eb4f636825931b966d78ca8f29f617f6e Mon Sep 17 00:00:00 2001 From: Lars-Magnus Skog Date: Sat, 8 Dec 2018 17:33:59 +0100 Subject: [PATCH 22/42] Implement dc_approximate_size() --- binding.cc | 70 +++++++++++++++++++++++++++++++++++++++++++++++++++ leveldown.js | 2 +- test/index.js | 6 ++--- 3 files changed, 74 insertions(+), 4 deletions(-) diff --git a/binding.cc b/binding.cc index 15c13514..27393aca 100644 --- a/binding.cc +++ b/binding.cc @@ -366,6 +366,12 @@ struct Database { return db_->Write(options, batch); } + uint64_t ApproximateSize (const leveldb::Range* range) { + uint64_t size = 0; + db_->GetApproximateSizes(range, 1, &size); + return size; + } + const leveldb::Snapshot* NewSnapshot () { return db_->GetSnapshot(); } @@ -763,6 +769,69 @@ NAPI_METHOD(db_del) { NAPI_RETURN_UNDEFINED(); } +/** + * Worker class for getting a value from a database. + */ +struct ApproximateSizeWorker : public BaseWorker { + ApproximateSizeWorker (napi_env env, + Database* database, + napi_value callback, + leveldb::Slice start, + leveldb::Slice end) + : BaseWorker(env, database, callback, "leveldown.db.approximate_size"), + start_(start), end_(end) {} + + virtual ~ApproximateSizeWorker () { + // TODO clean up start_ and end_ slices + // See DisposeStringOrBufferFromSlice() + } + + virtual void DoExecute () { + leveldb::Range range(start_, end_); + size_ = database_->ApproximateSize(&range); + } + + virtual void HandleOKCallback() { + const int argc = 2; + napi_value argv[argc]; + napi_get_null(env_, &argv[0]); + napi_create_uint32(env_, (uint32_t)size_, &argv[1]); + + // TODO move to base class + napi_value global; + napi_get_global(env_, &global); + napi_value callback; + napi_get_reference_value(env_, callbackRef_, &callback); + + napi_call_function(env_, global, callback, argc, argv, NULL); + } + + leveldb::Slice start_; + leveldb::Slice end_; + uint64_t size_; +}; + +/** + * Calculates the approximate size of a range in a database. + */ +NAPI_METHOD(db_approximate_size) { + NAPI_ARGV(4); + NAPI_DB_CONTEXT(); + + leveldb::Slice start = ToSlice(env, argv[1]); + leveldb::Slice end = ToSlice(env, argv[2]); + napi_value callback = argv[3]; + + ApproximateSizeWorker* worker = new ApproximateSizeWorker(env, + database, + callback, + start, + end); + worker->Queue(); + + NAPI_RETURN_UNDEFINED(); +} + /** * Owns a leveldb iterator. */ @@ -1702,6 +1771,7 @@ NAPI_INIT() { NAPI_EXPORT_FUNCTION(db_put); NAPI_EXPORT_FUNCTION(db_get); NAPI_EXPORT_FUNCTION(db_del); + NAPI_EXPORT_FUNCTION(db_approximate_size); /** * Iterator related functions. diff --git a/leveldown.js b/leveldown.js index 4018b26e..1b09b8e9 100644 --- a/leveldown.js +++ b/leveldown.js @@ -83,7 +83,7 @@ LevelDOWN.prototype.approximateSize = function (start, end, callback) { start = this._serializeKey(start) end = this._serializeKey(end) - this.binding.approximateSize(start, end, callback) + binding.db_approximate_size(this.context, start, end, callback) } LevelDOWN.prototype.compactRange = function (start, end, callback) { diff --git a/test/index.js b/test/index.js index cece63c4..e570a403 100644 --- a/test/index.js +++ b/test/index.js @@ -1,6 +1,6 @@ -require('./leveldown-test') -require('./abstract-leveldown-test') -// require('./approximate-size-test') +//require('./leveldown-test') +//require('./abstract-leveldown-test') +require('./approximate-size-test') // require('./cleanup-hanging-iterators-test') // require('./compact-range-test') // require('./compression-test') From 023deaee10a4332fb8c69ab0c9254e045ea0a79c Mon Sep 17 00:00:00 2001 From: Lars-Magnus Skog Date: Sat, 8 Dec 2018 23:03:45 +0100 Subject: [PATCH 23/42] Fix ending open iterators while closing database (+ reorder Iterator struct) --- binding.cc | 981 +++++++++++++++++++++++++------------------------- test/index.js | 4 +- 2 files changed, 488 insertions(+), 497 deletions(-) diff --git a/binding.cc b/binding.cc index 27393aca..7d59634b 100644 --- a/binding.cc +++ b/binding.cc @@ -15,6 +15,7 @@ struct Database; struct Iterator; struct EndWorker; +static void iterator_end_do (napi_env env, Iterator* iterator, napi_value cb); /** * Macros. @@ -423,6 +424,290 @@ static void FinalizeDatabase (napi_env env, void* data, void* hint) { } } +/** + * Owns a leveldb iterator. + */ +struct Iterator { + Iterator (Database* database, + uint32_t id, + leveldb::Slice* start, + std::string* end, + bool reverse, + bool keys, + bool values, + int limit, + std::string* lt, + std::string* lte, + std::string* gt, + std::string* gte, + bool fillCache, + bool keyAsBuffer, + bool valueAsBuffer, + uint32_t highWaterMark) + : database_(database), + id_(id), + start_(start), + end_(end), + reverse_(reverse), + keys_(keys), + values_(values), + limit_(limit), + lt_(lt), + lte_(lte), + gt_(gt), + gte_(gte), + keyAsBuffer_(keyAsBuffer), + valueAsBuffer_(valueAsBuffer), + highWaterMark_(highWaterMark), + dbIterator_(NULL), + count_(0), + target_(NULL), + seeking_(false), + landed_(false), + nexting_(false), + ended_(false), + endWorker_(NULL) { + options_ = new leveldb::ReadOptions(); + options_->fill_cache = fillCache; + options_->snapshot = database->NewSnapshot(); + } + + ~Iterator () { + ReleaseTarget(); + if (start_ != NULL) { + // Special case for `start` option: it won't be + // freed up by any of the delete calls below. + if (!((lt_ != NULL && reverse_) + || (lte_ != NULL && reverse_) + || (gt_ != NULL && !reverse_) + || (gte_ != NULL && !reverse_))) { + delete [] start_->data(); + } + delete start_; + } + if (end_ != NULL) { + delete end_; + } + if (lt_ != NULL) { + delete lt_; + } + if (gt_ != NULL) { + delete gt_; + } + if (lte_ != NULL) { + delete lte_; + } + if (gte_ != NULL) { + delete gte_; + } + } + + void ReleaseTarget () { + if (target_ != NULL) { + if (!target_->empty()) { + delete [] target_->data(); + } + delete target_; + target_ = NULL; + } + } + + void Release () { + database_->ReleaseIterator(id_); + } + + leveldb::Status IteratorStatus () { + return dbIterator_->status(); + } + + void IteratorEnd () { + delete dbIterator_; + dbIterator_ = NULL; + database_->ReleaseSnapshot(options_->snapshot); + } + + bool GetIterator () { + if (dbIterator_ == NULL) { + dbIterator_ = database_->NewIterator(options_); + + if (start_ != NULL) { + dbIterator_->Seek(*start_); + + if (reverse_) { + if (!dbIterator_->Valid()) { + // if it's past the last key, step back + dbIterator_->SeekToLast(); + } else { + std::string keyStr = dbIterator_->key().ToString(); + + if (lt_ != NULL) { + // TODO make a compount if statement + if (lt_->compare(keyStr) <= 0) { + dbIterator_->Prev(); + } + } else if (lte_ != NULL) { + // TODO make a compount if statement + if (lte_->compare(keyStr) < 0) { + dbIterator_->Prev(); + } + } else if (start_ != NULL) { + // TODO make a compount if statement + if (start_->compare(keyStr)) { + dbIterator_->Prev(); + } + } + } + + if (dbIterator_->Valid() && lt_ != NULL) { + if (lt_->compare(dbIterator_->key().ToString()) <= 0) { + dbIterator_->Prev(); + } + } + } else { + // TODO this could be an else if + if (dbIterator_->Valid() && gt_ != NULL + && gt_->compare(dbIterator_->key().ToString()) == 0) { + dbIterator_->Next(); + } + } + } else if (reverse_) { + dbIterator_->SeekToLast(); + } else { + dbIterator_->SeekToFirst(); + } + + return true; + } + + return false; + } + + bool Read (std::string& key, std::string& value) { + // if it's not the first call, move to next item. + if (!GetIterator() && !seeking_) { + if (reverse_) { + dbIterator_->Prev(); + } + else { + dbIterator_->Next(); + } + } + + seeking_ = false; + + // now check if this is the end or not, if not then return the key & value + if (dbIterator_->Valid()) { + std::string keyStr = dbIterator_->key().ToString(); + const int isEnd = end_ == NULL ? 1 : end_->compare(keyStr); + + if ((limit_ < 0 || ++count_ <= limit_) + && (end_ == NULL + || (reverse_ && (isEnd <= 0)) + || (!reverse_ && (isEnd >= 0))) + && ( lt_ != NULL ? (lt_->compare(keyStr) > 0) + : lte_ != NULL ? (lte_->compare(keyStr) >= 0) + : true ) + && ( gt_ != NULL ? (gt_->compare(keyStr) < 0) + : gte_ != NULL ? (gte_->compare(keyStr) <= 0) + : true ) + ) { + if (keys_) { + key.assign(dbIterator_->key().data(), dbIterator_->key().size()); + } + if (values_) { + value.assign(dbIterator_->value().data(), dbIterator_->value().size()); + } + return true; + } + } + + return false; + } + + bool OutOfRange (leveldb::Slice* target) { + if (lt_ != NULL) { + if (target->compare(*lt_) >= 0) + return true; + } else if (lte_ != NULL) { + if (target->compare(*lte_) > 0) + return true; + } else if (start_ != NULL && reverse_) { + if (target->compare(*start_) > 0) + return true; + } + + if (end_ != NULL) { + int d = target->compare(*end_); + if (reverse_ ? d < 0 : d > 0) + return true; + } + + if (gt_ != NULL) { + if (target->compare(*gt_) <= 0) + return true; + } else if (gte_ != NULL) { + if (target->compare(*gte_) < 0) + return true; + } else if (start_ != NULL && !reverse_) { + if (target->compare(*start_) < 0) + return true; + } + + return false; + } + + bool IteratorNext (std::vector >& result) { + size_t size = 0; + while (true) { + std::string key, value; + bool ok = Read(key, value); + + if (ok) { + result.push_back(std::make_pair(key, value)); + + if (!landed_) { + landed_ = true; + return true; + } + + size = size + key.size() + value.size(); + if (size > highWaterMark_) { + return true; + } + + } else { + return false; + } + } + } + + Database* database_; + uint32_t id_; + leveldb::Slice* start_; + std::string* end_; + bool reverse_; + bool keys_; + bool values_; + int limit_; + std::string* lt_; + std::string* lte_; + std::string* gt_; + std::string* gte_; + bool keyAsBuffer_; + bool valueAsBuffer_; + uint32_t highWaterMark_; + leveldb::Iterator* dbIterator_; + int count_; + leveldb::Slice* target_; + bool seeking_; + bool landed_; + bool nexting_; + bool ended_; + + leveldb::ReadOptions* options_; + EndWorker* endWorker_; +}; + /** * Returns a context object for a database. */ @@ -540,6 +825,10 @@ struct CloseWorker : public BaseWorker { } }; +napi_value noop_callback (napi_env env, napi_callback_info info) { + return 0; +} + /** * Close a database. */ @@ -553,216 +842,73 @@ NAPI_METHOD(db_close) { if (database->iterators_.empty()) { worker->Queue(); NAPI_RETURN_UNDEFINED(); - } - - // TODO fix me! - - /* - // yikes, we still have iterators open! naughty naughty. - // we have to queue up a CloseWorker and manually close each of them. - // the CloseWorker will be invoked once they are all cleaned up - database->pendingCloseWorker = worker; - - for ( - std::map< uint32_t, leveldown::Iterator * >::iterator it - = database->iterators.begin() - ; it != database->iterators.end() - ; ++it) { - - // for each iterator still open, first check if it's already in - // the process of ending (ended==true means an async End() is - // in progress), if not, then we call End() with an empty callback - // function and wait for it to hit ReleaseIterator() where our - // CloseWorker will be invoked - - leveldown::Iterator *iterator = it->second; - - if (!iterator->ended) { - v8::Local end = - v8::Local::Cast(iterator->handle()->Get( - Nan::New("end").ToLocalChecked())); - v8::Local argv[] = { - Nan::New(EmptyMethod)->GetFunction() // empty callback - }; - Nan::AsyncResource ar("leveldown:iterator.end"); - ar.runInAsyncScope(iterator->handle(), end, 1, argv); - } - } - - */ - - NAPI_RETURN_UNDEFINED(); -} - -/** - * Worker class for putting key/value to the database - */ -struct PutWorker : public BaseWorker { - PutWorker (napi_env env, - Database* database, - napi_value callback, - napi_value key, - napi_value value, - bool sync) - : BaseWorker(env, database, callback, "leveldown.db.put"), - key_(ToSlice(env, key)), value_(ToSlice(env, value)) { - options_.sync = sync; - } - - virtual ~PutWorker () { - // TODO clean up key_ and value_ if they aren't empty? - // See DisposeStringOrBufferFromSlice() - } - - virtual void DoExecute () { - SetStatus(database_->Put(options_, key_, value_)); - } - - leveldb::WriteOptions options_; - leveldb::Slice key_; - leveldb::Slice value_; -}; - -/** - * Puts a key and a value to a database. - */ -NAPI_METHOD(db_put) { - NAPI_ARGV(5); - NAPI_DB_CONTEXT(); - - napi_value key = argv[1]; - napi_value value = argv[2]; - bool sync = BooleanProperty(env, argv[3], "sync", false); - napi_value callback = argv[4]; - - PutWorker* worker = new PutWorker(env, - database, - callback, - key, - value, - sync); - worker->Queue(); - - NAPI_RETURN_UNDEFINED(); -} - -/** - * Worker class for getting a value from a database. - */ -struct GetWorker : public BaseWorker { - GetWorker (napi_env env, - Database* database, - napi_value callback, - napi_value key, - bool asBuffer, - bool fillCache) - : BaseWorker(env, database, callback, "leveldown.db.get"), - key_(ToSlice(env, key)), - asBuffer_(asBuffer) { - options_.fill_cache = fillCache; - } - - virtual ~GetWorker () { - // TODO clean up key_ if not empty? - // See DisposeStringOrBufferFromSlice() - } - - virtual void DoExecute () { - SetStatus(database_->Get(options_, key_, value_)); - } - - virtual void HandleOKCallback() { - const int argc = 2; - napi_value argv[argc]; - napi_get_null(env_, &argv[0]); - - if (asBuffer_) { - napi_create_buffer_copy(env_, value_.size(), value_.data(), NULL, &argv[1]); - } else { - napi_create_string_utf8(env_, value_.data(), value_.size(), &argv[1]); - } - - // TODO move to base class - napi_value global; - napi_get_global(env_, &global); - napi_value callback; - napi_get_reference_value(env_, callbackRef_, &callback); - - napi_call_function(env_, global, callback, argc, argv, NULL); - } - - leveldb::ReadOptions options_; - leveldb::Slice key_; - std::string value_; - bool asBuffer_; -}; - -/** - * Gets a value from a database. - */ -NAPI_METHOD(db_get) { - NAPI_ARGV(4); - NAPI_DB_CONTEXT(); - - napi_value key = argv[1]; - napi_value options = argv[2]; - bool asBuffer = BooleanProperty(env, options, "asBuffer", true); - bool fillCache = BooleanProperty(env, options, "fillCache", true); - napi_value callback = argv[3]; - - GetWorker* worker = new GetWorker(env, - database, - callback, - key, - asBuffer, - fillCache); - worker->Queue(); + } + + database->pendingCloseWorker_ = worker; + + napi_value noop; + napi_create_function(env, NULL, 0, noop_callback, NULL, &noop); + + std::map< uint32_t, Iterator * >::iterator it; + for (it = database->iterators_.begin(); + it != database->iterators_.end(); ++it) { + + Iterator *iterator = it->second; + if (!iterator->ended_) { + // Call same logic as iterator_end() but provide a noop callback. + iterator_end_do(env, iterator, noop); + } + } NAPI_RETURN_UNDEFINED(); } /** - * Worker class for getting a value from a database. + * Worker class for putting key/value to the database */ -struct DelWorker : public BaseWorker { - DelWorker (napi_env env, +struct PutWorker : public BaseWorker { + PutWorker (napi_env env, Database* database, napi_value callback, napi_value key, + napi_value value, bool sync) - : BaseWorker(env, database, callback, "leveldown.db.get"), - key_(ToSlice(env, key)) { + : BaseWorker(env, database, callback, "leveldown.db.put"), + key_(ToSlice(env, key)), value_(ToSlice(env, value)) { options_.sync = sync; } - virtual ~DelWorker () { - // TODO clean up key_ if not empty? + virtual ~PutWorker () { + // TODO clean up key_ and value_ if they aren't empty? // See DisposeStringOrBufferFromSlice() } virtual void DoExecute () { - SetStatus(database_->Del(options_, key_)); + SetStatus(database_->Put(options_, key_, value_)); } leveldb::WriteOptions options_; leveldb::Slice key_; + leveldb::Slice value_; }; /** - * Gets a value from a database. + * Puts a key and a value to a database. */ -NAPI_METHOD(db_del) { - NAPI_ARGV(4); +NAPI_METHOD(db_put) { + NAPI_ARGV(5); NAPI_DB_CONTEXT(); napi_value key = argv[1]; - bool sync = BooleanProperty(env, argv[2], "sync", false); - napi_value callback = argv[3]; + napi_value value = argv[2]; + bool sync = BooleanProperty(env, argv[3], "sync", false); + napi_value callback = argv[4]; - DelWorker* worker = new DelWorker(env, + PutWorker* worker = new PutWorker(env, database, callback, key, + value, sync); worker->Queue(); @@ -772,350 +918,189 @@ NAPI_METHOD(db_del) { /** * Worker class for getting a value from a database. */ -struct ApproximateSizeWorker : public BaseWorker { - ApproximateSizeWorker (napi_env env, - Database* database, - napi_value callback, - leveldb::Slice start, - leveldb::Slice end) - : BaseWorker(env, database, callback, "leveldown.db.approximate_size"), - start_(start), end_(end) {} +struct GetWorker : public BaseWorker { + GetWorker (napi_env env, + Database* database, + napi_value callback, + napi_value key, + bool asBuffer, + bool fillCache) + : BaseWorker(env, database, callback, "leveldown.db.get"), + key_(ToSlice(env, key)), + asBuffer_(asBuffer) { + options_.fill_cache = fillCache; + } - virtual ~ApproximateSizeWorker () { - // TODO clean up start_ and end_ slices + virtual ~GetWorker () { + // TODO clean up key_ if not empty? // See DisposeStringOrBufferFromSlice() } virtual void DoExecute () { - leveldb::Range range(start_, end_); - size_ = database_->ApproximateSize(&range); + SetStatus(database_->Get(options_, key_, value_)); } virtual void HandleOKCallback() { const int argc = 2; napi_value argv[argc]; napi_get_null(env_, &argv[0]); - napi_create_uint32(env_, (uint32_t)size_, &argv[1]); - - // TODO move to base class - napi_value global; - napi_get_global(env_, &global); - napi_value callback; - napi_get_reference_value(env_, callbackRef_, &callback); - - napi_call_function(env_, global, callback, argc, argv, NULL); - } - - leveldb::Slice start_; - leveldb::Slice end_; - uint64_t size_; -}; - -/** - * Calculates the approximate size of a range in a database. - */ -NAPI_METHOD(db_approximate_size) { - NAPI_ARGV(4); - NAPI_DB_CONTEXT(); - - leveldb::Slice start = ToSlice(env, argv[1]); - leveldb::Slice end = ToSlice(env, argv[2]); - napi_value callback = argv[3]; - - ApproximateSizeWorker* worker = new ApproximateSizeWorker(env, - database, - callback, - start, - end); - worker->Queue(); - - NAPI_RETURN_UNDEFINED(); -} - -/** - * Owns a leveldb iterator. - */ -struct Iterator { - Iterator (Database* database, - uint32_t id, - leveldb::Slice* start, - std::string* end, - bool reverse, - bool keys, - bool values, - int limit, - std::string* lt, - std::string* lte, - std::string* gt, - std::string* gte, - bool fillCache, - bool keyAsBuffer, - bool valueAsBuffer, - uint32_t highWaterMark) - : database_(database), - id_(id), - start_(start), - end_(end), - reverse_(reverse), - keys_(keys), - values_(values), - limit_(limit), - lt_(lt), - lte_(lte), - gt_(gt), - gte_(gte), - keyAsBuffer_(keyAsBuffer), - valueAsBuffer_(valueAsBuffer), - highWaterMark_(highWaterMark), - dbIterator_(NULL), - count_(0), - target_(NULL), - seeking_(false), - landed_(false), - nexting_(false), - ended_(false), - endWorker_(NULL) { - options_ = new leveldb::ReadOptions(); - options_->fill_cache = fillCache; - options_->snapshot = database->NewSnapshot(); - } - - ~Iterator () { - ReleaseTarget(); - if (start_ != NULL) { - // Special case for `start` option: it won't be - // freed up by any of the delete calls below. - if (!((lt_ != NULL && reverse_) - || (lte_ != NULL && reverse_) - || (gt_ != NULL && !reverse_) - || (gte_ != NULL && !reverse_))) { - delete [] start_->data(); - } - delete start_; - } - if (end_ != NULL) { - delete end_; - } - if (lt_ != NULL) { - delete lt_; - } - if (gt_ != NULL) { - delete gt_; - } - if (lte_ != NULL) { - delete lte_; - } - if (gte_ != NULL) { - delete gte_; - } - } - - void ReleaseTarget () { - if (target_ != NULL) { - if (!target_->empty()) { - delete [] target_->data(); - } - delete target_; - target_ = NULL; - } - } - - void Release () { - database_->ReleaseIterator(id_); - } - - leveldb::Status IteratorStatus () { - return dbIterator_->status(); - } - - void IteratorEnd () { - delete dbIterator_; - dbIterator_ = NULL; - database_->ReleaseSnapshot(options_->snapshot); - } - - bool GetIterator () { - if (dbIterator_ == NULL) { - dbIterator_ = database_->NewIterator(options_); - - if (start_ != NULL) { - dbIterator_->Seek(*start_); - - if (reverse_) { - if (!dbIterator_->Valid()) { - // if it's past the last key, step back - dbIterator_->SeekToLast(); - } else { - std::string keyStr = dbIterator_->key().ToString(); - - if (lt_ != NULL) { - // TODO make a compount if statement - if (lt_->compare(keyStr) <= 0) { - dbIterator_->Prev(); - } - } else if (lte_ != NULL) { - // TODO make a compount if statement - if (lte_->compare(keyStr) < 0) { - dbIterator_->Prev(); - } - } else if (start_ != NULL) { - // TODO make a compount if statement - if (start_->compare(keyStr)) { - dbIterator_->Prev(); - } - } - } - - if (dbIterator_->Valid() && lt_ != NULL) { - if (lt_->compare(dbIterator_->key().ToString()) <= 0) { - dbIterator_->Prev(); - } - } - } else { - // TODO this could be an else if - if (dbIterator_->Valid() && gt_ != NULL - && gt_->compare(dbIterator_->key().ToString()) == 0) { - dbIterator_->Next(); - } - } - } else if (reverse_) { - dbIterator_->SeekToLast(); - } else { - dbIterator_->SeekToFirst(); - } - return true; + if (asBuffer_) { + napi_create_buffer_copy(env_, value_.size(), value_.data(), NULL, &argv[1]); + } else { + napi_create_string_utf8(env_, value_.data(), value_.size(), &argv[1]); } - return false; + // TODO move to base class + napi_value global; + napi_get_global(env_, &global); + napi_value callback; + napi_get_reference_value(env_, callbackRef_, &callback); + + napi_call_function(env_, global, callback, argc, argv, NULL); } - bool Read (std::string& key, std::string& value) { - // if it's not the first call, move to next item. - if (!GetIterator() && !seeking_) { - if (reverse_) { - dbIterator_->Prev(); - } - else { - dbIterator_->Next(); - } - } + leveldb::ReadOptions options_; + leveldb::Slice key_; + std::string value_; + bool asBuffer_; +}; - seeking_ = false; +/** + * Gets a value from a database. + */ +NAPI_METHOD(db_get) { + NAPI_ARGV(4); + NAPI_DB_CONTEXT(); - // now check if this is the end or not, if not then return the key & value - if (dbIterator_->Valid()) { - std::string keyStr = dbIterator_->key().ToString(); - const int isEnd = end_ == NULL ? 1 : end_->compare(keyStr); + napi_value key = argv[1]; + napi_value options = argv[2]; + bool asBuffer = BooleanProperty(env, options, "asBuffer", true); + bool fillCache = BooleanProperty(env, options, "fillCache", true); + napi_value callback = argv[3]; - if ((limit_ < 0 || ++count_ <= limit_) - && (end_ == NULL - || (reverse_ && (isEnd <= 0)) - || (!reverse_ && (isEnd >= 0))) - && ( lt_ != NULL ? (lt_->compare(keyStr) > 0) - : lte_ != NULL ? (lte_->compare(keyStr) >= 0) - : true ) - && ( gt_ != NULL ? (gt_->compare(keyStr) < 0) - : gte_ != NULL ? (gte_->compare(keyStr) <= 0) - : true ) - ) { - if (keys_) { - key.assign(dbIterator_->key().data(), dbIterator_->key().size()); - } - if (values_) { - value.assign(dbIterator_->value().data(), dbIterator_->value().size()); - } - return true; - } - } + GetWorker* worker = new GetWorker(env, + database, + callback, + key, + asBuffer, + fillCache); + worker->Queue(); - return false; + NAPI_RETURN_UNDEFINED(); +} + +/** + * Worker class for getting a value from a database. + */ +struct DelWorker : public BaseWorker { + DelWorker (napi_env env, + Database* database, + napi_value callback, + napi_value key, + bool sync) + : BaseWorker(env, database, callback, "leveldown.db.get"), + key_(ToSlice(env, key)) { + options_.sync = sync; } - bool OutOfRange (leveldb::Slice* target) { - if (lt_ != NULL) { - if (target->compare(*lt_) >= 0) - return true; - } else if (lte_ != NULL) { - if (target->compare(*lte_) > 0) - return true; - } else if (start_ != NULL && reverse_) { - if (target->compare(*start_) > 0) - return true; - } + virtual ~DelWorker () { + // TODO clean up key_ if not empty? + // See DisposeStringOrBufferFromSlice() + } - if (end_ != NULL) { - int d = target->compare(*end_); - if (reverse_ ? d < 0 : d > 0) - return true; - } + virtual void DoExecute () { + SetStatus(database_->Del(options_, key_)); + } - if (gt_ != NULL) { - if (target->compare(*gt_) <= 0) - return true; - } else if (gte_ != NULL) { - if (target->compare(*gte_) < 0) - return true; - } else if (start_ != NULL && !reverse_) { - if (target->compare(*start_) < 0) - return true; - } + leveldb::WriteOptions options_; + leveldb::Slice key_; +}; - return false; - } +/** + * Gets a value from a database. + */ +NAPI_METHOD(db_del) { + NAPI_ARGV(4); + NAPI_DB_CONTEXT(); - bool IteratorNext (std::vector >& result) { - size_t size = 0; - while (true) { - std::string key, value; - bool ok = Read(key, value); + napi_value key = argv[1]; + bool sync = BooleanProperty(env, argv[2], "sync", false); + napi_value callback = argv[3]; - if (ok) { - result.push_back(std::make_pair(key, value)); + DelWorker* worker = new DelWorker(env, + database, + callback, + key, + sync); + worker->Queue(); - if (!landed_) { - landed_ = true; - return true; - } + NAPI_RETURN_UNDEFINED(); +} - size = size + key.size() + value.size(); - if (size > highWaterMark_) { - return true; - } +/** + * Worker class for getting a value from a database. + */ +struct ApproximateSizeWorker : public BaseWorker { + ApproximateSizeWorker (napi_env env, + Database* database, + napi_value callback, + leveldb::Slice start, + leveldb::Slice end) + : BaseWorker(env, database, callback, "leveldown.db.approximate_size"), + start_(start), end_(end) {} - } else { - return false; - } - } + virtual ~ApproximateSizeWorker () { + // TODO clean up start_ and end_ slices + // See DisposeStringOrBufferFromSlice() } - Database* database_; - uint32_t id_; - leveldb::Slice* start_; - std::string* end_; - bool reverse_; - bool keys_; - bool values_; - int limit_; - std::string* lt_; - std::string* lte_; - std::string* gt_; - std::string* gte_; - bool keyAsBuffer_; - bool valueAsBuffer_; - uint32_t highWaterMark_; - leveldb::Iterator* dbIterator_; - int count_; - leveldb::Slice* target_; - bool seeking_; - bool landed_; - bool nexting_; - bool ended_; + virtual void DoExecute () { + leveldb::Range range(start_, end_); + size_ = database_->ApproximateSize(&range); + } - leveldb::ReadOptions* options_; - EndWorker* endWorker_; + virtual void HandleOKCallback() { + const int argc = 2; + napi_value argv[argc]; + napi_get_null(env_, &argv[0]); + napi_create_uint32(env_, (uint32_t)size_, &argv[1]); + + // TODO move to base class + napi_value global; + napi_get_global(env_, &global); + napi_value callback; + napi_get_reference_value(env_, callbackRef_, &callback); + + napi_call_function(env_, global, callback, argc, argv, NULL); + } + + leveldb::Slice start_; + leveldb::Slice end_; + uint64_t size_; }; +/** + * Calculates the approximate size of a range in a database. + */ +NAPI_METHOD(db_approximate_size) { + NAPI_ARGV(4); + NAPI_DB_CONTEXT(); + + leveldb::Slice start = ToSlice(env, argv[1]); + leveldb::Slice end = ToSlice(env, argv[2]); + napi_value callback = argv[3]; + + ApproximateSizeWorker* worker = new ApproximateSizeWorker(env, + database, + callback, + start, + end); + worker->Queue(); + + NAPI_RETURN_UNDEFINED(); +} + /** * Runs when an Iterator is garbage collected. */ @@ -1371,16 +1356,12 @@ struct EndWorker : public BaseWorker { }; /** - * Ends an iterator. + * Called by NAPI_METHOD(iterator_end) and also when closing + * open iterators during NAPI_METHOD(db_close). */ -NAPI_METHOD(iterator_end) { - NAPI_ARGV(2); - NAPI_ITERATOR_CONTEXT(); - - napi_value callback = argv[1]; - +static void iterator_end_do (napi_env env, Iterator* iterator, napi_value cb) { if (!iterator->ended_) { - EndWorker* worker = new EndWorker(env, iterator, callback); + EndWorker* worker = new EndWorker(env, iterator, cb); iterator->ended_ = true; if (iterator->nexting_) { @@ -1390,6 +1371,16 @@ NAPI_METHOD(iterator_end) { worker->Queue(); } } +} + +/** + * Ends an iterator. + */ +NAPI_METHOD(iterator_end) { + NAPI_ARGV(2); + NAPI_ITERATOR_CONTEXT(); + + iterator_end_do(env, iterator, argv[1]); NAPI_RETURN_UNDEFINED(); } diff --git a/test/index.js b/test/index.js index e570a403..c677985c 100644 --- a/test/index.js +++ b/test/index.js @@ -1,7 +1,7 @@ //require('./leveldown-test') //require('./abstract-leveldown-test') -require('./approximate-size-test') -// require('./cleanup-hanging-iterators-test') +//require('./approximate-size-test') +require('./cleanup-hanging-iterators-test') // require('./compact-range-test') // require('./compression-test') // require('./destroy-test') From fc46554f34446306031d56a9f698a9e63d305290 Mon Sep 17 00:00:00 2001 From: Lars-Magnus Skog Date: Sat, 8 Dec 2018 23:23:13 +0100 Subject: [PATCH 24/42] Fix "iterator has ended" error --- binding.cc | 28 +++++++++++++++++++++------- test/index.js | 27 ++++++++++++++------------- 2 files changed, 35 insertions(+), 20 deletions(-) diff --git a/binding.cc b/binding.cc index 7d59634b..788ac893 100644 --- a/binding.cc +++ b/binding.cc @@ -84,6 +84,17 @@ static bool IsObject (napi_env env, napi_value value) { return type == napi_object; } +/** + * Create an error object. + */ +static napi_value CreateError (napi_env env, const char* str) { + napi_value msg; + napi_create_string_utf8(env, str, strlen(str), &msg); + napi_value error; + napi_create_error(env, NULL, msg, &error); + return error; +} + /** * Returns true if 'obj' has a property 'key'. */ @@ -271,12 +282,9 @@ struct BaseWorker { napi_value callback; napi_get_reference_value(env_, callbackRef_, &callback); - napi_value msg; - napi_create_string_utf8(env_, errMsg_, strlen(errMsg_), &msg); - const int argc = 1; napi_value argv[argc]; - napi_create_error(env_, NULL, msg, &argv[0]); + argv[0] = CreateError(env_, errMsg_); napi_call_function(env_, global, callback, argc, argv, NULL); } @@ -1486,9 +1494,15 @@ NAPI_METHOD(iterator_next) { napi_value callback = argv[1]; if (iterator->ended_) { - // TODO call back with error "iterator has ended" - //v8::Local argv[] = { Nan::Error("iterator has ended") }; - //LD_RUN_CALLBACK("leveldown:iterator.next", callback, 1, argv); + // TODO refactor with other callback code + napi_value global; + napi_get_global(env, &global); + + const int argc = 1; + napi_value argv[argc]; + argv[0] = CreateError(env, "iterator has ended"); + napi_call_function(env, global, callback, argc, argv, NULL); + NAPI_RETURN_UNDEFINED(); } diff --git a/test/index.js b/test/index.js index c677985c..235a8d18 100644 --- a/test/index.js +++ b/test/index.js @@ -1,14 +1,15 @@ -//require('./leveldown-test') -//require('./abstract-leveldown-test') -//require('./approximate-size-test') +require('./leveldown-test') +require('./abstract-leveldown-test') +require('./approximate-size-test') require('./cleanup-hanging-iterators-test') -// require('./compact-range-test') -// require('./compression-test') -// require('./destroy-test') -// require('./getproperty-test') -// require('./iterator-gc-test') -// require('./iterator-recursion-test') -// require('./iterator-test') -// require('./port-libuv-fix-test') -// require('./repair-test') -// require('./segfault-test') +require('./compression-test') +require('./iterator-gc-test') +require('./iterator-recursion-test') +require('./port-libuv-fix-test') +require('./segfault-test') +require('./iterator-test') + +//require('./destroy-test') +//require('./repair-test') +//require('./compact-range-test') +//require('./getproperty-test') From 0790470692ec1170e3ce5c58a88ff0aa029737dd Mon Sep 17 00:00:00 2001 From: Lars-Magnus Skog Date: Sat, 8 Dec 2018 23:35:23 +0100 Subject: [PATCH 25/42] Implement db_compact_range() --- binding.cc | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++++ leveldown.js | 17 +++-------------- 2 files changed, 55 insertions(+), 14 deletions(-) diff --git a/binding.cc b/binding.cc index 788ac893..96d81dba 100644 --- a/binding.cc +++ b/binding.cc @@ -381,6 +381,11 @@ struct Database { return size; } + void CompactRange (const leveldb::Slice* start, + const leveldb::Slice* end) { + db_->CompactRange(start, end); + } + const leveldb::Snapshot* NewSnapshot () { return db_->GetSnapshot(); } @@ -1109,6 +1114,52 @@ NAPI_METHOD(db_approximate_size) { NAPI_RETURN_UNDEFINED(); } +/** + * Worker class for getting a value from a database. + */ +struct CompactRangeWorker : public BaseWorker { + CompactRangeWorker (napi_env env, + Database* database, + napi_value callback, + leveldb::Slice start, + leveldb::Slice end) + : BaseWorker(env, database, callback, "leveldown.db.compact_range"), + start_(start), end_(end) {} + + virtual ~CompactRangeWorker () { + // TODO clean up start_ and end_ slices + // See DisposeStringOrBufferFromSlice() + } + + virtual void DoExecute () { + database_->CompactRange(&start_, &end_); + } + + leveldb::Slice start_; + leveldb::Slice end_; +}; + +/** + * Compacts a range. + */ +NAPI_METHOD(db_compact_range) { + NAPI_ARGV(4); + NAPI_DB_CONTEXT(); + + leveldb::Slice start = ToSlice(env, argv[1]); + leveldb::Slice end = ToSlice(env, argv[2]); + napi_value callback = argv[3]; + + CompactRangeWorker* worker = new CompactRangeWorker(env, + database, + callback, + start, + end); + worker->Queue(); + + NAPI_RETURN_UNDEFINED(); +} + /** * Runs when an Iterator is garbage collected. */ @@ -1777,6 +1828,7 @@ NAPI_INIT() { NAPI_EXPORT_FUNCTION(db_get); NAPI_EXPORT_FUNCTION(db_del); NAPI_EXPORT_FUNCTION(db_approximate_size); + NAPI_EXPORT_FUNCTION(db_compact_range); /** * Iterator related functions. diff --git a/leveldown.js b/leveldown.js index 1b09b8e9..a7c68c3e 100644 --- a/leveldown.js +++ b/leveldown.js @@ -22,12 +22,7 @@ function LevelDOWN (location) { util.inherits(LevelDOWN, AbstractLevelDOWN) LevelDOWN.prototype._open = function (options, callback) { - binding.db_open( - this.context, - this.location, - options, - callback - ) + binding.db_open(this.context, this.location, options, callback) } LevelDOWN.prototype._close = function (callback) { @@ -43,13 +38,7 @@ LevelDOWN.prototype._serializeValue = function (value) { } LevelDOWN.prototype._put = function (key, value, options, callback) { - binding.db_put( - this.context, - key, - value, - options, - callback - ) + binding.db_put(this.context, key, value, options, callback) } LevelDOWN.prototype._get = function (key, options, callback) { @@ -101,7 +90,7 @@ LevelDOWN.prototype.compactRange = function (start, end, callback) { start = this._serializeKey(start) end = this._serializeKey(end) - this.binding.compactRange(start, end, callback) + binding.db_compact_range(this.context, start, end, callback) } LevelDOWN.prototype.getProperty = function (property) { From ad079a561f55d197c438e58aaff4342de4c97651 Mon Sep 17 00:00:00 2001 From: Lars-Magnus Skog Date: Sun, 9 Dec 2018 00:01:58 +0100 Subject: [PATCH 26/42] Implement destroy_db() --- binding.cc | 63 ++++++++++++++++++++++++++++++++------------ leveldown.js | 2 +- test/destroy-test.js | 6 ++--- 3 files changed, 50 insertions(+), 21 deletions(-) diff --git a/binding.cc b/binding.cc index 96d81dba..47da4aa3 100644 --- a/binding.cc +++ b/binding.cc @@ -1004,7 +1004,7 @@ NAPI_METHOD(db_get) { } /** - * Worker class for getting a value from a database. + * Worker class for deleting a value from a database. */ struct DelWorker : public BaseWorker { DelWorker (napi_env env, @@ -1012,7 +1012,7 @@ struct DelWorker : public BaseWorker { napi_value callback, napi_value key, bool sync) - : BaseWorker(env, database, callback, "leveldown.db.get"), + : BaseWorker(env, database, callback, "leveldown.db.del"), key_(ToSlice(env, key)) { options_.sync = sync; } @@ -1031,7 +1031,7 @@ struct DelWorker : public BaseWorker { }; /** - * Gets a value from a database. + * Delete a value from a database. */ NAPI_METHOD(db_del) { NAPI_ARGV(4); @@ -1052,7 +1052,7 @@ NAPI_METHOD(db_del) { } /** - * Worker class for getting a value from a database. + * Worker class for calculating the size of a range. */ struct ApproximateSizeWorker : public BaseWorker { ApproximateSizeWorker (napi_env env, @@ -1115,7 +1115,7 @@ NAPI_METHOD(db_approximate_size) { } /** - * Worker class for getting a value from a database. + * Worker class for compacting a range in a database. */ struct CompactRangeWorker : public BaseWorker { CompactRangeWorker (napi_env env, @@ -1140,7 +1140,7 @@ struct CompactRangeWorker : public BaseWorker { }; /** - * Compacts a range. + * Compacts a range in a database. */ NAPI_METHOD(db_compact_range) { NAPI_ARGV(4); @@ -1160,6 +1160,44 @@ NAPI_METHOD(db_compact_range) { NAPI_RETURN_UNDEFINED(); } +/** + * Worker class for compacting a range in a database. + */ +struct DestroyWorker : public BaseWorker { + DestroyWorker (napi_env env, + const std::string* location, + napi_value callback) + // TODO turns out we don't even need database in BaseWorker() + : BaseWorker(env, NULL, callback, "leveldown.destroy_db"), + location_(location) {} + + virtual ~DestroyWorker () {} + + virtual void DoExecute () { + leveldb::Options options; + SetStatus(leveldb::DestroyDB(location_, options)); + } + + std::string location_; +}; + +/** + * Destroys a database. + */ +NAPI_METHOD(destroy_db) { + NAPI_ARGV(2); + // TODO replace with new + NAPI_ARGV_UTF8_MALLOC(location, 0); + napi_value callback = argv[1]; + + DestroyWorker* worker = new DestroyWorker(env, location, callback); + worker->Queue(); + + free(location); + + NAPI_RETURN_UNDEFINED(); +} + /** * Runs when an Iterator is garbage collected. */ @@ -1396,8 +1434,7 @@ struct EndWorker : public BaseWorker { EndWorker (napi_env env, Iterator* iterator, napi_value callback) - : BaseWorker(env, iterator->database_, callback, - "leveldown.iterator.end"), + : BaseWorker(env, iterator->database_, callback, "leveldown.iterator.end"), iterator_(iterator) {} virtual ~EndWorker () {} @@ -1818,9 +1855,6 @@ NAPI_METHOD(batch_write) { * All exported functions. */ NAPI_INIT() { - /** - * Database related functions. - */ NAPI_EXPORT_FUNCTION(db_init); NAPI_EXPORT_FUNCTION(db_open); NAPI_EXPORT_FUNCTION(db_close); @@ -1829,18 +1863,13 @@ NAPI_INIT() { NAPI_EXPORT_FUNCTION(db_del); NAPI_EXPORT_FUNCTION(db_approximate_size); NAPI_EXPORT_FUNCTION(db_compact_range); + NAPI_EXPORT_FUNCTION(destroy_db); - /** - * Iterator related functions. - */ NAPI_EXPORT_FUNCTION(iterator_init); NAPI_EXPORT_FUNCTION(iterator_seek); NAPI_EXPORT_FUNCTION(iterator_end); NAPI_EXPORT_FUNCTION(iterator_next); - /** - * Batch related functions. - */ NAPI_EXPORT_FUNCTION(batch_do); NAPI_EXPORT_FUNCTION(batch_init); NAPI_EXPORT_FUNCTION(batch_put); diff --git a/leveldown.js b/leveldown.js index a7c68c3e..66740d5d 100644 --- a/leveldown.js +++ b/leveldown.js @@ -121,7 +121,7 @@ LevelDOWN.destroy = function (location, callback) { throw new Error('destroy() requires a callback function argument') } - binding.destroy(location, callback) + binding.destroy_db(location, callback) } LevelDOWN.repair = function (location, callback) { diff --git a/test/destroy-test.js b/test/destroy-test.js index 0948cc22..eea7f01e 100644 --- a/test/destroy-test.js +++ b/test/destroy-test.js @@ -24,7 +24,7 @@ test('test callback-less, 1-arg, destroy() throws', function (t) { t.end() }) -test('test destroy non-existent directory', function (t) { +test.only('test destroy non-existent directory', function (t) { t.plan(4) var location = tempy.directory() @@ -37,8 +37,8 @@ test('test destroy non-existent directory', function (t) { rimraf(location, { glob: false }, function (err) { t.ifError(err, 'no error from rimraf()') - leveldown.destroy(location, function () { - t.is(arguments.length, 0, 'no arguments returned on callback') + leveldown.destroy(location, function (err) { + t.error(err, 'no error') // Assert that destroy() didn't inadvertently create the directory. // Or if it did, that it was at least cleaned up afterwards. From a1f94dab8473e2daf7d4ba9b215a610d743a2501 Mon Sep 17 00:00:00 2001 From: Lars-Magnus Skog Date: Sun, 9 Dec 2018 00:06:35 +0100 Subject: [PATCH 27/42] Implement repair_db() --- binding.cc | 43 +++++++++++++++++++++++++++++++++++++++++-- leveldown.js | 2 +- 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/binding.cc b/binding.cc index 47da4aa3..429a2694 100644 --- a/binding.cc +++ b/binding.cc @@ -1161,11 +1161,11 @@ NAPI_METHOD(db_compact_range) { } /** - * Worker class for compacting a range in a database. + * Worker class for destroying a database. */ struct DestroyWorker : public BaseWorker { DestroyWorker (napi_env env, - const std::string* location, + const std::string& location, napi_value callback) // TODO turns out we don't even need database in BaseWorker() : BaseWorker(env, NULL, callback, "leveldown.destroy_db"), @@ -1198,6 +1198,43 @@ NAPI_METHOD(destroy_db) { NAPI_RETURN_UNDEFINED(); } +/** + * Worker class for repairing a database. + */ +struct RepairWorker : public BaseWorker { + RepairWorker (napi_env env, + const std::string& location, + napi_value callback) + : BaseWorker(env, NULL, callback, "leveldown.repair_db"), + location_(location) {} + + virtual ~RepairWorker () {} + + virtual void DoExecute () { + leveldb::Options options; + SetStatus(leveldb::RepairDB(location_, options)); + } + + std::string location_; +}; + +/** + * Repairs a database. + */ +NAPI_METHOD(repair_db) { + NAPI_ARGV(2); + // TODO replace with new + NAPI_ARGV_UTF8_MALLOC(location, 0); + napi_value callback = argv[1]; + + RepairWorker* worker = new RepairWorker(env, location, callback); + worker->Queue(); + + free(location); + + NAPI_RETURN_UNDEFINED(); +} + /** * Runs when an Iterator is garbage collected. */ @@ -1863,7 +1900,9 @@ NAPI_INIT() { NAPI_EXPORT_FUNCTION(db_del); NAPI_EXPORT_FUNCTION(db_approximate_size); NAPI_EXPORT_FUNCTION(db_compact_range); + NAPI_EXPORT_FUNCTION(destroy_db); + NAPI_EXPORT_FUNCTION(repair_db); NAPI_EXPORT_FUNCTION(iterator_init); NAPI_EXPORT_FUNCTION(iterator_seek); diff --git a/leveldown.js b/leveldown.js index 66740d5d..3aec99c3 100644 --- a/leveldown.js +++ b/leveldown.js @@ -135,7 +135,7 @@ LevelDOWN.repair = function (location, callback) { throw new Error('repair() requires a callback function argument') } - binding.repair(location, callback) + binding.repair_db(location, callback) } module.exports = LevelDOWN.default = LevelDOWN From ce049ddd473649f5e5f1244a33110c418d6c56cf Mon Sep 17 00:00:00 2001 From: Lars-Magnus Skog Date: Sun, 9 Dec 2018 00:17:03 +0100 Subject: [PATCH 28/42] Implement db_get_property() --- binding.cc | 24 ++++++++++++++++++++++++ leveldown.js | 2 +- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/binding.cc b/binding.cc index 429a2694..8f0541f1 100644 --- a/binding.cc +++ b/binding.cc @@ -386,6 +386,10 @@ struct Database { db_->CompactRange(start, end); } + void GetProperty (const leveldb::Slice& property, std::string* value) { + db_->GetProperty(property, value); + } + const leveldb::Snapshot* NewSnapshot () { return db_->GetSnapshot(); } @@ -1160,6 +1164,25 @@ NAPI_METHOD(db_compact_range) { NAPI_RETURN_UNDEFINED(); } +/** + * Get a property from a database. + */ +NAPI_METHOD(db_get_property) { + NAPI_ARGV(2); + NAPI_DB_CONTEXT(); + + leveldb::Slice property = ToSlice(env, argv[1]); + + std::string value; + database->GetProperty(property, &value); + + napi_value result; + napi_create_string_utf8(env, value.data(), value.size(), &result); + + // TODO clean up property slice + return result; +} + /** * Worker class for destroying a database. */ @@ -1900,6 +1923,7 @@ NAPI_INIT() { NAPI_EXPORT_FUNCTION(db_del); NAPI_EXPORT_FUNCTION(db_approximate_size); NAPI_EXPORT_FUNCTION(db_compact_range); + NAPI_EXPORT_FUNCTION(db_get_property); NAPI_EXPORT_FUNCTION(destroy_db); NAPI_EXPORT_FUNCTION(repair_db); diff --git a/leveldown.js b/leveldown.js index 3aec99c3..4452190b 100644 --- a/leveldown.js +++ b/leveldown.js @@ -98,7 +98,7 @@ LevelDOWN.prototype.getProperty = function (property) { throw new Error('getProperty() requires a valid `property` argument') } - return this.binding.getProperty(property) + return binding.db_get_property(this.context, property) } LevelDOWN.prototype._iterator = function (options) { From 50062ad5fdd54e6824fb57ee99bcf78446ec5ab3 Mon Sep 17 00:00:00 2001 From: Lars-Magnus Skog Date: Sun, 9 Dec 2018 00:21:41 +0100 Subject: [PATCH 29/42] Accidentally disabled all tests but one :) --- test/destroy-test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/destroy-test.js b/test/destroy-test.js index eea7f01e..aba89104 100644 --- a/test/destroy-test.js +++ b/test/destroy-test.js @@ -24,7 +24,7 @@ test('test callback-less, 1-arg, destroy() throws', function (t) { t.end() }) -test.only('test destroy non-existent directory', function (t) { +test('test destroy non-existent directory', function (t) { t.plan(4) var location = tempy.directory() From 1dfe4343278e2ea8713a1c74108c7108721bc90d Mon Sep 17 00:00:00 2001 From: Lars-Magnus Skog Date: Sun, 9 Dec 2018 00:23:39 +0100 Subject: [PATCH 30/42] .destroy() now calls back with null --- test/destroy-test.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/destroy-test.js b/test/destroy-test.js index aba89104..32dfa976 100644 --- a/test/destroy-test.js +++ b/test/destroy-test.js @@ -55,8 +55,8 @@ test('test destroy non-existent parent directory', function (t) { t.notOk(fs.existsSync(parent), 'parent does not exist before') - leveldown.destroy(location, function () { - t.is(arguments.length, 0, 'no arguments returned on callback') + leveldown.destroy(location, function (err) { + t.error(err, 'no error') t.notOk(fs.existsSync(location), 'directory does not exist after') }) }) From 02ccedab25006743cd04920e591732fd358f0bfa Mon Sep 17 00:00:00 2001 From: Lars-Magnus Skog Date: Sun, 9 Dec 2018 00:28:09 +0100 Subject: [PATCH 31/42] Reset back to normal testing --- package.json | 1 - test/index.js | 15 --------------- 2 files changed, 16 deletions(-) delete mode 100644 test/index.js diff --git a/package.json b/package.json index d50dc8f8..6cb50de4 100644 --- a/package.json +++ b/package.json @@ -46,7 +46,6 @@ "scripts": { "install": "node-gyp-build", "test": "standard && verify-travis-appveyor && nyc tape test/*-test.js && prebuild-ci", - "test2": "node test", "coverage": "nyc report --reporter=text-lcov | coveralls", "rebuild": "prebuild --compile", "prebuild": "prebuild --all --strip --verbose", diff --git a/test/index.js b/test/index.js deleted file mode 100644 index 235a8d18..00000000 --- a/test/index.js +++ /dev/null @@ -1,15 +0,0 @@ -require('./leveldown-test') -require('./abstract-leveldown-test') -require('./approximate-size-test') -require('./cleanup-hanging-iterators-test') -require('./compression-test') -require('./iterator-gc-test') -require('./iterator-recursion-test') -require('./port-libuv-fix-test') -require('./segfault-test') -require('./iterator-test') - -//require('./destroy-test') -//require('./repair-test') -//require('./compact-range-test') -//require('./getproperty-test') From cb15d6e485dcb9b483a1c465b711b7b516bb0ca3 Mon Sep 17 00:00:00 2001 From: Lars-Magnus Skog Date: Sun, 9 Dec 2018 01:01:25 +0100 Subject: [PATCH 32/42] Clean up debug printf --- binding.cc | 3 --- 1 file changed, 3 deletions(-) diff --git a/binding.cc b/binding.cc index 8f0541f1..e9510ce9 100644 --- a/binding.cc +++ b/binding.cc @@ -44,13 +44,11 @@ static void iterator_end_do (napi_env env, Iterator* iterator, napi_value cb); to##Ch_ = new char[to##Sz_ + 1]; \ napi_get_value_string_utf8(env, from, to##Ch_, to##Sz_ + 1, &to##Sz_); \ to##Ch_[to##Sz_] = '\0'; \ - printf("-- LD_STRING_OR_BUFFER_TO_COPY STRING length: %zu content: %s\n", to##Sz_, to##Ch_); \ } else if (IsBuffer(env, from)) { \ char* buf = 0; \ napi_get_buffer_info(env, from, (void **)&buf, &to##Sz_); \ to##Ch_ = new char[to##Sz_]; \ memcpy(to##Ch_, buf, to##Sz_); \ - printf("-- LD_STRING_OR_BUFFER_TO_COPY BUFFER length: %zu content: %s\n", to##Sz_, to##Ch_); \ } /********************************************************************* @@ -1714,7 +1712,6 @@ NAPI_METHOD(batch_do) { if (!IsObject(env, element)) continue; std::string type = StringProperty(env, element, "type"); - printf("-- Batch element type: %s\n", type.c_str()); if (type == "del") { if (!HasProperty(env, element, "key")) continue; From d6001e1e05e962703b28e19f4c6c797aba3bb8ec Mon Sep 17 00:00:00 2001 From: Lars-Magnus Skog Date: Sun, 9 Dec 2018 01:29:12 +0100 Subject: [PATCH 33/42] Fix some napi commands + clean up old comments and reduce some if-statements --- binding.cc | 173 ++++++++++++++++++----------------------------------- 1 file changed, 57 insertions(+), 116 deletions(-) diff --git a/binding.cc b/binding.cc index e9510ce9..9b4a5889 100644 --- a/binding.cc +++ b/binding.cc @@ -97,11 +97,8 @@ static napi_value CreateError (napi_env env, const char* str) { * Returns true if 'obj' has a property 'key'. */ static bool HasProperty (napi_env env, napi_value obj, const char* key) { - // TODO switch to use napi_has_named_property() (no need to make a napi_value) bool has = false; - napi_value _key; - napi_create_string_utf8(env, key, strlen(key), &_key); - napi_has_property(env, obj, _key, &has); + napi_has_named_property(env, obj, key, &has); return has; } @@ -109,11 +106,8 @@ static bool HasProperty (napi_env env, napi_value obj, const char* key) { * Returns a property in napi_value form. */ static napi_value GetProperty (napi_env env, napi_value obj, const char* key) { - // TODO switch to use napi_get_named_property() (no need to make a napi_value) - napi_value _key; - napi_create_string_utf8(env, key, strlen(key), &_key); napi_value value; - napi_get_property(env, obj, _key, &value); + napi_get_named_property(env, obj, key, &value); return value; } @@ -401,14 +395,8 @@ struct Database { } void ReleaseIterator (uint32_t id) { - // called each time an Iterator is End()ed, in the main thread - // we have to remove our reference to it and if it's the last - // iterator we have to invoke a pending CloseWorker if there - // is one if there is a pending CloseWorker it means that - // we're waiting for iterators to end before we can close them iterators_.erase(id); if (iterators_.empty() && pendingCloseWorker_ != NULL) { - // TODO Test this! pendingCloseWorker_->Queue(); pendingCloseWorker_ = NULL; } @@ -422,10 +410,6 @@ struct Database { // time in the constructor? const leveldb::FilterPolicy* filterPolicy_; uint32_t currentIteratorId_; - // TODO this worker should have a proper type, now - // it's some form of function pointer bastard, it should - // just be a CloseWorker object - //void (*pendingCloseWorker_); BaseWorker *pendingCloseWorker_; std::map< uint32_t, Iterator * > iterators_; }; @@ -542,63 +526,46 @@ struct Iterator { } bool GetIterator () { - if (dbIterator_ == NULL) { - dbIterator_ = database_->NewIterator(options_); - - if (start_ != NULL) { - dbIterator_->Seek(*start_); - - if (reverse_) { - if (!dbIterator_->Valid()) { - // if it's past the last key, step back - dbIterator_->SeekToLast(); - } else { - std::string keyStr = dbIterator_->key().ToString(); - - if (lt_ != NULL) { - // TODO make a compount if statement - if (lt_->compare(keyStr) <= 0) { - dbIterator_->Prev(); - } - } else if (lte_ != NULL) { - // TODO make a compount if statement - if (lte_->compare(keyStr) < 0) { - dbIterator_->Prev(); - } - } else if (start_ != NULL) { - // TODO make a compount if statement - if (start_->compare(keyStr)) { - dbIterator_->Prev(); - } - } - } + if (dbIterator_ != NULL) return false; - if (dbIterator_->Valid() && lt_ != NULL) { - if (lt_->compare(dbIterator_->key().ToString()) <= 0) { - dbIterator_->Prev(); - } - } + dbIterator_ = database_->NewIterator(options_); + + if (start_ != NULL) { + dbIterator_->Seek(*start_); + + if (reverse_) { + if (!dbIterator_->Valid()) { + dbIterator_->SeekToLast(); } else { - // TODO this could be an else if - if (dbIterator_->Valid() && gt_ != NULL - && gt_->compare(dbIterator_->key().ToString()) == 0) { - dbIterator_->Next(); + std::string keyStr = dbIterator_->key().ToString(); + + if (lt_ != NULL && lt_->compare(keyStr) <= 0) { + dbIterator_->Prev(); + } else if (lte_ != NULL && lte_->compare(keyStr) < 0) { + dbIterator_->Prev(); + } else if (start_ != NULL && start_->compare(keyStr)) { + dbIterator_->Prev(); } } - } else if (reverse_) { - dbIterator_->SeekToLast(); - } else { - dbIterator_->SeekToFirst(); - } - return true; + if (dbIterator_->Valid() && lt_ != NULL + && lt_->compare(dbIterator_->key().ToString()) <= 0) { + dbIterator_->Prev(); + } + } else if (dbIterator_->Valid() && gt_ != NULL + && gt_->compare(dbIterator_->key().ToString()) == 0) { + dbIterator_->Next(); + } + } else if (reverse_) { + dbIterator_->SeekToLast(); + } else { + dbIterator_->SeekToFirst(); } - return false; + return true; } bool Read (std::string& key, std::string& value) { - // if it's not the first call, move to next item. if (!GetIterator() && !seeking_) { if (reverse_) { dbIterator_->Prev(); @@ -610,7 +577,6 @@ struct Iterator { seeking_ = false; - // now check if this is the end or not, if not then return the key & value if (dbIterator_->Valid()) { std::string keyStr = dbIterator_->key().ToString(); const int isEnd = end_ == NULL ? 1 : end_->compare(keyStr); @@ -640,35 +606,20 @@ struct Iterator { } bool OutOfRange (leveldb::Slice* target) { - if (lt_ != NULL) { - if (target->compare(*lt_) >= 0) - return true; - } else if (lte_ != NULL) { - if (target->compare(*lte_) > 0) - return true; - } else if (start_ != NULL && reverse_) { - if (target->compare(*start_) > 0) - return true; + if ((lt_ != NULL && target->compare(*lt_) >= 0) || + (lte_ != NULL && target->compare(*lte_) > 0) || + (start_ != NULL && reverse_ && target->compare(*start_) > 0)) { + return true; } if (end_ != NULL) { int d = target->compare(*end_); - if (reverse_ ? d < 0 : d > 0) - return true; - } - - if (gt_ != NULL) { - if (target->compare(*gt_) <= 0) - return true; - } else if (gte_ != NULL) { - if (target->compare(*gte_) < 0) - return true; - } else if (start_ != NULL && !reverse_) { - if (target->compare(*start_) < 0) - return true; + if (reverse_ ? d < 0 : d > 0) return true; } - return false; + return ((gt_ != NULL && target->compare(*gt_) <= 0) || + (gte_ != NULL && target->compare(*gte_) < 0) || + (start_ != NULL && !reverse_ && target->compare(*start_) < 0)); } bool IteratorNext (std::vector >& result) { @@ -686,9 +637,7 @@ struct Iterator { } size = size + key.size() + value.size(); - if (size > highWaterMark_) { - return true; - } + if (size > highWaterMark_) return true; } else { return false; @@ -790,7 +739,6 @@ NAPI_METHOD(db_open) { // so we have similar allocation/deallocation mechanisms everywhere NAPI_ARGV_UTF8_MALLOC(location, 1); - // Options object and properties. napi_value options = argv[2]; bool createIfMissing = BooleanProperty(env, options, "createIfMissing", true); bool errorIfExists = BooleanProperty(env, options, "errorIfExists", false); @@ -867,10 +815,8 @@ NAPI_METHOD(db_close) { std::map< uint32_t, Iterator * >::iterator it; for (it = database->iterators_.begin(); it != database->iterators_.end(); ++it) { - Iterator *iterator = it->second; if (!iterator->ended_) { - // Call same logic as iterator_end() but provide a noop callback. iterator_end_do(env, iterator, noop); } } @@ -947,7 +893,7 @@ struct GetWorker : public BaseWorker { } virtual ~GetWorker () { - // TODO clean up key_ if not empty? + // TODO clean up key_ if not empty // See DisposeStringOrBufferFromSlice() } @@ -971,7 +917,6 @@ struct GetWorker : public BaseWorker { napi_get_global(env_, &global); napi_value callback; napi_get_reference_value(env_, callbackRef_, &callback); - napi_call_function(env_, global, callback, argc, argv, NULL); } @@ -1086,7 +1031,6 @@ struct ApproximateSizeWorker : public BaseWorker { napi_get_global(env_, &global); napi_value callback; napi_get_reference_value(env_, callbackRef_, &callback); - napi_call_function(env_, global, callback, argc, argv, NULL); } @@ -1455,29 +1399,27 @@ NAPI_METHOD(iterator_seek) { dbIterator->Next(); } } - else { + else if (dbIterator->Valid()) { + int cmp = dbIterator->key().compare(*iterator->target_); + if (cmp > 0 && iterator->reverse_) { + dbIterator->Prev(); + } else if (cmp < 0 && !iterator->reverse_) { + dbIterator->Next(); + } + } else { + if (iterator->reverse_) { + dbIterator->SeekToLast(); + } else { + dbIterator->SeekToFirst(); + } if (dbIterator->Valid()) { int cmp = dbIterator->key().compare(*iterator->target_); if (cmp > 0 && iterator->reverse_) { + dbIterator->SeekToFirst(); dbIterator->Prev(); } else if (cmp < 0 && !iterator->reverse_) { - dbIterator->Next(); - } - } else { - if (iterator->reverse_) { dbIterator->SeekToLast(); - } else { - dbIterator->SeekToFirst(); - } - if (dbIterator->Valid()) { - int cmp = dbIterator->key().compare(*iterator->target_); - if (cmp > 0 && iterator->reverse_) { - dbIterator->SeekToFirst(); - dbIterator->Prev(); - } else if (cmp < 0 && !iterator->reverse_) { - dbIterator->SeekToLast(); - dbIterator->Next(); - } + dbIterator->Next(); } } } @@ -1519,7 +1461,6 @@ static void iterator_end_do (napi_env env, Iterator* iterator, napi_value cb) { iterator->ended_ = true; if (iterator->nexting_) { - // waiting for a next() to return, queue the end iterator->endWorker_ = worker; } else { worker->Queue(); From 885574974b70c07254f4cde17e01a6f6ad2c9325 Mon Sep 17 00:00:00 2001 From: Lars-Magnus Skog Date: Sun, 9 Dec 2018 02:17:22 +0100 Subject: [PATCH 34/42] Dispose data pointed to by instances of leveldb::Slice --- binding.cc | 46 ++++++++++++++++++++-------------------------- 1 file changed, 20 insertions(+), 26 deletions(-) diff --git a/binding.cc b/binding.cc index 9b4a5889..cb3ca8c8 100644 --- a/binding.cc +++ b/binding.cc @@ -183,6 +183,10 @@ static std::string StringProperty (napi_env env, napi_value obj, const char* key return ""; } +static void DisposeSliceBuffer (leveldb::Slice slice) { + if (!slice.empty()) delete [] slice.data(); +} + /** * Convert a napi_value to a leveldb::Slice. */ @@ -840,8 +844,8 @@ struct PutWorker : public BaseWorker { } virtual ~PutWorker () { - // TODO clean up key_ and value_ if they aren't empty? - // See DisposeStringOrBufferFromSlice() + DisposeSliceBuffer(key_); + DisposeSliceBuffer(value_); } virtual void DoExecute () { @@ -893,8 +897,7 @@ struct GetWorker : public BaseWorker { } virtual ~GetWorker () { - // TODO clean up key_ if not empty - // See DisposeStringOrBufferFromSlice() + DisposeSliceBuffer(key_); } virtual void DoExecute () { @@ -965,8 +968,7 @@ struct DelWorker : public BaseWorker { } virtual ~DelWorker () { - // TODO clean up key_ if not empty? - // See DisposeStringOrBufferFromSlice() + DisposeSliceBuffer(key_); } virtual void DoExecute () { @@ -1011,8 +1013,8 @@ struct ApproximateSizeWorker : public BaseWorker { start_(start), end_(end) {} virtual ~ApproximateSizeWorker () { - // TODO clean up start_ and end_ slices - // See DisposeStringOrBufferFromSlice() + DisposeSliceBuffer(start_); + DisposeSliceBuffer(end_); } virtual void DoExecute () { @@ -1073,8 +1075,8 @@ struct CompactRangeWorker : public BaseWorker { start_(start), end_(end) {} virtual ~CompactRangeWorker () { - // TODO clean up start_ and end_ slices - // See DisposeStringOrBufferFromSlice() + DisposeSliceBuffer(start_); + DisposeSliceBuffer(end_); } virtual void DoExecute () { @@ -1121,7 +1123,8 @@ NAPI_METHOD(db_get_property) { napi_value result; napi_create_string_utf8(env, value.data(), value.size(), &result); - // TODO clean up property slice + DisposeSliceBuffer(property); + return result; } @@ -1132,7 +1135,6 @@ struct DestroyWorker : public BaseWorker { DestroyWorker (napi_env env, const std::string& location, napi_value callback) - // TODO turns out we don't even need database in BaseWorker() : BaseWorker(env, NULL, callback, "leveldown.destroy_db"), location_(location) {} @@ -1661,8 +1663,7 @@ NAPI_METHOD(batch_do) { batch->Delete(key); if (!hasData) hasData = true; - // TODO clean up slices - //DisposeStringOrBufferFromSlice(keyBuffer, key); + DisposeSliceBuffer(key); } else if (type == "put") { if (!HasProperty(env, element, "key")) continue; if (!HasProperty(env, element, "value")) continue; @@ -1673,9 +1674,8 @@ NAPI_METHOD(batch_do) { batch->Put(key, value); if (!hasData) hasData = true; - // TODO clean up slices - //DisposeStringOrBufferFromSlice(keyBuffer, key); - //DisposeStringOrBufferFromSlice(valueBuffer, value); + DisposeSliceBuffer(key); + DisposeSliceBuffer(value); } } @@ -1771,12 +1771,9 @@ NAPI_METHOD(batch_put) { leveldb::Slice key = ToSlice(env, argv[1]); leveldb::Slice value = ToSlice(env, argv[2]); - batch->Put(key, value); - - // TODO clean up slices - //DisposeStringOrBufferFromSlice(keyBuffer, key); - //DisposeStringOrBufferFromSlice(valueBuffer, value); + DisposeSliceBuffer(key); + DisposeSliceBuffer(value); NAPI_RETURN_UNDEFINED(); } @@ -1789,11 +1786,8 @@ NAPI_METHOD(batch_del) { NAPI_BATCH_CONTEXT(); leveldb::Slice key = ToSlice(env, argv[1]); - batch->Del(key); - - // TODO clean up slice - //DisposeStringOrBufferFromSlice(keyBuffer, key); + DisposeSliceBuffer(key); NAPI_RETURN_UNDEFINED(); } From 6aca8c8ba6a7b3f3c5fc41dbc34d791558c137aa Mon Sep 17 00:00:00 2001 From: Lars-Magnus Skog Date: Sun, 9 Dec 2018 02:27:16 +0100 Subject: [PATCH 35/42] Only need to set filterPolicy once when Database object is created --- binding.cc | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/binding.cc b/binding.cc index cb3ca8c8..7831b77b 100644 --- a/binding.cc +++ b/binding.cc @@ -320,7 +320,7 @@ struct Database { : env_(env), db_(NULL), blockCache_(NULL), - filterPolicy_(NULL), + filterPolicy_(leveldb::NewBloomFilterPolicy(10)), currentIteratorId_(0), pendingCloseWorker_(NULL) {} @@ -343,10 +343,6 @@ struct Database { delete blockCache_; blockCache_ = NULL; } - if (filterPolicy_) { - delete filterPolicy_; - filterPolicy_ = NULL; - } } leveldb::Status Put (const leveldb::WriteOptions& options, @@ -409,9 +405,6 @@ struct Database { napi_env env_; leveldb::DB* db_; leveldb::Cache* blockCache_; - // TODO figure out if we can use filterPolicy_ across - // several Open()/Close(), i.e. can we create it _one_ - // time in the constructor? const leveldb::FilterPolicy* filterPolicy_; uint32_t currentIteratorId_; BaseWorker *pendingCloseWorker_; @@ -757,7 +750,6 @@ NAPI_METHOD(db_open) { uint32_t maxFileSize = Uint32Property(env, options, "maxFileSize", 2 << 20); database->blockCache_ = leveldb::NewLRUCache(cacheSize); - database->filterPolicy_ = leveldb::NewBloomFilterPolicy(10); napi_value callback = argv[3]; OpenWorker* worker = new OpenWorker(env, From cb34bf0ccb3cb26e1ca37bf7393d21292c185b30 Mon Sep 17 00:00:00 2001 From: Lars-Magnus Skog Date: Sun, 9 Dec 2018 02:38:08 +0100 Subject: [PATCH 36/42] Use new instead of malloc to keep similar memory handling everywhere --- binding.cc | 41 +++++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/binding.cc b/binding.cc index 7831b77b..a488b647 100644 --- a/binding.cc +++ b/binding.cc @@ -36,6 +36,16 @@ static void iterator_end_do (napi_env env, Iterator* iterator, napi_value cb); #define NAPI_RETURN_UNDEFINED() \ return 0; +#define NAPI_UTF8_NEW(name, val) \ + size_t name##_size = 0; \ + NAPI_STATUS_THROWS(napi_get_value_string_utf8(env, val, NULL, 0, &name##_size)) \ + char* name = new char[name##_size + 1]; \ + NAPI_STATUS_THROWS(napi_get_value_string_utf8(env, val, name, name##_size + 1, &name##_size)) \ + name[name##_size] = '\0'; + +#define NAPI_ARGV_UTF8_NEW(name, i) \ + NAPI_UTF8_NEW(name, argv[i]) + #define LD_STRING_OR_BUFFER_TO_COPY(env, from, to) \ char* to##Ch_ = 0; \ size_t to##Sz_ = 0; \ @@ -689,7 +699,7 @@ struct OpenWorker : public BaseWorker { OpenWorker (napi_env env, Database* database, napi_value callback, - char* location, + const std::string& location, bool createIfMissing, bool errorIfExists, bool compression, @@ -714,16 +724,14 @@ struct OpenWorker : public BaseWorker { options_.max_file_size = maxFileSize; } - virtual ~OpenWorker () { - free(location_); - } + virtual ~OpenWorker () {} virtual void DoExecute () { - SetStatus(database_->Open(options_, location_)); + SetStatus(database_->Open(options_, location_.c_str())); } leveldb::Options options_; - char* location_; + std::string location_; }; /** @@ -732,9 +740,7 @@ struct OpenWorker : public BaseWorker { NAPI_METHOD(db_open) { NAPI_ARGV(4); NAPI_DB_CONTEXT(); - // TODO create a NAPI_ARGV_UTF8_NEW() macro that uses new instead of malloc - // so we have similar allocation/deallocation mechanisms everywhere - NAPI_ARGV_UTF8_MALLOC(location, 1); + NAPI_ARGV_UTF8_NEW(location, 1); napi_value options = argv[2]; bool createIfMissing = BooleanProperty(env, options, "createIfMissing", true); @@ -765,6 +771,8 @@ NAPI_METHOD(db_open) { blockRestartInterval, maxFileSize); worker->Queue(); + delete [] location; + NAPI_RETURN_UNDEFINED(); } @@ -1145,14 +1153,13 @@ struct DestroyWorker : public BaseWorker { */ NAPI_METHOD(destroy_db) { NAPI_ARGV(2); - // TODO replace with new - NAPI_ARGV_UTF8_MALLOC(location, 0); + NAPI_ARGV_UTF8_NEW(location, 0); napi_value callback = argv[1]; DestroyWorker* worker = new DestroyWorker(env, location, callback); worker->Queue(); - free(location); + delete [] location; NAPI_RETURN_UNDEFINED(); } @@ -1182,14 +1189,13 @@ struct RepairWorker : public BaseWorker { */ NAPI_METHOD(repair_db) { NAPI_ARGV(2); - // TODO replace with new - NAPI_ARGV_UTF8_MALLOC(location, 0); + NAPI_ARGV_UTF8_NEW(location, 0); napi_value callback = argv[1]; RepairWorker* worker = new RepairWorker(env, location, callback); worker->Queue(); - free(location); + delete [] location; NAPI_RETURN_UNDEFINED(); } @@ -1199,11 +1205,6 @@ NAPI_METHOD(repair_db) { */ static void FinalizeIterator (napi_env env, void* data, void* hint) { if (data) { - // TODO this might be incorrect, at the moment mimicing the behavior - // of Database. We might need to review the life cycle of Iterator - // and if it's garbage collected it might be enough to unhook itself - // from the Database (it has a pointer to it and could do this from - // its destructor). delete (Iterator*)data; } } From 41c19a24ed09ecf634989bc98efcaf2a24c8572a Mon Sep 17 00:00:00 2001 From: Lars-Magnus Skog Date: Sun, 9 Dec 2018 02:53:56 +0100 Subject: [PATCH 37/42] Refactor calling code into CallFunction() --- binding.cc | 86 ++++++++++++++++++------------------------------------ 1 file changed, 28 insertions(+), 58 deletions(-) diff --git a/binding.cc b/binding.cc index a488b647..88da9b5f 100644 --- a/binding.cc +++ b/binding.cc @@ -221,6 +221,18 @@ static size_t StringOrBufferLength (napi_env env, napi_value value) { return size; } +/** + * Calls a function. + */ +static napi_status CallFunction (napi_env env, + napi_value callback, + const int argc, + napi_value* argv) { + napi_value global; + napi_get_global(env, &global); + return napi_call_function(env, global, callback, argc, argv, NULL); +} + /** * Base worker class. Handles the async work. */ @@ -280,32 +292,18 @@ struct BaseWorker { return HandleOKCallback(); } - // TODO the global, callback, and calling the function code - // could be refactored with HandleOKCallback() - - napi_value global; - napi_get_global(env_, &global); + napi_value argv = CreateError(env_, errMsg_); napi_value callback; napi_get_reference_value(env_, callbackRef_, &callback); - - const int argc = 1; - napi_value argv[argc]; - argv[0] = CreateError(env_, errMsg_); - - napi_call_function(env_, global, callback, argc, argv, NULL); + CallFunction(env_, callback, 1, &argv); } virtual void HandleOKCallback () { - napi_value global; - napi_get_global(env_, &global); + napi_value argv; + napi_get_null(env_, &argv); napi_value callback; napi_get_reference_value(env_, callbackRef_, &callback); - - const int argc = 1; - napi_value argv[argc]; - napi_get_null(env_, &argv[0]); - - napi_call_function(env_, global, callback, argc, argv, NULL); + CallFunction(env_, callback, 1, &argv); } void Queue () { @@ -905,8 +903,7 @@ struct GetWorker : public BaseWorker { } virtual void HandleOKCallback() { - const int argc = 2; - napi_value argv[argc]; + napi_value argv[2]; napi_get_null(env_, &argv[0]); if (asBuffer_) { @@ -915,12 +912,9 @@ struct GetWorker : public BaseWorker { napi_create_string_utf8(env_, value_.data(), value_.size(), &argv[1]); } - // TODO move to base class - napi_value global; - napi_get_global(env_, &global); napi_value callback; napi_get_reference_value(env_, callbackRef_, &callback); - napi_call_function(env_, global, callback, argc, argv, NULL); + CallFunction(env_, callback, 2, argv); } leveldb::ReadOptions options_; @@ -1023,17 +1017,12 @@ struct ApproximateSizeWorker : public BaseWorker { } virtual void HandleOKCallback() { - const int argc = 2; - napi_value argv[argc]; + napi_value argv[2]; napi_get_null(env_, &argv[0]); napi_create_uint32(env_, (uint32_t)size_, &argv[1]); - - // TODO move to base class - napi_value global; - napi_get_global(env_, &global); napi_value callback; napi_get_reference_value(env_, callbackRef_, &callback); - napi_call_function(env_, global, callback, argc, argv, NULL); + CallFunction(env_, callback, 2, argv); } leveldb::Slice start_; @@ -1543,20 +1532,13 @@ struct NextWorker : public BaseWorker { // TODO this should just do iterator_->CheckEndCallback(); localCallback_(iterator_); - const int argc = 3; - napi_value argv[argc]; - + napi_value argv[3]; napi_get_null(env_, &argv[0]); argv[1] = jsArray; napi_get_boolean(env_, !ok_, &argv[2]); - - // TODO move to base class - napi_value global; - napi_get_global(env_, &global); napi_value callback; napi_get_reference_value(env_, callbackRef_, &callback); - - napi_call_function(env_, global, callback, argc, argv, NULL); + CallFunction(env_, callback, 3, argv); } Iterator* iterator_; @@ -1576,14 +1558,8 @@ NAPI_METHOD(iterator_next) { napi_value callback = argv[1]; if (iterator->ended_) { - // TODO refactor with other callback code - napi_value global; - napi_get_global(env, &global); - - const int argc = 1; - napi_value argv[argc]; - argv[0] = CreateError(env, "iterator has ended"); - napi_call_function(env, global, callback, argc, argv, NULL); + napi_value argv = CreateError(env, "iterator has ended"); + CallFunction(env, callback, 1, &argv); NAPI_RETURN_UNDEFINED(); } @@ -1677,15 +1653,9 @@ NAPI_METHOD(batch_do) { worker->Queue(); } else { delete batch; - // TODO refactor with other callback code - napi_value global; - napi_get_global(env, &global); - - const int argc = 1; - napi_value argv[argc]; - napi_get_null(env, &argv[0]); - - napi_call_function(env, global, callback, argc, argv, NULL); + napi_value argv; + napi_get_null(env, &argv); + CallFunction(env, callback, 1, &argv); } NAPI_RETURN_UNDEFINED(); From 4bcffea3c8edef1b4af6755076e9b241712abc38 Mon Sep 17 00:00:00 2001 From: Lars-Magnus Skog Date: Sun, 9 Dec 2018 03:09:13 +0100 Subject: [PATCH 38/42] Refactor some of the hideous iterator property code --- binding.cc | 159 ++++++++++++++++++++++------------------------------- 1 file changed, 67 insertions(+), 92 deletions(-) diff --git a/binding.cc b/binding.cc index 88da9b5f..2310c463 100644 --- a/binding.cc +++ b/binding.cc @@ -1198,6 +1198,17 @@ static void FinalizeIterator (napi_env env, void* data, void* hint) { } } +#define CHECK_PROPERTY(name, code) \ + if (HasProperty(env, options, #name)) { \ + napi_value value = GetProperty(env, options, #name); \ + if (IsString(env, value) || IsBuffer(env, value)) { \ + if (StringOrBufferLength(env, value) > 0) { \ + LD_STRING_OR_BUFFER_TO_COPY(env, value, _##name); \ + code; \ + } \ + } \ + } \ + /** * Create an iterator. */ @@ -1220,116 +1231,80 @@ NAPI_METHOD(iterator_init) { leveldb::Slice* start = NULL; char *startStr = NULL; - if (HasProperty(env, options, "start")) { - napi_value value = GetProperty(env, options, "start"); - if (IsString(env, value) || IsBuffer(env, value)) { - if (StringOrBufferLength(env, value) > 0) { - LD_STRING_OR_BUFFER_TO_COPY(env, value, _start); - start = new leveldb::Slice(_startCh_, _startSz_); - startStr = _startCh_; - } - } - } + CHECK_PROPERTY(start, { + start = new leveldb::Slice(_startCh_, _startSz_); + startStr = _startCh_; + }); std::string* end = NULL; - if (HasProperty(env, options, "end")) { - napi_value value = GetProperty(env, options, "end"); - if (IsString(env, value) || IsBuffer(env, value)) { - if (StringOrBufferLength(env, value) > 0) { - LD_STRING_OR_BUFFER_TO_COPY(env, value, _end); - end = new std::string(_endCh_, _endSz_); - delete [] _endCh_; - } - } - } + CHECK_PROPERTY(end, { + end = new std::string(_endCh_, _endSz_); + delete [] _endCh_; + }); std::string* lt = NULL; - if (HasProperty(env, options, "lt")) { - napi_value value = GetProperty(env, options, "lt"); - if (IsString(env, value) || IsBuffer(env, value)) { - if (StringOrBufferLength(env, value) > 0) { - LD_STRING_OR_BUFFER_TO_COPY(env, value, _lt); - lt = new std::string(_ltCh_, _ltSz_); - delete [] _ltCh_; - if (reverse) { - if (startStr != NULL) { - delete [] startStr; - startStr = NULL; - } - if (start != NULL) { - delete start; - } - start = new leveldb::Slice(lt->data(), lt->size()); - } + CHECK_PROPERTY(lt, { + lt = new std::string(_ltCh_, _ltSz_); + delete [] _ltCh_; + if (reverse) { + if (startStr != NULL) { + delete [] startStr; + startStr = NULL; + } + if (start != NULL) { + delete start; } + start = new leveldb::Slice(lt->data(), lt->size()); } - } + }); std::string* lte = NULL; - if (HasProperty(env, options, "lte")) { - napi_value value = GetProperty(env, options, "lte"); - if (IsString(env, value) || IsBuffer(env, value)) { - if (StringOrBufferLength(env, value) > 0) { - LD_STRING_OR_BUFFER_TO_COPY(env, value, _lte); - lte = new std::string(_lteCh_, _lteSz_); - delete [] _lteCh_; - if (reverse) { - if (startStr != NULL) { - delete [] startStr; - startStr = NULL; - } - if (start != NULL) { - delete start; - } - start = new leveldb::Slice(lte->data(), lte->size()); - } + CHECK_PROPERTY(lte, { + lte = new std::string(_lteCh_, _lteSz_); + delete [] _lteCh_; + if (reverse) { + if (startStr != NULL) { + delete [] startStr; + startStr = NULL; + } + if (start != NULL) { + delete start; } + start = new leveldb::Slice(lte->data(), lte->size()); } - } + }); std::string* gt = NULL; - if (HasProperty(env, options, "gt")) { - napi_value value = GetProperty(env, options, "gt"); - if (IsString(env, value) || IsBuffer(env, value)) { - if (StringOrBufferLength(env, value) > 0) { - LD_STRING_OR_BUFFER_TO_COPY(env, value, _gt); - gt = new std::string(_gtCh_, _gtSz_); - delete [] _gtCh_; - if (!reverse) { - if (startStr != NULL) { - delete [] startStr; - startStr = NULL; - } - if (start != NULL) { - delete start; - } - start = new leveldb::Slice(gt->data(), gt->size()); - } + CHECK_PROPERTY(gt, { + gt = new std::string(_gtCh_, _gtSz_); + delete [] _gtCh_; + if (!reverse) { + if (startStr != NULL) { + delete [] startStr; + startStr = NULL; + } + if (start != NULL) { + delete start; } + start = new leveldb::Slice(gt->data(), gt->size()); } - } + }); std::string* gte = NULL; - if (HasProperty(env, options, "gte")) { - napi_value value = GetProperty(env, options, "gte"); - if (IsString(env, value) || IsBuffer(env, value)) { - if (StringOrBufferLength(env, value) > 0) { - LD_STRING_OR_BUFFER_TO_COPY(env, value, _gte); - gte = new std::string(_gteCh_, _gteSz_); - delete [] _gteCh_; - if (!reverse) { - if (startStr != NULL) { - delete [] startStr; - startStr = NULL; - } - if (start != NULL) { - delete start; - } - start = new leveldb::Slice(gte->data(), gte->size()); - } + CHECK_PROPERTY(gte, { + gte = new std::string(_gteCh_, _gteSz_); + delete [] _gteCh_; + if (!reverse) { + if (startStr != NULL) { + delete [] startStr; + startStr = NULL; + } + if (start != NULL) { + delete start; } + start = new leveldb::Slice(gte->data(), gte->size()); } - } + }); uint32_t id = database->currentIteratorId_++; Iterator* iterator = new Iterator(database, From 647c3dd77f7e7bc6b381051014ce252eca412726 Mon Sep 17 00:00:00 2001 From: Lars-Magnus Skog Date: Sun, 9 Dec 2018 03:13:05 +0100 Subject: [PATCH 39/42] Compress constructor arguments a little bit --- binding.cc | 70 ++++++++++++------------------------------------------ 1 file changed, 15 insertions(+), 55 deletions(-) diff --git a/binding.cc b/binding.cc index 2310c463..3657ba33 100644 --- a/binding.cc +++ b/binding.cc @@ -756,17 +756,10 @@ NAPI_METHOD(db_open) { database->blockCache_ = leveldb::NewLRUCache(cacheSize); napi_value callback = argv[3]; - OpenWorker* worker = new OpenWorker(env, - database, - callback, - location, - createIfMissing, - errorIfExists, - compression, - writeBufferSize, - blockSize, - maxOpenFiles, - blockRestartInterval, + OpenWorker* worker = new OpenWorker(env, database, callback, location, + createIfMissing, errorIfExists, + compression, writeBufferSize, blockSize, + maxOpenFiles, blockRestartInterval, maxFileSize); worker->Queue(); delete [] location; @@ -867,12 +860,7 @@ NAPI_METHOD(db_put) { bool sync = BooleanProperty(env, argv[3], "sync", false); napi_value callback = argv[4]; - PutWorker* worker = new PutWorker(env, - database, - callback, - key, - value, - sync); + PutWorker* worker = new PutWorker(env, database, callback, key, value, sync); worker->Queue(); NAPI_RETURN_UNDEFINED(); @@ -936,11 +924,7 @@ NAPI_METHOD(db_get) { bool fillCache = BooleanProperty(env, options, "fillCache", true); napi_value callback = argv[3]; - GetWorker* worker = new GetWorker(env, - database, - callback, - key, - asBuffer, + GetWorker* worker = new GetWorker(env, database, callback, key, asBuffer, fillCache); worker->Queue(); @@ -984,11 +968,7 @@ NAPI_METHOD(db_del) { bool sync = BooleanProperty(env, argv[2], "sync", false); napi_value callback = argv[3]; - DelWorker* worker = new DelWorker(env, - database, - callback, - key, - sync); + DelWorker* worker = new DelWorker(env, database, callback, key, sync); worker->Queue(); NAPI_RETURN_UNDEFINED(); @@ -1041,10 +1021,8 @@ NAPI_METHOD(db_approximate_size) { leveldb::Slice end = ToSlice(env, argv[2]); napi_value callback = argv[3]; - ApproximateSizeWorker* worker = new ApproximateSizeWorker(env, - database, - callback, - start, + ApproximateSizeWorker* worker = new ApproximateSizeWorker(env, database, + callback, start, end); worker->Queue(); @@ -1087,11 +1065,8 @@ NAPI_METHOD(db_compact_range) { leveldb::Slice end = ToSlice(env, argv[2]); napi_value callback = argv[3]; - CompactRangeWorker* worker = new CompactRangeWorker(env, - database, - callback, - start, - end); + CompactRangeWorker* worker = new CompactRangeWorker(env, database, callback, + start, end); worker->Queue(); NAPI_RETURN_UNDEFINED(); @@ -1307,22 +1282,9 @@ NAPI_METHOD(iterator_init) { }); uint32_t id = database->currentIteratorId_++; - Iterator* iterator = new Iterator(database, - id, - start, - end, - reverse, - keys, - values, - limit, - lt, - lte, - gt, - gte, - fillCache, - keyAsBuffer, - valueAsBuffer, - highWaterMark); + Iterator* iterator = new Iterator(database, id, start, end, reverse, keys, + values, limit, lt, lte, gt, gte, fillCache, + keyAsBuffer, valueAsBuffer, highWaterMark); database->iterators_[id] = iterator; napi_value result; @@ -1539,9 +1501,7 @@ NAPI_METHOD(iterator_next) { NAPI_RETURN_UNDEFINED(); } - NextWorker* worker = new NextWorker(env, - iterator, - callback, + NextWorker* worker = new NextWorker(env, iterator, callback, CheckEndCallback); iterator->nexting_ = true; worker->Queue(); From 42da176853d4ceef2559ea9fea5dba10aa5dc9a1 Mon Sep 17 00:00:00 2001 From: Lars-Magnus Skog Date: Sun, 9 Dec 2018 03:17:00 +0100 Subject: [PATCH 40/42] Remove nan/v8 based glue code --- src/async.h | 32 --- src/batch.cc | 148 ---------- src/batch.h | 40 --- src/batch_async.cc | 17 -- src/batch_async.h | 26 -- src/common.h | 33 --- src/database.cc | 514 ---------------------------------- src/database.h | 108 -------- src/database_async.cc | 290 ------------------- src/database_async.h | 168 ----------- src/iterator.cc | 613 ----------------------------------------- src/iterator.h | 94 ------- src/iterator_async.cc | 86 ------ src/iterator_async.h | 43 --- src/leveldown.cc | 70 ----- src/leveldown.h | 117 -------- src/leveldown_async.cc | 38 --- src/leveldown_async.h | 34 --- 18 files changed, 2471 deletions(-) delete mode 100644 src/async.h delete mode 100644 src/batch.cc delete mode 100644 src/batch.h delete mode 100644 src/batch_async.cc delete mode 100644 src/batch_async.h delete mode 100644 src/common.h delete mode 100644 src/database.cc delete mode 100644 src/database.h delete mode 100644 src/database_async.cc delete mode 100644 src/database_async.h delete mode 100644 src/iterator.cc delete mode 100644 src/iterator.h delete mode 100644 src/iterator_async.cc delete mode 100644 src/iterator_async.h delete mode 100644 src/leveldown.cc delete mode 100644 src/leveldown.h delete mode 100644 src/leveldown_async.cc delete mode 100644 src/leveldown_async.h diff --git a/src/async.h b/src/async.h deleted file mode 100644 index 72fc7278..00000000 --- a/src/async.h +++ /dev/null @@ -1,32 +0,0 @@ -#ifndef LD_ASYNC_H -#define LD_ASYNC_H - -#include -#include -#include "database.h" - -namespace leveldown { - -class Database; - -/* abstract */ class AsyncWorker : public Nan::AsyncWorker { -public: - AsyncWorker(leveldown::Database* database, - Nan::Callback *callback, - const char *resource_name) - : Nan::AsyncWorker(callback, resource_name), database(database) {} - -protected: - void SetStatus(leveldb::Status status) { - this->status = status; - if (!status.ok()) - SetErrorMessage(status.ToString().c_str()); - } - Database* database; -private: - leveldb::Status status; -}; - -} // namespace leveldown - -#endif diff --git a/src/batch.cc b/src/batch.cc deleted file mode 100644 index a66565df..00000000 --- a/src/batch.cc +++ /dev/null @@ -1,148 +0,0 @@ -#include -#include -#include - -#include "database.h" -#include "batch_async.h" -#include "batch.h" -#include "common.h" - -namespace leveldown { - -static Nan::Persistent batch_constructor; - -Batch::Batch (leveldown::Database* database, bool sync) : database(database) { - options = new leveldb::WriteOptions(); - options->sync = sync; - batch = new leveldb::WriteBatch(); - hasData = false; -} - -Batch::~Batch () { - delete options; - delete batch; -} - -leveldb::Status Batch::Write () { - return database->WriteBatchToDatabase(options, batch); -} - -void Batch::Init () { - v8::Local tpl = Nan::New(Batch::New); - batch_constructor.Reset(tpl); - tpl->SetClassName(Nan::New("Batch").ToLocalChecked()); - tpl->InstanceTemplate()->SetInternalFieldCount(1); - Nan::SetPrototypeMethod(tpl, "put", Batch::Put); - Nan::SetPrototypeMethod(tpl, "del", Batch::Del); - Nan::SetPrototypeMethod(tpl, "clear", Batch::Clear); - Nan::SetPrototypeMethod(tpl, "write", Batch::Write); -} - -NAN_METHOD(Batch::New) { - Database* database = Nan::ObjectWrap::Unwrap(info[0]->ToObject()); - v8::Local optionsObj; - - if (info.Length() > 1 && info[1]->IsObject()) { - optionsObj = v8::Local::Cast(info[1]); - } - - bool sync = BooleanOptionValue(optionsObj, "sync"); - - Batch* batch = new Batch(database, sync); - batch->Wrap(info.This()); - - info.GetReturnValue().Set(info.This()); -} - -v8::Local Batch::NewInstance ( - v8::Local database - , v8::Local optionsObj - ) { - - Nan::EscapableHandleScope scope; - - Nan::MaybeLocal maybeInstance; - v8::Local instance; - - v8::Local constructorHandle = - Nan::New(batch_constructor); - - if (optionsObj.IsEmpty()) { - v8::Local argv[1] = { database }; - maybeInstance = Nan::NewInstance(constructorHandle->GetFunction(), 1, argv); - } else { - v8::Local argv[2] = { database, optionsObj }; - maybeInstance = Nan::NewInstance(constructorHandle->GetFunction(), 2, argv); - } - - if (maybeInstance.IsEmpty()) - Nan::ThrowError("Could not create new Batch instance"); - else - instance = maybeInstance.ToLocalChecked(); - return scope.Escape(instance); -} - -NAN_METHOD(Batch::Put) { - Batch* batch = ObjectWrap::Unwrap(info.Holder()); - v8::Local callback; // purely for the error macros - - v8::Local keyBuffer = info[0]; - v8::Local valueBuffer = info[1]; - LD_STRING_OR_BUFFER_TO_SLICE(key, keyBuffer, key) - LD_STRING_OR_BUFFER_TO_SLICE(value, valueBuffer, value) - - batch->batch->Put(key, value); - if (!batch->hasData) - batch->hasData = true; - - DisposeStringOrBufferFromSlice(keyBuffer, key); - DisposeStringOrBufferFromSlice(valueBuffer, value); - - info.GetReturnValue().Set(info.Holder()); -} - -NAN_METHOD(Batch::Del) { - Batch* batch = ObjectWrap::Unwrap(info.Holder()); - - v8::Local callback; // purely for the error macros - - v8::Local keyBuffer = info[0]; - LD_STRING_OR_BUFFER_TO_SLICE(key, keyBuffer, key) - - batch->batch->Delete(key); - if (!batch->hasData) - batch->hasData = true; - - DisposeStringOrBufferFromSlice(keyBuffer, key); - - info.GetReturnValue().Set(info.Holder()); -} - -NAN_METHOD(Batch::Clear) { - Batch* batch = ObjectWrap::Unwrap(info.Holder()); - - batch->batch->Clear(); - batch->hasData = false; - - info.GetReturnValue().Set(info.Holder()); -} - -NAN_METHOD(Batch::Write) { - Batch* batch = ObjectWrap::Unwrap(info.Holder()); - - if (batch->hasData) { - Nan::Callback *callback = - new Nan::Callback(v8::Local::Cast(info[0])); - BatchWriteWorker* worker = new BatchWriteWorker(batch, callback); - // persist to prevent accidental GC - v8::Local _this = info.This(); - worker->SaveToPersistent("batch", _this); - Nan::AsyncQueueWorker(worker); - } else { - LD_RUN_CALLBACK("leveldown:batch.write", - v8::Local::Cast(info[0]), - 0, NULL); - } -} - -} // namespace leveldown diff --git a/src/batch.h b/src/batch.h deleted file mode 100644 index 39fb0685..00000000 --- a/src/batch.h +++ /dev/null @@ -1,40 +0,0 @@ -#ifndef LD_BATCH_H -#define LD_BATCH_H - -#include -#include - -#include - -#include "database.h" - -namespace leveldown { - -class Batch : public Nan::ObjectWrap { -public: - static void Init(); - static v8::Local NewInstance ( - v8::Local database - , v8::Local optionsObj - ); - - Batch (leveldown::Database* database, bool sync); - ~Batch (); - leveldb::Status Write (); - -private: - leveldown::Database* database; - leveldb::WriteOptions* options; - leveldb::WriteBatch* batch; - bool hasData; // keep track of whether we're writing data or not - - static NAN_METHOD(New); - static NAN_METHOD(Put); - static NAN_METHOD(Del); - static NAN_METHOD(Clear); - static NAN_METHOD(Write); -}; - -} // namespace leveldown - -#endif diff --git a/src/batch_async.cc b/src/batch_async.cc deleted file mode 100644 index 63893e11..00000000 --- a/src/batch_async.cc +++ /dev/null @@ -1,17 +0,0 @@ -#include -#include "batch.h" -#include "batch_async.h" - -namespace leveldown { - -/** NEXT WORKER **/ - -BatchWriteWorker::BatchWriteWorker(Batch* batch, Nan::Callback *callback) - : AsyncWorker(NULL, callback, "leveldown:batch.write"), batch(batch) -{} - -void BatchWriteWorker::Execute() { - SetStatus(batch->Write()); -} - -} // namespace leveldown diff --git a/src/batch_async.h b/src/batch_async.h deleted file mode 100644 index 623be74c..00000000 --- a/src/batch_async.h +++ /dev/null @@ -1,26 +0,0 @@ -#ifndef LD_BATCH_ASYNC_H -#define LD_BATCH_ASYNC_H - -#include -#include - -#include "async.h" -#include "batch.h" -#include "database.h" - -namespace leveldown { - -class BatchWriteWorker : public AsyncWorker { -public: - BatchWriteWorker(Batch* batch, Nan::Callback *callback); - - virtual ~BatchWriteWorker() {} - virtual void Execute(); - -private: - Batch* batch; -}; - -} // namespace leveldown - -#endif diff --git a/src/common.h b/src/common.h deleted file mode 100644 index c86a79b4..00000000 --- a/src/common.h +++ /dev/null @@ -1,33 +0,0 @@ -#ifndef LD_COMMON_H -#define LD_COMMON_H - -#include - -namespace leveldown { - -NAN_INLINE bool BooleanOptionValue(v8::Local options, - const char* _key, - bool def = false) { - Nan::HandleScope scope; - v8::Local key = Nan::New(_key).ToLocalChecked(); - return !options.IsEmpty() - && options->Has(key) - ? options->Get(key)->BooleanValue() - : def; -} - -NAN_INLINE uint32_t UInt32OptionValue(v8::Local options, - const char* _key, - uint32_t def) { - Nan::HandleScope scope; - v8::Local key = Nan::New(_key).ToLocalChecked(); - return !options.IsEmpty() - && options->Has(key) - && options->Get(key)->IsNumber() - ? options->Get(key)->Uint32Value() - : def; -} - -} // namespace leveldown - -#endif diff --git a/src/database.cc b/src/database.cc deleted file mode 100644 index a3c30f33..00000000 --- a/src/database.cc +++ /dev/null @@ -1,514 +0,0 @@ -#include -#include - -#include -#include - -#include "leveldown.h" -#include "database.h" -#include "async.h" -#include "database_async.h" -#include "batch.h" -#include "iterator.h" -#include "common.h" - -namespace leveldown { - -static Nan::Persistent database_constructor; - -Database::Database (const v8::Local& from) - : location(new Nan::Utf8String(from)) - , db(NULL) - , currentIteratorId(0) - , pendingCloseWorker(NULL) - , blockCache(NULL) - , filterPolicy(NULL) {}; - -Database::~Database () { - if (db != NULL) - delete db; - delete location; -}; - -/* Calls from worker threads, NO V8 HERE *****************************/ - -leveldb::Status Database::OpenDatabase ( - leveldb::Options* options - ) { - return leveldb::DB::Open(*options, **location, &db); -} - -leveldb::Status Database::PutToDatabase ( - leveldb::WriteOptions* options - , leveldb::Slice key - , leveldb::Slice value - ) { - return db->Put(*options, key, value); -} - -leveldb::Status Database::GetFromDatabase ( - leveldb::ReadOptions* options - , leveldb::Slice key - , std::string& value - ) { - return db->Get(*options, key, &value); -} - -leveldb::Status Database::DeleteFromDatabase ( - leveldb::WriteOptions* options - , leveldb::Slice key - ) { - return db->Delete(*options, key); -} - -leveldb::Status Database::WriteBatchToDatabase ( - leveldb::WriteOptions* options - , leveldb::WriteBatch* batch - ) { - return db->Write(*options, batch); -} - -uint64_t Database::ApproximateSizeFromDatabase (const leveldb::Range* range) { - uint64_t size; - db->GetApproximateSizes(range, 1, &size); - return size; -} - -void Database::CompactRangeFromDatabase (const leveldb::Slice* start, - const leveldb::Slice* end) { - db->CompactRange(start, end); -} - -void Database::GetPropertyFromDatabase ( - const leveldb::Slice& property - , std::string* value) { - - db->GetProperty(property, value); -} - -leveldb::Iterator* Database::NewIterator (leveldb::ReadOptions* options) { - return db->NewIterator(*options); -} - -const leveldb::Snapshot* Database::NewSnapshot () { - return db->GetSnapshot(); -} - -void Database::ReleaseSnapshot (const leveldb::Snapshot* snapshot) { - return db->ReleaseSnapshot(snapshot); -} - -void Database::ReleaseIterator (uint32_t id) { - // called each time an Iterator is End()ed, in the main thread - // we have to remove our reference to it and if it's the last iterator - // we have to invoke a pending CloseWorker if there is one - // if there is a pending CloseWorker it means that we're waiting for - // iterators to end before we can close them - iterators.erase(id); - if (iterators.empty() && pendingCloseWorker != NULL) { - Nan::AsyncQueueWorker((AsyncWorker*)pendingCloseWorker); - pendingCloseWorker = NULL; - } -} - -void Database::CloseDatabase () { - delete db; - db = NULL; - if (blockCache) { - delete blockCache; - blockCache = NULL; - } - if (filterPolicy) { - delete filterPolicy; - filterPolicy = NULL; - } -} - -/* V8 exposed functions *****************************/ - -NAN_METHOD(LevelDOWN) { - v8::Local location = info[0].As(); - info.GetReturnValue().Set(Database::NewInstance(location)); -} - -void Database::Init () { - v8::Local tpl = Nan::New(Database::New); - database_constructor.Reset(tpl); - tpl->SetClassName(Nan::New("Database").ToLocalChecked()); - tpl->InstanceTemplate()->SetInternalFieldCount(1); - Nan::SetPrototypeMethod(tpl, "open", Database::Open); - Nan::SetPrototypeMethod(tpl, "close", Database::Close); - Nan::SetPrototypeMethod(tpl, "put", Database::Put); - Nan::SetPrototypeMethod(tpl, "get", Database::Get); - Nan::SetPrototypeMethod(tpl, "del", Database::Delete); - Nan::SetPrototypeMethod(tpl, "batch", Database::Batch); - Nan::SetPrototypeMethod(tpl, "approximateSize", Database::ApproximateSize); - Nan::SetPrototypeMethod(tpl, "compactRange", Database::CompactRange); - Nan::SetPrototypeMethod(tpl, "getProperty", Database::GetProperty); - Nan::SetPrototypeMethod(tpl, "iterator", Database::Iterator); -} - -NAN_METHOD(Database::New) { - Database* obj = new Database(info[0]); - obj->Wrap(info.This()); - - info.GetReturnValue().Set(info.This()); -} - -v8::Local Database::NewInstance (v8::Local &location) { - Nan::EscapableHandleScope scope; - - Nan::MaybeLocal maybeInstance; - v8::Local instance; - - v8::Local constructorHandle = - Nan::New(database_constructor); - - v8::Local argv[] = { location }; - maybeInstance = Nan::NewInstance(constructorHandle->GetFunction(), 1, argv); - - if (maybeInstance.IsEmpty()) - Nan::ThrowError("Could not create new Database instance"); - else - instance = maybeInstance.ToLocalChecked(); - return scope.Escape(instance); -} - -NAN_METHOD(Database::Open) { - LD_METHOD_SETUP_COMMON(open, 0, 1) - - bool createIfMissing = BooleanOptionValue(optionsObj, "createIfMissing", true); - bool errorIfExists = BooleanOptionValue(optionsObj, "errorIfExists"); - bool compression = BooleanOptionValue(optionsObj, "compression", true); - - uint32_t cacheSize = UInt32OptionValue(optionsObj, "cacheSize", 8 << 20); - uint32_t writeBufferSize = UInt32OptionValue( - optionsObj - , "writeBufferSize" - , 4 << 20 - ); - uint32_t blockSize = UInt32OptionValue(optionsObj, "blockSize", 4096); - uint32_t maxOpenFiles = UInt32OptionValue(optionsObj, "maxOpenFiles", 1000); - uint32_t blockRestartInterval = UInt32OptionValue( - optionsObj - , "blockRestartInterval" - , 16 - ); - uint32_t maxFileSize = UInt32OptionValue(optionsObj, "maxFileSize", 2 << 20); - - database->blockCache = leveldb::NewLRUCache(cacheSize); - database->filterPolicy = leveldb::NewBloomFilterPolicy(10); - - OpenWorker* worker = new OpenWorker( - database - , new Nan::Callback(callback) - , database->blockCache - , database->filterPolicy - , createIfMissing - , errorIfExists - , compression - , writeBufferSize - , blockSize - , maxOpenFiles - , blockRestartInterval - , maxFileSize - ); - // persist to prevent accidental GC - v8::Local _this = info.This(); - worker->SaveToPersistent("database", _this); - Nan::AsyncQueueWorker(worker); -} - -// for an empty callback to iterator.end() -NAN_METHOD(EmptyMethod) { -} - -NAN_METHOD(Database::Close) { - LD_METHOD_SETUP_COMMON_ONEARG(close) - - CloseWorker* worker = new CloseWorker( - database - , new Nan::Callback(callback) - ); - // persist to prevent accidental GC - v8::Local _this = info.This(); - worker->SaveToPersistent("database", _this); - - if (!database->iterators.empty()) { - // yikes, we still have iterators open! naughty naughty. - // we have to queue up a CloseWorker and manually close each of them. - // the CloseWorker will be invoked once they are all cleaned up - database->pendingCloseWorker = worker; - - for ( - std::map< uint32_t, leveldown::Iterator * >::iterator it - = database->iterators.begin() - ; it != database->iterators.end() - ; ++it) { - - // for each iterator still open, first check if it's already in - // the process of ending (ended==true means an async End() is - // in progress), if not, then we call End() with an empty callback - // function and wait for it to hit ReleaseIterator() where our - // CloseWorker will be invoked - - leveldown::Iterator *iterator = it->second; - - if (!iterator->ended) { - v8::Local end = - v8::Local::Cast(iterator->handle()->Get( - Nan::New("end").ToLocalChecked())); - v8::Local argv[] = { - Nan::New(EmptyMethod)->GetFunction() // empty callback - }; - Nan::AsyncResource ar("leveldown:iterator.end"); - ar.runInAsyncScope(iterator->handle(), end, 1, argv); - } - } - } else { - Nan::AsyncQueueWorker(worker); - } -} - -NAN_METHOD(Database::Put) { - LD_METHOD_SETUP_COMMON(put, 2, 3) - - v8::Local keyHandle = info[0].As(); - v8::Local valueHandle = info[1].As(); - LD_STRING_OR_BUFFER_TO_SLICE(key, keyHandle, key); - LD_STRING_OR_BUFFER_TO_SLICE(value, valueHandle, value); - - bool sync = BooleanOptionValue(optionsObj, "sync"); - - WriteWorker* worker = new WriteWorker( - database - , new Nan::Callback(callback) - , key - , value - , sync - , keyHandle - , valueHandle - ); - - // persist to prevent accidental GC - v8::Local _this = info.This(); - worker->SaveToPersistent("database", _this); - Nan::AsyncQueueWorker(worker); -} - -NAN_METHOD(Database::Get) { - LD_METHOD_SETUP_COMMON(get, 1, 2) - - v8::Local keyHandle = info[0].As(); - LD_STRING_OR_BUFFER_TO_SLICE(key, keyHandle, key); - - bool asBuffer = BooleanOptionValue(optionsObj, "asBuffer", true); - bool fillCache = BooleanOptionValue(optionsObj, "fillCache", true); - - ReadWorker* worker = new ReadWorker( - database - , new Nan::Callback(callback) - , key - , asBuffer - , fillCache - , keyHandle - ); - // persist to prevent accidental GC - v8::Local _this = info.This(); - worker->SaveToPersistent("database", _this); - Nan::AsyncQueueWorker(worker); -} - -NAN_METHOD(Database::Delete) { - LD_METHOD_SETUP_COMMON(del, 1, 2) - - v8::Local keyHandle = info[0].As(); - LD_STRING_OR_BUFFER_TO_SLICE(key, keyHandle, key); - - bool sync = BooleanOptionValue(optionsObj, "sync"); - - DeleteWorker* worker = new DeleteWorker( - database - , new Nan::Callback(callback) - , key - , sync - , keyHandle - ); - // persist to prevent accidental GC - v8::Local _this = info.This(); - worker->SaveToPersistent("database", _this); - Nan::AsyncQueueWorker(worker); -} - -NAN_METHOD(Database::Batch) { - if ((info.Length() == 0 || info.Length() == 1) && !info[0]->IsArray()) { - v8::Local optionsObj; - if (info.Length() > 0 && info[0]->IsObject()) { - optionsObj = info[0].As(); - } - info.GetReturnValue().Set(Batch::NewInstance(info.This(), optionsObj)); - return; - } - - LD_METHOD_SETUP_COMMON(batch, 1, 2); - - bool sync = BooleanOptionValue(optionsObj, "sync"); - - v8::Local array = v8::Local::Cast(info[0]); - - leveldb::WriteBatch* batch = new leveldb::WriteBatch(); - bool hasData = false; - - for (unsigned int i = 0; i < array->Length(); i++) { - if (!array->Get(i)->IsObject()) - continue; - - v8::Local obj = v8::Local::Cast(array->Get(i)); - v8::Local keyBuffer = obj->Get(Nan::New("key").ToLocalChecked()); - v8::Local type = obj->Get(Nan::New("type").ToLocalChecked()); - - if (type->StrictEquals(Nan::New("del").ToLocalChecked())) { - LD_STRING_OR_BUFFER_TO_SLICE(key, keyBuffer, key) - - batch->Delete(key); - if (!hasData) - hasData = true; - - DisposeStringOrBufferFromSlice(keyBuffer, key); - } else if (type->StrictEquals(Nan::New("put").ToLocalChecked())) { - v8::Local valueBuffer = obj->Get(Nan::New("value").ToLocalChecked()); - - LD_STRING_OR_BUFFER_TO_SLICE(key, keyBuffer, key) - LD_STRING_OR_BUFFER_TO_SLICE(value, valueBuffer, value) - batch->Put(key, value); - if (!hasData) - hasData = true; - - DisposeStringOrBufferFromSlice(keyBuffer, key); - DisposeStringOrBufferFromSlice(valueBuffer, value); - } - } - - // don't allow an empty batch through - if (hasData) { - BatchWorker* worker = new BatchWorker( - database - , new Nan::Callback(callback) - , batch - , sync - ); - // persist to prevent accidental GC - v8::Local _this = info.This(); - worker->SaveToPersistent("database", _this); - Nan::AsyncQueueWorker(worker); - } else { - LD_RUN_CALLBACK("leveldown:db.batch", callback, 0, NULL); - } -} - -NAN_METHOD(Database::ApproximateSize) { - v8::Local startHandle = info[0].As(); - v8::Local endHandle = info[1].As(); - - LD_METHOD_SETUP_COMMON(approximateSize, -1, 2) - - LD_STRING_OR_BUFFER_TO_SLICE(start, startHandle, start) - LD_STRING_OR_BUFFER_TO_SLICE(end, endHandle, end) - - ApproximateSizeWorker* worker = new ApproximateSizeWorker( - database - , new Nan::Callback(callback) - , start - , end - , startHandle - , endHandle - ); - // persist to prevent accidental GC - v8::Local _this = info.This(); - worker->SaveToPersistent("database", _this); - Nan::AsyncQueueWorker(worker); -} - -NAN_METHOD(Database::CompactRange) { - v8::Local startHandle = info[0].As(); - v8::Local endHandle = info[1].As(); - - LD_METHOD_SETUP_COMMON(compactRange, -1, 2) - LD_STRING_OR_BUFFER_TO_SLICE(start, startHandle, start) - LD_STRING_OR_BUFFER_TO_SLICE(end, endHandle, end) - - CompactRangeWorker* worker = new CompactRangeWorker( - database - , new Nan::Callback(callback) - , start - , end - , startHandle - , endHandle - ); - // persist to prevent accidental GC - v8::Local _this = info.This(); - worker->SaveToPersistent("database", _this); - Nan::AsyncQueueWorker(worker); -} - -NAN_METHOD(Database::GetProperty) { - v8::Local propertyHandle = info[0].As(); - v8::Local callback; // for LD_STRING_OR_BUFFER_TO_SLICE - - LD_STRING_OR_BUFFER_TO_SLICE(property, propertyHandle, property) - - leveldown::Database* database = - Nan::ObjectWrap::Unwrap(info.This()); - - std::string* value = new std::string(); - database->GetPropertyFromDatabase(property, value); - v8::Local returnValue - = Nan::New(value->c_str(), value->length()).ToLocalChecked(); - delete value; - delete[] property.data(); - - info.GetReturnValue().Set(returnValue); -} - -NAN_METHOD(Database::Iterator) { - Database* database = Nan::ObjectWrap::Unwrap(info.This()); - - v8::Local optionsObj; - if (info.Length() > 0 && info[0]->IsObject()) { - optionsObj = v8::Local::Cast(info[0]); - } - - // each iterator gets a unique id for this Database, so we can - // easily store & lookup on our `iterators` map - uint32_t id = database->currentIteratorId++; - Nan::TryCatch try_catch; - v8::Local iteratorHandle = Iterator::NewInstance( - info.This() - , Nan::New(id) - , optionsObj - ); - if (try_catch.HasCaught()) { - // NB: node::FatalException can segfault here if there is no room on stack. - return Nan::ThrowError("Fatal Error in Database::Iterator!"); - } - - leveldown::Iterator *iterator = - Nan::ObjectWrap::Unwrap(iteratorHandle); - - database->iterators[id] = iterator; - - // register our iterator - /* - v8::Local obj = Nan::New(); - obj->Set(Nan::New("iterator"), iteratorHandle); - Nan::Persistent persistent; - persistent.Reset(nan_isolate, obj); - database->iterators.insert(std::pair< uint32_t, Nan::Persistent & > - (id, persistent)); - */ - - info.GetReturnValue().Set(iteratorHandle); -} - - -} // namespace leveldown diff --git a/src/database.h b/src/database.h deleted file mode 100644 index 6bbc05db..00000000 --- a/src/database.h +++ /dev/null @@ -1,108 +0,0 @@ -#ifndef LD_DATABASE_H -#define LD_DATABASE_H - -#include -#include -#include - -#include -#include -#include -#include - -#include "leveldown.h" -#include "iterator.h" - -namespace leveldown { - -NAN_METHOD(LevelDOWN); - -struct Reference { - Nan::Persistent handle; - leveldb::Slice slice; - - Reference(v8::Local obj, leveldb::Slice slice) : slice(slice) { - v8::Local _obj = Nan::New(); - _obj->Set(Nan::New("obj").ToLocalChecked(), obj); - handle.Reset(_obj); - }; -}; - -static inline void ClearReferences (std::vector *references) { - for (std::vector::iterator it = references->begin() - ; it != references->end() - ; ) { - DisposeStringOrBufferFromSlice((*it)->handle, (*it)->slice); - it = references->erase(it); - } - delete references; -} - -class Database : public Nan::ObjectWrap { -public: - static void Init (); - static v8::Local NewInstance (v8::Local &location); - - leveldb::Status OpenDatabase (leveldb::Options* options); - leveldb::Status PutToDatabase ( - leveldb::WriteOptions* options - , leveldb::Slice key - , leveldb::Slice value - ); - leveldb::Status GetFromDatabase ( - leveldb::ReadOptions* options - , leveldb::Slice key - , std::string& value - ); - leveldb::Status DeleteFromDatabase ( - leveldb::WriteOptions* options - , leveldb::Slice key - ); - leveldb::Status WriteBatchToDatabase ( - leveldb::WriteOptions* options - , leveldb::WriteBatch* batch - ); - uint64_t ApproximateSizeFromDatabase (const leveldb::Range* range); - void CompactRangeFromDatabase (const leveldb::Slice* start, const leveldb::Slice* end); - void GetPropertyFromDatabase (const leveldb::Slice& property, std::string* value); - leveldb::Iterator* NewIterator (leveldb::ReadOptions* options); - const leveldb::Snapshot* NewSnapshot (); - void ReleaseSnapshot (const leveldb::Snapshot* snapshot); - void CloseDatabase (); - void ReleaseIterator (uint32_t id); - - Database (const v8::Local& from); - ~Database (); - -private: - Nan::Utf8String* location; - leveldb::DB* db; - uint32_t currentIteratorId; - // What is this for? - void(*pendingCloseWorker); - leveldb::Cache* blockCache; - const leveldb::FilterPolicy* filterPolicy; - - std::map< uint32_t, leveldown::Iterator * > iterators; - - // What are these for? - static void WriteDoing(uv_work_t *req); - static void WriteAfter(uv_work_t *req); - - static NAN_METHOD(New); - static NAN_METHOD(Open); - static NAN_METHOD(Close); - static NAN_METHOD(Put); - static NAN_METHOD(Delete); - static NAN_METHOD(Get); - static NAN_METHOD(Batch); - static NAN_METHOD(Write); - static NAN_METHOD(Iterator); - static NAN_METHOD(ApproximateSize); - static NAN_METHOD(CompactRange); - static NAN_METHOD(GetProperty); -}; - -} // namespace leveldown - -#endif diff --git a/src/database_async.cc b/src/database_async.cc deleted file mode 100644 index ef044e42..00000000 --- a/src/database_async.cc +++ /dev/null @@ -1,290 +0,0 @@ -#include -#include - -#include -#include - -#include "database.h" -#include "leveldown.h" -#include "async.h" -#include "database_async.h" - -namespace leveldown { - -/** OPEN WORKER **/ - -OpenWorker::OpenWorker(Database *database, - Nan::Callback *callback, - leveldb::Cache* blockCache, - const leveldb::FilterPolicy* filterPolicy, - bool createIfMissing, - bool errorIfExists, - bool compression, - uint32_t writeBufferSize, - uint32_t blockSize, - uint32_t maxOpenFiles, - uint32_t blockRestartInterval, - uint32_t maxFileSize) -: AsyncWorker(database, callback, "leveldown:db.open") -{ - options = new leveldb::Options(); - options->block_cache = blockCache; - options->filter_policy = filterPolicy; - options->create_if_missing = createIfMissing; - options->error_if_exists = errorIfExists; - options->compression = compression - ? leveldb::kSnappyCompression - : leveldb::kNoCompression; - options->write_buffer_size = writeBufferSize; - options->block_size = blockSize; - options->max_open_files = maxOpenFiles; - options->block_restart_interval = blockRestartInterval; - options->max_file_size = maxFileSize; -}; - -OpenWorker::~OpenWorker() { - delete options; -} - -void OpenWorker::Execute() { - SetStatus(database->OpenDatabase(options)); -} - -/** CLOSE WORKER **/ - -CloseWorker::CloseWorker(Database *database, Nan::Callback *callback) - : AsyncWorker(database, callback, "leveldown:db.close") -{}; - -void CloseWorker::Execute() { - database->CloseDatabase(); -} - -void CloseWorker::WorkComplete() { - Nan::HandleScope scope; - HandleOKCallback(); - delete callback; - callback = NULL; -} - -/** IO WORKER (abstract) **/ - -IOWorker::IOWorker(Database *database, - Nan::Callback *callback, - const char *resource_name, - leveldb::Slice key, - v8::Local &keyHandle) - : AsyncWorker(database, callback, resource_name), key(key) -{ - Nan::HandleScope scope; - - SaveToPersistent("key", keyHandle); -}; - -void IOWorker::WorkComplete() { - Nan::HandleScope scope; - - DisposeStringOrBufferFromSlice(GetFromPersistent("key"), key); - AsyncWorker::WorkComplete(); -} - -/** READ WORKER **/ - -ReadWorker::ReadWorker(Database *database, - Nan::Callback *callback, - leveldb::Slice key, - bool asBuffer, - bool fillCache, - v8::Local &keyHandle) - : IOWorker(database, callback, "leveldown:db.get", key, keyHandle), - asBuffer(asBuffer) -{ - Nan::HandleScope scope; - - options = new leveldb::ReadOptions(); - options->fill_cache = fillCache; - SaveToPersistent("key", keyHandle); -}; - -ReadWorker::~ReadWorker() { - delete options; -} - -void ReadWorker::Execute() { - SetStatus(database->GetFromDatabase(options, key, value)); -} - -void ReadWorker::HandleOKCallback() { - Nan::HandleScope scope; - - v8::Local returnValue; - if (asBuffer) { - //TODO: could use NewBuffer if we carefully manage the lifecycle of `value` - //and avoid an an extra allocation. We'd have to clean up properly when not OK - //and let the new Buffer manage the data when OK - returnValue = Nan::CopyBuffer((char*)value.data(), value.size()).ToLocalChecked(); - } else { - returnValue = Nan::New((char*)value.data(), value.size()).ToLocalChecked(); - } - v8::Local argv[] = { - Nan::Null() - , returnValue - }; - callback->Call(2, argv, async_resource); -} - -/** DELETE WORKER **/ - -DeleteWorker::DeleteWorker(Database *database, - Nan::Callback *callback, - leveldb::Slice key, - bool sync, - v8::Local &keyHandle, - const char *resource_name) - : IOWorker(database, callback, resource_name, key, keyHandle) -{ - Nan::HandleScope scope; - - options = new leveldb::WriteOptions(); - options->sync = sync; - SaveToPersistent("key", keyHandle); -}; - -DeleteWorker::~DeleteWorker() { - delete options; -} - -void DeleteWorker::Execute() { - SetStatus(database->DeleteFromDatabase(options, key)); -} - -/** WRITE WORKER **/ - -WriteWorker::WriteWorker(Database *database, - Nan::Callback *callback, - leveldb::Slice key, - leveldb::Slice value, - bool sync, - v8::Local &keyHandle, - v8::Local &valueHandle) - : DeleteWorker(database, callback, key, sync, keyHandle, "leveldown:db.put"), - value(value) -{ - Nan::HandleScope scope; - - SaveToPersistent("value", valueHandle); -}; - -void WriteWorker::Execute() { - SetStatus(database->PutToDatabase(options, key, value)); -} - -void WriteWorker::WorkComplete() { - Nan::HandleScope scope; - - DisposeStringOrBufferFromSlice(GetFromPersistent("value"), value); - IOWorker::WorkComplete(); -} - -/** BATCH WORKER **/ - -BatchWorker::BatchWorker(Database *database, - Nan::Callback *callback, - leveldb::WriteBatch* batch, - bool sync) - : AsyncWorker(database, callback, "leveldown:db.batch"), batch(batch) -{ - options = new leveldb::WriteOptions(); - options->sync = sync; -}; - -BatchWorker::~BatchWorker() { - delete batch; - delete options; -} - -void BatchWorker::Execute() { - SetStatus(database->WriteBatchToDatabase(options, batch)); -} - -/** APPROXIMATE SIZE WORKER **/ - -ApproximateSizeWorker::ApproximateSizeWorker(Database *database, - Nan::Callback *callback, - leveldb::Slice start, - leveldb::Slice end, - v8::Local &startHandle, - v8::Local &endHandle) - : AsyncWorker(database, callback, "leveldown:db.approximateSize"), - range(start, end) -{ - Nan::HandleScope scope; - - SaveToPersistent("start", startHandle); - SaveToPersistent("end", endHandle); -}; - -void ApproximateSizeWorker::Execute() { - size = database->ApproximateSizeFromDatabase(&range); -} - -void ApproximateSizeWorker::WorkComplete() { - Nan::HandleScope scope; - - DisposeStringOrBufferFromSlice(GetFromPersistent("start"), range.start); - DisposeStringOrBufferFromSlice(GetFromPersistent("end"), range.limit); - AsyncWorker::WorkComplete(); -} - -void ApproximateSizeWorker::HandleOKCallback() { - Nan::HandleScope scope; - - v8::Local returnValue = Nan::New((double) size); - v8::Local argv[] = { - Nan::Null() - , returnValue - }; - callback->Call(2, argv, async_resource); -} - -/** COMPACT RANGE WORKER **/ - -CompactRangeWorker::CompactRangeWorker(Database *database, - Nan::Callback *callback, - leveldb::Slice start, - leveldb::Slice end, - v8::Local &startHandle, - v8::Local &endHandle) - : AsyncWorker(database, callback, "leveldown:db.compactRange") -{ - Nan::HandleScope scope; - - rangeStart = start; - rangeEnd = end; - - SaveToPersistent("compactStart", startHandle); - SaveToPersistent("compactEnd", endHandle); -}; - -void CompactRangeWorker::Execute() { - database->CompactRangeFromDatabase(&rangeStart, &rangeEnd); -} - -void CompactRangeWorker::WorkComplete() { - Nan::HandleScope scope; - - DisposeStringOrBufferFromSlice(GetFromPersistent("compactStart"), rangeStart); - DisposeStringOrBufferFromSlice(GetFromPersistent("compactEnd"), rangeEnd); - AsyncWorker::WorkComplete(); -} - -void CompactRangeWorker::HandleOKCallback() { - Nan::HandleScope scope; - - v8::Local argv[] = { - Nan::Null() - }; - callback->Call(1, argv, async_resource); -} - -} // namespace leveldown diff --git a/src/database_async.h b/src/database_async.h deleted file mode 100644 index f150812c..00000000 --- a/src/database_async.h +++ /dev/null @@ -1,168 +0,0 @@ -#ifndef LD_DATABASE_ASYNC_H -#define LD_DATABASE_ASYNC_H - -#include -#include - -#include - -#include "async.h" - -namespace leveldown { - -class OpenWorker : public AsyncWorker { -public: - OpenWorker(Database *database, - Nan::Callback *callback, - leveldb::Cache* blockCache, - const leveldb::FilterPolicy* filterPolicy, - bool createIfMissing, - bool errorIfExists, - bool compression, - uint32_t writeBufferSize, - uint32_t blockSize, - uint32_t maxOpenFiles, - uint32_t blockRestartInterval, - uint32_t maxFileSize); - - virtual ~OpenWorker(); - virtual void Execute(); - -private: - leveldb::Options* options; -}; - -class CloseWorker : public AsyncWorker { -public: - CloseWorker(Database *database, Nan::Callback *callback); - - virtual ~CloseWorker() {} - virtual void Execute(); - virtual void WorkComplete(); -}; - -class IOWorker : public AsyncWorker { -public: - IOWorker(Database *database, - Nan::Callback *callback, - const char *resource_name, - leveldb::Slice key, - v8::Local &keyHandle); - - virtual ~IOWorker() {} - virtual void WorkComplete(); - -protected: - leveldb::Slice key; -}; - -class ReadWorker : public IOWorker { -public: - ReadWorker(Database *database, - Nan::Callback *callback, - leveldb::Slice key, - bool asBuffer, - bool fillCache, - v8::Local &keyHandle); - - virtual ~ReadWorker(); - virtual void Execute(); - virtual void HandleOKCallback(); - -private: - bool asBuffer; - leveldb::ReadOptions* options; - std::string value; -}; - -class DeleteWorker : public IOWorker { -public: - DeleteWorker(Database *database, - Nan::Callback *callback, - leveldb::Slice key, - bool sync, - v8::Local &keyHandle, - const char *resource_name = "leveldown:db.del"); - - virtual ~DeleteWorker(); - virtual void Execute(); - -protected: - leveldb::WriteOptions* options; -}; - -class WriteWorker : public DeleteWorker { -public: - WriteWorker(Database *database, - Nan::Callback *callback, - leveldb::Slice key, - leveldb::Slice value, - bool sync, - v8::Local &keyHandle, - v8::Local &valueHandle); - - virtual ~WriteWorker() {} - virtual void Execute(); - virtual void WorkComplete(); - -private: - leveldb::Slice value; -}; - -class BatchWorker : public AsyncWorker { -public: - BatchWorker(Database *database, - Nan::Callback *callback, - leveldb::WriteBatch* batch, - bool sync); - - virtual ~BatchWorker(); - virtual void Execute(); - -private: - leveldb::WriteOptions* options; - leveldb::WriteBatch* batch; -}; - -class ApproximateSizeWorker : public AsyncWorker { -public: - ApproximateSizeWorker(Database *database, - Nan::Callback *callback, - leveldb::Slice start, - leveldb::Slice end, - v8::Local &startHandle, - v8::Local &endHandle); - - virtual ~ApproximateSizeWorker() {} - virtual void Execute(); - virtual void HandleOKCallback(); - virtual void WorkComplete(); - - private: - leveldb::Range range; - uint64_t size; -}; - -class CompactRangeWorker : public AsyncWorker { -public: - CompactRangeWorker(Database *database, - Nan::Callback *callback, - leveldb::Slice start, - leveldb::Slice end, - v8::Local &startHandle, - v8::Local &endHandle); - - virtual ~CompactRangeWorker() {} - virtual void Execute(); - virtual void HandleOKCallback(); - virtual void WorkComplete(); - - private: - leveldb::Slice rangeStart; - leveldb::Slice rangeEnd; -}; - - -} // namespace leveldown - -#endif diff --git a/src/iterator.cc b/src/iterator.cc deleted file mode 100644 index b4620c6d..00000000 --- a/src/iterator.cc +++ /dev/null @@ -1,613 +0,0 @@ -#include -#include - -#include "database.h" -#include "iterator.h" -#include "iterator_async.h" -#include "common.h" - -namespace leveldown { - -static Nan::Persistent iterator_constructor; - -Iterator::Iterator ( - Database* database - , uint32_t id - , leveldb::Slice* start - , std::string* end - , bool reverse - , bool keys - , bool values - , int limit - , std::string* lt - , std::string* lte - , std::string* gt - , std::string* gte - , bool fillCache - , bool keyAsBuffer - , bool valueAsBuffer - , size_t highWaterMark -) : database(database) - , id(id) - , start(start) - , end(end) - , reverse(reverse) - , keys(keys) - , values(values) - , limit(limit) - , lt(lt) - , lte(lte) - , gt(gt) - , gte(gte) - , highWaterMark(highWaterMark) - , keyAsBuffer(keyAsBuffer) - , valueAsBuffer(valueAsBuffer) -{ - Nan::HandleScope scope; - - options = new leveldb::ReadOptions(); - options->fill_cache = fillCache; - // get a snapshot of the current state - options->snapshot = database->NewSnapshot(); - dbIterator = NULL; - count = 0; - target = NULL; - seeking = false; - landed = false; - nexting = false; - ended = false; - endWorker = NULL; -}; - -Iterator::~Iterator () { - delete options; - ReleaseTarget(); - if (start != NULL) { - // Special case for `start` option: it won't be - // freed up by any of the delete calls below. - if (!((lt != NULL && reverse) - || (lte != NULL && reverse) - || (gt != NULL && !reverse) - || (gte != NULL && !reverse))) { - delete[] start->data(); - } - delete start; - } - if (end != NULL) - delete end; - if (lt != NULL) - delete lt; - if (gt != NULL) - delete gt; - if (lte != NULL) - delete lte; - if (gte != NULL) - delete gte; -}; - -bool Iterator::GetIterator () { - if (dbIterator == NULL) { - dbIterator = database->NewIterator(options); - - if (start != NULL) { - dbIterator->Seek(*start); - - if (reverse) { - if (!dbIterator->Valid()) { - // if it's past the last key, step back - dbIterator->SeekToLast(); - } else { - std::string key_ = dbIterator->key().ToString(); - - if (lt != NULL) { - if (lt->compare(key_) <= 0) - dbIterator->Prev(); - } else if (lte != NULL) { - if (lte->compare(key_) < 0) - dbIterator->Prev(); - } else if (start != NULL) { - if (start->compare(key_)) - dbIterator->Prev(); - } - } - - if (dbIterator->Valid() && lt != NULL) { - if (lt->compare(dbIterator->key().ToString()) <= 0) - dbIterator->Prev(); - } - } else { - if (dbIterator->Valid() && gt != NULL - && gt->compare(dbIterator->key().ToString()) == 0) - dbIterator->Next(); - } - } else if (reverse) { - dbIterator->SeekToLast(); - } else { - dbIterator->SeekToFirst(); - } - - return true; - } - return false; -} - -bool Iterator::Read (std::string& key, std::string& value) { - // if it's not the first call, move to next item. - if (!GetIterator() && !seeking) { - if (reverse) - dbIterator->Prev(); - else - dbIterator->Next(); - } - - seeking = false; - - // now check if this is the end or not, if not then return the key & value - if (dbIterator->Valid()) { - std::string key_ = dbIterator->key().ToString(); - int isEnd = end == NULL ? 1 : end->compare(key_); - - if ((limit < 0 || ++count <= limit) - && (end == NULL - || (reverse && (isEnd <= 0)) - || (!reverse && (isEnd >= 0))) - && ( lt != NULL ? (lt->compare(key_) > 0) - : lte != NULL ? (lte->compare(key_) >= 0) - : true ) - && ( gt != NULL ? (gt->compare(key_) < 0) - : gte != NULL ? (gte->compare(key_) <= 0) - : true ) - ) { - if (keys) - key.assign(dbIterator->key().data(), dbIterator->key().size()); - if (values) - value.assign(dbIterator->value().data(), dbIterator->value().size()); - return true; - } - } - - return false; -} - -bool Iterator::OutOfRange (leveldb::Slice* target) { - if (lt != NULL) { - if (target->compare(*lt) >= 0) - return true; - } else if (lte != NULL) { - if (target->compare(*lte) > 0) - return true; - } else if (start != NULL && reverse) { - if (target->compare(*start) > 0) - return true; - } - - if (end != NULL) { - int d = target->compare(*end); - if (reverse ? d < 0 : d > 0) - return true; - } - - if (gt != NULL) { - if (target->compare(*gt) <= 0) - return true; - } else if (gte != NULL) { - if (target->compare(*gte) < 0) - return true; - } else if (start != NULL && !reverse) { - if (target->compare(*start) < 0) - return true; - } - - return false; -} - -bool Iterator::IteratorNext (std::vector >& result) { - size_t size = 0; - while(true) { - std::string key, value; - bool ok = Read(key, value); - - if (ok) { - result.push_back(std::make_pair(key, value)); - - if (!landed) { - landed = true; - return true; - } - - size = size + key.size() + value.size(); - if (size > highWaterMark) - return true; - - } else { - return false; - } - } -} - -leveldb::Status Iterator::IteratorStatus () { - return dbIterator->status(); -} - -void Iterator::IteratorEnd () { - //TODO: could return it->status() - delete dbIterator; - dbIterator = NULL; - database->ReleaseSnapshot(options->snapshot); -} - -void Iterator::Release () { - database->ReleaseIterator(id); -} - -void Iterator::ReleaseTarget () { - if (target != NULL) { - - if (!target->empty()) - delete[] target->data(); - - delete target; - target = NULL; - } -} - -void checkEndCallback (Iterator* iterator) { - iterator->ReleaseTarget(); - iterator->nexting = false; - if (iterator->endWorker != NULL) { - Nan::AsyncQueueWorker(iterator->endWorker); - iterator->endWorker = NULL; - } -} - -NAN_METHOD(Iterator::Seek) { - Iterator* iterator = Nan::ObjectWrap::Unwrap(info.This()); - - iterator->ReleaseTarget(); - - v8::Local targetBuffer = info[0].As(); - LD_STRING_OR_BUFFER_TO_COPY(_target, targetBuffer, target); - iterator->target = new leveldb::Slice(_targetCh_, _targetSz_); - - iterator->GetIterator(); - leveldb::Iterator* dbIterator = iterator->dbIterator; - - dbIterator->Seek(*iterator->target); - iterator->seeking = true; - iterator->landed = false; - - if (iterator->OutOfRange(iterator->target)) { - if (iterator->reverse) { - dbIterator->SeekToFirst(); - dbIterator->Prev(); - } else { - dbIterator->SeekToLast(); - dbIterator->Next(); - } - } - else { - if (dbIterator->Valid()) { - int cmp = dbIterator->key().compare(*iterator->target); - if (cmp > 0 && iterator->reverse) { - dbIterator->Prev(); - } else if (cmp < 0 && !iterator->reverse) { - dbIterator->Next(); - } - } else { - if (iterator->reverse) { - dbIterator->SeekToLast(); - } else { - dbIterator->SeekToFirst(); - } - if (dbIterator->Valid()) { - int cmp = dbIterator->key().compare(*iterator->target); - if (cmp > 0 && iterator->reverse) { - dbIterator->SeekToFirst(); - dbIterator->Prev(); - } else if (cmp < 0 && !iterator->reverse) { - dbIterator->SeekToLast(); - dbIterator->Next(); - } - } - } - } - - info.GetReturnValue().Set(info.Holder()); -} - -NAN_METHOD(Iterator::Next) { - Iterator* iterator = Nan::ObjectWrap::Unwrap(info.This()); - - if (!info[0]->IsFunction()) { - return Nan::ThrowError("next() requires a callback argument"); - } - - v8::Local callback = info[0].As(); - - if (iterator->ended) { - if (!callback.IsEmpty() && callback->IsFunction()) { - v8::Local argv[] = { Nan::Error("iterator has ended") }; - LD_RUN_CALLBACK("leveldown:iterator.next", callback, 1, argv); - info.GetReturnValue().SetUndefined(); - return; - } - return Nan::ThrowError("iterator has ended"); - } - - NextWorker* worker = new NextWorker( - iterator - , new Nan::Callback(callback) - , checkEndCallback - ); - // persist to prevent accidental GC - v8::Local _this = info.This(); - worker->SaveToPersistent("iterator", _this); - iterator->nexting = true; - Nan::AsyncQueueWorker(worker); - - info.GetReturnValue().Set(info.Holder()); -} - -NAN_METHOD(Iterator::End) { - Iterator* iterator = Nan::ObjectWrap::Unwrap(info.This()); - - if (!info[0]->IsFunction()) { - return Nan::ThrowError("end() requires a callback argument"); - } - - if (!iterator->ended) { - v8::Local callback = v8::Local::Cast(info[0]); - - EndWorker* worker = new EndWorker( - iterator - , new Nan::Callback(callback) - ); - // persist to prevent accidental GC - v8::Local _this = info.This(); - worker->SaveToPersistent("iterator", _this); - iterator->ended = true; - - if (iterator->nexting) { - // waiting for a next() to return, queue the end - iterator->endWorker = worker; - } else { - Nan::AsyncQueueWorker(worker); - } - } - - info.GetReturnValue().Set(info.Holder()); -} - -void Iterator::Init () { - v8::Local tpl = - Nan::New(Iterator::New); - iterator_constructor.Reset(tpl); - tpl->SetClassName(Nan::New("Iterator").ToLocalChecked()); - tpl->InstanceTemplate()->SetInternalFieldCount(1); - Nan::SetPrototypeMethod(tpl, "seek", Iterator::Seek); - Nan::SetPrototypeMethod(tpl, "next", Iterator::Next); - Nan::SetPrototypeMethod(tpl, "end", Iterator::End); -} - -v8::Local Iterator::NewInstance ( - v8::Local database - , v8::Local id - , v8::Local optionsObj - ) { - - Nan::EscapableHandleScope scope; - - Nan::MaybeLocal maybeInstance; - v8::Local instance; - v8::Local constructorHandle = - Nan::New(iterator_constructor); - - if (optionsObj.IsEmpty()) { - v8::Local argv[2] = { database, id }; - maybeInstance = Nan::NewInstance(constructorHandle->GetFunction(), 2, argv); - } else { - v8::Local argv[3] = { database, id, optionsObj }; - maybeInstance = Nan::NewInstance(constructorHandle->GetFunction(), 3, argv); - } - - if (maybeInstance.IsEmpty()) - Nan::ThrowError("Could not create new Iterator instance"); - else - instance = maybeInstance.ToLocalChecked(); - return scope.Escape(instance); -} - -NAN_METHOD(Iterator::New) { - Database* database = Nan::ObjectWrap::Unwrap(info[0]->ToObject()); - - leveldb::Slice* start = NULL; - std::string* end = NULL; - int limit = -1; - // default highWaterMark from Readble-streams - size_t highWaterMark = 16 * 1024; - - v8::Local id = info[1]; - - v8::Local optionsObj; - - v8::Local ltHandle; - v8::Local lteHandle; - v8::Local gtHandle; - v8::Local gteHandle; - - char *startStr = NULL; - std::string* lt = NULL; - std::string* lte = NULL; - std::string* gt = NULL; - std::string* gte = NULL; - - //default to forward. - bool reverse = false; - - if (info.Length() > 1 && info[2]->IsObject()) { - optionsObj = v8::Local::Cast(info[2]); - - reverse = BooleanOptionValue(optionsObj, "reverse"); - - if (optionsObj->Has(Nan::New("start").ToLocalChecked()) - && (node::Buffer::HasInstance(optionsObj->Get(Nan::New("start").ToLocalChecked())) - || optionsObj->Get(Nan::New("start").ToLocalChecked())->IsString())) { - - v8::Local startBuffer = optionsObj->Get(Nan::New("start").ToLocalChecked()); - - // ignore start if it has size 0 since a Slice can't have length 0 - if (StringOrBufferLength(startBuffer) > 0) { - LD_STRING_OR_BUFFER_TO_COPY(_start, startBuffer, start) - start = new leveldb::Slice(_startCh_, _startSz_); - startStr = _startCh_; - } - } - - if (optionsObj->Has(Nan::New("end").ToLocalChecked()) - && (node::Buffer::HasInstance(optionsObj->Get(Nan::New("end").ToLocalChecked())) - || optionsObj->Get(Nan::New("end").ToLocalChecked())->IsString())) { - - v8::Local endBuffer = optionsObj->Get(Nan::New("end").ToLocalChecked()); - - // ignore end if it has size 0 since a Slice can't have length 0 - if (StringOrBufferLength(endBuffer) > 0) { - LD_STRING_OR_BUFFER_TO_COPY(_end, endBuffer, end) - end = new std::string(_endCh_, _endSz_); - delete[] _endCh_; - } - } - - if (!optionsObj.IsEmpty() && optionsObj->Has(Nan::New("limit").ToLocalChecked())) { - limit = v8::Local::Cast(optionsObj->Get( - Nan::New("limit").ToLocalChecked()))->Value(); - } - - if (optionsObj->Has(Nan::New("highWaterMark").ToLocalChecked())) { - highWaterMark = v8::Local::Cast(optionsObj->Get( - Nan::New("highWaterMark").ToLocalChecked()))->Value(); - } - - if (optionsObj->Has(Nan::New("lt").ToLocalChecked()) - && (node::Buffer::HasInstance(optionsObj->Get(Nan::New("lt").ToLocalChecked())) - || optionsObj->Get(Nan::New("lt").ToLocalChecked())->IsString())) { - - v8::Local ltBuffer = optionsObj->Get(Nan::New("lt").ToLocalChecked()); - - // ignore end if it has size 0 since a Slice can't have length 0 - if (StringOrBufferLength(ltBuffer) > 0) { - LD_STRING_OR_BUFFER_TO_COPY(_lt, ltBuffer, lt) - lt = new std::string(_ltCh_, _ltSz_); - delete[] _ltCh_; - if (reverse) { - if (startStr != NULL) { - delete[] startStr; - startStr = NULL; - } - if (start != NULL) - delete start; - start = new leveldb::Slice(lt->data(), lt->size()); - } - } - } - - if (optionsObj->Has(Nan::New("lte").ToLocalChecked()) - && (node::Buffer::HasInstance(optionsObj->Get(Nan::New("lte").ToLocalChecked())) - || optionsObj->Get(Nan::New("lte").ToLocalChecked())->IsString())) { - - v8::Local lteBuffer = optionsObj->Get(Nan::New("lte").ToLocalChecked()); - - // ignore end if it has size 0 since a Slice can't have length 0 - if (StringOrBufferLength(lteBuffer) > 0) { - LD_STRING_OR_BUFFER_TO_COPY(_lte, lteBuffer, lte) - lte = new std::string(_lteCh_, _lteSz_); - delete[] _lteCh_; - if (reverse) { - if (startStr != NULL) { - delete[] startStr; - startStr = NULL; - } - if (start != NULL) - delete start; - start = new leveldb::Slice(lte->data(), lte->size()); - } - } - } - - if (optionsObj->Has(Nan::New("gt").ToLocalChecked()) - && (node::Buffer::HasInstance(optionsObj->Get(Nan::New("gt").ToLocalChecked())) - || optionsObj->Get(Nan::New("gt").ToLocalChecked())->IsString())) { - - v8::Local gtBuffer = optionsObj->Get(Nan::New("gt").ToLocalChecked()); - - // ignore end if it has size 0 since a Slice can't have length 0 - if (StringOrBufferLength(gtBuffer) > 0) { - LD_STRING_OR_BUFFER_TO_COPY(_gt, gtBuffer, gt) - gt = new std::string(_gtCh_, _gtSz_); - delete[] _gtCh_; - if (!reverse) { - if (startStr != NULL) { - delete[] startStr; - startStr = NULL; - } - if (start != NULL) - delete start; - start = new leveldb::Slice(gt->data(), gt->size()); - } - } - } - - if (optionsObj->Has(Nan::New("gte").ToLocalChecked()) - && (node::Buffer::HasInstance(optionsObj->Get(Nan::New("gte").ToLocalChecked())) - || optionsObj->Get(Nan::New("gte").ToLocalChecked())->IsString())) { - - v8::Local gteBuffer = optionsObj->Get(Nan::New("gte").ToLocalChecked()); - - // ignore end if it has size 0 since a Slice can't have length 0 - if (StringOrBufferLength(gteBuffer) > 0) { - LD_STRING_OR_BUFFER_TO_COPY(_gte, gteBuffer, gte) - gte = new std::string(_gteCh_, _gteSz_); - delete[] _gteCh_; - if (!reverse) { - if (startStr != NULL) { - delete[] startStr; - startStr = NULL; - } - if (start != NULL) - delete start; - start = new leveldb::Slice(gte->data(), gte->size()); - } - } - } - - } - - bool keys = BooleanOptionValue(optionsObj, "keys", true); - bool values = BooleanOptionValue(optionsObj, "values", true); - bool keyAsBuffer = BooleanOptionValue(optionsObj, "keyAsBuffer", true); - bool valueAsBuffer = BooleanOptionValue(optionsObj, "valueAsBuffer", true); - bool fillCache = BooleanOptionValue(optionsObj, "fillCache"); - - Iterator* iterator = new Iterator( - database - , (uint32_t)id->Int32Value() - , start - , end - , reverse - , keys - , values - , limit - , lt - , lte - , gt - , gte - , fillCache - , keyAsBuffer - , valueAsBuffer - , highWaterMark - ); - iterator->Wrap(info.This()); - - info.GetReturnValue().Set(info.This()); -} - -} // namespace leveldown diff --git a/src/iterator.h b/src/iterator.h deleted file mode 100644 index 50e8891c..00000000 --- a/src/iterator.h +++ /dev/null @@ -1,94 +0,0 @@ -#ifndef LD_ITERATOR_H -#define LD_ITERATOR_H - -#include -#include -#include - -#include "leveldown.h" -#include "database.h" -#include "async.h" - -namespace leveldown { - -class Database; -class AsyncWorker; - -class Iterator : public Nan::ObjectWrap { -public: - static void Init (); - static v8::Local NewInstance ( - v8::Local database - , v8::Local id - , v8::Local optionsObj - ); - - Iterator ( - Database* database - , uint32_t id - , leveldb::Slice* start - , std::string* end - , bool reverse - , bool keys - , bool values - , int limit - , std::string* lt - , std::string* lte - , std::string* gt - , std::string* gte - , bool fillCache - , bool keyAsBuffer - , bool valueAsBuffer - , size_t highWaterMark - ); - - ~Iterator (); - - bool IteratorNext (std::vector >& result); - leveldb::Status IteratorStatus (); - void IteratorEnd (); - void Release (); - void ReleaseTarget (); - -private: - Database* database; - uint32_t id; - leveldb::Iterator* dbIterator; - leveldb::ReadOptions* options; - leveldb::Slice* start; - leveldb::Slice* target; - std::string* end; - bool seeking; - bool landed; - bool reverse; - bool keys; - bool values; - int limit; - std::string* lt; - std::string* lte; - std::string* gt; - std::string* gte; - int count; - size_t highWaterMark; - -public: - bool keyAsBuffer; - bool valueAsBuffer; - bool nexting; - bool ended; - AsyncWorker* endWorker; - -private: - bool Read (std::string& key, std::string& value); - bool GetIterator (); - bool OutOfRange (leveldb::Slice* target); - - static NAN_METHOD(New); - static NAN_METHOD(Seek); - static NAN_METHOD(Next); - static NAN_METHOD(End); -}; - -} // namespace leveldown - -#endif diff --git a/src/iterator_async.cc b/src/iterator_async.cc deleted file mode 100644 index 6cc3308e..00000000 --- a/src/iterator_async.cc +++ /dev/null @@ -1,86 +0,0 @@ -#include -#include - -#include "database.h" -#include "leveldown.h" -#include "async.h" -#include "iterator_async.h" - -namespace leveldown { - -/** NEXT-MULTI WORKER **/ - -NextWorker::NextWorker(Iterator* iterator, - Nan::Callback *callback, - void (*localCallback)(Iterator*)) - : AsyncWorker(NULL, callback, "leveldown:iterator.next"), iterator(iterator), - localCallback(localCallback) -{} - -void NextWorker::Execute() { - ok = iterator->IteratorNext(result); - if (!ok) - SetStatus(iterator->IteratorStatus()); -} - -void NextWorker::HandleOKCallback() { - Nan::HandleScope scope; - size_t idx = 0; - - size_t arraySize = result.size() * 2; - v8::Local returnArray = Nan::New(arraySize); - - for(idx = 0; idx < result.size(); ++idx) { - std::pair row = result[idx]; - std::string key = row.first; - std::string value = row.second; - - v8::Local returnKey; - if (iterator->keyAsBuffer) { - //TODO: use NewBuffer, see database_async.cc - returnKey = Nan::CopyBuffer((char*)key.data(), key.size()).ToLocalChecked(); - } else { - returnKey = Nan::New((char*)key.data(), key.size()).ToLocalChecked(); - } - - v8::Local returnValue; - if (iterator->valueAsBuffer) { - //TODO: use NewBuffer, see database_async.cc - returnValue = Nan::CopyBuffer((char*)value.data(), value.size()).ToLocalChecked(); - } else { - returnValue = Nan::New((char*)value.data(), value.size()).ToLocalChecked(); - } - - // put the key & value in a descending order, so that they can be .pop:ed in javascript-land - returnArray->Set(Nan::New(static_cast(arraySize - idx * 2 - 1)), returnKey); - returnArray->Set(Nan::New(static_cast(arraySize - idx * 2 - 2)), returnValue); - } - - // clean up & handle the next/end state see iterator.cc/checkEndCallback - localCallback(iterator); - - v8::Local argv[] = { - Nan::Null() - , returnArray - // when ok === false all data has been read, so it's then finished - , Nan::New(!ok) - }; - callback->Call(3, argv, async_resource); -} - -/** END WORKER **/ - -EndWorker::EndWorker(Iterator* iterator, Nan::Callback *callback) - : AsyncWorker(NULL, callback, "leveldown:iterator.end"), iterator(iterator) -{} - -void EndWorker::Execute() { - iterator->IteratorEnd(); -} - -void EndWorker::HandleOKCallback() { - iterator->Release(); - callback->Call(0, NULL, async_resource); -} - -} // namespace leveldown diff --git a/src/iterator_async.h b/src/iterator_async.h deleted file mode 100644 index e08728ba..00000000 --- a/src/iterator_async.h +++ /dev/null @@ -1,43 +0,0 @@ -#ifndef LD_ITERATOR_ASYNC_H -#define LD_ITERATOR_ASYNC_H - -#include -#include - -#include "async.h" -#include "iterator.h" - -namespace leveldown { - -class NextWorker : public AsyncWorker { -public: - NextWorker(Iterator* iterator, - Nan::Callback *callback, - void (*localCallback)(Iterator*)); - - virtual ~NextWorker() {} - virtual void Execute(); - virtual void HandleOKCallback(); - -private: - Iterator* iterator; - void (*localCallback)(Iterator*); - std::vector > result; - bool ok; -}; - -class EndWorker : public AsyncWorker { -public: - EndWorker(Iterator* iterator, Nan::Callback *callback); - - virtual ~EndWorker() {} - virtual void Execute(); - virtual void HandleOKCallback(); - -private: - Iterator* iterator; -}; - -} // namespace leveldown - -#endif diff --git a/src/leveldown.cc b/src/leveldown.cc deleted file mode 100644 index 2e7f6c1d..00000000 --- a/src/leveldown.cc +++ /dev/null @@ -1,70 +0,0 @@ -#include - -#include "leveldown.h" -#include "database.h" -#include "iterator.h" -#include "batch.h" -#include "leveldown_async.h" - -namespace leveldown { - -NAN_METHOD(DestroyDB) { - Nan::HandleScope scope; - - Nan::Utf8String* location = new Nan::Utf8String(info[0]); - - Nan::Callback* callback = new Nan::Callback( - v8::Local::Cast(info[1])); - - DestroyWorker* worker = new DestroyWorker( - location - , callback - ); - - Nan::AsyncQueueWorker(worker); - - info.GetReturnValue().SetUndefined(); -} - -NAN_METHOD(RepairDB) { - Nan::HandleScope scope; - - Nan::Utf8String* location = new Nan::Utf8String(info[0]); - - Nan::Callback* callback = new Nan::Callback( - v8::Local::Cast(info[1])); - - RepairWorker* worker = new RepairWorker( - location - , callback - ); - - Nan::AsyncQueueWorker(worker); - - info.GetReturnValue().SetUndefined(); -} - -void Init (v8::Local target) { - Database::Init(); - leveldown::Iterator::Init(); - leveldown::Batch::Init(); - - v8::Local leveldown = - Nan::New(LevelDOWN)->GetFunction(); - - leveldown->Set( - Nan::New("destroy").ToLocalChecked() - , Nan::New(DestroyDB)->GetFunction() - ); - - leveldown->Set( - Nan::New("repair").ToLocalChecked() - , Nan::New(RepairDB)->GetFunction() - ); - - target->Set(Nan::New("leveldown").ToLocalChecked(), leveldown); -} - -NODE_MODULE(leveldown, Init) - -} // namespace leveldown diff --git a/src/leveldown.h b/src/leveldown.h deleted file mode 100644 index 053344f7..00000000 --- a/src/leveldown.h +++ /dev/null @@ -1,117 +0,0 @@ -#ifndef LD_LEVELDOWN_H -#define LD_LEVELDOWN_H - -#include -#include -#include -#include - -static inline size_t StringOrBufferLength(v8::Local obj) { - Nan::HandleScope scope; - - return (!obj->ToObject().IsEmpty() - && node::Buffer::HasInstance(obj->ToObject())) - ? node::Buffer::Length(obj->ToObject()) - : obj->ToString()->Utf8Length(); -} - -// NOTE: this MUST be called on objects created by -// LD_STRING_OR_BUFFER_TO_SLICE -static inline void DisposeStringOrBufferFromSlice( - Nan::Persistent &handle - , leveldb::Slice slice) { - Nan::HandleScope scope; - - if (!slice.empty()) { - v8::Local obj = Nan::New(handle)->Get(Nan::New("obj").ToLocalChecked()); - if (!node::Buffer::HasInstance(obj)) - delete[] slice.data(); - } - - handle.Reset(); -} - -static inline void DisposeStringOrBufferFromSlice( - v8::Local handle - , leveldb::Slice slice) { - - if (!slice.empty() && !node::Buffer::HasInstance(handle)) - delete[] slice.data(); -} - -// NOTE: must call DisposeStringOrBufferFromSlice() on objects created here -#define LD_STRING_OR_BUFFER_TO_SLICE(to, from, name) \ - size_t to ## Sz_; \ - char* to ## Ch_; \ - if (from->IsNull() || from->IsUndefined()) { \ - to ## Sz_ = 0; \ - to ## Ch_ = 0; \ - } else if (!from->ToObject().IsEmpty() \ - && node::Buffer::HasInstance(from->ToObject())) { \ - to ## Sz_ = node::Buffer::Length(from->ToObject()); \ - to ## Ch_ = node::Buffer::Data(from->ToObject()); \ - } else { \ - v8::Local to ## Str = from->ToString(); \ - to ## Sz_ = to ## Str->Utf8Length(); \ - to ## Ch_ = new char[to ## Sz_]; \ - to ## Str->WriteUtf8( \ - to ## Ch_ \ - , -1 \ - , NULL, v8::String::NO_NULL_TERMINATION \ - ); \ - } \ - leveldb::Slice to(to ## Ch_, to ## Sz_); - -#define LD_STRING_OR_BUFFER_TO_COPY(to, from, name) \ - size_t to ## Sz_; \ - char* to ## Ch_; \ - if (!from->ToObject().IsEmpty() \ - && node::Buffer::HasInstance(from->ToObject())) { \ - to ## Sz_ = node::Buffer::Length(from->ToObject()); \ - to ## Ch_ = new char[to ## Sz_]; \ - memcpy(to ## Ch_, node::Buffer::Data(from->ToObject()), to ## Sz_); \ - } else { \ - v8::Local to ## Str = from->ToString(); \ - to ## Sz_ = to ## Str->Utf8Length(); \ - to ## Ch_ = new char[to ## Sz_]; \ - to ## Str->WriteUtf8( \ - to ## Ch_ \ - , -1 \ - , NULL, v8::String::NO_NULL_TERMINATION \ - ); \ - } - -#define LD_RUN_CALLBACK(resource, callback, argc, argv) \ - Nan::AsyncResource ar(resource); \ - ar.runInAsyncScope(Nan::GetCurrentContext()->Global(), \ - callback, argc, argv); - -/* LD_METHOD_SETUP_COMMON setup the following objects: - * - Database* database - * - v8::Local optionsObj (may be empty) - * - Nan::Persistent callback (won't be empty) - * Will throw/return if there isn't a callback in arg 0 or 1 - */ -#define LD_METHOD_SETUP_COMMON(name, optionPos, callbackPos) \ - if (info.Length() == 0) \ - return Nan::ThrowError(#name "() requires a callback argument"); \ - leveldown::Database* database = \ - Nan::ObjectWrap::Unwrap(info.This()); \ - v8::Local optionsObj; \ - v8::Local callback; \ - if (optionPos == -1 && info[callbackPos]->IsFunction()) { \ - callback = info[callbackPos].As(); \ - } else if (optionPos != -1 && info[callbackPos - 1]->IsFunction()) { \ - callback = info[callbackPos - 1].As(); \ - } else if (optionPos != -1 \ - && info[optionPos]->IsObject() \ - && info[callbackPos]->IsFunction()) { \ - optionsObj = info[optionPos].As(); \ - callback = info[callbackPos].As(); \ - } else { \ - return Nan::ThrowError(#name "() requires a callback argument"); \ - } - -#define LD_METHOD_SETUP_COMMON_ONEARG(name) LD_METHOD_SETUP_COMMON(name, -1, 0) - -#endif diff --git a/src/leveldown_async.cc b/src/leveldown_async.cc deleted file mode 100644 index 19d731d0..00000000 --- a/src/leveldown_async.cc +++ /dev/null @@ -1,38 +0,0 @@ -#include - -#include "leveldown.h" -#include "leveldown_async.h" - -namespace leveldown { - -/** DESTROY WORKER **/ - -DestroyWorker::DestroyWorker(Nan::Utf8String* location, Nan::Callback *callback) - : AsyncWorker(NULL, callback, "leveldown:destroy"), location(location) -{} - -DestroyWorker::~DestroyWorker() { - delete location; -} - -void DestroyWorker::Execute() { - leveldb::Options options; - SetStatus(leveldb::DestroyDB(**location, options)); -} - -/** REPAIR WORKER **/ - -RepairWorker::RepairWorker(Nan::Utf8String* location, Nan::Callback *callback) - : AsyncWorker(NULL, callback, "leveldown:repair"), location(location) -{} - -RepairWorker::~RepairWorker() { - delete location; -} - -void RepairWorker::Execute() { - leveldb::Options options; - SetStatus(leveldb::RepairDB(**location, options)); -} - -} // namespace leveldown diff --git a/src/leveldown_async.h b/src/leveldown_async.h deleted file mode 100644 index 028a55ee..00000000 --- a/src/leveldown_async.h +++ /dev/null @@ -1,34 +0,0 @@ -#ifndef LD_LEVELDOWN_ASYNC_H -#define LD_LEVELDOWN_ASYNC_H - -#include - -#include "async.h" - -namespace leveldown { - -class DestroyWorker : public AsyncWorker { -public: - DestroyWorker(Nan::Utf8String* location, Nan::Callback *callback); - - virtual ~DestroyWorker(); - virtual void Execute(); - -private: - Nan::Utf8String* location; -}; - -class RepairWorker : public AsyncWorker { -public: - RepairWorker(Nan::Utf8String* location, Nan::Callback *callback); - - virtual ~RepairWorker(); - virtual void Execute(); - -private: - Nan::Utf8String* location; -}; - -} // namespace leveldown - -#endif From d4dd7f047a3d8ffa9af2965b6a902eb6c07e61ab Mon Sep 17 00:00:00 2001 From: Lars-Magnus Skog Date: Sun, 9 Dec 2018 14:51:09 +0100 Subject: [PATCH 41/42] Tweak back if-statements to be more clear --- binding.cc | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/binding.cc b/binding.cc index 3657ba33..a0c9fd72 100644 --- a/binding.cc +++ b/binding.cc @@ -544,22 +544,26 @@ struct Iterator { } else { std::string keyStr = dbIterator_->key().ToString(); - if (lt_ != NULL && lt_->compare(keyStr) <= 0) { - dbIterator_->Prev(); - } else if (lte_ != NULL && lte_->compare(keyStr) < 0) { - dbIterator_->Prev(); - } else if (start_ != NULL && start_->compare(keyStr)) { - dbIterator_->Prev(); + if (lt_ != NULL) { + if (lt_->compare(keyStr) <= 0) + dbIterator_->Prev(); + } else if (lte_ != NULL) { + if (lte_->compare(keyStr) < 0) + dbIterator_->Prev(); + } else if (start_ != NULL) { + if (start_->compare(keyStr)) + dbIterator_->Prev(); } } - if (dbIterator_->Valid() && lt_ != NULL - && lt_->compare(dbIterator_->key().ToString()) <= 0) { - dbIterator_->Prev(); + if (dbIterator_->Valid() && lt_ != NULL) { + if (lt_->compare(dbIterator_->key().ToString()) <= 0) + dbIterator_->Prev(); } - } else if (dbIterator_->Valid() && gt_ != NULL - && gt_->compare(dbIterator_->key().ToString()) == 0) { - dbIterator_->Next(); + } else { + if (dbIterator_->Valid() && gt_ != NULL + && gt_->compare(dbIterator_->key().ToString()) == 0) + dbIterator_->Next(); } } else if (reverse_) { dbIterator_->SeekToLast(); From e80be43b9bca1226563c3772f67bb6fdf8b92fda Mon Sep 17 00:00:00 2001 From: Lars-Magnus Skog Date: Sun, 9 Dec 2018 16:37:43 +0100 Subject: [PATCH 42/42] Pass in leveldb::Slice into workers and not napi values This is more for consistency, there were variants on this. Also makes it more clear where the slices are created --- binding.cc | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/binding.cc b/binding.cc index a0c9fd72..4ada59bd 100644 --- a/binding.cc +++ b/binding.cc @@ -830,11 +830,11 @@ struct PutWorker : public BaseWorker { PutWorker (napi_env env, Database* database, napi_value callback, - napi_value key, - napi_value value, + leveldb::Slice key, + leveldb::Slice value, bool sync) : BaseWorker(env, database, callback, "leveldown.db.put"), - key_(ToSlice(env, key)), value_(ToSlice(env, value)) { + key_(key), value_(value) { options_.sync = sync; } @@ -859,8 +859,8 @@ NAPI_METHOD(db_put) { NAPI_ARGV(5); NAPI_DB_CONTEXT(); - napi_value key = argv[1]; - napi_value value = argv[2]; + leveldb::Slice key = ToSlice(env, argv[1]); + leveldb::Slice value = ToSlice(env, argv[2]); bool sync = BooleanProperty(env, argv[3], "sync", false); napi_value callback = argv[4]; @@ -877,11 +877,11 @@ struct GetWorker : public BaseWorker { GetWorker (napi_env env, Database* database, napi_value callback, - napi_value key, + leveldb::Slice key, bool asBuffer, bool fillCache) : BaseWorker(env, database, callback, "leveldown.db.get"), - key_(ToSlice(env, key)), + key_(key), asBuffer_(asBuffer) { options_.fill_cache = fillCache; } @@ -922,7 +922,7 @@ NAPI_METHOD(db_get) { NAPI_ARGV(4); NAPI_DB_CONTEXT(); - napi_value key = argv[1]; + leveldb::Slice key = ToSlice(env, argv[1]); napi_value options = argv[2]; bool asBuffer = BooleanProperty(env, options, "asBuffer", true); bool fillCache = BooleanProperty(env, options, "fillCache", true); @@ -942,10 +942,10 @@ struct DelWorker : public BaseWorker { DelWorker (napi_env env, Database* database, napi_value callback, - napi_value key, + leveldb::Slice key, bool sync) : BaseWorker(env, database, callback, "leveldown.db.del"), - key_(ToSlice(env, key)) { + key_(key) { options_.sync = sync; } @@ -968,7 +968,7 @@ NAPI_METHOD(db_del) { NAPI_ARGV(4); NAPI_DB_CONTEXT(); - napi_value key = argv[1]; + leveldb::Slice key = ToSlice(env, argv[1]); bool sync = BooleanProperty(env, argv[2], "sync", false); napi_value callback = argv[3]; @@ -1023,6 +1023,7 @@ NAPI_METHOD(db_approximate_size) { leveldb::Slice start = ToSlice(env, argv[1]); leveldb::Slice end = ToSlice(env, argv[2]); + napi_value callback = argv[3]; ApproximateSizeWorker* worker = new ApproximateSizeWorker(env, database,