LMDB proof-of-concept implementation

.github/workflows/build-linux.yml

@@ -25,6 +25,14 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-konan-
 
+      - name: glibc install
+        run: |
+          sudo apt-get install glibc-source
+
+      - name: LMDB install
+        run: |
+          sudo apt-get install liblmdb-dev
+
       - name: Linux build
         run: |
           ./gradlew build publishToMavenLocal --no-daemon --stacktrace

convention-plugins/src/main/kotlin/standard-configuration-without-android.gradle.kts

@@ -17,6 +17,7 @@
 import org.gradle.accessors.dm.LibrariesForLibs
 import org.jetbrains.kotlin.gradle.ExperimentalKotlinGradlePluginApi
 import org.jetbrains.kotlin.gradle.dsl.JvmTarget
+import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinNativeTarget
 import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
 
 plugins {
@@ -40,6 +41,14 @@ kotlin {
 //                allWarningsAsErrors = true
 //            }
         }
+
+        if (this is KotlinNativeTarget && this.name.startsWith("linux")) {
+            binaries.configureEach {
+                // lmdb appears to be using a newer gcc than Kotlin. This lets us still work as long as host has newer gcc too
+                // https://stackoverflow.com/a/78267398
+                linkerOpts += "--allow-shlib-undefined"
+            }
+        }
     }
 
     @OptIn(ExperimentalKotlinGradlePluginApi::class)

multiplatform-settings/api/multiplatform-settings.klib.api

@@ -225,6 +225,38 @@ final class com.russhwolf.settings/StorageSettings : com.russhwolf.settings/Sett
     final fun remove(kotlin/String) // com.russhwolf.settings/StorageSettings.remove|remove(kotlin.String){}[0]
 }
 
+// Targets: [linuxX64]
+final class com.russhwolf.settings/LmdbSettings : com.russhwolf.settings/Settings { // com.russhwolf.settings/LmdbSettings|null[0]
+    constructor <init>(kotlin/String) // com.russhwolf.settings/LmdbSettings.<init>|<init>(kotlin.String){}[0]
+
+    final val keys // com.russhwolf.settings/LmdbSettings.keys|{}keys[0]
+        final fun <get-keys>(): kotlin.collections/Set<kotlin/String> // com.russhwolf.settings/LmdbSettings.keys.<get-keys>|<get-keys>(){}[0]
+    final val size // com.russhwolf.settings/LmdbSettings.size|{}size[0]
+        final fun <get-size>(): kotlin/Int // com.russhwolf.settings/LmdbSettings.size.<get-size>|<get-size>(){}[0]
+
+    final fun clear() // com.russhwolf.settings/LmdbSettings.clear|clear(){}[0]
+    final fun getBoolean(kotlin/String, kotlin/Boolean): kotlin/Boolean // com.russhwolf.settings/LmdbSettings.getBoolean|getBoolean(kotlin.String;kotlin.Boolean){}[0]
+    final fun getBooleanOrNull(kotlin/String): kotlin/Boolean? // com.russhwolf.settings/LmdbSettings.getBooleanOrNull|getBooleanOrNull(kotlin.String){}[0]
+    final fun getDouble(kotlin/String, kotlin/Double): kotlin/Double // com.russhwolf.settings/LmdbSettings.getDouble|getDouble(kotlin.String;kotlin.Double){}[0]
+    final fun getDoubleOrNull(kotlin/String): kotlin/Double? // com.russhwolf.settings/LmdbSettings.getDoubleOrNull|getDoubleOrNull(kotlin.String){}[0]
+    final fun getFloat(kotlin/String, kotlin/Float): kotlin/Float // com.russhwolf.settings/LmdbSettings.getFloat|getFloat(kotlin.String;kotlin.Float){}[0]
+    final fun getFloatOrNull(kotlin/String): kotlin/Float? // com.russhwolf.settings/LmdbSettings.getFloatOrNull|getFloatOrNull(kotlin.String){}[0]
+    final fun getInt(kotlin/String, kotlin/Int): kotlin/Int // com.russhwolf.settings/LmdbSettings.getInt|getInt(kotlin.String;kotlin.Int){}[0]
+    final fun getIntOrNull(kotlin/String): kotlin/Int? // com.russhwolf.settings/LmdbSettings.getIntOrNull|getIntOrNull(kotlin.String){}[0]
+    final fun getLong(kotlin/String, kotlin/Long): kotlin/Long // com.russhwolf.settings/LmdbSettings.getLong|getLong(kotlin.String;kotlin.Long){}[0]
+    final fun getLongOrNull(kotlin/String): kotlin/Long? // com.russhwolf.settings/LmdbSettings.getLongOrNull|getLongOrNull(kotlin.String){}[0]
+    final fun getString(kotlin/String, kotlin/String): kotlin/String // com.russhwolf.settings/LmdbSettings.getString|getString(kotlin.String;kotlin.String){}[0]
+    final fun getStringOrNull(kotlin/String): kotlin/String? // com.russhwolf.settings/LmdbSettings.getStringOrNull|getStringOrNull(kotlin.String){}[0]
+    final fun hasKey(kotlin/String): kotlin/Boolean // com.russhwolf.settings/LmdbSettings.hasKey|hasKey(kotlin.String){}[0]
+    final fun putBoolean(kotlin/String, kotlin/Boolean) // com.russhwolf.settings/LmdbSettings.putBoolean|putBoolean(kotlin.String;kotlin.Boolean){}[0]
+    final fun putDouble(kotlin/String, kotlin/Double) // com.russhwolf.settings/LmdbSettings.putDouble|putDouble(kotlin.String;kotlin.Double){}[0]
+    final fun putFloat(kotlin/String, kotlin/Float) // com.russhwolf.settings/LmdbSettings.putFloat|putFloat(kotlin.String;kotlin.Float){}[0]
+    final fun putInt(kotlin/String, kotlin/Int) // com.russhwolf.settings/LmdbSettings.putInt|putInt(kotlin.String;kotlin.Int){}[0]
+    final fun putLong(kotlin/String, kotlin/Long) // com.russhwolf.settings/LmdbSettings.putLong|putLong(kotlin.String;kotlin.Long){}[0]
+    final fun putString(kotlin/String, kotlin/String) // com.russhwolf.settings/LmdbSettings.putString|putString(kotlin.String;kotlin.String){}[0]
+    final fun remove(kotlin/String) // com.russhwolf.settings/LmdbSettings.remove|remove(kotlin.String){}[0]
+}
+
 // Targets: [mingwX64]
 final class com.russhwolf.settings/RegistrySettings : com.russhwolf.settings/Settings { // com.russhwolf.settings/RegistrySettings|null[0]
     constructor <init>(kotlin/String) // com.russhwolf.settings/RegistrySettings.<init>|<init>(kotlin.String){}[0]

multiplatform-settings/build.gradle.kts

@@ -1,6 +1,3 @@
-import org.jetbrains.kotlin.gradle.ExperimentalKotlinGradlePluginApi
-import org.jetbrains.kotlin.gradle.targets.js.dsl.ExperimentalWasmDsl
-
 /*
  * Copyright 2019 Russell Wolf
  *
@@ -17,6 +14,10 @@ import org.jetbrains.kotlin.gradle.targets.js.dsl.ExperimentalWasmDsl
  * limitations under the License.
  */
 
+import org.jetbrains.kotlin.gradle.ExperimentalKotlinGradlePluginApi
+import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinNativeTarget
+import org.jetbrains.kotlin.gradle.targets.js.dsl.ExperimentalWasmDsl
+
 plugins {
     id("standard-configuration")
     id("module-publication")
@@ -27,6 +28,9 @@ standardConfig {
 }
 
 kotlin {
+    targets.getByName<KotlinNativeTarget>("linuxX64") {
+        compilations["main"].cinterops.create("lmdb")
+    }
     @OptIn(ExperimentalKotlinGradlePluginApi::class)
     compilerOptions {
         freeCompilerArgs.add("-Xexpect-actual-classes")

multiplatform-settings/src/linuxX64Main/kotlin/com/russhwolf/settings/LmdbSettings.kt

@@ -0,0 +1,184 @@
+/*
+ * Copyright 2022 Russell Wolf
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.russhwolf.settings
+
+import cnames.structs.MDB_cursor
+import cnames.structs.MDB_env
+import cnames.structs.MDB_txn
+import com.russhwolf.settings.cinterop.lmdb.MDB_CREATE
+import com.russhwolf.settings.cinterop.lmdb.MDB_NOTFOUND
+import com.russhwolf.settings.cinterop.lmdb.MDB_SUCCESS
+import com.russhwolf.settings.cinterop.lmdb.MDB_cursor_op
+import com.russhwolf.settings.cinterop.lmdb.MDB_dbi
+import com.russhwolf.settings.cinterop.lmdb.MDB_dbiVar
+import com.russhwolf.settings.cinterop.lmdb.MDB_stat
+import com.russhwolf.settings.cinterop.lmdb.MDB_val
+import com.russhwolf.settings.cinterop.lmdb.mdb_cursor_close
+import com.russhwolf.settings.cinterop.lmdb.mdb_cursor_del
+import com.russhwolf.settings.cinterop.lmdb.mdb_cursor_get
+import com.russhwolf.settings.cinterop.lmdb.mdb_cursor_open
+import com.russhwolf.settings.cinterop.lmdb.mdb_dbi_close
+import com.russhwolf.settings.cinterop.lmdb.mdb_dbi_open
+import com.russhwolf.settings.cinterop.lmdb.mdb_del
+import com.russhwolf.settings.cinterop.lmdb.mdb_env_close
+import com.russhwolf.settings.cinterop.lmdb.mdb_env_create
+import com.russhwolf.settings.cinterop.lmdb.mdb_env_open
+import com.russhwolf.settings.cinterop.lmdb.mdb_get
+import com.russhwolf.settings.cinterop.lmdb.mdb_put
+import com.russhwolf.settings.cinterop.lmdb.mdb_stat
+import com.russhwolf.settings.cinterop.lmdb.mdb_strerror
+import com.russhwolf.settings.cinterop.lmdb.mdb_txn_begin
+import com.russhwolf.settings.cinterop.lmdb.mdb_txn_commit
+import kotlinx.cinterop.ByteVar
+import kotlinx.cinterop.CPointer
+import kotlinx.cinterop.ExperimentalForeignApi
+import kotlinx.cinterop.MemScope
+import kotlinx.cinterop.alloc
+import kotlinx.cinterop.allocPointerTo
+import kotlinx.cinterop.cstr
+import kotlinx.cinterop.memScoped
+import kotlinx.cinterop.ptr
+import kotlinx.cinterop.reinterpret
+import kotlinx.cinterop.toKString
+import kotlinx.cinterop.value
+import platform.posix.S_IRWXG
+import platform.posix.S_IRWXO
+import platform.posix.S_IRWXU
+import platform.posix.mkdir
+import kotlin.experimental.ExperimentalNativeApi
+
+@OptIn(ExperimentalForeignApi::class)
+@ExperimentalSettingsImplementation
+public class LmdbSettings(private val path: String) : Settings {
+    override val keys: Set<String>
+        get() = buildSet {
+            lmdbCursorTransaction { _, key ->
+                add(key.mv_data!!.reinterpret<ByteVar>().toKString())
+            }
+        }
+
+    override val size: Int
+        get() = lmdbTransaction { txn, dbi ->
+            val stat = alloc<MDB_stat>()
+            mdb_stat(txn, dbi, stat.ptr)
+            stat.ms_entries.toInt()
+        }
+
+    public override fun clear(): Unit = lmdbCursorTransaction { cursor, _ ->
+        mdb_cursor_del(cursor, 0.toUInt())
+    }
+
+    public override fun remove(key: String): Unit =
+        lmdbTransaction { txn, dbi -> mdb_del(txn, dbi, createMdbVal(key).ptr, null).checkError(MDB_NOTFOUND) }
+
+    public override fun hasKey(key: String): Boolean = loadString(key) != null
+
+    public override fun putInt(key: String, value: Int): Unit = saveString(key, value.toString())
+    public override fun getInt(key: String, defaultValue: Int): Int = getIntOrNull(key) ?: defaultValue
+    public override fun getIntOrNull(key: String): Int? = loadString(key)?.toInt()
+
+    public override fun putLong(key: String, value: Long): Unit = saveString(key, value.toString())
+    public override fun getLong(key: String, defaultValue: Long): Long = getLongOrNull(key) ?: defaultValue
+    public override fun getLongOrNull(key: String): Long? = loadString(key)?.toLong()
+
+    public override fun putString(key: String, value: String): Unit = saveString(key, value)
+    public override fun getString(key: String, defaultValue: String): String = getStringOrNull(key) ?: defaultValue
+    public override fun getStringOrNull(key: String): String? = loadString(key)
+
+    public override fun putFloat(key: String, value: Float): Unit = saveString(key, value.toString())
+    public override fun getFloat(key: String, defaultValue: Float): Float = getFloatOrNull(key) ?: defaultValue
+    public override fun getFloatOrNull(key: String): Float? = loadString(key)?.toFloat()
+
+    public override fun putDouble(key: String, value: Double): Unit = saveString(key, value.toString())
+    public override fun getDouble(key: String, defaultValue: Double): Double = getDoubleOrNull(key) ?: defaultValue
+    public override fun getDoubleOrNull(key: String): Double? = loadString(key)?.toDouble()
+
+    public override fun putBoolean(key: String, value: Boolean): Unit = saveString(key, value.toString())
+    public override fun getBoolean(key: String, defaultValue: Boolean): Boolean = getBooleanOrNull(key) ?: defaultValue
+    public override fun getBooleanOrNull(key: String): Boolean? = loadString(key)?.toBoolean()
+
+    private fun saveString(key: String, value: String): Unit = lmdbTransaction { txn, dbi ->
+        mdb_put(txn, dbi, createMdbVal(key).ptr, createMdbVal(value).ptr, 0.toUInt()).checkError()
+    }
+
+    private fun loadString(key: String): String? = lmdbTransaction { txn, dbi ->
+        val value = alloc<MDB_val>()
+        mdb_get(txn, dbi, createMdbVal(key).ptr, value.ptr).checkError(MDB_NOTFOUND)
+        value.mv_data?.reinterpret<ByteVar>()?.toKString()
+    }
+
+    private fun lmdbCursorTransaction(action: MemScope.(cursor: CPointer<MDB_cursor>?, key: MDB_val) -> Unit) =
+        lmdbTransaction { txn, dbi ->
+            val cursor = allocPointerTo<MDB_cursor>()
+            mdb_cursor_open(txn, dbi, cursor.ptr).checkError()
+            val mdbKey = alloc<MDB_val>()
+            val mdbValue = alloc<MDB_val>()
+            mdb_cursor_get(cursor.value, mdbKey.ptr, mdbValue.ptr, MDB_cursor_op.MDB_FIRST).checkError(
+                MDB_NOTFOUND
+            )
+            while (true) {
+                if (mdbKey.mv_data == null) {
+                    break
+                } else {
+                    action(cursor.value, mdbKey)
+                }
+                mdbKey.mv_data = null
+                mdbKey.mv_size = 0u
+                mdb_cursor_get(cursor.value, mdbKey.ptr, mdbValue.ptr, MDB_cursor_op.MDB_NEXT).checkError(
+                    MDB_NOTFOUND
+                )
+            }
+            mdb_cursor_close(cursor.value)
+
+        }
+
+    private fun MemScope.createMdbVal(string: String): MDB_val {
+        val mdbVal = alloc<MDB_val>()
+        val cstr = string.cstr
+        mdbVal.mv_data = cstr.ptr
+        mdbVal.mv_size = cstr.size.toULong()
+        return mdbVal
+    }
+
+    private inline fun <T> lmdbTransaction(action: MemScope.(txn: CPointer<MDB_txn>?, dbi: MDB_dbi) -> T): T =
+        memScoped {
+            val env = allocPointerTo<MDB_env>()
+            mdb_env_create(env.ptr).checkError()
+            val mode = (S_IRWXU or S_IRWXG or S_IRWXO).toUInt()
+            mkdir(path, mode)
+            mdb_env_open(env.value, path, 0.toUInt(), mode).checkError()
+
+            val txn = allocPointerTo<MDB_txn>()
+            mdb_txn_begin(env.value, null, 0.toUInt(), txn.ptr).checkError()
+
+            val dbi = alloc<MDB_dbiVar>()
+            mdb_dbi_open(txn.value, null, MDB_CREATE.toUInt(), dbi.ptr).checkError()
+
+            val out = action(txn.value, dbi.value)
+
+            mdb_dbi_close(env.value, dbi.value)
+            mdb_txn_commit(txn.value).checkError()
+            mdb_env_close(env.value)
+
+            out
+        }
+
+    private fun Int.checkError(vararg expectedErrors: Int) {
+        @OptIn(ExperimentalNativeApi::class)
+        assert(this == MDB_SUCCESS || this in expectedErrors) { "Error: ${mdb_strerror(this)?.toKString()}" }
+    }
+}

multiplatform-settings/src/linuxX64Test/kotlin/com/russhwolf/settings/LmdbSettingsTest.kt

@@ -0,0 +1,30 @@
+/*
+ * Copyright 2022 Russell Wolf
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.russhwolf.settings
+
+@OptIn(ExperimentalSettingsImplementation::class)
+public class LmdbSettingsTest
+    : BaseSettingsTest(
+    platformFactory = object : Settings.Factory {
+        override fun create(name: String?): Settings = LmdbSettings(name ?: "lmdbTest")
+    },
+    hasListeners = false
+) {
+    // TODO add test cases to verify that we write to the files we think we do
+
+    // TODO add cleanup methods so we don't leave test DBs lying around
+}

multiplatform-settings/src/nativeInterop/cinterop/lmdb.def

@@ -0,0 +1,4 @@
+headers = lmdb.h
+package = com.russhwolf.settings.cinterop.lmdb
+compilerOpts = -I/usr/include -I/usr/include/x86_64-linux-gnu/
+linkerOpts = -llmdb  -L/usr/lib -L/usr/lib/x86_64-linux-gnu/