diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ba0f28aa..82200a56 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -420,12 +420,9 @@ jobs: env: TURBO_CACHE_DIR: .turbo/android steps: - - name: Checkout - uses: actions/checkout@v4 + - uses: actions/checkout@v4 - - name: Turn on libsql - run: | - node ./scripts/turnOnLibsql.js + - run: node ./scripts/turnOnLibsql.js - name: Setup uses: ./.github/actions/setup diff --git a/.gitignore b/.gitignore index ed9f7eda..1db2b1aa 100644 --- a/.gitignore +++ b/.gitignore @@ -81,4 +81,6 @@ android/gradle/ !.yarn/plugins !.yarn/releases !.yarn/sdks -!.yarn/versions \ No newline at end of file +!.yarn/versions + +android/c_sources \ No newline at end of file diff --git a/Gemfile b/Gemfile deleted file mode 100644 index f963de13..00000000 --- a/Gemfile +++ /dev/null @@ -1,5 +0,0 @@ -source 'https://rubygems.org' -# You may use http://rbenv.org/ or https://rvm.io/ to install and use this version -ruby '>= 2.7.6' -gem 'cocoapods', '>= 1.13', '!= 1.15.0', '!= 1.15.1' -gem 'activesupport', '>= 6.1.7.5', '!= 7.1.0' \ No newline at end of file diff --git a/android/CMakeLists.txt b/android/CMakeLists.txt index 5ff0655d..8df9623a 100644 --- a/android/CMakeLists.txt +++ b/android/CMakeLists.txt @@ -10,6 +10,7 @@ set (BUILD_DIR ${CMAKE_SOURCE_DIR}/build) ../cpp ../cpp/sqlcipher ../cpp/libsql +# ../example/c_sources ) add_definitions( @@ -72,6 +73,13 @@ find_package(ReactAndroid REQUIRED CONFIG) find_package(fbjni REQUIRED CONFIG) find_library(LOG_LIB log) +# Add user defined files +if (USER_DEFINED_SOURCE_FILES) + target_sources(${PACKAGE_NAME} PRIVATE ${USER_DEFINED_SOURCE_FILES}) + + add_definitions("-DTOKENIZERS_HEADER_PATH=\"${USER_DEFINED_TOKENIZERS_HEADER_PATH}\"") +endif() + if (USE_SQLCIPHER) if (ReactAndroid_VERSION_MINOR GREATER_EQUAL 76) target_link_libraries( diff --git a/android/build.gradle b/android/build.gradle index 4d5b8338..632d76f1 100644 --- a/android/build.gradle +++ b/android/build.gradle @@ -35,9 +35,18 @@ def sqliteFlags = "" def enableFTS5 = false def useSqliteVec = false def enableRtree = false +def tokenizers = [] -def packageJsonFile = new File("$rootDir/../package.json") -def packageJson = new JsonSlurper().parseText(packageJsonFile.text) +def isInsideNodeModules = rootDir.absolutePath.contains("node_modules") +def packageJson + +if ( isInsideNodeModules ) { + def packageJsonFile = new File("$rootDir/../../../package.json") + packageJson = new JsonSlurper().parseText(packageJsonFile.text) +} else { + def packageJsonFile = new File("$rootDir/../package.json") + packageJson = new JsonSlurper().parseText(packageJsonFile.text) +} def opsqliteConfig = packageJson["op-sqlite"] if(opsqliteConfig) { @@ -49,6 +58,7 @@ if(opsqliteConfig) { enableFTS5 = opsqliteConfig["fts5"] useLibsql = opsqliteConfig["libsql"] enableRtree = opsqliteConfig["rtree"] + tokenizers = opsqliteConfig["tokenizers"] ? opsqliteConfig["tokenizers"] : [] } if(useSQLCipher) { @@ -83,6 +93,11 @@ if(useSqliteVec) { println "[OP-SQLITE] Sqlite Vec enabled! ↗️" } + +if (!tokenizers.isEmpty()) { + println "[OP-SQLITE] Tokenizers enabled! 🧾 Tokenizers: " + tokenizers +} + if (isNewArchitectureEnabled()) { apply plugin: "com.facebook.react" } @@ -153,14 +168,32 @@ android { cppFlags += "-DOP_SQLITE_USE_SQLITE_VEC=1" } - cppFlags "-O2", "-fexceptions", "-frtti", "-std=c++1y", "-DONANDROID" + // This are zeroes because they will be passed as C flags, so they become falsy + def sourceFiles = 0 + // def tokenizerInitStrings = 0 + def tokenizersHeaderPath = 0 + if (!tokenizers.isEmpty()) { + def sourceDir = isInsideNodeModules ? file("$rootDir/../../../c_sources") : file("$rootDir/../c_sources") + def destDir = file("$buildscript.sourceFile.parentFile/c_sources") + copy { + from sourceDir + into destDir + include "**/*.cpp", "**/*.h" + } + sourceFiles = fileTree(dir: destDir, include: ["**/*.cpp", "**/*.h"]).files.join(";") + tokenizersHeaderPath = "../c_sources/tokenizers.h" + } + + cppFlags "-O2", "-fexceptions", "-DONANDROID" abiFilters 'x86', 'x86_64', 'armeabi-v7a', 'arm64-v8a' arguments "-DANDROID_STL=c++_shared", "-DSQLITE_FLAGS='$sqliteFlags'", "-DUSE_SQLCIPHER=${useSQLCipher ? 1 : 0}", "-DUSE_CRSQLITE=${useCRSQLite ? 1 : 0}", "-DUSE_LIBSQL=${useLibsql ? 1 : 0}", - "-DUSE_SQLITE_VEC=${useSqliteVec ? 1 : 0}" + "-DUSE_SQLITE_VEC=${useSqliteVec ? 1 : 0}", + "-DUSER_DEFINED_SOURCE_FILES=${sourceFiles}", + "-DUSER_DEFINED_TOKENIZERS_HEADER_PATH='${tokenizersHeaderPath}'" } } diff --git a/c_sources/tokenizers.cpp b/c_sources/tokenizers.cpp new file mode 100644 index 00000000..5b0a977d --- /dev/null +++ b/c_sources/tokenizers.cpp @@ -0,0 +1,88 @@ +#include "tokenizers.h" +#include +#include +#include + +namespace opsqlite { + +fts5_api *fts5_api_from_db(sqlite3 *db) { + fts5_api *pRet = 0; + sqlite3_stmt *pStmt = 0; + + if (SQLITE_OK == sqlite3_prepare_v2(db, "SELECT fts5(?1)", -1, &pStmt, 0)) { + sqlite3_bind_pointer(pStmt, 1, (void *)&pRet, "fts5_api_ptr", NULL); + sqlite3_step(pStmt); + } + sqlite3_finalize(pStmt); + return pRet; +} + +class WordTokenizer { +public: + WordTokenizer() = default; + ~WordTokenizer() = default; +}; + +// Define `xCreate`, which initializes the tokenizer +int wordTokenizerCreate(void *pUnused, const char **azArg, int nArg, + Fts5Tokenizer **ppOut) { + auto tokenizer = std::make_unique(); + *ppOut = reinterpret_cast( + tokenizer.release()); // Cast to Fts5Tokenizer* + return SQLITE_OK; +} + +// Define `xDelete`, which frees the tokenizer +void wordTokenizerDelete(Fts5Tokenizer *pTokenizer) { + delete reinterpret_cast(pTokenizer); +} + +// Define `xTokenize`, which performs the actual tokenization +int wordTokenizerTokenize(Fts5Tokenizer *pTokenizer, void *pCtx, int flags, + const char *pText, int nText, + int (*xToken)(void *, int, const char *, int, int, + int)) { + int start = 0; + int i = 0; + + while (i <= nText) { + if (i == nText || !std::isalnum(static_cast(pText[i]))) { + if (start < i) { // Found a token + int rc = xToken(pCtx, 0, pText + start, i - start, start, i); + if (rc != SQLITE_OK) + return rc; + } + start = i + 1; + } + i++; + } + return SQLITE_OK; +} + +int opsqlite_wordtokenizer_init(sqlite3 *db, char **error, + sqlite3_api_routines const *api) { + fts5_tokenizer wordtokenizer = {wordTokenizerCreate, wordTokenizerDelete, + wordTokenizerTokenize}; + + fts5_api *ftsApi = (fts5_api *)fts5_api_from_db(db); + if (ftsApi == NULL) + return SQLITE_ERROR; + + return ftsApi->xCreateTokenizer(ftsApi, "wordtokenizer", NULL, &wordtokenizer, + NULL); +} + +int opsqlite_porter_init(sqlite3 *db, char **error, + sqlite3_api_routines const *api) { + fts5_tokenizer porter_tokenizer = {wordTokenizerCreate, wordTokenizerDelete, + wordTokenizerTokenize}; + + fts5_api *ftsApi = (fts5_api *)fts5_api_from_db(db); + if (ftsApi == nullptr) + return SQLITE_ERROR; + + return ftsApi->xCreateTokenizer(ftsApi, "portertokenizer", NULL, + &porter_tokenizer, NULL); +} + +} // namespace opsqlite diff --git a/c_sources/tokenizers.h b/c_sources/tokenizers.h new file mode 100644 index 00000000..ccbf95be --- /dev/null +++ b/c_sources/tokenizers.h @@ -0,0 +1,15 @@ +#ifndef TOKENIZERS_H +#define TOKENIZERS_H + +#define TOKENIZER_LIST opsqlite_wordtokenizer_init(db,&errMsg,nullptr);opsqlite_porter_init(db,&errMsg,nullptr); + +#include "sqlite3.h" + +namespace opsqlite { + +int opsqlite_wordtokenizer_init(sqlite3 *db, char **error, sqlite3_api_routines const *api); +int opsqlite_porter_init(sqlite3 *db, char **error, sqlite3_api_routines const *api); + +} // namespace opsqlite + +#endif // TOKENIZERS_H diff --git a/cpp/bindings.cpp b/cpp/bindings.cpp index 1aa502ce..168998e0 100644 --- a/cpp/bindings.cpp +++ b/cpp/bindings.cpp @@ -46,7 +46,8 @@ void clearState() { thread_pool->restartPool(); } -void install(jsi::Runtime &rt, const std::shared_ptr& invoker, +void install(jsi::Runtime &rt, + const std::shared_ptr &invoker, const char *base_path, const char *crsqlite_path, const char *sqlite_vec_path) { invalidated = false; diff --git a/cpp/bridge.cpp b/cpp/bridge.cpp index 65e07efc..c4da7657 100644 --- a/cpp/bridge.cpp +++ b/cpp/bridge.cpp @@ -4,9 +4,16 @@ #include "logs.h" #include "utils.h" #include +#include #include #include +#ifdef TOKENIZERS_HEADER_PATH +#include TOKENIZERS_HEADER_PATH +#else +#define TOKENIZER_LIST +#endif + namespace opsqlite { /// Maps to hold the different objects @@ -109,9 +116,10 @@ BridgeResult opsqlite_open(std::string const &name, if (errMsg != nullptr) { return {.type = SQLiteError, .message = errMsg}; } - #endif + TOKENIZER_LIST + return {.type = SQLiteOk, .affectedRows = 0}; } diff --git a/cpp/utils.cpp b/cpp/utils.cpp index c1ee3e71..79f0ee80 100644 --- a/cpp/utils.cpp +++ b/cpp/utils.cpp @@ -341,4 +341,14 @@ int mkdir(std::string const &path) { return 0; } +std::vector parse_string_list(const std::string& str) { + std::vector result; + std::istringstream stream(str); + std::string token; + while (std::getline(stream, token, ',')) { + result.push_back(token); + } + return result; +} + } // namespace opsqlite diff --git a/cpp/utils.h b/cpp/utils.h index 2950d9fb..cd2383b5 100644 --- a/cpp/utils.h +++ b/cpp/utils.h @@ -9,20 +9,28 @@ #include #include #include +#include namespace opsqlite { namespace jsi = facebook::jsi; jsi::Value toJSI(jsi::Runtime &rt, const JSVariant &value); + JSVariant toVariant(jsi::Runtime &rt, jsi::Value const &value); + std::vector to_string_vec(jsi::Runtime &rt, jsi::Value const &xs); + std::vector to_variant_vec(jsi::Runtime &rt, jsi::Value const &xs); + std::vector to_int_vec(jsi::Runtime &rt, jsi::Value const &xs); + jsi::Value createResult(jsi::Runtime &rt, BridgeResult status, std::vector *results, std::shared_ptr> metadata); + jsi::Value create_js_rows(jsi::Runtime &rt, const BridgeResult &status); + jsi::Value create_raw_result(jsi::Runtime &rt, BridgeResult status, const std::vector> *results); @@ -38,6 +46,8 @@ bool folder_exists(const std::string &foldername); bool file_exists(const std::string &path); +std::vector parse_string_list(const std::string& str); + } // namespace opsqlite #endif /* utils_h */ diff --git a/example/Gemfile b/example/Gemfile index 5f7f7e4e..bc0f3637 100644 --- a/example/Gemfile +++ b/example/Gemfile @@ -2,5 +2,7 @@ source 'https://rubygems.org' ruby '>= 2.7.6' -gem 'cocoapods', '>= 1.13', '!= 1.15.0', '!= 1.15.1' -gem 'activesupport', '>= 6.1.7.5', '!= 7.1.0' \ No newline at end of file +gem 'cocoapods', '=1.15.2' +gem 'activesupport', '>= 6.1.7.5', '!= 7.1.0' +gem 'bigdecimal' +gem 'mutex_m' \ No newline at end of file diff --git a/example/Gemfile.lock b/example/Gemfile.lock index d7b8e7d5..3a7d5735 100644 --- a/example/Gemfile.lock +++ b/example/Gemfile.lock @@ -10,18 +10,19 @@ GEM i18n (>= 1.6, < 2) minitest (>= 5.1) tzinfo (~> 2.0) - addressable (2.8.6) - public_suffix (>= 2.0.2, < 6.0) + addressable (2.8.7) + public_suffix (>= 2.0.2, < 7.0) algoliasearch (1.27.5) httpclient (~> 2.8, >= 2.8.3) json (>= 1.5.1) atomos (0.1.3) base64 (0.2.0) + bigdecimal (3.1.5) claide (1.1.0) - cocoapods (1.14.3) + cocoapods (1.15.2) addressable (~> 2.8) claide (>= 1.0.2, < 2.0) - cocoapods-core (= 1.14.3) + cocoapods-core (= 1.15.2) cocoapods-deintegrate (>= 1.0.3, < 2.0) cocoapods-downloader (>= 2.1, < 3.0) cocoapods-plugins (>= 1.0.0, < 2.0) @@ -36,7 +37,7 @@ GEM nap (~> 1.0) ruby-macho (>= 2.3.0, < 3.0) xcodeproj (>= 1.23.0, < 2.0) - cocoapods-core (1.14.3) + cocoapods-core (1.15.2) activesupport (>= 5.0, < 8) addressable (~> 2.8) algoliasearch (~> 1.0) @@ -60,41 +61,44 @@ GEM escape (0.0.4) ethon (0.16.0) ffi (>= 1.15.0) - ffi (1.16.3) + ffi (1.17.0) fourflusher (2.3.1) fuzzy_match (2.0.4) gh_inspector (1.1.3) httpclient (2.8.3) i18n (1.14.4) concurrent-ruby (~> 1.0) - json (2.7.2) + json (2.7.5) minitest (5.22.3) molinillo (0.8.0) - nanaimo (0.3.0) + mutex_m (0.2.0) + nanaimo (0.4.0) nap (1.1.0) netrc (0.11.0) nkf (0.2.0) public_suffix (4.0.7) - rexml (3.2.6) + rexml (3.3.9) ruby-macho (2.5.1) typhoeus (1.4.1) ethon (>= 0.9.0) tzinfo (2.0.6) concurrent-ruby (~> 1.0) - xcodeproj (1.24.0) + xcodeproj (1.27.0) CFPropertyList (>= 2.3.3, < 4.0) atomos (~> 0.1.3) claide (>= 1.0.2, < 2.0) colored2 (~> 3.1) - nanaimo (~> 0.3.0) - rexml (~> 3.2.4) + nanaimo (~> 0.4.0) + rexml (>= 3.3.6, < 4.0) PLATFORMS ruby DEPENDENCIES activesupport (>= 6.1.7.5, != 7.1.0) - cocoapods (>= 1.13, != 1.15.1, != 1.15.0) + bigdecimal + cocoapods (= 1.15.2) + mutex_m RUBY VERSION ruby 3.3.1p55 diff --git a/example/c_sources/tokenizers.cpp b/example/c_sources/tokenizers.cpp new file mode 100644 index 00000000..5b0a977d --- /dev/null +++ b/example/c_sources/tokenizers.cpp @@ -0,0 +1,88 @@ +#include "tokenizers.h" +#include +#include +#include + +namespace opsqlite { + +fts5_api *fts5_api_from_db(sqlite3 *db) { + fts5_api *pRet = 0; + sqlite3_stmt *pStmt = 0; + + if (SQLITE_OK == sqlite3_prepare_v2(db, "SELECT fts5(?1)", -1, &pStmt, 0)) { + sqlite3_bind_pointer(pStmt, 1, (void *)&pRet, "fts5_api_ptr", NULL); + sqlite3_step(pStmt); + } + sqlite3_finalize(pStmt); + return pRet; +} + +class WordTokenizer { +public: + WordTokenizer() = default; + ~WordTokenizer() = default; +}; + +// Define `xCreate`, which initializes the tokenizer +int wordTokenizerCreate(void *pUnused, const char **azArg, int nArg, + Fts5Tokenizer **ppOut) { + auto tokenizer = std::make_unique(); + *ppOut = reinterpret_cast( + tokenizer.release()); // Cast to Fts5Tokenizer* + return SQLITE_OK; +} + +// Define `xDelete`, which frees the tokenizer +void wordTokenizerDelete(Fts5Tokenizer *pTokenizer) { + delete reinterpret_cast(pTokenizer); +} + +// Define `xTokenize`, which performs the actual tokenization +int wordTokenizerTokenize(Fts5Tokenizer *pTokenizer, void *pCtx, int flags, + const char *pText, int nText, + int (*xToken)(void *, int, const char *, int, int, + int)) { + int start = 0; + int i = 0; + + while (i <= nText) { + if (i == nText || !std::isalnum(static_cast(pText[i]))) { + if (start < i) { // Found a token + int rc = xToken(pCtx, 0, pText + start, i - start, start, i); + if (rc != SQLITE_OK) + return rc; + } + start = i + 1; + } + i++; + } + return SQLITE_OK; +} + +int opsqlite_wordtokenizer_init(sqlite3 *db, char **error, + sqlite3_api_routines const *api) { + fts5_tokenizer wordtokenizer = {wordTokenizerCreate, wordTokenizerDelete, + wordTokenizerTokenize}; + + fts5_api *ftsApi = (fts5_api *)fts5_api_from_db(db); + if (ftsApi == NULL) + return SQLITE_ERROR; + + return ftsApi->xCreateTokenizer(ftsApi, "wordtokenizer", NULL, &wordtokenizer, + NULL); +} + +int opsqlite_porter_init(sqlite3 *db, char **error, + sqlite3_api_routines const *api) { + fts5_tokenizer porter_tokenizer = {wordTokenizerCreate, wordTokenizerDelete, + wordTokenizerTokenize}; + + fts5_api *ftsApi = (fts5_api *)fts5_api_from_db(db); + if (ftsApi == nullptr) + return SQLITE_ERROR; + + return ftsApi->xCreateTokenizer(ftsApi, "portertokenizer", NULL, + &porter_tokenizer, NULL); +} + +} // namespace opsqlite diff --git a/example/c_sources/tokenizers.h b/example/c_sources/tokenizers.h new file mode 100644 index 00000000..ccbf95be --- /dev/null +++ b/example/c_sources/tokenizers.h @@ -0,0 +1,15 @@ +#ifndef TOKENIZERS_H +#define TOKENIZERS_H + +#define TOKENIZER_LIST opsqlite_wordtokenizer_init(db,&errMsg,nullptr);opsqlite_porter_init(db,&errMsg,nullptr); + +#include "sqlite3.h" + +namespace opsqlite { + +int opsqlite_wordtokenizer_init(sqlite3 *db, char **error, sqlite3_api_routines const *api); +int opsqlite_porter_init(sqlite3 *db, char **error, sqlite3_api_routines const *api); + +} // namespace opsqlite + +#endif // TOKENIZERS_H diff --git a/example/ios/Podfile.lock b/example/ios/Podfile.lock index 899770f3..3a80ed7e 100644 --- a/example/ios/Podfile.lock +++ b/example/ios/Podfile.lock @@ -10,7 +10,7 @@ PODS: - hermes-engine (0.76.1): - hermes-engine/Pre-built (= 0.76.1) - hermes-engine/Pre-built (0.76.1) - - op-sqlite (9.3.0): + - op-sqlite (10.0.0-tokenizers-beta9): - DoubleConversion - glog - hermes-engine @@ -1540,7 +1540,7 @@ PODS: - React-logger (= 0.76.1) - React-perflogger (= 0.76.1) - React-utils (= 0.76.1) - - RNShare (11.0.3): + - RNShare (11.0.4): - DoubleConversion - glog - hermes-engine @@ -1785,7 +1785,7 @@ SPEC CHECKSUMS: GCDWebServer: 2c156a56c8226e2d5c0c3f208a3621ccffbe3ce4 glog: 08b301085f15bcbb6ff8632a8ebaf239aae04e6a hermes-engine: 46f1ffbf0297f4298862068dd4c274d4ac17a1fd - op-sqlite: 1d96ea1e6fcab14883b7315d4975374ff91eb899 + op-sqlite: 63400939931ca67186f99f8b536a02b048133710 RCT-Folly: bf5c0376ffe4dd2cf438dcf86db385df9fdce648 RCTDeprecation: fde92935b3caa6cb65cbff9fbb7d3a9867ffb259 RCTRequired: 75c6cee42d21c1530a6f204ba32ff57335d19007 @@ -1844,7 +1844,7 @@ SPEC CHECKSUMS: React-utils: 5362bd16a9563f9916e7a56c011ddc533507650f ReactCodegen: 865bafc5c17ec2181620ced1a32c39c38ab2951d ReactCommon: 422e364463f33e336fc4db196aeb50fd801d90d6 - RNShare: e1721a8818a3bf111ed686ed5d8c1dc76b91c8ad + RNShare: 4305edead1b8f614ab994046c68193e8d50aaadc SocketRocket: d4aabe649be1e368d1318fdf28a022d714d65748 Yoga: db69236006b8b1c6d55ab453390c882306cbf219 diff --git a/example/package.json b/example/package.json index 3ffc9adb..42cf6b87 100644 --- a/example/package.json +++ b/example/package.json @@ -69,10 +69,15 @@ "sqlcipher": false, "crsqlite": false, "performanceMode": "2", + "sqliteFlags": "-DSQLITE_TEMP_STORE=2", "iosSqlite": false, "fts5": true, "rtree": true, "libsql": false, - "sqliteVec": true + "sqliteVec": true, + "tokenizers": [ + "wordtokenizer", + "porter" + ] } } diff --git a/example/src/App.tsx b/example/src/App.tsx index 77d84b4f..0f510ef3 100644 --- a/example/src/App.tsx +++ b/example/src/App.tsx @@ -1,3 +1,4 @@ +import {open} from '@op-engineering/op-sqlite'; import clsx from 'clsx'; import {useEffect, useState} from 'react'; import { @@ -8,17 +9,22 @@ import { Text, View, } from 'react-native'; +import RNRestart from 'react-native-restart'; +import Share from 'react-native-share'; import 'reflect-metadata'; +import {createLargeDB, queryLargeDB} from './Database'; +import { + setServerError, + setServerResults, + startServer, + stopServer, +} from './server'; import {constantsTests} from './tests/constants.spec'; import {registerHooksTests} from './tests/hooks.spec'; import {blobTests, dbSetupTests, queriesTests, runTests} from './tests/index'; import {preparedStatementsTests} from './tests/preparedStatements.spec'; import {reactiveTests} from './tests/reactive.spec'; -import {setServerResults, startServer, stopServer} from './server'; -import {open} from '@op-engineering/op-sqlite'; -import Share from 'react-native-share'; -import {createLargeDB, queryLargeDB} from './Database'; -import RNRestart from 'react-native-restart'; +import {tokenizerTests} from './tests/tokenizer.spec'; export default function App() { const [times, setTimes] = useState([]); @@ -38,10 +44,15 @@ export default function App() { preparedStatementsTests, constantsTests, reactiveTests, - ).then(results => { - setServerResults(results as any); - setResults(results); - }); + tokenizerTests, + ) + .then(results => { + setServerResults(results as any); + setResults(results); + }) + .catch(e => { + setServerError(e); + }); startServer(); diff --git a/example/src/server.ts b/example/src/server.ts index 5adf82b4..c5451896 100644 --- a/example/src/server.ts +++ b/example/src/server.ts @@ -4,7 +4,6 @@ let results: any[] = []; const server = new BridgeServer('http_service', true); server.get('/ping', async (_req, _res) => { - // console.log("🟦 🟦🟦🟦🟦🟦🟦 🟦 Received request for '/ping'"); return {message: 'pong'}; }); @@ -23,5 +22,11 @@ export function stopServer() { } export function setServerResults(r: any[]) { + console.log('Setting server results'); results = r; } + +export function setServerError(e: any) { + console.log('Setting server error'); + results = e; +} diff --git a/example/src/tests/tokenizer.spec.ts b/example/src/tests/tokenizer.spec.ts new file mode 100644 index 00000000..21534a4e --- /dev/null +++ b/example/src/tests/tokenizer.spec.ts @@ -0,0 +1,47 @@ +import {isLibsql, open, type DB} from '@op-engineering/op-sqlite'; +import chai from 'chai'; +import {afterEach, beforeEach, describe, it} from './MochaRNAdapter'; + +const expect = chai.expect; + +export function tokenizerTests() { + let db: DB; + + describe('Tokenizer tests', () => { + beforeEach(async () => { + db = open({ + name: 'tokenizers.sqlite', + encryptionKey: 'test', + }); + + if (!isLibsql()) { + await db.execute( + `CREATE VIRTUAL TABLE tokenizer_table USING fts5(content, tokenize = 'wordtokenizer');`, + ); + } + }); + + afterEach(() => { + if (db) { + db.close(); + db.delete(); + // @ts-ignore + db = null; + } + }); + + if (!isLibsql()) { + it('Should match the word split by the tokenizer', async () => { + await db.execute('INSERT INTO tokenizer_table(content) VALUES (?)', [ + 'This is a test document', + ]); + const res = await db.execute( + 'SELECT content FROM tokenizer_table WHERE content MATCH ?', + ['test'], + ); + expect(res.rows.length).to.be.equal(1); + expect(res.rows[0]!.content).to.be.equal('This is a test document'); + }); + } + }); +} diff --git a/generate_tokenizers_header_file.rb b/generate_tokenizers_header_file.rb new file mode 100644 index 00000000..9e34afd4 --- /dev/null +++ b/generate_tokenizers_header_file.rb @@ -0,0 +1,29 @@ +require 'fileutils' + +def generate_tokenizers_header_file(names, file_path) + # Ensure the directory exists + dir_path = File.dirname(file_path) + FileUtils.mkdir_p(dir_path) unless Dir.exist?(dir_path) + tokenizer_list = names.map { |name| "opsqlite_#{name}_init(db,&errMsg,nullptr);" }.join + + File.open(file_path, 'w') do |file| + file.puts "#ifndef TOKENIZERS_H" + file.puts "#define TOKENIZERS_H" + file.puts + file.puts "#define TOKENIZER_LIST #{tokenizer_list}" + file.puts + file.puts "#include \"sqlite3.h\"" + file.puts + file.puts "namespace opsqlite {" + file.puts + + names.each do |name| + file.puts "int opsqlite_#{name}_init(sqlite3 *db, char **error, sqlite3_api_routines const *api);" + end + + file.puts + file.puts "} // namespace opsqlite" + file.puts + file.puts "#endif // TOKENIZERS_H" + end +end \ No newline at end of file diff --git a/op-sqlite.podspec b/op-sqlite.podspec index d14e7d93..6b45dfff 100644 --- a/op-sqlite.podspec +++ b/op-sqlite.podspec @@ -1,9 +1,11 @@ require "json" +require_relative "./generate_tokenizers_header_file" log_message = lambda do |message| puts "\e[34m#{message}\e[0m" end +is_user_app = __dir__.include?("node_modules") package = JSON.parse(File.read(File.join(__dir__, "package.json"))) folly_compiler_flags = '-DFOLLY_NO_CONFIG -DFOLLY_MOBILE=1 -DFOLLY_USE_LIBCPP=1 -Wno-comma -Wno-shorten-64-to-32' fabric_enabled = ENV['RCT_NEW_ARCH_ENABLED'] == '1' @@ -11,7 +13,7 @@ fabric_enabled = ENV['RCT_NEW_ARCH_ENABLED'] == '1' parent_folder_name = File.basename(__dir__) app_package = nil # When installed on user node_modules lives inside node_modules/@op-engineering/op-sqlite -if __dir__.include?("node_modules") +if is_user_app app_package = JSON.parse(File.read(File.join(__dir__, "..", "..", "..", "package.json"))) # When running on the example app else @@ -28,6 +30,7 @@ sqlite_flags = "" fts5 = false rtree = false use_sqlite_vec = false +tokenizers = [] if(op_sqlite_config != nil) use_sqlcipher = op_sqlite_config["sqlcipher"] == true @@ -39,6 +42,7 @@ if(op_sqlite_config != nil) fts5 = op_sqlite_config["fts5"] == true rtree = op_sqlite_config["rtree"] == true use_sqlite_vec = op_sqlite_config["sqliteVec"] == true + tokenizers = op_sqlite_config["tokenizers"] || [] end if phone_version then @@ -74,12 +78,37 @@ Pod::Spec.new do |s| s.platforms = { :ios => "13.0", :osx => "10.15", :visionos => "1.0" } s.source = { :git => "https://github.com/op-engineering/op-sqlite.git", :tag => "#{s.version}" } - s.source_files = "ios/**/*.{h,m,mm}", "cpp/**/*.{h,cpp,c}" + # Base source files + source_files = Dir.glob("ios/**/*.{h,m,mm}") + Dir.glob("cpp/**/*.{h,cpp,c}") + + # Set the path to the `c_sources` directory based on environment + if is_user_app + c_sources_dir = File.join("..", "..", "..", "c_sources") + else + c_sources_dir = File.join("example", "c_sources") + end + + + if tokenizers.any? + generate_tokenizers_header_file(tokenizers, File.join(c_sources_dir, "tokenizers.h")) + FileUtils.cp_r(c_sources_dir, __dir__) + # puts "Current directory: #{__dir__}" + # c_sources_dir_output = Dir.glob(File.join(c_sources_dir, "**/*.{h,cpp}")) + + # puts "c_sources_dir: #{c_sources_dir_output}" + + # # Add all .h and .c files from the `c_sources` directory + source_files += Dir.glob(File.join("c_sources", "**/*.{h,cpp}")) + # source_files += ["../../c_sources/tokenizers.h", "../../c_sources/tokenizers.cpp"] + # puts "Source files: #{source_files}" + end + + # Assign the collected source files to `s.source_files` + s.source_files = source_files xcconfig = { :GCC_PREPROCESSOR_DEFINITIONS => "HAVE_FULLFSYNC=1", :WARNING_CFLAGS => "-Wno-shorten-64-to-32 -Wno-comma -Wno-unreachable-code -Wno-conditional-uninitialized -Wno-deprecated-declarations", - :USE_HEADERMAP => "No", :CLANG_CXX_LANGUAGE_STANDARD => "c++17", } @@ -87,7 +116,7 @@ Pod::Spec.new do |s| if use_sqlcipher then log_message.call("[OP-SQLITE] using SQLCipher πŸ”’") - s.exclude_files = "cpp/sqlite3.c", "cpp/sqlite3.h", "cpp/libsql/bridge.c", "cpp/libsql/bridge.h" + s.exclude_files = "cpp/sqlite3.c", "cpp/sqlite3.h", "cpp/libsql/bridge.c", "cpp/libsql/bridge.h", "cpp/libsql/bridge.cpp", "cpp/libsql/libsql.h" xcconfig[:GCC_PREPROCESSOR_DEFINITIONS] += " OP_SQLITE_USE_SQLCIPHER=1 HAVE_FULLFSYNC=1 SQLITE_HAS_CODEC SQLITE_TEMP_STORE=2" s.dependency "OpenSSL-Universal" elsif use_libsql then @@ -95,7 +124,7 @@ Pod::Spec.new do |s| s.exclude_files = "cpp/sqlite3.c", "cpp/sqlite3.h", "cpp/sqlcipher/sqlite3.c", "cpp/sqlcipher/sqlite3.h", "cpp/bridge.h", "cpp/bridge.cpp" else log_message.call("[OP-SQLITE] using vanilla SQLite πŸ“¦") - s.exclude_files = "cpp/sqlcipher/sqlite3.c", "cpp/sqlcipher/sqlite3.h", "cpp/libsql/bridge.c", "cpp/libsql/bridge.h" + s.exclude_files = "cpp/sqlcipher/sqlite3.c", "cpp/sqlcipher/sqlite3.h", "cpp/libsql/bridge.c", "cpp/libsql/bridge.h", "cpp/libsql/bridge.cpp", "cpp/libsql/libsql.h" end s.dependency "React-callinvoker" @@ -129,12 +158,12 @@ Pod::Spec.new do |s| if performance_mode == '1' then log_message.call("[OP-SQLITE] Thread unsafe (1) performance mode enabled. Use only transactions! πŸš€πŸš€") - xcconfig[:OTHER_CFLAGS] = optimizedCflags + ' -DSQLITE_THREADSAFE=0 ' + other_cflags = optimizedCflags + ' -DSQLITE_THREADSAFE=0 ' end if performance_mode == '2' then log_message.call("[OP-SQLITE] Thread safe (2) performance mode enabled πŸš€") - xcconfig[:OTHER_CFLAGS] = optimizedCflags + ' -DSQLITE_THREADSAFE=1 ' + other_cflags = optimizedCflags + ' -DSQLITE_THREADSAFE=1 ' end if use_crsqlite then @@ -160,9 +189,19 @@ Pod::Spec.new do |s| if sqlite_flags != "" then log_message.call("[OP-SQLITE] Custom SQLite flags: #{sqlite_flags}") - xcconfig[:OTHER_CFLAGS] += " #{sqlite_flags}" + other_cflags += " #{sqlite_flags}" + end + + if tokenizers.any? then + log_message.call("[OP_SQLITE] Tokenizers enabled: #{tokenizers}") + if is_user_app then + other_cflags += " -DTOKENIZERS_HEADER_PATH=\\\"../c_sources/tokenizers.h\\\"" + else + other_cflags += " -DTOKENIZERS_HEADER_PATH=\\\"../example/c_sources/tokenizers.h\\\"" + end end + xcconfig[:OTHER_CFLAGS] = other_cflags s.pod_target_xcconfig = xcconfig s.vendored_frameworks = frameworks end diff --git a/package.json b/package.json index 72151c37..9313580c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@op-engineering/op-sqlite", - "version": "10.0.0", + "version": "10.0.0-tokenizers-beta9", "description": "Next generation SQLite for React Native", "main": "lib/commonjs/index", "module": "lib/module/index", @@ -14,6 +14,7 @@ "ios", "cpp", "op-sqlite.podspec", + "generate_tokenizers_header_file.rb", "ios/**.xcframework", "!lib/typescript/example", "!android/build", diff --git a/scripts/poll-in-app-server.js b/scripts/poll-in-app-server.js index b0c508f9..9fecb135 100644 --- a/scripts/poll-in-app-server.js +++ b/scripts/poll-in-app-server.js @@ -10,16 +10,25 @@ async function pollInAppServer() { const response = await makeHttpRequest('http://127.0.0.1:9000/results'); if (response !== null) { - let parsed_response = JSON.parse(response); - const allTestsPassed = parsed_response.results.reduce((acc, r) => { + let parsedResponse = JSON.parse(response); + + // Wait until some results are returned + if (parsedResponse.results.length === 0) { + continue; + } + + const allTestsPassed = parsedResponse.results.reduce((acc, r) => { + console.log(`- ${r.description} : ${r.type}`); return acc && r.type !== 'incorrect'; }, true); if (allTestsPassed) { - console.log('🟒🟒🟒🟒🟒 All tests passed!'); + console.log( + `🟒🟒🟒🟒🟒 ${parsedResponse.results.length} tests passed!` + ); process.exit(0); } else { - parsed_response.results.forEach((r) => { + parsedResponse.results.forEach((r) => { if (r.type === 'incorrect') { console.log(`πŸŸ₯Failed: ${JSON.stringify(r, null, 2)}`); } diff --git a/scripts/test-android.sh b/scripts/test-android.sh index 55607b4c..84f9627f 100755 --- a/scripts/test-android.sh +++ b/scripts/test-android.sh @@ -1,4 +1,5 @@ -JAVA_OPTS=-XX:MaxHeapSize=6g yarn turbo run run:android:release --cache-dir=.turbo/android +cd example +JAVA_OPTS=-XX:MaxHeapSize=6g yarn run:android:release adb forward tcp:9000 tcp:9000 echo "Polling in-app server..." -node ./scripts/poll-in-app-server.js \ No newline at end of file +node ../scripts/poll-in-app-server.js \ No newline at end of file diff --git a/scripts/turnOnLibsql.js b/scripts/turnOnLibsql.js index 6e13c013..9b125f88 100644 --- a/scripts/turnOnLibsql.js +++ b/scripts/turnOnLibsql.js @@ -1,7 +1,5 @@ const fs = require('fs'); -console.log('Current working directory:', process.cwd()); - // Read the package.json file const packageJson = JSON.parse(fs.readFileSync('./example/package.json')); @@ -15,4 +13,4 @@ fs.writeFileSync( JSON.stringify(packageJson, null, 2) ); -console.log('package.json updated successfully!'); +console.log('Turned on libsql in package.json', packageJson); diff --git a/src/index.ts b/src/index.ts index 01963067..44edfad6 100644 --- a/src/index.ts +++ b/src/index.ts @@ -177,6 +177,13 @@ export type DB = { }[]; callback: (response: any) => void; }) => () => void; + /** This function is only available for libsql. + * Allows to trigger a sync the database with it's remote replica + * In order for this function to work you need to use openSync or openRemote functions + * with libsql: true in the package.json + * + * The database is hosted in turso + **/ sync: () => void; flushPendingReactiveQueries: () => Promise; }; @@ -405,6 +412,9 @@ function enhanceDB(db: DB, options: any): DB { return enhancedDb; } +/** Open a replicating connection via libsql to a turso db + * libsql needs to be enabled on your package.json + */ export const openSync = (options: { url: string; authToken: string; @@ -422,6 +432,9 @@ export const openSync = (options: { return enhancedDb; }; +/** Open a remote connection via libsql to a turso db + * libsql needs to be enabled on your package.json + */ export const openRemote = (options: { url: string; authToken: string }): DB => { if (!isLibsql()) { throw new Error('This function is only available for libsql'); diff --git a/turbo.json b/turbo.json index cde2f601..e5c3058c 100644 --- a/turbo.json +++ b/turbo.json @@ -7,7 +7,7 @@ "android", "!android/build", "src/*.ts", - "src/*.tsx", + "src/tests/*.ts", "example/package.json", "example/android", "cpp", @@ -24,7 +24,7 @@ "ios", "cpp", "src/*.ts", - "src/*.tsx", + "src/tests/*.ts", "example/package.json", "example/ios", "!example/ios/build",