in_tail: new 'db.locking' option and other perf improvements (fluent#…

…2564) This patch adjust sqlite synchronization mode by default to 'normal', it sets journal mode to WAL and it adds a new option called 'db.locking' (default: false) which helps to reduce the number of syscalls on every commit but at the price of locking the access to the database file to third party programs. The new adjustments makes a significant performance increase reducing database I/O in about 40%. Signed-off-by: Eduardo Silva <[email protected]>
sirwio · Oct 7, 2020 · b22f91d · b22f91d
1 parent ef47f26
commit b22f91d
Show file tree

Hide file tree

Showing 5 changed files with 22 additions and 3 deletions.
diff --git a/plugins/in_tail/tail.c b/plugins/in_tail/tail.c
@@ -581,10 +581,16 @@ static struct flb_config_map config_map[] = {
      "set a database file to keep track of monitored files and it offsets."
     },
     {
-     FLB_CONFIG_MAP_STR, "db.sync", "full",
+     FLB_CONFIG_MAP_STR, "db.sync", "normal",
      0, FLB_FALSE, 0,
      "set a database sync method. values: extra, full, normal and off."
     },
+    {
+     FLB_CONFIG_MAP_BOOL, "db.locking", "false",
+     0, FLB_TRUE, offsetof(struct flb_tail_config, db_locking),
+     "set exclusive locking mode, increase performance but don't allow "
+     "external connections to the database file."
+    },
 #endif
 
     /* Multiline Options */

diff --git a/plugins/in_tail/tail_config.c b/plugins/in_tail/tail_config.c
@@ -54,7 +54,7 @@ struct flb_tail_config *flb_tail_config_create(struct flb_input_instance *ins,
     ctx->ignore_older = 0;
     ctx->skip_long_lines = FLB_FALSE;
 #ifdef FLB_HAVE_SQLDB
-    ctx->db_sync = -1;
+    ctx->db_sync = 1;  /* sqlite sync 'normal' */
 #endif
 
     /* Load the config map */

diff --git a/plugins/in_tail/tail_config.h b/plugins/in_tail/tail_config.h
@@ -83,6 +83,7 @@ struct flb_tail_config {
 #ifdef FLB_HAVE_SQLDB
     struct flb_sqldb *db;
     int db_sync;
+    int db_locking;
     sqlite3_stmt *stmt_get_file;
     sqlite3_stmt *stmt_insert_file;
     sqlite3_stmt *stmt_delete_file;

diff --git a/plugins/in_tail/tail_db.c b/plugins/in_tail/tail_db.c
@@ -74,6 +74,15 @@ struct flb_sqldb *flb_tail_db_open(const char *path,
         return NULL;
     }
 
+    if (ctx->db_locking == FLB_TRUE) {
+        ret = flb_sqldb_query(db, SQL_PRAGMA_LOCKING_MODE, NULL, NULL);
+        if (ret != FLB_OK) {
+            flb_plg_error(ctx->ins, "db: could not set pragma 'locking_mode'");
+            flb_sqldb_close(db);
+            return NULL;
+        }
+    }
+
     return db;
 }
 

diff --git a/plugins/in_tail/tail_sql.h b/plugins/in_tail/tail_sql.h
@@ -57,6 +57,9 @@
     "PRAGMA synchronous=%i;"
 
 #define SQL_PRAGMA_JOURNAL_MODE                 \
-    "PRAGMA journal_mode=OFF;"
+    "PRAGMA journal_mode=WAL;"
+
+#define SQL_PRAGMA_LOCKING_MODE                 \
+    "PRAGMA locking_mode=EXCLUSIVE;"
 
 #endif