From b22f91dcc9e72cdc3432c43b086af4770c83048f Mon Sep 17 00:00:00 2001
From: Eduardo Silva <eduardo@treasure-data.com>
Date: Mon, 5 Oct 2020 17:26:33 -0600
Subject: [PATCH] in_tail: new 'db.locking' option and other perf improvements
 (#2564)

This patch adjust sqlite synchronization mode by default
to 'normal', it sets journal mode to WAL and it adds a
new option called 'db.locking' (default: false) which helps
to reduce the number of syscalls on every commit but at the price
of locking the access to the database file to third party programs.

The new adjustments makes a significant performance increase reducing
database I/O in about 40%.

Signed-off-by: Eduardo Silva <eduardo@treasure-data.com>
---
 plugins/in_tail/tail.c        | 8 +++++++-
 plugins/in_tail/tail_config.c | 2 +-
 plugins/in_tail/tail_config.h | 1 +
 plugins/in_tail/tail_db.c     | 9 +++++++++
 plugins/in_tail/tail_sql.h    | 5 ++++-
 5 files changed, 22 insertions(+), 3 deletions(-)

diff --git a/plugins/in_tail/tail.c b/plugins/in_tail/tail.c
index 8f0a35271cf..67f844bb2f3 100644
--- a/plugins/in_tail/tail.c
+++ b/plugins/in_tail/tail.c
@@ -581,10 +581,16 @@ static struct flb_config_map config_map[] = {
      "set a database file to keep track of monitored files and it offsets."
     },
     {
-     FLB_CONFIG_MAP_STR, "db.sync", "full",
+     FLB_CONFIG_MAP_STR, "db.sync", "normal",
      0, FLB_FALSE, 0,
      "set a database sync method. values: extra, full, normal and off."
     },
+    {
+     FLB_CONFIG_MAP_BOOL, "db.locking", "false",
+     0, FLB_TRUE, offsetof(struct flb_tail_config, db_locking),
+     "set exclusive locking mode, increase performance but don't allow "
+     "external connections to the database file."
+    },
 #endif
 
     /* Multiline Options */
diff --git a/plugins/in_tail/tail_config.c b/plugins/in_tail/tail_config.c
index c32fdbe9a8e..01a3443d04f 100644
--- a/plugins/in_tail/tail_config.c
+++ b/plugins/in_tail/tail_config.c
@@ -54,7 +54,7 @@ struct flb_tail_config *flb_tail_config_create(struct flb_input_instance *ins,
     ctx->ignore_older = 0;
     ctx->skip_long_lines = FLB_FALSE;
 #ifdef FLB_HAVE_SQLDB
-    ctx->db_sync = -1;
+    ctx->db_sync = 1;  /* sqlite sync 'normal' */
 #endif
 
     /* Load the config map */
diff --git a/plugins/in_tail/tail_config.h b/plugins/in_tail/tail_config.h
index ae87bbb90d5..385d69a253d 100644
--- a/plugins/in_tail/tail_config.h
+++ b/plugins/in_tail/tail_config.h
@@ -83,6 +83,7 @@ struct flb_tail_config {
 #ifdef FLB_HAVE_SQLDB
     struct flb_sqldb *db;
     int db_sync;
+    int db_locking;
     sqlite3_stmt *stmt_get_file;
     sqlite3_stmt *stmt_insert_file;
     sqlite3_stmt *stmt_delete_file;
diff --git a/plugins/in_tail/tail_db.c b/plugins/in_tail/tail_db.c
index 0b577767387..d37f8b5fa6c 100644
--- a/plugins/in_tail/tail_db.c
+++ b/plugins/in_tail/tail_db.c
@@ -74,6 +74,15 @@ struct flb_sqldb *flb_tail_db_open(const char *path,
         return NULL;
     }
 
+    if (ctx->db_locking == FLB_TRUE) {
+        ret = flb_sqldb_query(db, SQL_PRAGMA_LOCKING_MODE, NULL, NULL);
+        if (ret != FLB_OK) {
+            flb_plg_error(ctx->ins, "db: could not set pragma 'locking_mode'");
+            flb_sqldb_close(db);
+            return NULL;
+        }
+    }
+
     return db;
 }
 
diff --git a/plugins/in_tail/tail_sql.h b/plugins/in_tail/tail_sql.h
index 79feb01ea1f..6b7f00b0960 100644
--- a/plugins/in_tail/tail_sql.h
+++ b/plugins/in_tail/tail_sql.h
@@ -57,6 +57,9 @@
     "PRAGMA synchronous=%i;"
 
 #define SQL_PRAGMA_JOURNAL_MODE                 \
-    "PRAGMA journal_mode=OFF;"
+    "PRAGMA journal_mode=WAL;"
+
+#define SQL_PRAGMA_LOCKING_MODE                 \
+    "PRAGMA locking_mode=EXCLUSIVE;"
 
 #endif