Merge pull request #327 from matrix-org/erikj/search

Implement rank function for SQLite FTS

synapse/storage/engines/sqlite3.py

@@ -17,6 +17,8 @@
     prepare_database, prepare_sqlite3_database
 )
 
+import struct
+
 
 class Sqlite3Engine(object):
     single_threaded = True
@@ -32,6 +34,7 @@ def convert_param_style(self, sql):
 
     def on_new_connection(self, db_conn):
         self.prepare_database(db_conn)
+        db_conn.create_function("rank", 1, _rank)
 
     def prepare_database(self, db_conn):
         prepare_sqlite3_database(db_conn)
@@ -45,3 +48,27 @@ def is_connection_closed(self, conn):
 
     def lock_table(self, txn, table):
         return
+
+
+# Following functions taken from: https://github.com/coleifer/peewee
+
+def _parse_match_info(buf):
+    bufsize = len(buf)
+    return [struct.unpack('@I', buf[i:i+4])[0] for i in range(0, bufsize, 4)]
+
+
+def _rank(raw_match_info):
+    """Handle match_info called w/default args 'pcx' - based on the example rank
+    function http://sqlite.org/fts3.html#appendix_a
+    """
+    match_info = _parse_match_info(raw_match_info)
+    score = 0.0
+    p, c = match_info[:2]
+    for phrase_num in range(p):
+        phrase_info_idx = 2 + (phrase_num * c * 3)
+        for col_num in range(c):
+            col_idx = phrase_info_idx + (col_num * 3)
+            x1, x2 = match_info[col_idx:col_idx + 2]
+            if x1 > 0:
+                score += float(x1) / x2
+    return score

synapse/storage/schema/delta/25/fts.py

@@ -54,7 +54,7 @@
 
 SQLITE_TABLE = (
     "CREATE VIRTUAL TABLE IF NOT EXISTS event_search"
-    " USING fts3 ( event_id, room_id, key, value)"
+    " USING fts4 ( event_id, room_id, key, value )"
 )
 
 

synapse/storage/search.py

@@ -72,7 +72,8 @@ def search_msgs(self, room_ids, search_term, keys):
             )
         elif isinstance(self.database_engine, Sqlite3Engine):
             sql = (
-                "SELECT 0 as rank, room_id, event_id FROM event_search"
+                "SELECT rank(matchinfo(event_search)) as rank, room_id, event_id"
+                " FROM event_search"
                 " WHERE value MATCH ?"
             )
         else: