feat: faster timestamp queries (#479)

* fix: postgres timestamp casting should include timezone Signed-off-by: Clément Salaün <[email protected]> * fix: fix: test Signed-off-by: Clément Salaün <[email protected]> * feat: faster endtime based queries Signed-off-by: Clément Salaün <[email protected]> * feat: add temporal index on transactions Signed-off-by: Clément Salaün <[email protected]> * fix: enable optimisation only when pageSize is one Signed-off-by: Clément Salaün <[email protected]> * fix: apply optimized filter only for postgres Signed-off-by: Clément Salaün <[email protected]> --------- Signed-off-by: Clément Salaün <[email protected]>
formancehq · Feb 27, 2024 · 187ea53 · 187ea53
1 parent 010ab3b
commit 187ea53
Show file tree

Hide file tree

Showing 3 changed files with 25 additions and 1 deletion.
diff --git a/pkg/storage/sqlstorage/migrates/21-optimized-temporal/postgres.sql b/pkg/storage/sqlstorage/migrates/21-optimized-temporal/postgres.sql
@@ -0,0 +1,2 @@
+--statement
+CREATE INDEX IF NOT EXISTS transactions_ts_desc ON "VAR_LEDGER_NAME".transactions ("timestamp" DESC);
diff --git a/pkg/storage/sqlstorage/transactions.go b/pkg/storage/sqlstorage/transactions.go
@@ -122,6 +122,25 @@ func (s *Store) buildTransactionsQuery(flavor Flavor, p ledger.TransactionsQuery
 	}
 	if !endTime.IsZero() {
 		sb.Where(sb.L("timestamp", endTime.UTC()))
+
+		if flavor == PostgreSQL {
+			// We nudge the query planner in the right direction,
+			// by reducing the search space according to the end time.
+			// We have to use a raw query as the sqlbuilder
+			// does not support LTE+subqueries in the where clause.
+			sb.SQL(fmt.Sprintf(`
+				AND "id" <= (
+					SELECT "id"
+					FROM "%s".transactions
+					WHERE "timestamp" < '%s'::timestamptz
+					ORDER BY "timestamp" DESC, "id" DESC
+					LIMIT 1
+				)`,
+				s.schema.Name(),
+				endTime.UTC().Format(time.RFC3339),
+			))
+		}
+
 		t.EndTime = endTime
 	}
 
@@ -461,7 +480,7 @@ func (s *Store) insertTransactions(ctx context.Context, txs ...core.ExpandedTran
                                pre_commit_volumes,
                                post_commit_volumes) (SELECT * FROM unnest(
                                    $1::int[],
-                                   $2::timestamp[],
+                                   $2::timestamptz[],
                                    $3::varchar[],
                                    $4::jsonb[],
                                    $5::jsonb[],

diff --git a/pkg/storage/sqlstorage/transactions_test.go b/pkg/storage/sqlstorage/transactions_test.go
@@ -353,6 +353,9 @@ func testTransactions(t *testing.T, store *sqlstorage.Store) {
 		// Should get only the first transaction.
 		require.Equal(t, 1, cursor.PageSize)
 
+		// Transaction timestamp fetched should be equal to the timestamp of the committed transaction.
+		require.True(t, cursor.Data[0].Timestamp.Equal(tx3.Timestamp))
+
 		cursor, err = store.GetTransactions(context.Background(), ledger.TransactionsQuery{
 			AfterTxID: cursor.Data[0].ID,
 			PageSize:  1,
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		--statement
		CREATE INDEX IF NOT EXISTS transactions_ts_desc ON "VAR_LEDGER_NAME".transactions ("timestamp" DESC);