From b427c94a624da18ba817e507efadb8a743c593a3 Mon Sep 17 00:00:00 2001
From: Andrew Kane <andrew@ankane.org>
Date: Tue, 27 Feb 2024 00:22:53 -0800
Subject: [PATCH] Fixed error with hypothetical index limit - fixes #49

---
 CHANGELOG.md          |  1 +
 lib/dexter/indexer.rb | 24 ++++++++++++++++--------
 lib/dexter/query.rb   |  2 +-
 3 files changed, 18 insertions(+), 9 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index cfb9d9f2..bb96ebab 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,6 @@
 ## 0.5.3 (unreleased)
 
+- Fixed error with hypothetical index limit
 - Fixed error with foreign tables
 
 ## 0.5.2 (2024-01-10)
diff --git a/lib/dexter/indexer.rb b/lib/dexter/indexer.rb
index 119ce29b..b867e689 100644
--- a/lib/dexter/indexer.rb
+++ b/lib/dexter/indexer.rb
@@ -96,10 +96,16 @@ def process_queries(queries)
       analyze_tables(tables) if tables.any? && (@analyze || @log_level == "debug2")
 
       # create hypothetical indexes and explain queries
-      candidates = tables.any? ? create_hypothetical_indexes(queries.select(&:candidate_tables)) : {}
+      if tables.any?
+        # process in batches to prevent "hypopg: not more oid available" error
+        # https://hypopg.readthedocs.io/en/rel1_stable/usage.html#configuration
+        queries.select(&:candidate_tables).each_slice(500) do |batch|
+          create_hypothetical_indexes(batch)
+        end
+      end
 
       # see if new indexes were used and meet bar
-      new_indexes = determine_indexes(queries, candidates, tables)
+      new_indexes = determine_indexes(queries, tables)
 
       # display and create new indexes
       show_and_create_indexes(new_indexes, queries)
@@ -228,7 +234,9 @@ def create_hypothetical_indexes(queries)
         calculate_plan(explainable_queries)
       end
 
-      candidates
+      queries.each do |query|
+        query.candidates = candidates
+      end
     end
 
     def find_columns(plan)
@@ -282,7 +290,7 @@ def hypo_indexes_from_plan(index_name_to_columns, plan, index_set)
       query_indexes
     end
 
-    def determine_indexes(queries, index_name_to_columns, tables)
+    def determine_indexes(queries, tables)
       new_indexes = {}
 
       # filter out existing indexes
@@ -312,11 +320,11 @@ def determine_indexes(queries, index_name_to_columns, tables)
           cost_savings2 = new_cost > 100 && new_cost2 < new_cost * savings_ratio
 
           key = cost_savings2 ? 2 : 1
-          query_indexes = hypo_indexes_from_plan(index_name_to_columns, query.plans[key], index_set)
+          query_indexes = hypo_indexes_from_plan(query.candidates, query.plans[key], index_set)
 
           # likely a bad suggestion, so try single column
           if cost_savings2 && query_indexes.size > 1
-            query_indexes = hypo_indexes_from_plan(index_name_to_columns, query.plans[1], index_set)
+            query_indexes = hypo_indexes_from_plan(query.candidates, query.plans[1], index_set)
             cost_savings2 = false
           end
 
@@ -389,8 +397,8 @@ def determine_indexes(queries, index_name_to_columns, tables)
 
           # TODO optimize
           if @log_level.start_with?("debug")
-            query.pass1_indexes = hypo_indexes_from_plan(index_name_to_columns, query.plans[1], index_set)
-            query.pass2_indexes = hypo_indexes_from_plan(index_name_to_columns, query.plans[2], index_set)
+            query.pass1_indexes = hypo_indexes_from_plan(query.candidates, query.plans[1], index_set)
+            query.pass2_indexes = hypo_indexes_from_plan(query.candidates, query.plans[2], index_set)
           end
         end
       end
diff --git a/lib/dexter/query.rb b/lib/dexter/query.rb
index fea4a123..67e49e02 100644
--- a/lib/dexter/query.rb
+++ b/lib/dexter/query.rb
@@ -2,7 +2,7 @@ module Dexter
   class Query
     attr_reader :statement, :fingerprint, :plans
     attr_writer :tables
-    attr_accessor :missing_tables, :new_cost, :total_time, :calls, :indexes, :suggest_index, :pass1_indexes, :pass2_indexes, :pass3_indexes, :candidate_tables, :tables_from_views
+    attr_accessor :missing_tables, :new_cost, :total_time, :calls, :indexes, :suggest_index, :pass1_indexes, :pass2_indexes, :pass3_indexes, :candidate_tables, :tables_from_views, :candidates
 
     def initialize(statement, fingerprint = nil)
       @statement = statement