Merge to master - Release 2.3.5 - Optimize order by and filters queries

lib/goo/base/resource.rb

@@ -137,7 +137,7 @@ def unmmaped_to_array
       end
 
       def unmapped(*args)
-        @unmapped.transform_values do  |language_values|
+        @unmapped&.transform_values do  |language_values|
           self.class.not_show_all_languages?(language_values, args) ?  language_values.values.flatten: language_values
         end
       end

lib/goo/base/where.rb

@@ -209,6 +209,11 @@ def process_query_intl(count=false)
 
         options_load[:ids] = ids if ids
         models_by_id = {}
+        if @page_i && (options_load[:models].length > 0)
+          options_load.delete(:filters)
+          options_load.delete(:order_by)
+        end
+
         if (@page_i && options_load[:models].length > 0) ||
             (!@page_i && (@count.nil? || @count > 0))
           models_by_id = Goo::SPARQL::Queries.model_load(options_load)

lib/goo/sparql/mixins/solution_lang_filter.rb

@@ -25,8 +25,8 @@ def fill_models_with_all_languages(models_by_id)
         end
 
 
-        def set_model_value(model, predicate, values)
-          set_value(model, predicate, values) do
+        def set_model_value(model, predicate, values, value)
+          set_value(model, predicate, value) do
             model.send("#{predicate}=", values, on_load: true)
           end
         end

lib/goo/sparql/query_builder.rb

@@ -25,21 +25,18 @@ def build_select_query(ids, variables, graphs, patterns,
                              query_options, properties_to_include)
 
         patterns = graph_match(@collection, @graph_match, graphs, @klass, patterns, query_options, @unions)
-
-        aggregate_projections, aggregate_vars,
-          variables, optional_patterns = get_aggregate_vars(@aggregate, @collection, graphs,
-                                                            @klass, @unions, variables)
-
-        @order_by, variables, optional_patterns = init_order_by(@count, @klass, @order_by, optional_patterns, variables,patterns, query_options, graphs)
         variables, patterns = add_some_type_to_id(patterns, query_options, variables)
-
+        aggregate_projections, aggregate_vars, variables, optional_patterns = get_aggregate_vars(@aggregate, @collection, graphs, @klass, @unions, variables)
         query_filter_str, patterns, optional_patterns, filter_variables =
           filter_query_strings(@collection, graphs, @klass, optional_patterns, patterns, @query_filters)
+        @order_by, variables, optional_patterns = init_order_by(@count, @klass, @order_by, optional_patterns, variables,patterns, query_options, graphs)
+        order_by_str, order_variables = order_by_string
+
 
         variables = [] if @count
         variables.delete :some_type
 
-        select_distinct(variables, aggregate_projections, filter_variables)
+        select_distinct(variables, aggregate_projections, filter_variables, order_variables)
           .from(graphs)
           .where(patterns)
           .union_bind_in_where(properties_to_include)
@@ -55,7 +52,10 @@ def build_select_query(ids, variables, graphs, patterns,
         @query.union(*@unions) unless @unions.empty?
 
         ids_filter(ids) if ids
-        order_by if @order_by
+
+
+        @query.order_by(*order_by_str) if @order_by
+
 
         put_query_aggregate_vars(aggregate_vars) if aggregate_vars
         count if @count
@@ -117,16 +117,17 @@ def put_query_aggregate_vars(aggregate_vars)
         self
       end
 
-      def order_by
-        order_by_str = @order_by.map do |attr, order|
+      def order_by_string
+        order_variables = []
+        order_str = @order_by&.map do |attr, order|
           if order.is_a?(Hash)
             sub_attr, order = order.first
-            attr = @internal_variables_map[sub_attr]
+            attr =  @internal_variables_map.select{ |internal_var, attr_var| attr_var.eql?({attr => sub_attr}) || attr_var.eql?(sub_attr)}.keys.last
           end
+          order_variables << attr
           "#{order.to_s.upcase}(?#{attr})"
         end
-        @query.order_by(*order_by_str)
-        self
+        [order_str,order_variables]
       end
 
       def from(graphs)
@@ -141,10 +142,11 @@ def from(graphs)
         self
       end
 
-      def select_distinct(variables, aggregate_projections, filter_variables)
+      def select_distinct(variables, aggregate_variables, filter_variables, order_variables)
         select_vars = variables.dup
-        reject_aggregations_from_vars(select_vars, aggregate_projections) if aggregate_projections
-        select_vars = (select_vars + filter_variables).uniq  if @page # Fix for 4store pagination with a filter
+        reject_aggregations_from_vars(select_vars, aggregate_variables) if aggregate_variables
+        # Fix for 4store pagination with a filter https://github.com/ontoportal-lirmm/ontologies_api/issues/25
+        select_vars = (select_vars + filter_variables + order_variables).uniq  if @page
         @query = @query.select(*select_vars).distinct(true)
         self
       end
@@ -165,23 +167,24 @@ def ids_filter(ids)
       def patterns_for_match(klass, attr, value, graphs, patterns, unions,
                              internal_variables, subject = :id, in_union = false,
                              in_aggregate = false, query_options = {}, collection = nil)
+        new_internal_var = value
         if value.respond_to?(:each) || value.instance_of?(Symbol)
           next_pattern = value.instance_of?(Array) ? value.first : value
 
           #for filters
           next_pattern = { next_pattern => [] } if next_pattern.instance_of?(Symbol)
 
-          value = "internal_join_var_#{internal_variables.length}".to_sym
+          new_internal_var = "internal_join_var_#{internal_variables.length}".to_sym
           if in_aggregate
-            value = "#{attr}_agg_#{in_aggregate}".to_sym
+            new_internal_var = "#{attr}_agg_#{in_aggregate}".to_sym
           end
-          internal_variables << value
-          @internal_variables_map[attr] = value
+          internal_variables << new_internal_var
+          @internal_variables_map[new_internal_var] = value.empty? ? attr : {attr => value}
         end
 
         add_rules(attr, klass, query_options)
         graph, pattern =
-          query_pattern(klass, attr, value: value, subject: subject, collection: collection)
+          query_pattern(klass, attr, value: new_internal_var, subject: subject, collection: collection)
         if pattern
           if !in_union
             patterns << pattern
@@ -194,7 +197,7 @@ def patterns_for_match(klass, attr, value, graphs, patterns, unions,
           range = klass.range(attr)
           next_pattern.each do |next_attr, next_value|
             patterns_for_match(range, next_attr, next_value, graphs,
-                               patterns, unions, internal_variables, subject = value,
+                               patterns, unions, internal_variables, subject = new_internal_var,
                                in_union, in_aggregate, collection = collection)
           end
         end
@@ -270,15 +273,35 @@ def init_order_by(count, klass, order_by, optional_patterns, variables, patterns
           order_by.each do |attr, direction|
 
             if direction.is_a?(Hash)
+              # TODO this part can be improved/refactored, the complexity was added because order by don't work
+              # if the pattern is in the mandatory ones (variable `patterns`)
+              # and optional (variable `optional_patterns`) at the same time
               sub_attr, direction = direction.first
               graph_match_iteration = Goo::Base::PatternIteration.new(Goo::Base::Pattern.new({attr => [sub_attr]}))
               old_internal = internal_variables.dup
+              old_patterns = optional_patterns.dup
+
               walk_pattern(klass, graph_match_iteration, graphs, optional_patterns, @unions, internal_variables, in_aggregate = false, query_options, @collection)
-              variables << (internal_variables - old_internal).last
+              new_variables = (internal_variables - old_internal)
+              internal_variables.delete(new_variables)
+              new_patterns = optional_patterns - old_patterns
+              already_existent_pattern = patterns.select{|x| x[1].eql?(new_patterns.last[1])}.first
+
+              if already_existent_pattern
+                already_existent_variable = already_existent_pattern[2]
+                optional_patterns = old_patterns
+                key = @internal_variables_map.select{|key, value|  key.eql?(new_variables.last)}.keys.first
+                @internal_variables_map[key] = (already_existent_variable || new_variables.last) if key
+
+                #variables << already_existent_variable
+              else
+                #variables <<  new_variables.last
+              end
+
             else
               quad = query_pattern(klass, attr)
               optional_patterns << quad[1]
-              variables << attr
+              #variables << attr
             end
 
             #patterns << quad[1]
@@ -325,7 +348,12 @@ def query_filter_sparql(klass, filter, filter_patterns, filter_graphs,
           end
           filter_var = inspected_patterns[filter_pattern_match]
 
-          unless filter_operation.value.instance_of?(Goo::Filter)
+          if filter_operation.value.instance_of?(Goo::Filter)
+            filter_operations << "#{sparql_op_string(filter_operation.operator)}"
+            query_filter_sparql(klass, filter_operation.value, filter_patterns,
+                                filter_graphs, filter_operations,
+                                internal_variables, inspected_patterns, collection)
+          else
             case filter_operation.operator
             when  :unbound
               filter_operations << "!BOUND(?#{filter_var.to_s})"
@@ -349,11 +377,6 @@ def query_filter_sparql(klass, filter, filter_patterns, filter_graphs,
                   " #{value.to_ntriples}")
             end
 
-          else
-            filter_operations << "#{sparql_op_string(filter_operation.operator)}"
-            query_filter_sparql(klass, filter_operation.value, filter_patterns,
-                                filter_graphs, filter_operations,
-                                internal_variables, inspected_patterns, collection)
           end
         end
       end
@@ -399,7 +422,7 @@ def add_some_type_to_id(patterns, query_options, variables)
       end
 
       def internal_variables
-        @internal_variables_map.values
+        @internal_variables_map.keys
       end
     end
   end