Skip to content

Commit

Permalink
Feature: Improve Browse page performance (#572)
Browse files Browse the repository at this point in the history
* optimize fetched  attributes in the browse page in API mode

* update ontologies browse to use API or Index

* add cache for the fair_score and submission endpoint

* internationalize a sentence in the browse and ontologies selector  views

* fix browse hasFormality and isOfType filters

* prfioritize API filtring  over the index in the browse page

* fix a bug with edit submission returning nil error on submission
  • Loading branch information
syphax-bouazzouni authored Apr 10, 2024
1 parent 067bb7e commit 11f9fe1
Showing 14 changed files with 199 additions and 105 deletions.
3 changes: 1 addition & 2 deletions app/controllers/application_controller.rb
Original file line number Diff line number Diff line change
@@ -710,9 +710,8 @@ def init_trial_license

# Get the submission metadata from the REST API.
def submission_metadata
@metadata ||= JSON.parse(LinkedData::Client::HTTP.get("#{REST_URI}/submission_metadata", {}, raw: true))
@metadata ||= helpers.submission_metadata
end
helper_method :submission_metadata

def request_lang
helpers.request_lang
164 changes: 131 additions & 33 deletions app/controllers/concerns/submission_filter.rb
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
module SubmissionFilter
extend ActiveSupport::Concern

include SearchContent

BROWSE_ATTRIBUTES = ['ontology', 'submissionStatus', 'description', 'pullLocation', 'creationDate',
'contact', 'released', 'naturalLanguage', 'hasOntologyLanguage',
'hasFormalityLevel', 'isOfType', 'deprecated', 'status', 'metrics']
@@ -10,34 +12,54 @@ def init_filters(params)
@show_private_only = params[:private_only]&.eql?('true')
@show_retired = params[:show_retired]&.eql?('true')
@selected_format = params[:format]
@selected_sort_by = params[:sort_by].blank? ? 'visits' : params[:sort_by]
@sort_by = params[:sort_by].blank? ? 'visits' : params[:sort_by]
@search = params[:search]
end

def submissions_paginate_filter(params)
request_params = filters_params(params, page: nil)
request_params = filters_params(params, page: params[:page], pagesize: 10)
filter_params = params.permit(@filters.keys).to_h
init_filters(params)
# pagination disabled because is not supported by 4store,
# see https://github.com/ontoportal-lirmm/ontologies_api/issues/25
# @page = LinkedData::Client::Models::OntologySubmission.all(request_params)
@page = OpenStruct.new(page: 1, next_page: nil)
submissions = LinkedData::Client::Models::OntologySubmission.all(request_params)
@analytics = helpers.ontologies_analytics

@analytics = Rails.cache.fetch("ontologies_analytics-#{Time.now.year}-#{Time.now.month}") do
helpers.ontologies_analytics
end
@ontologies = LinkedData::Client::Models::Ontology.all(include: 'notes,projects', also_include_views: @show_views, display_links: false, display_context: false)

# get fair scores of all ontologies
@fair_scores = fairness_service_enabled? ? get_fair_score('all') : nil
submissions = submissions.reject { |sub| sub.ontology.nil? }.map { |sub| ontology_hash(sub) }

if @selected_sort_by.eql?('visits')
submissions = submissions.sort_by { |x| -x[:popularity] }
elsif @selected_sort_by.eql?('fair')
submissions = submissions.sort_by { |x| -x[:fairScore] }
elsif @selected_sort_by.eql?('notes')
submissions = submissions.sort_by { |x| -x[:note_count] }
elsif @selected_sort_by.eql?('projects')
submissions = submissions.sort_by { |x| -x[:project_count] }
@total_ontologies = @ontologies.size
search_backend = params[:search_backend]
params = { query: @search,
status: request_params[:status],
show_views: @show_views,
private_only: @show_private_only,
languages: request_params[:naturalLanguage],
page_size: @total_ontologies,
formality_level: request_params[:hasFormalityLevel],
is_of_type: request_params[:isOfType],
groups: request_params[:group], categories: request_params[:hasDomain],
formats: request_params[:hasOntologyLanguage] }
submissions = []

if search_backend.eql?('index')
submissions = filter_using_index(**params)
else
submissions = filter_using_data(**params)
end
submissions

submissions = submissions.reject { |sub| sub.ontology.nil? }.map { |sub| ontology_hash(sub, @ontologies) }


submissions = sort_submission_by(submissions, @sort_by) if @search.blank?


@page = paginate_submissions(submissions, request_params[:page].to_i, request_params[:pagesize].to_i)

count = @page.page.eql?(1) ? count_objects(submissions) : {}

[@page.collection, @page.totalCount, count, filter_params]
end

def ontologies_filter_url(filters, page: 1, count: false)
@@ -46,10 +68,78 @@ def ontologies_filter_url(filters, page: 1, count: false)

private

def filter_using_index(query:, status:, show_views:, private_only:, languages:, page_size:, formality_level:, is_of_type:, groups:, categories:, formats:)
search_ontologies(
query: query,
status: status,
show_views: show_views,
private_only: private_only,
languages: languages,
page_size: page_size,
formality_level: formality_level,
is_of_type: is_of_type,
groups: groups, categories: categories,
formats: formats
)

end

def filter_using_data(query:, status:, show_views:, private_only:, languages:, page_size:, formality_level:, is_of_type:, groups:, categories:, formats:)
submissions = LinkedData::Client::Models::OntologySubmission.all(include: BROWSE_ATTRIBUTES.join(','), also_include_views: show_views, display_links: false, display_context: false)

submissions.select do |s|
out = !s.ontology.nil?
out = out && ((s.ontology.viewingRestriction.eql?('public') && !private_only) || private_only && s.ontology.viewingRestriction.eql?('private'))
out = out && (query.blank? || [s.description, s.ontology.name, s.ontology.acronym].any? { |x| x.downcase.include?(query.downcase) })
out = out && (groups.blank? || (s.ontology.group.map { |x| helpers.link_last_part(x) } & groups.split(',')).any?)
out = out && (categories.blank? || (s.ontology.hasDomain.map { |x| helpers.link_last_part(x) } & categories.split(',')).any?)
out = out && (status.blank? || status.split(',').include?(s.status))
out = out && (formats.blank? || formats.split(',').any? { |f| s.hasOntologyLanguage.eql?(f) })
out = out && (is_of_type.blank? || is_of_type.split(',').any? { |f| helpers.link_last_part(s.isOfType).eql?(f) })
out = out && (formality_level.blank? || formality_level.split(',').any? { |f| helpers.link_last_part(s.hasFormalityLevel).eql?(f) })
out = out && (languages.blank? || languages.split(',').any? { |f| s.naturalLanguage.any? { |n| helpers.link_last_part(n).eql?(f) } })
out
end
end

def paginate_submissions(all_submissions, page, size)
current_page = page
page_size = size

start_index = (current_page - 1) * page_size
end_index = start_index + page_size - 1
next_page = current_page * page_size < all_submissions.size ? current_page + 1 : nil
OpenStruct.new(page: current_page, nextPage: next_page, totalCount: all_submissions.size,
collection: all_submissions[start_index..end_index])
end

def sort_submission_by(submissions, sort_by)
if sort_by.eql?('visits')
submissions = submissions.sort_by { |x| -x[:popularity] }
elsif sort_by.eql?('fair')
submissions = submissions.sort_by { |x| -x[:fairScore] }
elsif sort_by.eql?('notes')
submissions = submissions.sort_by { |x| -x[:note_count] }
elsif sort_by.eql?('projects')
submissions = submissions.sort_by { |x| -x[:project_count] }
elsif sort_by.eql?('metrics_classes')
submissions = submissions.sort_by { |x| -x[:class_count] }
elsif sort_by.eql?('metrics_individuals')
submissions = submissions.sort_by { |x| -x[:individual_count] }
elsif sort_by.eql?('creationDate')
submissions = submissions.sort_by { |x| -x[:creationDate] }
elsif sort_by.eql?('released')
submissions = submissions.sort_by { |x| -x[:released] }
elsif sort_by.eql?('ontology_name')
submissions = submissions.sort_by { |x| -x[:name] }
end
submissions
end

def filters_params(params, includes: BROWSE_ATTRIBUTES.join(','), page: 1, pagesize: 5)
request_params = { display_links: false, display_context: false,
include: includes, include_status: 'RDF' }
request_params.merge!(page: page, pagesize: pagesize) if page
request_params.merge!(page: page.to_i, pagesize: pagesize.to_i) if page
filters_values_map = {
categories: :hasDomain,
groups: :group,
@@ -83,7 +173,6 @@ def filters_params(params, includes: BROWSE_ATTRIBUTES.join(','), page: 1, pages
@filters[:show_retired] = 'true'
end


filters_values_map.each do |filter, api_key|
next if params[filter].nil? || params[filter].empty?

@@ -93,13 +182,23 @@ def filters_params(params, includes: BROWSE_ATTRIBUTES.join(','), page: 1, pages
end
end

unless params[:sort_by].blank?
@filters[:sort_by] = params[:sort_by]
end

unless params[:search].blank?
@filters[:search] = params[:search]
end

request_params.delete(:order_by) if %w[visits fair].include?(request_params[:sort_by].to_s)
request_params
end

def ontology_hash(sub)
def ontology_hash(sub, ontologies)
o = {}
ont = sub.ontology
ont.notes = ontologies.select { |x| x.acronym.eql?(sub.ontology.acronym) }.first&.notes || []
ont.projects = ontologies.select { |x| x.acronym.eql?(sub.ontology.acronym) }.first&.projects || []

add_ontology_attributes(o, ont)
add_submission_attributes(o, sub)
@@ -144,12 +243,12 @@ def add_submission_attributes(ont_hash, sub)
ont_hash[:hasFormalityLevel] = sub.hasFormalityLevel
ont_hash[:isOfType] = sub.isOfType
ont_hash[:submissionStatusFormatted] = submission_status2string(sub).gsub(/\(|\)/, '')
ont_hash[:format] = sub.hasOntologyLanguage
ont_hash[:contact] = sub.contact.map(&:name).first unless sub.contact.nil?
ont_hash[:format] = sub.hasOntologyLanguage&.split('/').last
ont_hash[:contact] = sub.contact.map { |c| c.is_a?(String) ? c.split('|').first : c.name }.first unless sub.contact.nil?
end

def add_ontology_attributes(ont_hash, ont)
return if ont.nil?
return if ont.nil?

ont_hash[:id] = ont.id
ont_hash[:type] = ont.viewOf.nil? ? 'ontology' : 'ontology_view'
@@ -182,11 +281,11 @@ def ontology_filters_init(categories, groups)
end

@formalityLevel = submission_metadata.select { |x| x['@id']['hasFormalityLevel'] }.first['enforcedValues'].map do |id, name|
{ 'id' => id, 'name' => name, 'acronym' => name.camelize(:lower), 'value' => name.delete(' ')}
{ 'id' => id, 'name' => helpers.link_last_part(id), 'acronym' => name, 'value' => helpers.link_last_part(id) }
end

@isOfType = submission_metadata.select { |x| x['@id']['isOfType'] }.first['enforcedValues'].map do |id, name|
{ 'id' => id, 'name' => name, 'acronym' => name.camelize(:lower), 'value' => name.delete(' ') }
{ 'id' => id, 'name' => helpers.link_last_part(id), 'acronym' => name, 'value' => helpers.link_last_part(id) }
end

@formats = [[t("submissions.filter.all_formats"), ''], 'OBO', 'OWL', 'SKOS', 'UMLS']
@@ -217,10 +316,10 @@ def ontology_filters_init(categories, groups)
{
categories: object_filter(categories, :categories),
groups: object_filter(groups, :groups),
naturalLanguage: object_filter(@languages, :naturalLanguage),
naturalLanguage: object_filter(@languages, :naturalLanguage, "value"),
hasFormalityLevel: object_filter(@formalityLevel, :hasFormalityLevel),
isOfType: object_filter(@isOfType, :isOfType),
#missingStatus: object_filter(@missingStatus, :missingStatus)
# missingStatus: object_filter(@missingStatus, :missingStatus)
}
end

@@ -229,12 +328,9 @@ def check_id(name_value, objects, name_key)
selected_category.first && selected_category.first['id']
end

def object_checks(key)
params[key]&.split(',')
end

def object_filter(objects, object_name, name_key = 'acronym')
checks = object_checks(object_name) || []
checks = params[object_name]&.split(',') || []
checks = checks.map { |x| check_id(x, objects, name_key) }.compact

ids = objects.map { |x| x['id'] }
@@ -246,10 +342,10 @@ def count_objects(ontologies)
objects_count = {}
@categories = LinkedData::Client::Models::Category.all(display_links: false, display_context: false)
@groups = LinkedData::Client::Models::Group.all(display_links: false, display_context: false)

@filters = ontology_filters_init(@categories, @groups)
object_names = @filters.keys


@filters.each do |filter, values|
objects = values.first
objects_count[filter] = objects.map { |v| [v['id'], 0] }.to_h
@@ -259,6 +355,8 @@ def count_objects(ontologies)
object_names.each do |name|
values = Array(ontology[name])
values.each do |v|
v.gsub!('http://data.bioontology.org', rest_url)

objects_count[name] = {} unless objects_count[name]
objects_count[name][v] = (objects_count[name][v] || 0) + 1
end
59 changes: 20 additions & 39 deletions app/controllers/ontologies_controller.rb
Original file line number Diff line number Diff line change
@@ -32,45 +32,33 @@ class OntologiesController < ApplicationController
def index
@categories = LinkedData::Client::Models::Category.all(display_links: false, display_context: false)
@groups = LinkedData::Client::Models::Group.all(display_links: false, display_context: false)

@filters = ontology_filters_init(@categories, @groups)
init_filters(params)
render 'ontologies/browser/browse'
end

def ontologies_filter
@ontologies = submissions_paginate_filter(params)
@object_count = count_objects(@ontologies)

update_filters_counts = @object_count.map do |section, values_count|
values_count.map do |value, count|
replace("count_#{section}_#{value}") do
helpers.turbo_frame_tag("count_#{section}_#{value}") do
helpers.content_tag(:span, count.to_s, class: "hide-if-loading #{count.zero? ? 'disabled' : ''}")
@time = Benchmark.realtime do
@ontologies, @count, @count_objects, @request_params = submissions_paginate_filter(params)
end

if @page.page.eql?(1)
streams = [prepend("ontologies_list_view-page-#{@page.page}", partial: 'ontologies/browser/ontologies')]
streams += @count_objects.map do |section, values_count|
values_count.map do |value, count|
replace("count_#{section}_#{value}") do
helpers.turbo_frame_tag("count_#{section}_#{value}") do
helpers.content_tag(:span, count.to_s, class: "hide-if-loading #{count.zero? ? 'disabled' : ''}")
end
end
end
end
end.flatten
end.flatten
else
streams = [replace("ontologies_list_view-page-#{@page.page}", partial: 'ontologies/browser/ontologies')]
end

count_streams = [
replace('ontologies_filter_count_request') do
helpers.content_tag(:p, class: "browse-desc-text", style: "margin-bottom: 12px !important;") { t("ontologies.showing_ontologies_size", ontologies_size: @ontologies.size, analytics_size: @analytics.keys.size) }
end
] + update_filters_counts

streams = if params[:page].nil?
[
prepend('ontologies_list_container', partial: 'ontologies/browser/ontologies'),
prepend('ontologies_list_container') {
helpers.turbo_frame_tag("ontologies_filter_count_request") do
helpers.browser_counter_loader
end
}
]
else
[replace("ontologies_list_view-page-1", partial: 'ontologies/browser/ontologies')]
end

render turbo_stream: streams + count_streams
render turbo_stream: streams
end

def classes
@@ -222,8 +210,8 @@ def sparql

def content_serializer
@result = serialize_content(ontology_acronym: params[:acronym],
concept_id: params[:id],
format: params[:output_format])
concept_id: params[:id],
format: params[:output_format])

render 'ontologies/content_serializer', layout: nil
end
@@ -553,13 +541,6 @@ def properties_hash_values(properties, sub: @submission_latest, custom_labels: {
properties.map { |x| [x.to_s, [sub.send(x.to_s), custom_labels[x.to_sym]]] }.to_h
end

def get_metrics_hash
metrics_hash = {}
# TODO: Metrics do not return for views on the backend, need to enable include_views param there
@metrics = LinkedData::Client::Models::Metrics.all(include_views: true)
@metrics.each { |m| metrics_hash[m.links['ontology']] = m }
return metrics_hash
end

def determine_layout
case action_name
2 changes: 1 addition & 1 deletion app/controllers/submissions_controller.rb
Original file line number Diff line number Diff line change
@@ -60,7 +60,7 @@ def edit_properties
display_submission_attributes params[:ontology_id], params[:properties]&.split(','), submissionId: params[:submission_id],
inline_save: params[:inline_save]&.eql?('true')

attribute_template_output = render_to_string(inline: helpers.render_submission_inputs(params[:container_id] || 'metadata_by_ontology'))
attribute_template_output = render_to_string(inline: helpers.render_submission_inputs(params[:container_id] || 'metadata_by_ontology', @submission))

render inline: attribute_template_output

2 changes: 2 additions & 0 deletions app/helpers/application_helper.rb
Original file line number Diff line number Diff line change
@@ -197,6 +197,8 @@ def onts_for_select
end

def link_last_part(url)
return "" if url.nil?

if url.include?('#')
url.split('#').last
else
Loading

0 comments on commit 11f9fe1

Please sign in to comment.