Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feat: Sample Searching using SearchKick (Part One) #846

Merged
merged 9 commits into from
Dec 13, 2024
Merged
4 changes: 4 additions & 0 deletions .github/workflows/ruby.yml
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,10 @@ jobs:
cache: "pnpm"
- name: Install dependencies
run: pnpm install
- name: Set up OpenSearch
uses: ankane/setup-opensearch@v1
with:
opensearch-version: 2
- name: Precompile assets
run: |
bin/rails assets:precompile
Expand Down
5 changes: 5 additions & 0 deletions Gemfile
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,11 @@ gem 'caxlsx'
# renders client's local time zone
gem 'local_time', '~> 3.0', '>= 3.0.2'

# Advanced searching using SearchKick
gem 'searchkick'

gem 'opensearch-ruby'

group :development, :test do
# See https://guides.rubyonrails.org/debugging_rails_applications.html#debugging-with-the-debug-gem
gem 'debug', platforms: %i[mri windows], require: 'debug/prelude'
Expand Down
8 changes: 8 additions & 0 deletions Gemfile.lock
Original file line number Diff line number Diff line change
Expand Up @@ -410,6 +410,9 @@ GEM
omniauth-saml (2.1.2)
omniauth (~> 2.1)
ruby-saml (~> 1.17)
opensearch-ruby (3.4.0)
faraday (>= 1.0, < 3)
multi_json (>= 1.0)
orm_adapter (0.5.0)
os (1.1.4)
pagy (9.0.5)
Expand Down Expand Up @@ -538,6 +541,9 @@ GEM
rubyzip (2.3.2)
search_syntax (0.1.3)
treetop (~> 1.6)
searchkick (5.4.0)
activemodel (>= 6.1)
hashie
securerandom (0.3.1)
signet (0.19.0)
addressable (~> 2.8)
Expand Down Expand Up @@ -682,6 +688,7 @@ DEPENDENCIES
omniauth-entra-id
omniauth-rails_csrf_protection
omniauth-saml
opensearch-ruby
pagy (~> 9.0.5)
paranoia
pathogen_view_components!
Expand All @@ -696,6 +703,7 @@ DEPENDENCIES
rubocop-graphql
rubocop-rails
search_syntax
searchkick
simplecov
sprockets-rails
stimulus-rails
Expand Down
10 changes: 10 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ bundle && pnpm install
```

Generate credentials:

```bash
EDITOR=nano bin/rails credentials:edit
```
Expand Down Expand Up @@ -98,6 +99,15 @@ Start postgresql service:
sudo systemctl start postgresql.service
```

## OpenSearch Setup

Install [OpenSearch](https://opensearch.org/downloads.html). For [Homebrew](https://brew.sh/), use:

```sh
brew install opensearch
brew services start opensearch
```

## Serve

```bash
Expand Down
4 changes: 2 additions & 2 deletions app/components/samples/table_component.html.erb
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@
<% end %>
<% if column == :puid || column == :name %>
<%= link_to(
project_sample_path(sample.project, sample),
namespace_project_sample_path(sample.project.namespace.parent, sample.project, sample),
data: { turbo: false },
class: "text-slate-700 dark:text-slate-300 font-semibold hover:underline"
) do %>
Expand All @@ -116,7 +116,7 @@
<% end %>
<% elsif column == :project_id %>
<%= link_to sample.project.puid,
project_samples_path(sample.project),
namespace_project_samples_path(sample.project.namespace.parent, sample.project),
data: {
turbo: false,
},
Expand Down
15 changes: 8 additions & 7 deletions app/controllers/groups/samples_controller.rb
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ class SamplesController < Groups::ApplicationController

def index
@timestamp = DateTime.current
@pagy, @samples = pagy(@query.results, limit: params[:limit] || 20)
@pagy, @samples = pagy_searchkick(@query.results(:searchkick_pagy), limit: params[:limit] || 20)
@has_samples = authorized_samples.count.positive?
end

Expand All @@ -26,7 +26,9 @@ def select
respond_to do |format|
format.turbo_stream do
if params[:select].present?
@sample_ids = @query.results.where(updated_at: ..params[:timestamp].to_datetime).select(:id).pluck(:id)
@sample_ids = @query.results(:searchkick)
.where(updated_at: ..params[:timestamp].to_datetime)
.select(:id).pluck(:id)
end
end
end
Expand All @@ -38,10 +40,6 @@ def group
@group = Group.find_by_full_path(params[:group_id]) # rubocop:disable Rails/DynamicFindBy
end

def authorized_projects
authorized_scope(Project, type: :relation, as: :group_projects, scope_options: { group: @group })
end

def authorized_samples
authorized_scope(Sample, type: :relation, as: :namespace_samples,
scope_options: { namespace: @group }).includes(project: { namespace: [{ parent: :route },
Expand Down Expand Up @@ -80,7 +78,10 @@ def query
@search_params = search_params
set_metadata_fields

@query = Sample::Query.new(@search_params.except(:metadata).merge({ project_ids: authorized_projects.select(:id) }))
project_ids =
authorized_scope(Project, type: :relation, as: :group_projects, scope_options: { group: @group }).pluck(:id)

@query = Sample::Query.new(@search_params.except(:metadata).merge({ project_ids: project_ids }))
end

def search_params
Expand Down
6 changes: 4 additions & 2 deletions app/controllers/projects/samples_controller.rb
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ class SamplesController < Projects::ApplicationController # rubocop:disable Metr

def index
@timestamp = DateTime.current
@pagy, @samples = pagy(@query.results, limit: params[:limit] || 20)
@pagy, @samples = pagy_searchkick(@query.results(:searchkick_pagy), limit: params[:limit] || 20)
@has_samples = @project.samples.size.positive?
end

Expand Down Expand Up @@ -83,7 +83,9 @@ def select
respond_to do |format|
format.turbo_stream do
if params[:select].present?
@sample_ids = @query.results.where(updated_at: ..params[:timestamp].to_datetime).select(:id).pluck(:id)
@sample_ids = @query.results(:searchkick)
.where(updated_at: ..params[:timestamp].to_datetime)
.select(:id).pluck(:id)
end
end
end
Expand Down
22 changes: 22 additions & 0 deletions app/models/sample.rb
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,15 @@ class Sample < ApplicationRecord
include HasPuid
include History

extend Pagy::Searchkick

has_logidze
acts_as_paranoid

searchkick \
deep_paging: true,
text_middle: %i[name puid]

belongs_to :project, counter_cache: true

broadcasts_refreshes_to :project
Expand Down Expand Up @@ -79,4 +85,20 @@ def sort_files

{ singles:, pe_forward:, pe_reverse: }
end

def search_data
{
name: name,
puid: puid,
project_id: project_id,
metadata: metadata.as_json,
created_at: created_at,
updated_at: updated_at,
attachments_updated_at: attachments_updated_at
}
end

def should_index?
!deleted?
end
end
51 changes: 47 additions & 4 deletions app/models/sample/query.rb
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ class Sample::Query # rubocop:disable Style/ClassAndModuleChildren
include ActiveModel::Model
include ActiveModel::Attributes

ResultTypeError = Class.new(StandardError)

attribute :column, :string
attribute :direction, :string
attribute :name_or_puid_cont, :string
Expand All @@ -23,16 +25,57 @@ def initialize(...)
def sort=(value)
super
column, direction = sort.split
column = column.gsub('metadata_', 'metadata.') if column.match?(/metadata_/)
assign_attributes(column:, direction:)
end

def results
def results(type = :ransack)
case type
when :ransack
ransack_results
when :searchkick
searchkick_results
when :searchkick_pagy
searchkick_pagy_results
else
raise ResultTypeError, "Unrecognized type: #{type}"
end
end

private

def ransack_results
return Sample.none unless valid?

sort_samples.ransack(ransack_params).result
end

private
def searchkick_pagy_results
return Sample.pagy_search('') unless valid?

Sample.pagy_search(name_or_puid_cont.presence || '*', **searchkick_kwargs)
end

def searchkick_results
return Sample.search('') unless valid?

Sample.search(name_or_puid_cont.presence || '*', **searchkick_kwargs)
end

def searchkick_kwargs
{ fields: [{ name: :text_middle }, { puid: :text_middle }],
misspellings: false,
where: { project_id: project_ids }.merge((
if name_or_puid_in.present?
{ _or: [{ name: name_or_puid_in },
{ puid: name_or_puid_in }] }
else
{}
end
)),
order: { "#{column}": { order: direction, unmapped_type: 'long' } },
includes: [project: { namespace: [{ parent: :route }, :route] }] }
end

def ransack_params
{
Expand All @@ -42,8 +85,8 @@ def ransack_params
end

def sort_samples(scope = Sample.where(project_id: project_ids))
if column.starts_with? 'metadata_'
field = column.gsub('metadata_', '')
if column.starts_with? 'metadata.'
field = column.gsub('metadata.', '')
scope.order(Sample.metadata_sort(field, direction))
else
scope.order("#{column} #{direction}")
Expand Down
5 changes: 5 additions & 0 deletions app/services/samples/destroy_service.rb
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ def destroy_single
sample_destroyed = sample.destroy

if sample_destroyed
Sample.search_index.refresh

@project.namespace.create_activity key: 'namespaces_project_namespace.samples.destroy',
owner: current_user,
parameters:
Expand All @@ -46,10 +48,13 @@ def destroy_multiple # rubocop:disable Metrics/MethodLength
samples = samples.destroy_all

samples.each do |sample|
# Sample.searchkick_index.remove(sample)
update_metadata_summary(sample)
samples_deleted_puids << sample.puid
end

Sample.search_index.refresh

update_samples_count(samples_to_delete_count) if @project.parent.type == 'Group'

@project.namespace.create_activity key: 'namespaces_project_namespace.samples.destroy_multiple',
Expand Down
4 changes: 2 additions & 2 deletions config/initializers/pagy.rb
Original file line number Diff line number Diff line change
Expand Up @@ -103,9 +103,9 @@
# DEFAULT[:searchkick_pagy_search] = :pagy_search
# Default original :search method called internally to do the actual search
# Pagy::DEFAULT[:searchkick_search] = :search
# require 'pagy/extras/searchkick'
require 'pagy/extras/searchkick'
# uncomment if you are going to use Searchkick.pagy_search
# Searchkick.extend Pagy::Searchkick
Searchkick.extend Pagy::Searchkick

# Frontend Extras

Expand Down
8 changes: 8 additions & 0 deletions db/migrate/20241212164410_index_existing_samples.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
# frozen_string_literal: true

# Index existing Samples using SearchKick
class IndexExistingSamples < ActiveRecord::Migration[7.2]
def up
Sample.reindex
end
end
36 changes: 18 additions & 18 deletions db/schema.rb

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 3 additions & 0 deletions db/seeds.rb
Original file line number Diff line number Diff line change
Expand Up @@ -803,4 +803,7 @@ def seed_exports # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metric
DataExport.suppressing_turbo_broadcasts do
seed_exports
end

# index samples using searchkick
Sample.reindex
end
Loading
Loading