From 65b87720bb78b0e813a52c1a1970b084cbd9ed8c Mon Sep 17 00:00:00 2001 From: Alexei Peters Date: Thu, 2 May 2019 13:25:01 -0700 Subject: [PATCH] get backend search working with the term-search, re #4771 --- .../js/views/components/search/term-filter.js | 19 ++-- arches/app/media/js/views/search.js | 5 +- arches/app/search/components/term_filter.py | 47 +++++++++- arches/app/views/search.py | 93 +++++++++++-------- 4 files changed, 110 insertions(+), 54 deletions(-) diff --git a/arches/app/media/js/views/components/search/term-filter.js b/arches/app/media/js/views/components/search/term-filter.js index 0c5c30a9a55..85fa1a8db81 100644 --- a/arches/app/media/js/views/components/search/term-filter.js +++ b/arches/app/media/js/views/components/search/term-filter.js @@ -5,7 +5,8 @@ define([ 'views/components/search/base-filter', 'bindings/term-search' ], function(ko, koMapping, _, BaseFilter, termSearchComponent) { - return ko.components.register('term-filter', { + var component_name = 'term-filter'; + return ko.components.register(component_name, { viewModel: BaseFilter.extend({ initialize: function(options) { options.name = 'Term Filter'; @@ -21,7 +22,7 @@ define([ this.updateQuery(); }, this); - options.filters['term-filter'](this); + options.filters[component_name](this); this.restoreState(); }, @@ -33,22 +34,22 @@ define([ var queryObj = this.query(); if (terms.length > 0){ - queryObj.termFilter = ko.toJSON(terms); + queryObj[component_name] = ko.toJSON(terms); } else { - delete queryObj.termFilter; + delete queryObj[component_name]; } this.query(queryObj); }, restoreState: function() { var query = this.query(); - if ('termFilter' in query) { - query.termFilter = JSON.parse(query.termFilter); - if (query.termFilter.length > 0) { - query.termFilter.forEach(function(term){ + if (component_name in query) { + query[component_name] = JSON.parse(query[component_name]); + if (query[component_name].length > 0) { + query[component_name].forEach(function(term){ term.inverted = ko.observable(term.inverted); }); - this.filter.terms(query.termFilter); + this.filter.terms(query[component_name]); } } }, diff --git a/arches/app/media/js/views/search.js b/arches/app/media/js/views/search.js index 6cb6d3b60d6..674bac9015c 100644 --- a/arches/app/media/js/views/search.js +++ b/arches/app/media/js/views/search.js @@ -83,13 +83,12 @@ define([ this.viewModel.sharedStateObject = new CommonSearchViewModel(); _.extend(this, this.viewModel.sharedStateObject); - filtersLoaded = ko.computed(function(){ + var filtersLoaded = ko.computed(function(){ var allLoaded = true; var filters = _.filter(this.filters, function(filter, key) { - var filter = _.find(this.filtersList, function(f) { + return _.find(this.filtersList, function(f) { return f.enabled && f.componentname === key; }, this); - return filter; }, this); _.each(filters, function(value, key, list) { if (!value()) { diff --git a/arches/app/search/components/term_filter.py b/arches/app/search/components/term_filter.py index 7886fa2e437..00614e97fb9 100644 --- a/arches/app/search/components/term_filter.py +++ b/arches/app/search/components/term_filter.py @@ -1,3 +1,6 @@ +from arches.app.utils.betterJSONSerializer import JSONDeserializer +from arches.app.search.elasticsearch_dsl_builder import Bool, Match, Query, Nested, Term, Terms, GeoShape, Range, MinAgg, MaxAgg, RangeAgg, Aggregation, GeoHashGridAgg, GeoBoundsAgg, FiltersAgg, NestedAgg + details = { "searchcomponentid": "", "name": "Term Filter", @@ -15,5 +18,45 @@ class TermFilter(): - def append_dsl(self, dsl): - pass + def append_dsl(self, querysting_params, search_query, permitted_nodegroups, include_provisional): + for term in JSONDeserializer().deserialize(querysting_params): + term_query = Bool() + provisional_term_filter = Bool() + if term['type'] == 'term' or term['type'] == 'string': + string_filter = Bool() + if term['type'] == 'term': + string_filter.must(Match(field='strings.string', query=term['value'], type='phrase')) + elif term['type'] == 'string': + string_filter.should(Match(field='strings.string', query=term['value'], type='phrase_prefix')) + string_filter.should(Match(field='strings.string.folded', query=term['value'], type='phrase_prefix')) + + if include_provisional == False: + string_filter.must_not(Match(field='strings.provisional', query='true', type='phrase')) + elif include_provisional == 'only provisional': + string_filter.must_not(Match(field='strings.provisional', query='false', type='phrase')) + + string_filter.filter(Terms(field='strings.nodegroup_id', terms=permitted_nodegroups)) + nested_string_filter = Nested(path='strings', query=string_filter) + if term['inverted']: + search_query.must_not(nested_string_filter) + else: + search_query.must(nested_string_filter) + # need to set min_score because the query returns results with score 0 and those have to be removed, which I don't think it should be doing + query.min_score('0.01') + elif term['type'] == 'concept': + concept_ids = _get_child_concepts(term['value']) + conceptid_filter = Bool() + conceptid_filter.filter(Terms(field='domains.conceptid', terms=concept_ids)) + conceptid_filter.filter(Terms(field='domains.nodegroup_id', terms=permitted_nodegroups)) + + if include_provisional == False: + conceptid_filter.must_not(Match(field='domains.provisional', query='true', type='phrase')) + elif include_provisional == 'only provisional': + conceptid_filter.must_not(Match(field='domains.provisional', query='false', type='phrase')) + + nested_conceptid_filter = Nested(path='domains', query=conceptid_filter) + if term['inverted']: + search_query.must_not(nested_conceptid_filter) + else: + search_query.filter(nested_conceptid_filter) + diff --git a/arches/app/views/search.py b/arches/app/views/search.py index 2ca3f90329a..e8d740326fb 100644 --- a/arches/app/views/search.py +++ b/arches/app/views/search.py @@ -34,6 +34,7 @@ from arches.app.models.system_settings import settings from arches.app.utils.pagination import get_paginator from arches.app.utils.response import JSONResponse +from arches.app.utils.module_importer import get_class_from_modulename from arches.app.utils.betterJSONSerializer import JSONSerializer, JSONDeserializer from arches.app.utils.date_utils import ExtendedDateFormat from arches.app.search.search_engine_factory import SearchEngineFactory @@ -357,47 +358,59 @@ def build_search_results_dsl(request): nested_agg.add_aggregation(nested_agg_filter) query.add_aggregation(nested_agg) - if term_filter != '': - for term in JSONDeserializer().deserialize(term_filter): - term_query = Bool() - provisional_term_filter = Bool() - if term['type'] == 'term' or term['type'] == 'string': - string_filter = Bool() - if term['type'] == 'term': - string_filter.must(Match(field='strings.string', query=term['value'], type='phrase')) - elif term['type'] == 'string': - string_filter.should(Match(field='strings.string', query=term['value'], type='phrase_prefix')) - string_filter.should(Match(field='strings.string.folded', query=term['value'], type='phrase_prefix')) - if include_provisional == False: - string_filter.must_not(Match(field='strings.provisional', query='true', type='phrase')) - elif include_provisional == 'only provisional': - string_filter.must_not(Match(field='strings.provisional', query='false', type='phrase')) - - string_filter.filter(Terms(field='strings.nodegroup_id', terms=permitted_nodegroups)) - nested_string_filter = Nested(path='strings', query=string_filter) - if term['inverted']: - search_query.must_not(nested_string_filter) - else: - search_query.must(nested_string_filter) - # need to set min_score because the query returns results with score 0 and those have to be removed, which I don't think it should be doing - query.min_score('0.01') - elif term['type'] == 'concept': - concept_ids = _get_child_concepts(term['value']) - conceptid_filter = Bool() - conceptid_filter.filter(Terms(field='domains.conceptid', terms=concept_ids)) - conceptid_filter.filter(Terms(field='domains.nodegroup_id', terms=permitted_nodegroups)) - - if include_provisional == False: - conceptid_filter.must_not(Match(field='domains.provisional', query='true', type='phrase')) - elif include_provisional == 'only provisional': - conceptid_filter.must_not(Match(field='domains.provisional', query='false', type='phrase')) - - nested_conceptid_filter = Nested(path='domains', query=conceptid_filter) - if term['inverted']: - search_query.must_not(nested_conceptid_filter) - else: - search_query.filter(nested_conceptid_filter) + search_components = models.SearchComponent.objects.all() + def get_filter(filtertype): + for component in search_components: + if component.componentname == 'term-filter': + return get_class_from_modulename(component.modulename, component.classname, settings.SEARCH_COMPONENT_LOCATIONS) + + for filter_type, querystring in request.GET.items(): + if filter_type == 'term-filter': + search_filter = get_filter(filter_type)() + search_filter.append_dsl(querystring, search_query, permitted_nodegroups, include_provisional) + + # if term_filter != '': + # for term in JSONDeserializer().deserialize(term_filter): + # term_query = Bool() + # provisional_term_filter = Bool() + # if term['type'] == 'term' or term['type'] == 'string': + # string_filter = Bool() + # if term['type'] == 'term': + # string_filter.must(Match(field='strings.string', query=term['value'], type='phrase')) + # elif term['type'] == 'string': + # string_filter.should(Match(field='strings.string', query=term['value'], type='phrase_prefix')) + # string_filter.should(Match(field='strings.string.folded', query=term['value'], type='phrase_prefix')) + + # if include_provisional == False: + # string_filter.must_not(Match(field='strings.provisional', query='true', type='phrase')) + # elif include_provisional == 'only provisional': + # string_filter.must_not(Match(field='strings.provisional', query='false', type='phrase')) + + # string_filter.filter(Terms(field='strings.nodegroup_id', terms=permitted_nodegroups)) + # nested_string_filter = Nested(path='strings', query=string_filter) + # if term['inverted']: + # search_query.must_not(nested_string_filter) + # else: + # search_query.must(nested_string_filter) + # # need to set min_score because the query returns results with score 0 and those have to be removed, which I don't think it should be doing + # query.min_score('0.01') + # elif term['type'] == 'concept': + # concept_ids = _get_child_concepts(term['value']) + # conceptid_filter = Bool() + # conceptid_filter.filter(Terms(field='domains.conceptid', terms=concept_ids)) + # conceptid_filter.filter(Terms(field='domains.nodegroup_id', terms=permitted_nodegroups)) + + # if include_provisional == False: + # conceptid_filter.must_not(Match(field='domains.provisional', query='true', type='phrase')) + # elif include_provisional == 'only provisional': + # conceptid_filter.must_not(Match(field='domains.provisional', query='false', type='phrase')) + + # nested_conceptid_filter = Nested(path='domains', query=conceptid_filter) + # if term['inverted']: + # search_query.must_not(nested_conceptid_filter) + # else: + # search_query.filter(nested_conceptid_filter) if type_filter != '': for resouceTypeFilter in JSONDeserializer().deserialize(type_filter):