Skip to content

Commit

Permalink
search: add prefix filter; filter by cms working group
Browse files Browse the repository at this point in the history 
Signed-off-by: Anna Trzcinska <[email protected]>
  • Loading branch information
@annatrz
annatrz committed Nov 12, 2019
1 parent 4a72255 commit b371189
Show file tree
Hide file tree
Showing 6 changed files with 121 additions and 185 deletions.
2 changes: 1 addition & 1 deletion .travis.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ services:
- rabbitmq

env:
- REQUIREMENTS=lowest E2E="no" SQLALCHEMY_DATABASE_URI="postgresql+psycopg2://postgres@localhost:5432/cap" ES_VERSION=5.0.1
- REQUIREMENTS=lowest E2E="no" SQLALCHEMY_DATABASE_URI="postgresql+psycopg2://postgres@localhost:5432/cap" ES_VERSION=5.6.4

before_install:
- "./scripts/ci/prebuild.sh"
Expand Down
264 changes: 91 additions & 173 deletions cap/config.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@
from cap.modules.oauthclient.rest_handlers import (authorized_signup_handler,
signup_handler)
from cap.modules.records.permissions import ReadRecordPermission
from cap.modules.search.facets import nested_filter
from cap.modules.search.facets import nested_filter, prefix_filter


def _(x):
Expand Down Expand Up @@ -138,17 +138,20 @@ def _(x):
#: It should be changed before deploying.
SECRET_KEY = 'CHANGE_ME'
#: Max upload size for form data via application/mulitpart-formdata.
MAX_CONTENT_LENGTH = 100 * 1024 * 1024 # 100 MiB
MAX_CONTENT_LENGTH = 100 * 1024 * 1024 # 100 MiB
#: Sets cookie with the secure flag by default
SESSION_COOKIE_SECURE = False
#: Since HAProxy and Nginx route all requests no matter the host header
#: provided, the allowed hosts variable is set to localhost. In production it
#: should be set to the correct host and it is strongly recommended to only
#: route correct hosts to the application.
APP_ALLOWED_HOSTS = [
'localhost', 'analysispreservation.web.cern.ch',
'analysispreservation.cern.ch', 'analysispreservation-dev.web.cern.ch',
'analysispreservation-dev.cern.ch', 'analysispreservation-qa.web.cern.ch',
'localhost',
'analysispreservation.web.cern.ch',
'analysispreservation.cern.ch',
'analysispreservation-dev.web.cern.ch',
'analysispreservation-dev.cern.ch',
'analysispreservation-qa.web.cern.ch',
'analysispreservation-qa.cern.ch'
]

Expand Down Expand Up @@ -231,181 +234,101 @@ def _(x):
# =======
#: Records sort/facets options
RECORDS_REST_SORT_OPTIONS = dict(records=dict(
bestmatch=dict(
title=_('Best match'),
fields=['_score'],
order=1,
),
mostrecent=dict(
title=_('Most recent'),
fields=['_updated'],
default_order='desc',
order=2,
),
bestmatch=dict(title=_('Best match'), fields=['_score'], order=1, ),
mostrecent=dict(title=_('Most recent'),
fields=['_updated'],
default_order='desc',
order=2,
),
))

RECORDS_REST_SORT_OPTIONS.update(DEPOSIT_REST_SORT_OPTIONS)

#: Record search facets.
# for aggregations, only ones starting with facet_ will be displayed on a page
RECORDS_REST_FACETS = {
'deposits': {
'aggs': {
'facet_status': {
'terms': {
'field': 'status.keyword'
}
},
'facet_type': {
'terms': {
'field': '_type'
}
},
'facet_cadi_status': {
'terms': {
'field': 'cadi_status'
}
},
'facet_publication_status': {
'terms': {
'field': 'publication_status.keyword'
}
},
'facet_cms_wg': {
'terms': {
"script": {
"inline":
"doc.containsKey('cadi_id') ? doc['cadi_id.keyword'].value.substring(0,3) : null" # noqa
}
}
},
"particles": {
"nested": {
"path": "main_measurements.signal_event_selection"
".physics_objects"
},
"aggs": {
"facet_physics_objects": {
"terms": {
"field": "main_measurements.signal_event_selection"
".physics_objects.object",
"exclude": ""
},
"aggs": {
"doc_count": {
"reverse_nested": {}
},
"facet_physics_objects_type": {
"terms": {
"field": "main_measurements"
".signal_event_selection"
".physics_objects"
".object_type.keyword"
},
"aggs": {
"doc_count": {
"reverse_nested": {}
}
}
}
}
},
}
},
CAP_FACETS = {
'aggs': {
'facet_type': {
'terms': {
'field': '_type'
}
},
'post_filters': {
'type': terms_filter('_type'),
'status': terms_filter('status.keyword'),
'cadi_status': terms_filter('cadi_status'),
'cms_wg': terms_filter('cadi_id'),
'publication_status': terms_filter('publication_status.keyword'),
'conference': terms_filter('conference'),
'physics_objects': nested_filter(
'main_measurements.signal_event_selection.physics_objects',
'main_measurements.signal_event_selection'
'.physics_objects.object'),
'physics_objects_type': nested_filter(
'main_measurements.signal_event_selection.physics_objects',
'main_measurements.signal_event_selection.physics_objects'
'.object_type.keyword'),
}
},
'records': {
'aggs': {
'facet_type': {
'terms': {
'field': '_type'
}
},
'facet_cadi_status': {
'terms': {
'field': 'cadi_status'
}
},
'facet_publication_status': {
'terms': {
'field': 'publication_status.keyword'
}
'facet_cadi_status': {
'terms': {
'field': 'cadi_status'
}
},
'facet_publication_status': {
'terms': {
'field': 'publication_status.keyword'
}
},
'facet_cms_working_group': {
'terms': {
"script": "doc.containsKey('cadi_id') ? doc['cadi_id'].value?.substring(0,3) : null" # noqa
}
},
"particles": {
"nested": {
"path": "main_measurements.signal_event_selection"
".physics_objects"
},
"particles": {
"nested": {
"path": "main_measurements.signal_event_selection"
".physics_objects"
},
"aggs": {
"facet_physics_objects": {
"terms": {
"field": "main_measurements.signal_event_selection"
".physics_objects.object",
"exclude": ""
"aggs": {
"facet_physics_objects": {
"terms": {
"field": "main_measurements.signal_event_selection"
".physics_objects.object",
"exclude": ""
},
"aggs": {
"doc_count": {
"reverse_nested": {}
},
"aggs": {
"doc_count": {
"reverse_nested": {}
"facet_physics_objects_type": {
"terms": {
"field": "main_measurements"
".signal_event_selection"
".physics_objects"
".object_type.keyword"
},
"facet_physics_objects_type": {
"terms": {
"field": "main_measurements"
".signal_event_selection"
".physics_objects"
".object_type.keyword"
},
"aggs": {
"doc_count": {
"reverse_nested": {}
}
"aggs": {
"doc_count": {
"reverse_nested": {}
}
}
}
},
}
},
}
},
}
},
'post_filters': {
'type': terms_filter('_type'),
'cadi_status': terms_filter('cadi_status'),
'publication_status': terms_filter('publication_status.keyword'),
'conference': terms_filter('conference'),
'physics_objects': nested_filter(
'main_measurements.signal_event_selection.physics_objects',
'main_measurements.signal_event_selection'
'.physics_objects.object'),
'physics_objects_type': nested_filter(
'main_measurements.signal_event_selection.physics_objects',
'main_measurements.signal_event_selection.physics_objects'
'.object_type.keyword'),
}
},
'post_filters': {
'type': terms_filter('_type'),
'cms_working_group': prefix_filter('cadi_id'),
'publication_status': terms_filter('publication_status.keyword'),
'cadi_status': terms_filter('cadi_status'),
'conference': terms_filter('conference'),
'physics_objects': nested_filter(
'main_measurements.signal_event_selection.physics_objects',
'main_measurements.signal_event_selection'
'.physics_objects.object'),
'physics_objects_type': nested_filter(
'main_measurements.signal_event_selection.physics_objects',
'main_measurements.signal_event_selection.physics_objects'
'.object_type.keyword')
}
}

RECORDS_REST_FACETS = {'deposits': CAP_FACETS, 'records': CAP_FACETS}

#: Records REST API endpoints.
RECORDS_REST_ENDPOINTS = copy.deepcopy(RECORDS_REST_ENDPOINTS)

RECORDS_REST_ENDPOINTS['recid'].update({
'record_class': 'cap.modules.records.api:CAPRecord',
'pid_fetcher': 'cap_record_fetcher',
'search_class': 'cap.modules.records.search:CAPRecordSearch',
'search_factory_imp': 'cap.modules.search.query'
':cap_search_factory',
'search_factory_imp': 'cap.modules.search.query:cap_search_factory',
'record_serializers': {
'application/json': ('cap.modules.records.serializers'
':record_json_v1_response'),
Expand Down Expand Up @@ -505,10 +428,9 @@ def _(x):
# Update CERN OAuth handlers - due to REST - mostly only redirect urls
# and error flashing
CERN_REMOTE_APP.update(
dict(
authorized_handler=authorized_signup_handler,
disconnect_handler=disconnect_handler,
))
dict(authorized_handler=authorized_signup_handler,
disconnect_handler=disconnect_handler,
))

CERN_REMOTE_APP['signup_handler']['view'] = signup_handler

Expand All @@ -525,9 +447,9 @@ def _(x):

JSONSCHEMAS_LOADER_CLS = json_loader_factory(
JSONResolver(plugins=[
'cap.modules.schemas.resolvers',
'cap.modules.schemas.resolvers_api',
], ))
'cap.modules.schemas.resolvers', 'cap.modules.schemas.resolvers_api'
],
))

# WARNING: Do not share the secret key - especially do not commit it to
# version control.
Expand Down Expand Up @@ -609,16 +531,14 @@ def _(x):
':basic_json_v1_search')
},
'files_serializers': {
'application/json': ('cap.modules.deposit.serializers'
':files_response'),
'application/json': ('cap.modules.deposit.serializers:files_response'),
},
'search_class': 'cap.modules.deposit.search:CAPDepositSearch',
'search_factory_imp': 'cap.modules.search.query'
':cap_search_factory',
'search_factory_imp': 'cap.modules.search.query:cap_search_factory',
'item_route': '/deposits/<{0}:pid_value>'.format(_PID),
'file_list_route': '/deposits/<{0}:pid_value>/files'.format(_PID),
'file_item_route':
'/deposits/<{0}:pid_value>/files/<path:key>'.format(_PID),
'file_item_route': '/deposits/<{0}:pid_value>/files/<path:key>'.format(
_PID),
'create_permission_factory_imp': check_oauth2_scope(
lambda record: CreateDepositPermission(record).can(), write_scope.id),
'read_permission_factory_imp': check_oauth2_scope(
Expand Down Expand Up @@ -675,9 +595,7 @@ def _(x):
# Keytabs
KEYTABS_LOCATION = os.environ.get('APP_KEYTABS_LOCATION', '/etc/keytabs')

KRB_PRINCIPALS = {
'CADI': (CMS_USER_PRINCIPAL, CMS_USER_KEYTAB),
}
KRB_PRINCIPALS = {'CADI': (CMS_USER_PRINCIPAL, CMS_USER_KEYTAB)}

CERN_CERTS_PEM = os.environ.get('APP_CERN_CERTS_PEM')

Expand Down
3 changes: 3 additions & 0 deletions cap/modules/fixtures/schemas/cms.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -425,6 +425,9 @@
"type": "date",
"copy_to": "updated"
},
"cadi_id": {
"type": "keyword"
},
"basic_info": {
"type": "object",
"properties": {
Expand Down
Loading

0 comments on commit b371189

Please sign in to comment.