Merge pull request #104 from raccoongang/golub-sergey/BD-19/transitio…

…n-from-ES1.5-to-ES7

[BD-19] Transition to the Elasticsearch 7.8.0 version

.travis.yml

@@ -8,27 +8,21 @@ envs:
 - TOXENV=django22
 - TOXENV=quality
 
-addons:
-  apt:
-    packages:
-      - openjdk-8-jdk
+services:
+  - docker
 
 # Cache the pip directory. "cache: pip" doesn't work due to install override. See https://github.com/travis-ci/travis-ci/issues/3239.
 cache:
   - directories:
     - $HOME/.cache/pip
 
-before_install:
-  - export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-amd64
-  - export PATH=${PATH/\/usr\/local\/lib\/jvm\/openjdk11\/bin/$JAVA_HOME\/bin}
-  - curl -O https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-1.5.2.zip
-  - unzip elasticsearch-1.5.2.zip
-  - elasticsearch-1.5.2/bin/elasticsearch -d
+before_install: make test.start_elasticsearch
 
 install:
   - pip install -r requirements/travis.txt
 
-script: tox
+# sleep needs for waiting on elasticsearch
+script: sleep 10 && tox
 
 after_success: codecov
 

Makefile

@@ -23,6 +23,16 @@ requirements:
 validate: clean
 	tox
 
+test.start_elasticsearch:
+	docker-compose up -d
+
+test.stop_elasticsearch:
+	docker-compose stop
+
+test_with_es: clean test.start_elasticsearch
+	coverage run --source='.' manage.py test
+	make test.stop_elasticsearch
+
 upgrade: export CUSTOM_COMPILE_COMMAND=make upgrade
 upgrade: ## update the requirements/*.txt files with the latest packages satisfying requirements/*.in
 	pip install -qr requirements/pip-tools.txt

README.md

@@ -252,3 +252,17 @@ Users of the `views.do_search` view can choose to override the SearchResultProce
 In particular, the base SearchResultProcessor adds in each python property alongside the properties within the results. So, the override class simply defines properties that it desires to be exposed alongside the results - for example, the LMS includes an `LmsSearchResultProcessor` that defines properties `url` and `location` which are used to infer the url / location for each search result. In this fashion, the search client can blend in information that it infers from the result fields, but it knows how to format / calculate.
 
 Also, SearchResultProcessor overriders can override the member `should_remove` which allows the client app to determine if access should be excluded to the search result - for example, the LMS includes an implementation of this member that calls the LMS `has_access` as a last safety resort in case the end user does not have access to the result returned.
+
+#### Testing
+Tests use an Elasticsearch Docker container. To run tests locally use command:
+```
+make test_with_es
+```
+To simply run the container without starting the tests, run:
+```
+make test.start_elasticsearch
+```
+To stop an Elasticsearch Docker container, run:
+```
+make test.stop_elasticsearch
+```

docker-compose.yml

@@ -0,0 +1,25 @@
+version: '2.2'
+services:
+
+  test_elasticsearch:
+    image: elasticsearch:7.8.0
+    container_name: test_elasticsearch
+    environment:
+      - node.name=test_elasticsearch
+      - cluster.name=docker-cluster
+      - cluster.initial_master_nodes=test_elasticsearch
+      - bootstrap.memory_lock=true
+      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+      - http.port=9200
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+    volumes:
+      - data01:/usr/share/elasticsearch/data
+    ports:
+      - "9200:9200"
+
+volumes:
+  data01:
+    driver: local

requirements/base.in

@@ -12,5 +12,5 @@
 
 
 Django                       # Web application framework
-elasticsearch>=1.0.0,<2.0.0
+elasticsearch>=7.8.0,<8.0.0
 event-tracking

requirements/base.txt

@@ -4,11 +4,12 @@
 #
 #    make upgrade
 #
-django==2.2.14            # via -r requirements/base.in, event-tracking
-elasticsearch==1.9.0      # via -r requirements/base.in
+certifi==2020.6.20        # via elasticsearch
+django==2.2.15            # via -r requirements/base.in, event-tracking
+elasticsearch==7.8.1      # via -r requirements/base.in
 event-tracking==0.3.3     # via -r requirements/base.in
-pymongo==3.10.1           # via event-tracking
+pymongo==3.11.0           # via event-tracking
 pytz==2020.1              # via django, event-tracking
 six==1.15.0               # via event-tracking
 sqlparse==0.3.1           # via django
-urllib3==1.25.9           # via elasticsearch
+urllib3==1.25.10          # via elasticsearch

requirements/constraints.txt

@@ -12,7 +12,7 @@
 ddt < 1.4.0
 
 # newer version is causing failure
-tox-battery==0.5.2
+tox-battery==0.6.1
 
 
 coverage<5.1

requirements/dev.txt

@@ -7,52 +7,52 @@
 appdirs==1.4.4            # via -r requirements/travis.txt, virtualenv
 astroid==2.3.3            # via -r requirements/quality.txt, pylint, pylint-celery
 attrs==19.3.0             # via -r requirements/quality.txt, -r requirements/testing.txt, pytest
-certifi==2020.6.20        # via -r requirements/travis.txt, requests
+certifi==2020.6.20        # via -r requirements/quality.txt, -r requirements/testing.txt, -r requirements/travis.txt, elasticsearch, requests
 chardet==3.0.4            # via -r requirements/travis.txt, requests
 click-log==0.3.2          # via -r requirements/quality.txt, edx-lint
 click==7.1.2              # via -r requirements/pip-tools.txt, -r requirements/quality.txt, click-log, edx-lint, pip-tools
 codecov==2.1.8            # via -r requirements/travis.txt
 coverage==5.0.4           # via -c requirements/constraints.txt, -r requirements/quality.txt, -r requirements/testing.txt, -r requirements/travis.txt, codecov, pytest-cov
 ddt==1.3.1                # via -c requirements/constraints.txt, -r requirements/quality.txt, -r requirements/testing.txt
 distlib==0.3.1            # via -r requirements/travis.txt, virtualenv
-django==2.2.14            # via -r requirements/quality.txt, -r requirements/testing.txt, event-tracking
+django==2.2.15            # via -r requirements/quality.txt, -r requirements/testing.txt, event-tracking
 edx-lint==1.5.0           # via -r requirements/quality.txt
-elasticsearch==1.9.0      # via -r requirements/quality.txt, -r requirements/testing.txt
+elasticsearch==7.8.1      # via -r requirements/quality.txt, -r requirements/testing.txt
 event-tracking==0.3.3     # via -r requirements/quality.txt, -r requirements/testing.txt
 filelock==3.0.12          # via -r requirements/travis.txt, tox, virtualenv
 idna==2.10                # via -r requirements/travis.txt, requests
 importlib-metadata==1.7.0  # via -r requirements/quality.txt, -r requirements/testing.txt, -r requirements/travis.txt, pluggy, pytest, tox, virtualenv
 importlib-resources==3.0.0  # via -r requirements/travis.txt, virtualenv
+iniconfig==1.0.1          # via -r requirements/quality.txt, -r requirements/testing.txt, pytest
 isort==4.3.21             # via -r requirements/quality.txt, pylint
 lazy-object-proxy==1.4.3  # via -r requirements/quality.txt, astroid
 mccabe==0.6.1             # via -r requirements/quality.txt, pylint
 mock==3.0.5               # via -r requirements/quality.txt, -r requirements/testing.txt
 more-itertools==8.4.0     # via -r requirements/quality.txt, -r requirements/testing.txt, pytest
 packaging==20.4           # via -r requirements/quality.txt, -r requirements/testing.txt, -r requirements/travis.txt, pytest, tox
 pathlib2==2.3.5           # via -r requirements/quality.txt, -r requirements/testing.txt, pytest
-pip-tools==5.2.1          # via -r requirements/pip-tools.txt
+pip-tools==5.3.1          # via -r requirements/pip-tools.txt
 pluggy==0.13.1            # via -r requirements/quality.txt, -r requirements/testing.txt, -r requirements/travis.txt, pytest, tox
 py==1.9.0                 # via -r requirements/quality.txt, -r requirements/testing.txt, -r requirements/travis.txt, pytest, tox
 pycodestyle==2.6.0        # via -r requirements/quality.txt
 pylint-celery==0.3        # via -r requirements/quality.txt, edx-lint
 pylint-django==2.0.11     # via -r requirements/quality.txt, edx-lint
 pylint-plugin-utils==0.6  # via -r requirements/quality.txt, pylint-celery, pylint-django
 pylint==2.4.4             # via -r requirements/quality.txt, edx-lint, pylint-celery, pylint-django, pylint-plugin-utils
-pymongo==3.10.1           # via -r requirements/quality.txt, -r requirements/testing.txt, event-tracking
+pymongo==3.11.0           # via -r requirements/quality.txt, -r requirements/testing.txt, event-tracking
 pyparsing==2.4.7          # via -r requirements/quality.txt, -r requirements/testing.txt, -r requirements/travis.txt, packaging
 pytest-cov==2.10.0        # via -r requirements/quality.txt, -r requirements/testing.txt
-pytest==5.4.3             # via -r requirements/quality.txt, -r requirements/testing.txt, pytest-cov
+pytest==6.0.1             # via -r requirements/quality.txt, -r requirements/testing.txt, pytest-cov
 pytz==2020.1              # via -r requirements/quality.txt, -r requirements/testing.txt, django, event-tracking
 requests==2.24.0          # via -r requirements/travis.txt, codecov
 six==1.15.0               # via -r requirements/pip-tools.txt, -r requirements/quality.txt, -r requirements/testing.txt, -r requirements/travis.txt, astroid, edx-lint, event-tracking, mock, packaging, pathlib2, pip-tools, tox, virtualenv
 sqlparse==0.3.1           # via -r requirements/quality.txt, -r requirements/testing.txt, django
-toml==0.10.1              # via -r requirements/travis.txt, tox
-tox-battery==0.5.2        # via -c requirements/constraints.txt, -r requirements/travis.txt
-tox==3.17.1               # via -r requirements/travis.txt, tox-battery
+toml==0.10.1              # via -r requirements/quality.txt, -r requirements/testing.txt, -r requirements/travis.txt, pytest, tox
+tox-battery==0.6.1        # via -c requirements/constraints.txt, -r requirements/travis.txt
+tox==3.18.1               # via -r requirements/travis.txt, tox-battery
 typed-ast==1.4.1          # via -r requirements/quality.txt, astroid
-urllib3==1.25.9           # via -r requirements/quality.txt, -r requirements/testing.txt, -r requirements/travis.txt, elasticsearch, requests
-virtualenv==20.0.27       # via -r requirements/travis.txt, tox
-wcwidth==0.2.5            # via -r requirements/quality.txt, -r requirements/testing.txt, pytest
+urllib3==1.25.10          # via -r requirements/quality.txt, -r requirements/testing.txt, -r requirements/travis.txt, elasticsearch, requests
+virtualenv==20.0.29       # via -r requirements/travis.txt, tox
 wrapt==1.11.2             # via -r requirements/quality.txt, astroid
 zipp==1.2.0               # via -r requirements/quality.txt, -r requirements/testing.txt, -r requirements/travis.txt, importlib-metadata, importlib-resources
 

requirements/pip-tools.txt

@@ -5,7 +5,7 @@
 #    make upgrade
 #
 click==7.1.2              # via pip-tools
-pip-tools==5.2.1          # via -r requirements/pip-tools.in
+pip-tools==5.3.1          # via -r requirements/pip-tools.in
 six==1.15.0               # via pip-tools
 
 # The following packages are considered to be unsafe in a requirements file:

requirements/quality.txt

@@ -6,15 +6,17 @@
 #
 astroid==2.3.3            # via pylint, pylint-celery
 attrs==19.3.0             # via -r requirements/testing.txt, pytest
+certifi==2020.6.20        # via -r requirements/testing.txt, elasticsearch
 click-log==0.3.2          # via edx-lint
 click==7.1.2              # via click-log, edx-lint
 coverage==5.0.4           # via -c requirements/constraints.txt, -r requirements/quality.in, -r requirements/testing.txt, pytest-cov
 ddt==1.3.1                # via -c requirements/constraints.txt, -r requirements/testing.txt
-django==2.2.14            # via -r requirements/testing.txt, event-tracking
+django==2.2.15            # via -r requirements/testing.txt, event-tracking
 edx-lint==1.5.0           # via -r requirements/quality.in
-elasticsearch==1.9.0      # via -r requirements/testing.txt
+elasticsearch==7.8.1      # via -r requirements/testing.txt
 event-tracking==0.3.3     # via -r requirements/testing.txt
 importlib-metadata==1.7.0  # via -r requirements/testing.txt, pluggy, pytest
+iniconfig==1.0.1          # via -r requirements/testing.txt, pytest
 isort==4.3.21             # via pylint
 lazy-object-proxy==1.4.3  # via astroid
 mccabe==0.6.1             # via pylint
@@ -29,15 +31,15 @@ pylint-celery==0.3        # via edx-lint
 pylint-django==2.0.11     # via edx-lint
 pylint-plugin-utils==0.6  # via pylint-celery, pylint-django
 pylint==2.4.4             # via edx-lint, pylint-celery, pylint-django, pylint-plugin-utils
-pymongo==3.10.1           # via -r requirements/testing.txt, event-tracking
+pymongo==3.11.0           # via -r requirements/testing.txt, event-tracking
 pyparsing==2.4.7          # via -r requirements/testing.txt, packaging
 pytest-cov==2.10.0        # via -r requirements/testing.txt
-pytest==5.4.3             # via -r requirements/testing.txt, pytest-cov
+pytest==6.0.1             # via -r requirements/testing.txt, pytest-cov
 pytz==2020.1              # via -r requirements/testing.txt, django, event-tracking
 six==1.15.0               # via -r requirements/testing.txt, astroid, edx-lint, event-tracking, mock, packaging, pathlib2
 sqlparse==0.3.1           # via -r requirements/testing.txt, django
+toml==0.10.1              # via -r requirements/testing.txt, pytest
 typed-ast==1.4.1          # via astroid
-urllib3==1.25.9           # via -r requirements/testing.txt, elasticsearch
-wcwidth==0.2.5            # via -r requirements/testing.txt, pytest
+urllib3==1.25.10          # via -r requirements/testing.txt, elasticsearch
 wrapt==1.11.2             # via astroid
 zipp==1.2.0               # via -r requirements/testing.txt, importlib-metadata

requirements/testing.txt

@@ -5,24 +5,26 @@
 #    make upgrade
 #
 attrs==19.3.0             # via pytest
+certifi==2020.6.20        # via -r requirements/base.txt, elasticsearch
 coverage==5.0.4           # via -c requirements/constraints.txt, -r requirements/testing.in, pytest-cov
 ddt==1.3.1                # via -c requirements/constraints.txt, -r requirements/testing.in
-elasticsearch==1.9.0      # via -r requirements/base.txt
+elasticsearch==7.8.1      # via -r requirements/base.txt
 event-tracking==0.3.3     # via -r requirements/base.txt
 importlib-metadata==1.7.0  # via pluggy, pytest
+iniconfig==1.0.1          # via pytest
 mock==3.0.5               # via -r requirements/testing.in
 more-itertools==8.4.0     # via pytest
 packaging==20.4           # via pytest
 pathlib2==2.3.5           # via pytest
 pluggy==0.13.1            # via pytest
 py==1.9.0                 # via pytest
-pymongo==3.10.1           # via -r requirements/base.txt, event-tracking
+pymongo==3.11.0           # via -r requirements/base.txt, event-tracking
 pyparsing==2.4.7          # via packaging
 pytest-cov==2.10.0        # via -r requirements/testing.in
-pytest==5.4.3             # via pytest-cov
+pytest==6.0.1             # via pytest-cov
 pytz==2020.1              # via -r requirements/base.txt, django, event-tracking
 six==1.15.0               # via -r requirements/base.txt, event-tracking, mock, packaging, pathlib2
 sqlparse==0.3.1           # via -r requirements/base.txt, django
-urllib3==1.25.9           # via -r requirements/base.txt, elasticsearch
-wcwidth==0.2.5            # via pytest
+toml==0.10.1              # via pytest
+urllib3==1.25.10          # via -r requirements/base.txt, elasticsearch
 zipp==1.2.0               # via importlib-metadata

requirements/travis.txt

@@ -21,8 +21,8 @@ pyparsing==2.4.7          # via packaging
 requests==2.24.0          # via codecov
 six==1.15.0               # via packaging, tox, virtualenv
 toml==0.10.1              # via tox
-tox-battery==0.5.2        # via -c requirements/constraints.txt, -r requirements/travis.in
-tox==3.17.1               # via -r requirements/travis.in, tox-battery
-urllib3==1.25.9           # via requests
-virtualenv==20.0.27       # via tox
+tox-battery==0.6.1        # via -c requirements/constraints.txt, -r requirements/travis.in
+tox==3.18.1               # via -r requirements/travis.in, tox-battery
+urllib3==1.25.10          # via requests
+virtualenv==20.0.29       # via tox
 zipp==1.2.0               # via importlib-metadata, importlib-resources

search/api.py

@@ -14,26 +14,30 @@
 
 
 def course_discovery_filter_fields():
-    """ look up the desired list of course discovery filter fields """
+    """
+    Look up the desired list of course discovery filter fields.
+    """
     return getattr(settings, "COURSE_DISCOVERY_FILTERS", DEFAULT_FILTER_FIELDS)
 
 
-def course_discovery_facets():
-    """ Discovery facets to include, by default we specify each filter field with unspecified size attribute """
-    return getattr(settings, "COURSE_DISCOVERY_FACETS", {field: {} for field in course_discovery_filter_fields()})
-
-
-class NoSearchEngineError(Exception):
-    """ NoSearchEngineError exception to be thrown if no search engine is specified """
+def course_discovery_aggregations():
+    """
+    Discovery aggregations to include bucket names.
 
+    By default we specify each filter field with unspecified size attribute.
+    """
+    return getattr(
+        settings,
+        "COURSE_DISCOVERY_AGGREGATIONS",
+        {field: {} for field in course_discovery_filter_fields()}
+    )
 
-class QueryParseError(Exception):
-    """QueryParseError will be thrown if the query is malformed.
 
-    If a query has mismatched quotes (e.g. '"some phrase', return a
-    more specific exception so the view can provide a more helpful
-    error message to the user.
+class NoSearchEngineError(Exception):
+    """
+    NoSearchEngineError exception.
 
+    It is thrown if no search engine is specified.
     """
 
 
@@ -43,15 +47,19 @@ def perform_search(
         size=10,
         from_=0,
         course_id=None):
-    """ Call the search engine with the appropriate parameters """
+    """
+    Call the search engine with the appropriate parameters
+    """
     # field_, filter_ and exclude_dictionary(s) can be overridden by calling application
     # field_dictionary includes course if course_id provided
     (field_dictionary, filter_dictionary, exclude_dictionary) = SearchFilterGenerator.generate_field_filters(
         user=user,
         course_id=course_id
     )
 
-    searcher = SearchEngine.get_search_engine(getattr(settings, "COURSEWARE_INDEX_NAME", "courseware_index"))
+    searcher = SearchEngine.get_search_engine(
+        getattr(settings, "COURSEWARE_CONTENT_INDEX_NAME", "courseware_content")
+    )
     if not searcher:
         raise NoSearchEngineError("No search engine specified in settings.SEARCH_ENGINE")
 
@@ -62,7 +70,6 @@ def perform_search(
         exclude_dictionary=exclude_dictionary,
         size=size,
         from_=from_,
-        doc_type="courseware_content",
     )
 
     # post-process the result
@@ -83,28 +90,31 @@ def course_discovery_search(search_term=None, size=20, from_=0, field_dictionary
     # dictionary, and use our own logic upon enrollment dates for these
     use_search_fields = ["org"]
     (search_fields, _, exclude_dictionary) = SearchFilterGenerator.generate_field_filters()
-    use_field_dictionary = {}
-    use_field_dictionary.update({field: search_fields[field] for field in search_fields if field in use_search_fields})
+    use_field_dictionary = {
+        field: search_fields[field]
+        for field in search_fields if field in use_search_fields
+    }
     if field_dictionary:
         use_field_dictionary.update(field_dictionary)
     if not getattr(settings, "SEARCH_SKIP_ENROLLMENT_START_DATE_FILTERING", False):
         use_field_dictionary["enrollment_start"] = DateRange(None, datetime.utcnow())
 
-    searcher = SearchEngine.get_search_engine(getattr(settings, "COURSEWARE_INDEX_NAME", "courseware_index"))
+    searcher = SearchEngine.get_search_engine(
+        getattr(settings, "COURSEWARE_INFO_INDEX_NAME", "course_info")
+    )
     if not searcher:
         raise NoSearchEngineError("No search engine specified in settings.SEARCH_ENGINE")
 
     results = searcher.search(
         query_string=search_term,
-        doc_type="course_info",
         size=size,
         from_=from_,
         # only show when enrollment start IS provided and is before now
         field_dictionary=use_field_dictionary,
         # show if no enrollment end is provided and has not yet been reached
         filter_dictionary={"enrollment_end": DateRange(datetime.utcnow(), None)},
         exclude_dictionary=exclude_dictionary,
-        facet_terms=course_discovery_facets(),
+        aggregation_terms=course_discovery_aggregations(),
     )
 
     return results