Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Allow setting java options when launching Elasticsearch / OpenSearch #5002

Merged
merged 6 commits into from
May 25, 2023
Merged

feat: Allow setting java options when launching Elasticsearch / OpenSearch #5002

Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
2e0d2a6
Allow launching Elasticsearch and OpenSearch with java options and de…
bogdankostic May 23, 2023
fe0f745
Merge branch 'main' into launch_docstore_options
bogdankostic May 23, 2023
3f0d698
Remove unneeded imports
bogdankostic May 23, 2023
1b45d7a
Merge branch 'main' into launch_docstore_options
bogdankostic May 24, 2023
33acecc
Merge branch 'main' into launch_docstore_options
bogdankostic May 24, 2023
46dc326
Simplify java opts tring generation
bogdankostic May 24, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 26 additions & 13 deletions haystack/utils/doc_store.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import time
import logging
import subprocess
from typing import Optional


logger = logging.getLogger(__name__)
Expand All @@ -9,20 +10,25 @@
WEAVIATE_CONTAINER_NAME = "weaviate"


def launch_es(sleep=15, delete_existing=False):
def launch_es(sleep=15, delete_existing=False, java_opts: Optional[str] = None):
"""
Start an Elasticsearch server via Docker.
"""

logger.debug("Starting Elasticsearch ...")
if delete_existing:
_ = subprocess.run([f"docker rm --force {ELASTICSEARCH_CONTAINER_NAME}"], shell=True, stdout=subprocess.DEVNULL)
status = subprocess.run(
[
f'docker start {ELASTICSEARCH_CONTAINER_NAME} > /dev/null 2>&1 || docker run -d -p 9200:9200 -e "discovery.type=single-node" --name {ELASTICSEARCH_CONTAINER_NAME} elasticsearch:7.17.6'
],
shell=True,

java_opts_str = f"-e ES_JAVA_OPTS='{java_opts}' " if java_opts is not None else ""

command = (
f"docker start {ELASTICSEARCH_CONTAINER_NAME} > /dev/null 2>&1 || docker run -d "
f'-p 9200:9200 -p 9600:9600 -e "discovery.type=single-node" '
f"{java_opts_str}"
f"--name {ELASTICSEARCH_CONTAINER_NAME} elasticsearch:7.17.6"
)

status = subprocess.run([command], shell=True)
if status.returncode:
logger.warning(
"Tried to start Elasticsearch through Docker but this failed. "
Expand All @@ -32,7 +38,7 @@ def launch_es(sleep=15, delete_existing=False):
time.sleep(sleep)


def launch_opensearch(sleep=15, delete_existing=False, local_port=9200):
def launch_opensearch(sleep=15, delete_existing=False, local_port=9200, java_opts: Optional[str] = None):
"""
Start an OpenSearch server via Docker.
"""
Expand All @@ -41,12 +47,17 @@ def launch_opensearch(sleep=15, delete_existing=False, local_port=9200):
# docker rm only succeeds if the container is stopped, not if it is running
if delete_existing:
_ = subprocess.run([f"docker rm --force {OPENSEARCH_CONTAINER_NAME}"], shell=True, stdout=subprocess.DEVNULL)
status = subprocess.run(
[
f'docker start {OPENSEARCH_CONTAINER_NAME} > /dev/null 2>&1 || docker run -d -p {local_port}:9200 -p 9600:9600 -e "discovery.type=single-node" --name {OPENSEARCH_CONTAINER_NAME} opensearchproject/opensearch:1.3.5'
],
shell=True,

java_opts_str = f"-e OPENSEARCH_JAVA_OPTS='{java_opts}' " if java_opts is not None else ""

command = (
f"docker start {OPENSEARCH_CONTAINER_NAME} > /dev/null 2>&1 || docker run -d "
f'-p {local_port}:9200 -p 9600:9600 -e "discovery.type=single-node" '
f"{java_opts_str}"
f"--name {OPENSEARCH_CONTAINER_NAME} opensearchproject/opensearch:1.3.5"
)

status = subprocess.run([command], shell=True)
if status.returncode:
logger.warning(
"Tried to start OpenSearch through Docker but this failed. "
Expand All @@ -56,12 +67,14 @@ def launch_opensearch(sleep=15, delete_existing=False, local_port=9200):
time.sleep(sleep)


def launch_weaviate(sleep=15):
def launch_weaviate(sleep=15, delete_existing=False):
"""
Start a Weaviate server via Docker.
"""

logger.debug("Starting Weaviate ...")
if delete_existing:
_ = subprocess.run([f"docker rm --force {WEAVIATE_CONTAINER_NAME}"], shell=True, stdout=subprocess.DEVNULL)
status = subprocess.run(
[
f"docker start {WEAVIATE_CONTAINER_NAME} > /dev/null 2>&1 || docker run -d -p 8080:8080 --env AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED='true' --env PERSISTENCE_DATA_PATH='/var/lib/weaviate' --name {WEAVIATE_CONTAINER_NAME} semitechnologies/weaviate:latest"
Expand Down