Improve dependency management

.github/workflows/cml.yaml

@@ -78,8 +78,7 @@ jobs:
         run: |
           apt-get update -y
           apt-get install python3-dev -y
-          pip install -r requirements.txt
-          pip install .
+          pip install .[elasticsearch,faiss,milvus,weaviate,graphdb,ray,rest,ui,dev]
           cd test/benchmarks && python run.py --retriever_index --retriever_query --reader --ci --save_markdown
           echo -en "## Benchmarks: Retriever Indexing\n" >> report.md
           cat retriever_index_results.md >> report.md

.github/workflows/linux_ci.yml

@@ -34,14 +34,12 @@ jobs:
         uses: actions/cache@v2
         with:
           path: ${{ env.pythonLocation }}
-          key: linux-${{ env.pythonLocation }}-${{ env.date }}-${{ hashFiles('setup.py') }}-${{ hashFiles('requirements.txt') }}-${{ hashFiles('requirements-dev.txt') }}
+          key: linux-${{ env.pythonLocation }}-${{ env.date }}-${{ hashFiles('setup.py') }}-${{ hashFiles('setup.cfg') }}-${{ hashFiles('pyproject.toml') }}
       - name: Install dependencies
         if: steps.cache-python-env.outputs.cache-hit != 'true'
         run: |
           python -m pip install --upgrade pip
-          pip install --upgrade --upgrade-strategy eager -r requirements-dev.txt -e .
-          pip install --upgrade --upgrade-strategy eager -r requirements.txt -e .
-          pip install --upgrade --upgrade-strategy eager -r ui/requirements.txt -e .
+          pip install --upgrade --upgrade-strategy eager .[test]
           pip install torch-scatter -f https://data.pyg.org/whl/torch-1.10.0+cpu.html
 
   prepare-build:
@@ -51,7 +49,7 @@ jobs:
       - uses: actions/checkout@v2
       - id: set-matrix
         run: |
-          echo "::set-output name=matrix::$(cd test && ls -d test_*.py | jq -R . | jq -cs .)"
+          echo "::set-output name=matrix::$(find $(find . -type d -name test -not -path "./*env*/*") -type f -name test_*.py | jq -SR . | jq -cs .)"
     outputs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
   build:
@@ -72,7 +70,7 @@ jobs:
       uses: actions/cache@v2
       with:
         path: ${{ env.pythonLocation }}
-        key: linux-${{ env.pythonLocation }}-${{ env.date }}-${{ hashFiles('setup.py') }}-${{ hashFiles('requirements.txt') }}-${{ hashFiles('requirements-dev.txt') }}
+        key: linux-${{ env.pythonLocation }}-${{ env.date }}-${{ hashFiles('setup.py') }}-${{ hashFiles('setup.cfg') }}-${{ hashFiles('pyproject.toml') }}
     - name: Run Elasticsearch
       run: docker run -d -p 9200:9200 -e "discovery.type=single-node" -e "ES_JAVA_OPTS=-Xms128m -Xmx128m" elasticsearch:7.9.2
 
@@ -101,4 +99,4 @@ jobs:
       run: sudo apt-get install tesseract-ocr libtesseract-dev poppler-utils
 
     - name: Run tests
-      run: cd test && pytest -s ${{ matrix.test-path }}
+      run: pytest -s ${{ matrix.test-path }}

.github/workflows/windows_ci.yml

@@ -35,17 +35,15 @@ jobs:
         uses: actions/cache@v2
         with:
           path: ${{ env.pythonLocation }}
-          key: windows-${{ env.pythonLocation }}-${{ env.date }}-${{ hashFiles('setup.py') }}-${{ hashFiles('requirements.txt') }}-${{ hashFiles('requirements-dev.txt') }}
+          key: windows-${{ env.pythonLocation }}-${{ env.date }}-${{ hashFiles('setup.py') }}-${{ hashFiles('setup.cfg') }}-${{ hashFiles('pyproject.toml') }}
       - name: Install Pytorch on windows
         run: |
           pip install torch==1.8.1+cpu -f https://download.pytorch.org/whl/lts/1.8/torch_lts.html
       - name: Install dependencies
         if: steps.cache-python-env.outputs.cache-hit != 'true'
         run: |
           python -m pip install --upgrade pip
-          pip install --upgrade --upgrade-strategy eager -r requirements-dev.txt -e .
-          pip install --upgrade --upgrade-strategy eager -r ui/requirements.txt -e .
-          pip install --upgrade --upgrade-strategy eager -f https://download.pytorch.org/whl/torch_stable.html -r requirements.txt -e .
+          pip install --upgrade --upgrade-strategy eager .[test]
           pip install torch-scatter -f https://data.pyg.org/whl/torch-1.10.0+cpu.html
 
   prepare-build:
@@ -56,7 +54,7 @@ jobs:
       - uses: actions/checkout@v2
       - id: set-matrix
         run: |
-          echo "::set-output name=matrix::$(cd test && ls -d test_*.py | jq -R . | jq -cs .)"
+          echo "::set-output name=matrix::$(find $(find . -type d -name test -not -path "./*env*/*") -type f -name test_*.py | jq -SR . | jq -cs .)"
     outputs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
 
@@ -78,7 +76,7 @@ jobs:
       uses: actions/cache@v2
       with:
         path: ${{ env.pythonLocation }}
-        key: windows-${{ env.pythonLocation }}-${{ env.date }}-${{ hashFiles('setup.py') }}-${{ hashFiles('requirements.txt') }}-${{ hashFiles('requirements-dev.txt') }}
+        key: windows-${{ env.pythonLocation }}-${{ env.date }}-${{ hashFiles('setup.py') }}-${{ hashFiles('setup.cfg') }}-${{ hashFiles('pyproject.toml') }}
 
     # Windows runner can't run Linux containers. Refer https://github.com/actions/virtual-environments/issues/1143
     - name: Set up Windows test env
@@ -96,4 +94,4 @@ jobs:
     # Removing test_ray, test_utils, test_preprocessor, test_knowledge_graph and test_connector
     - name: Run tests
       if: ${{ !contains(fromJSON('["test_ray.py", "test_knowledge_graph.py", "test_connector.py", "test_summarizer_translation.py"]'), matrix.test-path) }}
-      run: cd test && pytest --document_store_type=memory,faiss,elasticsearch -m "not tika and not graphdb" -k "not test_parsr_converter" -s ${{ matrix.test-path }}
+      run: pytest --document_store_type=memory,faiss,elasticsearch -m "not tika and not graphdb" -k "not test_parsr_converter" -s ${{ matrix.test-path }}

Dockerfile

@@ -17,25 +17,25 @@ RUN apt-get update && apt-get install -y \
 RUN wget --no-check-certificate https://dl.xpdfreader.com/xpdf-tools-linux-4.03.tar.gz && \
     tar -xvf xpdf-tools-linux-4.03.tar.gz && cp xpdf-tools-linux-4.03/bin64/pdftotext /usr/local/bin
 
-# copy code
-COPY haystack /home/user/haystack
-
 # install as a package
-COPY setup.py requirements.txt README.md /home/user/
+COPY setup.py setup.cfg pyproject.toml VERSION.txt LICENSE README.md \
+    # Haystack code
+    haystack \
+    # Saved models
+    models* \
+    # REST API cpde
+    rest_api \
+    /home/user/
 RUN pip install --upgrade pip
-RUN pip install --no-cache-dir -r requirements.txt
-RUN pip install -e .
+RUN pip install --no-cache-dir -e .[docstores,crawler,preprocessing,ocr,ray,rest]
+RUN ls /home/user
+RUN pip freeze
 RUN python3 -c "from haystack.utils.docker import cache_models;cache_models()"
 
 # create folder for /file-upload API endpoint with write permissions, this might be adjusted depending on FILE_UPLOAD_PATH
 RUN mkdir -p /home/user/file-upload
 RUN chmod 777 /home/user/file-upload
 
-# copy saved models
-COPY README.md models* /home/user/models/
-
-# Copy REST API code
-COPY rest_api /home/user/rest_api
 
 # optional : copy sqlite db if needed for testing
 #COPY qa.db /home/user/

Dockerfile-GPU

@@ -35,27 +35,23 @@ RUN curl -s https://dl.xpdfreader.com/xpdf-tools-linux-4.03.tar.gz | tar -xvzf -
 RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.7 1 && \
     update-alternatives --set python3 /usr/bin/python3.7
 
-# Copy package setup files
-COPY setup.py requirements.txt README.md /home/user/
+# Copy package files
+COPY setup.py setup.cfg pyproject.toml VERSION.txt LICENSE README.md \
+    # Haystack code
+    haystack \
+    # Saved models
+    models* \
+    # REST API cpde
+    rest_api \
+    /home/user/
 
 RUN pip install --upgrade pip
 RUN echo "Install required packages" && \
     # Install PyTorch for CUDA 11
-    pip3 install --no-cache-dir torch==1.10.1+cu111 -f https://download.pytorch.org/whl/torch_stable.html && \
-    # Install from requirements.txt
-    pip3 install --no-cache-dir -r requirements.txt
-
-# copy saved models
-COPY README.md models* /home/user/models/
-
-# Copy REST API code
-COPY rest_api /home/user/rest_api
-
-# copy code
-COPY haystack /home/user/haystack
+    pip3 install --no-cache-dir torch==1.10.1+cu111 -f https://download.pytorch.org/whl/torch_stable.html
 
 # Install package
-RUN pip3 install -e .
+RUN pip install --no-cache .[docstores_gpu,crawler,preprocessing,ocr,ray,rest]
 
 # Cache Roberta and NLTK data
 RUN python3 -c "from haystack.utils.docker import cache_models;cache_models()"

VERSION.txt

@@ -0,0 +1 @@
+1.1.0

docs/_src/api/api/retriever.md

@@ -96,12 +96,12 @@ class ElasticsearchRetriever(BaseRetriever)
 #### \_\_init\_\_
 
 ```python
- | __init__(document_store: ElasticsearchDocumentStore, top_k: int = 10, custom_query: str = None)
+ | __init__(document_store: "ElasticsearchDocumentStore", top_k: int = 10, custom_query: str = None)
 ```
 
 **Arguments**:
 
-- `document_store`: an instance of a DocumentStore to retrieve documents from.
+- `document_store`: an instance of an ElasticsearchDocumentStore to retrieve documents from.
 - `custom_query`: query string as per Elasticsearch DSL with a mandatory query placeholder(query).
 
                      Optionally, ES `filter` clause can be added where the values of `terms` are placeholders

haystack/__init__.py

@@ -1,19 +1,29 @@
-import logging
+from typing import Union
+from types import ModuleType
+
+try:
+    import importlib.metadata as metadata
+except ModuleNotFoundError:
+    # Python <= 3.7
+    import importlib_metadata as metadata  # type: ignore
+
+__version__ = metadata.version('farm-haystack')
 
 # This configuration must be done before any import to apply to all submodules
+import logging
 logging.basicConfig(format="%(levelname)s - %(name)s -  %(message)s", datefmt="%m/%d/%Y %H:%M:%S", level=logging.WARNING)
 logging.getLogger("haystack").setLevel(logging.INFO)
 
 from haystack import pipelines
 from haystack.schema import Document, Answer, Label, MultiLabel, Span
 from haystack.nodes import BaseComponent
 from haystack.pipelines import Pipeline
-from haystack._version import __version__
 
 import pandas as pd
 pd.options.display.max_colwidth = 80
 
 
+
 # ###########################################
 # Enable old style imports (temporary)
 import sys
@@ -57,9 +67,29 @@ def __getattr__(self, attr):
     summarizer,
     translator
 )
-from haystack import document_stores
-from haystack.nodes.retriever import text2sparql as graph_retriever
-from haystack.document_stores import graphdb as knowledge_graph
+
+# Note that we ignore the ImportError here because if the user did not install
+# the correct dependency group for a document store, we don't need to setup 
+# import warnings for that, so the import here is useless and should fail silently.
+
+document_stores: Union[ModuleType, None] = None
+try:
+    from haystack import document_stores
+except ImportError:
+    pass
+
+graph_retriever: Union[ModuleType, None] = None
+try:
+    from haystack.nodes.retriever import text2sparql as graph_retriever
+except ImportError:
+    pass
+
+knowledge_graph: Union[ModuleType, None] = None
+try:
+    from haystack.document_stores import graphdb as knowledge_graph
+except ImportError:
+    pass
+
 from haystack.modeling.evaluation import eval
 from haystack.modeling.logger import MLFlowLogger, StdoutLogger, TensorBoardLogger
 from haystack.nodes.other import JoinDocuments, Docs2Answers
@@ -73,8 +103,9 @@ def __getattr__(self, attr):
 # modules need to be set as attributes of their parent model.
 # To make chain imports work (`from haystack.reader import FARMReader`) the module
 # needs to be also present in sys.modules with its complete import path.
-setattr(knowledge_graph, "graphdb", DeprecatedModule(knowledge_graph))
-sys.modules["haystack.knowledge_graph.graphdb"] = DeprecatedModule(knowledge_graph)
+if knowledge_graph:
+    setattr(knowledge_graph, "graphdb", DeprecatedModule(knowledge_graph))
+    sys.modules["haystack.knowledge_graph.graphdb"] = DeprecatedModule(knowledge_graph)
 
 setattr(preprocessor, "utils", DeprecatedModule(preprocessing))
 setattr(preprocessor, "cleaning", DeprecatedModule(cleaning))
@@ -88,7 +119,6 @@ def __getattr__(self, attr):
 setattr(haystack, "extractor", DeprecatedModule(extractor))
 setattr(haystack, "eval", DeprecatedModule(eval))
 setattr(haystack, "file_converter", DeprecatedModule(file_converter, deprecated_attributes=["FileTypeClassifier"]))
-setattr(haystack, "graph_retriever", DeprecatedModule(graph_retriever))
 setattr(haystack, "knowledge_graph", DeprecatedModule(knowledge_graph, deprecated_attributes=["graphdb"]))
 setattr(haystack, "pipeline", DeprecatedModule(pipelines, deprecated_attributes=["JoinDocuments", "Docs2Answers", "SklearnQueryClassifier", "TransformersQueryClassifier"]))
 setattr(haystack, "preprocessor", DeprecatedModule(preprocessor, deprecated_attributes=["utils", "cleaning"]))
@@ -105,7 +135,6 @@ def __getattr__(self, attr):
 sys.modules["haystack.extractor"] = DeprecatedModule(extractor)
 sys.modules["haystack.eval"] = DeprecatedModule(eval)
 sys.modules["haystack.file_converter"] = DeprecatedModule(file_converter)
-sys.modules["haystack.graph_retriever"] = DeprecatedModule(graph_retriever)
 sys.modules["haystack.knowledge_graph"] = DeprecatedModule(knowledge_graph)
 sys.modules["haystack.pipeline"] = DeprecatedModule(pipelines)
 sys.modules["haystack.preprocessor"] = DeprecatedModule(preprocessor, deprecated_attributes=["utils", "cleaning"])
@@ -115,6 +144,9 @@ def __getattr__(self, attr):
 sys.modules["haystack.retriever"] = DeprecatedModule(retriever)
 sys.modules["haystack.summarizer"] = DeprecatedModule(summarizer)
 sys.modules["haystack.translator"] = DeprecatedModule(translator)
+if graph_retriever:
+    setattr(haystack, "graph_retriever", DeprecatedModule(graph_retriever))
+    sys.modules["haystack.graph_retriever"] = DeprecatedModule(graph_retriever)
 
 # To be imported from modules, classes need only to be set as attributes, 
 # they don't need to be present in sys.modules too.
@@ -138,7 +170,6 @@ def __getattr__(self, attr):
     "extractor",
     "eval",
     "file_converter",
-    "graph_retriever",
     "knowledge_graph",
     "pipeline",
     "preprocessor",
@@ -149,4 +180,6 @@ def __getattr__(self, attr):
     "summarizer",
     "translator"
 ]
-sys.modules["haystack"] = DeprecatedModule(haystack, is_module_deprecated=False, deprecated_attributes=deprecated_attributes)
+if graph_retriever:
+    deprecated_attributes.append("graph_retriever")
+sys.modules["haystack"] = DeprecatedModule(haystack, is_module_deprecated=False, deprecated_attributes=deprecated_attributes)

haystack/document_stores/__init__.py

@@ -1,16 +1,20 @@
+import os
+import importlib
+from haystack.utils.import_utils import safe_import
 from haystack.document_stores.base import BaseDocumentStore, BaseKnowledgeGraph
-from haystack.document_stores.elasticsearch import ElasticsearchDocumentStore, OpenDistroElasticsearchDocumentStore, OpenSearchDocumentStore
-from haystack.document_stores.faiss import FAISSDocumentStore
-from haystack.document_stores.memory import InMemoryDocumentStore
 
-import os
+ElasticsearchDocumentStore = safe_import("haystack.document_stores.elasticsearch", "ElasticsearchDocumentStore", "elasticsearch")
+OpenDistroElasticsearchDocumentStore = safe_import("haystack.document_stores.elasticsearch", "OpenDistroElasticsearchDocumentStore", "elasticsearch")
+OpenSearchDocumentStore = safe_import("haystack.document_stores.elasticsearch", "OpenSearchDocumentStore", "elasticsearch")
+
+SQLDocumentStore = safe_import("haystack.document_stores.sql", "SQLDocumentStore", "sql")
+FAISSDocumentStore = safe_import("haystack.document_stores.faiss", "FAISSDocumentStore", "faiss")
 if os.getenv("MILVUS2_ENABLED"):
-    print("Using experimental Milvus2DocumentStore")
-    from haystack.document_stores.milvus2x import Milvus2DocumentStore as MilvusDocumentStore
+    MilvusDocumentStore = safe_import("haystack.document_stores.milvus2x", "MilvusDocumentStore", "milvus2")
 else:
-    from haystack.document_stores.milvus import MilvusDocumentStore  # type: ignore    
+    MilvusDocumentStore = safe_import("haystack.document_stores.milvus", "MilvusDocumentStore", "milvus")
+WeaviateDocumentStore = safe_import("haystack.document_stores.weaviate", "WeaviateDocumentStore", "weaviate")
+GraphDBKnowledgeGraph = safe_import("haystack.document_stores.graphdb", "GraphDBKnowledgeGraph", "graphdb")
 
-from haystack.document_stores.sql import SQLDocumentStore
-from haystack.document_stores.weaviate import WeaviateDocumentStore
-from haystack.document_stores.graphdb import GraphDBKnowledgeGraph
+from haystack.document_stores.memory import InMemoryDocumentStore
 from haystack.document_stores.utils import eval_data_from_json, eval_data_from_jsonl, squad_json_to_jsonl

haystack/document_stores/elasticsearch.py

@@ -1,3 +1,4 @@
+from modulefinder import Module
 from typing import List, Optional, Union, Dict, Any, Generator
 
 import json
@@ -8,11 +9,16 @@
 import numpy as np
 from scipy.special import expit
 from tqdm.auto import tqdm
-from elasticsearch import Elasticsearch, RequestsHttpConnection
-from elasticsearch.helpers import bulk, scan
-from elasticsearch.exceptions import RequestError
 import pandas as pd
 
+try:
+    from elasticsearch import Elasticsearch, RequestsHttpConnection
+    from elasticsearch.helpers import bulk, scan
+    from elasticsearch.exceptions import RequestError
+except (ImportError, ModuleNotFoundError) as ie:
+    from haystack.utils.import_utils import _optional_component_not_installed
+    _optional_component_not_installed(__name__, "elasticsearch", ie)
+
 from haystack.document_stores import BaseDocumentStore
 from haystack.schema import Document, Label
 from haystack.document_stores.base import get_batches_from_generator

haystack/document_stores/faiss.py

@@ -9,17 +9,19 @@
 from typing import Union, List, Optional, Dict, Generator
 from tqdm.auto import tqdm
 import warnings
+import numpy as np
+from inspect import Signature, signature
 
 try:
     import faiss
-except ImportError:
-    faiss = None
-import numpy as np
+    from haystack.document_stores.sql import SQLDocumentStore  # its deps are optional, but get installed with the `faiss` extra
+except (ImportError, ModuleNotFoundError) as ie:
+    from haystack.utils.import_utils import _optional_component_not_installed
+    _optional_component_not_installed(__name__, "faiss", ie)
+
 
 from haystack.schema import Document
-from haystack.document_stores.sql import SQLDocumentStore
 from haystack.document_stores.base import get_batches_from_generator
-from inspect import Signature, signature
 
 
 logger = logging.getLogger(__name__)