Validate YAML files without loading the nodes

docs/_src/api/pydoc/pipelines.yml

@@ -1,7 +1,7 @@
 loaders:
   - type: python
     search_path: [../../../../haystack/pipelines]
-    modules: ['base', 'standard_pipelines']
+    modules: ['base', 'ray', 'standard_pipelines']
     ignore_when_discovered: ['__init__']
 processors:
   - type: filter

haystack/document_stores/faiss.py

@@ -180,8 +180,8 @@ def _validate_index_sync(self):
         # used when creating the original FAISS index
         if not self.get_document_count() == self.get_embedding_count():
             raise ValueError(
-                "The number of documents present in the SQL database does not "
-                "match the number of embeddings in FAISS. Make sure your FAISS "
+                f"The number of documents present in the SQL database ({self.get_document_count()}) does not "
+                f"match the number of embeddings in FAISS ({self.get_embedding_count()}). Make sure your FAISS "
                 "configuration file correctly points to the same database that "
                 "was used when creating the original index."
             )

haystack/json-schemas/haystack-pipeline-1.0.0.schema.json

@@ -1,6 +1,6 @@
 {
   "$schema": "http://json-schema.org/draft-07/schema",
-  "$id": "https://haystack.deepset.ai/json-schemas/haystack-pipeline-1.0.0.schema.json",
+  "$id": "https://raw.githubusercontent.com/deepset-ai/haystack/master/haystack/json-schemas/haystack-pipeline-1.0.0.schema.json",
   "title": "Haystack Pipeline",
   "description": "Haystack Pipeline YAML file describing the nodes of the pipelines. For more info read the docs at: https://haystack.deepset.ai/components/pipelines#yaml-file-definitions",
   "type": "object",

haystack/json-schemas/haystack-pipeline-1.1.0.schema.json

@@ -1,6 +1,6 @@
 {
   "$schema": "http://json-schema.org/draft-07/schema",
-  "$id": "https://haystack.deepset.ai/json-schemas/haystack-pipeline-1.1.0.schema.json",
+  "$id": "https://raw.githubusercontent.com/deepset-ai/haystack/master/haystack/json-schemas/haystack-pipeline-1.1.0.schema.json",
   "title": "Haystack Pipeline",
   "description": "Haystack Pipeline YAML file describing the nodes of the pipelines. For more info read the docs at: https://haystack.deepset.ai/components/pipelines#yaml-file-definitions",
   "type": "object",

haystack/json-schemas/haystack-pipeline-1.2.0.schema.json

@@ -1,6 +1,6 @@
 {
   "$schema": "http://json-schema.org/draft-07/schema",
-  "$id": "https://haystack.deepset.ai/json-schemas/haystack-pipeline-1.2.0.schema.json",
+  "$id": "https://raw.githubusercontent.com/deepset-ai/haystack/master/haystack/json-schemas/haystack-pipeline-1.2.0.schema.json",
   "title": "Haystack Pipeline",
   "description": "Haystack Pipeline YAML file describing the nodes of the pipelines. For more info read the docs at: https://haystack.deepset.ai/components/pipelines#yaml-file-definitions",
   "type": "object",

haystack/json-schemas/haystack-pipeline-1.3.0.schema.json

@@ -1,6 +1,6 @@
 {
   "$schema": "http://json-schema.org/draft-07/schema",
-  "$id": "https://haystack.deepset.ai/haystack/json-schemas/haystack-pipeline-1.2.1rc0.schema.json",
+  "$id": "https://raw.githubusercontent.com/deepset-ai/haystack/master/haystack/json-schemas/haystack-pipeline-1.2.1rc0.schema.json",
   "title": "Haystack Pipeline",
   "description": "Haystack Pipeline YAML file describing the nodes of the pipelines. For more info read the docs at: https://haystack.deepset.ai/components/pipelines#yaml-file-definitions",
   "type": "object",

haystack/json-schemas/haystack-pipeline-master.schema.json

@@ -1,6 +1,6 @@
 {
   "$schema": "http://json-schema.org/draft-07/schema",
-  "$id": "https://haystack.deepset.ai/haystack/json-schemas/haystack-pipeline-master.schema.json",
+  "$id": "https://raw.githubusercontent.com/deepset-ai/haystack/master/haystack/json-schemas/haystack-pipeline-master.schema.json",
   "title": "Haystack Pipeline",
   "description": "Haystack Pipeline YAML file describing the nodes of the pipelines. For more info read the docs at: https://haystack.deepset.ai/components/pipelines#yaml-file-definitions",
   "type": "object",

haystack/json-schemas/haystack-pipeline.schema.json

@@ -1,6 +1,6 @@
 {
   "$schema": "http://json-schema.org/draft-07/schema",
-  "$id": "https://haystack.deepset.ai/json-schemas/haystack-pipeline.schema.json",
+  "$id": "https://raw.githubusercontent.com/deepset-ai/haystack/master/haystack/json-schemas/haystack-pipeline.schema.json",
   "title": "Haystack Pipeline",
   "description": "Haystack Pipeline YAML file describing the nodes of the pipelines. For more info read the docs at: https://haystack.deepset.ai/components/pipelines#yaml-file-definitions",
   "type": "object",
@@ -15,7 +15,7 @@
           }
         },
         {
-          "$ref": "https://raw.githubusercontent.com/deepset-ai/haystack/master/json-schemas/haystack-pipeline-master.schema.json"
+          "$ref": "https://raw.githubusercontent.com/deepset-ai/haystack/master/haystack/json-schemas/haystack-pipeline-master.schema.json"
         }
       ]
     },
@@ -29,7 +29,7 @@
           }
         },
         {
-          "$ref": "https://raw.githubusercontent.com/deepset-ai/haystack/master/json-schemas/haystack-pipeline-1.0.0.schema.json"
+          "$ref": "https://raw.githubusercontent.com/deepset-ai/haystack/master/haystack/json-schemas/haystack-pipeline-1.0.0.schema.json"
         }
       ]
     },
@@ -43,7 +43,7 @@
           }
         },
         {
-          "$ref": "https://raw.githubusercontent.com/deepset-ai/haystack/master/json-schemas/haystack-pipeline-1.1.0.schema.json"
+          "$ref": "https://raw.githubusercontent.com/deepset-ai/haystack/master/haystack/json-schemas/haystack-pipeline-1.1.0.schema.json"
         }
       ]
     },
@@ -57,7 +57,7 @@
           }
         },
         {
-          "$ref": "https://raw.githubusercontent.com/deepset-ai/haystack/master/json-schemas/haystack-pipeline-1.2.0.schema.json"
+          "$ref": "https://raw.githubusercontent.com/deepset-ai/haystack/master/haystack/json-schemas/haystack-pipeline-1.2.0.schema.json"
         }
       ]
     },
@@ -71,7 +71,7 @@
           }
         },
         {
-          "$ref": "https://raw.githubusercontent.com/deepset-ai/haystack/master/json-schemas/haystack-pipeline-1.3.0.schema.json"
+          "$ref": "https://raw.githubusercontent.com/deepset-ai/haystack/master/haystack/json-schemas/haystack-pipeline-1.3.0.schema.json"
         }
       ]
     }

haystack/nodes/_json_schema.py

@@ -28,7 +28,7 @@
 
 
 JSON_SCHEMAS_PATH = Path(__file__).parent.parent.parent / "haystack" / "json-schemas"
-SCHEMA_URL = "https://haystack.deepset.ai/haystack/json-schemas/"
+SCHEMA_URL = "https://raw.githubusercontent.com/deepset-ai/haystack/master/haystack/json-schemas/"
 
 # Allows accessory classes (like enums and helpers) to be registered as valid input for
 # custom node's init parameters. For now we disable this feature, but flipping this variables
@@ -351,7 +351,7 @@ def update_json_schema(destination_path: Path = JSON_SCHEMAS_PATH):
                     "allOf": [
                         {"properties": {"version": {"const": haystack_version}}},
                         {
-                            "$ref": "https://raw.githubusercontent.com/deepset-ai/haystack/master/json-schemas/"
+                            "$ref": "https://raw.githubusercontent.com/deepset-ai/haystack/master/haystack/json-schemas/"
                             f"haystack-pipeline-{haystack_version}.schema.json"
                         },
                     ]

haystack/nodes/base.py

@@ -1,5 +1,5 @@
 from __future__ import annotations
-from typing import Any, Optional, Dict, List, Tuple
+from typing import Any, Optional, Dict, List, Tuple, Type
 
 from copy import deepcopy
 from abc import ABC, abstractmethod
@@ -109,47 +109,24 @@ def get_params(self, return_defaults: bool = False) -> Dict[str, Any]:
         return params
 
     @classmethod
-    def get_subclass(cls, component_type: str):
+    def get_subclass(cls, component_type: str) -> Type[BaseComponent]:
         if component_type not in cls._subclasses.keys():
             raise PipelineSchemaError(f"Haystack component with the name '{component_type}' not found.")
         subclass = cls._subclasses[component_type]
         return subclass
 
     @classmethod
-    def load_from_args(cls, component_type: str, **kwargs):
+    def _create_instance(cls, component_type: str, component_params: Dict[str, Any]):
         """
-        Load a component instance of the given type using the kwargs.
+        Returns an instance of the given subclass of BaseComponent.
 
         :param component_type: name of the component class to load.
-        :param kwargs: parameters to pass to the __init__() for the component.
+        :param component_params: parameters to pass to the __init__() for the component.
         """
         subclass = cls.get_subclass(component_type)
-        instance = subclass(**kwargs)
+        instance = subclass(**component_params)
         return instance
 
-    @classmethod
-    def load_from_pipeline_config(cls, pipeline_config: dict, component_name: str):
-        """
-        Load an individual component from a YAML config for Pipelines.
-
-        :param pipeline_config: the Pipelines YAML config parsed as a dict.
-        :param component_name: the name of the component to load.
-        """
-        if pipeline_config:
-            all_component_configs = pipeline_config["components"]
-            all_component_names = [comp["name"] for comp in all_component_configs]
-            component_config = next(comp for comp in all_component_configs if comp["name"] == component_name)
-            component_params = component_config["params"]
-
-            for key, value in component_params.items():
-                if value in all_component_names:  # check if the param value is a reference to another component
-                    component_params[key] = cls.load_from_pipeline_config(pipeline_config, value)
-
-            component_instance = cls.load_from_args(component_config["type"], **component_params)
-        else:
-            component_instance = cls.load_from_args(component_name)
-        return component_instance
-
     @abstractmethod
     def run(
         self,
@@ -251,3 +228,14 @@ def _get_signature(cls) -> Dict[str, inspect.Parameter]:
             for param_key, parameter in inspect.signature(class_).parameters.items()
         }
         return component_signature
+
+
+class RootNode(BaseComponent):
+    """
+    RootNode feeds inputs together with corresponding params to a Pipeline.
+    """
+
+    outgoing_edges = 1
+
+    def run(self):  # type: ignore
+        return {}, "output_1"

haystack/pipelines/__init__.py

@@ -1,4 +1,5 @@
-from haystack.pipelines.base import Pipeline, RootNode, RayPipeline
+from haystack.pipelines.base import Pipeline, RootNode
+from haystack.pipelines.ray import RayPipeline
 from haystack.pipelines.standard_pipelines import (
     BaseStandardPipeline,
     DocumentSearchPipeline,