Merge branch 'force-deterministic-order' of https://github.com/jina-a…

…i/jina into force-deterministic-order
jina-ai · Nov 9, 2022 · a4ea275 · a4ea275
2 parents 97771a7 + a7db021
commit a4ea275
Show file tree

Hide file tree

Showing 4 changed files with 22 additions and 12 deletions.
diff --git a/docs/fundamentals/executor/executor-methods.md b/docs/fundamentals/executor/executor-methods.md
@@ -50,11 +50,14 @@ with f:
 ```
 
 ```shell
-           Flow@18048[I]:🎉 Flow is ready to use!                                                   
-	🔗 Protocol: 		GRPC
-	🏠 Local access:	0.0.0.0:52255
-	🔒 Private network:	192.168.1.187:52255
-	🌐 Public address:	212.231.186.65:52255
+────────────────────────── 🎉 Flow is ready to serve! ──────────────────────────
+╭────────────── 🔗 Endpoint ───────────────╮
+│  ⛓     Protocol                    GRPC  │
+│  🏠       Local           0.0.0.0:55925  │
+│  🔒     Private     192.168.1.187:55925  │
+│  🌍      Public    212.231.186.65:55925  │
+╰──────────────────────────────────────────╯
+
 Calling foo
 Calling bar
 Calling foo
@@ -120,7 +123,7 @@ any other `list`-like object in a Python function.
 
 - `docs_matrix`:  This is the least common parameter to be used for an Executor. This is needed when an Executor is used inside a Flow to merge or reduce the output of more than one other Executor.
 
-
+- `tracing_context`: Context needed if you want to add custom traces. Check {ref}`how to add custom traces in your Executor <instrumenting-executor>`
 
 
 ````{admonition} Hint

diff --git a/docs/fundamentals/flow/executor-args.md b/docs/fundamentals/flow/executor-args.md
@@ -17,6 +17,8 @@
 | `py_modules` | The customized python modules need to be imported before loading the executor<br><br>Note that the recommended way is to only import a single module - a simple python file, if your<br>executor can be defined in a single file, or an ``__init__.py`` file if you have multiple files,<br>which should be structured as a python package. For more details, please see the<br>`Executor cookbook <https://docs.jina.ai/fundamentals/executor/executor-files/>`__ | `array` | `None` |
 | `output_array_type` | The type of array `tensor` and `embedding` will be serialized to.<br><br>Supports the same types as `docarray.to_protobuf(.., ndarray_type=...)`, which can be found <br>`here <https://docarray.jina.ai/fundamentals/document/serialization/#from-to-protobuf>`.<br>Defaults to retaining whatever type is returned by the Executor. | `string` | `None` |
 | `exit_on_exceptions` | List of exceptions that will cause the Executor to shut down. | `array` | `[]` |
+| `disable_reduce` | Disable the built-in reduce mechanism, set this if the reduction is to be handled by the Executor, if the Executor is expected to handle `docs_matrix` | `boolean` | `False` |
+| `docs_by_executor` | If enabled, in combination with `--disable-reduce`,the `docs_matrix` received by Executor will be a dictionary with previous Executor name in the key and docs as values | `boolean` | `False` |
 | `port` | The port for input data to bind to, default is a random port between [49152, 65535]. In the case of an external Executor (`--external` or `external=True`) this can be a list of ports, separated by commas. Then, every resulting address will be considered as one replica of the Executor. | `string` | `random in [49152, 65535]` |
 | `host_in` | The host address for binding to, by default it is 0.0.0.0 | `string` | `0.0.0.0` |
 | `grpc_server_options` | Dictionary of kwargs arguments that will be passed to the grpc server as options when starting the server, example : {'grpc.max_send_message_length': -1} | `object` | `None` |
@@ -47,5 +49,4 @@
 | `uses_before_address` | The address of the uses-before runtime | `string` | `None` |
 | `uses_after_address` | The address of the uses-before runtime | `string` | `None` |
 | `connection_list` | dictionary JSON with a list of connections to configure | `string` | `None` |
-| `disable_reduce` | Disable the built-in reduce mechanism, set this if the reduction is to be handled by the Executor connected to this Head | `boolean` | `False` |
 | `timeout_send` | The timeout in milliseconds used when sending data requests to Executors, -1 means no timeout, disabled by default | `number` | `None` |
diff --git a/jina/orchestrate/flow/base.py b/jina/orchestrate/flow/base.py
@@ -835,6 +835,7 @@ def add(
         disable_auto_volume: Optional[bool] = False,
         disable_reduce: Optional[bool] = False,
         docker_kwargs: Optional[dict] = None,
+        docs_by_executor: Optional[bool] = False,
         entrypoint: Optional[str] = None,
         env: Optional[dict] = None,
         exit_on_exceptions: Optional[List[str]] = [],
@@ -892,11 +893,12 @@ def add(
         :param compression: The compression mechanism used when sending requests from the Head to the WorkerRuntimes. For more details, check https://grpc.github.io/grpc/python/grpc.html#compression.
         :param connection_list: dictionary JSON with a list of connections to configure
         :param disable_auto_volume: Do not automatically mount a volume for dockerized Executors.
-        :param disable_reduce: Disable the built-in reduce mechanism, set this if the reduction is to be handled by the Executor connected to this Head
+        :param disable_reduce: Disable the built-in reduce mechanism, set this if the reduction is to be handled by the Executor, if the Executor is expected to handle `docs_matrix`
         :param docker_kwargs: Dictionary of kwargs arguments that will be passed to Docker SDK when starting the docker '
           container.
 
           More details can be found in the Docker SDK docs:  https://docker-py.readthedocs.io/en/stable/
+        :param docs_by_executor: If enabled, in combination with `--disable-reduce`,the `docs_matrix` received by Executor will be a dictionary with previous Executor name in the key and docs as values
         :param entrypoint: The entrypoint command overrides the ENTRYPOINT in Docker image. when not set then the Docker image ENTRYPOINT takes effective.
         :param env: The map of environment variables that are available inside runtime
         :param exit_on_exceptions: List of exceptions that will cause the Executor to shut down.
@@ -1048,11 +1050,12 @@ def add(
         :param compression: The compression mechanism used when sending requests from the Head to the WorkerRuntimes. For more details, check https://grpc.github.io/grpc/python/grpc.html#compression.
         :param connection_list: dictionary JSON with a list of connections to configure
         :param disable_auto_volume: Do not automatically mount a volume for dockerized Executors.
-        :param disable_reduce: Disable the built-in reduce mechanism, set this if the reduction is to be handled by the Executor connected to this Head
+        :param disable_reduce: Disable the built-in reduce mechanism, set this if the reduction is to be handled by the Executor, if the Executor is expected to handle `docs_matrix`
         :param docker_kwargs: Dictionary of kwargs arguments that will be passed to Docker SDK when starting the docker '
           container.
 
           More details can be found in the Docker SDK docs:  https://docker-py.readthedocs.io/en/stable/
+        :param docs_by_executor: If enabled, in combination with `--disable-reduce`,the `docs_matrix` received by Executor will be a dictionary with previous Executor name in the key and docs as values
         :param entrypoint: The entrypoint command overrides the ENTRYPOINT in Docker image. when not set then the Docker image ENTRYPOINT takes effective.
         :param env: The map of environment variables that are available inside runtime
         :param exit_on_exceptions: List of exceptions that will cause the Executor to shut down.

diff --git a/jina_cli/autocomplete.py b/jina_cli/autocomplete.py
@@ -40,6 +40,8 @@
             '--py-modules',
             '--output-array-type',
             '--exit-on-exceptions',
+            '--disable-reduce',
+            '--docs-by-executor',
             '--port-in',
             '--host-in',
             '--grpc-server-options',
@@ -75,7 +77,6 @@
             '--uses-before-address',
             '--uses-after-address',
             '--connection-list',
-            '--disable-reduce',
             '--timeout-send',
         ],
         'flow': [
@@ -252,6 +253,8 @@
             '--py-modules',
             '--output-array-type',
             '--exit-on-exceptions',
+            '--disable-reduce',
+            '--docs-by-executor',
             '--port-in',
             '--host-in',
             '--grpc-server-options',
@@ -287,7 +290,6 @@
             '--uses-before-address',
             '--uses-after-address',
             '--connection-list',
-            '--disable-reduce',
             '--timeout-send',
         ],
         'deployment': [
@@ -312,6 +314,8 @@
             '--py-modules',
             '--output-array-type',
             '--exit-on-exceptions',
+            '--disable-reduce',
+            '--docs-by-executor',
             '--port-in',
             '--host-in',
             '--grpc-server-options',
@@ -347,7 +351,6 @@
             '--uses-before-address',
             '--uses-after-address',
             '--connection-list',
-            '--disable-reduce',
             '--timeout-send',
             '--uses-before',
             '--uses-after',