From f228567e8c5ab7fa66df8b9db72c7b057d4557f8 Mon Sep 17 00:00:00 2001 From: George Panchuk Date: Thu, 12 Oct 2023 17:31:23 +0200 Subject: [PATCH] new: update generator launch script, update async files --- qdrant_client/async_client_base.py | 12 +-- qdrant_client/async_qdrant_client.py | 53 ++++++++---- qdrant_client/async_qdrant_remote.py | 86 +++++++++---------- .../client_generator.py | 2 +- .../fastembed_generator.py | 2 +- .../qdrant_remote_generator.py | 3 +- tools/generate_async_client.sh | 20 +++-- 7 files changed, 102 insertions(+), 76 deletions(-) diff --git a/qdrant_client/async_client_base.py b/qdrant_client/async_client_base.py index ee563157..5ca53f4a 100644 --- a/qdrant_client/async_client_base.py +++ b/qdrant_client/async_client_base.py @@ -57,8 +57,8 @@ async def recommend_batch( async def recommend( self, collection_name: str, - positive: Sequence[types.PointId], - negative: Optional[Sequence[types.PointId]] = None, + positive: Optional[Sequence[types.RecommendExample]] = None, + negative: Optional[Sequence[types.RecommendExample]] = None, query_filter: Optional[types.Filter] = None, search_params: Optional[types.SearchParams] = None, limit: int = 10, @@ -67,7 +67,8 @@ async def recommend( with_vectors: Union[bool, List[str]] = False, score_threshold: Optional[float] = None, using: Optional[str] = None, - lookup_from: Optional[models.LookupLocation] = None, + lookup_from: Optional[types.LookupLocation] = None, + strategy: Optional[types.RecommendStrategy] = None, **kwargs: Any, ) -> List[types.ScoredPoint]: raise NotImplementedError() @@ -76,8 +77,8 @@ async def recommend_groups( self, collection_name: str, group_by: str, - positive: Sequence[types.PointId], - negative: Optional[Sequence[types.PointId]] = None, + positive: Optional[Sequence[types.RecommendExample]] = None, + negative: Optional[Sequence[types.RecommendExample]] = None, query_filter: Optional[models.Filter] = None, search_params: Optional[models.SearchParams] = None, limit: int = 10, @@ -88,6 +89,7 @@ async def recommend_groups( using: Optional[str] = None, lookup_from: Optional[models.LookupLocation] = None, with_lookup: Optional[types.WithLookupInterface] = None, + strategy: Optional[types.RecommendStrategy] = None, **kwargs: Any, ) -> types.GroupsResult: raise NotImplementedError() diff --git a/qdrant_client/async_qdrant_client.py b/qdrant_client/async_qdrant_client.py index 814f6a89..49c96a18 100644 --- a/qdrant_client/async_qdrant_client.py +++ b/qdrant_client/async_qdrant_client.py @@ -388,8 +388,8 @@ async def recommend_batch( async def recommend( self, collection_name: str, - positive: Sequence[types.PointId], - negative: Optional[Sequence[types.PointId]] = None, + positive: Optional[Sequence[types.RecommendExample]] = None, + negative: Optional[Sequence[types.RecommendExample]] = None, query_filter: Optional[types.Filter] = None, search_params: Optional[types.SearchParams] = None, limit: int = 10, @@ -399,6 +399,7 @@ async def recommend( score_threshold: Optional[float] = None, using: Optional[str] = None, lookup_from: Optional[types.LookupLocation] = None, + strategy: Optional[types.RecommendStrategy] = None, consistency: Optional[types.ReadConsistency] = None, **kwargs: Any, ) -> List[types.ScoredPoint]: @@ -410,14 +411,16 @@ async def recommend( Args: collection_name: Collection to search in positive: - List of stored point IDs, which should be used as reference for similarity search. - If there is only one ID provided - this request is equivalent to the regular search with vector of that + List of stored point IDs or vectors, which should be used as reference for similarity search. + If there is only one example - this request is equivalent to the regular search with vector of that point. - If there are more than one IDs, Qdrant will attempt to search for similar to all of them. - Recommendation for multiple vectors is experimental. Its behaviour may change in the future. + If there are more than one example, Qdrant will attempt to search for similar to all of them. + Recommendation for multiple vectors is experimental. + Its behaviour may change depending on selected strategy. negative: - List of stored point IDs, which should be dissimilar to the search result. - Negative examples is an experimental functionality. Its behaviour may change in the future. + List of stored point IDs or vectors, which should be dissimilar to the search result. + Negative examples is an experimental functionality. + Its behaviour may change depending on selected strategy. query_filter: - Exclude vectors which doesn't fit given conditions. - If `None` - search among all vectors @@ -457,6 +460,12 @@ async def recommend( - 'majority' - query all replicas, but return values present in the majority of replicas - 'quorum' - query the majority of replicas, return values present in all of them - 'all' - query all replicas, and return values present in all replicas + strategy: + Strategy to use for recommendation. + Strategy defines how to combine multiple examples into a recommendation query. + Possible values: + - 'average_vector' - calculates average vector of all examples and uses it for search + - 'best_score' - finds the result which is closer to positive examples and further from negative Returns: List of recommended points with similarity scores. @@ -476,6 +485,7 @@ async def recommend( using=using, lookup_from=lookup_from, consistency=consistency, + strategy=strategy, **kwargs, ) @@ -483,8 +493,8 @@ async def recommend_groups( self, collection_name: str, group_by: str, - positive: Sequence[types.PointId], - negative: Optional[Sequence[types.PointId]] = None, + positive: Optional[Sequence[types.RecommendExample]] = None, + negative: Optional[Sequence[types.RecommendExample]] = None, query_filter: Optional[types.Filter] = None, search_params: Optional[types.SearchParams] = None, limit: int = 10, @@ -495,6 +505,7 @@ async def recommend_groups( using: Optional[str] = None, lookup_from: Optional[types.LookupLocation] = None, with_lookup: Optional[types.WithLookupInterface] = None, + strategy: Optional[types.RecommendStrategy] = None, consistency: Optional[types.ReadConsistency] = None, **kwargs: Any, ) -> types.GroupsResult: @@ -508,14 +519,16 @@ async def recommend_groups( Args: collection_name: Collection to search in positive: - List of stored point IDs, which should be used as reference for similarity search. - If there is only one ID provided - this request is equivalent to the regular search with vector of that + List of stored point IDs or vectors, which should be used as reference for similarity search. + If there is only one example - this request is equivalent to the regular search with vector of that point. - If there are more than one IDs, Qdrant will attempt to search for similar to all of them. - Recommendation for multiple vectors is experimental. Its behaviour may change in the future. + If there are more than one example, Qdrant will attempt to search for similar to all of them. + Recommendation for multiple vectors is experimental. + Its behaviour may change depending on selected strategy. negative: - List of stored point IDs, which should be dissimilar to the search result. - Negative examples is an experimental functionality. Its behaviour may change in the future. + List of stored point IDs or vectors, which should be dissimilar to the search result. + Negative examples is an experimental functionality. + Its behaviour may change depending on selected strategy. group_by: Name of the payload field to group by. Field must be of type "keyword" or "integer". Nested fields are specified using dot notation, e.g. "nested_field.subfield". @@ -560,6 +573,12 @@ async def recommend_groups( - 'majority' - query all replicas, but return values present in the majority of replicas - 'quorum' - query the majority of replicas, return values present in all of them - 'all' - query all replicas, and return values present in all replicas + strategy: + Strategy to use for recommendation. + Strategy defines how to combine multiple examples into a recommendation query. + Possible values: + - 'average_vector' - calculates average vector of all examples and uses it for search + - 'best_score' - finds the result which is closer to positive examples and further from negative Returns: List of groups with not more than `group_size` hits in each group. @@ -583,6 +602,7 @@ async def recommend_groups( lookup_from=lookup_from, consistency=consistency, with_lookup=with_lookup, + strategy=strategy, **kwargs, ) @@ -1409,6 +1429,7 @@ def upload_records( parallel=parallel, method=method, max_retries=max_retries, + wait=wait, **kwargs, ) diff --git a/qdrant_client/async_qdrant_remote.py b/qdrant_client/async_qdrant_remote.py index cd2eba26..fb8d8d6c 100644 --- a/qdrant_client/async_qdrant_remote.py +++ b/qdrant_client/async_qdrant_remote.py @@ -491,8 +491,8 @@ async def recommend_batch( async def recommend( self, collection_name: str, - positive: Sequence[types.PointId], - negative: Optional[Sequence[types.PointId]] = None, + positive: Optional[Sequence[types.RecommendExample]] = None, + negative: Optional[Sequence[types.RecommendExample]] = None, query_filter: Optional[types.Filter] = None, search_params: Optional[types.SearchParams] = None, limit: int = 10, @@ -502,24 +502,19 @@ async def recommend( score_threshold: Optional[float] = None, using: Optional[str] = None, lookup_from: Optional[types.LookupLocation] = None, + strategy: Optional[types.RecommendStrategy] = None, consistency: Optional[types.ReadConsistency] = None, **kwargs: Any, ) -> List[types.ScoredPoint]: + if positive is None: + positive = [] if negative is None: negative = [] if self._prefer_grpc: - positive = [ - RestToGrpc.convert_extended_point_id(point_id) - if isinstance(point_id, get_args_subscribed(models.ExtendedPointId)) - else point_id - for point_id in positive - ] - negative = [ - RestToGrpc.convert_extended_point_id(point_id) - if isinstance(point_id, get_args_subscribed(models.ExtendedPointId)) - else point_id - for point_id in negative - ] + positive_ids = RestToGrpc.convert_recommend_examples_to_ids(positive) + positive_vectors = RestToGrpc.convert_recommend_examples_to_vectors(positive) + negative_ids = RestToGrpc.convert_recommend_examples_to_ids(negative) + negative_vectors = RestToGrpc.convert_recommend_examples_to_vectors(negative) if isinstance(query_filter, models.Filter): query_filter = RestToGrpc.convert_filter(model=query_filter) if isinstance(search_params, models.SearchParams): @@ -532,11 +527,13 @@ async def recommend( lookup_from = RestToGrpc.convert_lookup_location(lookup_from) if isinstance(consistency, get_args_subscribed(models.ReadConsistency)): consistency = RestToGrpc.convert_read_consistency(consistency) + if isinstance(strategy, models.RecommendStrategy): + strategy = RestToGrpc.convert_recommend_strategy(strategy) res: grpc.SearchResponse = await self.grpc_points.Recommend( grpc.RecommendPoints( collection_name=collection_name, - positive=positive, - negative=negative, + positive=positive_ids, + negative=negative_ids, filter=query_filter, limit=limit, offset=offset, @@ -547,22 +544,25 @@ async def recommend( using=using, lookup_from=lookup_from, read_consistency=consistency, + strategy=strategy, + positive_vectors=positive_vectors, + negative_vectors=negative_vectors, ), timeout=self._timeout, ) return [GrpcToRest.convert_scored_point(hit) for hit in res.result] else: positive = [ - GrpcToRest.convert_point_id(point_id) - if isinstance(point_id, grpc.PointId) - else point_id - for point_id in positive + GrpcToRest.convert_point_id(example) + if isinstance(example, grpc.PointId) + else example + for example in positive ] negative = [ - GrpcToRest.convert_point_id(point_id) - if isinstance(point_id, grpc.PointId) - else point_id - for point_id in negative + GrpcToRest.convert_point_id(example) + if isinstance(example, grpc.PointId) + else example + for example in negative ] if isinstance(query_filter, grpc.Filter): query_filter = GrpcToRest.convert_filter(model=query_filter) @@ -588,6 +588,7 @@ async def recommend( score_threshold=score_threshold, lookup_from=lookup_from, using=using, + strategy=strategy, ), ) ).result @@ -598,8 +599,8 @@ async def recommend_groups( self, collection_name: str, group_by: str, - positive: Sequence[types.PointId], - negative: Optional[Sequence[types.PointId]] = None, + positive: Optional[Sequence[Union[types.PointId, List[float]]]] = None, + negative: Optional[Sequence[Union[types.PointId, List[float]]]] = None, query_filter: Optional[models.Filter] = None, search_params: Optional[models.SearchParams] = None, limit: int = 10, @@ -610,28 +611,21 @@ async def recommend_groups( using: Optional[str] = None, lookup_from: Optional[models.LookupLocation] = None, with_lookup: Optional[types.WithLookupInterface] = None, - consistency: Optional[models.ReadConsistencyType] = None, + strategy: Optional[types.RecommendStrategy] = None, + consistency: Optional[types.ReadConsistency] = None, **kwargs: Any, ) -> types.GroupsResult: - if negative is None: - negative = [] + positive = positive if positive is not None else [] + negative = negative if negative is not None else [] if self._prefer_grpc: if isinstance(with_lookup, models.WithLookup): with_lookup = RestToGrpc.convert_with_lookup(with_lookup) if isinstance(with_lookup, str): with_lookup = grpc.WithLookup(lookup_index=with_lookup) - positive = [ - RestToGrpc.convert_extended_point_id(point_id) - if isinstance(point_id, get_args_subscribed(models.ExtendedPointId)) - else point_id - for point_id in positive - ] - negative = [ - RestToGrpc.convert_extended_point_id(point_id) - if isinstance(point_id, get_args_subscribed(models.ExtendedPointId)) - else point_id - for point_id in negative - ] + positive_ids = RestToGrpc.convert_recommend_examples_to_ids(positive) + positive_vectors = RestToGrpc.convert_recommend_examples_to_vectors(positive) + negative_ids = RestToGrpc.convert_recommend_examples_to_ids(negative) + negative_vectors = RestToGrpc.convert_recommend_examples_to_vectors(negative) if isinstance(query_filter, models.Filter): query_filter = RestToGrpc.convert_filter(model=query_filter) if isinstance(search_params, models.SearchParams): @@ -644,12 +638,14 @@ async def recommend_groups( lookup_from = RestToGrpc.convert_lookup_location(lookup_from) if isinstance(consistency, get_args_subscribed(models.ReadConsistency)): consistency = RestToGrpc.convert_read_consistency(consistency) + if isinstance(strategy, models.RecommendStrategy): + strategy = RestToGrpc.convert_recommend_strategy(strategy) res: grpc.GroupsResult = ( await self.grpc_points.RecommendGroups( grpc.RecommendPointGroups( collection_name=collection_name, - positive=positive, - negative=negative, + positive=positive_ids, + negative=negative_ids, filter=query_filter, group_by=group_by, limit=limit, @@ -662,6 +658,9 @@ async def recommend_groups( lookup_from=lookup_from, read_consistency=consistency, with_lookup=with_lookup, + strategy=strategy, + positive_vectors=positive_vectors, + negative_vectors=negative_vectors, ), timeout=self._timeout, ) @@ -710,6 +709,7 @@ async def recommend_groups( lookup_from=lookup_from, using=using, with_lookup=with_lookup, + strategy=strategy, ), ) ).result diff --git a/tools/async_client_generator/client_generator.py b/tools/async_client_generator/client_generator.py index 1c946cf7..a919b6bc 100644 --- a/tools/async_client_generator/client_generator.py +++ b/tools/async_client_generator/client_generator.py @@ -4,7 +4,7 @@ import inspect from typing import Optional -from qdrant_client.async_client_base import AsyncQdrantBase +from .async_client_base import AsyncQdrantBase class AsyncAwaitTransformer(ast.NodeTransformer): diff --git a/tools/async_client_generator/fastembed_generator.py b/tools/async_client_generator/fastembed_generator.py index 76ac3a1b..cafc6417 100644 --- a/tools/async_client_generator/fastembed_generator.py +++ b/tools/async_client_generator/fastembed_generator.py @@ -4,7 +4,7 @@ import inspect from typing import Optional -from qdrant_client.async_client_base import AsyncQdrantBase +from .async_client_base import AsyncQdrantBase # Define a custom AST transformer to add 'await' before method calls diff --git a/tools/async_client_generator/qdrant_remote_generator.py b/tools/async_client_generator/qdrant_remote_generator.py index af21ef1a..dcf0016d 100644 --- a/tools/async_client_generator/qdrant_remote_generator.py +++ b/tools/async_client_generator/qdrant_remote_generator.py @@ -4,7 +4,6 @@ import inspect from typing import Optional -from qdrant_client.async_client_base import AsyncQdrantBase from qdrant_client.grpc import CollectionsStub, PointsStub, SnapshotsStub from qdrant_client.http import AsyncApiClient from qdrant_client.http.api.cluster_api import AsyncClusterApi @@ -13,6 +12,8 @@ from qdrant_client.http.api.service_api import AsyncServiceApi from qdrant_client.http.api.snapshots_api import AsyncSnapshotsApi +from .async_client_base import AsyncQdrantBase + class AsyncAwaitTransformer(ast.NodeTransformer): def __init__( diff --git a/tools/generate_async_client.sh b/tools/generate_async_client.sh index f757332d..13ec8a68 100755 --- a/tools/generate_async_client.sh +++ b/tools/generate_async_client.sh @@ -2,20 +2,22 @@ set -e -PROJECT_ROOT="$(dirname "$0")/../" +RELATIVE_PROJECT_ROOT="$(dirname "$0")/.." +cd $RELATIVE_PROJECT_ROOT +ABSOLUTE_PROJECT_ROOT=$(pwd) -cd $(mktemp -d) +cd $ABSOLUTE_PROJECT_ROOT/tools/async_client_generator python3 -m tools.async_client_generator.base_client_generator python3 -m tools.async_client_generator.fastembed_generator python3 -m tools.async_client_generator.client_generator python3 -m tools.async_client_generator.qdrant_remote_generator -ls -1 | autoflake --recursive --imports qdrant_client --remove-unused-variables --in-place ./ -ls -1 | xargs -I {} isort -w 99 --multi-line 3 --trailing-comma --force-grid-wrap 0 --combine-as {} -ls -1 | xargs -I {} black --fast -l 99 --target-version py38 {} +ls -1 | autoflake --recursive --imports qdrant_client --remove-unused-variables --in-place async*.py +ls -1 | grep async | xargs -I {} isort -w 99 --multi-line 3 --trailing-comma --force-grid-wrap 0 --combine-as {} +ls -1 | grep async | xargs -I {} black --fast -l 99 --target-version py38 {} -mv async_client_base.py $PROJECT_ROOT/qdrant_client/async_client_base.py -mv async_qdrant_client.py $PROJECT_ROOT/qdrant_client/async_qdrant_client.py -mv async_qdrant_fastembed.py $PROJECT_ROOT/qdrant_client/async_qdrant_fastembed.py -mv async_qdrant_remote.py $PROJECT_ROOT/qdrant_client/async_qdrant_remote.py +mv async_client_base.py $ABSOLUTE_PROJECT_ROOT/qdrant_client/async_client_base.py +mv async_qdrant_client.py $ABSOLUTE_PROJECT_ROOT/qdrant_client/async_qdrant_client.py +mv async_qdrant_fastembed.py $ABSOLUTE_PROJECT_ROOT/qdrant_client/async_qdrant_fastembed.py +mv async_qdrant_remote.py $ABSOLUTE_PROJECT_ROOT/qdrant_client/async_qdrant_remote.py