From 67b9dcd7f965682062fcd6243498b1fafe120076 Mon Sep 17 00:00:00 2001 From: Iman Tabrizian Date: Fri, 8 Sep 2023 13:24:40 -0400 Subject: [PATCH] Add L0_request_cancellation (#6252) * Add L0_request_cancellation * Remove unittest test * Add cancellation to gRPC server error handling * Fix up * Use identity model --- Dockerfile.QA | 1 + .../models/model/config.pbtxt | 43 ++++++++++++++ qa/L0_request_cancellation/test.sh | 57 +++++++++++++++++++ src/grpc/grpc_utils.cc | 2 + 4 files changed, 103 insertions(+) create mode 100644 qa/L0_request_cancellation/models/model/config.pbtxt create mode 100755 qa/L0_request_cancellation/test.sh diff --git a/Dockerfile.QA b/Dockerfile.QA index 52f66f33335..7fc33e71954 100644 --- a/Dockerfile.QA +++ b/Dockerfile.QA @@ -139,6 +139,7 @@ RUN mkdir -p qa/common && \ cp bin/data_compressor_test qa/L0_data_compression/. && \ cp bin/metrics_api_test qa/L0_metrics/. && \ cp bin/response_cache_test qa/L0_response_cache/. && \ + cp bin/request_cancellation_test qa/L0_request_cancellation/. && \ cp bin/triton_json_test qa/L0_json/. && \ cp bin/backend_output_detail_test qa/L0_backend_output_detail/. && \ cp -r deploy/mlflow-triton-plugin qa/L0_mlflow/. diff --git a/qa/L0_request_cancellation/models/model/config.pbtxt b/qa/L0_request_cancellation/models/model/config.pbtxt new file mode 100644 index 00000000000..7776515b499 --- /dev/null +++ b/qa/L0_request_cancellation/models/model/config.pbtxt @@ -0,0 +1,43 @@ +# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +backend: "identity" +name: "model" +max_batch_size: 64 +input { +name: "INPUT0" +data_type: TYPE_INT32 +dims: 1000 +} +output { +name: "OUTPUT0" +data_type: TYPE_INT32 +dims: 1000 +} +instance_group { +kind: KIND_CPU +} + diff --git a/qa/L0_request_cancellation/test.sh b/qa/L0_request_cancellation/test.sh new file mode 100755 index 00000000000..7a359ebaecc --- /dev/null +++ b/qa/L0_request_cancellation/test.sh @@ -0,0 +1,57 @@ +#!/bin/bash +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +REPO_VERSION=${NVIDIA_TRITON_SERVER_VERSION} +if [ "$#" -ge 1 ]; then + REPO_VERSION=$1 +fi +if [ -z "$REPO_VERSION" ]; then + echo -e "Repository version must be specified" + echo -e "\n***\n*** Test Failed\n***" + exit 1 +fi +if [ ! -z "$TEST_REPO_ARCH" ]; then + REPO_VERSION=${REPO_VERSION}_${TEST_REPO_ARCH} +fi +DATADIR=${DATADIR:="/data/inferenceserver/${REPO_VERSION}"} + +RET=0 + +mkdir -p models/model/1 + +SERVER_LOG=server.log +LD_LIBRARY_PATH=/opt/tritonserver/lib:$LD_LIBRARY_PATH ./request_cancellation_test > $SERVER_LOG +if [ $? -ne 0 ]; then + cat $SERVER_LOG + RET=1 +fi + +if [ $RET -eq 0 ]; then + echo -e "\n***\n*** Test Passed\n***" +else + echo -e "\n***\n*** Test FAILED\n***" +fi diff --git a/src/grpc/grpc_utils.cc b/src/grpc/grpc_utils.cc index 11742ff5ed0..6cb211542db 100644 --- a/src/grpc/grpc_utils.cc +++ b/src/grpc/grpc_utils.cc @@ -60,6 +60,8 @@ GrpcStatusUtil::CodeToStatus(TRITONSERVER_Error_Code code) return ::grpc::StatusCode::UNIMPLEMENTED; case TRITONSERVER_ERROR_ALREADY_EXISTS: return ::grpc::StatusCode::ALREADY_EXISTS; + case TRITONSERVER_ERROR_CANCELLED: + return ::grpc::StatusCode::CANCELLED; } return ::grpc::StatusCode::UNKNOWN;