From f13ecb8d268c7d1eca71c5ba49d15677d1a3ba06 Mon Sep 17 00:00:00 2001 From: Iman Tabrizian Date: Thu, 31 Aug 2023 00:48:01 -0400 Subject: [PATCH 1/5] Add L0_request_cancellation --- Dockerfile.QA | 1 + qa/L0_request_cancellation/test.sh | 59 ++++++++++++++++++++++++++++++ src/test/CMakeLists.txt | 1 + 3 files changed, 61 insertions(+) create mode 100755 qa/L0_request_cancellation/test.sh diff --git a/Dockerfile.QA b/Dockerfile.QA index 5007a97455..5b38c62a40 100644 --- a/Dockerfile.QA +++ b/Dockerfile.QA @@ -139,6 +139,7 @@ RUN mkdir -p qa/common && \ cp bin/data_compressor_test qa/L0_data_compression/. && \ cp bin/metrics_api_test qa/L0_metrics/. && \ cp bin/response_cache_test qa/L0_response_cache/. && \ + cp bin/request_cancellation_test qa/L0_request_cancellation/. && \ cp bin/triton_json_test qa/L0_json/. && \ cp -r deploy/mlflow-triton-plugin qa/L0_mlflow/. diff --git a/qa/L0_request_cancellation/test.sh b/qa/L0_request_cancellation/test.sh new file mode 100755 index 0000000000..09320ad00b --- /dev/null +++ b/qa/L0_request_cancellation/test.sh @@ -0,0 +1,59 @@ +#!/bin/bash +# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +REPO_VERSION=${NVIDIA_TRITON_SERVER_VERSION} +if [ "$#" -ge 1 ]; then + REPO_VERSION=$1 +fi +if [ -z "$REPO_VERSION" ]; then + echo -e "Repository version must be specified" + echo -e "\n***\n*** Test Failed\n***" + exit 1 +fi +if [ ! -z "$TEST_REPO_ARCH" ]; then + REPO_VERSION=${REPO_VERSION}_${TEST_REPO_ARCH} +fi +DATADIR=${DATADIR:="/data/inferenceserver/${REPO_VERSION}"} + +RET=0 + +mkdir -p models/ +cp -r $DATADIR/qa_identity_model_repository/libtorch_nobatch_zero_1_bool models/model +sed -i 's/name:.*/name: "model"/' models/model/config.pbtxt + +LOG_FILE=server.log +LD_LIBRARY_PATH=/opt/tritonserver/lib:$LD_LIBRARY_PATH ./request_cancellation_test > $SERVER_LOG +if [ $? -ne 0 ]; then + cat $SERVER_LOG + RET=1 +fi + +if [ $RET -eq 0 ]; then + echo -e "\n***\n*** Test Passed\n***" +else + echo -e "\n***\n*** Test FAILED\n***" +fi diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt index d021a51a15..a80cda9c84 100644 --- a/src/test/CMakeLists.txt +++ b/src/test/CMakeLists.txt @@ -102,6 +102,7 @@ add_subdirectory(distributed_addsub distributed_addsub) add_subdirectory(dyna_sequence dyna_sequence) add_subdirectory(implicit_state implicit_state) add_subdirectory(query_backend query_backend) +add_subdirectory(unittest unittest) if(${TRITON_ENABLE_GPU}) add_subdirectory(sequence sequence) From 0dea83f6ca754e2d0654cd4a3b6f89fadad66c82 Mon Sep 17 00:00:00 2001 From: Iman Tabrizian Date: Thu, 31 Aug 2023 17:15:53 -0400 Subject: [PATCH 2/5] Remove unittest test --- src/test/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt index a80cda9c84..d021a51a15 100644 --- a/src/test/CMakeLists.txt +++ b/src/test/CMakeLists.txt @@ -102,7 +102,6 @@ add_subdirectory(distributed_addsub distributed_addsub) add_subdirectory(dyna_sequence dyna_sequence) add_subdirectory(implicit_state implicit_state) add_subdirectory(query_backend query_backend) -add_subdirectory(unittest unittest) if(${TRITON_ENABLE_GPU}) add_subdirectory(sequence sequence) From 4d1a2732d73f01c026ead380b72464ba88cd2229 Mon Sep 17 00:00:00 2001 From: Iman Tabrizian Date: Thu, 31 Aug 2023 18:21:19 -0400 Subject: [PATCH 3/5] Add cancellation to gRPC server error handling --- src/grpc/grpc_utils.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/grpc/grpc_utils.cc b/src/grpc/grpc_utils.cc index 11742ff5ed..6cb211542d 100644 --- a/src/grpc/grpc_utils.cc +++ b/src/grpc/grpc_utils.cc @@ -60,6 +60,8 @@ GrpcStatusUtil::CodeToStatus(TRITONSERVER_Error_Code code) return ::grpc::StatusCode::UNIMPLEMENTED; case TRITONSERVER_ERROR_ALREADY_EXISTS: return ::grpc::StatusCode::ALREADY_EXISTS; + case TRITONSERVER_ERROR_CANCELLED: + return ::grpc::StatusCode::CANCELLED; } return ::grpc::StatusCode::UNKNOWN; From 0311ad33174b0f54a8783b497890f79532170642 Mon Sep 17 00:00:00 2001 From: Iman Tabrizian Date: Fri, 1 Sep 2023 11:32:00 -0400 Subject: [PATCH 4/5] Fix up --- qa/L0_request_cancellation/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qa/L0_request_cancellation/test.sh b/qa/L0_request_cancellation/test.sh index 09320ad00b..60b99ff7c7 100755 --- a/qa/L0_request_cancellation/test.sh +++ b/qa/L0_request_cancellation/test.sh @@ -45,7 +45,7 @@ mkdir -p models/ cp -r $DATADIR/qa_identity_model_repository/libtorch_nobatch_zero_1_bool models/model sed -i 's/name:.*/name: "model"/' models/model/config.pbtxt -LOG_FILE=server.log +SERVER_LOG=server.log LD_LIBRARY_PATH=/opt/tritonserver/lib:$LD_LIBRARY_PATH ./request_cancellation_test > $SERVER_LOG if [ $? -ne 0 ]; then cat $SERVER_LOG From 028abe39148771d69d7c80b5a931f4f586e0d79d Mon Sep 17 00:00:00 2001 From: Iman Tabrizian Date: Wed, 6 Sep 2023 11:52:56 -0400 Subject: [PATCH 5/5] Use identity model --- .../models/model/config.pbtxt | 43 +++++++++++++++++++ qa/L0_request_cancellation/test.sh | 4 +- 2 files changed, 44 insertions(+), 3 deletions(-) create mode 100644 qa/L0_request_cancellation/models/model/config.pbtxt diff --git a/qa/L0_request_cancellation/models/model/config.pbtxt b/qa/L0_request_cancellation/models/model/config.pbtxt new file mode 100644 index 0000000000..7776515b49 --- /dev/null +++ b/qa/L0_request_cancellation/models/model/config.pbtxt @@ -0,0 +1,43 @@ +# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +backend: "identity" +name: "model" +max_batch_size: 64 +input { +name: "INPUT0" +data_type: TYPE_INT32 +dims: 1000 +} +output { +name: "OUTPUT0" +data_type: TYPE_INT32 +dims: 1000 +} +instance_group { +kind: KIND_CPU +} + diff --git a/qa/L0_request_cancellation/test.sh b/qa/L0_request_cancellation/test.sh index 60b99ff7c7..7a359ebaec 100755 --- a/qa/L0_request_cancellation/test.sh +++ b/qa/L0_request_cancellation/test.sh @@ -41,9 +41,7 @@ DATADIR=${DATADIR:="/data/inferenceserver/${REPO_VERSION}"} RET=0 -mkdir -p models/ -cp -r $DATADIR/qa_identity_model_repository/libtorch_nobatch_zero_1_bool models/model -sed -i 's/name:.*/name: "model"/' models/model/config.pbtxt +mkdir -p models/model/1 SERVER_LOG=server.log LD_LIBRARY_PATH=/opt/tritonserver/lib:$LD_LIBRARY_PATH ./request_cancellation_test > $SERVER_LOG