Skip to content

Commit

Permalink
Initial functional tests
Browse files Browse the repository at this point in the history
Co-authored-by: Trawinski, Dariusz <[email protected]>
  • Loading branch information
bstrzele and dtrawins authored Dec 6, 2024
1 parent 5c00350 commit 0a3fb4c
Show file tree
Hide file tree
Showing 20 changed files with 640 additions and 0 deletions.
18 changes: 18 additions & 0 deletions tests/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
# Running tests

Installing and running tests
```bash
pip install -r requirements.txt
pytest
```

Running tests with gpu
```bash
pytest --gpu
```

Run tests while caching downloaded models
```bash
pytest --model-cache ./cache
```

Empty file added tests/__init__.py
Empty file.
20 changes: 20 additions & 0 deletions tests/configs/dynamic.pbtxt
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
input [
{
name: "data"
data_type: TYPE_FP32
dims: [ -1, 3, -1, -1]
}
]
output [
{
name: "detection_out"
data_type: TYPE_FP32
dims: [ 1, 1, -1, 7]
}
]
parameters: {
key: "RESHAPE_IO_LAYERS"
value: {
string_value:"yes"
}
}
28 changes: 28 additions & 0 deletions tests/configs/dynamic_gpu.pbtxt
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
input: [
{
name: "data"
data_type: TYPE_FP32
dims: [ -1, 3, -1, -1]
}
]
output: [
{
name: "detection_out"
data_type: TYPE_FP32
dims: [ 1, 1, -1, 7]
}
]
parameters: [
{
key: "RESHAPE_IO_LAYERS"
value: {
string_value:"yes"
}
},
{
key: "TARGET_DEVICE"
value: {
string_value: "GPU"
}
}
]
14 changes: 14 additions & 0 deletions tests/configs/ir.pbtxt
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
parameters: [
{
key: "NUM_STREAMS"
value: {
string_value: "1"
}
},
{
key: "PERFORMANCE_HINT"
value: {
string_value: "LATENCY"
}
}
]
20 changes: 20 additions & 0 deletions tests/configs/ir_gpu.pbtxt
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
parameters: [
{
key: "NUM_STREAMS"
value: {
string_value: "1"
}
},
{
key: "PERFORMANCE_HINT"
value: {
string_value: "LATENCY"
}
},
{
key: "TARGET_DEVICE"
value: {
string_value: "GPU"
}
}
]
23 changes: 23 additions & 0 deletions tests/configs/onnx.pbtxt
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
backend: "openvino"
input: [
{
name: "gpu_0/data_0"
data_type: TYPE_FP32
dims: [ -1, 3, 224, 224]
}
]
parameters: [
{
key: "NUM_STREAMS"
value: {
string_value: "1"
}
},
{
key: "PERFORMANCE_HINT"
value: {
string_value: "LATENCY"
}
}
]
default_model_filename: "model.onnx"
30 changes: 30 additions & 0 deletions tests/configs/onnx_gpu.pbtxt
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
backend: "openvino"
input: [
{
name: "gpu_0/data_0"
data_type: TYPE_FP32
dims: [ -1, 3, 224, 224]
}
]
parameters: [
{
key: "NUM_STREAMS"
value: {
string_value: "1"
}
},
{
key: "PERFORMANCE_HINT"
value: {
string_value: "LATENCY"
}
},
{
key: "TARGET_DEVICE"
value: {
string_value: "GPU"
}
}
]
default_model_filename: "model.onnx"
31 changes: 31 additions & 0 deletions tests/configs/paddle.pbtxt
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
backend: "openvino"
input: [
{
name: "inputs"
data_type: TYPE_FP32
dims: [ -1, 3, 224, 224]
}
]
output: [
{
name: "save_infer_model/scale_0.tmp_1"
data_type: TYPE_FP32
dims: [ -1, 1000]
}
]

parameters: [
{
key: "NUM_STREAMS"
value: {
string_value: "1"
}
},
{
key: "PERFORMANCE_HINT"
value: {
string_value: "LATENCY"
}
}
]
default_model_filename: "model.pdmodel"
31 changes: 31 additions & 0 deletions tests/configs/paddle_gpu.pbtxt
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
backend: "openvino"
input: [
{
name: "inputs"
data_type: TYPE_FP32
dims: [ -1, 3, 224, 224]
}
]
output: [
{
name: "save_infer_model/scale_0.tmp_1"
data_type: TYPE_FP32
dims: [ -1, 1000]
}
]

parameters: [
{
key: "NUM_STREAMS"
value: {
string_value: "1"
}
},
{
key: "PERFORMANCE_HINT"
value: {
string_value: "LATENCY"
}
}
]
default_model_filename: "model.pdmodel"
30 changes: 30 additions & 0 deletions tests/configs/saved_model.pbtxt
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
backend: "openvino"
parameters: [
{
key: "NUM_STREAMS"
value: {
string_value: "1"
}
},
{
key: "PERFORMANCE_HINT"
value: {
string_value: "LATENCY"
}
}
]
input [
{
name: "input_1"
data_type: TYPE_FP32
dims: [-1, 224, 224, 3 ]
}
]
output [
{
name: "activation_49"
data_type: TYPE_FP32
dims: [-1, 1001]
}
]
default_model_filename: "model.saved_model"
16 changes: 16 additions & 0 deletions tests/configs/tflite.pbtxt
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
backend: "openvino"
parameters: [
{
key: "NUM_STREAMS"
value: {
string_value: "1"
}
},
{
key: "PERFORMANCE_HINT"
value: {
string_value: "LATENCY"
}
}
]
default_model_filename: "model.tflite"
22 changes: 22 additions & 0 deletions tests/configs/tflite_gpu.pbtxt
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
backend: "openvino"
parameters: [
{
key: "NUM_STREAMS"
value: {
string_value: "1"
}
},
{
key: "PERFORMANCE_HINT"
value: {
string_value: "LATENCY"
}
},
{
key: "TARGET_DEVICE"
value: {
string_value: "GPU"
}
}
]
default_model_filename: "model.tflite"
29 changes: 29 additions & 0 deletions tests/conftest.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
import shutil
import subprocess
import sys

import pytest


def pytest_addoption(parser):
parser.addoption("--gpu", action="store_true")
parser.addoption("--model-cache", action="store")
parser.addoption("--image", action="store")


def pytest_configure(config):
config.addinivalue_line(
"markers",
"gpu: run tests on GPU device",
)


def pytest_collection_modifyitems(config, items):
for item in items:
if "gpu" in item.keywords:
if not config.getoption("--gpu"):
item.add_marker(
pytest.mark.skip(
"Test requires --gpu flag to be set and Intel GPU device on the host machine"
)
)
26 changes: 26 additions & 0 deletions tests/functional/common.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
import os

import grpc
import numpy as np
import tritonclient.grpc as grpcclient
from tritonclient.grpc import service_pb2_grpc
from tritonclient.utils import np_to_triton_dtype


def prepare_inputs(input_name, shape, dtype):
inputs = []
inputs.append(grpcclient.InferInput(input_name, shape, np_to_triton_dtype(dtype)))

inputs[0].set_data_from_numpy(np.ones(shape, dtype=dtype))
return inputs


def prepare_grpc_stub(port):
channel = grpc.insecure_channel(f"localhost:{port}")
return service_pb2_grpc.GRPCInferenceServiceStub(channel)


def prepare_triton_client(port):
return grpcclient.InferenceServerClient(
url=f"localhost:{port}", verbose=os.environ.get("LOG_LEVEL") == "DEBUG"
)
Loading

0 comments on commit 0a3fb4c

Please sign in to comment.