Load tests

examples/grpc_client/Makefile

@@ -15,4 +15,4 @@
 #
 
 style:
-	flake8 --max-line-length 100 images_2_numpy.py grpc_client_utils.py grpc_client.py
+	flake8 --max-line-length 100 images_2_numpy.py grpc_client_utils.py grpc_client.py load_testing/image_locust.py

examples/grpc_client/grpc_client.py

@@ -145,7 +145,7 @@ def main(**kwargs):
         imgs = prepare_images(kwargs)
 
         if kwargs['transpose_input']:
-            imgs = imgs.transpose((0, 2, 3, 1))
+            imgs = imgs.transpose((0, 3, 1, 2))
 
         output = inference(stub, request, imgs, kwargs)
 

examples/grpc_client/load_testing/README.md

@@ -0,0 +1,48 @@
+# Load Test
+
+## Prerequisites
+* Python 3.6
+* Install [requirements](../requirements.txt) needed to launch [grpc client](../README.md)
+* Install Locust requirements ```pip install -r requirements-locust.txt```
+
+
+## Description
+Script available under this directory allow users to execute on their own load tests. Prepared [Locust class](image_locust.py)
+on start loads environment variables and images also create grpc stub. In hatching phase this client only perform request to GRPC service using earlier loaded images. 
+The number of comma-delimited photo paths determines the size of the batch. Now this tests only support resnet model.
+
+## Configurable parameters
+````
+    GRPC_ADDRESS # address to grpc endpoint
+    MODEL_NAME = # model name; default value=resnet
+    TENSOR_NAME # input tensor name' default value=in
+    IMAGES # path to images, separated by comma; example=/test/image.jpeg,/test/image2.jpeg
+    SERVER_CERT # path to server certificate
+    CLIENT_CERT # path to client certificate
+    CLIENT_KEY # path to client key
+    TRANSPOSE # set to True if you want to transpose input or completly unset env to not transpose
+    TIMEOUT # timeout used in grpc request; default value=10.0
+````
+
+## Start Locust
+
+A script was prepared to launch specific amount of locust instances, by passing integer to bash script. 
+Without passing any parameter script will launch Locust in distributed mode with 2 slaves.
+If you want to perform as many RPS as possible we recommend to spawn 1 slave for 1 CPU core.
+Also this script creates temporary file `pid_locust` which keep PID of processes launched by this script.
+
+```./start_locust.sh```
+
+Examples:
+```
+   ./start_locust.sh 1 # Launch Locust in distributed mode with 1 slave
+   ./start_locust.sh 0 # Launch Locust in single instance mode
+   ./start_locust.sh 4 # Launch Locust in distributed mode with 4 slaves
+```
+
+After using this command Locust will be available under [http://localhost:8089](http://localhost:8089)
+
+## Stop Locust
+
+We run all instances as background process, so to stop load tests you can use our script ```./stop_locust```
+, which also remove temporary file created earlier or read PIDs in `pid_locust` file and simply kill processes listed in that file.

examples/grpc_client/load_testing/image_locust.py

@@ -0,0 +1,72 @@
+import sys
+import os
+from locust import Locust, TaskSet, task, events
+import datetime
+import tensorflow.contrib.util as tf_contrib_util
+
+sys.path.append(os.path.realpath(os.path.join(os.path.realpath(__file__), '../../')))  # noqa
+from grpc_client_utils import INFERENCE_REQUEST
+from images_2_numpy import load_images_from_list
+from grpc_client import get_stub_and_request
+
+
+class MyTaskSet(TaskSet):
+    data_for_requests = []
+    GRPC_ADDRESS = os.environ.get('GRPC_ADDRESS', "URL")
+    MODEL_NAME = os.environ.get('MODEL_NAME', "resnet")
+    TENSOR_NAME = os.environ.get('TENSOR_NAME', "in")
+    IMAGES = os.environ.get('IMAGES', None)
+    SERVER_CERT = os.environ.get('SERVER_CERT', None)
+    CLIENT_CERT = os.environ.get('CLIENT_CERT', None)
+    CLIENT_KEY = os.environ.get('CLIENT_KEY', None)
+    TRANSPOSE = bool(os.environ.get('TRANSPOSE', False))
+    TIMEOUT = float(os.environ.get('TIMEOUT', 10))
+    stub = None
+    request = None
+    imgs = None
+
+    def on_start(self):
+        if self.GRPC_ADDRESS in 'URL':
+            print("Url to service is not set")
+            sys.exit(0)
+        if self.IMAGES is not None:
+            images = self.IMAGES.split(',')
+            self.imgs = load_images_from_list(images, 224, len(images))
+        if self.TRANSPOSE:
+            self.imgs = self.imgs.transpose((0, 3, 1, 2))
+        certs = dict()
+        certs['server_cert'] = self.SERVER_CERT
+        certs['client_cert'] = self.CLIENT_CERT
+        certs['client_key'] = self.CLIENT_KEY
+        self.stub, self.request = get_stub_and_request(self.GRPC_ADDRESS, self.MODEL_NAME, certs,
+                                                       True, None, INFERENCE_REQUEST)
+        self.request.inputs[self.TENSOR_NAME].CopyFrom(
+            tf_contrib_util.make_tensor_proto(self.imgs, shape=(self.imgs.shape)))
+
+    def on_stop(self):
+        print("Tasks was stopped")
+
+    @task
+    def my_task(self):
+        start_time = datetime.datetime.now()
+        try:
+            response = self.stub.Predict(self.request, self.TIMEOUT)
+        except Exception as e:
+            end_time = datetime.datetime.now()
+            duration = (end_time - start_time).total_seconds() * 1000
+            events.request_failure.fire(request_type="grpc", name='grpc', response_time=duration,
+                                        exception=e)
+        else:
+            end_time = datetime.datetime.now()
+            duration = (end_time - start_time).total_seconds() * 1000
+            events.request_success.fire(request_type="grpc", name='grpc', response_time=duration,
+                                        response_length=response.ByteSize())
+
+
+class MyLocust(Locust):
+    task_set = MyTaskSet
+    min_wait = 0
+    max_wait = 0
+
+    def teardown(self):
+        print("Locust ends his tasks")

examples/grpc_client/load_testing/requirements-locust.txt

@@ -0,0 +1 @@
+locustio==0.9.0

examples/grpc_client/load_testing/start_locust.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+SLAVES=${1:-2}
+
+if [ "$SLAVES" == "0" ]
+then
+  LOCUST_OPTS=""
+  echo "Launch locust single instance"
+else
+  LOCUST_OPTS="--master"
+  echo "Launch locust master"
+fi
+
+locust -f image_locust.py ${LOCUST_OPTS} &
+echo $! >> pid_locust
+for (( c=1; c<=$SLAVES; c++ ))
+do
+   echo "Launch locust slave $i"
+   locust -f image_locust.py --master-host=localhost --slave &
+   echo $! >> pid_locust
+done

examples/grpc_client/load_testing/stop_locust.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+while read p; do
+  kill -9 $p
+done <pid_locust
+echo "Now all locust instances have been killed"
+
+rm pid_locust