From 5874f1039bf961f47ccb70191397e5dcd166360e Mon Sep 17 00:00:00 2001
From: RunningLeon <mnsheng@yeah.net>
Date: Fri, 2 Sep 2022 10:55:53 +0800
Subject: [PATCH] Add CI to test full pipeline (#966)

* add mmcls full pipeline test ci

* update

* update

* add mmcv

* install torch

* install mmdeploy

* change clone with https

* install mmcls

* update

* change mmcls version

* add mmcv version

* update  mmcls version

* test sdk

* tast with imagnet

* sed pipeline

* print env

* update

* move to backend-ort ci

* install mim
---
 .github/scripts/test_mmcls_full_pipeline.sh | 69 +++++++++++++++++++++
 .github/workflows/backend-ort.yml           |  9 +++
 2 files changed, 78 insertions(+)
 create mode 100644 .github/scripts/test_mmcls_full_pipeline.sh

diff --git a/.github/scripts/test_mmcls_full_pipeline.sh b/.github/scripts/test_mmcls_full_pipeline.sh
new file mode 100644
index 0000000000..e378af8f72
--- /dev/null
+++ b/.github/scripts/test_mmcls_full_pipeline.sh
@@ -0,0 +1,69 @@
+#!/bin/sh
+
+set -e
+# print env
+python3 tools/check_env.py
+
+deploy_cfg=configs/mmcls/classification_onnxruntime_dynamic.py
+device=cpu
+model_cfg=../mmclassification/configs/resnet/resnet18_8xb32_in1k.py
+checkpoint=https://download.openmmlab.com/mmclassification/v0/resnet/resnet18_8xb32_in1k_20210831-fbbb1da6.pth
+sdk_cfg=-configs/mmcls/classification_sdk_dynamic.py
+input_img=../mmclassification/demo/demo.JPEG
+work_dir=work_dir
+
+echo "------------------------------------------------------------------------------------------------------------"
+echo "deploy_cfg=$deploy_cfg"
+echo "model_cfg=$model_cfg"
+echo "checkpoint=$checkpoint"
+echo "device=$device"
+echo "------------------------------------------------------------------------------------------------------------"
+
+mkdir -p $work_dir
+
+python3 tools/deploy.py \
+  $deploy_cfg \
+  $model_cfg \
+  $checkpoint \
+  $input_img \
+  --device $device \
+  --work-dir $work_dir \
+  --dump-info
+
+# prepare dataset
+wget -P data/ https://github.com/open-mmlab/mmdeploy/files/9401216/imagenet-val100.zip
+unzip data/imagenet-val100.zip -d data/
+
+echo "Running test with ort"
+python3 tools/test.py \
+  $deploy_cfg \
+  $model_cfg \
+  --model $work_dir/end2end.onnx \
+  --device $device \
+  --out $work_dir/ort_out.pkl \
+  --metrics accuracy \
+  --device $device \
+  --log2file $work_dir/test_ort.log \
+  --speed-test \
+  --log-interval 50 \
+  --warmup 20 \
+  --batch-size 32
+
+echo "Running test with sdk"
+
+# change topk for test
+sed -i 's/"topk": 5/"topk": 1000/g' work_dir/pipeline.json
+
+python3 tools/test.py \
+  $sdk_cfg \
+  $model_cfg \
+  --model $work_dir \
+  --device $device \
+  --out $work_dir/sdk_out.pkl \
+  --metrics accuracy \
+  --device $device \
+  --log2file $work_dir/test_sdk.log \
+  --speed-test \
+  --log-interval 50 \
+  --warmup 20 \
+  --batch-size 1
diff --git a/.github/workflows/backend-ort.yml b/.github/workflows/backend-ort.yml
index cc5c97767b..7546e66a59 100644
--- a/.github/workflows/backend-ort.yml
+++ b/.github/workflows/backend-ort.yml
@@ -37,3 +37,12 @@ jobs:
           python3 -m pip install torch==1.8.2 torchvision==0.9.2 --extra-index-url https://download.pytorch.org/whl/lts/1.8/cpu
           python3 -m pip install mmcv-full==1.5.1  -f https://download.openmmlab.com/mmcv/dist/cpu/torch1.8.0/index.html
           python3 -c 'import mmdeploy.apis.onnxruntime as ort_api; assert ort_api.is_available() and ort_api.is_custom_ops_available()'
+      - name: test mmcls full pipeline
+        run: |
+          pip isntall openmim
+          mim install mmcls
+          git clone  --depth 1 --single-branch --branch master https://github.com/open-mmlab/mmclassification.git ../mmclassification
+          export MMDEPLOY_DIR=$(pwd)
+          export ONNXRUNTIME_DIR=$MMDEPLOY_DIR/../mmdeploy-dep/onnxruntime-linux-x64-1.8.1
+          export LD_LIBRARY_PATH=$ONNXRUNTIME_DIR/lib:$MMDEPLOY_DIR/build/install/lib:$LD_LIBRARY_PATH
+          bash .github/scripts/test_mmcls_full_pipeline.sh