Fix CI tests (#2220)

* 🔧 setup CI

* 🐛 Fix path

* cd

* set working dir

* fix openpose tests

* Fix web_api tests

* Fix batch hijack tests

* change coverage dir

* 📝 Add doc

.github/workflows/tests.yml

@@ -10,28 +10,82 @@ jobs:
     steps:
       - name: Checkout Code
         uses: actions/checkout@v3
-        with: 
+        with:
           repository: 'AUTOMATIC1111/stable-diffusion-webui'
           path: 'stable-diffusion-webui'
-          ref: '5ab7f213bec2f816f9c5644becb32eb72c8ffb89'
-
+          ref: '4afaaf8a020c1df457bcf7250cb1c7f609699fa7'
       - name: Checkout Code
         uses: actions/checkout@v3
         with:
           repository: 'Mikubill/sd-webui-controlnet'
           path: 'stable-diffusion-webui/extensions/sd-webui-controlnet'
-
       - name: Set up Python 3.10
         uses: actions/setup-python@v4
         with:
           python-version: 3.10.6
           cache: pip
           cache-dependency-path: |
             **/requirements*txt
-            stable-diffusion-webui/requirements*txt
-            
-      - run: |
-          pip install torch torchvision
-          curl -Lo stable-diffusion-webui/extensions/sd-webui-controlnet/models/control_canny-fp16.safetensors https://huggingface.co/webui/ControlNet-modules-safetensors/resolve/main/control_canny-fp16.safetensors
-          cd stable-diffusion-webui && python launch.py --no-half --disable-opt-split-attention --use-cpu all --skip-torch-cuda-test --api --tests ./extensions/sd-webui-controlnet/tests
-          rm -fr stable-diffusion-webui/extensions/sd-webui-controlnet/models/control_canny-fp16.safetensors
+            launch.py
+      - name: Install test dependencies
+        run: |
+          pip install wait-for-it
+          pip install -r requirements-test.txt
+        working-directory: stable-diffusion-webui
+        env:
+          PIP_DISABLE_PIP_VERSION_CHECK: "1"
+          PIP_PROGRESS_BAR: "off"
+      - name: Setup environment
+        run: python launch.py --skip-torch-cuda-test --exit
+        working-directory: stable-diffusion-webui
+        env:
+          PIP_DISABLE_PIP_VERSION_CHECK: "1"
+          PIP_PROGRESS_BAR: "off"
+          TORCH_INDEX_URL: https://download.pytorch.org/whl/cpu
+          WEBUI_LAUNCH_LIVE_OUTPUT: "1"
+          PYTHONUNBUFFERED: "1"
+      - name: Download controlnet model for testing
+        run: |
+          curl -Lo extensions/sd-webui-controlnet/models/control_canny-fp16.safetensors https://huggingface.co/webui/ControlNet-modules-safetensors/resolve/main/control_canny-fp16.safetensors
+        working-directory: stable-diffusion-webui
+      - name: Start test server
+        run: >
+          python -m coverage run
+          --data-file=.coverage.server
+          launch.py
+          --skip-prepare-environment
+          --skip-torch-cuda-test
+          --test-server
+          --do-not-download-clip
+          --no-half
+          --disable-opt-split-attention
+          --use-cpu all
+          --api-server-stop
+          2>&1 | tee output.txt &
+        working-directory: stable-diffusion-webui
+      - name: Run tests
+        run: |
+          wait-for-it --service 127.0.0.1:7860 -t 600
+          python -m pytest -vv --junitxml=test/results.xml --cov ./extensions/sd-webui-controlnet --cov-report=xml --verify-base-url ./extensions/sd-webui-controlnet/tests
+        working-directory: stable-diffusion-webui
+      - name: Kill test server
+        if: always()
+        run: curl -vv -XPOST http://127.0.0.1:7860/sdapi/v1/server-stop && sleep 10
+      - name: Show coverage
+        run: |
+          python -m coverage combine .coverage*
+          python -m coverage report -i
+          python -m coverage html -i
+        working-directory: stable-diffusion-webui
+      - name: Upload main app output
+        uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: output
+          path: output.txt
+      - name: Upload coverage HTML
+        uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: htmlcov
+          path: htmlcov

tests/README.md

@@ -0,0 +1,41 @@
+# Tests
+There are 2 types of tests:
+- unittest: backend based tests that directly import A1111 shared modules
+- api test: test functionality through A1111 web API
+
+# Run tests locally
+Make sure the current working directory is A1111 root.
+
+## Install test dependencies
+`pip install -r requirements-test.txt`
+
+## Start test server
+```shell
+python -m coverage run
+          --data-file=.coverage.server
+          launch.py
+          --skip-prepare-environment
+          --skip-torch-cuda-test
+          --test-server
+          --do-not-download-clip
+          --no-half
+          --disable-opt-split-attention
+          --use-cpu all
+          --api-server-stop
+```
+
+## Run test
+```shell
+python -m pytest -vv --junitxml=test/results.xml --cov ./extensions/sd-webui-controlnet --cov-report=xml --verify-base-url ./extensions/sd-webui-controlnet/tests
+```
+
+## Check code coverage
+Text report
+```shell
+python -m coverage report -i
+```
+
+HTML report
+```shell
+python -m coverage html -i
+```

tests/annotator_tests/openpose_tests/openpose_e2e_test.py

@@ -1,6 +1,7 @@
 import unittest
 import cv2
 import numpy as np
+from pathlib import Path
 from typing import Dict
 
 
@@ -11,7 +12,7 @@
 from annotator.openpose import OpenposeDetector
 
 class TestOpenposeDetector(unittest.TestCase):
-    image_path = './tests/images'
+    image_path = str(Path(__file__).parent.parent.parent / 'images')
     def setUp(self) -> None:
         self.detector = OpenposeDetector()
         self.detector.load_model()

tests/cn_script/batch_hijack_test.py

@@ -42,6 +42,7 @@ def test_do_hijack__multiple_times__hijacks_once(self, process_images_inner_mock
 class TestGetControlNetBatchesWorks(unittest.TestCase):
     def setUp(self):
         self.p = unittest.mock.MagicMock()
+        assert scripts.scripts_txt2img is not None
         self.p.scripts = scripts.scripts_txt2img
         self.cn_script = controlnet.Script()
         self.p.scripts.alwayson_scripts = [self.cn_script]
@@ -193,6 +194,7 @@ class TestProcessImagesPatchWorks(unittest.TestCase):
     def setUp(self, on_script_unloaded_mock):
         self.on_script_unloaded_mock = on_script_unloaded_mock
         self.p = unittest.mock.MagicMock()
+        assert scripts.scripts_txt2img is not None
         self.p.scripts = scripts.scripts_txt2img
         self.cn_script = controlnet.Script()
         self.p.scripts.alwayson_scripts = [self.cn_script]

tests/utils.py

@@ -2,16 +2,28 @@
 import sys
 import cv2
 from base64 import b64encode
+from pathlib import Path
 
 import requests
 
 BASE_URL = "http://localhost:7860"
 
 
 def setup_test_env():
-    ext_root = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
-    if ext_root not in sys.path:
-        sys.path.append(ext_root)
+    os.environ['IGNORE_CMD_ARGS_ERRORS'] = 'True'
+
+    file_path = Path(__file__).resolve()
+    ext_root = file_path.parent.parent
+    a1111_root = ext_root.parent.parent
+
+    for p in (ext_root, a1111_root):
+        if p not in sys.path:
+            sys.path.append(str(p))
+
+    # Initialize shared opts.
+    from modules import initialize
+    initialize.imports()
+    initialize.initialize()
 
 
 def readImage(path):
@@ -21,16 +33,23 @@ def readImage(path):
     return b64img
 
 
-def get_model():
+def get_model(use_sd15: bool = True) -> str:
     r = requests.get(BASE_URL+"/controlnet/model_list")
     result = r.json()
-    if "model_list" in result:
-        result = result["model_list"]
-        for item in result:
-            print("Using model: ", item)
-            return item
-    return "None"
-
+    if "model_list" not in result:
+        return "None"
+
+    def is_sd15(model_name: str) -> bool:
+        return 'sd15' in model_name
+
+    candidates = [
+        model
+        for model in result["model_list"]
+        if (use_sd15 and is_sd15(model)) or (not use_sd15 and not is_sd15(model))
+    ]
+
+    return candidates[0] if candidates else "None"
+
 
 def get_modules():
     return requests.get(f"{BASE_URL}/controlnet/module_list").json()

tests/web_api/img2img_test.py

@@ -75,13 +75,6 @@ def setup_route(self, setup_args):
 
     def assert_status_ok(self):
         self.assertEqual(requests.post(self.url_img2img, json=self.simple_img2img).status_code, 200)
-        stderr = ""
-        with open('test/stderr.txt') as f:
-            stderr = f.read().lower()
-        with open('test/stderr.txt', 'w') as f:
-            # clear stderr file so we can easily parse the next test
-            f.write("")
-        self.assertFalse('error' in stderr, "Errors in stderr: \n" + stderr)
 
     def test_img2img_simple_performed(self):
         self.assert_status_ok()

tests/web_api/txt2img_test.py

@@ -68,13 +68,6 @@ def setup_controlnet_params(self, setup_args):
 
     def assert_status_ok(self, msg=None):
         self.assertEqual(requests.post(self.url_txt2img, json=self.simple_txt2img).status_code, 200, msg)
-        stderr = ""
-        with open('test/stderr.txt') as f:
-            stderr = f.read().lower()
-        with open('test/stderr.txt', 'w') as f:
-            # clear stderr file so that we can easily parse the next test
-            f.write("")
-        self.assertFalse('error' in stderr, "Errors in stderr: \n" + stderr)
 
     def test_txt2img_simple_performed(self):
         self.assert_status_ok()