Public Backend API

.github/workflows/package-release.yml

@@ -39,6 +39,8 @@ jobs:
         uses: actions/setup-python@v4
         with:
           python-version: '3.10'
+          cache: 'pip'
+          cache-dependency-path: '**/win-linux-cuda.txt'
       - name: Install dependencies into target
         shell: bash
         run: 'python -m pip install -r requirements/win-linux-cuda.txt --no-cache-dir --target .python_dependencies'
@@ -65,6 +67,8 @@ jobs:
         uses: actions/setup-python@v4
         with:
           python-version: '3.10'
+          cache: 'pip'
+          cache-dependency-path: '**/win-dml.txt'
       - name: Install dependencies into target
         shell: bash
         run: 'python -m pip install -r requirements/win-dml.txt --no-cache-dir --target .python_dependencies'
@@ -79,58 +83,4 @@ jobs:
         uses: actions/upload-artifact@v3
         with:
           name: dream_textures-windows-directml
-          path: dream_textures-windows-directml.zip
-  windows-dreamstudio:
-    runs-on: windows-latest
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v3
-        with:
-          path: dream_textures
-      - name: Setup Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.10'
-      - name: Install dependencies into target
-        shell: bash
-        run: 'python -m pip install -r requirements/dreamstudio.txt --no-cache-dir --target .python_dependencies'
-        working-directory: dream_textures
-      - name: Archive Release
-        uses: thedoctor0/zip-release@main
-        with:
-          type: zip
-          filename: dream_textures-windows-dreamstudio.zip
-          exclusions: '*.git*'
-      - name: Archive and upload artifact
-        uses: actions/upload-artifact@v3
-        with:
-          name: dream_textures-windows-dreamstudio
-          path: dream_textures-windows-dreamstudio.zip
-  # No universal wheels for grpcio means we need separate macOS installations for Intel/ARM.
-  # GitHub Actions does not have Apple Silicon runners, so that build still needs to be created manually.
-  macos-intel-dreamstudio:
-    runs-on: macos-latest
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v3
-        with:
-          path: dream_textures
-      - name: Setup Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.10'
-      - name: Install dependencies into target
-        shell: bash
-        run: 'python -m pip install -r requirements/dreamstudio.txt --no-cache-dir --target .python_dependencies'
-        working-directory: dream_textures
-      - name: Archive Release
-        uses: thedoctor0/zip-release@main
-        with:
-          type: zip
-          filename: dream_textures-macos-intel-dreamstudio.zip
-          exclusions: '*.git*'
-      - name: Archive and upload artifact
-        uses: actions/upload-artifact@v3
-        with:
-          name: dream_textures-macos-intel-dreamstudio
-          path: dream_textures-macos-intel-dreamstudio.zip
+          path: dream_textures-windows-directml.zip

__init__.py

@@ -49,6 +49,8 @@ def clear_modules():
 
     from . import engine
 
+    from .diffusers_backend import DiffusersBackend
+
     requirements_path_items = (
         ('requirements/win-linux-cuda.txt', 'Linux/Windows (CUDA)', 'Linux or Windows with NVIDIA GPU'),
         ('requirements/mac-mps-cpu.txt', 'Apple Silicon', 'Apple M1/M2'),
@@ -127,6 +129,9 @@ def project_use_controlnet(self, context):
         register_render_pass()
 
         register_default_presets()
+
+        # Register the default backend.
+        bpy.utils.register_class(DiffusersBackend)
 
     def unregister():
         for cls in PREFERENCE_CLASSES:
@@ -143,4 +148,7 @@ def unregister():
 
         unregister_render_pass()
 
+        # Unregister the default backend
+        bpy.utils.unregister_class(DiffusersBackend)
+
         kill_generator()

api/__init__.py

@@ -0,0 +1,2 @@
+from .models import *
+from .backend import *

api/backend/__init__.py

@@ -0,0 +1 @@
+from .backend import *

api/backend/backend.py

@@ -0,0 +1,143 @@
+try:
+    import bpy
+    from typing import Callable, List, Tuple
+    from ..models.generation_arguments import GenerationArguments
+    from ..models.generation_result import GenerationResult
+    from ..models.model import Model
+
+    StepCallback = Callable[[List[GenerationResult]], bool]
+    Callback = Callable[[List[GenerationResult] | Exception], None]
+
+    class Backend(bpy.types.PropertyGroup):
+        """A backend for Dream Textures.
+
+        Provide the following methods to create a valid backend.
+
+        ```python
+        def list_models(self) -> List[Model]
+        def generate(
+            self,
+            arguments: GenerationArguments,
+
+            step_callback: StepCallback,
+            callback: Callback
+        )
+        ```
+        """
+
+        @classmethod
+        def register(cls):
+            from ...property_groups.dream_prompt import DreamPrompt
+            setattr(DreamPrompt, cls._attribute(), bpy.props.PointerProperty(type=cls))
+
+        @classmethod
+        def unregister(cls):
+            from ...property_groups.dream_prompt import DreamPrompt
+            delattr(DreamPrompt, cls._attribute())
+
+        @classmethod
+        def _id(cls) -> str:
+            return f"{cls.__module__}.{cls.__name__}"
+
+        @classmethod
+        def _attribute(cls) -> str:
+            return cls._id().replace('.', '_')
+
+        @classmethod
+        def _lookup(cls, id):
+            return next(
+                (backend for backend in cls._list_backends() if backend._id() == id),
+                next(iter(cls._list_backends()), None)
+            )
+
+        @classmethod
+        def _list_backends(cls):
+            return cls.__subclasses__()
+
+        def list_models(self, context) -> List[Model]:
+            """Provide a list of available models.
+
+            The `id` of the model will be provided.
+            """
+            ...
+
+        def list_controlnet_models(self, context) -> List[Model]:
+            """Provide a list of available ControlNet models.
+
+            The `id` of the model will be provided.
+            """
+            return []
+
+        def list_schedulers(self, context) -> List[str]:
+            """Provide a list of available schedulers."""
+            ...
+
+        def draw_prompt(self, layout, context):
+            """Draw additional UI in the 'Prompt' panel"""
+            ...
+
+        def draw_advanced(self, layout, context):
+            """Draw additional UI in the 'Advanced' panel"""
+            ...
+
+        def draw_speed_optimizations(self, layout, context):
+            """Draw additional UI in the 'Speed Optimizations' panel"""
+            ...
+
+        def draw_memory_optimizations(self, layout, context):
+            """Draw additional UI in the 'Memory Optimizations' panel"""
+            ...
+
+        def draw_extra(self, layout, context):
+            """Draw additional UI in the panel"""
+            ...
+
+        def get_batch_size(self, context) -> int:
+            """Return the selected batch size for the backend (if applicable).
+
+            A default implementation is provided that returns `1`.
+            """
+            return 1
+
+        def generate(
+            self,
+            arguments: GenerationArguments,
+            step_callback: StepCallback,
+            callback: Callback
+        ):
+            """
+            A request to generate an image.
+
+            If the `step_callback` returns `False`, the generation should be cancelled.
+            After cancelling, `callback` should be called with an `InterruptedError`.
+            """
+            ...
+
+        def validate(
+            self,
+            arguments: GenerationArguments
+        ):
+            """Validates the given arguments in the UI without generating.
+
+            This validation should occur as quickly as possible.
+
+            To report problems with the inputs, raise a `ValueError`.
+            Use the `FixItError` to provide a solution to the problem as well.
+
+            ```python
+            if arguments.steps % 2 == 0:
+                throw FixItError(
+                    "The number of steps is even",
+                    solution=FixItError.UpdateGenerationArgumentsSolution(
+                        title="Add 1 more step",
+                        arguments=dataclasses.replace(
+                            arguments,
+                            steps=arguments.steps + 1
+                        )
+                    )
+                )
+            ```
+            """
+            ...
+except:
+    pass

api/models/__init__.py

@@ -0,0 +1,7 @@
+from .generation_result import *
+from .model import *
+from .prompt import *
+from .seamless_axes import *
+from .step_preview_mode import *
+from .task import *
+from .fix_it_error import *

api/models/control_net.py

@@ -0,0 +1,14 @@
+from dataclasses import dataclass
+from typing import Tuple, List
+from numpy.typing import NDArray
+
+@dataclass
+class ControlNet:
+    model: str
+    """The selected ControlNet model used for generation"""
+
+    image: NDArray
+    """The control image"""
+
+    strength: float
+    """The strength of the ControlNet's influence"""

api/models/fix_it_error.py

@@ -0,0 +1,41 @@
+from typing import Callable, Any
+from .generation_arguments import GenerationArguments
+from dataclasses import dataclass
+
+class FixItError(Exception):
+    """An exception with a solution.
+
+    Call the `draw` method to render the UI elements responsible for resolving this error.
+    """
+    def __init__(self, message, solution: 'Solution'):
+        super().__init__(message)
+
+        self._solution = solution
+
+    def _draw(self, dream_prompt, context, layout):
+        self._solution._draw(dream_prompt, context, layout)
+
+    @dataclass
+    class Solution:
+        def _draw(self, dream_prompt, context, layout):
+            ...
+
+    @dataclass
+    class ChangeProperty(Solution):
+        """Prompts the user to change the given `property` of the `GenerationArguments`."""
+        property: str
+
+        def _draw(self, dream_prompt, context, layout):
+            layout.prop(dream_prompt, self.property)
+
+    @dataclass
+    class RunOperator(Solution):
+        """Runs the given operator"""
+        title: str
+        operator: str
+        modify_operator: Callable[[Any], None]
+
+        def _draw(self, dream_prompt, context, layout):
+            self.modify_operator(
+                layout.operator(self.operator, text=self.title)
+            )