Skip to content

Commit

Permalink
Add image generation to assistant (#181)
Browse files Browse the repository at this point in the history
* wip

* Add image character inference in novel builder

* wip emotions inference

* fix img gen

* Fix seed, fix inference for multiple emotions

* Add pending generations widget

* disable emotions gen if no default image

* Add cretion credits for character and emotions

* add poses to character generation, wip novel assistant with imggen

* wip emotion group generation

* Fix pose image inference, fix premium novel assistant

* add pricing constraint to novel assistant, remove parts of disclaimer

* Add spend approval modal

* Add item and background generation

* remove old generation prompts

* add background and item image generation to novel assistant

* minor fixes

* revert configs
  • Loading branch information
miku448 authored Feb 26, 2025
1 parent 2223f52 commit b80aed0
Show file tree
Hide file tree
Showing 38 changed files with 2,488 additions and 161 deletions.
79 changes: 79 additions & 0 deletions apps/novel-builder/src/components/CreditsWidget.scss
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
@import '../styles/variables';
.CreditsWidget {
position: relative; // Ensure the dropdown is positioned relative to the widget
display: flex;
justify-content: center;
align-items: center;
gap: 10px;
font-size: 0.9rem;
font-weight: normal;

&__credits {
display: flex;
align-items: center;
gap: 0.5rem;
}

&__pending {
cursor: pointer;
display: inline-flex;
}

.rotating-cog {
font-size: 1.2rem;
/* Rotating animation for the cog */
animation: rotation 2s linear infinite;
color: $text-2;
transition: color 0.2s ease-out;

&:hover {
color: #fff;
}
}

@keyframes rotation {
from {
transform: rotate(0deg);
}
to {
transform: rotate(359deg);
}
}

&__dropdown {
position: absolute;
top: 110%;
right: 0;
background-color: $background-1;
border: 1px solid $text-2;
width: 300px;
max-height: 200px;
overflow-y: auto;
box-shadow: 0 2px 10px rgba(0, 0, 0, 0.2);
z-index: 10;
padding: 0.5rem;
border-radius: 4px;
}

&__dropdown-header {
font-size: 0.9rem;
font-weight: normal;
padding: 0.2rem;
text-align: right;
}

&__dropdown-content {
padding: 0.2rem;
overflow: auto;
max-height: 150px;
}

&__inference {
background-color: $background-2;
padding: 4px;
margin-bottom: 4px;
&:last-child {
border-bottom: none;
}
}
}
109 changes: 109 additions & 0 deletions apps/novel-builder/src/components/CreditsWidget.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
import { useAppSelector, useAppDispatch } from '../state/store';
import { fetchPrices, fetchUser } from '../state/slices/userSlice';
import { useEffect, useState } from 'react';
import { PiCoinsLight } from 'react-icons/pi';
import { IoCog } from 'react-icons/io5';
import './CreditsWidget.scss';

interface PendingInference {
inferenceId: string;
inferenceType: 'character' | 'emotion' | 'background' | 'item';
prompt: string;
characterId?: string;
outfitId?: string;
emotionId?: string;
backgroundId?: string;
itemId?: string;
}

export default function CreditsWidget() {
const dispatch = useAppDispatch();
const user = useAppSelector((state) => state.user.user);
const pendingInferences = useAppSelector((state) => state.novel.pendingInferences);
const characters = useAppSelector((state) => state.novel.characters);
const inventory = useAppSelector((state) => state.novel.inventory);
const backgrounds = useAppSelector((state) => state.novel.backgrounds);

const [showDropdown, setShowDropdown] = useState(false);

useEffect(() => {
dispatch(fetchUser());
dispatch(fetchPrices());
}, [dispatch]);

useEffect(() => {
if (!pendingInferences || pendingInferences.length === 0) {
setShowDropdown(false);
}
}, [pendingInferences]);

const renderInference = (inference: PendingInference) => {
switch (inference.inferenceType) {
case 'character': {
const character = characters.find((c) => c.id === inference.characterId);
let outfitName = '';
if (character && character.card && character.card.data && character.card.data.extensions?.mikugg_v2?.outfits) {
const outfit = character.card.data.extensions.mikugg_v2.outfits.find((o) => o.id === inference.outfitId);
outfitName = outfit ? outfit.name : inference.outfitId || '';
}
return (
<div>
{character ? character.name : inference.characterId} - {outfitName} outfit
</div>
);
}
case 'emotion': {
const character = characters.find((c) => c.id === inference.characterId);
let outfitName = '';
if (character && character.card && character.card.data && character.card.data.extensions?.mikugg_v2?.outfits) {
const outfit = character.card.data.extensions.mikugg_v2.outfits.find((o) => o.id === inference.outfitId);
outfitName = outfit ? outfit.name : inference.outfitId || '';
}
return (
<div>
{character ? character.name : inference.characterId} - {outfitName}, {inference.emotionId}
</div>
);
}
case 'item': {
const item = inventory?.find((i) => i.id === inference.itemId);
return <div>{item ? item.name : inference.itemId} item</div>;
}
case 'background': {
const bg = backgrounds.find((b) => b.id === inference.backgroundId);
return <div>{bg ? bg.name : inference.backgroundId} background</div>;
}
default:
return <div>Unknown inference</div>;
}
};

return (
<div className="CreditsWidget">
<div className="CreditsWidget__credits">
<PiCoinsLight /> {user?.credits || 0} credits
</div>
{pendingInferences?.length ? (
<div
className="CreditsWidget__pending"
onClick={() => setShowDropdown((prev) => !prev)}
title="View pending inferences"
>
<IoCog className="rotating-cog" />
</div>
) : null}
{showDropdown && pendingInferences?.length && (
<div className="CreditsWidget__dropdown">
<div className="CreditsWidget__dropdown-header">Pending Generations</div>
<div className="CreditsWidget__dropdown-content scrollbar">
{pendingInferences.map((inference: PendingInference) => (
<div key={inference.inferenceId} className="CreditsWidget__inference">
{renderInference(inference)}
</div>
))}
</div>
</div>
)}
</div>
);
}
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
import { Button, Modal } from '@mikugg/ui-kit';
import { HiSparkles } from 'react-icons/hi';
import { BiSolidError } from 'react-icons/bi';
import { MdOutlineTimer, MdPhotoCamera } from 'react-icons/md';
import { MdOutlineTimer } from 'react-icons/md';
import './DisclaimerModal.scss';

interface DisclaimerModalProps {
Expand All @@ -13,17 +12,6 @@ export default function DisclaimerModal({ opened, onClose }: DisclaimerModalProp
return (
<Modal opened={opened} title="Assistant Disclaimer" shouldCloseOnOverlayClick={false} onCloseModal={onClose}>
<div className="disclaimer-modal__content">
<div className="disclaimer-modal__section disclaimer-modal__section--premium">
<HiSparkles className="disclaimer-modal__icon" />
<div className="disclaimer-modal__text-container">
<h3 className="disclaimer-modal__title">Premium Feature</h3>
<p className="disclaimer-modal__text">
This feature is exclusively available for premium members at the moment. Please keep in mind that the
assistant might NOT be available even for premium members in the future due to high costs.
</p>
</div>
</div>

<div className="disclaimer-modal__section disclaimer-modal__section--warning">
<BiSolidError className="disclaimer-modal__icon" />
<div className="disclaimer-modal__text-container">
Expand All @@ -47,16 +35,6 @@ export default function DisclaimerModal({ opened, onClose }: DisclaimerModalProp
</div>
</div>

<div className="disclaimer-modal__section disclaimer-modal__section--info">
<MdPhotoCamera className="disclaimer-modal__icon" />
<div className="disclaimer-modal__text-container">
<h3 className="disclaimer-modal__title">Text Only</h3>
<p className="disclaimer-modal__text">
The assistant can help with text content only. It cannot generate, modify, or manipulate images.
</p>
</div>
</div>

<div className="disclaimer-modal__button-container">
<Button onClick={onClose} theme="primary">
I understand
Expand Down
Original file line number Diff line number Diff line change
@@ -1,17 +1,17 @@
import ChatBot, { Button, Params, RcbToggleChatWindowEvent } from 'react-chatbotify';
import { ChatCompletion, ChatCompletionMessageParam } from 'openai/src/resources/index.js';
import axios from 'axios';
import { useState, useEffect } from 'react';
import DisclaimerModal from './DisclaimerModal';
import config from '../../config';

import './NovelAssistant.scss';
import { FunctionAction, FunctionRegistry } from './prompt/FunctionDefinitions';
import { FunctionRegistry } from './prompt/FunctionDefinitions';
import { NovelManager } from './prompt/NovelSpec';
import { NovelV3 } from '@mikugg/bot-utils';
import { useAppDispatch, useAppSelector } from '../../state/store';
import { store, useAppDispatch } from '../../state/store';
import { loadCompleteState } from '../../state/slices/novelFormSlice';
import { SERVICES_ENDPOINT } from '../../libs/utils';
import { callChatCompletion, FunctionAction } from '../../libs/assistantCall';
import '../../libs/sdPromptImprover';

function getFunctionActionColor(action: FunctionAction): string {
switch (action) {
Expand Down Expand Up @@ -54,39 +54,6 @@ const functions = functionRegistry.getFunctionDefinitions();
// Load existing history if it exists
const conversationHistory: ChatCompletionMessageParam[] = [];

// Load

const callChatCompletion = async (
messages: ChatCompletionMessageParam[],
// eslint-disable-next-line @typescript-eslint/no-explicit-any
tools: any[],
parallel_tool_calls: boolean,
tool_choice: 'none' | 'auto',
): Promise<ChatCompletion> => {
const response = await axios.post(
SERVICES_ENDPOINT + '/assistant',
{
messages,
tools,
parallel_tool_calls,
tool_choice,
},
{
method: 'POST',
withCredentials: true,
headers: {
'Content-Type': 'application/json',
},
},
);

if (response.status !== 200) {
throw new Error('Failed to get completion from proxy');
}

return response.data;
};

const call_openai = async (params: Params, replaceState: (state: NovelV3.NovelState) => void) => {
try {
conversationHistory.push({ role: 'user', content: params.userInput });
Expand All @@ -105,6 +72,7 @@ const call_openai = async (params: Params, replaceState: (state: NovelV3.NovelSt
for (response = await askResponse(); response?.choices[0].message?.tool_calls; response = await askResponse()) {
const message = response.choices[0].message;
conversationHistory.push(message);
novelManager.replaceState(store.getState().novel);
if (message?.tool_calls) {
for (const toolCall of message.tool_calls) {
const fnName = toolCall.function.name;
Expand Down Expand Up @@ -150,7 +118,6 @@ const call_openai = async (params: Params, replaceState: (state: NovelV3.NovelSt

export default function NovelAssistant() {
const dispatch = useAppDispatch();
const state = useAppSelector((state) => state.novel);
const [showDisclaimer, setShowDisclaimer] = useState(false);
const [hasAcceptedDisclaimer, setHasAcceptedDisclaimer] = useState(() => false);
const [isPremium, setIsPremium] = useState(false);
Expand All @@ -172,10 +139,6 @@ export default function NovelAssistant() {
checkPremiumStatus();
}, []);

useEffect(() => {
novelManager.replaceState(state);
}, [state.title]);

useEffect(() => {
const handleToggleChatWindow = (event: RcbToggleChatWindowEvent) => {
const shouldOpen = event.data.newState;
Expand Down
Loading

0 comments on commit b80aed0

Please sign in to comment.