Skip to content

Commit

Permalink
feat: add screen capture in frontend
Browse files Browse the repository at this point in the history
  • Loading branch information
ylxmf2005 committed Jan 20, 2025
1 parent 726b461 commit fc5ef9f
Show file tree
Hide file tree
Showing 10 changed files with 378 additions and 63 deletions.
115 changes: 59 additions & 56 deletions src/renderer/src/App.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import TitleBar from './components/electron/title-bar';
import { Live2DModelProvider } from './context/live2d-model-context';
import { InputSubtitle } from './components/electron/input-subtitle';
import { ProactiveSpeakProvider } from './context/proactive-speak-context';
import { ScreenCaptureProvider } from './context/screen-capture-context';

function App(): JSX.Element {
const [showSidebar, setShowSidebar] = useState(true);
Expand Down Expand Up @@ -69,66 +70,68 @@ function App(): JSX.Element {
<ChakraProvider value={defaultSystem}>
<Live2DModelProvider>
<CameraProvider>
<ChatHistoryProvider>
<AiStateProvider>
<ProactiveSpeakProvider>
<CharacterConfigProvider>
<Live2DConfigProvider>
<SubtitleProvider>
<VADProvider>
<BgUrlProvider>
<WebSocketHandler>
<Toaster />
{mode === 'window' ? (
<>
{isElectron && <TitleBar />}
<Flex {...layoutStyles.appContainer}>
<Box
{...layoutStyles.sidebar}
{...(!showSidebar && { width: '24px' })}
>
<Sidebar
isCollapsed={!showSidebar}
onToggle={() => setShowSidebar(!showSidebar)}
/>
</Box>
<Box {...layoutStyles.mainContent}>
{/* <Box {...layoutStyles.canvas}> */}
<Canvas />
{/* <InputSubtitle isPet={false} /> */}
{/* </Box> */}
<ScreenCaptureProvider>
<ChatHistoryProvider>
<AiStateProvider>
<ProactiveSpeakProvider>
<CharacterConfigProvider>
<Live2DConfigProvider>
<SubtitleProvider>
<VADProvider>
<BgUrlProvider>
<WebSocketHandler>
<Toaster />
{mode === 'window' ? (
<>
{isElectron && <TitleBar />}
<Flex {...layoutStyles.appContainer}>
<Box
{...layoutStyles.footer}
{...(isFooterCollapsed
&& layoutStyles.collapsedFooter)}
{...layoutStyles.sidebar}
{...(!showSidebar && { width: '24px' })}
>
<Footer
isCollapsed={isFooterCollapsed}
onToggle={() => setIsFooterCollapsed(
!isFooterCollapsed,
)}
<Sidebar
isCollapsed={!showSidebar}
onToggle={() => setShowSidebar(!showSidebar)}
/>
</Box>
</Box>
</Flex>
</>
) : (
<>
<Live2D isPet={mode === 'pet'} />
{mode === 'pet' && (
<InputSubtitle isPet={mode === 'pet'} />
)}
</>
)}
</WebSocketHandler>
</BgUrlProvider>
</VADProvider>
</SubtitleProvider>
</Live2DConfigProvider>
</CharacterConfigProvider>
</ProactiveSpeakProvider>
</AiStateProvider>
</ChatHistoryProvider>
<Box {...layoutStyles.mainContent}>
{/* <Box {...layoutStyles.canvas}> */}
<Canvas />
{/* <InputSubtitle isPet={false} /> */}
{/* </Box> */}
<Box
{...layoutStyles.footer}
{...(isFooterCollapsed
&& layoutStyles.collapsedFooter)}
>
<Footer
isCollapsed={isFooterCollapsed}
onToggle={() => setIsFooterCollapsed(
!isFooterCollapsed,
)}
/>
</Box>
</Box>
</Flex>
</>
) : (
<>
<Live2D isPet={mode === 'pet'} />
{mode === 'pet' && (
<InputSubtitle isPet={mode === 'pet'} />
)}
</>
)}
</WebSocketHandler>
</BgUrlProvider>
</VADProvider>
</SubtitleProvider>
</Live2DConfigProvider>
</CharacterConfigProvider>
</ProactiveSpeakProvider>
</AiStateProvider>
</ChatHistoryProvider>
</ScreenCaptureProvider>
</CameraProvider>
</Live2DModelProvider>
</ChakraProvider>
Expand Down
37 changes: 37 additions & 0 deletions src/renderer/src/components/sidebar/bottom-tab.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
/* eslint-disable */
import { Tabs } from '@chakra-ui/react'
import { FiCamera, FiMonitor } from 'react-icons/fi'
import { sidebarStyles } from './sidebar-styles'
import CameraPanel from './camera-panel'
import ScreenPanel from './screen-panel'

function BottomTab(): JSX.Element {
return (
<Tabs.Root
defaultValue="camera"
variant="plain"
{...sidebarStyles.bottomTab.container}
>
<Tabs.List {...sidebarStyles.bottomTab.list}>
<Tabs.Trigger value="camera" {...sidebarStyles.bottomTab.trigger}>
<FiCamera />
Camera
</Tabs.Trigger>
<Tabs.Trigger value="screen" {...sidebarStyles.bottomTab.trigger}>
<FiMonitor />
Screen
</Tabs.Trigger>
</Tabs.List>

<Tabs.Content value="camera">
<CameraPanel />
</Tabs.Content>

<Tabs.Content value="screen">
<ScreenPanel />
</Tabs.Content>
</Tabs.Root>
);
}

export default BottomTab
1 change: 0 additions & 1 deletion src/renderer/src/components/sidebar/camera-panel.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,6 @@ function CameraPanel(): JSX.Element {
return (
<Box {...sidebarStyles.cameraPanel.container}>
<Box {...sidebarStyles.cameraPanel.header}>
<Text {...sidebarStyles.cameraPanel.title}>Camera</Text>
{isStreaming && <LiveIndicator />}
</Box>

Expand Down
113 changes: 113 additions & 0 deletions src/renderer/src/components/sidebar/screen-panel.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
/* eslint-disable */
import { Box, Text } from "@chakra-ui/react";
import { FiMonitor } from "react-icons/fi";
import { Tooltip } from "@/components/ui/tooltip";
import { sidebarStyles } from "./sidebar-styles";
import { useCaptureScreen } from "@/hooks/sidebar/use-capture-screen";

// Reusable components
function ScreenIndicator() {
return (
<Box color="red.500" display="flex" alignItems="center" gap={2}>
<Box
w="8px"
h="8px"
borderRadius="full"
bg="red.500"
animation="pulse 2s infinite"
/>
<Text fontSize="sm">Screen</Text>
</Box>
);
}

function ScreenPlaceholder() {
return (
<Box
position="absolute"
display="flex"
flexDirection="column"
alignItems="center"
gap={2}
>
<FiMonitor size={24} />
<Text color="whiteAlpha.600" fontSize="sm" textAlign="center">
Click to start screen capture
</Text>
</Box>
);
}

function VideoStream({
videoRef,
isStreaming,
}: {
videoRef: React.RefObject<HTMLVideoElement>;
isStreaming: boolean;
}) {
return (
<video
ref={videoRef}
autoPlay
playsInline
muted
style={sidebarStyles.screenPanel.video}
{...(isStreaming ? {} : { display: "none" })}
/>
);
}

function ScreenPanel(): JSX.Element {
const {
videoRef,
error,
isHovering,
isStreaming,
toggleCapture,
handleMouseEnter,
handleMouseLeave,
} = useCaptureScreen();

return (
<Box {...sidebarStyles.screenPanel.container}>
<Box {...sidebarStyles.screenPanel.header}>
{isStreaming && <ScreenIndicator />}
</Box>

<Tooltip
showArrow
content={
isStreaming
? "Click to stop screen capture"
: "Click to start screen capture"
}
open={isHovering && !error}
>
<Box
{...sidebarStyles.screenPanel.screenContainer}
onClick={toggleCapture}
onMouseEnter={handleMouseEnter}
onMouseLeave={handleMouseLeave}
cursor="pointer"
position="relative"
_hover={{
bg: "whiteAlpha.100",
}}
>
{error ? (
<Text color="red.300" fontSize="sm" textAlign="center">
{error}
</Text>
) : (
<>
<VideoStream videoRef={videoRef} isStreaming={isStreaming} />
{!isStreaming && <ScreenPlaceholder />}
</>
)}
</Box>
</Tooltip>
</Box>
);
}

export default ScreenPanel;
67 changes: 67 additions & 0 deletions src/renderer/src/components/sidebar/sidebar-styles.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -267,4 +267,71 @@ export const sidebarStyles = {
display: 'block',
} as const,
},

screenPanel: {
container: {
width: '97%',
overflow: 'hidden',
px: 4,
minH: '240px',
},
header: {
display: 'flex',
justifyContent: 'space-between',
alignItems: 'center',
mb: 4,
},
title: commonStyles.title,
screenContainer: {
...commonStyles.panel,
width: '100%',
height: '240px',
display: 'flex',
alignItems: 'center',
justifyContent: 'center',
overflow: 'hidden',
transition: 'all 0.2s',
},
video: {
width: '100%',
height: '100%',
objectFit: 'cover' as const,
borderRadius: '8px',
display: 'block',
} as const,
},

bottomTab: {
container: {
width: '100%',
px: 4,
},
tabs: {
width: '100%',
bg: 'whiteAlpha.50',
borderRadius: 'lg',
p: '1',
},
list: {
borderBottom: 'none',
gap: '2',
},
trigger: {
color: 'whiteAlpha.700',
display: 'flex',
alignItems: 'center',
gap: 2,
px: 3,
py: 2,
borderRadius: 'md',
_hover: {
color: 'white',
bg: 'whiteAlpha.50',
},
_selected: {
color: 'white',
bg: 'whiteAlpha.200',
},
},
},
};
5 changes: 3 additions & 2 deletions src/renderer/src/components/sidebar/sidebar.tsx
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
/* eslint-disable react/require-default-props */
import { Box, Button } from '@chakra-ui/react';
import {
FiSettings, FiClock, FiPlus, FiChevronLeft,
Expand All @@ -6,7 +7,7 @@ import { memo } from 'react';
import { sidebarStyles } from './sidebar-styles';
import SettingUI from './setting/setting-ui';
import ChatHistoryPanel from './chat-history-panel';
import CameraPanel from './camera-panel';
import BottomTab from './bottom-tab';
import HistoryDrawer from './history-drawer';
import { useSidebar } from '@/hooks/sidebar/use-sidebar';

Expand Down Expand Up @@ -68,7 +69,7 @@ const SidebarContent = memo(({ onSettingsOpen, onNewHistory }: HeaderButtonsProp
/>
</Box>
<ChatHistoryPanel />
<CameraPanel />
<BottomTab />
</Box>
));

Expand Down
3 changes: 0 additions & 3 deletions src/renderer/src/context/live2d-config-context.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -51,9 +51,6 @@ export interface ModelInfo {
/** Initial Y position shift */
initialYshift: number | string;

/** X-axis offset coefficient */
kXOffset?: number | string;

/** Idle motion group name */
idleMotionGroupName?: string;

Expand Down
Loading

0 comments on commit fc5ef9f

Please sign in to comment.