From fc5ef9fcd5125917c6cf6ed95648e2deb8bce1f3 Mon Sep 17 00:00:00 2001
From: ylxmf <ethanelift@gmail.com>
Date: Mon, 20 Jan 2025 16:47:53 +0800
Subject: [PATCH] feat: add screen capture in frontend

---
 src/renderer/src/App.tsx                      | 115 +++++++++---------
 .../src/components/sidebar/bottom-tab.tsx     |  37 ++++++
 .../src/components/sidebar/camera-panel.tsx   |   1 -
 .../src/components/sidebar/screen-panel.tsx   | 113 +++++++++++++++++
 .../src/components/sidebar/sidebar-styles.tsx |  67 ++++++++++
 .../src/components/sidebar/sidebar.tsx        |   5 +-
 .../src/context/live2d-config-context.tsx     |   3 -
 .../src/context/screen-capture-context.tsx    |  63 ++++++++++
 .../sidebar/setting/use-live2d-settings.ts    |   1 -
 .../src/hooks/sidebar/use-capture-screen.ts   |  36 ++++++
 10 files changed, 378 insertions(+), 63 deletions(-)
 create mode 100644 src/renderer/src/components/sidebar/bottom-tab.tsx
 create mode 100644 src/renderer/src/components/sidebar/screen-panel.tsx
 create mode 100644 src/renderer/src/context/screen-capture-context.tsx
 create mode 100644 src/renderer/src/hooks/sidebar/use-capture-screen.ts

diff --git a/src/renderer/src/App.tsx b/src/renderer/src/App.tsx
index d083769..9747528 100644
--- a/src/renderer/src/App.tsx
+++ b/src/renderer/src/App.tsx
@@ -22,6 +22,7 @@ import TitleBar from './components/electron/title-bar';
 import { Live2DModelProvider } from './context/live2d-model-context';
 import { InputSubtitle } from './components/electron/input-subtitle';
 import { ProactiveSpeakProvider } from './context/proactive-speak-context';
+import { ScreenCaptureProvider } from './context/screen-capture-context';
 
 function App(): JSX.Element {
   const [showSidebar, setShowSidebar] = useState(true);
@@ -69,66 +70,68 @@ function App(): JSX.Element {
     <ChakraProvider value={defaultSystem}>
       <Live2DModelProvider>
         <CameraProvider>
-          <ChatHistoryProvider>
-            <AiStateProvider>
-              <ProactiveSpeakProvider>
-                <CharacterConfigProvider>
-                  <Live2DConfigProvider>
-                    <SubtitleProvider>
-                      <VADProvider>
-                        <BgUrlProvider>
-                          <WebSocketHandler>
-                            <Toaster />
-                            {mode === 'window' ? (
-                              <>
-                                {isElectron && <TitleBar />}
-                                <Flex {...layoutStyles.appContainer}>
-                                  <Box
-                                    {...layoutStyles.sidebar}
-                                    {...(!showSidebar && { width: '24px' })}
-                                  >
-                                    <Sidebar
-                                      isCollapsed={!showSidebar}
-                                      onToggle={() => setShowSidebar(!showSidebar)}
-                                    />
-                                  </Box>
-                                  <Box {...layoutStyles.mainContent}>
-                                    {/* <Box {...layoutStyles.canvas}> */}
-                                    <Canvas />
-                                    {/* <InputSubtitle isPet={false} /> */}
-                                    {/* </Box> */}
+          <ScreenCaptureProvider>
+            <ChatHistoryProvider>
+              <AiStateProvider>
+                <ProactiveSpeakProvider>
+                  <CharacterConfigProvider>
+                    <Live2DConfigProvider>
+                      <SubtitleProvider>
+                        <VADProvider>
+                          <BgUrlProvider>
+                            <WebSocketHandler>
+                              <Toaster />
+                              {mode === 'window' ? (
+                                <>
+                                  {isElectron && <TitleBar />}
+                                  <Flex {...layoutStyles.appContainer}>
                                     <Box
-                                      {...layoutStyles.footer}
-                                      {...(isFooterCollapsed
-                                        && layoutStyles.collapsedFooter)}
+                                      {...layoutStyles.sidebar}
+                                      {...(!showSidebar && { width: '24px' })}
                                     >
-                                      <Footer
-                                        isCollapsed={isFooterCollapsed}
-                                        onToggle={() => setIsFooterCollapsed(
-                                          !isFooterCollapsed,
-                                        )}
+                                      <Sidebar
+                                        isCollapsed={!showSidebar}
+                                        onToggle={() => setShowSidebar(!showSidebar)}
                                       />
                                     </Box>
-                                  </Box>
-                                </Flex>
-                              </>
-                            ) : (
-                              <>
-                                <Live2D isPet={mode === 'pet'} />
-                                {mode === 'pet' && (
-                                  <InputSubtitle isPet={mode === 'pet'} />
-                                )}
-                              </>
-                            )}
-                          </WebSocketHandler>
-                        </BgUrlProvider>
-                      </VADProvider>
-                    </SubtitleProvider>
-                  </Live2DConfigProvider>
-                </CharacterConfigProvider>
-              </ProactiveSpeakProvider>
-            </AiStateProvider>
-          </ChatHistoryProvider>
+                                    <Box {...layoutStyles.mainContent}>
+                                      {/* <Box {...layoutStyles.canvas}> */}
+                                      <Canvas />
+                                      {/* <InputSubtitle isPet={false} /> */}
+                                      {/* </Box> */}
+                                      <Box
+                                        {...layoutStyles.footer}
+                                        {...(isFooterCollapsed
+                                          && layoutStyles.collapsedFooter)}
+                                      >
+                                        <Footer
+                                          isCollapsed={isFooterCollapsed}
+                                          onToggle={() => setIsFooterCollapsed(
+                                            !isFooterCollapsed,
+                                          )}
+                                        />
+                                      </Box>
+                                    </Box>
+                                  </Flex>
+                                </>
+                              ) : (
+                                <>
+                                  <Live2D isPet={mode === 'pet'} />
+                                  {mode === 'pet' && (
+                                    <InputSubtitle isPet={mode === 'pet'} />
+                                  )}
+                                </>
+                              )}
+                            </WebSocketHandler>
+                          </BgUrlProvider>
+                        </VADProvider>
+                      </SubtitleProvider>
+                    </Live2DConfigProvider>
+                  </CharacterConfigProvider>
+                </ProactiveSpeakProvider>
+              </AiStateProvider>
+            </ChatHistoryProvider>
+          </ScreenCaptureProvider>
         </CameraProvider>
       </Live2DModelProvider>
     </ChakraProvider>
diff --git a/src/renderer/src/components/sidebar/bottom-tab.tsx b/src/renderer/src/components/sidebar/bottom-tab.tsx
new file mode 100644
index 0000000..c44b13e
--- /dev/null
+++ b/src/renderer/src/components/sidebar/bottom-tab.tsx
@@ -0,0 +1,37 @@
+/* eslint-disable */
+import { Tabs } from '@chakra-ui/react'
+import { FiCamera, FiMonitor } from 'react-icons/fi'
+import { sidebarStyles } from './sidebar-styles'
+import CameraPanel from './camera-panel'
+import ScreenPanel from './screen-panel'
+
+function BottomTab(): JSX.Element {
+  return (
+    <Tabs.Root 
+      defaultValue="camera" 
+      variant="plain"
+      {...sidebarStyles.bottomTab.container}
+    >
+      <Tabs.List {...sidebarStyles.bottomTab.list}>
+        <Tabs.Trigger value="camera" {...sidebarStyles.bottomTab.trigger}>
+          <FiCamera />
+          Camera
+        </Tabs.Trigger>
+        <Tabs.Trigger value="screen" {...sidebarStyles.bottomTab.trigger}>
+          <FiMonitor />
+          Screen
+        </Tabs.Trigger>
+      </Tabs.List>
+
+      <Tabs.Content value="camera">
+        <CameraPanel />
+      </Tabs.Content>
+      
+      <Tabs.Content value="screen">
+        <ScreenPanel />
+      </Tabs.Content>
+    </Tabs.Root>
+  );
+}
+
+export default BottomTab
diff --git a/src/renderer/src/components/sidebar/camera-panel.tsx b/src/renderer/src/components/sidebar/camera-panel.tsx
index a7f312e..8762a52 100644
--- a/src/renderer/src/components/sidebar/camera-panel.tsx
+++ b/src/renderer/src/components/sidebar/camera-panel.tsx
@@ -73,7 +73,6 @@ function CameraPanel(): JSX.Element {
   return (
     <Box {...sidebarStyles.cameraPanel.container}>
       <Box {...sidebarStyles.cameraPanel.header}>
-        <Text {...sidebarStyles.cameraPanel.title}>Camera</Text>
         {isStreaming && <LiveIndicator />}
       </Box>
 
diff --git a/src/renderer/src/components/sidebar/screen-panel.tsx b/src/renderer/src/components/sidebar/screen-panel.tsx
new file mode 100644
index 0000000..d0a74b6
--- /dev/null
+++ b/src/renderer/src/components/sidebar/screen-panel.tsx
@@ -0,0 +1,113 @@
+/* eslint-disable */
+import { Box, Text } from "@chakra-ui/react";
+import { FiMonitor } from "react-icons/fi";
+import { Tooltip } from "@/components/ui/tooltip";
+import { sidebarStyles } from "./sidebar-styles";
+import { useCaptureScreen } from "@/hooks/sidebar/use-capture-screen";
+
+// Reusable components
+function ScreenIndicator() {
+  return (
+    <Box color="red.500" display="flex" alignItems="center" gap={2}>
+      <Box
+        w="8px"
+        h="8px"
+        borderRadius="full"
+        bg="red.500"
+        animation="pulse 2s infinite"
+      />
+      <Text fontSize="sm">Screen</Text>
+    </Box>
+  );
+}
+
+function ScreenPlaceholder() {
+  return (
+    <Box
+      position="absolute"
+      display="flex"
+      flexDirection="column"
+      alignItems="center"
+      gap={2}
+    >
+      <FiMonitor size={24} />
+      <Text color="whiteAlpha.600" fontSize="sm" textAlign="center">
+        Click to start screen capture
+      </Text>
+    </Box>
+  );
+}
+
+function VideoStream({
+  videoRef,
+  isStreaming,
+}: {
+  videoRef: React.RefObject<HTMLVideoElement>;
+  isStreaming: boolean;
+}) {
+  return (
+    <video
+      ref={videoRef}
+      autoPlay
+      playsInline
+      muted
+      style={sidebarStyles.screenPanel.video}
+      {...(isStreaming ? {} : { display: "none" })}
+    />
+  );
+}
+
+function ScreenPanel(): JSX.Element {
+  const {
+    videoRef,
+    error,
+    isHovering,
+    isStreaming,
+    toggleCapture,
+    handleMouseEnter,
+    handleMouseLeave,
+  } = useCaptureScreen();
+
+  return (
+    <Box {...sidebarStyles.screenPanel.container}>
+      <Box {...sidebarStyles.screenPanel.header}>
+        {isStreaming && <ScreenIndicator />}
+      </Box>
+
+      <Tooltip
+        showArrow
+        content={
+          isStreaming
+            ? "Click to stop screen capture"
+            : "Click to start screen capture"
+        }
+        open={isHovering && !error}
+      >
+        <Box
+          {...sidebarStyles.screenPanel.screenContainer}
+          onClick={toggleCapture}
+          onMouseEnter={handleMouseEnter}
+          onMouseLeave={handleMouseLeave}
+          cursor="pointer"
+          position="relative"
+          _hover={{
+            bg: "whiteAlpha.100",
+          }}
+        >
+          {error ? (
+            <Text color="red.300" fontSize="sm" textAlign="center">
+              {error}
+            </Text>
+          ) : (
+            <>
+              <VideoStream videoRef={videoRef} isStreaming={isStreaming} />
+              {!isStreaming && <ScreenPlaceholder />}
+            </>
+          )}
+        </Box>
+      </Tooltip>
+    </Box>
+  );
+}
+
+export default ScreenPanel;
diff --git a/src/renderer/src/components/sidebar/sidebar-styles.tsx b/src/renderer/src/components/sidebar/sidebar-styles.tsx
index 78586f3..89d2631 100644
--- a/src/renderer/src/components/sidebar/sidebar-styles.tsx
+++ b/src/renderer/src/components/sidebar/sidebar-styles.tsx
@@ -267,4 +267,71 @@ export const sidebarStyles = {
       display: 'block',
     } as const,
   },
+
+  screenPanel: {
+    container: {
+      width: '97%',
+      overflow: 'hidden',
+      px: 4,
+      minH: '240px',
+    },
+    header: {
+      display: 'flex',
+      justifyContent: 'space-between',
+      alignItems: 'center',
+      mb: 4,
+    },
+    title: commonStyles.title,
+    screenContainer: {
+      ...commonStyles.panel,
+      width: '100%',
+      height: '240px',
+      display: 'flex',
+      alignItems: 'center',
+      justifyContent: 'center',
+      overflow: 'hidden',
+      transition: 'all 0.2s',
+    },
+    video: {
+      width: '100%',
+      height: '100%',
+      objectFit: 'cover' as const,
+      borderRadius: '8px',
+      display: 'block',
+    } as const,
+  },
+
+  bottomTab: {
+    container: {
+      width: '100%',
+      px: 4,
+    },
+    tabs: {
+      width: '100%',
+      bg: 'whiteAlpha.50',
+      borderRadius: 'lg',
+      p: '1',
+    },
+    list: {
+      borderBottom: 'none',
+      gap: '2',
+    },
+    trigger: {
+      color: 'whiteAlpha.700',
+      display: 'flex',
+      alignItems: 'center',
+      gap: 2,
+      px: 3,
+      py: 2,
+      borderRadius: 'md',
+      _hover: {
+        color: 'white',
+        bg: 'whiteAlpha.50',
+      },
+      _selected: {
+        color: 'white',
+        bg: 'whiteAlpha.200',
+      },
+    },
+  },
 };
diff --git a/src/renderer/src/components/sidebar/sidebar.tsx b/src/renderer/src/components/sidebar/sidebar.tsx
index 4afabfc..5353365 100644
--- a/src/renderer/src/components/sidebar/sidebar.tsx
+++ b/src/renderer/src/components/sidebar/sidebar.tsx
@@ -1,3 +1,4 @@
+/* eslint-disable react/require-default-props */
 import { Box, Button } from '@chakra-ui/react';
 import {
   FiSettings, FiClock, FiPlus, FiChevronLeft,
@@ -6,7 +7,7 @@ import { memo } from 'react';
 import { sidebarStyles } from './sidebar-styles';
 import SettingUI from './setting/setting-ui';
 import ChatHistoryPanel from './chat-history-panel';
-import CameraPanel from './camera-panel';
+import BottomTab from './bottom-tab';
 import HistoryDrawer from './history-drawer';
 import { useSidebar } from '@/hooks/sidebar/use-sidebar';
 
@@ -68,7 +69,7 @@ const SidebarContent = memo(({ onSettingsOpen, onNewHistory }: HeaderButtonsProp
       />
     </Box>
     <ChatHistoryPanel />
-    <CameraPanel />
+    <BottomTab />
   </Box>
 ));
 
diff --git a/src/renderer/src/context/live2d-config-context.tsx b/src/renderer/src/context/live2d-config-context.tsx
index d498852..49cc2a4 100644
--- a/src/renderer/src/context/live2d-config-context.tsx
+++ b/src/renderer/src/context/live2d-config-context.tsx
@@ -51,9 +51,6 @@ export interface ModelInfo {
   /** Initial Y position shift */
   initialYshift: number | string;
 
-  /** X-axis offset coefficient */
-  kXOffset?: number | string;
-
   /** Idle motion group name */
   idleMotionGroupName?: string;
 
diff --git a/src/renderer/src/context/screen-capture-context.tsx b/src/renderer/src/context/screen-capture-context.tsx
new file mode 100644
index 0000000..fdebcf8
--- /dev/null
+++ b/src/renderer/src/context/screen-capture-context.tsx
@@ -0,0 +1,63 @@
+import { createContext, useContext, useState, ReactNode } from 'react';
+
+interface ScreenCaptureContextType {
+  stream: MediaStream | null;
+  isStreaming: boolean;
+  error: string;
+  startCapture: () => Promise<void>;
+  stopCapture: () => void;
+}
+
+const ScreenCaptureContext = createContext<ScreenCaptureContextType | undefined>(undefined);
+
+export function ScreenCaptureProvider({ children }: { children: ReactNode }) {
+  const [stream, setStream] = useState<MediaStream | null>(null);
+  const [isStreaming, setIsStreaming] = useState(false);
+  const [error, setError] = useState('');
+
+  const startCapture = async () => {
+    try {
+      const displayMediaOptions: DisplayMediaStreamOptions = {
+        video: true,
+        audio: false,
+      };
+      const mediaStream = await navigator.mediaDevices.getDisplayMedia(displayMediaOptions);
+      setStream(mediaStream);
+      setIsStreaming(true);
+      setError('');
+    } catch (err) {
+      setError('Failed to start screen capture');
+      console.error(err);
+    }
+  };
+
+  const stopCapture = () => {
+    if (stream) {
+      stream.getTracks().forEach((track) => track.stop());
+      setStream(null);
+      setIsStreaming(false);
+    }
+  };
+
+  return (
+    <ScreenCaptureContext.Provider
+      value={{
+        stream,
+        isStreaming,
+        error,
+        startCapture,
+        stopCapture,
+      }}
+    >
+      {children}
+    </ScreenCaptureContext.Provider>
+  );
+}
+
+export const useScreenCaptureContext = () => {
+  const context = useContext(ScreenCaptureContext);
+  if (context === undefined) {
+    throw new Error('useScreenCaptureContext must be used within a ScreenCaptureProvider');
+  }
+  return context;
+};
diff --git a/src/renderer/src/hooks/sidebar/setting/use-live2d-settings.ts b/src/renderer/src/hooks/sidebar/setting/use-live2d-settings.ts
index 899cd74..1e3a677 100644
--- a/src/renderer/src/hooks/sidebar/setting/use-live2d-settings.ts
+++ b/src/renderer/src/hooks/sidebar/setting/use-live2d-settings.ts
@@ -9,7 +9,6 @@ export const useLive2dSettings = () => {
     kScale: 0.000625,
     initialXshift: 0,
     initialYshift: 0,
-    kXOffset: 1150,
     emotionMap: {},
     scrollToResize: true,
   };
diff --git a/src/renderer/src/hooks/sidebar/use-capture-screen.ts b/src/renderer/src/hooks/sidebar/use-capture-screen.ts
new file mode 100644
index 0000000..5967c9a
--- /dev/null
+++ b/src/renderer/src/hooks/sidebar/use-capture-screen.ts
@@ -0,0 +1,36 @@
+import { useRef, useState, useEffect } from 'react';
+import { useScreenCaptureContext } from '@/context/screen-capture-context';
+
+export function useCaptureScreen() {
+  const videoRef = useRef<HTMLVideoElement>(null);
+  const [isHovering, setIsHovering] = useState(false);
+  const { stream, isStreaming, error, startCapture, stopCapture } = useScreenCaptureContext();
+
+  const toggleCapture = () => {
+    if (isStreaming) {
+      stopCapture();
+    } else {
+      startCapture();
+    }
+  };
+
+  const handleMouseEnter = () => setIsHovering(true);
+  const handleMouseLeave = () => setIsHovering(false);
+
+  useEffect(() => {
+    if (videoRef.current) {
+      videoRef.current.srcObject = stream;
+    }
+  }, [stream]);
+
+  return {
+    videoRef,
+    error,
+    isHovering,
+    isStreaming,
+    stream,
+    toggleCapture,
+    handleMouseEnter,
+    handleMouseLeave,
+  };
+}