Feat: Scrolling knowledge base list infiniflow#3695

cike8899 · Nov 27, 2024 · c44e513 · c44e513
1 parent 57208d8
commit c44e513
Show file tree

Hide file tree

Showing 11 changed files with 155 additions and 36 deletions.
diff --git a/api/apps/kb_app.py b/api/apps/kb_app.py
@@ -125,15 +125,16 @@ def detail():
 @manager.route('/list', methods=['GET'])
 @login_required
 def list_kbs():
+    keywords = request.args.get("keywords", "")
     page_number = int(request.args.get("page", 1))
     items_per_page = int(request.args.get("page_size", 150))
     orderby = request.args.get("orderby", "create_time")
     desc = request.args.get("desc", True)
     try:
         tenants = TenantService.get_joined_tenants_by_user_id(current_user.id)
-        kbs = KnowledgebaseService.get_by_tenant_ids(
-            [m["tenant_id"] for m in tenants], current_user.id, page_number, items_per_page, orderby, desc)
-        return get_json_result(data=kbs)
+        kbs, total = KnowledgebaseService.get_by_tenant_ids(
+            [m["tenant_id"] for m in tenants], current_user.id, page_number, items_per_page, orderby, desc, keywords)
+        return get_json_result(data={"kbs": kbs, "total": total})
     except Exception as e:
         return server_error_response(e)
 

diff --git a/api/db/services/knowledgebase_service.py b/api/db/services/knowledgebase_service.py
@@ -34,7 +34,7 @@ def list_documents_by_ids(cls,kb_ids):
     @classmethod
     @DB.connection_context()
     def get_by_tenant_ids(cls, joined_tenant_ids, user_id,
-                          page_number, items_per_page, orderby, desc):
+                          page_number, items_per_page, orderby, desc, keywords):
         fields = [
             cls.model.id,
             cls.model.avatar,
@@ -51,20 +51,31 @@ def get_by_tenant_ids(cls, joined_tenant_ids, user_id,
             User.avatar.alias('tenant_avatar'),
             cls.model.update_time
         ]
-        kbs = cls.model.select(*fields).join(User, on=(cls.model.tenant_id == User.id)).where(
-            ((cls.model.tenant_id.in_(joined_tenant_ids) & (cls.model.permission ==
-                                                            TenantPermission.TEAM.value)) | (
-                     cls.model.tenant_id == user_id))
-            & (cls.model.status == StatusEnum.VALID.value)
-        )
+        if keywords:
+            kbs = cls.model.select(*fields).join(User, on=(cls.model.tenant_id == User.id)).where(
+                ((cls.model.tenant_id.in_(joined_tenant_ids) & (cls.model.permission ==
+                                                                TenantPermission.TEAM.value)) | (
+                    cls.model.tenant_id == user_id))
+                & (cls.model.status == StatusEnum.VALID.value),
+                (fn.LOWER(cls.model.name).contains(keywords.lower()))
+            )
+        else:
+            kbs = cls.model.select(*fields).join(User, on=(cls.model.tenant_id == User.id)).where(
+                ((cls.model.tenant_id.in_(joined_tenant_ids) & (cls.model.permission ==
+                                                                TenantPermission.TEAM.value)) | (
+                    cls.model.tenant_id == user_id))
+                & (cls.model.status == StatusEnum.VALID.value)
+            )
         if desc:
             kbs = kbs.order_by(cls.model.getter_by(orderby).desc())
         else:
             kbs = kbs.order_by(cls.model.getter_by(orderby).asc())
 
+        count = kbs.count()
+
         kbs = kbs.paginate(page_number, items_per_page)
 
-        return list(kbs.dicts())
+        return list(kbs.dicts()), count
 
     @classmethod
     @DB.connection_context()

diff --git a/web/.umirc.ts b/web/.umirc.ts
@@ -34,7 +34,7 @@ export default defineConfig({
   proxy: [
     {
       context: ['/api', '/v1'],
-      target: 'http://127.0.0.1:9456/',
+      target: 'http://127.0.0.1:9380/',
       changeOrigin: true,
       ws: true,
       logger: console,

diff --git a/web/package-lock.json b/web/package-lock.json
diff --git a/web/package.json b/web/package.json
@@ -68,6 +68,7 @@
     "react-force-graph": "^1.44.4",
     "react-hook-form": "^7.53.1",
     "react-i18next": "^14.0.0",
+    "react-infinite-scroll-component": "^6.1.0",
     "react-markdown": "^9.0.1",
     "react-pdf-highlighter": "^6.1.0",
     "react-string-replace": "^1.1.1",

diff --git a/web/src/hooks/knowledge-hooks.ts b/web/src/hooks/knowledge-hooks.ts
@@ -3,14 +3,17 @@ import { IKnowledge, ITestingResult } from '@/interfaces/database/knowledge';
 import i18n from '@/locales/config';
 import kbService from '@/services/knowledge-service';
 import {
+  useInfiniteQuery,
   useIsMutating,
   useMutation,
   useMutationState,
   useQuery,
   useQueryClient,
 } from '@tanstack/react-query';
+import { useDebounce } from 'ahooks';
 import { message } from 'antd';
 import { useSearchParams } from 'umi';
+import { useHandleSearchChange } from './logic-hooks';
 import { useSetPaginationParams } from './route-hook';
 
 export const useKnowledgeBaseId = (): string => {
@@ -50,7 +53,7 @@ export const useNextFetchKnowledgeList = (
     gcTime: 0, // https://tanstack.com/query/latest/docs/framework/react/guides/caching?from=reactQueryV3
     queryFn: async () => {
       const { data } = await kbService.getList();
-      const list = data?.data ?? [];
+      const list = data?.data?.kbs ?? [];
       return shouldFilterListWithoutDocument
         ? list.filter((x: IKnowledge) => x.chunk_num > 0)
         : list;
@@ -60,6 +63,52 @@ export const useNextFetchKnowledgeList = (
   return { list: data, loading };
 };
 
+export const useInfiniteFetchKnowledgeList = () => {
+  const { searchString, handleInputChange } = useHandleSearchChange();
+  const debouncedSearchString = useDebounce(searchString, { wait: 500 });
+
+  const PageSize = 10;
+  const {
+    data,
+    error,
+    fetchNextPage,
+    hasNextPage,
+    isFetching,
+    isFetchingNextPage,
+    status,
+  } = useInfiniteQuery({
+    queryKey: ['infiniteFetchKnowledgeList', debouncedSearchString],
+    queryFn: async ({ pageParam }) => {
+      const { data } = await kbService.getList({
+        page: pageParam,
+        page_size: PageSize,
+        keywords: debouncedSearchString,
+      });
+      const list = data?.data ?? [];
+      return list;
+    },
+    initialPageParam: 1,
+    getNextPageParam: (lastPage, pages, lastPageParam) => {
+      if (lastPageParam * PageSize <= lastPage.total) {
+        return lastPageParam + 1;
+      }
+      return undefined;
+    },
+  });
+  return {
+    data,
+    loading: isFetching,
+    error,
+    fetchNextPage,
+    hasNextPage,
+    isFetching,
+    isFetchingNextPage,
+    status,
+    handleInputChange,
+    searchString,
+  };
+};
+
 export const useCreateKnowledge = () => {
   const queryClient = useQueryClient();
   const {
@@ -95,7 +144,9 @@ export const useDeleteKnowledge = () => {
       const { data } = await kbService.rmKb({ kb_id: id });
       if (data.code === 0) {
         message.success(i18n.t(`message.deleted`));
-        queryClient.invalidateQueries({ queryKey: ['fetchKnowledgeList'] });
+        queryClient.invalidateQueries({
+          queryKey: ['infiniteFetchKnowledgeList'],
+        });
       }
       return data?.data ?? [];
     },

diff --git a/web/src/locales/en.ts b/web/src/locales/en.ts
@@ -75,6 +75,7 @@ export default {
       namePlaceholder: 'Please input name!',
       doc: 'Docs',
       searchKnowledgePlaceholder: 'Search',
+      noMoreData: 'It is all, nothing more',
     },
     knowledgeDetails: {
       dataset: 'Dataset',

diff --git a/web/src/locales/zh-traditional.ts b/web/src/locales/zh-traditional.ts
@@ -75,6 +75,7 @@ export default {
       namePlaceholder: '請輸入名稱',
       doc: '文件',
       searchKnowledgePlaceholder: '搜索',
+      noMoreData: 'It is all, nothing more',
     },
     knowledgeDetails: {
       dataset: '數據集',

diff --git a/web/src/locales/zh.ts b/web/src/locales/zh.ts
@@ -75,6 +75,7 @@ export default {
       namePlaceholder: '请输入名称',
       doc: '文档',
       searchKnowledgePlaceholder: '搜索',
+      noMoreData: '沒有更多的數據了',
     },
     knowledgeDetails: {
       dataset: '数据集',

diff --git a/web/src/pages/knowledge/index.less b/web/src/pages/knowledge/index.less
@@ -2,6 +2,7 @@
 
 .knowledge {
   padding: 48px 0;
+  overflow: auto;
 }
 
 .topWrapper {

diff --git a/web/src/pages/knowledge/index.tsx b/web/src/pages/knowledge/index.tsx
@@ -1,18 +1,26 @@
-import { useNextFetchKnowledgeList } from '@/hooks/knowledge-hooks';
+import { useInfiniteFetchKnowledgeList } from '@/hooks/knowledge-hooks';
 import { useFetchUserInfo } from '@/hooks/user-setting-hooks';
 import { PlusOutlined, SearchOutlined } from '@ant-design/icons';
-import { Button, Empty, Flex, Input, Space, Spin } from 'antd';
+import {
+  Button,
+  Divider,
+  Empty,
+  Flex,
+  Input,
+  Skeleton,
+  Space,
+  Spin,
+} from 'antd';
+import { useTranslation } from 'react-i18next';
+import InfiniteScroll from 'react-infinite-scroll-component';
+import { useSaveKnowledge } from './hooks';
 import KnowledgeCard from './knowledge-card';
 import KnowledgeCreatingModal from './knowledge-creating-modal';
 
-import { useTranslation } from 'react-i18next';
-import { useSaveKnowledge, useSearchKnowledge } from './hooks';
+import { useMemo } from 'react';
 import styles from './index.less';
 
 const KnowledgeList = () => {
-  const { searchString, handleInputChange } = useSearchKnowledge();
-  const { loading, list: data } = useNextFetchKnowledgeList();
-  const list = data.filter((x) => x.name.includes(searchString));
   const { data: userInfo } = useFetchUserInfo();
   const { t } = useTranslation('translation', { keyPrefix: 'knowledgeList' });
   const {
@@ -22,9 +30,23 @@ const KnowledgeList = () => {
     onCreateOk,
     loading: creatingLoading,
   } = useSaveKnowledge();
+  const {
+    fetchNextPage,
+    data,
+    hasNextPage,
+    searchString,
+    handleInputChange,
+    loading,
+  } = useInfiniteFetchKnowledgeList();
+  console.log('🚀 ~ KnowledgeList ~ data:', data);
+  const nextList = data?.pages?.flatMap((x) => x.kbs) ?? [];
+
+  const total = useMemo(() => {
+    return data?.pages.at(-1).total ?? 0;
+  }, [data?.pages]);
 
   return (
-    <Flex className={styles.knowledge} vertical flex={1}>
+    <Flex className={styles.knowledge} vertical flex={1} id="scrollableDiv">
       <div className={styles.topWrapper}>
         <div>
           <span className={styles.title}>
@@ -53,21 +75,30 @@ const KnowledgeList = () => {
         </Space>
       </div>
       <Spin spinning={loading}>
-        <Flex
-          gap={'large'}
-          wrap="wrap"
-          className={styles.knowledgeCardContainer}
+        <InfiniteScroll
+          dataLength={nextList?.length ?? 0}
+          next={fetchNextPage}
+          hasMore={hasNextPage}
+          loader={<Skeleton avatar paragraph={{ rows: 1 }} active />}
+          endMessage={total && <Divider plain>{t('noMoreData')} 🤐</Divider>}
+          scrollableTarget="scrollableDiv"
         >
-          {list.length > 0 ? (
-            list.map((item: any) => {
-              return (
-                <KnowledgeCard item={item} key={item.name}></KnowledgeCard>
-              );
-            })
-          ) : (
-            <Empty className={styles.knowledgeEmpty}></Empty>
-          )}
-        </Flex>
+          <Flex
+            gap={'large'}
+            wrap="wrap"
+            className={styles.knowledgeCardContainer}
+          >
+            {nextList?.length > 0 ? (
+              nextList.map((item: any) => {
+                return (
+                  <KnowledgeCard item={item} key={item.name}></KnowledgeCard>
+                );
+              })
+            ) : (
+              <Empty className={styles.knowledgeEmpty}></Empty>
+            )}
+          </Flex>
+        </InfiniteScroll>
       </Spin>
       <KnowledgeCreatingModal
         loading={creatingLoading}