Project tasks loading only when needed (#3223)

* Tasks loading only when needed * Fixed project page * Added CHANGELOG, increased packages versions * Update CHANGELOG.md Co-authored-by: Boris Sekachev <[email protected]> * Fixed comments * Fixed overflow issue * Fixed reducer issue * Fixed cvat-core tests Co-authored-by: Boris Sekachev <[email protected]>
cvat-ai · May 25, 2021 · a17d054 · a17d054
1 parent ae175d9
commit a17d054
Show file tree

Hide file tree

Showing 18 changed files with 152 additions and 108 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -25,7 +25,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Fixed
 
--
+- Project page requests took a long time and did many DB queries (<https://github.com/openvinotoolkit/cvat/pull/3223>)
 
 ### Security
 

diff --git a/cvat-core/package-lock.json b/cvat-core/package-lock.json
diff --git a/cvat-core/package.json b/cvat-core/package.json
@@ -1,6 +1,6 @@
 {
   "name": "cvat-core",
-  "version": "3.12.3",
+  "version": "3.13.0",
   "description": "Part of Computer Vision Tool which presents an interface for client-side integration",
   "main": "babel.config.js",
   "scripts": {

diff --git a/cvat-core/src/api-implementation.js b/cvat-core/src/api-implementation.js
@@ -227,6 +227,14 @@
 
             checkExclusiveFields(filter, ['id', 'search'], ['page', 'withoutTasks']);
 
+            if (typeof filter.withoutTasks === 'undefined') {
+                if (typeof filter.id === 'undefined') {
+                    filter.withoutTasks = true;
+                } else {
+                    filter.withoutTasks = false;
+                }
+            }
+
             const searchParams = new URLSearchParams();
             for (const field of ['name', 'assignee', 'owner', 'search', 'status', 'id', 'page', 'withoutTasks']) {
                 if (Object.prototype.hasOwnProperty.call(filter, field)) {
@@ -238,7 +246,10 @@
             // prettier-ignore
             const projects = projectsData.map((project) => {
                 if (filter.withoutTasks) {
+                    project.task_ids = project.tasks;
                     project.tasks = [];
+                } else {
+                    project.task_ids = project.tasks.map((task) => task.id);
                 }
                 return project;
             }).map((project) => new Project(project));

diff --git a/cvat-core/src/project.js b/cvat-core/src/project.js
@@ -8,6 +8,7 @@
     const { ArgumentError } = require('./exceptions');
     const { Task } = require('./session');
     const { Label } = require('./labels');
+    const { getPreview } = require('./frames');
     const User = require('./user');
 
     /**
@@ -34,6 +35,7 @@
                 updated_date: undefined,
                 task_subsets: undefined,
                 training_project: undefined,
+                task_ids: undefined,
             };
 
             for (const property in data) {
@@ -58,9 +60,9 @@
                     data.tasks.push(taskInstance);
                 }
             }
-            if (!data.task_subsets && data.tasks.length) {
+            if (!data.task_subsets) {
                 const subsetsSet = new Set();
-                for (const task in data.tasks) {
+                for (const task of data.tasks) {
                     if (task.subset) subsetsSet.add(task.subset);
                 }
                 data.task_subsets = Array.from(subsetsSet);
@@ -254,6 +256,22 @@
             );
         }
 
+        /**
+         * Get the first frame of the first task of a project for preview
+         * @method preview
+         * @memberof Project
+         * @returns {string} - jpeg encoded image
+         * @instance
+         * @async
+         * @throws {module:API.cvat.exceptions.PluginError}
+         * @throws {module:API.cvat.exceptions.ServerError}
+         * @throws {module:API.cvat.exceptions.ArgumentError}
+         */
+        async preview() {
+            const result = await PluginRegistry.apiWrapper.call(this, Project.prototype.preview);
+            return result;
+        }
+
         /**
          * Method updates data of a created project or creates new project from scratch
          * @method save
@@ -331,4 +349,12 @@
         const result = await serverProxy.projects.delete(this.id);
         return result;
     };
+
+    Project.prototype.preview.implementation = async function () {
+        if (!this._internalData.task_ids.length) {
+            return '';
+        }
+        const frameData = await getPreview(this._internalData.task_ids[0]);
+        return frameData;
+    };
 })();
diff --git a/cvat-core/tests/api/projects.js b/cvat-core/tests/api/projects.js
@@ -16,7 +16,7 @@ const { Project } = require('../../src/project');
 
 describe('Feature: get projects', () => {
     test('get all projects', async () => {
-        const result = await window.cvat.projects.get();
+        const result = await window.cvat.projects.get({ withoutTasks: false });
         expect(Array.isArray(result)).toBeTruthy();
         expect(result).toHaveLength(2);
         for (const el of result) {

diff --git a/cvat-core/tests/mocks/server-proxy.mock.js b/cvat-core/tests/mocks/server-proxy.mock.js
@@ -14,16 +14,18 @@ const {
     frameMetaDummyData,
 } = require('./dummy-data.mock');
 
-function QueryStringToJSON(query) {
+function QueryStringToJSON(query, ignoreList = []) {
     const pairs = [...new URLSearchParams(query).entries()];
 
     const result = {};
     for (const pair of pairs) {
         const [key, value] = pair;
-        if (['id'].includes(key)) {
-            result[key] = +value;
-        } else {
-            result[key] = value;
+        if (!ignoreList.includes(key)) {
+            if (['id'].includes(key)) {
+                result[key] = +value;
+            } else {
+                result[key] = value;
+            }
         }
     }
 
@@ -73,7 +75,7 @@ class ServerProxy {
         }
 
         async function getProjects(filter = '') {
-            const queries = QueryStringToJSON(filter);
+            const queries = QueryStringToJSON(filter, ['without_tasks']);
             const result = projectsDummyData.results.filter((x) => {
                 for (const key in queries) {
                     if (Object.prototype.hasOwnProperty.call(queries, key)) {

diff --git a/cvat-ui/package-lock.json b/cvat-ui/package-lock.json
diff --git a/cvat-ui/package.json b/cvat-ui/package.json
@@ -1,6 +1,6 @@
 {
   "name": "cvat-ui",
-  "version": "1.20.0",
+  "version": "1.20.1",
   "description": "CVAT single-page application",
   "main": "src/index.tsx",
   "scripts": {

diff --git a/cvat-ui/src/actions/projects-actions.ts b/cvat-ui/src/actions/projects-actions.ts
@@ -5,9 +5,8 @@
 import { Dispatch, ActionCreator } from 'redux';
 
 import { ActionUnion, createAction, ThunkAction } from 'utils/redux';
-import { ProjectsQuery, CombinedState } from 'reducers/interfaces';
+import { ProjectsQuery } from 'reducers/interfaces';
 import { getTasksSuccess, updateTaskSuccess } from 'actions/tasks-actions';
-import { getCVATStore } from 'cvat-store';
 import getCore from 'cvat-core-wrapper';
 
 const cvat = getCore();
@@ -31,8 +30,8 @@ export enum ProjectsActionTypes {
 // prettier-ignore
 const projectActions = {
     getProjects: () => createAction(ProjectsActionTypes.GET_PROJECTS),
-    getProjectsSuccess: (array: any[], count: number) => (
-        createAction(ProjectsActionTypes.GET_PROJECTS_SUCCESS, { array, count })
+    getProjectsSuccess: (array: any[], previews: string[], count: number) => (
+        createAction(ProjectsActionTypes.GET_PROJECTS_SUCCESS, { array, previews, count })
     ),
     getProjectsFailed: (error: any) => createAction(ProjectsActionTypes.GET_PROJECTS_FAILED, { error }),
     updateProjectsGettingQuery: (query: Partial<ProjectsQuery>) => (
@@ -60,7 +59,7 @@ const projectActions = {
 export type ProjectActions = ActionUnion<typeof projectActions>;
 
 export function getProjectsAsync(query: Partial<ProjectsQuery>): ThunkAction {
-    return async (dispatch: ActionCreator<Dispatch>): Promise<void> => {
+    return async (dispatch: ActionCreator<Dispatch>, getState): Promise<void> => {
         dispatch(projectActions.getProjects());
         dispatch(projectActions.updateProjectsGettingQuery(query));
 
@@ -69,6 +68,7 @@ export function getProjectsAsync(query: Partial<ProjectsQuery>): ThunkAction {
             page: 1,
             ...query,
         };
+
         for (const key in filteredQuery) {
             if (filteredQuery[key] === null || typeof filteredQuery[key] === 'undefined') {
                 delete filteredQuery[key];
@@ -85,38 +85,38 @@ export function getProjectsAsync(query: Partial<ProjectsQuery>): ThunkAction {
 
         const array = Array.from(result);
 
-        const tasks: any[] = [];
-        const taskPreviewPromises: Promise<any>[] = [];
-
-        for (const project of array) {
-            taskPreviewPromises.push(
-                ...(project as any).tasks.map((task: any): string => {
-                    tasks.push(task);
-                    return (task as any).frames.preview().catch(() => '');
-                }),
-            );
-        }
+        // Appropriate tasks fetching proccess needs with retrieving only a single project
+        if (Object.keys(filteredQuery).includes('id')) {
+            const tasks: any[] = [];
+            const [project] = array;
+            const taskPreviewPromises: Promise<string>[] = (project as any).tasks.map((task: any): string => {
+                tasks.push(task);
+                return (task as any).frames.preview().catch(() => '');
+            });
 
-        const taskPreviews = await Promise.all(taskPreviewPromises);
-
-        dispatch(projectActions.getProjectsSuccess(array, result.count));
-
-        const store = getCVATStore();
-        const state: CombinedState = store.getState();
-
-        if (!state.tasks.fetching) {
-            dispatch(
-                getTasksSuccess(tasks, taskPreviews, tasks.length, {
-                    page: 1,
-                    assignee: null,
-                    id: null,
-                    mode: null,
-                    name: null,
-                    owner: null,
-                    search: null,
-                    status: null,
-                }),
-            );
+            const taskPreviews = await Promise.all(taskPreviewPromises);
+
+            const state = getState();
+
+            dispatch(projectActions.getProjectsSuccess(array, taskPreviews, result.count));
+
+            if (!state.tasks.fetching) {
+                dispatch(
+                    getTasksSuccess(tasks, taskPreviews, tasks.length, {
+                        page: 1,
+                        assignee: null,
+                        id: null,
+                        mode: null,
+                        name: null,
+                        owner: null,
+                        search: null,
+                        status: null,
+                    }),
+                );
+            }
+        } else {
+            const previewPromises = array.map((project): string => (project as any).preview().catch(() => ''));
+            dispatch(projectActions.getProjectsSuccess(array, await Promise.all(previewPromises), result.count));
         }
     };
 }

diff --git a/cvat-ui/src/components/project-page/project-page.tsx b/cvat-ui/src/components/project-page/project-page.tsx
@@ -28,18 +28,16 @@ export default function ProjectPageComponent(): JSX.Element {
     const id = +useParams<ParamType>().id;
     const dispatch = useDispatch();
     const history = useHistory();
-    const projects = useSelector((state: CombinedState) => state.projects.current);
+    const projects = useSelector((state: CombinedState) => state.projects.current).map((project) => project.instance);
     const projectsFetching = useSelector((state: CombinedState) => state.projects.fetching);
     const deletes = useSelector((state: CombinedState) => state.projects.activities.deletes);
     const taskDeletes = useSelector((state: CombinedState) => state.tasks.activities.deletes);
     const tasksActiveInferences = useSelector((state: CombinedState) => state.models.inferences);
     const tasks = useSelector((state: CombinedState) => state.tasks.current);
-    const projectSubsets = useSelector((state: CombinedState) => {
-        const [project] = state.projects.current.filter((_project) => _project.id === id);
-        return project ? ([...new Set(project.tasks.map((task: any) => task.subset))] as string[]) : [];
-    });
 
     const [project] = projects.filter((_project) => _project.id === id);
+    const projectSubsets = [''];
+    if (project) projectSubsets.push(...project.subsets);
     const deleteActivity = project && id in deletes ? deletes[id] : null;
 
     useEffect(() => {
@@ -90,7 +88,7 @@ export default function ProjectPageComponent(): JSX.Element {
                         </Button>
                     </Col>
                 </Row>
-                {projectSubsets.map((subset) => (
+                {projectSubsets.map((subset: string) => (
                     <React.Fragment key={subset}>
                         {subset && <Title level={4}>{subset}</Title>}
                         {tasks

diff --git a/cvat-ui/src/components/project-page/styles.scss b/cvat-ui/src/components/project-page/styles.scss
@@ -4,6 +4,11 @@
 
 @import '../../base.scss';
 
+.cvat-project-page {
+    overflow-y: auto;
+    height: 100%;
+}
+
 .cvat-project-details {
     width: 100%;
     height: auto;

diff --git a/cvat-ui/src/components/projects-page/project-item.tsx b/cvat-ui/src/components/projects-page/project-item.tsx
@@ -1,4 +1,4 @@
-// Copyright (C) 2020 Intel Corporation
+// Copyright (C) 2020-2021 Intel Corporation
 //
 // SPDX-License-Identifier: MIT
 
@@ -22,24 +22,18 @@ interface Props {
 }
 
 export default function ProjectItemComponent(props: Props): JSX.Element {
-    const { projectInstance } = props;
+    const {
+        projectInstance: { instance, preview },
+    } = props;
 
     const history = useHistory();
-    const ownerName = projectInstance.owner ? projectInstance.owner.username : null;
-    const updated = moment(projectInstance.updatedDate).fromNow();
+    const ownerName = instance.owner ? instance.owner.username : null;
+    const updated = moment(instance.updatedDate).fromNow();
     const deletes = useSelector((state: CombinedState) => state.projects.activities.deletes);
-    const deleted = projectInstance.id in deletes ? deletes[projectInstance.id] : false;
-
-    let projectPreview = null;
-    if (projectInstance.tasks.length) {
-        // prettier-ignore
-        projectPreview = useSelector((state: CombinedState) => (
-            state.tasks.current.find((task) => task.instance.id === projectInstance.tasks[0].id)?.preview
-        ));
-    }
+    const deleted = instance.id in deletes ? deletes[instance.id] : false;
 
     const onOpenProject = (): void => {
-        history.push(`/projects/${projectInstance.id}`);
+        history.push(`/projects/${instance.id}`);
     };
 
     const style: React.CSSProperties = {};
@@ -52,10 +46,10 @@ export default function ProjectItemComponent(props: Props): JSX.Element {
     return (
         <Card
             cover={
-                projectPreview ? (
+                preview ? (
                     <img
                         className='cvat-projects-project-item-card-preview'
-                        src={projectPreview}
+                        src={preview}
                         alt='Preview'
                         onClick={onOpenProject}
                         aria-hidden
@@ -73,7 +67,7 @@ export default function ProjectItemComponent(props: Props): JSX.Element {
             <Meta
                 title={(
                     <span onClick={onOpenProject} className='cvat-projects-project-item-title' aria-hidden>
-                        {projectInstance.name}
+                        {instance.name}
                     </span>
                 )}
                 description={(
@@ -88,7 +82,7 @@ export default function ProjectItemComponent(props: Props): JSX.Element {
                             <Text type='secondary'>{`Last updated ${updated}`}</Text>
                         </div>
                         <div>
-                            <Dropdown overlay={<ProjectActionsMenuComponent projectInstance={projectInstance} />}>
+                            <Dropdown overlay={<ProjectActionsMenuComponent projectInstance={instance} />}>
                                 <Button type='link' size='large' icon={<MoreOutlined />} />
                             </Dropdown>
                         </div>
-Original file line number
+Diff line change
@@ Expand Up @@
     ### Fixed
-    -
+    - Project page requests took a long time and did many DB queries (<https://github.com/openvinotoolkit/cvat/pull/3223>)
     ### Security
@@ Expand Down @@