diff --git a/.vscode/launch.json b/.vscode/launch.json index 4b65fad0ba25..55811c5356f7 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -5,7 +5,7 @@ "version": "0.2.0", "configurations": [ { - "type": "chrome", + "type": "pwa-chrome", "request": "launch", "preLaunchTask": "npm: start - cvat-ui", "name": "ui.js: debug", @@ -59,7 +59,7 @@ }, { "name": "server: chrome", - "type": "chrome", + "type": "pwa-chrome", "request": "launch", "url": "http://localhost:7000/", "disableNetworkCache":true, diff --git a/CHANGELOG.md b/CHANGELOG.md index f5e168ec4f2a..ba874209df24 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added intelligent scissors blocking feature () - Support cloud storage status () - Support cloud storage preview () +- cvat-core: support cloud storages () ### Changed @@ -37,6 +38,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Fixed multiple tasks moving () - Fixed task creating CLI parameter () +- Fixed import for MOTS format () ### Security diff --git a/cvat-core/package-lock.json b/cvat-core/package-lock.json index 15be69a6b42f..f0a18dc0ffb8 100644 --- a/cvat-core/package-lock.json +++ b/cvat-core/package-lock.json @@ -1,6 +1,6 @@ { "name": "cvat-core", - "version": "3.15.0", + "version": "3.16.0", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -3785,11 +3785,11 @@ "integrity": "sha512-3YDiu347mtVtjpyV3u5kVqQLP242c06zwDOgpeRnybmXlYYsLbtTrUBUm8i8srONt+FWobl5aibnU1030PeeuA==" }, "axios": { - "version": "0.21.1", - "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.1.tgz", - "integrity": "sha512-dKQiRHxGD9PPRIUNIWvZhPTPpl1rf/OxTYKsqKUDjBwYylTvV7SjSHJb9ratfyzM6wCdLCOYLzs73qpg5c4iGA==", + "version": "0.21.3", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.3.tgz", + "integrity": "sha512-JtoZ3Ndke/+Iwt5n+BgSli/3idTvpt5OjKyoCmz4LX5+lPiY5l7C1colYezhlxThjNa/NhngCUWZSZFypIFuaA==", "requires": { - "follow-redirects": "^1.10.0" + "follow-redirects": "^1.14.0" } }, "babel-code-frame": { @@ -13562,9 +13562,9 @@ } }, "follow-redirects": { - "version": "1.13.1", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.13.1.tgz", - "integrity": "sha512-SSG5xmZh1mkPGyKzjZP8zLjltIfpW32Y5QpdNJyjcfGxK3qo3NDDkZOZSFiGn1A6SclQxY9GzEwAHQ3dmYRWpg==" + "version": "1.14.3", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.3.tgz", + "integrity": "sha512-3MkHxknWMUtb23apkgz/83fDoe+y+qr0TdgacGIA7bew+QLBo3vdgEN2xEsuXNivpFy4CyDhBBZnNZOtalmenw==" }, "for-in": { "version": "1.0.2", diff --git a/cvat-core/package.json b/cvat-core/package.json index 229b9bec6ae8..2ca27952b6ec 100644 --- a/cvat-core/package.json +++ b/cvat-core/package.json @@ -1,6 +1,6 @@ { "name": "cvat-core", - "version": "3.15.0", + "version": "3.16.0", "description": "Part of Computer Vision Tool which presents an interface for client-side integration", "main": "babel.config.js", "scripts": { @@ -39,7 +39,7 @@ "webpack-cli": "^3.3.2" }, "dependencies": { - "axios": "^0.21.1", + "axios": "^0.21.3", "browser-or-node": "^1.2.1", "cvat-data": "../cvat-data", "detect-browser": "^5.2.0", diff --git a/cvat-core/src/api-implementation.js b/cvat-core/src/api-implementation.js index 787c4303f90e..0301fdec6031 100644 --- a/cvat-core/src/api-implementation.js +++ b/cvat-core/src/api-implementation.js @@ -16,13 +16,20 @@ camelToSnake, } = require('./common'); - const { TaskStatus, TaskMode, DimensionType } = require('./enums'); + const { + TaskStatus, + TaskMode, + DimensionType, + CloudStorageProviderType, + CloudStorageCredentialsType, + } = require('./enums'); const User = require('./user'); const { AnnotationFormats } = require('./annotation-formats'); const { ArgumentError } = require('./exceptions'); const { Task } = require('./session'); const { Project } = require('./project'); + const { CloudStorage } = require('./cloud-storage'); function implementAPI(cvat) { cvat.plugins.list.implementation = PluginRegistry.list; @@ -262,6 +269,49 @@ cvat.projects.searchNames.implementation = async (search, limit) => serverProxy.projects.searchNames(search, limit); + cvat.cloudStorages.get.implementation = async (filter) => { + checkFilter(filter, { + page: isInteger, + displayName: isString, + resourceName: isString, + description: isString, + id: isInteger, + owner: isString, + search: isString, + providerType: isEnum.bind(CloudStorageProviderType), + credentialsType: isEnum.bind(CloudStorageCredentialsType), + }); + + checkExclusiveFields(filter, ['id', 'search'], ['page']); + + const searchParams = new URLSearchParams(); + for (const field of [ + 'displayName', + 'credentialsType', + 'providerType', + 'owner', + 'search', + 'id', + 'page', + 'description', + ]) { + if (Object.prototype.hasOwnProperty.call(filter, field)) { + searchParams.set(camelToSnake(field), filter[field]); + } + } + + if (Object.prototype.hasOwnProperty.call(filter, 'resourceName')) { + searchParams.set('resource', filter.resourceName); + } + + const cloudStoragesData = await serverProxy.cloudStorages.get(searchParams.toString()); + const cloudStorages = cloudStoragesData.map((cloudStorage) => new CloudStorage(cloudStorage)); + + cloudStorages.count = cloudStoragesData.count; + + return cloudStorages; + }; + return cvat; } diff --git a/cvat-core/src/api.js b/cvat-core/src/api.js index d517b444caa3..e50cdbc8f8cb 100644 --- a/cvat-core/src/api.js +++ b/cvat-core/src/api.js @@ -22,6 +22,8 @@ function build() { const { Attribute, Label } = require('./labels'); const MLModel = require('./ml-model'); const { FrameData } = require('./frames'); + const { CloudStorage } = require('./cloud-storage'); + const enums = require('./enums'); @@ -748,6 +750,41 @@ function build() { PluginError, ServerError, }, + /** + * Namespace is used for getting cloud storages + * @namespace cloudStorages + * @memberof module:API.cvat + */ + cloudStorages: { + /** + * @typedef {Object} CloudStorageFilter + * @property {string} displayName Check if displayName contains this value + * @property {string} resourceName Check if resourceName contains this value + * @property {module:API.cvat.enums.ProviderType} providerType Check if providerType equal this value + * @property {integer} id Check if id equals this value + * @property {integer} page Get specific page + * (default REST API returns 20 clouds storages per request. + * In order to get more, it is need to specify next page) + * @property {string} owner Check if an owner name contains this value + * @property {string} search Combined search of contains among all the fields + * @global + */ + + /** + * Method returns a list of cloud storages corresponding to a filter + * @method get + * @async + * @memberof module:API.cvat.cloudStorages + * @param {CloudStorageFilter} [filter={}] cloud storage filter + * @returns {module:API.cvat.classes.CloudStorage[]} + * @throws {module:API.cvat.exceptions.PluginError} + * @throws {module:API.cvat.exceptions.ServerError} + */ + async get(filter = {}) { + const result = await PluginRegistry.apiWrapper(cvat.cloudStorages.get, filter); + return result; + }, + }, /** * Namespace is used for access to classes * @namespace classes @@ -768,6 +805,7 @@ function build() { Issue, Review, FrameData, + CloudStorage, }, }; @@ -780,6 +818,7 @@ function build() { cvat.lambda = Object.freeze(cvat.lambda); cvat.client = Object.freeze(cvat.client); cvat.enums = Object.freeze(cvat.enums); + cvat.cloudStorages = Object.freeze(cvat.cloudStorages); const implementAPI = require('./api-implementation'); diff --git a/cvat-core/src/cloud-storage.js b/cvat-core/src/cloud-storage.js new file mode 100644 index 000000000000..4fd8bd3a96e5 --- /dev/null +++ b/cvat-core/src/cloud-storage.js @@ -0,0 +1,520 @@ +// Copyright (C) 2021 Intel Corporation +// +// SPDX-License-Identifier: MIT + +(() => { + const PluginRegistry = require('./plugins'); + const serverProxy = require('./server-proxy'); + const { isBrowser, isNode } = require('browser-or-node'); + const { ArgumentError } = require('./exceptions'); + const { CloudStorageCredentialsType, CloudStorageProviderType } = require('./enums'); + + /** + * Class representing a cloud storage + * @memberof module:API.cvat.classes + */ + class CloudStorage { + // TODO: add storage availability status (avaliable/unavaliable) + constructor(initialData) { + const data = { + id: undefined, + display_name: undefined, + description: undefined, + credentials_type: undefined, + provider_type: undefined, + resource: undefined, + account_name: undefined, + key: undefined, + secret_key: undefined, + session_token: undefined, + specific_attributes: undefined, + owner: undefined, + created_date: undefined, + updated_date: undefined, + manifest_path: undefined, + manifests: undefined, + }; + + for (const property in data) { + if (Object.prototype.hasOwnProperty.call(data, property) && property in initialData) { + data[property] = initialData[property]; + } + } + + Object.defineProperties( + this, + Object.freeze({ + /** + * @name id + * @type {integer} + * @memberof module:API.cvat.classes.CloudStorage + * @readonly + * @instance + */ + id: { + get: () => data.id, + }, + /** + * Storage name + * @name displayName + * @type {string} + * @memberof module:API.cvat.classes.CloudStorage + * @instance + * @throws {module:API.cvat.exceptions.ArgumentError} + */ + displayName: { + get: () => data.display_name, + set: (value) => { + if (typeof value !== 'string') { + throw new ArgumentError(`Value must be string. ${typeof value} was found`); + } else if (!value.trim().length) { + throw new ArgumentError('Value must not be empty string'); + } + data.display_name = value; + }, + }, + /** + * Storage description + * @name description + * @type {string} + * @memberof module:API.cvat.classes.CloudStorage + * @instance + * @throws {module:API.cvat.exceptions.ArgumentError} + */ + description: { + get: () => data.description, + set: (value) => { + if (typeof value !== 'string') { + throw new ArgumentError('Value must be string'); + } + data.description = value; + }, + }, + /** + * Azure account name + * @name accountName + * @type {string} + * @memberof module:API.cvat.classes.CloudStorage + * @instance + * @throws {module:API.cvat.exceptions.ArgumentError} + */ + accountName: { + get: () => data.account_name, + set: (value) => { + if (typeof value === 'string') { + if (value.trim().length) { + data.account_name = value; + } else { + throw new ArgumentError('Value must not be empty'); + } + } else { + throw new ArgumentError(`Value must be a string. ${typeof value} was found`); + } + }, + }, + /** + * AWS access key id + * @name accessKey + * @type {string} + * @memberof module:API.cvat.classes.CloudStorage + * @instance + * @throws {module:API.cvat.exceptions.ArgumentError} + */ + accessKey: { + get: () => data.key, + set: (value) => { + if (typeof value === 'string') { + if (value.trim().length) { + data.key = value; + } else { + throw new ArgumentError('Value must not be empty'); + } + } else { + throw new ArgumentError(`Value must be a string. ${typeof value} was found`); + } + }, + }, + /** + * AWS secret key + * @name secretKey + * @type {string} + * @memberof module:API.cvat.classes.CloudStorage + * @instance + * @throws {module:API.cvat.exceptions.ArgumentError} + */ + secretKey: { + get: () => data.secret_key, + set: (value) => { + if (typeof value === 'string') { + if (value.trim().length) { + data.secret_key = value; + } else { + throw new ArgumentError('Value must not be empty'); + } + } else { + throw new ArgumentError(`Value must be a string. ${typeof value} was found`); + } + }, + }, + /** + * Session token + * @name token + * @type {string} + * @memberof module:API.cvat.classes.CloudStorage + * @instance + * @throws {module:API.cvat.exceptions.ArgumentError} + */ + token: { + get: () => data.session_token, + set: (value) => { + if (typeof value === 'string') { + if (value.trim().length) { + data.session_token = value; + } else { + throw new ArgumentError('Value must not be empty'); + } + } else { + throw new ArgumentError(`Value must be a string. ${typeof value} was found`); + } + }, + }, + /** + * Unique resource name + * @name resourceName + * @type {string} + * @memberof module:API.cvat.classes.CloudStorage + * @instance + * @throws {module:API.cvat.exceptions.ArgumentError} + */ + resourceName: { + get: () => data.resource, + set: (value) => { + if (typeof value !== 'string') { + throw new ArgumentError(`Value must be string. ${typeof value} was found`); + } else if (!value.trim().length) { + throw new ArgumentError('Value must not be empty'); + } + data.resource = value; + }, + }, + /** + * @name manifestPath + * @type {string} + * @memberof module:API.cvat.classes.CloudStorage + * @instance + * @throws {module:API.cvat.exceptions.ArgumentError} + */ + manifestPath: { + get: () => data.manifest_path, + set: (value) => { + if (typeof value === 'string') { + data.manifest_path = value; + } else { + throw new ArgumentError('Value must be a string'); + } + }, + }, + /** + * @name providerType + * @type {module:API.cvat.enums.ProviderType} + * @memberof module:API.cvat.classes.CloudStorage + * @instance + * @throws {module:API.cvat.exceptions.ArgumentError} + */ + providerType: { + get: () => data.provider_type, + set: (key) => { + if (key !== undefined && !!CloudStorageProviderType[key]) { + data.provider_type = CloudStorageProviderType[key]; + } else { + throw new ArgumentError('Value must be one CloudStorageProviderType keys'); + } + }, + }, + /** + * @name credentialsType + * @type {module:API.cvat.enums.CloudStorageCredentialsType} + * @memberof module:API.cvat.classes.CloudStorage + * @instance + * @throws {module:API.cvat.exceptions.ArgumentError} + */ + credentialsType: { + get: () => data.credentials_type, + set: (key) => { + if (key !== undefined && !!CloudStorageCredentialsType[key]) { + data.credentials_type = CloudStorageCredentialsType[key]; + } else { + throw new ArgumentError('Value must be one CloudStorageCredentialsType keys'); + } + }, + }, + /** + * @name specificAttributes + * @type {string} + * @memberof module:API.cvat.classes.CloudStorage + * @instance + * @throws {module:API.cvat.exceptions.ArgumentError} + */ + specificAttributes: { + get: () => data.specific_attributes, + set: (attributesValue) => { + if (typeof attributesValue === 'string') { + const attrValues = new URLSearchParams( + Array.from(new URLSearchParams(attributesValue).entries()).filter( + ([key, value]) => !!key && !!value, + ), + ).toString(); + if (!attrValues) { + throw new ArgumentError('Value must match the key1=value1&key2=value2'); + } + data.specific_attributes = attributesValue; + } else { + throw new ArgumentError('Value must be a string'); + } + }, + }, + /** + * Instance of a user who has created the cloud storage + * @name owner + * @type {module:API.cvat.classes.User} + * @memberof module:API.cvat.classes.CloudStorage + * @readonly + * @instance + */ + owner: { + get: () => data.owner, + }, + /** + * @name createdDate + * @type {string} + * @memberof module:API.cvat.classes.CloudStorage + * @readonly + * @instance + */ + createdDate: { + get: () => data.created_date, + }, + /** + * @name updatedDate + * @type {string} + * @memberof module:API.cvat.classes.CloudStorage + * @readonly + * @instance + */ + updatedDate: { + get: () => data.updated_date, + }, + /** + * @name manifests + * @type {string[]} + * @memberof module:API.cvat.classes.CloudStorage + * @instance + * @throws {module:API.cvat.exceptions.ArgumentError} + */ + manifests: { + get: () => data.manifests, + set: (manifests) => { + if (Array.isArray(manifests)) { + for (const elem of manifests) { + if (typeof elem !== 'string') { + throw new ArgumentError('Each element of the manifests array must be a string'); + } + } + data.manifests = manifests; + } else { + throw new ArgumentError('Value must be an array'); + } + }, + }, + }), + ); + } + + /** + * Method updates data of a created cloud storage or creates new cloud storage + * @method save + * @returns {module:API.cvat.classes.CloudStorage} + * @memberof module:API.cvat.classes.CloudStorage + * @readonly + * @instance + * @async + * @throws {module:API.cvat.exceptions.ServerError} + * @throws {module:API.cvat.exceptions.PluginError} + */ + async save() { + const result = await PluginRegistry.apiWrapper.call(this, CloudStorage.prototype.save); + return result; + } + + /** + * Method deletes a cloud storage from a server + * @method delete + * @memberof module:API.cvat.classes.CloudStorage + * @readonly + * @instance + * @async + * @throws {module:API.cvat.exceptions.ServerError} + * @throws {module:API.cvat.exceptions.PluginError} + */ + async delete() { + const result = await PluginRegistry.apiWrapper.call(this, CloudStorage.prototype.delete); + return result; + } + + /** + * Method returns cloud storage content + * @method getContent + * @memberof module:API.cvat.classes.CloudStorage + * @readonly + * @instance + * @async + * @throws {module:API.cvat.exceptions.ServerError} + * @throws {module:API.cvat.exceptions.PluginError} + */ + async getContent() { + const result = await PluginRegistry.apiWrapper.call(this, CloudStorage.prototype.getContent); + return result; + } + + /** + * Method returns the cloud storage preview + * @method getPreview + * @memberof module:API.cvat.classes.CloudStorage + * @readonly + * @instance + * @async + * @throws {module:API.cvat.exceptions.ServerError} + * @throws {module:API.cvat.exceptions.PluginError} + */ + async getPreview() { + const result = await PluginRegistry.apiWrapper.call(this, CloudStorage.prototype.getPreview); + return result; + } + + /** + * Method returns cloud storage status + * @method getStatus + * @memberof module:API.cvat.classes.CloudStorage + * @readonly + * @instance + * @async + * @throws {module:API.cvat.exceptions.ServerError} + * @throws {module:API.cvat.exceptions.PluginError} + */ + async getStatus() { + const result = await PluginRegistry.apiWrapper.call(this, CloudStorage.prototype.getStatus); + return result; + } + } + + CloudStorage.prototype.save.implementation = async function () { + function prepareOptionalFields(cloudStorageInstance) { + const data = {}; + if (cloudStorageInstance.description) { + data.description = cloudStorageInstance.description; + } + + if (cloudStorageInstance.accountName) { + data.account_name = cloudStorageInstance.accountName; + } + + if (cloudStorageInstance.accessKey) { + data.key = cloudStorageInstance.accessKey; + } + + if (cloudStorageInstance.secretKey) { + data.secret_key = cloudStorageInstance.secretKey; + } + + if (cloudStorageInstance.token) { + data.session_token = cloudStorageInstance.token; + } + + if (cloudStorageInstance.specificAttributes) { + data.specific_attributes = cloudStorageInstance.specificAttributes; + } + return data; + } + // update + if (typeof this.id !== 'undefined') { + // providr_type and recource should not change; + // send to the server only the values that have changed + const initialData = {}; + if (this.displayName) { + initialData.display_name = this.displayName; + } + if (this.credentialsType) { + initialData.credentials_type = this.credentialsType; + } + + if (this.manifests) { + initialData.manifests = this.manifests; + } + + const cloudStorageData = { + ...initialData, + ...prepareOptionalFields(this), + }; + + await serverProxy.cloudStorages.update(this.id, cloudStorageData); + return this; + } + + // create + const initialData = { + display_name: this.displayName, + credentials_type: this.credentialsType, + provider_type: this.providerType, + resource: this.resourceName, + manifests: this.manifests, + }; + + const cloudStorageData = { + ...initialData, + ...prepareOptionalFields(this), + }; + + const cloudStorage = await serverProxy.cloudStorages.create(cloudStorageData); + return new CloudStorage(cloudStorage); + }; + + CloudStorage.prototype.delete.implementation = async function () { + const result = await serverProxy.cloudStorages.delete(this.id); + return result; + }; + + CloudStorage.prototype.getContent.implementation = async function () { + const result = await serverProxy.cloudStorages.getContent(this.id, this.manifestPath); + return result; + }; + + CloudStorage.prototype.getPreview.implementation = async function getPreview() { + return new Promise((resolve, reject) => { + serverProxy.cloudStorages + .getPreview(this.id) + .then((result) => { + if (isNode) { + resolve(global.Buffer.from(result, 'binary').toString('base64')); + } else if (isBrowser) { + const reader = new FileReader(); + reader.onload = () => { + resolve(reader.result); + }; + reader.readAsDataURL(result); + } + }) + .catch((error) => { + reject(error); + }); + }); + }; + + CloudStorage.prototype.getStatus.implementation = async function () { + const result = await serverProxy.cloudStorages.getStatus(this.id); + return result; + }; + + module.exports = { + CloudStorage, + }; +})(); diff --git a/cvat-core/src/enums.js b/cvat-core/src/enums.js index 4ce6d80c08c6..c8ecab559650 100644 --- a/cvat-core/src/enums.js +++ b/cvat-core/src/enums.js @@ -1,4 +1,4 @@ -// Copyright (C) 2019-2020 Intel Corporation +// Copyright (C) 2019-2021 Intel Corporation // // SPDX-License-Identifier: MIT @@ -333,6 +333,36 @@ '#733380', ]; + /** + * Types of cloud storage providers + * @enum {string} + * @name CloudStorageProviderType + * @memberof module:API.cvat.enums + * @property {string} AWS_S3 'AWS_S3_BUCKET' + * @property {string} AZURE 'AZURE_CONTAINER' + * @readonly + */ + const CloudStorageProviderType = Object.freeze({ + AWS_S3_BUCKET: 'AWS_S3_BUCKET', + AZURE_CONTAINER: 'AZURE_CONTAINER', + }); + + /** + * Types of cloud storage credentials + * @enum {string} + * @name CloudStorageCredentialsType + * @memberof module:API.cvat.enums + * @property {string} KEY_SECRET_KEY_PAIR 'KEY_SECRET_KEY_PAIR' + * @property {string} ACCOUNT_NAME_TOKEN_PAIR 'ACCOUNT_NAME_TOKEN_PAIR' + * @property {string} ANONYMOUS_ACCESS 'ANONYMOUS_ACCESS' + * @readonly + */ + const CloudStorageCredentialsType = Object.freeze({ + KEY_SECRET_KEY_PAIR: 'KEY_SECRET_KEY_PAIR', + ACCOUNT_NAME_TOKEN_PAIR: 'ACCOUNT_NAME_TOKEN_PAIR', + ANONYMOUS_ACCESS: 'ANONYMOUS_ACCESS', + }); + module.exports = { ShareFileType, TaskStatus, @@ -348,5 +378,7 @@ colors, Source, DimensionType, + CloudStorageProviderType, + CloudStorageCredentialsType, }; })(); diff --git a/cvat-core/src/server-proxy.js b/cvat-core/src/server-proxy.js index 3d914ce9db7a..cdc98940f0c3 100644 --- a/cvat-core/src/server-proxy.js +++ b/cvat-core/src/server-proxy.js @@ -1145,9 +1145,7 @@ const closureId = Date.now(); predictAnnotations.latestRequest.id = closureId; - const predicate = () => ( - !predictAnnotations.latestRequest.fetching || predictAnnotations.latestRequest.id !== closureId - ); + const predicate = () => !predictAnnotations.latestRequest.fetching || predictAnnotations.latestRequest.id !== closureId; if (predictAnnotations.latestRequest.fetching) { waitFor(5, predicate).then(() => { if (predictAnnotations.latestRequest.id !== closureId) { @@ -1181,6 +1179,121 @@ } } + async function createCloudStorage(storageDetail) { + const { backendAPI } = config; + + try { + const response = await Axios.post(`${backendAPI}/cloudstorages`, JSON.stringify(storageDetail), { + proxy: config.proxy, + headers: { + 'Content-Type': 'application/json', + }, + }); + return response.data; + } catch (errorData) { + throw generateError(errorData); + } + } + + async function updateCloudStorage(id, cloudStorageData) { + const { backendAPI } = config; + + try { + await Axios.patch(`${backendAPI}/cloudstorages/${id}`, JSON.stringify(cloudStorageData), { + proxy: config.proxy, + headers: { + 'Content-Type': 'application/json', + }, + }); + } catch (errorData) { + throw generateError(errorData); + } + } + + async function getCloudStorages(filter = '') { + const { backendAPI } = config; + + let response = null; + try { + response = await Axios.get(`${backendAPI}/cloudstorages?page_size=12&${filter}`, { + proxy: config.proxy, + }); + } catch (errorData) { + throw generateError(errorData); + } + + response.data.results.count = response.data.count; + return response.data.results; + } + + async function getCloudStorageContent(id, manifestPath) { + const { backendAPI } = config; + + let response = null; + try { + const url = `${backendAPI}/cloudstorages/${id}/content${ + manifestPath ? `?manifest_path=${manifestPath}` : '' + }`; + response = await Axios.get(url, { + proxy: config.proxy, + }); + } catch (errorData) { + throw generateError(errorData); + } + + return response.data; + } + + async function getCloudStoragePreview(id) { + const { backendAPI } = config; + + let response = null; + try { + const url = `${backendAPI}/cloudstorages/${id}/preview`; + response = await workerAxios.get(url, { + proxy: config.proxy, + responseType: 'arraybuffer', + }); + } catch (errorData) { + throw generateError({ + ...errorData, + message: '', + response: { + ...errorData.response, + data: String.fromCharCode.apply(null, new Uint8Array(errorData.response.data)), + }, + }); + } + + return new Blob([new Uint8Array(response)]); + } + + async function getCloudStorageStatus(id) { + const { backendAPI } = config; + + let response = null; + try { + const url = `${backendAPI}/cloudstorages/${id}/status`; + response = await Axios.get(url, { + proxy: config.proxy, + }); + } catch (errorData) { + throw generateError(errorData); + } + + return response.data; + } + + async function deleteCloudStorage(id) { + const { backendAPI } = config; + + try { + await Axios.delete(`${backendAPI}/cloudstorages/${id}`); + } catch (errorData) { + throw generateError(errorData); + } + } + Object.defineProperties( this, Object.freeze({ @@ -1310,6 +1423,19 @@ }), writable: false, }, + + cloudStorages: { + value: Object.freeze({ + get: getCloudStorages, + getContent: getCloudStorageContent, + getPreview: getCloudStoragePreview, + getStatus: getCloudStorageStatus, + create: createCloudStorage, + delete: deleteCloudStorage, + update: updateCloudStorage, + }), + writable: false, + }, }), ); } diff --git a/cvat-core/src/session.js b/cvat-core/src/session.js index c5366ea1d5f5..eaba48bc078f 100644 --- a/cvat-core/src/session.js +++ b/cvat-core/src/session.js @@ -1012,6 +1012,7 @@ use_cache: undefined, copy_data: undefined, dimension: undefined, + cloud_storage_id: undefined, }; const updatedFields = new FieldUpdateTrigger({ @@ -1373,7 +1374,7 @@ get: () => [...data.jobs], }, /** - * List of files from shared resource + * List of files from shared resource or list of cloud storage files * @name serverFiles * @type {string[]} * @memberof module:API.cvat.classes.Task @@ -1535,6 +1536,15 @@ */ get: () => data.dimension, }, + /** + * @name cloudStorageId + * @type {integer|null} + * @memberof module:API.cvat.classes.Task + * @instance + */ + cloudStorageId: { + get: () => data.cloud_storage_id, + }, _internalData: { get: () => data, }, @@ -2062,6 +2072,9 @@ if (typeof this.copyData !== 'undefined') { taskDataSpec.copy_data = this.copyData; } + if (typeof this.cloudStorageId !== 'undefined') { + taskDataSpec.cloud_storage_id = this.cloudStorageId; + } const task = await serverProxy.tasks.createTask(taskSpec, taskDataSpec, onUpdate); return new Task(task); diff --git a/cvat-core/tests/api/cloud-storages.js b/cvat-core/tests/api/cloud-storages.js new file mode 100644 index 000000000000..7b5d1bf4ed0e --- /dev/null +++ b/cvat-core/tests/api/cloud-storages.js @@ -0,0 +1,178 @@ +// Copyright (C) 2021 Intel Corporation +// +// SPDX-License-Identifier: MIT + +// Setup mock for a server +jest.mock('../../src/server-proxy', () => { + const mock = require('../mocks/server-proxy.mock'); + return mock; +}); + +// Initialize api +window.cvat = require('../../src/api'); + +const { CloudStorage } = require('../../src/cloud-storage'); +const { cloudStoragesDummyData } = require('../mocks/dummy-data.mock'); + +describe('Feature: get cloud storages', () => { + test('get all cloud storages', async () => { + const result = await window.cvat.cloudStorages.get(); + expect(Array.isArray(result)).toBeTruthy(); + expect(result).toHaveLength(cloudStoragesDummyData.count); + for (const item of result) { + expect(item).toBeInstanceOf(CloudStorage); + } + }); + + test('get cloud storage by id', async () => { + const result = await window.cvat.cloudStorages.get({ + id: 1, + }); + const cloudStorage = result[0]; + + expect(Array.isArray(result)).toBeTruthy(); + expect(result).toHaveLength(1); + expect(cloudStorage).toBeInstanceOf(CloudStorage); + expect(cloudStorage.id).toBe(1); + expect(cloudStorage.providerType).toBe('AWS_S3_BUCKET'); + expect(cloudStorage.credentialsType).toBe('KEY_SECRET_KEY_PAIR'); + expect(cloudStorage.resourceName).toBe('bucket'); + expect(cloudStorage.displayName).toBe('Demonstration bucket'); + expect(cloudStorage.manifests).toHaveLength(1); + expect(cloudStorage.manifests[0]).toBe('manifest.jsonl'); + expect(cloudStorage.specificAttributes).toBe(''); + expect(cloudStorage.description).toBe('It is first bucket'); + }); + + test('get a cloud storage by an unknown id', async () => { + const result = await window.cvat.cloudStorages.get({ + id: 10, + }); + expect(Array.isArray(result)).toBeTruthy(); + expect(result).toHaveLength(0); + }); + + test('get a cloud storage by an invalid id', async () => { + expect( + window.cvat.cloudStorages.get({ + id: '1', + }), + ).rejects.toThrow(window.cvat.exceptions.ArgumentError); + }); + + test('get cloud storages by filters', async () => { + const filters = new Map([ + ['providerType', 'AWS_S3_BUCKET'], + ['resourceName', 'bucket'], + ['displayName', 'Demonstration bucket'], + ['credentialsType', 'KEY_SECRET_KEY_PAIR'], + ['description', 'It is first bucket'], + ]); + + const result = await window.cvat.cloudStorages.get(Object.fromEntries(filters)); + + const [cloudStorage] = result; + expect(Array.isArray(result)).toBeTruthy(); + expect(result).toHaveLength(1); + expect(cloudStorage).toBeInstanceOf(CloudStorage); + expect(cloudStorage.id).toBe(1); + filters.forEach((value, key) => { + expect(cloudStorage[key]).toBe(value); + }); + }); + + test('get cloud storage by invalid filters', async () => { + expect( + window.cvat.cloudStorages.get({ + unknown: '5', + }), + ).rejects.toThrow(window.cvat.exceptions.ArgumentError); + }); +}); + +describe('Feature: create a cloud storage', () => { + test('create new cloud storage without an id', async () => { + const cloudStorage = new window.cvat.classes.CloudStorage({ + display_name: 'new cloud storage', + provider_type: 'AZURE_CONTAINER', + resource: 'newcontainer', + credentials_type: 'ACCOUNT_NAME_TOKEN_PAIR', + account_name: 'accountname', + session_token: 'x'.repeat(135), + manifests: ['manifest.jsonl'], + }); + + const result = await cloudStorage.save(); + expect(typeof result.id).toBe('number'); + }); +}); + +describe('Feature: update a cloud storage', () => { + test('update cloud storage with some new field values', async () => { + const newValues = new Map([ + ['displayName', 'new display name'], + ['credentialsType', 'ANONYMOUS_ACCESS'], + ['description', 'new description'], + ['specificAttributes', 'region=eu-west-1'], + ]); + + let result = await window.cvat.cloudStorages.get({ + id: 1, + }); + + let [cloudStorage] = result; + + for (const [key, value] of newValues) { + cloudStorage[key] = value; + } + + cloudStorage.save(); + + result = await window.cvat.cloudStorages.get({ + id: 1, + }); + [cloudStorage] = result; + + newValues.forEach((value, key) => { + expect(cloudStorage[key]).toBe(value); + }); + }); + + test('Update manifests in a cloud storage', async () => { + const newManifests = [ + 'sub1/manifest.jsonl', + 'sub2/manifest.jsonl', + ]; + + let result = await window.cvat.cloudStorages.get({ + id: 1, + }); + let [cloudStorage] = result; + + cloudStorage.manifests = newManifests; + cloudStorage.save(); + + result = await window.cvat.cloudStorages.get({ + id: 1, + }); + [cloudStorage] = result; + + expect(cloudStorage.manifests).toEqual(newManifests); + }); +}); + +describe('Feature: delete a cloud storage', () => { + test('delete a cloud storage', async () => { + let result = await window.cvat.cloudStorages.get({ + id: 2, + }); + + await result[0].delete(); + result = await window.cvat.cloudStorages.get({ + id: 2, + }); + + expect(Array.isArray(result)).toBeTruthy(); + expect(result).toHaveLength(0); + }); +}); diff --git a/cvat-core/tests/mocks/dummy-data.mock.js b/cvat-core/tests/mocks/dummy-data.mock.js index 17e886ca5e34..7b67d00ea890 100644 --- a/cvat-core/tests/mocks/dummy-data.mock.js +++ b/cvat-core/tests/mocks/dummy-data.mock.js @@ -2547,6 +2547,56 @@ const frameMetaDummyData = { }, }; +const cloudStoragesDummyData = { + count: 2, + next: null, + previous: null, + results: [ + { + id: 2, + owner: { + url: 'http://localhost:7000/api/v1/users/1', + id: 1, + username: 'maya', + first_name: '', + last_name: '' + }, + manifests: [ + 'manifest.jsonl' + ], + provider_type: 'AZURE_CONTAINER', + resource: 'container', + display_name: 'Demonstration container', + created_date: '2021-09-01T09:29:47.094244Z', + updated_date: '2021-09-01T09:29:47.103264Z', + credentials_type: 'ACCOUNT_NAME_TOKEN_PAIR', + specific_attributes: '', + description: 'It is first container' + }, + { + id: 1, + owner: { + url: 'http://localhost:7000/api/v1/users/1', + id: 1, + username: 'maya', + first_name: '', + last_name: '' + }, + manifests: [ + 'manifest.jsonl' + ], + provider_type: 'AWS_S3_BUCKET', + resource: 'bucket', + display_name: 'Demonstration bucket', + created_date: '2021-08-31T09:03:09.350817Z', + updated_date: '2021-08-31T15:16:21.394773Z', + credentials_type: 'KEY_SECRET_KEY_PAIR', + specific_attributes: '', + description: 'It is first bucket' + } + ] +}; + module.exports = { tasksDummyData, projectsDummyData, @@ -2557,4 +2607,5 @@ module.exports = { jobAnnotationsDummyData, frameMetaDummyData, formatsDummyData, + cloudStoragesDummyData, }; diff --git a/cvat-core/tests/mocks/server-proxy.mock.js b/cvat-core/tests/mocks/server-proxy.mock.js index a5111756cd97..7c9e2e155731 100644 --- a/cvat-core/tests/mocks/server-proxy.mock.js +++ b/cvat-core/tests/mocks/server-proxy.mock.js @@ -12,6 +12,7 @@ const { taskAnnotationsDummyData, jobAnnotationsDummyData, frameMetaDummyData, + cloudStoragesDummyData, } = require('./dummy-data.mock'); function QueryStringToJSON(query, ignoreList = []) { @@ -318,6 +319,63 @@ class ServerProxy { return null; } + async function getCloudStorages(filter = '') { + const queries = QueryStringToJSON(filter); + const result = cloudStoragesDummyData.results.filter((item) => { + for (const key in queries) { + if (Object.prototype.hasOwnProperty.call(queries, key)) { + if (queries[key] !== item[key]) { + return false; + } + } + } + return true; + }); + return result; + } + + async function updateCloudStorage(id, cloudStorageData) { + const cloudStorage = cloudStoragesDummyData.results.find((item) => item.id === id); + if (cloudStorage) { + for (const prop in cloudStorageData) { + if ( + Object.prototype.hasOwnProperty.call(cloudStorageData, prop) + && Object.prototype.hasOwnProperty.call(cloudStorage, prop) + ) { + cloudStorage[prop] = cloudStorageData[prop]; + } + } + } + } + + async function createCloudStorage(cloudStorageData) { + const id = Math.max(...cloudStoragesDummyData.results.map((item) => item.id)) + 1; + cloudStoragesDummyData.results.push({ + id, + provider_type: cloudStorageData.provider_type, + resource: cloudStorageData.resource, + display_name: cloudStorageData.display_name, + credentials_type: cloudStorageData.credentials_type, + specific_attributes: cloudStorageData.specific_attributes, + description: cloudStorageData.description, + owner: 1, + created_date: '2021-09-01T09:29:47.094244+03:00', + updated_date: '2021-09-01T09:29:47.103264+03:00', + }); + + const result = await getCloudStorages(`?id=${id}`); + return result[0]; + } + + async function deleteCloudStorage(id) { + const cloudStorages = cloudStoragesDummyData.results; + const cloudStorageId = cloudStorages.findIndex((item) => item.id === id); + if (cloudStorageId !== -1) { + cloudStorages.splice(cloudStorageId); + } + } + + Object.defineProperties( this, Object.freeze({ @@ -384,6 +442,16 @@ class ServerProxy { getAnnotations, }, }, + + cloudStorages: { + value: Object.freeze({ + get: getCloudStorages, + update: updateCloudStorage, + create: createCloudStorage, + delete: deleteCloudStorage, + }), + writable: false, + }, }), ); } diff --git a/cvat-ui/package-lock.json b/cvat-ui/package-lock.json index 8e9d08769a59..f7934f420b15 100644 --- a/cvat-ui/package-lock.json +++ b/cvat-ui/package-lock.json @@ -54132,9 +54132,9 @@ } }, "react-cookie": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/react-cookie/-/react-cookie-4.1.0.tgz", - "integrity": "sha512-CUq222HHxGPt/XOcQjV/1/5shClAxHlp0pLSbsRfCk/DHgLgREW3TxD0SaZOxaPR4QmTHIEq189zF52c8Vb0lA==", + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/react-cookie/-/react-cookie-4.1.1.tgz", + "integrity": "sha512-ffn7Y7G4bXiFbnE+dKhHhbP+b8I34mH9jqnm8Llhj89zF4nPxPutxHT1suUqMeCEhLDBI7InYwf1tpaSoK5w8A==", "requires": { "@types/hoist-non-react-statics": "^3.0.1", "hoist-non-react-statics": "^3.0.0", diff --git a/cvat-ui/package.json b/cvat-ui/package.json index b7c9181b1e3e..5b347dd50046 100644 --- a/cvat-ui/package.json +++ b/cvat-ui/package.json @@ -83,7 +83,7 @@ "react": "^16.14.0", "react-awesome-query-builder": "^3.0.0", "react-color": "^2.19.3", - "react-cookie": "^4.1.0", + "react-cookie": "^4.1.1", "react-dom": "^16.14.0", "react-moment": "^1.1.1", "react-redux": "^7.2.4", diff --git a/cvat/apps/dataset_manager/bindings.py b/cvat/apps/dataset_manager/bindings.py index 2062e0333fba..fba124b5c887 100644 --- a/cvat/apps/dataset_manager/bindings.py +++ b/cvat/apps/dataset_manager/bindings.py @@ -908,7 +908,7 @@ def _make_image(i, **kwargs): attributes["labels"].append({"label_id": idx, "name": label["name"], "color": label["color"]}) attributes["track_id"] = -1 - dm_item = datumaro.DatasetItem(id=osp.split(frame_data.name)[-1].split('.')[0], + dm_item = datumaro.DatasetItem(id=osp.splitext(osp.split(frame_data.name)[-1])[0], annotations=dm_anno, point_cloud=dm_image[0], related_images=dm_image[1], attributes=attributes) diff --git a/cvat/apps/dataset_manager/formats/mots.py b/cvat/apps/dataset_manager/formats/mots.py index fc2d69edea99..9ba5b2268bce 100644 --- a/cvat/apps/dataset_manager/formats/mots.py +++ b/cvat/apps/dataset_manager/formats/mots.py @@ -46,15 +46,27 @@ def _import(src_file, task_data): root_hint = find_dataset_root(dataset, task_data) + shift = 0 for item in dataset: frame_number = task_data.abs_frame_id( match_dm_item(item, task_data, root_hint=root_hint)) + track_ids = set() + for ann in item.annotations: if ann.type != AnnotationType.polygon: continue track_id = ann.attributes['track_id'] + group_id = track_id + + if track_id in track_ids: + # use negative id for tracks with the same id on the same frame + shift -= 1 + track_id = shift + else: + track_ids.add(track_id) + shape = task_data.TrackedShape( type='polygon', points=ann.points, @@ -65,6 +77,7 @@ def _import(src_file, task_data): frame=frame_number, attributes=[], source='manual', + group=group_id ) # build trajectories as lists of shapes in track dict diff --git a/cvat/apps/dataset_manager/tests/test_formats.py b/cvat/apps/dataset_manager/tests/test_formats.py index 8f7edd2f7450..69c7e34797ea 100644 --- a/cvat/apps/dataset_manager/tests/test_formats.py +++ b/cvat/apps/dataset_manager/tests/test_formats.py @@ -3,22 +3,27 @@ # # SPDX-License-Identifier: MIT - -from io import BytesIO +import numpy as np import os.path as osp import tempfile import zipfile +from io import BytesIO import datumaro +from datumaro.components.dataset import Dataset, DatasetItem +from datumaro.components.extractor import Mask +from django.contrib.auth.models import Group, User from PIL import Image -from django.contrib.auth.models import User, Group -from rest_framework.test import APITestCase, APIClient + from rest_framework import status +from rest_framework.test import APIClient, APITestCase import cvat.apps.dataset_manager as dm from cvat.apps.dataset_manager.annotation import AnnotationIR -from cvat.apps.dataset_manager.bindings import TaskData, find_dataset_root, CvatTaskDataExtractor +from cvat.apps.dataset_manager.bindings import (CvatTaskDataExtractor, + TaskData, find_dataset_root) from cvat.apps.dataset_manager.task import TaskAnnotation +from cvat.apps.dataset_manager.util import make_zip_archive from cvat.apps.engine.models import Task @@ -501,7 +506,6 @@ def test_frames_outside_are_not_generated(self): self.assertTrue(frame.frame in range(6, 10)) self.assertEqual(i + 1, 4) - class FrameMatchingTest(_DbTestBase): def _generate_task_images(self, paths): # pylint: disable=no-self-use f = BytesIO() @@ -598,9 +602,10 @@ def _generate_custom_annotations(self, annotations, task): self._put_api_v1_task_id_annotations(task["id"], annotations) return annotations - def _generate_task_images(self, count, name="image"): + def _generate_task_images(self, count, name="image", **image_params): images = { - "client_files[%d]" % i: generate_image_file("image_%d.jpg" % i) + "client_files[%d]" % i: generate_image_file("%s_%d.jpg" % (name, i), + **image_params) for i in range(count) } images["image_quality"] = 75 @@ -916,3 +921,36 @@ def test_can_import_annotations_for_image_with_dots_in_filename(self): self.skipTest("Format is disabled") self._test_can_import_annotations(task, format_name) + + def test_can_import_mots_annotations_with_splited_masks(self): + #https://github.com/openvinotoolkit/cvat/issues/3360 + + format_name = 'MOTS PNG 1.0' + source_dataset = Dataset.from_iterable([ + DatasetItem(id='image_0', + annotations=[ + Mask(np.array([[1, 1, 1, 0, 1, 1, 1]] * 5), + label=0, attributes={'track_id': 0}) + ] + ) + ], categories=['label_0']) + + with tempfile.TemporaryDirectory() as temp_dir: + dataset_dir = osp.join(temp_dir, 'dataset') + source_dataset.export(dataset_dir, 'mots_png') + dataset_path = osp.join(temp_dir, 'annotations.zip') + make_zip_archive(dataset_dir, dataset_path) + + images = self._generate_task_images(1, size=(5, 7)) + task = { + 'name': 'test', + "overlap": 0, + "segment_size": 100, + "labels": [{'name': 'label_0'}] + } + task.update() + task = self._create_task(task, images) + + dm.task.import_task_annotations(task['id'], dataset_path, format_name) + self._test_can_import_annotations(task, format_name) + diff --git a/cvat/requirements/base.txt b/cvat/requirements/base.txt index 4f8190ac4794..60569a23661d 100644 --- a/cvat/requirements/base.txt +++ b/cvat/requirements/base.txt @@ -6,7 +6,7 @@ django-cacheops==5.0.1 django-compressor==2.4 django-rq==2.3.2 EasyProcess==0.3 -Pillow==8.3.0 +Pillow==8.3.2 numpy==1.19.5 python-ldap==3.3.1 pytz==2020.1 @@ -14,7 +14,7 @@ pyunpack==0.2.1 rcssmin==1.0.6 redis==3.5.3 rjsmin==1.1.0 -requests==2.24.0 +requests==2.26.0 rq==1.5.1 rq-scheduler==0.10.0 sqlparse==0.3.1 @@ -54,3 +54,4 @@ google-cloud-storage==1.42.0 # of pycocotools and tensorflow 2.4.1 # when pycocotools is installed by wheel in python 3.8+ datumaro==0.1.10.1 --no-binary=datumaro --no-binary=pycocotools +urllib3>=1.26.5 # not directly required, pinned by Snyk to avoid a vulnerability diff --git a/site/content/en/docs/manual/basics/creating_an_annotation_task.md b/site/content/en/docs/manual/basics/creating_an_annotation_task.md index 80a3af83c97f..ebd26bde4fba 100644 --- a/site/content/en/docs/manual/basics/creating_an_annotation_task.md +++ b/site/content/en/docs/manual/basics/creating_an_annotation_task.md @@ -5,246 +5,221 @@ weight: 2 description: 'Instructions on how to create and configure an annotation task.' --- -1. Create an annotation task pressing `Create new task` button on the tasks page or on the project page. - ![](/images/image004.jpg) +Create an annotation task pressing `Create new task` button on the tasks page or on the project page. +![](/images/image004.jpg) -1. Specify parameters of the task: - - ## Basic configuration - - ### Name - - The name of the task to be created. - - ![](/images/image005.jpg) - - ### Projects - - The project that this task will be related with. - - ![](/images/image193.jpg) - - ### Labels - - There are two ways of working with labels (available only if the task is not related to the project): +Specify parameters of the task: + +## Basic configuration + +### Name + +The name of the task to be created. + +![](/images/image005.jpg) + +### Projects + +The project that this task will be related with. + +![](/images/image193.jpg) + +### Labels + +There are two ways of working with labels (available only if the task is not related to the project): - - The `Constructor` is a simple way to add and adjust labels. To add a new label click the `Add label` button. - ![](/images/image123.jpg) +- The `Constructor` is a simple way to add and adjust labels. To add a new label click the `Add label` button. + ![](/images/image123.jpg) - You can set a name of the label in the `Label name` field and choose a color for each label. - - ![](/images/image124.jpg) + You can set a name of the label in the `Label name` field and choose a color for each label. + + ![](/images/image124.jpg) - If necessary you can add an attribute and set its properties by clicking `Add an attribute`: + If necessary you can add an attribute and set its properties by clicking `Add an attribute`: - ![](/images/image125.jpg) + ![](/images/image125.jpg) - The following actions are available here: + The following actions are available here: - 1. Set the attribute’s name. - 1. Choose the way to display the attribute: - - Select — drop down list of value - - Radio — is used when it is necessary to choose just one option out of few suggested. - - Checkbox — is used when it is necessary to choose any number of options out of suggested. - - Text — is used when an attribute is entered as a text. - - Number — is used when an attribute is entered as a number. - 1. Set values for the attribute. The values could be separated by pressing `Enter`. - The entered value is displayed as a separate element which could be deleted - by pressing `Backspace` or clicking the close button (x). - If the specified way of displaying the attribute is Text or Number, - the entered value will be displayed as text by default (e.g. you can specify the text format). - 1. Checkbox `Mutable` determines if an attribute would be changed frame to frame. - 1. You can delete the attribute by clicking the close button (x). - - Click the `Continue` button to add more labels. - If you need to cancel adding a label - press the `Cancel` button. - After all the necessary labels are added click the `Done` button. - After clicking `Done` the added labels would be displayed as separate elements of different colour. - You can edit or delete labels by clicking `Update attributes` or `Delete label`. - - - The `Raw` is a way of working with labels for an advanced user. - Raw presents label data in _json_ format with an option of editing and copying labels as a text. - The `Done` button applies the changes and the `Reset` button cancels the changes. - ![](/images/image126.jpg) - - In `Raw` and `Constructor` mode, you can press the `Copy` button to copy the list of labels. - - ### Select files - - Press tab `My computer` to choose some files for annotation from your PC. - If you select tab `Connected file share` you can choose files for annotation from your network. - If you select ` Remote source` , you'll see a field where you can enter a list of URLs (one URL per line). - If you upload a video or dataset with images and select `Use cache` option, you can attach a `manifest.jsonl` file. - You can find how to prepare it [here](/docs/manual/advanced/dataset_manifest/). - - ![](/images/image127.jpg) - - ### Data formats for a 3D task - - To create a 3D task, you need to use the following directory structures: - {{< tabpane >}} - {{< tab header="Velodyne" >}} - VELODYNE FORMAT - Structure: - velodyne_points/ - data/ - image_01.bin - IMAGE_00 # unknown dirname, Generally image_01.png can be under IMAGE_00, IMAGE_01, IMAGE_02, IMAGE_03, etc - data/ - image_01.png - {{< /tab >}} - {{< tab header="3D pointcloud" >}} - 3D POINTCLOUD DATA FORMAT - Structure: - pointcloud/ - 00001.pcd - related_images/ - 00001_pcd/ - image_01.png # or any other image - {{< /tab >}} - {{< tab header="3D Option 1" >}} - 3D, DEFAULT DATAFORMAT Option 1 - Structure: - data/ - image.pcd - image.png - {{< /tab >}} - {{< tab header="3D Option 2" >}} - 3D, DEFAULT DATAFORMAT Option 2 - Structure: - data/ - image_1/ - image_1.pcd - context_1.png # or any other name - context_2.jpg - {{< /tab >}} - {{< /tabpane >}} + 1. Set the attribute’s name. + 1. Choose the way to display the attribute: + - Select — drop down list of value + - Radio — is used when it is necessary to choose just one option out of few suggested. + - Checkbox — is used when it is necessary to choose any number of options out of suggested. + - Text — is used when an attribute is entered as a text. + - Number — is used when an attribute is entered as a number. + 1. Set values for the attribute. The values could be separated by pressing `Enter`. + The entered value is displayed as a separate element which could be deleted + by pressing `Backspace` or clicking the close button (x). + If the specified way of displaying the attribute is Text or Number, + the entered value will be displayed as text by default (e.g. you can specify the text format). + 1. Checkbox `Mutable` determines if an attribute would be changed frame to frame. + 1. You can delete the attribute by clicking the close button (x). + + Click the `Continue` button to add more labels. + If you need to cancel adding a label - press the `Cancel` button. + After all the necessary labels are added click the `Done` button. + After clicking `Done` the added labels would be displayed as separate elements of different colour. + You can edit or delete labels by clicking `Update attributes` or `Delete label`. + +- The `Raw` is a way of working with labels for an advanced user. + Raw presents label data in _json_ format with an option of editing and copying labels as a text. + The `Done` button applies the changes and the `Reset` button cancels the changes. + ![](/images/image126.jpg) + +In `Raw` and `Constructor` mode, you can press the `Copy` button to copy the list of labels. + +### Select files + +Press tab `My computer` to choose some files for annotation from your PC. +If you select tab `Connected file share` you can choose files for annotation from your network. +If you select ` Remote source` , you'll see a field where you can enter a list of URLs (one URL per line). +If you upload a video or dataset with images and select `Use cache` option, you can attach a `manifest.jsonl` file. +You can find how to prepare it [here](/docs/manual/advanced/dataset_manifest/). + +![](/images/image127.jpg) + +### Data formats for a 3D task + +To create a 3D task, you need to use the following directory structures: +{{< tabpane >}} + {{< tab header="Velodyne" >}} + VELODYNE FORMAT + Structure: + velodyne_points/ + data/ + image_01.bin + IMAGE_00 # unknown dirname, Generally image_01.png can be under IMAGE_00, IMAGE_01, IMAGE_02, IMAGE_03, etc + data/ + image_01.png + {{< /tab >}} + {{< tab header="3D pointcloud" >}} + 3D POINTCLOUD DATA FORMAT + Structure: + pointcloud/ + 00001.pcd + related_images/ + 00001_pcd/ + image_01.png # or any other image + {{< /tab >}} + {{< tab header="3D Option 1" >}} + 3D, DEFAULT DATAFORMAT Option 1 + Structure: + data/ + image.pcd + image.png + {{< /tab >}} + {{< tab header="3D Option 2" >}} + 3D, DEFAULT DATAFORMAT Option 2 + Structure: + data/ + image_1/ + image_1.pcd + context_1.png # or any other name + context_2.jpg + {{< /tab >}} +{{< /tabpane >}} - ## Advanced configuration +## Advanced configuration - ![](/images/image128_use_cache.jpg) +![](/images/image128_use_cache.jpg) - ### Use zip chunks +### Use zip chunks - Force to use zip chunks as compressed data. Actual for videos only. +Force to use zip chunks as compressed data. Actual for videos only. + +### Use cache + +Defines how to work with data. Select the checkbox to switch to the "on-the-fly data processing", +which will reduce the task creation time (by preparing chunks when requests are received) +and store data in a cache of limited size with a policy of evicting less popular items. +See more [here](/docs/manual/advanced/data_on_fly/). + +### Image Quality + +Use this option to specify quality of uploaded images. +The option helps to load high resolution datasets faster. +Use the value from `5` (almost completely compressed images) to `100` (not compressed images). + +## Overlap Size + +Use this option to make overlapped segments. +The option makes tracks continuous from one segment into another. +Use it for interpolation mode. There are several options for using the parameter: + +- For an interpolation task (video sequence). + If you annotate a bounding box on two adjacent segments they will be merged into one bounding box. + If overlap equals to zero or annotation is poor on adjacent segments inside a dumped annotation file, + you will have several tracks, one for each segment, which corresponds to the object. +- For an annotation task (independent images). + If an object exists on overlapped segments, the overlap is greater than zero + and the annotation is good enough on adjacent segments, it will be automatically merged into one object. + If overlap equals to zero or annotation is poor on adjacent segments inside a dumped annotation file, + you will have several bounding boxes for the same object. + Thus, you annotate an object on the first segment. + You annotate the same object on second segment, and if you do it right, you + will have one track inside the annotations. + If annotations on different segments (on overlapped frames) + are very different, you will have two shapes for the same object. + This functionality works only for bounding boxes. + Polygons, polylines, points don't support automatic merge on overlapped segments + even the overlap parameter isn't zero and match between corresponding shapes on adjacent segments is perfect. - ### Use cache +### Segment size - Defines how to work with data. Select the checkbox to switch to the "on-the-fly data processing", - which will reduce the task creation time (by preparing chunks when requests are received) - and store data in a cache of limited size with a policy of evicting less popular items. - See more [here](/docs/manual/advanced/data_on_fly/). - - ### Image Quality - - Use this option to specify quality of uploaded images. - The option helps to load high resolution datasets faster. - Use the value from `5` (almost completely compressed images) to `100` (not compressed images). - - ## Overlap Size - - Use this option to make overlapped segments. - The option makes tracks continuous from one segment into another. - Use it for interpolation mode. There are several options for using the parameter: - - - For an interpolation task (video sequence). - If you annotate a bounding box on two adjacent segments they will be merged into one bounding box. - If overlap equals to zero or annotation is poor on adjacent segments inside a dumped annotation file, - you will have several tracks, one for each segment, which corresponds to the object. - - For an annotation task (independent images). - If an object exists on overlapped segments, the overlap is greater than zero - and the annotation is good enough on adjacent segments, it will be automatically merged into one object. - If overlap equals to zero or annotation is poor on adjacent segments inside a dumped annotation file, - you will have several bounding boxes for the same object. - Thus, you annotate an object on the first segment. - You annotate the same object on second segment, and if you do it right, you - will have one track inside the annotations. - If annotations on different segments (on overlapped frames) - are very different, you will have two shapes for the same object. - This functionality works only for bounding boxes. - Polygons, polylines, points don't support automatic merge on overlapped segments - even the overlap parameter isn't zero and match between corresponding shapes on adjacent segments is perfect. - - ### Segment size - - Use this option to divide a huge dataset into a few smaller segments. - For example, one job cannot be annotated by several labelers (it isn't supported). - Thus using "segment size" you can create several jobs for the same annotation task. - It will help you to parallel data annotation process. - - ### Start frame - - Frame from which video in task begins. - - ### Stop frame - - Frame on which video in task ends. - - ### Frame Step - - Use this option to filter video frames. - For example, enter `25` to leave every twenty fifth frame in the video or every twenty fifth image. - - ### Chunk size - - Defines a number of frames to be packed in a chunk when send from client to server. - Server defines automatically if empty. - - Recommended values: - - - 1080p or less: 36 - - 2k or less: 8 - 16 - - 4k or less: 4 - 8 - - More: 1 - 4 +Use this option to divide a huge dataset into a few smaller segments. +For example, one job cannot be annotated by several labelers (it isn't supported). +Thus using "segment size" you can create several jobs for the same annotation task. +It will help you to parallel data annotation process. + +### Start frame + +Frame from which video in task begins. + +### Stop frame + +Frame on which video in task ends. + +### Frame Step + +Use this option to filter video frames. +For example, enter `25` to leave every twenty fifth frame in the video or every twenty fifth image. + +### Chunk size + +Defines a number of frames to be packed in a chunk when send from client to server. +Server defines automatically if empty. - ### Dataset Repository +Recommended values: - URL link of the repository optionally specifies the path to the repository for storage - (`default: annotation / .zip`). - The .zip and .xml file extension of annotation are supported. - Field format: `URL [PATH]` example: `https://github.com/project/repos.git [1/2/3/4/annotation.xml]` +- 1080p or less: 36 +- 2k or less: 8 - 16 +- 4k or less: 4 - 8 +- More: 1 - 4 - Supported URL formats : +### Dataset Repository - - `https://github.com/project/repos[.git]` - - `github.com/project/repos[.git]` - - `git@github.com:project/repos[.git]` +URL link of the repository optionally specifies the path to the repository for storage +(`default: annotation / .zip`). +The .zip and .xml file extension of annotation are supported. +Field format: `URL [PATH]` example: `https://github.com/project/repos.git [1/2/3/4/annotation.xml]` - The task will be highlighted in red after creation if annotation isn't synchronized with the repository. +Supported URL formats : - ### Use LFS +- `https://github.com/project/repos[.git]` +- `github.com/project/repos[.git]` +- `git@github.com:project/repos[.git]` - If the annotation file is large, you can create a repository with - [LFS](https://git-lfs.github.com/) support. +After the task is created, the synchronization status is displayed on the task page. - ### Issue tracker +### Use LFS - Specify full issue tracker's URL if it's necessary. +If the annotation file is large, you can create a repository with +[LFS](https://git-lfs.github.com/) support. - Push `Submit` button and it will be added into the list of annotation tasks. - Then, the created task will be displayed on a tasks page: +### Issue tracker - ![](/images/image006_detrac.jpg) +Specify full issue tracker's URL if it's necessary. -1. The tasks page contains elements and each of them relates to a separate task. They are sorted in creation order. - Each element contains: task name, preview, progress bar, button `Open`, and menu `Actions`. - Each button is responsible for a in menu `Actions` specific function: - - - `Export task dataset` — download annotations or annotations and images in a specific format. - More information is available in the [Downloading annotations](/docs/manual/advanced/downloading-annotations/) - section. - - `Upload annotation` is available in the same formats as in `Export task dataset`. The - [CVAT](/docs/manual/advanced/xml_format/) format accepts both video and image sub-formats. - - `Automatic Annotation` — automatic annotation with OpenVINO toolkit. - Presence depends on how you build CVAT instance. - - `Export task` — Export a task into a zip archive. - Read more in the [export/import a task](/docs/manual/advanced/export-import/) section. - - `Move to project` — Moving the task to the project (can be used to move a task from one project to another). - Note that attributes reset during the moving process. In case of label mismatch, - you can create or delete necessary labels in the project/task. - Some task labels can be matched with the target project labels. - - `Delete` — delete task. - ---- - -Push `Open` button to go to [task details](/docs/manual/basics/task-details/). +Push `Submit` button and it will be added into the list of annotation tasks. +Then, the created task will be displayed on a [tasks page](/docs/manual/basics/tasks-page/). diff --git a/site/content/en/docs/manual/basics/tasks-page.md b/site/content/en/docs/manual/basics/tasks-page.md new file mode 100644 index 000000000000..4aabbdf6286d --- /dev/null +++ b/site/content/en/docs/manual/basics/tasks-page.md @@ -0,0 +1,31 @@ +--- +title: 'Tasks page' +linkTitle: 'Tasks page' +weight: 2.5 +description: 'Overview of the Tasks page.' +--- + +![](/images/image006_detrac.jpg) + +The tasks page contains elements and each of them relates to a separate task. They are sorted in creation order. +Each element contains: task name, preview, progress bar, button `Open`, and menu `Actions`. +Each button is responsible for a in menu `Actions` specific function: + +- `Export task dataset` — download annotations or annotations and images in a specific format. + More information is available in the [Downloading annotations](/docs/manual/advanced/downloading-annotations/) + section. +- `Upload annotation` is available in the same formats as in `Export task dataset`. The + [CVAT](/docs/manual/advanced/xml_format/) format accepts both video and image sub-formats. +- `Automatic Annotation` — automatic annotation with OpenVINO toolkit. + Presence depends on how you build CVAT instance. +- `Export task` — Export a task into a zip archive. + Read more in the [export/import a task](/docs/manual/advanced/export-import/) section. +- `Move to project` — Moving the task to the project (can be used to move a task from one project to another). + Note that attributes reset during the moving process. In case of label mismatch, + you can create or delete necessary labels in the project/task. + Some task labels can be matched with the target project labels. +- `Delete` — delete task. + +--- + +Push `Open` button to go to [task details](/docs/manual/basics/task-details/).