From ffa4ccf97e08fafd1c057b2662f55abaf73948af Mon Sep 17 00:00:00 2001 From: shafeeqd959 Date: Thu, 24 Oct 2024 18:35:08 +0530 Subject: [PATCH] asset mapper file creation --- .../src/config/index.ts | 4 +- .../src/import/modules/assets.ts | 121 ++++++++++++ .../src/import/modules/base-setup.ts | 177 +++++++++++++++++- .../src/import/modules/extensions.ts | 5 +- .../src/types/default-config.ts | 2 + .../src/types/index.ts | 44 +++++ 6 files changed, 349 insertions(+), 4 deletions(-) diff --git a/packages/contentstack-import-setup/src/config/index.ts b/packages/contentstack-import-setup/src/config/index.ts index f211dbb22e..a1b0e6cac3 100644 --- a/packages/contentstack-import-setup/src/config/index.ts +++ b/packages/contentstack-import-setup/src/config/index.ts @@ -31,11 +31,12 @@ const config: DefaultConfig = { assets: { dirName: 'assets', fileName: 'assets.json', + fetchConcurrency: 5, }, 'content-types': { dirName: 'content_types', fileName: 'content_types.json', - dependencies: ['extensions'], + dependencies: ['extensions', 'assets'], }, entries: { dirName: 'entries', @@ -56,6 +57,7 @@ const config: DefaultConfig = { fileName: 'taxonomies.json', }, }, + fetchConcurrency: 5, }; export default config; diff --git a/packages/contentstack-import-setup/src/import/modules/assets.ts b/packages/contentstack-import-setup/src/import/modules/assets.ts index e69de29bb2..5a3b0cb803 100644 --- a/packages/contentstack-import-setup/src/import/modules/assets.ts +++ b/packages/contentstack-import-setup/src/import/modules/assets.ts @@ -0,0 +1,121 @@ +import * as chalk from 'chalk'; +import { log, fsUtil } from '../../utils'; +import { join } from 'path'; +import { AssetRecord, ImportConfig, ModuleClassParams } from '../../types'; +import { isEmpty, orderBy, values } from 'lodash'; +import { formatError, FsUtility } from '@contentstack/cli-utilities'; +import BaseImportSetup from './base-setup'; + +export default class AssetImportSetup extends BaseImportSetup { + private assetsFilePath: string; + private assetUidMapper: Record; + private assetUrlMapper: Record; + private duplicateAssets: Record; + private assetsConfig: ImportConfig['modules']['assets']; + private mapperDirPath: string; + private assetsFolderPath: string; + private assetUidMapperPath: string; + private assetUrlMapperPath: string; + private duplicateAssetPath: string; + + constructor({ config, stackAPIClient, dependencies }: ModuleClassParams) { + super({ config, stackAPIClient, dependencies }); + this.assetsFolderPath = join(this.config.contentDir, 'assets'); + this.assetsFilePath = join(this.config.contentDir, 'assets', 'assets.json'); + this.assetsConfig = config.modules.assets; + this.mapperDirPath = join(this.config.backupDir, 'mapper', 'assets'); + this.assetUidMapperPath = join(this.config.backupDir, 'mapper', 'assets', 'uid-mapping.json'); + this.assetUrlMapperPath = join(this.config.backupDir, 'mapper', 'assets', 'url-mapping.json'); + this.duplicateAssetPath = join(this.config.backupDir, 'mapper', 'assets', 'duplicate-assets.json'); + this.assetUidMapper = {}; + this.assetUrlMapper = {}; + this.duplicateAssets = {}; + } + + /** + * Start the asset import setup + * This method reads the assets from the content folder and generates a mapper file + * @returns {Promise} + */ + async start() { + try { + fsUtil.makeDirectory(this.mapperDirPath); + await this.fetchAndMapAssets(); + log(this.config, `Generated required setup files for asset`, 'success'); + } catch (error) { + log(this.config, `Error generating asset mapper: ${formatError(error)}`, 'error'); + } + } + + /** + * @method importAssets + * @param {boolean} isVersion boolean + * @returns {Promise} Promise + */ + async fetchAndMapAssets(): Promise { + const processName = 'mapping assets'; + const indexFileName = 'assets.json'; + const basePath = this.assetsFolderPath; + const fs = new FsUtility({ basePath, indexFileName }); + const indexer = fs.indexFileContent; + const indexerCount = values(indexer).length; + + const onSuccess = ({ + response: { items = [] as AssetRecord[] } = {}, + apiData: { uid, url, title } = undefined, + }: any) => { + if (items.length === 1) { + this.assetUidMapper[uid] = items[0].uid; + this.assetUrlMapper[url] = items[0].url; + log(this.config, `Mapped asset: '${title}'`, 'info'); + } else if (items.length > 1) { + this.duplicateAssets[uid] = items.map((asset: any) => { + return { uid: asset.uid, title: asset.title, url: asset.url }; + }); + log(this.config, `Multiple assets found with title '${title}'`, 'error'); + } else { + log(this.config, `Asset with title '${title}' not found in the stack!`, 'error'); + } + }; + const onReject = ({ error, apiData: { title } = undefined }: any) => { + log(this.config, `${title} asset mapping failed.!`, 'error'); + log(this.config, formatError(error), 'error'); + }; + + /* eslint-disable @typescript-eslint/no-unused-vars, guard-for-in */ + for (const index in indexer) { + const chunk = await fs.readChunkFiles.next().catch((error) => { + log(this.config, error, 'error'); + }); + + if (chunk) { + let apiContent = orderBy(values(chunk as Record[]), '_version'); + + await this.makeConcurrentCall( + { + apiContent, + processName, + indexerCount, + currentIndexer: +index, + apiParams: { + reject: onReject, + resolve: onSuccess, + entity: 'fetch-assets', + includeParamOnCompletion: true, + }, + concurrencyLimit: this.assetsConfig.fetchConcurrency, + }, + undefined, + ); + } + } + + if (!isEmpty(this.assetUidMapper) || !isEmpty(this.assetUrlMapper)) { + fsUtil.writeFile(this.assetUidMapperPath, this.assetUidMapper); + fsUtil.writeFile(this.assetUrlMapperPath, this.assetUrlMapper); + } + if (!isEmpty(this.duplicateAssets)) { + fsUtil.writeFile(this.duplicateAssetPath, this.duplicateAssets); + } + } +} diff --git a/packages/contentstack-import-setup/src/import/modules/base-setup.ts b/packages/contentstack-import-setup/src/import/modules/base-setup.ts index 3e2aa3c864..5385a0ea49 100644 --- a/packages/contentstack-import-setup/src/import/modules/base-setup.ts +++ b/packages/contentstack-import-setup/src/import/modules/base-setup.ts @@ -1,5 +1,6 @@ import { log, fsUtil } from '../../utils'; -import { ImportConfig, ModuleClassParams } from '../../types'; +import { ApiOptions, CustomPromiseHandler, EnvType, ImportConfig, ModuleClassParams } from '../../types'; +import { chunk, entries, isEmpty, isEqual, last } from 'lodash'; export default class BaseImportSetup { public config: ImportConfig; @@ -30,4 +31,178 @@ export default class BaseImportSetup { } } } + + /** + * @method delay + * @param {number} ms number + * @returns {Promise} Promise + */ + delay(ms: number): Promise { + /* eslint-disable no-promise-executor-return */ + return new Promise((resolve) => setTimeout(resolve, ms <= 0 ? 0 : ms)); + } + + /** + * @method makeConcurrentCall + * @param {Record} env EnvType + * @param {CustomPromiseHandler} promisifyHandler CustomPromiseHandler + * @param {boolean} logBatchCompletionMsg boolean + * @returns {Promise} Promise + */ + makeConcurrentCall( + env: EnvType, + promisifyHandler?: CustomPromiseHandler, + logBatchCompletionMsg = true, + ): Promise { + const { + apiParams, + apiContent, + processName, + indexerCount, + currentIndexer, + concurrencyLimit = this.config.fetchConcurrency, + } = env; + + /* eslint-disable no-async-promise-executor */ + return new Promise(async (resolve) => { + let batchNo = 0; + let isLastRequest = false; + const batches: Array> = chunk(apiContent, concurrencyLimit); + + /* eslint-disable no-promise-executor-return */ + if (isEmpty(batches)) return resolve(); + + for (const [batchIndex, batch] of entries(batches)) { + batchNo += 1; + const allPromise = []; + const start = Date.now(); + + for (const [index, element] of entries(batch)) { + let promise = Promise.resolve(); + isLastRequest = isEqual(last(batch as ArrayLike), element) && isEqual(last(batches), batch); + + if (promisifyHandler instanceof Function) { + promise = promisifyHandler({ + apiParams, + isLastRequest, + element, + index: Number(index), + batchIndex: Number(batchIndex), + }); + } else if (apiParams) { + apiParams.apiData = element; + promise = this.makeAPICall(apiParams, isLastRequest); + } + + allPromise.push(promise); + } + + /* eslint-disable no-await-in-loop */ + await Promise.allSettled(allPromise); + + /* eslint-disable no-await-in-loop */ + await this.logMsgAndWaitIfRequired( + processName, + start, + batches.length, + batchNo, + logBatchCompletionMsg, + indexerCount, + currentIndexer, + ); + + if (isLastRequest) resolve(); + } + }); + } + + /** + * @method logMsgAndWaitIfRequired + * @param {string} processName string + * @param {number} start number + * @param {number} batchNo - number + * @returns {Promise} Promise + */ + async logMsgAndWaitIfRequired( + processName: string, + start: number, + totelBatches: number, + batchNo: number, + logBatchCompletionMsg = true, + indexerCount?: number, + currentIndexer?: number, + ): Promise { + const end = Date.now(); + const exeTime = end - start; + + if (logBatchCompletionMsg) { + let batchMsg = ''; + // info: Batch No. 20 of import assets is complete + if (currentIndexer) batchMsg += `Current chunk processing is (${currentIndexer}/${indexerCount})`; + + log(this.config, `Batch No. (${batchNo}/${totelBatches}) of ${processName} is complete`, 'success'); + } + + // if (this.config.modules.assets.displayExecutionTime) { + // console.log( + // `Time taken to execute: ${exeTime} milliseconds; wait time: ${ + // exeTime < 1000 ? 1000 - exeTime : 0 + // } milliseconds`, + // ); + // } + + if (exeTime < 1000) await this.delay(1000 - exeTime); + } + + /** + * @method makeAPICall + * @param {Record} apiOptions - Api related params + * @param {Record} isLastRequest - Boolean + * @return {Promise} Promise + */ + makeAPICall(apiOptions: ApiOptions, isLastRequest = false): Promise { + if (apiOptions.serializeData instanceof Function) { + apiOptions = apiOptions.serializeData(apiOptions); + } + + const { uid, entity, reject, resolve, apiData, additionalInfo = {}, includeParamOnCompletion } = apiOptions; + + const onSuccess = (response: any) => + resolve({ + response, + isLastRequest, + additionalInfo, + apiData: includeParamOnCompletion ? apiData : undefined, + }); + const onReject = (error: Error) => + reject({ + error, + isLastRequest, + additionalInfo, + apiData: includeParamOnCompletion ? apiData : undefined, + }); + + if (!apiData) { + return Promise.resolve(); + } + switch (entity) { + case 'fetch-assets': + return this.stackAPIClient + .asset() + .query({ + query: { + $and: [ + { file_size: Number(apiData.file_size) }, + { filename: apiData.filename }, + { title: apiData.title }, + ], + }, + }) + .find() + .then(onSuccess) + .catch(onReject); + default: + return Promise.resolve(); + } + } } diff --git a/packages/contentstack-import-setup/src/import/modules/extensions.ts b/packages/contentstack-import-setup/src/import/modules/extensions.ts index f9e85445e2..68aaabc47c 100644 --- a/packages/contentstack-import-setup/src/import/modules/extensions.ts +++ b/packages/contentstack-import-setup/src/import/modules/extensions.ts @@ -3,6 +3,7 @@ import { log, fsUtil } from '../../utils'; import { join } from 'path'; import { ImportConfig, ModuleClassParams } from '../../types'; import { isEmpty } from 'lodash'; +import { formatError } from '@contentstack/cli-utilities'; export default class ExtensionImportSetup { private config: ImportConfig; @@ -48,12 +49,12 @@ export default class ExtensionImportSetup { await fsUtil.writeFile(this.extUidMapperPath, this.extensionMapper); - log(this.config, `Generate required setup files for extension`, 'success'); + log(this.config, `Generated required setup files for extension`, 'success'); } else { log(this.config, 'No extensions found in the content folder!', 'error'); } } catch (error) { - log(this.config, `Error generating extension mapper: ${error.message}`, 'error'); + log(this.config, `Error generating extension mapper: ${formatError(error)}`, 'error'); } } diff --git a/packages/contentstack-import-setup/src/types/default-config.ts b/packages/contentstack-import-setup/src/types/default-config.ts index d0d1fa57f3..bce1ffee03 100644 --- a/packages/contentstack-import-setup/src/types/default-config.ts +++ b/packages/contentstack-import-setup/src/types/default-config.ts @@ -22,6 +22,7 @@ export default interface DefaultConfig { dirName: string; fileName: string; dependencies?: Modules[]; + fetchConcurrency: number; }; 'content-types': { dirName: string; @@ -49,4 +50,5 @@ export default interface DefaultConfig { dependencies?: Modules[]; }; }; + fetchConcurrency: number; } diff --git a/packages/contentstack-import-setup/src/types/index.ts b/packages/contentstack-import-setup/src/types/index.ts index 984a661dbc..468c2ddbe7 100644 --- a/packages/contentstack-import-setup/src/types/index.ts +++ b/packages/contentstack-import-setup/src/types/index.ts @@ -19,6 +19,44 @@ export interface PrintOptions { color?: string; } +export type AdditionalKeys = { + backupDir: string; +}; + +export type ApiModuleType = 'fetch-assets'; + +export type ApiOptions = { + uid?: string; + url?: string; + entity: ApiModuleType; + apiData?: Record | any; + resolve: (value: any) => Promise | void; + reject: (error: any) => Promise | void; + additionalInfo?: Record; + includeParamOnCompletion?: boolean; + serializeData?: (input: ApiOptions) => any; +}; + +export type EnvType = { + processName: string; + totalCount?: number; + indexerCount?: number; + currentIndexer?: number; + apiParams?: ApiOptions; + concurrencyLimit?: number; + apiContent: Record[]; +}; + +export type CustomPromiseHandlerInput = { + index: number; + batchIndex: number; + element?: Record; + apiParams?: ApiOptions; + isLastRequest: boolean; +}; + +export type CustomPromiseHandler = (input: CustomPromiseHandlerInput) => Promise; + export interface InquirePayload { type: string; name: string; @@ -87,6 +125,12 @@ export interface CustomRoleConfig { customRolesLocalesFileName: string; } +export type AssetRecord = { + uid: string; + url: string; + title: string; +}; + export interface TaxonomiesConfig { dirName: string; fileName: string;