diff --git a/.talismanrc b/.talismanrc index 86b85bda..20cce845 100644 --- a/.talismanrc +++ b/.talismanrc @@ -1,26 +1,24 @@ fileignoreconfig: - - filename: packages/contentstack-audit/test/unit/mock/contents/environments/environments.json - checksum: d983bc17ab56937c66a6d25f449ecbe285d00c807923ed56deacb3a571aa3448 - - filename: package-lock.json - checksum: 8fd93d7b01c7d064fa797fba81d09da612b5639468c02003e11b2f3b599e49a2 - - filename: pnpm-lock.yaml - checksum: 458cb8d135905583080023055430cf8344f46e92dd1c9e90cdb6f78bbd02eecb - - filename: packages/contentstack-seed/src/commands/cm/stacks/seed.ts - checksum: 5c59296f3d5ba078f16bca23a47c920dc2180cff3b8250a341176185a4dabc39 - - filename: packages/contentstack-seed/README.md - checksum: d2a017a8206aae1058d4a91d445a7f7d50e919a0d2dd69605a66529c4f4ebe2e - - filename: packages/contentstack-seed/src/seed/github/client.ts - checksum: 44d491ab5253ebb6c24bb0ac5c8d985320bdc4cc3711de77adfe79f7fb1874a1 - - filename: packages/contentstack-seed/test/commands/cm/stacks/seed.test.ts - checksum: 999b1afe970452691318c76d5e9abd8852384fcf2d826cecda19f156de75fb59 - - filename: packages/contentstack-seed/src/seed/index.ts - checksum: 2bd73b2562618a37e02247459141432515471277ee5b85f5053f169891a54eb5 - - filename: packages/contentstack-export-to-csv/src/utils/interactive.ts - checksum: 8aa3870a6694e404f4f8df3ed884dd69521099fe66851c4f8c9860276254da9d - - filename: packages/contentstack-import/test/unit/import/modules/marketplace-apps.test.ts - checksum: e927e670f70374bfa09a4866faf2af0a65476709412882122ea2811717e528aa - - filename: packages/contentstack-export/test/unit/export/modules/marketplace-apps.test.ts - checksum: 6cb665ee2ea09372b3f80c3d2c38d1c3c3889ce0a1ba7d8488614a73aedf17b7 - - filename: packages/contentstack-import/test/unit/import/module-importer.test.ts - checksum: baf0ffc77d2afe9084da2d1fecac4a3a6ef875739677ef6186cd7de278886406 + - filename: packages/contentstack-export/src/export/modules/assets.ts + checksum: 1eacc8e86cb50fe283febe6688965854f420e02cf1b49555a15661fa0c3e3c7a + - filename: packages/contentstack-import/src/utils/import-config-handler.ts + checksum: f831cef1b7c3bd97bdbc170cff452350cee0f448d97df02e25aa41d6c4d64ad3 + - filename: packages/contentstack-asset-management/src/import/asset-types.ts + checksum: a39caa373b2a736d1e57063326cfb2073ae78376efa931b27d2c7110997708a5 + - filename: packages/contentstack-asset-management/src/export/spaces.ts + checksum: 3ec11c8f710b60ae495c69344025587df2e6195c872a0c82feaf04ac044ecefa + - filename: packages/contentstack-import/src/import/modules/assets.ts + checksum: d9f4a29a29e8b8a2a36e498f2380d39e1c5c0ec13ff894ef450abd817f2a646e + - filename: packages/contentstack-asset-management/src/import/fields.ts + checksum: bbae69c28ec69bf67c2c7b4df3620380ef3fca488b3288e137b65a60ee738b9e + - filename: packages/contentstack-asset-management/src/import/spaces.ts + checksum: 79cf2f1b55523d28c218d970155f887255a00dc095a941556b709d1f19c6a8a0 + - filename: packages/contentstack-asset-management/src/types/asset-management-api.ts + checksum: 716df03dcba70b2cc0f77b1f6338524553ba740080d7087a8699147c3ce8f0ba + - filename: packages/contentstack-asset-management/src/utils/asset-management-api-adapter.ts + checksum: 92bcad2feabc1954ead89b370d284b7af5f38ec1dca60a41752371977ef106ff + - filename: packages/contentstack-asset-management/src/import/base.ts + checksum: 513b55cf7e92bdbe8b815141822ba10579b6a728bec4424fc664794246ad33bb + - filename: packages/contentstack-asset-management/src/import/assets.ts + checksum: 2d34fa57f5ab269f6c535dff3242cc1135dbe1decd84fa0bc8997d0410d520b2 version: '1.0' diff --git a/packages/contentstack-asset-management/src/constants/index.ts b/packages/contentstack-asset-management/src/constants/index.ts index 96ff8490..6763629d 100644 --- a/packages/contentstack-asset-management/src/constants/index.ts +++ b/packages/contentstack-asset-management/src/constants/index.ts @@ -17,10 +17,15 @@ export const PROCESS_NAMES = { AM_FIELDS: 'Fields', AM_ASSET_TYPES: 'Asset types', AM_DOWNLOADS: 'Asset downloads', + // Import process names + AM_IMPORT_FIELDS: 'Import fields', + AM_IMPORT_ASSET_TYPES: 'Import asset types', + AM_IMPORT_FOLDERS: 'Import folders', + AM_IMPORT_ASSETS: 'Import assets', } as const; /** - * Status messages for each process (exporting, fetching, failed). + * Status messages for each process (exporting, fetching, importing, failed). */ export const PROCESS_STATUS = { [PROCESS_NAMES.AM_SPACE_METADATA]: { @@ -47,4 +52,20 @@ export const PROCESS_STATUS = { DOWNLOADING: 'Downloading asset files...', FAILED: 'Failed to download assets.', }, + [PROCESS_NAMES.AM_IMPORT_FIELDS]: { + IMPORTING: 'Importing shared fields...', + FAILED: 'Failed to import fields.', + }, + [PROCESS_NAMES.AM_IMPORT_ASSET_TYPES]: { + IMPORTING: 'Importing shared asset types...', + FAILED: 'Failed to import asset types.', + }, + [PROCESS_NAMES.AM_IMPORT_FOLDERS]: { + IMPORTING: 'Importing folders...', + FAILED: 'Failed to import folders.', + }, + [PROCESS_NAMES.AM_IMPORT_ASSETS]: { + IMPORTING: 'Importing assets...', + FAILED: 'Failed to import assets.', + }, } as const; diff --git a/packages/contentstack-asset-management/src/export/assets.ts b/packages/contentstack-asset-management/src/export/assets.ts index 140b5664..28016691 100644 --- a/packages/contentstack-asset-management/src/export/assets.ts +++ b/packages/contentstack-asset-management/src/export/assets.ts @@ -21,8 +21,8 @@ export default class ExportAssets extends AssetManagementExportAdapter { log.debug(`Fetching folders and assets for space ${workspace.space_uid}`, this.exportContext.context); const [folders, assetsData] = await Promise.all([ - this.getWorkspaceFolders(workspace.space_uid), - this.getWorkspaceAssets(workspace.space_uid), + this.getWorkspaceFolders(workspace.space_uid, workspace.uid), + this.getWorkspaceAssets(workspace.space_uid, workspace.uid), ]); await writeFile(pResolve(assetsDir, 'folders.json'), JSON.stringify(folders, null, 2)); diff --git a/packages/contentstack-asset-management/src/export/spaces.ts b/packages/contentstack-asset-management/src/export/spaces.ts index cf3ff2c3..a5abd54a 100644 --- a/packages/contentstack-asset-management/src/export/spaces.ts +++ b/packages/contentstack-asset-management/src/export/spaces.ts @@ -29,15 +29,8 @@ export class ExportSpaces { } async start(): Promise { - const { - linkedWorkspaces, - exportDir, - branchName, - assetManagementUrl, - org_uid, - context, - securedAssets, - } = this.options; + const { linkedWorkspaces, exportDir, branchName, assetManagementUrl, org_uid, apiKey, context, securedAssets } = + this.options; if (!linkedWorkspaces.length) { log.debug('No linked workspaces to export', context); @@ -54,7 +47,9 @@ export class ExportSpaces { const totalSteps = 2 + linkedWorkspaces.length * 4; const progress = this.createProgress(); progress.addProcess(AM_MAIN_PROCESS_NAME, totalSteps); - progress.startProcess(AM_MAIN_PROCESS_NAME).updateStatus(PROCESS_STATUS[PROCESS_NAMES.AM_FIELDS].FETCHING, AM_MAIN_PROCESS_NAME); + progress + .startProcess(AM_MAIN_PROCESS_NAME) + .updateStatus(PROCESS_STATUS[PROCESS_NAMES.AM_FIELDS].FETCHING, AM_MAIN_PROCESS_NAME); const apiConfig: AssetManagementAPIConfig = { baseURL: assetManagementUrl, diff --git a/packages/contentstack-asset-management/src/import/asset-types.ts b/packages/contentstack-asset-management/src/import/asset-types.ts new file mode 100644 index 00000000..9305657c --- /dev/null +++ b/packages/contentstack-asset-management/src/import/asset-types.ts @@ -0,0 +1,88 @@ +import omit from 'lodash/omit'; +import isEqual from 'lodash/isEqual'; +import { log } from '@contentstack/cli-utilities'; + +import type { AssetManagementAPIConfig, ImportContext } from '../types/asset-management-api'; +import { AssetManagementImportAdapter } from './base'; +import { PROCESS_NAMES, PROCESS_STATUS } from '../constants/index'; + +const STRIP_KEYS = ['created_at', 'created_by', 'updated_at', 'updated_by', 'is_system', 'category', 'preview_image_url', 'category_detail']; + +/** + * Reads shared asset types from `spaces/asset_types/asset-types.json` and POSTs + * each to the target org-level AM endpoint (`POST /api/asset_types`). + * + * Strategy: Fetch → Diff → Create only missing, warn on conflict + * 1. Fetch asset types that already exist in the target org. + * 2. Skip entries where is_system=true (platform-owned, cannot be created via API). + * 3. If uid already exists and definition differs → warn and skip. + * 4. If uid already exists and definition matches → silently skip. + * 5. Strip read-only/computed keys from the POST body before creating new asset types. + */ +export default class ImportAssetTypes extends AssetManagementImportAdapter { + constructor(apiConfig: AssetManagementAPIConfig, importContext: ImportContext) { + super(apiConfig, importContext); + } + + async start(): Promise { + await this.init(); + + const dir = this.getAssetTypesDir(); + const items = await this.readAllChunkedJson>(dir, 'asset-types.json'); + + if (items.length === 0) { + log.debug('No shared asset types to import', this.importContext.context); + return; + } + + // Fetch existing asset types from the target org keyed by uid for diff comparison. + // Asset types are org-level; the spaceUid param in getWorkspaceAssetTypes is unused in the path. + const existingByUid = new Map>(); + try { + const existing = await this.getWorkspaceAssetTypes(''); + for (const at of existing.asset_types ?? []) { + existingByUid.set(at.uid, at as Record); + } + log.debug(`Target org has ${existingByUid.size} existing asset type(s)`, this.importContext.context); + } catch (e) { + log.debug(`Could not fetch existing asset types, will attempt to create all: ${e}`, this.importContext.context); + } + + this.updateStatus(PROCESS_STATUS[PROCESS_NAMES.AM_IMPORT_ASSET_TYPES].IMPORTING, PROCESS_NAMES.AM_IMPORT_ASSET_TYPES); + + for (const assetType of items) { + const uid = assetType.uid as string; + + if (assetType.is_system) { + log.debug(`Skipping system asset type: ${uid}`, this.importContext.context); + continue; + } + + const existing = existingByUid.get(uid); + if (existing) { + const exportedClean = omit(assetType, STRIP_KEYS); + const existingClean = omit(existing, STRIP_KEYS); + if (!isEqual(exportedClean, existingClean)) { + log.warn( + `Asset type "${uid}" already exists in the target org with a different definition. Skipping — to apply the exported definition, delete the asset type from the target org first.`, + this.importContext.context, + ); + } else { + log.debug(`Asset type "${uid}" already exists with matching definition, skipping`, this.importContext.context); + } + this.tick(true, `asset-type: ${uid} (skipped, already exists)`, null, PROCESS_NAMES.AM_IMPORT_ASSET_TYPES); + continue; + } + + const payload = omit(assetType, STRIP_KEYS); + try { + await this.createAssetType(payload as any); + this.tick(true, `asset-type: ${uid}`, null, PROCESS_NAMES.AM_IMPORT_ASSET_TYPES); + log.debug(`Imported asset type: ${uid}`, this.importContext.context); + } catch (e) { + this.tick(false, `asset-type: ${uid}`, (e as Error)?.message ?? PROCESS_STATUS[PROCESS_NAMES.AM_IMPORT_ASSET_TYPES].FAILED, PROCESS_NAMES.AM_IMPORT_ASSET_TYPES); + log.debug(`Failed to import asset type ${uid}: ${e}`, this.importContext.context); + } + } + } +} diff --git a/packages/contentstack-asset-management/src/import/assets.ts b/packages/contentstack-asset-management/src/import/assets.ts new file mode 100644 index 00000000..ab7986db --- /dev/null +++ b/packages/contentstack-asset-management/src/import/assets.ts @@ -0,0 +1,227 @@ +import { resolve as pResolve, join } from 'node:path'; +import { existsSync, readFileSync } from 'node:fs'; +import { log } from '@contentstack/cli-utilities'; + +import type { AssetManagementAPIConfig, ImportContext } from '../types/asset-management-api'; +import { AssetManagementImportAdapter } from './base'; +import { getArrayFromResponse } from '../utils/export-helpers'; +import { PROCESS_NAMES, PROCESS_STATUS } from '../constants/index'; + +type FolderRecord = { + uid: string; + title: string; + description?: string; + parent_uid?: string; +}; + +type AssetRecord = { + uid: string; + url: string; + filename?: string; + file_name?: string; + parent_uid?: string; + title?: string; + description?: string; +}; + +/** + * Imports folders and assets for a single AM space. + * - Reads `spaces/{spaceUid}/assets/folders.json` → creates folders, builds folderUidMap + * - Reads chunked `assets.json` → uploads each file from `files/{oldUid}/{filename}` + * - Builds UID and URL mapper entries for entries.ts consumption + * Mirrors ExportAssets. + */ +export default class ImportAssets extends AssetManagementImportAdapter { + constructor(apiConfig: AssetManagementAPIConfig, importContext: ImportContext) { + super(apiConfig, importContext); + } + + /** + * Loads chunked `assets.json` when present; shared by reuse (identity maps) and upload path. + */ + private async loadExportedAssetItems(spaceDir: string): Promise { + const assetsDir = pResolve(spaceDir, 'assets'); + if (!existsSync(join(assetsDir, 'assets.json'))) { + return null; + } + return this.readAllChunkedJson(assetsDir, 'assets.json'); + } + + /** + * Build identity uid/url mappers from export JSON only (reuse path — no upload). + * Keys and values are equal so lookupAssets contract is satisfied without remapping. + */ + async buildIdentityMappersFromExport( + spaceDir: string, + ): Promise<{ uidMap: Record; urlMap: Record }> { + const uidMap: Record = {}; + const urlMap: Record = {}; + + const assetItems = await this.loadExportedAssetItems(spaceDir); + if (!assetItems) { + log.debug( + `No assets.json index in ${pResolve(spaceDir, 'assets')}, identity mappers empty`, + this.importContext.context, + ); + return { uidMap, urlMap }; + } + log.debug( + `Building identity mappers for ${assetItems.length} exported asset(s) (reuse path)`, + this.importContext.context, + ); + + for (const asset of assetItems) { + if (asset.uid) { + uidMap[asset.uid] = asset.uid; + } + if (asset.url) { + urlMap[asset.url] = asset.url; + } + } + + return { uidMap, urlMap }; + } + + async start( + newSpaceUid: string, + spaceDir: string, + ): Promise<{ uidMap: Record; urlMap: Record }> { + const assetsDir = pResolve(spaceDir, 'assets'); + const uidMap: Record = {}; + const urlMap: Record = {}; + + // ----------------------------------------------------------------------- + // 1. Import folders + // ----------------------------------------------------------------------- + const folderUidMap: Record = {}; + const foldersFilePath = join(assetsDir, 'folders.json'); + + if (existsSync(foldersFilePath)) { + let foldersData: unknown; + try { + foldersData = JSON.parse(readFileSync(foldersFilePath, 'utf8')); + } catch (e) { + log.debug(`Could not read folders.json: ${e}`, this.importContext.context); + } + + if (foldersData) { + const folders = getArrayFromResponse(foldersData, 'folders') as FolderRecord[]; + this.updateStatus(PROCESS_STATUS[PROCESS_NAMES.AM_IMPORT_FOLDERS].IMPORTING, PROCESS_NAMES.AM_IMPORT_FOLDERS); + log.debug(`Importing ${folders.length} folder(s) for space ${newSpaceUid}`, this.importContext.context); + await this.importFolders(newSpaceUid, folders, folderUidMap); + } + } + + // ----------------------------------------------------------------------- + // 2. Import assets (chunked) + // ----------------------------------------------------------------------- + const assetItems = await this.loadExportedAssetItems(spaceDir); + if (!assetItems) { + log.debug(`No assets.json index found in ${assetsDir}, skipping asset upload`, this.importContext.context); + return { uidMap, urlMap }; + } + this.updateStatus(PROCESS_STATUS[PROCESS_NAMES.AM_IMPORT_ASSETS].IMPORTING, PROCESS_NAMES.AM_IMPORT_ASSETS); + log.debug(`Uploading ${assetItems.length} asset(s) for space ${newSpaceUid}`, this.importContext.context); + + for (const asset of assetItems) { + const oldUid = asset.uid; + const filename = asset.filename ?? asset.file_name ?? 'asset'; + const filePath = pResolve(assetsDir, 'files', oldUid, filename); + + if (!existsSync(filePath)) { + log.debug(`Asset file not found: ${filePath}, skipping`, this.importContext.context); + this.tick(false, `asset: ${oldUid}`, 'File not found on disk', PROCESS_NAMES.AM_IMPORT_ASSETS); + continue; + } + + const assetParent = asset.parent_uid && asset.parent_uid !== 'root' ? asset.parent_uid : undefined; + const mappedParentUid = assetParent ? folderUidMap[assetParent] ?? undefined : undefined; + + try { + const { asset: created } = await this.uploadAsset(newSpaceUid, filePath, { + title: asset.title ?? filename, + description: asset.description, + parent_uid: mappedParentUid, + }); + + uidMap[oldUid] = created.uid; + + // Map old AM direct URL → new AM direct URL. + if (asset.url && created.url) { + urlMap[asset.url] = created.url; + } + + this.tick(true, `asset: ${oldUid}`, null, PROCESS_NAMES.AM_IMPORT_ASSETS); + log.debug(`Uploaded asset ${oldUid} → ${created.uid}`, this.importContext.context); + } catch (e) { + this.tick( + false, + `asset: ${oldUid}`, + (e as Error)?.message ?? PROCESS_STATUS[PROCESS_NAMES.AM_IMPORT_ASSETS].FAILED, + PROCESS_NAMES.AM_IMPORT_ASSETS, + ); + log.debug(`Failed to upload asset ${oldUid}: ${e}`, this.importContext.context); + } + } + + return { uidMap, urlMap }; + } + + /** + * Creates folders respecting hierarchy: parents before children. + * Uses multiple passes to handle arbitrary depth without requiring sorted input. + */ + private async importFolders( + newSpaceUid: string, + folders: FolderRecord[], + folderUidMap: Record, + ): Promise { + let remaining = [...folders]; + let prevLength = -1; + + while (remaining.length > 0 && remaining.length !== prevLength) { + prevLength = remaining.length; + const nextPass: FolderRecord[] = []; + + for (const folder of remaining) { + const { parent_uid: parentUid } = folder; + // "root" is the AM API sentinel for a top-level folder + const isRootParent = !parentUid || parentUid === 'root'; + const parentMapped = isRootParent || folderUidMap[parentUid] !== undefined; + + if (!parentMapped) { + nextPass.push(folder); + continue; + } + + try { + const { folder: created } = await this.createFolder(newSpaceUid, { + title: folder.title, + description: folder.description, + parent_uid: isRootParent ? undefined : folderUidMap[parentUid], + }); + folderUidMap[folder.uid] = created.uid; + this.tick(true, `folder: ${folder.uid}`, null, PROCESS_NAMES.AM_IMPORT_FOLDERS); + log.debug(`Created folder ${folder.uid} → ${created.uid}`, this.importContext.context); + } catch (e) { + this.tick( + false, + `folder: ${folder.uid}`, + (e as Error)?.message ?? PROCESS_STATUS[PROCESS_NAMES.AM_IMPORT_FOLDERS].FAILED, + PROCESS_NAMES.AM_IMPORT_FOLDERS, + ); + log.debug(`Failed to create folder ${folder.uid}: ${e}`, this.importContext.context); + } + } + + remaining = nextPass; + } + + if (remaining.length > 0) { + log.debug( + `${remaining.length} folder(s) could not be imported (unresolved parent UIDs)`, + this.importContext.context, + ); + } + } +} diff --git a/packages/contentstack-asset-management/src/import/base.ts b/packages/contentstack-asset-management/src/import/base.ts new file mode 100644 index 00000000..db82f146 --- /dev/null +++ b/packages/contentstack-asset-management/src/import/base.ts @@ -0,0 +1,96 @@ +import { resolve as pResolve } from 'node:path'; +import { FsUtility, log, CLIProgressManager, configHandler } from '@contentstack/cli-utilities'; + +import type { AssetManagementAPIConfig, ImportContext } from '../types/asset-management-api'; +import { AssetManagementAdapter } from '../utils/asset-management-api-adapter'; +import { AM_MAIN_PROCESS_NAME } from '../constants/index'; + +export type { ImportContext }; + +/** + * Base class for all AM 2.0 import modules. Mirrors AssetManagementExportAdapter + * but carries ImportContext (spacesRootPath, apiKey, host, etc.) instead of ExportContext. + */ +export class AssetManagementImportAdapter extends AssetManagementAdapter { + protected readonly apiConfig: AssetManagementAPIConfig; + protected readonly importContext: ImportContext; + protected progressManager: CLIProgressManager | null = null; + protected parentProgressManager: CLIProgressManager | null = null; + protected readonly processName: string = AM_MAIN_PROCESS_NAME; + + constructor(apiConfig: AssetManagementAPIConfig, importContext: ImportContext) { + super(apiConfig); + this.apiConfig = apiConfig; + this.importContext = importContext; + } + + public setParentProgressManager(parent: CLIProgressManager): void { + this.parentProgressManager = parent; + } + + protected get progressOrParent(): CLIProgressManager | null { + return this.parentProgressManager ?? this.progressManager; + } + + protected createNestedProgress(moduleName: string): CLIProgressManager { + if (this.parentProgressManager) { + this.progressManager = this.parentProgressManager; + return this.parentProgressManager; + } + const logConfig = configHandler.get('log') || {}; + const showConsoleLogs = logConfig.showConsoleLogs ?? false; + this.progressManager = CLIProgressManager.createNested(moduleName, showConsoleLogs); + return this.progressManager; + } + + protected tick(success: boolean, itemName: string, error: string | null, processName?: string): void { + this.progressOrParent?.tick?.(success, itemName, error, processName ?? this.processName); + } + + protected updateStatus(message: string, processName?: string): void { + this.progressOrParent?.updateStatus?.(message, processName ?? this.processName); + } + + protected completeProcess(processName: string, success: boolean): void { + if (!this.parentProgressManager) { + this.progressManager?.completeProcess?.(processName, success); + } + } + + protected get spacesRootPath(): string { + return this.importContext.spacesRootPath; + } + + protected getAssetTypesDir(): string { + return pResolve(this.importContext.spacesRootPath, 'asset_types'); + } + + protected getFieldsDir(): string { + return pResolve(this.importContext.spacesRootPath, 'fields'); + } + + /** + * Reads all items from a FsUtility chunked JSON store (index file + chunk files). + * Returns a flat array of all items across all chunks. + */ + protected async readAllChunkedJson>(dir: string, indexFileName: string): Promise { + try { + const fs = new FsUtility({ basePath: dir, indexFileName }); + const indexer = fs.indexFileContent; + const items: T[] = []; + for (const _ in indexer) { + const chunk = await fs.readChunkFiles.next().catch((err: unknown): null => { + log.debug(`Error reading chunk: ${err}`, this.importContext.context); + return null; + }); + if (chunk) { + items.push(...(Object.values(chunk as Record))); + } + } + return items; + } catch (err) { + log.debug(`readAllChunkedJson failed for ${dir}/${indexFileName}: ${err}`, this.importContext.context); + return []; + } + } +} diff --git a/packages/contentstack-asset-management/src/import/fields.ts b/packages/contentstack-asset-management/src/import/fields.ts new file mode 100644 index 00000000..86f45086 --- /dev/null +++ b/packages/contentstack-asset-management/src/import/fields.ts @@ -0,0 +1,88 @@ +import omit from 'lodash/omit'; +import isEqual from 'lodash/isEqual'; +import { log } from '@contentstack/cli-utilities'; + +import type { AssetManagementAPIConfig, ImportContext } from '../types/asset-management-api'; +import { AssetManagementImportAdapter } from './base'; +import { PROCESS_NAMES, PROCESS_STATUS } from '../constants/index'; + +const STRIP_KEYS = ['created_at', 'created_by', 'updated_at', 'updated_by', 'is_system', 'asset_types_count']; + +/** + * Reads shared fields from `spaces/fields/fields.json` and POSTs each to the + * target org-level AM fields endpoint (`POST /api/fields`). + * + * Strategy: Fetch → Diff → Create only missing, warn on conflict + * 1. Fetch fields that already exist in the target org. + * 2. Skip entries where is_system=true (platform-owned, cannot be created via API). + * 3. If uid already exists and definition differs → warn and skip. + * 4. If uid already exists and definition matches → silently skip. + * 5. Strip read-only/computed keys from the POST body before creating new fields. + */ +export default class ImportFields extends AssetManagementImportAdapter { + constructor(apiConfig: AssetManagementAPIConfig, importContext: ImportContext) { + super(apiConfig, importContext); + } + + async start(): Promise { + await this.init(); + + const dir = this.getFieldsDir(); + const items = await this.readAllChunkedJson>(dir, 'fields.json'); + + if (items.length === 0) { + log.debug('No shared fields to import', this.importContext.context); + return; + } + + // Fetch existing fields from the target org keyed by uid for diff comparison. + // Fields are org-level; the spaceUid param in getWorkspaceFields is unused in the path. + const existingByUid = new Map>(); + try { + const existing = await this.getWorkspaceFields(''); + for (const f of existing.fields ?? []) { + existingByUid.set(f.uid, f as Record); + } + log.debug(`Target org has ${existingByUid.size} existing field(s)`, this.importContext.context); + } catch (e) { + log.debug(`Could not fetch existing fields, will attempt to create all: ${e}`, this.importContext.context); + } + + this.updateStatus(PROCESS_STATUS[PROCESS_NAMES.AM_IMPORT_FIELDS].IMPORTING, PROCESS_NAMES.AM_IMPORT_FIELDS); + + for (const field of items) { + const uid = field.uid as string; + + if (field.is_system) { + log.debug(`Skipping system field: ${uid}`, this.importContext.context); + continue; + } + + const existing = existingByUid.get(uid); + if (existing) { + const exportedClean = omit(field, STRIP_KEYS); + const existingClean = omit(existing, STRIP_KEYS); + if (!isEqual(exportedClean, existingClean)) { + log.warn( + `Field "${uid}" already exists in the target org with a different definition. Skipping — to apply the exported definition, delete the field from the target org first.`, + this.importContext.context, + ); + } else { + log.debug(`Field "${uid}" already exists with matching definition, skipping`, this.importContext.context); + } + this.tick(true, `field: ${uid} (skipped, already exists)`, null, PROCESS_NAMES.AM_IMPORT_FIELDS); + continue; + } + + const payload = omit(field, STRIP_KEYS); + try { + await this.createField(payload as any); + this.tick(true, `field: ${uid}`, null, PROCESS_NAMES.AM_IMPORT_FIELDS); + log.debug(`Imported field: ${uid}`, this.importContext.context); + } catch (e) { + this.tick(false, `field: ${uid}`, (e as Error)?.message ?? PROCESS_STATUS[PROCESS_NAMES.AM_IMPORT_FIELDS].FAILED, PROCESS_NAMES.AM_IMPORT_FIELDS); + log.debug(`Failed to import field ${uid}: ${e}`, this.importContext.context); + } + } + } +} diff --git a/packages/contentstack-asset-management/src/import/index.ts b/packages/contentstack-asset-management/src/import/index.ts new file mode 100644 index 00000000..61d8a457 --- /dev/null +++ b/packages/contentstack-asset-management/src/import/index.ts @@ -0,0 +1,7 @@ +export { ImportSpaces } from './spaces'; +export { default as ImportWorkspace } from './workspaces'; +export { default as ImportAssets } from './assets'; +export { default as ImportFields } from './fields'; +export { default as ImportAssetTypes } from './asset-types'; +export { AssetManagementImportAdapter } from './base'; +export type { ImportContext } from './base'; diff --git a/packages/contentstack-asset-management/src/import/spaces.ts b/packages/contentstack-asset-management/src/import/spaces.ts new file mode 100644 index 00000000..f4388515 --- /dev/null +++ b/packages/contentstack-asset-management/src/import/spaces.ts @@ -0,0 +1,165 @@ +import { resolve as pResolve, join } from 'node:path'; +import { readdirSync, statSync } from 'node:fs'; +import { log, CLIProgressManager, configHandler } from '@contentstack/cli-utilities'; + +import type { + AssetManagementAPIConfig, + AssetManagementImportOptions, + ImportContext, + ImportResult, + SpaceMapping, +} from '../types/asset-management-api'; +import { AM_MAIN_PROCESS_NAME } from '../constants/index'; +import { AssetManagementAdapter } from '../utils/asset-management-api-adapter'; +import ImportAssetTypes from './asset-types'; +import ImportFields from './fields'; +import ImportWorkspace from './workspaces'; + +/** + * Top-level orchestrator for AM 2.0 import. + * Mirrors ExportSpaces: creates shared fields + asset types, then imports each space. + * Returns combined uidMap, urlMap, and spaceMappings for the bridge module. + */ +export class ImportSpaces { + private readonly options: AssetManagementImportOptions; + private parentProgressManager: CLIProgressManager | null = null; + private progressManager: CLIProgressManager | null = null; + + constructor(options: AssetManagementImportOptions) { + this.options = options; + } + + public setParentProgressManager(parent: CLIProgressManager): void { + this.parentProgressManager = parent; + } + + async start(): Promise { + const { contentDir, assetManagementUrl, org_uid, apiKey, host, sourceApiKey, context } = this.options; + + const spacesRootPath = pResolve(contentDir, 'spaces'); + + const importContext: ImportContext = { + spacesRootPath, + sourceApiKey, + apiKey, + host, + org_uid, + context, + }; + + const apiConfig: AssetManagementAPIConfig = { + baseURL: assetManagementUrl, + headers: { organization_uid: org_uid }, + context, + }; + + // Discover space directories + let spaceDirs: string[] = []; + try { + spaceDirs = readdirSync(spacesRootPath).filter((entry) => { + try { + return statSync(join(spacesRootPath, entry)).isDirectory() && entry.startsWith('am'); + } catch { + return false; + } + }); + } catch (e) { + log.debug(`Could not read spaces root path ${spacesRootPath}: ${e}`, context); + } + + const totalSteps = 2 + spaceDirs.length * 2; + const progress = this.createProgress(); + progress.addProcess(AM_MAIN_PROCESS_NAME, totalSteps); + progress.startProcess(AM_MAIN_PROCESS_NAME); + + const allUidMap: Record = {}; + const allUrlMap: Record = {}; + const allSpaceUidMap: Record = {}; + const spaceMappings: SpaceMapping[] = []; + let hasFailures = false; + + // Space UIDs already present in the target org — reuse when export dir name matches a uid here. + const existingSpaceUids = new Set(); + try { + const adapterForList = new AssetManagementAdapter(apiConfig); + await adapterForList.init(); + const { spaces } = await adapterForList.listSpaces(); + for (const s of spaces) { + if (s.uid) existingSpaceUids.add(s.uid); + } + log.debug(`Found ${existingSpaceUids.size} existing space uid(s) in target org`, context); + } catch (e) { + log.debug(`Could not fetch existing spaces — reuse-by-uid disabled: ${e}`, context); + } + + try { + // 1. Import shared fields + progress.updateStatus(`Importing shared fields...`, AM_MAIN_PROCESS_NAME); + const fieldsImporter = new ImportFields(apiConfig, importContext); + fieldsImporter.setParentProgressManager(progress); + await fieldsImporter.start(); + + // 2. Import shared asset types + progress.updateStatus('Importing shared asset types...', AM_MAIN_PROCESS_NAME); + const assetTypesImporter = new ImportAssetTypes(apiConfig, importContext); + assetTypesImporter.setParentProgressManager(progress); + await assetTypesImporter.start(); + + // 3. Import each space — continue on failure so partially-imported data is never lost + for (const spaceUid of spaceDirs) { + const spaceDir = join(spacesRootPath, spaceUid); + progress.updateStatus(`Importing space: ${spaceUid}...`, AM_MAIN_PROCESS_NAME); + log.debug(`Importing space: ${spaceUid}`, context); + + try { + const workspaceImporter = new ImportWorkspace(apiConfig, importContext); + workspaceImporter.setParentProgressManager(progress); + const result = await workspaceImporter.start(spaceUid, spaceDir, existingSpaceUids); + + // Newly created spaces get a new uid — add so later iterations in this run see it. + existingSpaceUids.add(result.newSpaceUid); + + Object.assign(allUidMap, result.uidMap); + Object.assign(allUrlMap, result.urlMap); + allSpaceUidMap[result.oldSpaceUid] = result.newSpaceUid; + spaceMappings.push({ + oldSpaceUid: result.oldSpaceUid, + newSpaceUid: result.newSpaceUid, + workspaceUid: result.workspaceUid, + isDefault: result.isDefault, + }); + + log.debug(`Imported space ${spaceUid} → ${result.newSpaceUid}`, context); + } catch (err) { + hasFailures = true; + progress.tick( + false, + `space: ${spaceUid}`, + (err as Error)?.message ?? 'Failed to import space', + AM_MAIN_PROCESS_NAME, + ); + log.debug(`Failed to import space ${spaceUid}: ${err}`, context); + } + } + + progress.completeProcess(AM_MAIN_PROCESS_NAME, !hasFailures); + log.debug('Asset Management 2.0 import completed', context); + } catch (err) { + progress.completeProcess(AM_MAIN_PROCESS_NAME, false); + throw err; + } + + return { uidMap: allUidMap, urlMap: allUrlMap, spaceMappings, spaceUidMap: allSpaceUidMap }; + } + + private createProgress(): CLIProgressManager { + if (this.parentProgressManager) { + this.progressManager = this.parentProgressManager; + return this.parentProgressManager; + } + const logConfig = configHandler.get('log') || {}; + const showConsoleLogs = logConfig.showConsoleLogs ?? false; + this.progressManager = CLIProgressManager.createNested(AM_MAIN_PROCESS_NAME, showConsoleLogs); + return this.progressManager; + } +} diff --git a/packages/contentstack-asset-management/src/import/workspaces.ts b/packages/contentstack-asset-management/src/import/workspaces.ts new file mode 100644 index 00000000..5d13450d --- /dev/null +++ b/packages/contentstack-asset-management/src/import/workspaces.ts @@ -0,0 +1,82 @@ +import { join } from 'node:path'; +import { readFileSync } from 'node:fs'; +import { log } from '@contentstack/cli-utilities'; + +import type { AssetManagementAPIConfig, ImportContext, SpaceMapping } from '../types/asset-management-api'; +import { AssetManagementImportAdapter } from './base'; +import ImportAssets from './assets'; +import { PROCESS_NAMES } from '../constants/index'; + +type WorkspaceResult = SpaceMapping & { + uidMap: Record; + urlMap: Record; +}; + +/** + * Handles import for a single AM 2.0 space directory. + * Reads `metadata.json`, creates the space in the target org when its uid is not + * already present, or reuses the existing space and emits identity mappers only. + * Returns the SpaceMapping plus UID/URL maps for the mapper files. + */ +export default class ImportWorkspace extends AssetManagementImportAdapter { + constructor(apiConfig: AssetManagementAPIConfig, importContext: ImportContext) { + super(apiConfig, importContext); + } + + async start( + oldSpaceUid: string, + spaceDir: string, + existingSpaceUids: Set = new Set(), + ): Promise { + await this.init(); + + // Read exported metadata + const metadataPath = join(spaceDir, 'metadata.json'); + let metadata: Record = {}; + try { + metadata = JSON.parse(readFileSync(metadataPath, 'utf8')) as Record; + } catch (e) { + log.debug(`Could not read metadata.json for space ${oldSpaceUid}: ${e}`, this.importContext.context); + } + + const exportedTitle = (metadata.title as string) ?? oldSpaceUid; + const description = metadata.description as string | undefined; + const isDefault = (metadata.is_default as boolean) ?? false; + const workspaceUid = 'main'; + + const assetsImporter = new ImportAssets(this.apiConfig, this.importContext); + if (this.progressOrParent) assetsImporter.setParentProgressManager(this.progressOrParent); + + // Reuse: target org already has a space with the same uid as the export directory. + if (existingSpaceUids.has(oldSpaceUid)) { + log.info( + `Reusing existing AM space "${oldSpaceUid}" (uid matches export directory); skipping create and upload.`, + this.importContext.context, + ); + const newSpaceUid = oldSpaceUid; + const { uidMap, urlMap } = await assetsImporter.buildIdentityMappersFromExport(spaceDir); + this.tick(true, `space: ${oldSpaceUid} → ${newSpaceUid} (reused)`, null, PROCESS_NAMES.AM_SPACE_METADATA); + return { + oldSpaceUid, + newSpaceUid, + workspaceUid, + isDefault, + uidMap, + urlMap, + }; + } + + // Create new space with exact exported title + log.debug(`Creating space "${exportedTitle}" (old uid: ${oldSpaceUid})`, this.importContext.context); + + const { space } = await this.createSpace({ title: exportedTitle, description }); + const newSpaceUid = space.uid; + + log.debug(`Created space ${newSpaceUid} (old: ${oldSpaceUid})`, this.importContext.context); + this.tick(true, `space: ${oldSpaceUid} → ${newSpaceUid}`, null, PROCESS_NAMES.AM_SPACE_METADATA); + + const { uidMap, urlMap } = await assetsImporter.start(newSpaceUid, spaceDir); + + return { oldSpaceUid, newSpaceUid, workspaceUid, isDefault, uidMap, urlMap }; + } +} diff --git a/packages/contentstack-asset-management/src/index.ts b/packages/contentstack-asset-management/src/index.ts index f0ff59bd..c66c638d 100644 --- a/packages/contentstack-asset-management/src/index.ts +++ b/packages/contentstack-asset-management/src/index.ts @@ -2,3 +2,4 @@ export * from './constants/index'; export * from './types'; export * from './utils'; export * from './export'; +export * from './import'; diff --git a/packages/contentstack-asset-management/src/types/asset-management-api.ts b/packages/contentstack-asset-management/src/types/asset-management-api.ts index 733ecada..3d3b01d1 100644 --- a/packages/contentstack-asset-management/src/types/asset-management-api.ts +++ b/packages/contentstack-asset-management/src/types/asset-management-api.ts @@ -36,6 +36,9 @@ export type Space = { /** Response shape of GET /api/spaces/{space_uid}. */ export type SpaceResponse = { space: Space }; +/** Response shape of GET /api/spaces (list all spaces in the org). */ +export type SpacesListResponse = { spaces: Space[]; count?: number }; + /** * Field structure from GET /api/fields (org-level). */ @@ -118,10 +121,11 @@ export type AssetManagementAPIConfig = { */ export interface IAssetManagementAdapter { init(): Promise; + listSpaces(): Promise; getSpace(spaceUid: string): Promise; getWorkspaceFields(spaceUid: string): Promise; - getWorkspaceAssets(spaceUid: string): Promise; - getWorkspaceFolders(spaceUid: string): Promise; + getWorkspaceAssets(spaceUid: string, workspaceUid?: string): Promise; + getWorkspaceFolders(spaceUid: string, workspaceUid?: string): Promise; getWorkspaceAssetTypes(spaceUid: string): Promise; } @@ -137,4 +141,119 @@ export type AssetManagementExportOptions = { context?: Record; /** When true, the AM package will add authtoken to asset download URLs. */ securedAssets?: boolean; + /** + * API key of the stack being exported. + * Saved to `spaces/export-metadata.json` so that during import the URL mapper + * can reconstruct old CMA proxy URLs (format: /v3/assets/{apiKey}/{amUid}/...). + */ + apiKey?: string; +}; + +// --------------------------------------------------------------------------- +// Import types +// --------------------------------------------------------------------------- + +/** + * Context passed down to every import adapter class. + * Mirrors ExportContext but carries the import-specific fields needed for + * URL mapper reconstruction and API calls. + */ +export type ImportContext = { + /** Absolute path to the root `spaces/` directory inside the backup/content dir. */ + spacesRootPath: string; + /** Source stack API key — used to reconstruct old CMA proxy URLs. */ + sourceApiKey?: string; + /** Target stack API key — used to build new CMA proxy URLs. */ + apiKey: string; + /** Target CMA host (may include /v3), e.g. "https://api.contentstack.io/v3". */ + host: string; + /** Target org UID — required as `x-organization-uid` header when creating spaces. */ + org_uid: string; + /** Optional logging context (same shape as ExportConfig.context). */ + context?: Record; +}; + +/** + * Options accepted by the top-level `ImportSpaces` class. + */ +export type AssetManagementImportOptions = { + /** Absolute path to the root content / backup directory. */ + contentDir: string; + /** AM 2.0 base URL (e.g. "https://am.contentstack.io"). */ + assetManagementUrl: string; + /** Target organisation UID. */ + org_uid: string; + /** Target stack API key. */ + apiKey: string; + /** Target CMA host. */ + host: string; + /** Source stack API key — used for old CMA proxy URL reconstruction. */ + sourceApiKey?: string; + /** Optional logging context. */ + context?: Record; +}; + +/** + * Maps an old source-org space UID to the newly created target-org space UID. + */ +export type SpaceMapping = { + oldSpaceUid: string; + newSpaceUid: string; + /** Workspace identifier inside the space (typically "main"). */ + workspaceUid: string; + isDefault: boolean; +}; + +/** + * The value returned by `ImportSpaces.start()`. + * Written to `mapper/assets/uid-mapping.json` and `mapper/assets/url-mapping.json` + * by the bridge module so `entries.ts` can resolve asset references. + */ +export type ImportResult = { + uidMap: Record; + urlMap: Record; + spaceMappings: SpaceMapping[]; + /** old space UID → new space UID, written to mapper/assets/space-uid-mapping.json */ + spaceUidMap: Record; +}; + +// --------------------------------------------------------------------------- +// Import payload types (confirmed from Postman collection) +// --------------------------------------------------------------------------- + +export type CreateSpacePayload = { + title: string; + description?: string; +}; + +export type CreateFolderPayload = { + title: string; + description?: string; + parent_uid?: string; +}; + +export type CreateAssetMetadata = { + title?: string; + description?: string; + parent_uid?: string; +}; + +export type CreateFieldPayload = { + uid: string; + title: string; + display_type?: string; + child?: unknown[]; + is_mandatory?: boolean; + is_multiple?: boolean; + [key: string]: unknown; +}; + +export type CreateAssetTypePayload = { + uid: string; + title: string; + description?: string; + content_type?: string; + file_extension?: string | string[]; + fields?: string[]; + [key: string]: unknown; }; diff --git a/packages/contentstack-asset-management/src/utils/asset-management-api-adapter.ts b/packages/contentstack-asset-management/src/utils/asset-management-api-adapter.ts index b159cc33..b26b2664 100644 --- a/packages/contentstack-asset-management/src/utils/asset-management-api-adapter.ts +++ b/packages/contentstack-asset-management/src/utils/asset-management-api-adapter.ts @@ -1,11 +1,20 @@ +import { readFileSync } from 'node:fs'; +import { basename } from 'node:path'; import { HttpClient, log, authenticationHandler } from '@contentstack/cli-utilities'; import type { AssetManagementAPIConfig, AssetTypesResponse, + CreateAssetMetadata, + CreateAssetTypePayload, + CreateFieldPayload, + CreateFolderPayload, + CreateSpacePayload, FieldsResponse, IAssetManagementAdapter, + Space, SpaceResponse, + SpacesListResponse, } from '../types/asset-management-api'; export class AssetManagementAdapter implements IAssetManagementAdapter { @@ -89,6 +98,13 @@ export class AssetManagementAdapter implements IAssetManagementAdapter { } } + async listSpaces(): Promise { + log.debug('Fetching all spaces in org', this.config.context); + const result = await this.getSpaceLevel('', '/api/spaces', {}); + log.debug(`Fetched ${result?.count ?? result?.spaces?.length ?? '?'} space(s)`, this.config.context); + return result; + } + async getSpace(spaceUid: string): Promise { log.debug(`Fetching space: ${spaceUid}`, this.config.context); const path = `/api/spaces/${spaceUid}`; @@ -114,27 +130,30 @@ export class AssetManagementAdapter implements IAssetManagementAdapter { spaceUid: string, path: string, logLabel: string, + queryParams: Record = {}, ): Promise { log.debug(`Fetching ${logLabel} for space: ${spaceUid}`, this.config.context); - const result = await this.getSpaceLevel(spaceUid, path, {}); + const result = await this.getSpaceLevel(spaceUid, path, queryParams); const count = (result as { count?: number })?.count ?? (Array.isArray(result) ? result.length : '?'); log.debug(`Fetched ${logLabel} (count: ${count})`, this.config.context); return result; } - async getWorkspaceAssets(spaceUid: string): Promise { + async getWorkspaceAssets(spaceUid: string, workspaceUid?: string): Promise { return this.getWorkspaceCollection( spaceUid, `/api/spaces/${encodeURIComponent(spaceUid)}/assets`, 'assets', + workspaceUid ? { workspace: workspaceUid } : {}, ); } - async getWorkspaceFolders(spaceUid: string): Promise { + async getWorkspaceFolders(spaceUid: string, workspaceUid?: string): Promise { return this.getWorkspaceCollection( spaceUid, `/api/spaces/${encodeURIComponent(spaceUid)}/folders`, 'folders', + workspaceUid ? { workspace: workspaceUid } : {}, ); } @@ -146,4 +165,118 @@ export class AssetManagementAdapter implements IAssetManagementAdapter { log.debug(`Fetched asset types (count: ${result?.count ?? '?'})`, this.config.context); return result; } + + // --------------------------------------------------------------------------- + // POST helpers + // --------------------------------------------------------------------------- + + /** + * Build headers for outgoing POST requests. + */ + private async getPostHeaders(extraHeaders: Record = {}): Promise> { + await authenticationHandler.getAuthDetails(); + const token = authenticationHandler.accessToken; + const authHeader: Record = authenticationHandler.isOauthEnabled + ? { authorization: token } + : { access_token: token }; + return { + Accept: 'application/json', + 'x-cs-api-version': '4', + ...(this.config.headers ?? {}), + ...authHeader, + ...extraHeaders, + }; + } + + private async postJson(path: string, body: unknown, extraHeaders: Record = {}): Promise { + const baseUrl = this.config.baseURL?.replace(/\/$/, '') ?? ''; + const headers = await this.getPostHeaders({ 'Content-Type': 'application/json', ...extraHeaders }); + log.debug(`POST ${path}`, this.config.context); + const response = await fetch(`${baseUrl}${path}`, { + method: 'POST', + headers, + body: JSON.stringify(body), + }); + if (!response.ok) { + const text = await response.text().catch(() => ''); + throw new Error(`AM API POST error: status ${response.status}, path ${path}, body: ${text}`); + } + return response.json() as Promise; + } + + private async postMultipart(path: string, form: FormData, extraHeaders: Record = {}): Promise { + const baseUrl = this.config.baseURL?.replace(/\/$/, '') ?? ''; + const headers = await this.getPostHeaders(extraHeaders); + log.debug(`POST (multipart) ${path}`, this.config.context); + const response = await fetch(`${baseUrl}${path}`, { + method: 'POST', + headers, + body: form, + }); + if (!response.ok) { + const text = await response.text().catch(() => ''); + throw new Error(`AM API multipart POST error: status ${response.status}, path ${path}, body: ${text}`); + } + return response.json() as Promise; + } + + // --------------------------------------------------------------------------- + // Import API methods + // --------------------------------------------------------------------------- + + /** + * POST /api/spaces — creates a new space in the target org. + */ + async createSpace(payload: CreateSpacePayload): Promise<{ space: Space }> { + const orgUid = (this.config.headers as Record | undefined)?.organization_uid ?? ''; + return this.postJson<{ space: Space }>('/api/spaces', payload, { + 'x-organization-uid': orgUid, + }); + } + + /** + * POST /api/spaces/{spaceUid}/folders — creates a folder inside a space. + */ + async createFolder(spaceUid: string, payload: CreateFolderPayload): Promise<{ folder: { uid: string } }> { + return this.postJson<{ folder: { uid: string } }>(`/api/spaces/${encodeURIComponent(spaceUid)}/folders`, payload, { + space_key: spaceUid, + }); + } + + /** + * POST /api/spaces/{spaceUid}/assets — uploads an asset file as multipart form-data. + */ + async uploadAsset( + spaceUid: string, + filePath: string, + metadata: CreateAssetMetadata, + ): Promise<{ asset: { uid: string; url: string } }> { + const filename = basename(filePath); + const fileBuffer = readFileSync(filePath); + const blob = new Blob([fileBuffer]); + const form = new FormData(); + form.append('file', blob, filename); + if (metadata.title) form.append('title', metadata.title); + if (metadata.description) form.append('description', metadata.description); + if (metadata.parent_uid) form.append('parent_uid', metadata.parent_uid); + return this.postMultipart<{ asset: { uid: string; url: string } }>( + `/api/spaces/${encodeURIComponent(spaceUid)}/assets`, + form, + { space_key: spaceUid }, + ); + } + + /** + * POST /api/fields — creates a shared field. + */ + async createField(payload: CreateFieldPayload): Promise<{ field: { uid: string } }> { + return this.postJson<{ field: { uid: string } }>('/api/fields', payload); + } + + /** + * POST /api/asset_types — creates a shared asset type. + */ + async createAssetType(payload: CreateAssetTypePayload): Promise<{ asset_type: { uid: string } }> { + return this.postJson<{ asset_type: { uid: string } }>('/api/asset_types', payload); + } } diff --git a/packages/contentstack-export/src/export/modules/assets.ts b/packages/contentstack-export/src/export/modules/assets.ts index 1812ba0a..d896af4e 100644 --- a/packages/contentstack-export/src/export/modules/assets.ts +++ b/packages/contentstack-export/src/export/modules/assets.ts @@ -75,6 +75,7 @@ export default class ExportAssets extends BaseClass { branchName: this.exportConfig.branchName || 'main', assetManagementUrl, org_uid: this.exportConfig.org_uid ?? '', + apiKey: this.exportConfig.apiKey, context: this.exportConfig.context as unknown as Record, securedAssets: this.exportConfig.securedAssets, }); diff --git a/packages/contentstack-import/package.json b/packages/contentstack-import/package.json index a1cddafb..b6424707 100644 --- a/packages/contentstack-import/package.json +++ b/packages/contentstack-import/package.json @@ -5,6 +5,7 @@ "author": "Contentstack", "bugs": "https://github.com/contentstack/cli/issues", "dependencies": { + "@contentstack/cli-asset-management": "1.0.0", "@contentstack/cli-audit": "~2.0.0-beta.6", "@contentstack/cli-command": "~2.0.0-beta.2", "@contentstack/cli-utilities": "~2.0.0-beta.2", @@ -90,4 +91,4 @@ } }, "repository": "https://github.com/contentstack/cli" -} +} \ No newline at end of file diff --git a/packages/contentstack-import/src/config/index.ts b/packages/contentstack-import/src/config/index.ts index a18d7d07..fd0a5d1e 100644 --- a/packages/contentstack-import/src/config/index.ts +++ b/packages/contentstack-import/src/config/index.ts @@ -101,6 +101,15 @@ const config: DefaultConfig = { folderValidKeys: ['name', 'parent_uid'], validKeys: ['title', 'parent_uid', 'description', 'tags'], }, + 'asset-management': { + dirName: 'spaces', + fieldsDir: 'fields', + assetTypesDir: 'asset_types', + foldersFileName: 'folders.json', + assetsFileName: 'assets.json', + uploadAssetsConcurrency: 2, + importFoldersConcurrency: 1, + }, 'assets-old': { dirName: 'assets', fileName: 'assets.json', diff --git a/packages/contentstack-import/src/constants/index.ts b/packages/contentstack-import/src/constants/index.ts index 5872ec29..e3455787 100644 --- a/packages/contentstack-import/src/constants/index.ts +++ b/packages/contentstack-import/src/constants/index.ts @@ -20,6 +20,7 @@ export const PATH_CONSTANTS = { INDEX: 'index.json', FOLDER_MAPPING: 'folder-mapping.json', VERSIONED_ASSETS: 'versioned-assets.json', + SPACE_UID_MAPPING: 'space-uid-mapping.json', }, /** Module subdirectory names within mapper */ diff --git a/packages/contentstack-import/src/import/modules/assets.ts b/packages/contentstack-import/src/import/modules/assets.ts index 007f2187..215d2775 100644 --- a/packages/contentstack-import/src/import/modules/assets.ts +++ b/packages/contentstack-import/src/import/modules/assets.ts @@ -5,20 +5,17 @@ import unionBy from 'lodash/unionBy'; import orderBy from 'lodash/orderBy'; import isEmpty from 'lodash/isEmpty'; import uniq from 'lodash/uniq'; -import { existsSync } from 'node:fs'; +import { existsSync, mkdirSync } from 'node:fs'; import includes from 'lodash/includes'; import { v4 as uuid } from 'uuid'; import { resolve as pResolve, join } from 'node:path'; -import { - FsUtility, - log, - handleAndLogError, -} from '@contentstack/cli-utilities'; +import { FsUtility, log, handleAndLogError } from '@contentstack/cli-utilities'; +import { ImportSpaces } from '@contentstack/cli-asset-management'; import { PATH_CONSTANTS } from '../../constants'; import config from '../../config'; import { ModuleClassParams } from '../../types'; -import { formatDate, PROCESS_NAMES, MODULE_CONTEXTS, MODULE_NAMES, PROCESS_STATUS } from '../../utils'; +import { formatDate, fsUtil, PROCESS_NAMES, MODULE_CONTEXTS, MODULE_NAMES, PROCESS_STATUS } from '../../utils'; import BaseClass, { ApiOptions } from './base-class'; export default class ImportAssets extends BaseClass { @@ -42,22 +39,14 @@ export default class ImportAssets extends BaseClass { this.currentModuleName = MODULE_NAMES[MODULE_CONTEXTS.ASSETS]; this.assetsPath = join(this.importConfig.backupDir, PATH_CONSTANTS.CONTENT_DIRS.ASSETS); - this.mapperDirPath = join( - this.importConfig.backupDir, - PATH_CONSTANTS.MAPPER, - PATH_CONSTANTS.MAPPER_MODULES.ASSETS, - ); + this.mapperDirPath = join(this.importConfig.backupDir, PATH_CONSTANTS.MAPPER, PATH_CONSTANTS.MAPPER_MODULES.ASSETS); this.assetUidMapperPath = join(this.mapperDirPath, PATH_CONSTANTS.FILES.UID_MAPPING); this.assetUrlMapperPath = join(this.mapperDirPath, PATH_CONSTANTS.FILES.URL_MAPPING); this.assetFolderUidMapperPath = join(this.mapperDirPath, PATH_CONSTANTS.FILES.FOLDER_MAPPING); this.assetsRootPath = join(this.importConfig.backupDir, this.assetConfig.dirName); this.fs = new FsUtility({ basePath: this.mapperDirPath }); this.environments = this.fs.readFile( - join( - this.importConfig.backupDir, - PATH_CONSTANTS.CONTENT_DIRS.ENVIRONMENTS, - PATH_CONSTANTS.FILES.ENVIRONMENTS, - ), + join(this.importConfig.backupDir, PATH_CONSTANTS.CONTENT_DIRS.ENVIRONMENTS, PATH_CONSTANTS.FILES.ENVIRONMENTS), true, ) as Record; } @@ -70,6 +59,95 @@ export default class ImportAssets extends BaseClass { try { log.debug('Starting assets import process...', this.importConfig.context); + // AM 2.0: assetManagementEnabled is set in the config handler when spaces/ + am_v2 are detected. + if (this.importConfig.assetManagementEnabled) { + const assetManagementUrl = this.importConfig.assetManagementUrl; + if (!assetManagementUrl) { + log.info( + 'AM 2.0 export detected but assetManagementUrl is not configured in the region settings. Skipping AM 2.0 asset import.', + this.importConfig.context, + ); + return; + } + + const progress = this.createNestedProgress(this.currentModuleName); + try { + const importer = new ImportSpaces({ + contentDir: this.importConfig.contentDir, + assetManagementUrl, + org_uid: this.importConfig.org_uid ?? '', + apiKey: this.importConfig.apiKey, + host: this.importConfig.region?.cma ?? this.importConfig.host ?? '', + sourceApiKey: this.importConfig.source_stack, + context: this.importConfig.context as unknown as Record, + }); + importer.setParentProgressManager(progress); + + const { uidMap, urlMap, spaceMappings, spaceUidMap } = await importer.start(); + + const mapperDirPath = join( + this.importConfig.backupDir, + PATH_CONSTANTS.MAPPER, + PATH_CONSTANTS.MAPPER_MODULES.ASSETS, + ); + mkdirSync(mapperDirPath, { recursive: true }); + await fsUtil.writeFile(join(mapperDirPath, PATH_CONSTANTS.FILES.UID_MAPPING), uidMap); + await fsUtil.writeFile(join(mapperDirPath, PATH_CONSTANTS.FILES.URL_MAPPING), urlMap); + await fsUtil.writeFile(join(mapperDirPath, PATH_CONSTANTS.FILES.SPACE_UID_MAPPING), spaceUidMap); + log.debug('Wrote AM 2.0 mapper files (uid, url, space-uid)', this.importConfig.context); + + // Link newly-created spaces to the target stack via branch settings POST. + if (spaceMappings.length > 0) { + try { + const branchUid = this.importConfig.branchName ?? 'main'; + + // Fetch the current branch settings to get already-linked workspaces. + const branchData = (await this.stack.branch(branchUid).fetch({ include_settings: true })) as Record< + string, + any + >; + const currentLinked = (branchData?.settings?.am_v2?.linked_workspaces ?? []) as Array<{ + uid: string; + space_uid: string; + is_default: boolean; + operation?: string; + }>; + + const newWorkspaces = spaceMappings.map(({ newSpaceUid, workspaceUid }) => ({ + uid: workspaceUid, + space_uid: newSpaceUid, + is_default: false, + operation: 'LINK', + })); + + const combinedWorkspaces = [...currentLinked, ...newWorkspaces]; + + await this.stack.branch(branchUid).updateSettings({ + branch: { settings: { am_v2: { linked_workspaces: combinedWorkspaces } } }, + }); + log.success( + `Linked ${newWorkspaces.length} space(s) to branch "${branchUid}"`, + this.importConfig.context, + ); + } catch (linkErr) { + log.warn( + `AM 2.0 spaces were imported but could not be linked to the target stack: ${ + (linkErr as Error)?.message + }. Re-run the import or link manually.`, + this.importConfig.context, + ); + } + } + + this.completeProgressWithMessage(); + } catch (error) { + this.completeProgress(false, (error as Error)?.message ?? 'AM 2.0 asset import failed'); + throw error; + } + return; + } + // Legacy flow continues below + // Step 1: Analyze import data const [foldersCount, assetsCount, versionedAssetsCount, publishableAssetsCount] = await this.withLoadingSpinner( 'ASSETS: Analyzing import data...', @@ -221,9 +299,7 @@ export default class ImportAssets extends BaseClass { */ async importAssets(isVersion = false): Promise { const processName = isVersion ? 'import versioned assets' : 'import assets'; - const indexFileName = isVersion - ? PATH_CONSTANTS.FILES.VERSIONED_ASSETS - : this.assetConfig.fileName; + const indexFileName = isVersion ? PATH_CONSTANTS.FILES.VERSIONED_ASSETS : this.assetConfig.fileName; const basePath = isVersion ? join(this.assetsPath, 'versions') : this.assetsPath; const progressProcessName = isVersion ? PROCESS_NAMES.ASSET_VERSIONS : PROCESS_NAMES.ASSET_UPLOAD; diff --git a/packages/contentstack-import/src/import/modules/stack.ts b/packages/contentstack-import/src/import/modules/stack.ts index d6d920b0..969ad4c9 100644 --- a/packages/contentstack-import/src/import/modules/stack.ts +++ b/packages/contentstack-import/src/import/modules/stack.ts @@ -1,4 +1,5 @@ import { join } from 'node:path'; +import { existsSync } from 'node:fs'; import { log, handleAndLogError } from '@contentstack/cli-utilities'; import { PATH_CONSTANTS } from '../../constants'; @@ -59,7 +60,6 @@ export default class ImportStack extends BaseClass { await this.importStackSettings(); this.completeProgressWithMessage(); - } catch (error) { this.completeProgress(false, 'Stack settings import failed'); handleAndLogError(error, { ...this.importConfig.context }); @@ -69,6 +69,17 @@ export default class ImportStack extends BaseClass { private async importStackSettings(): Promise { log.debug('Processing stack settings for import', this.importConfig.context); + // Old source-org space UIDs must not be written to the target stack — + // the asset-management module will apply the correct am_v2.linked_workspaces. + if (existsSync(join(this.importConfig.contentDir, 'spaces'))) { + const { am_v2, ...settingsWithoutAm } = this.stackSettings as any; + this.stackSettings = settingsWithoutAm; + log.debug( + 'Stripped am_v2 from stack settings; asset-management module will apply it after space creation', + this.importConfig.context, + ); + } + // Update environment UID mapping if live preview is configured if (this.stackSettings?.live_preview && this.stackSettings?.live_preview['default-env'] !== undefined) { const oldEnvUid = this.stackSettings.live_preview['default-env']; diff --git a/packages/contentstack-import/src/types/default-config.ts b/packages/contentstack-import/src/types/default-config.ts index 2b7c3bd9..dc7ce315 100644 --- a/packages/contentstack-import/src/types/default-config.ts +++ b/packages/contentstack-import/src/types/default-config.ts @@ -72,6 +72,15 @@ export default interface DefaultConfig { uploadAssetsConcurrency: number; importFoldersConcurrency: number; }; + 'asset-management': { + dirName: string; + fieldsDir: string; + assetTypesDir: string; + foldersFileName: string; + assetsFileName: string; + uploadAssetsConcurrency: number; + importFoldersConcurrency: number; + }; content_types: { dirName: string; fileName: string; diff --git a/packages/contentstack-import/src/types/import-config.ts b/packages/contentstack-import/src/types/import-config.ts index 86db5668..5025c82a 100644 --- a/packages/contentstack-import/src/types/import-config.ts +++ b/packages/contentstack-import/src/types/import-config.ts @@ -58,6 +58,8 @@ export default interface ImportConfig extends DefaultConfig, ExternalConfig { personalizeProjectName?: string; 'exclude-global-modules': false; context: Context; + assetManagementUrl?: string; + assetManagementEnabled?: boolean; } type branch = { diff --git a/packages/contentstack-import/src/types/index.ts b/packages/contentstack-import/src/types/index.ts index 70bf9110..a73584b3 100644 --- a/packages/contentstack-import/src/types/index.ts +++ b/packages/contentstack-import/src/types/index.ts @@ -19,6 +19,7 @@ export interface Region { cma: string; cda: string; uiHost: string; + assetManagementUrl?: string; } export interface InquirePayload { diff --git a/packages/contentstack-import/src/utils/import-config-handler.ts b/packages/contentstack-import/src/utils/import-config-handler.ts index 0eb0ee29..927e3448 100644 --- a/packages/contentstack-import/src/utils/import-config-handler.ts +++ b/packages/contentstack-import/src/utils/import-config-handler.ts @@ -1,5 +1,6 @@ import merge from 'merge'; import * as path from 'path'; +import { existsSync, readFileSync } from 'node:fs'; import { omit, filter, includes, isArray } from 'lodash'; import { configHandler, isAuthenticated, cliux, sanitizePath, log } from '@contentstack/cli-utilities'; import defaultConfig from '../config'; @@ -21,7 +22,6 @@ const setupConfig = async (importCmdFlags: any): Promise => { if (importCmdFlags['config']) { let externalConfig = await readFile(importCmdFlags['config']); - if (isArray(externalConfig['modules'])) { config.modules.types = filter(config.modules.types, (module) => includes(externalConfig['modules'], module)); externalConfig = omit(externalConfig, ['modules']); @@ -126,6 +126,40 @@ const setupConfig = async (importCmdFlags: any): Promise => { config['exclude-global-modules'] = importCmdFlags['exclude-global-modules']; } + const spacesDir = path.join(config.contentDir, 'spaces'); + const stackSettingsPath = path.join(config.contentDir, 'stack', 'settings.json'); + + if (existsSync(spacesDir) && existsSync(stackSettingsPath)) { + try { + const stackSettings = JSON.parse(readFileSync(stackSettingsPath, 'utf8')); + if (stackSettings?.am_v2) { + config.assetManagementEnabled = true; + config.assetManagementUrl = configHandler.get('region')?.assetManagementUrl; + + const branchesJsonCandidates = [ + path.join(config.contentDir, 'branches.json'), + path.join(config.contentDir, '..', 'branches.json'), + ]; + for (const branchesJsonPath of branchesJsonCandidates) { + if (existsSync(branchesJsonPath)) { + try { + const branches = JSON.parse(readFileSync(branchesJsonPath, 'utf8')); + const apiKey = branches?.[0]?.stackHeaders?.api_key; + if (apiKey) { + config.source_stack = apiKey; + } + } catch { + // branches.json unreadable — URL mapping will be skipped + } + break; + } + } + } + } catch { + // stack settings unreadable — not an AM 2.0 export we can process + } + } + // Add authentication details to config for context tracking config.authenticationMethod = authenticationMethod; log.debug('Import configuration setup completed.', { ...config });