Skip to content

feat(extractEntries, extractItems, schemaMapper): enhance UID generat… #1110

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
umesh-more-cstk merged 6 commits into from
Jul 9, 2026
+86 −41
Merged

feat(extractEntries, extractItems, schemaMapper): enhance UID generat… #1110

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
a38946d
feat(extractEntries, extractItems, schemaMapper): enhance UID generat…
aishwarya-cstk Jul 8, 2026
d97e459
fix: correct UID generation for core/buttons and update test expectat…
aishwarya-cstk Jul 8, 2026
fd3ff1b
refactor: improve UID generation logic in extractEntries, extractItem…
aishwarya-cstk Jul 8, 2026
c207fec
fix: update UID assignment logic in extractEntries and schemaMapper f…
aishwarya-cstk Jul 8, 2026
5c53d43
Merge branch 'dev' of github.com:contentstack/migration-v2 into featu…
aishwarya-cstk Jul 8, 2026
fe0e936
fix: add constants for WordPress blocks and update extractEntries and…
aishwarya-cstk Jul 9, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 8 additions & 3 deletions .../migration-wordpress/constants/index.json → ...pi/migration-wordpress/constants/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
{
export const blocks = {
"core/paragraph": "text",
"core/image": "image",
"core/heading": "heading",
Expand Down Expand Up @@ -40,5 +40,10 @@
"core/post-author": "post_author",
"core/comments-template": "comments_template",
"core/comments-count": "comments_count",
"core/comment-author-name": "comment_author_name"
}
"core/comment-author-name": "comment_author_name"
}
export const EXCLUDED_POST_TYPES = new Set(['attachment', 'wp_global_styles', 'wp_navigation']);

export const ALLOWED_POST_STATUSES = new Set(['publish', 'inherit']);

export const MAX_SUFFIX_LEN = 40;
18 changes: 8 additions & 10 deletions upload-api/migration-wordpress/libs/extractEntries.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
import fs from 'fs';
import path from 'path';
import config from '../config/index.json';
import { EXCLUDED_POST_TYPES, ALLOWED_POST_STATUSES } from '../constants/index';


const { contentTypes: contentTypesConfig } = config?.modules;
const contentTypeFolderPath = path.resolve(config?.data, contentTypesConfig?.dirName);

const EXCLUDED_POST_TYPES = new Set(['attachment', 'wp_global_styles', 'wp_navigation']);

const normalizeArray = <T>(value: T | T[] | undefined): T[] => {
if (!value) return [];
return Array.isArray(value) ? value : [value];
Expand Down Expand Up @@ -47,10 +47,7 @@ const getEntryName = (item: any): string => {
return 'Untitled Entry';
};

/**
* All WordPress source entry keys use `posts_${...}` (any content type) so they align with
* wordpress.service export JSON and CLI uid-mapping.
*/

const getSourceEntryUid = (item: any): string => {
const postId = item?.['wp:post_id'];
if (postId != null && String(postId).trim() !== '') {
Expand All @@ -59,12 +56,12 @@ const getSourceEntryUid = (item: any): string => {

const authorId = item?.['wp:author_id'];
if (authorId != null && String(authorId).trim() !== '') {
return idCorrector(`posts_${authorId}`);
return idCorrector(`authors_${authorId}`);
}
Comment thread
aishwarya-cstk marked this conversation as resolved.

const termId = item?.['wp:term_id'];
if (termId != null && String(termId).trim() !== '') {
return idCorrector(`posts_${termId}`);
return idCorrector(`terms_${termId}`);
}
Comment thread
aishwarya-cstk marked this conversation as resolved.
Comment thread
aishwarya-cstk marked this conversation as resolved.

const candidate =
Expand Down Expand Up @@ -102,6 +99,8 @@ const extractEntries = async (filePath: string, contentTypeData: any[] = []) =>
const groupedByType = items?.reduce((acc: Record<string, any[]>, item: any) => {
const postType = item?.['wp:post_type'] || 'unknown';
if (EXCLUDED_POST_TYPES.has(postType)) return acc;
const postStatus = String(item?.['wp:status'] || '').toLowerCase();
if (!ALLOWED_POST_STATUSES.has(postStatus)) return acc;
if (!acc[postType]) acc[postType] = [];
acc[postType].push(item);
return acc;
Expand All @@ -112,7 +111,7 @@ const extractEntries = async (filePath: string, contentTypeData: any[] = []) =>
if (authorData) {
const authorEntries = normalizeArray(authorData).map((author: any) => ({
'wp:post_type': 'author',
'wp:post_id': author?.['wp:author_id'],
'wp:author_id': author?.['wp:author_id'],
title: author?.['wp:author_display_name'] || author?.['wp:author_login'],
'wp:author_login': author?.['wp:author_login'],
'wp:author_email': author?.['wp:author_email'],
Expand All @@ -130,7 +129,6 @@ const extractEntries = async (filePath: string, contentTypeData: any[] = []) =>
if (termData) {
const termEntries = normalizeArray(termData).map((term: any) => ({
'wp:post_type': 'terms',
'wp:post_id': term?.['wp:term_id'],
title: term?.['wp:term_name'] || term?.['wp:term_slug'],
'wp:term_id': term?.['wp:term_id'],
'wp:term_taxonomy': term?.['wp:term_taxonomy'],
Expand Down
11 changes: 7 additions & 4 deletions upload-api/migration-wordpress/libs/extractItems.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import * as cheerio from 'cheerio';


import { setupWordPressBlocks } from "../utils/parseUtil";
import { clientIdForUid, getFieldName, getFieldUid, schemaMapper } from "./schemaMapper";
import { stableSuffix, getFieldName, getFieldUid, schemaMapper } from "./schemaMapper";
import helper from "../utils/helper";
import config from '../config/index.json';
import extractTaxonomy from './extractTaxonomy';
Expand Down Expand Up @@ -365,7 +365,10 @@ const extractItems = async (item: any, config: DataConfig, type: string, affix:
// Track processed similar blocks to avoid duplicates

for (const field of blocksJson) {
const fieldUid = getFieldUid(`${field?.name}_${clientIdForUid(field?.clientId)}`|| '', affix || '');
// Guard a missing block name (freeform/whitespace separators parse with name: null)
// so the UID never gets a literal "undefined" prefix.
const blockName = field?.name || 'block';
const fieldUid = getFieldUid(`${blockName}${stableSuffix(field)}`, affix || '');
const contentstackFieldName = getFieldName(resolveBlockName(field));
Comment thread
aishwarya-cstk marked this conversation as resolved.

const similarBlocks = findSimilarBlocks(result, field?.clientId);
Expand Down Expand Up @@ -428,7 +431,7 @@ const extractItems = async (item: any, config: DataConfig, type: string, affix:
// No duplicate found - add the modular block child
if(Schema?.length > 0){
CT?.push?.({
"uid": `modular_blocks.${getFieldUid(`${field?.name}_${clientIdForUid(field?.clientId)}`, affix)}`,
"uid": `modular_blocks.${getFieldUid(`${blockName}${stableSuffix(field)}`, affix)}`,
"backupFieldUid": `modular_blocks.${fieldUid}`,
"contentstackFieldUid": `modular_blocks.${fieldUid}`,
"otherCmsField": contentstackFieldName,
Expand Down Expand Up @@ -484,7 +487,7 @@ const extractItems = async (item: any, config: DataConfig, type: string, affix:
// No duplicate found - add the modular block child
if(Schema?.length > 0){
CT?.push?.({
"uid": `modular_blocks.${getFieldUid(`${field?.name}_${clientIdForUid(field?.clientId)}`, affix)}`,
"uid": `modular_blocks.${getFieldUid(`${blockName}${stableSuffix(field)}`, affix)}`,
"backupFieldUid": `modular_blocks.${fieldUid}`,
"contentstackFieldUid": `modular_blocks.${fieldUid}`,
"otherCmsField": contentstackFieldName,
Expand Down
79 changes: 59 additions & 20 deletions upload-api/migration-wordpress/libs/schemaMapper.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import { Field, WordPressBlock } from '../interface/interface';
import restrictedUid from '../utils/index';
import GenerateSchema from 'generate-schema';
import { MAX_SUFFIX_LEN } from '../constants/index';

const MEDIA_BLOCK_NAMES = ['core/image', 'core/video', 'core/audio', 'core/file'];

Expand Down Expand Up @@ -50,6 +51,44 @@ export function clientIdForUid(clientId: string | undefined): string {
const compact = clientId?.replace?.(/-/g, '')?.toLowerCase();
return compact?.slice?.(0, 4) || '0';
}

/** Slugify an author-supplied identifier into a UID-safe token (lowercase, `_`-joined). */
const slugifyIdentifier = (value: unknown): string => {
if (value == null) return '';
return String(value)
.toLowerCase()
.replace(/[^a-z0-9]+/g, '_')
.replace(/^_+|_+$/g, '');
};


/**
* Deterministic, iteration-stable suffix for a block's field UID.
*
* The UID used to embed `clientIdForUid(block.clientId)`, but Gutenberg's parse()
* mints a fresh random clientId on every parse — so the UID changed on each
* re-migration and Contentstack treated the field as brand new, never updating
* the existing content on a 2nd iteration.
*
* This keys off `attributes.metadata.name` — the explicit author "Rename block"
* label — which survives re-parse, content edits, and reordering. It is the ONLY
* attribute the block de-duplication (`resolveBlockName`) uses to decide whether
* two same-named blocks are separate fields, so it is the correct — and only
* meaningful — disambiguator.
*
* We deliberately do NOT use `anchor`/`id`: those don't affect field separation
* and are frequently auto-derived from the block's text, which would make UIDs
* long and couple them to content (an edit would change the UID). Anonymous
* blocks therefore get NO suffix — the block name alone is a stable, unique UID
* within its scope.
*
* Returns a leading-underscore token (e.g. "_hero") or "" when anonymous, so
* callers can append it directly after the block name.
*/
export function stableSuffix(key: any): string {
const identity = slugifyIdentifier(key?.attributes?.metadata?.name).slice(0, MAX_SUFFIX_LEN);
return identity ? `_${identity}` : '';
}


async function processInnerBlocks(key: WordPressBlock, parentUid: string | null = null, parentFieldName: string | null = null, affix: string | null = null): Promise<any[]> {
Expand Down Expand Up @@ -207,8 +246,8 @@ async function schemaMapper (key: WordPressBlock | WordPressBlock[], parentUid:
case 'core/verse':
case 'core/code': {
const rteUid = parentUid ?
`${parentUid}.${getFieldUid(`${key?.name}_${clientIdForUid(key?.clientId)}`, affix)}`
: getFieldUid(`${key?.name}_${clientIdForUid(key?.clientId)}`, affix);
`${parentUid}.${getFieldUid(`${key?.name}${stableSuffix(key)}`, affix)}`
: getFieldUid(`${key?.name}${stableSuffix(key)}`, affix);
return {
uid: rteUid,
otherCmsField: getFieldName(key?.name),
Expand All @@ -223,8 +262,8 @@ async function schemaMapper (key: WordPressBlock | WordPressBlock[], parentUid:
}
case 'core/missing':
const rteUid = parentUid ?
`${parentUid}.${getFieldUid(`${key?.name}_${clientIdForUid(key?.clientId)}`, affix)}`
: getFieldUid(`${key?.name}_${clientIdForUid(key?.clientId)}`, affix);
`${parentUid}.${getFieldUid(`${key?.name}${stableSuffix(key)}`, affix)}`
: getFieldUid(`${key?.name}${stableSuffix(key)}`, affix);
if(key?.attributes?.originalName === 'jetpack/markdown'){
return {
uid: rteUid,
Expand Down Expand Up @@ -296,7 +335,7 @@ async function schemaMapper (key: WordPressBlock | WordPressBlock[], parentUid:
case 'core/audio':
case 'core/video':
case 'core/file': {
const fileUid = parentUid ? `${parentUid}.${getFieldUid(`${key?.name}_${clientIdForUid(key?.clientId)}`, affix)}` : getFieldUid(`${key?.name}_${clientIdForUid(key?.clientId)}`, affix);
const fileUid = parentUid ? `${parentUid}.${getFieldUid(`${key?.name}${stableSuffix(key)}`, affix)}` : getFieldUid(`${key?.name}${stableSuffix(key)}`, affix);

return {
uid: fileUid,
Expand All @@ -314,7 +353,7 @@ async function schemaMapper (key: WordPressBlock | WordPressBlock[], parentUid:
case 'core/heading':
case 'core/accordion-heading':
case 'core/list-item': {
const textUid = parentUid ? `${parentUid}.${getFieldUid(`${key?.name}_${clientIdForUid(key?.clientId)}`, affix)}` : getFieldUid(`${key?.name}_${clientIdForUid(key?.clientId)}`, affix);
const textUid = parentUid ? `${parentUid}.${getFieldUid(`${key?.name}${stableSuffix(key)}`, affix)}` : getFieldUid(`${key?.name}${stableSuffix(key)}`, affix);
return {
uid: textUid,
otherCmsField: getFieldName(key?.name),
Expand All @@ -331,7 +370,7 @@ async function schemaMapper (key: WordPressBlock | WordPressBlock[], parentUid:
case 'core/social-link':
case 'core/navigation-link': {

const LinkUid = parentUid ? `${parentUid}.${getFieldUid(key?.name, affix)}` : getFieldUid(`${key?.name}_${clientIdForUid(key?.clientId)}`, affix);
const LinkUid = parentUid ? `${parentUid}.${getFieldUid(`${key?.name}${stableSuffix(key)}`, affix)}` : getFieldUid(`${key?.name}${stableSuffix(key)}`, affix);
return {
Comment thread
aishwarya-cstk marked this conversation as resolved.
uid: LinkUid,
otherCmsField: getFieldName(key?.name),
Expand Down Expand Up @@ -377,7 +416,7 @@ async function schemaMapper (key: WordPressBlock | WordPressBlock[], parentUid:
}

const groupSchema: Field[] = [];
const groupUid = parentUid ? `${parentUid}.${getFieldUid(`${key?.name}_${clientIdForUid(key?.clientId)}`, affix)}` : getFieldUid(`${key?.name}_${clientIdForUid(key?.clientId)}`, affix);
const groupUid = parentUid ? `${parentUid}.${getFieldUid(`${key?.name}${stableSuffix(key)}`, affix)}` : getFieldUid(`${key?.name}${stableSuffix(key)}`, affix);

const innerBlocks = await processInnerBlocks(
key,
Expand Down Expand Up @@ -423,7 +462,7 @@ async function schemaMapper (key: WordPressBlock | WordPressBlock[], parentUid:
case 'core/accordion-panel':
case 'core/navigation': {
const groupSchema: Field[] = [];
const groupUid = parentUid ? `${parentUid}.${getFieldUid(`${key?.name}_${clientIdForUid(key?.clientId)}`, affix)}` : getFieldUid(`${key?.name}_${clientIdForUid(key?.clientId)}`, affix);
const groupUid = parentUid ? `${parentUid}.${getFieldUid(`${key?.name}${stableSuffix(key)}`, affix)}` : getFieldUid(`${key?.name}${stableSuffix(key)}`, affix);

const innerBlocks = await processInnerBlocks(
key,
Expand Down Expand Up @@ -465,14 +504,14 @@ async function schemaMapper (key: WordPressBlock | WordPressBlock[], parentUid:
const coverSchema: Field[] = []
if(key?.attributes?.url){
coverSchema.push({
uid: `${parentUid}.${getFieldUid(`${key?.name}_${clientIdForUid(key?.clientId)}`, affix)}`,
uid: `${parentUid}.${getFieldUid(`${key?.name}${stableSuffix(key)}`, affix)}`,
otherCmsField: 'media',
Comment thread
aishwarya-cstk marked this conversation as resolved.
otherCmsType: getFieldName(key?.attributes?.metadata?.name ?? key?.name),
contentstackField: 'media',
contentstackFieldUid: `${parentUid}.${getFieldUid(`${key?.name}_${clientIdForUid(key?.clientId)}`, affix)}`,
contentstackFieldUid: `${parentUid}.${getFieldUid(`${key?.name}${stableSuffix(key)}`, affix)}`,
contentstackFieldType: 'file',
backupFieldType: 'file',
backupFieldUid: `${parentUid}.${getFieldUid(`${key?.name}_${clientIdForUid(key?.clientId)}`, affix)}`,
backupFieldUid: `${parentUid}.${getFieldUid(`${key?.name}${stableSuffix(key)}`, affix)}`,
advanced: {}
});
}
Expand All @@ -498,7 +537,7 @@ async function schemaMapper (key: WordPressBlock | WordPressBlock[], parentUid:


case 'core/search': {
const searchEleUid = parentUid ? `${parentUid}.${getFieldUid(`${key?.name}_${clientIdForUid(key?.clientId)}`, affix)}` : getFieldUid(`${key?.name}_${clientIdForUid(key?.clientId)}`, affix);
const searchEleUid = parentUid ? `${parentUid}.${getFieldUid(`${key?.name}${stableSuffix(key)}`, affix)}` : getFieldUid(`${key?.name}${stableSuffix(key)}`, affix);
const searchEle = await processAttributes(key, searchEleUid,fieldName, affix);
const groupSchema: Field[] = [];
searchEle?.length > 0 && groupSchema?.push({
Expand Down Expand Up @@ -526,7 +565,7 @@ async function schemaMapper (key: WordPressBlock | WordPressBlock[], parentUid:

case 'core/button': {
const fieldName = parentFieldName ? `${parentFieldName} > ${getFieldName(key?.attributes?.metadata?.name ?? key?.name)}` : `${getFieldName(key?.attributes?.metadata?.name ?? key?.name)}` ;
const buttonUid = parentUid ? `${parentUid}.${getFieldUid(`${key?.name}_${clientIdForUid(key?.clientId)}`, affix)}` : getFieldUid(`${key?.name}_${clientIdForUid(key?.clientId)}`, affix);
const buttonUid = parentUid ? `${parentUid}.${getFieldUid(`${key?.name}${stableSuffix(key)}`, affix)}` : getFieldUid(`${key?.name}${stableSuffix(key)}`, affix);

return {
uid: buttonUid,
Expand All @@ -544,7 +583,7 @@ async function schemaMapper (key: WordPressBlock | WordPressBlock[], parentUid:

case 'core/buttons': {
const groupSchema: Field[] = [];
const groupUid = parentUid ? `${parentUid}.${getFieldUid(`${key?.name}_${clientIdForUid(key?.clientId)}`, affix)}` : getFieldUid(`${key?.name}_${clientIdForUid(key?.clientId)}`, affix);
const groupUid = parentUid ? `${parentUid}.${getFieldUid(`${key?.name}${stableSuffix(key)}`, affix)}` : getFieldUid(`${key?.name}${stableSuffix(key)}`, affix);

const innerBlocks = await processInnerBlocks(
key,
Expand All @@ -555,13 +594,13 @@ async function schemaMapper (key: WordPressBlock | WordPressBlock[], parentUid:
if (innerBlocks?.length === 1) {
const items = Array.isArray(innerBlocks[0]) ? innerBlocks[0] : [innerBlocks[0]];
items?.forEach((item: Field) => {
item.uid = `${parentUid}.${getFieldUid(`${key?.name}_${clientIdForUid(key?.clientId)}`, affix)}`;

item.uid = groupUid;
item.otherCmsField = getFieldName(resolveBlockName(key));
item.otherCmsType = getFieldName(resolveBlockName(key));
item.contentstackField = `${parentFieldName} > ${getFieldName(resolveBlockName(key))}`;
item.contentstackFieldUid = `${parentUid}.${getFieldUid(`${key?.name}_${clientIdForUid(key?.clientId)}`, affix)}`;
item.backupFieldUid = `${parentUid}.${getFieldUid(`${key?.name}_${clientIdForUid(key?.clientId)}`, affix)}`;
item.contentstackFieldUid = groupUid;
item.backupFieldUid = groupUid;
Comment thread
aishwarya-cstk marked this conversation as resolved.
});
return items;
}
Expand Down Expand Up @@ -596,7 +635,7 @@ async function schemaMapper (key: WordPressBlock | WordPressBlock[], parentUid:

case 'core/media-text': {
const mediaTextSchema: Field[] = [];
const mediaTextUid = parentUid ? `${parentUid}.${getFieldUid(`${key?.name}_${clientIdForUid(key?.clientId)}`, affix)}` : getFieldUid(`${key?.name}_${clientIdForUid(key?.clientId)}`, affix);
const mediaTextUid = parentUid ? `${parentUid}.${getFieldUid(`${key?.name}${stableSuffix(key)}`, affix)}` : getFieldUid(`${key?.name}${stableSuffix(key)}`, affix);
const innerBlocks =
key?.innerBlocks && key?.innerBlocks?.length > 0
? await processInnerBlocks(key, parentUid, parentFieldName, affix)
Expand Down
8 changes: 4 additions & 4 deletions upload-api/tests/unit/migration-wordpress/schemaMapper.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -212,9 +212,9 @@ describe('schemaMapper', () => {
const result = await schemaMapper(block, 'page', 'Page', affix);

result.forEach((field: any) => {
expect(field.contentstackFieldUid).toBe(`page.${getFieldUid('core/buttons_btns', affix)}`);
expect(field.uid).toBe(`page.${getFieldUid('core/buttons_btns', affix)}`);
expect(field.backupFieldUid).toBe(`page.${getFieldUid('core/buttons_btns', affix)}`);
expect(field.contentstackFieldUid).toBe(`page.${getFieldUid('core/buttons', affix)}`);
expect(field.uid).toBe(`page.${getFieldUid('core/buttons', affix)}`);
expect(field.backupFieldUid).toBe(`page.${getFieldUid('core/buttons', affix)}`);
expect(field.contentstackField).toContain('Page > ');
});
});
Expand Down Expand Up @@ -259,7 +259,7 @@ describe('schemaMapper', () => {

expect(Array.isArray(result)).toBe(true);
const p = result[0] as any;
expect(p.uid).toMatch(/^panel_uid\.paragraph_/);
expect(p.uid).toBe('panel_uid.paragraph');
expect(p.contentstackField).toBe('Panel > paragraph');
});

Expand Down
Loading