Skip to content

HCK-16468: Improve logging and run queries sequential #195

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
bigorn0 merged 5 commits into from
Jun 10, 2026
+118 −40
Merged

HCK-16468: Improve logging and run queries sequential #195

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
22c3634
HCK-16468: Make all call sequential
Nightlngale Jun 9, 2026
0950563
HCK-16468: Skip memory optimized query if unsupported
Nightlngale Jun 10, 2026
b644b00
HCK-16468: Fix remarks
Nightlngale Jun 10, 2026
58c1161
HCK-16468: Logged normalized query
Nightlngale Jun 10, 2026
8c77dd0
HCK-16468: Improve query formatting in logs
Nightlngale Jun 10, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
107 changes: 89 additions & 18 deletions reverse_engineering/databaseService/databaseService.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,15 @@ const { parseProcedure } = require('../helpers/parsers/parseProcedure');

const PERMISSION_DENIED_CODE = 297;

const normalizeQueryForLogging = queryParams =>
queryParams.map(param => {
if (typeof param === 'string') {
return param.replace(/\t/g, ' ');
}

return param;
});

const addPermissionDeniedMetaData = ({ error, meta }) => {
error.message =
'The user does not have permission to perform ' +
Expand All @@ -33,7 +42,12 @@ const getClient = async ({ client, dbName, meta, logger }) => {
},
async query(...queryParams) {
try {
return await currentDbConnectionClient.query(...queryParams);
logger.log('info', { query: normalizeQueryForLogging(queryParams) }, 'Performing query');
const start = Date.now();
const result = await currentDbConnectionClient.query(...queryParams);
const duration = Date.now() - start;
logger.log('info', { 'action': meta.action, duration: `${duration}ms` }, 'Query executed');
return result;
} catch (error) {
if (meta) {
if (error.number === PERMISSION_DENIED_CODE) {
Expand Down Expand Up @@ -265,6 +279,7 @@ const getDatabaseIndexes = async ({ client, dbName, tablesInfo, logger }) => {
});

logger.log('info', { message: `Get '${dbName}' database indexes.` }, 'Reverse Engineering');
logger.progress({ message: 'Discovering table index metadata', containerName: dbName, entityName: '' });

const tableAlias = 't';
const leftJoinClauseParts = Object.entries(tablesInfo).map(([schemaName, tableNames]) => {
Expand Down Expand Up @@ -315,6 +330,8 @@ const getIndexesBucketCount = async ({ client, dbName, indexesId, logger }) => {
logger,
});

logger.log('info', { message: `Get '${dbName}' database indexes bucket count.` }, 'Reverse Engineering');

return mapResponse(
await currentDbConnectionClient.query(`
SELECT hs.total_bucket_count, hs.index_id
Expand Down Expand Up @@ -343,6 +360,7 @@ const getSpatialIndexes = async ({ client, dbName, allUniqueSchemasAndTables, lo
});

logger.log('info', { message: `Get '${dbName}' database spatial indexes.` }, 'Reverse Engineering');
logger.progress({ message: 'Discovering spatial index metadata', containerName: dbName, entityName: '' });

const tableAlias = 't';
const whereClauseParts = getWhereClauseForUniqueSchemasAndTables({ tableAlias, allUniqueSchemasAndTables });
Expand Down Expand Up @@ -400,6 +418,7 @@ const getFullTextIndexes = async ({ client, dbName, allUniqueSchemasAndTables, l
});

logger.log('info', { message: `Get '${dbName}' database full text indexes.` }, 'Reverse Engineering');
logger.progress({ message: 'Discovering full-text index metadata', containerName: dbName, entityName: '' });

const tableAlias = 'F';
const whereClauseParts = getWhereClauseForUniqueSchemasAndTables({ tableAlias, allUniqueSchemasAndTables });
Expand Down Expand Up @@ -458,6 +477,7 @@ const getViewsIndexes = async ({ client, dbName, logger }) => {
});

logger.log('info', { message: `Get '${dbName}' database views indexes.` }, 'Reverse Engineering');
logger.progress({ message: 'Discovering view index metadata', containerName: dbName, entityName: '' });

const tableAlias = 'ind';

Expand Down Expand Up @@ -536,21 +556,56 @@ const getDatabaseMemoryOptimizedTables = async ({ client, dbName, logger }) => {
logger,
});

logger.log('info', { message: `Get '${dbName}' database memory optimized indexes.` }, 'Reverse Engineering');
logger.log(
'info',
{ message: `Detecting if '${dbName}' database supports memory optimized tables.` },
'Reverse Engineering',
);
logger.progress({ message: 'Discovering memory-optimized table metadata', containerName: dbName, entityName: '' });

const { isMemoryOptimizedTableSupported, isHistoryTableSupported } = await getSysTablesCatalogCapabilities({
client,
dbName,
});

if (!isMemoryOptimizedTableSupported) {
logger.log(
'info',
{ message: `Memory optimized tables are not supported in '${dbName}' database.` },
'Reverse Engineering',
);
return [];
}

if (isHistoryTableSupported) {
return mapResponse(
await currentDbConnectionClient.query`
SELECT
T.name,
T.durability,
T.durability_desc,
OBJECT_NAME(T.history_table_id) AS history_table,
SCHEMA_NAME(O.schema_id) AS history_schema,
T.temporal_type_desc,
T.is_memory_optimized
FROM sys.tables T
LEFT JOIN sys.objects O
ON T.history_table_id = O.object_id
WHERE T.is_memory_optimized = 1
`,
);
}

logger.log('info', { message: `History tables are not supported in '${dbName}' database.` }, 'Reverse Engineering');

return mapResponse(
await currentDbConnectionClient.query`
SELECT
T.name,
T.durability,
T.durability_desc,
OBJECT_NAME(T.history_table_id) AS history_table,
SCHEMA_NAME(O.schema_id) AS history_schema,
T.temporal_type_desc,
T.is_memory_optimized
FROM sys.tables T
LEFT JOIN sys.objects O
ON T.history_table_id = O.object_id
WHERE T.is_memory_optimized = 1
`,
);
Expand All @@ -569,6 +624,7 @@ const getDatabaseCheckConstraints = async ({ client, dbName, allUniqueSchemasAnd
});

logger.log('info', { message: `Get '${dbName}' database check constraints.` }, 'Reverse Engineering');
logger.progress({ message: 'Discovering check constraint metadata', containerName: dbName, entityName: '' });

const tableAlias = 't';
const whereClauseParts = getWhereClauseForUniqueSchemasAndTables({ tableAlias, allUniqueSchemasAndTables });
Expand Down Expand Up @@ -790,11 +846,10 @@ const getDatabaseXmlSchemaCollection = async ({ client, dbName, allUniqueSchemas
});

logger.log('info', { message: `Get '${dbName}' database xml schema collection.` }, 'Reverse Engineering');
logger.progress({ message: 'Discovering XML schema collection metadata', containerName: dbName, entityName: '' });

const schemaAlias = 'xsc';
const tableAlias = 'xcu';
const whereClauseParts = getWhereClauseForUniqueSchemasAndTables({
schemaAlias,
tableAlias,
allUniqueSchemasAndTables,
});
Expand Down Expand Up @@ -860,6 +915,7 @@ const getDatabaseUserDefinedTypes = async ({ client, dbName, logger }) => {
});

logger.log('info', { message: `Get '${dbName}' database UDTs.` }, 'Reverse Engineering');
logger.progress({ message: 'Discovering user-defined type metadata', containerName: dbName, entityName: '' });

return mapResponse(currentDbConnectionClient.query`
SELECT * FROM sys.types
Expand Down Expand Up @@ -889,6 +945,24 @@ const mapResponse = async (response = Promise.resolve({})) => {
return resp.recordset ? resp.recordset : resp;
};

const getSysTablesCatalogCapabilities = async ({ client, dbName }) => {
const connectionClient = await getNewConnectionClientByDb(client, dbName);
const [{ Has_HistoryTableId, Has_IsMemoryOptimized } = {}] = await mapResponse(
connectionClient.query(`
SELECT
COUNT(CASE WHEN name = 'history_table_id' THEN 1 END) AS Has_HistoryTableId,
COUNT(CASE WHEN name = 'is_memory_optimized' THEN 1 END) AS Has_IsMemoryOptimized
FROM sys.all_columns
WHERE object_id = OBJECT_ID('sys.tables')
`),
);

return {
isHistoryTableSupported: Boolean(Has_HistoryTableId),
isMemoryOptimizedTableSupported: Boolean(Has_IsMemoryOptimized),
};
};

const getDescriptionComments = async ({ client, dbName, schema, entity, logger }) => {
const currentDbConnectionClient = await getClient({
client,
Expand Down Expand Up @@ -922,12 +996,8 @@ const buildDescriptionCommentsRetrieveQuery = ({ schema, entity }) => {
return `SELECT objtype, objname, value FROM fn_listextendedproperty ('MS_Description', ${schemaTemplate}, ${entityTemplate});`;
};

const getWhereClauseForUniqueSchemasAndTables = ({
schemaAlias,
tableAlias,
allUniqueSchemasAndTables: { schemas, tables },
}) =>
`OBJECT_SCHEMA_NAME(${schemaAlias || tableAlias}.object_id) IN (${[...schemas].join(', ')})
const getWhereClauseForUniqueSchemasAndTables = ({ tableAlias, allUniqueSchemasAndTables: { schemas, tables } }) =>
`OBJECT_SCHEMA_NAME(${tableAlias}.object_id) IN (${[...schemas].join(', ')})
AND OBJECT_NAME(${tableAlias}.object_id) IN (${[...tables].join(', ')})`;

const getDatabaseProcedures = async ({ client, dbName, logger }) => {
Expand All @@ -943,6 +1013,7 @@ const getDatabaseProcedures = async ({ client, dbName, logger }) => {
});

logger.log('info', { message: `Get '${dbName}' database procedures.` }, 'Reverse Engineering');
logger.progress({ message: 'Discovering stored procedure metadata', containerName: dbName, entityName: '' });

const response = await currentDbConnectionClient.query(`
SELECT
Expand All @@ -951,11 +1022,11 @@ const getDatabaseProcedures = async ({ client, dbName, logger }) => {
sm.definition AS procedure_body,
ep.value AS description
FROM sys.procedures p
JOIN sys.schemas s
JOIN sys.schemas s
ON p.schema_id = s.schema_id
LEFT JOIN sys.sql_modules sm
LEFT JOIN sys.sql_modules sm
ON p.object_id = sm.object_id
LEFT JOIN sys.extended_properties ep
LEFT JOIN sys.extended_properties ep
ON ep.major_id = p.object_id
AND ep.minor_id = 0
AND ep.name = 'MS_Description'
Expand Down
51 changes: 29 additions & 22 deletions reverse_engineering/reverseEngineeringService/reverseEngineeringService.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -288,34 +288,41 @@ const addTotalBucketCountToDatabaseIndexes = ({ databaseIndexes, indexesBucketCo
const fetchDatabaseMetadata = async ({ client, dbName, tablesInfo, logger }) => {
const allUniqueSchemasAndTables = getAllUniqueSchemasAndTables({ tablesInfo });

const [
rawDatabaseIndexes,
databaseMemoryOptimizedTables,
databaseCheckConstraints,
xmlSchemaCollections,
databaseUDT,
viewsIndexes,
fullTextIndexes,
spatialIndexes,
procedures,
] = await Promise.all([
getDatabaseIndexes({ client, dbName, tablesInfo, logger }),
getDatabaseMemoryOptimizedTables({ client, dbName, logger }),
getDatabaseCheckConstraints({ client, dbName, allUniqueSchemasAndTables, logger }),
getDatabaseXmlSchemaCollection({ client, dbName, allUniqueSchemasAndTables, logger }),
getDatabaseUserDefinedTypes({ client, dbName, logger }),
getViewsIndexes({ client, dbName, logger }),
getFullTextIndexes({ client, dbName, allUniqueSchemasAndTables, logger }),
getSpatialIndexes({ client, dbName, allUniqueSchemasAndTables, logger }),
getDatabaseProcedures({ client, dbName, logger }),
]);

const rawDatabaseIndexes = await getDatabaseIndexes({ client, dbName, tablesInfo, logger });
const databaseCheckConstraints = await getDatabaseCheckConstraints({
client,
dbName,
allUniqueSchemasAndTables,
logger,
});
const viewsIndexes = await getViewsIndexes({ client, dbName, logger });
const fullTextIndexes = await getFullTextIndexes({
client,
dbName,
allUniqueSchemasAndTables,
logger,
});
const spatialIndexes = await getSpatialIndexes({
client,
dbName,
allUniqueSchemasAndTables,
logger,
});
const procedures = await getDatabaseProcedures({ client, dbName, logger });
const indexesBucketCount = await getIndexesBucketCount({
client,
dbName,
indexesId: rawDatabaseIndexes.map(i => i.index_id),
logger,
});
const databaseUDT = await getDatabaseUserDefinedTypes({ client, dbName, logger });
const databaseMemoryOptimizedTables = await getDatabaseMemoryOptimizedTables({ client, dbName, logger });
const xmlSchemaCollections = await getDatabaseXmlSchemaCollection({
client,
dbName,
allUniqueSchemasAndTables,
logger,
});

const uniqueDatabaseIndexesColumns = getUniqueIndexesColumns({ indexesColumns: rawDatabaseIndexes });
const databaseIndexes = addTotalBucketCountToDatabaseIndexes({
Expand Down