From 916fea1f0efc04b02f346ec60a3e40618aa9b622 Mon Sep 17 00:00:00 2001 From: Mike Wild Date: Mon, 30 Mar 2026 16:39:31 +0100 Subject: [PATCH 01/15] Add IT for DLQ alarms --- tests/integration/dlq-alarms.test.ts | 83 +++++++++++++++++++ tests/integration/dlq-redrive.test.ts | 8 +- tests/integration/helpers/sqs.ts | 4 +- .../inbound-sqs-to-webhook.test.ts | 8 +- tests/integration/metrics.test.ts | 8 +- tests/integration/package.json | 1 + 6 files changed, 103 insertions(+), 9 deletions(-) create mode 100644 tests/integration/dlq-alarms.test.ts diff --git a/tests/integration/dlq-alarms.test.ts b/tests/integration/dlq-alarms.test.ts new file mode 100644 index 00000000..cb476f70 --- /dev/null +++ b/tests/integration/dlq-alarms.test.ts @@ -0,0 +1,83 @@ +import { + CloudWatchClient, + DescribeAlarmsCommand, + type MetricAlarm, +} from "@aws-sdk/client-cloudwatch"; +import type { DeploymentDetails } from "@nhs-notify-client-callbacks/test-support/helpers"; +import { getDeploymentDetails } from "@nhs-notify-client-callbacks/test-support/helpers"; +import { + getMockItClient2Config, + getMockItClientConfig, +} from "./helpers/mock-client-config"; +import { buildMockClientDlqQueueUrl } from "./helpers/sqs"; + +function buildDlqDepthAlarmName( + { component, environment, project }: DeploymentDetails, + targetId: string, +): string { + return `${project}-${environment}-${component}-${targetId}-dlq-depth`; +} + +function getQueueNameFromUrl(queueUrl: string): string { + const queueName = queueUrl.split("/").pop(); + if (!queueName) { + throw new Error(`Unable to derive queue name from URL: ${queueUrl}`); + } + + return queueName; +} + +describe("DLQ alarms", () => { + let cloudWatchClient: CloudWatchClient; + let deploymentDetails: DeploymentDetails; + let targetIds: string[]; + + beforeAll(() => { + deploymentDetails = getDeploymentDetails(); + cloudWatchClient = new CloudWatchClient({ + region: deploymentDetails.region, + }); + + targetIds = [ + ...getMockItClientConfig().targets.map(({ targetId }) => targetId), + ...getMockItClient2Config().targets.map(({ targetId }) => targetId), + ]; + }); + + afterAll(() => { + cloudWatchClient.destroy(); + }); + + it("should create a DLQ depth alarm for every target DLQ", async () => { + const uniqueTargetIds = [...new Set(targetIds)]; + expect(uniqueTargetIds.length).toBeGreaterThan(0); + + for (const targetId of uniqueTargetIds) { + const alarmName = buildDlqDepthAlarmName(deploymentDetails, targetId); + const targetDlqQueueUrl = buildMockClientDlqQueueUrl(deploymentDetails, [ + { targetId }, + ]); + const targetDlqQueueName = getQueueNameFromUrl(targetDlqQueueUrl); + const response = await cloudWatchClient.send( + new DescribeAlarmsCommand({ + AlarmNames: [alarmName], + }), + ); + + const alarm: MetricAlarm | undefined = response.MetricAlarms?.[0]; + + expect(alarm?.AlarmName).toBe(alarmName); + expect(alarm?.MetricName).toBe("ApproximateNumberOfMessagesVisible"); + expect(alarm?.Namespace).toBe("AWS/SQS"); + expect(alarm?.Threshold).toBe(0); + expect(alarm?.Dimensions).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + Name: "QueueName", + Value: targetDlqQueueName, + }), + ]), + ); + } + }, 120_000); +}); diff --git a/tests/integration/dlq-redrive.test.ts b/tests/integration/dlq-redrive.test.ts index 3360a792..3847d9da 100644 --- a/tests/integration/dlq-redrive.test.ts +++ b/tests/integration/dlq-redrive.test.ts @@ -18,7 +18,10 @@ import { purgeQueues, sendSqsEvent, } from "./helpers/sqs"; -import { buildMockWebhookTargetPath } from "./helpers/mock-client-config"; +import { + buildMockWebhookTargetPath, + getMockItClientConfig, +} from "./helpers/mock-client-config"; import { awaitSignedCallbacksFromWebhookLogGroup } from "./helpers/cloudwatch"; import { createMessageStatusPublishEvent } from "./helpers/event-factories"; import sendEventToDlqAndRedrive from "./helpers/redrive"; @@ -32,12 +35,13 @@ describe("DLQ Redrive", () => { beforeAll(async () => { const deploymentDetails = getDeploymentDetails(); + const { targets } = getMockItClientConfig(); sqsClient = createSqsClient(deploymentDetails); cloudWatchClient = createCloudWatchLogsClient(deploymentDetails); inboundQueueUrl = buildInboundEventQueueUrl(deploymentDetails); - dlqQueueUrl = buildMockClientDlqQueueUrl(deploymentDetails); + dlqQueueUrl = buildMockClientDlqQueueUrl(deploymentDetails, targets); webhookLogGroupName = buildLambdaLogGroupName( deploymentDetails, "mock-webhook", diff --git a/tests/integration/helpers/sqs.ts b/tests/integration/helpers/sqs.ts index 7df2bee9..c98ea4ca 100644 --- a/tests/integration/helpers/sqs.ts +++ b/tests/integration/helpers/sqs.ts @@ -12,8 +12,6 @@ import { logger } from "@nhs-notify-client-callbacks/logger"; import type { DeploymentDetails } from "@nhs-notify-client-callbacks/test-support/helpers/deployment"; import { waitUntil } from "async-wait-until"; -import { getMockItClientConfig } from "./mock-client-config"; - const QUEUE_WAIT_TIMEOUT_MS = 60_000; const POLL_INTERVAL_MS = 500; const SQS_MAX_NUMBER_OF_MESSAGES = 1; @@ -48,8 +46,8 @@ function buildQueueUrl( export function buildMockClientDlqQueueUrl( deploymentDetails: DeploymentDetails, + targets: { targetId: string }[], ): string { - const { targets } = getMockItClientConfig(); return buildQueueUrl(deploymentDetails, `${targets[0].targetId}-dlq`); } diff --git a/tests/integration/inbound-sqs-to-webhook.test.ts b/tests/integration/inbound-sqs-to-webhook.test.ts index 3e61d090..7bf6c824 100644 --- a/tests/integration/inbound-sqs-to-webhook.test.ts +++ b/tests/integration/inbound-sqs-to-webhook.test.ts @@ -14,7 +14,10 @@ import { getDeploymentDetails, } from "@nhs-notify-client-callbacks/test-support/helpers"; import { assertCallbackHeaders } from "./helpers/signature"; -import { buildMockWebhookTargetPath } from "./helpers/mock-client-config"; +import { + buildMockWebhookTargetPath, + getMockItClientConfig, +} from "./helpers/mock-client-config"; import { awaitQueueMessage, awaitQueueMessageByMessageId, @@ -43,11 +46,12 @@ describe("SQS to Webhook Integration", () => { beforeAll(async () => { const deploymentDetails = getDeploymentDetails(); + const { targets } = getMockItClientConfig(); sqsClient = createSqsClient(deploymentDetails); cloudWatchClient = createCloudWatchLogsClient(deploymentDetails); callbackEventQueueUrl = buildInboundEventQueueUrl(deploymentDetails); - clientDlqQueueUrl = buildMockClientDlqQueueUrl(deploymentDetails); + clientDlqQueueUrl = buildMockClientDlqQueueUrl(deploymentDetails, targets); inboundEventDlqQueueUrl = buildInboundEventDlqQueueUrl(deploymentDetails); webhookLogGroupName = buildLambdaLogGroupName( deploymentDetails, diff --git a/tests/integration/metrics.test.ts b/tests/integration/metrics.test.ts index 6391c093..2f314f85 100644 --- a/tests/integration/metrics.test.ts +++ b/tests/integration/metrics.test.ts @@ -19,7 +19,10 @@ import { purgeQueues, sendSqsEvent, } from "./helpers/sqs"; -import { buildMockWebhookTargetPath } from "./helpers/mock-client-config"; +import { + buildMockWebhookTargetPath, + getMockItClientConfig, +} from "./helpers/mock-client-config"; import { awaitAllEmfMetricsInLogGroup, awaitSignedCallbacksFromWebhookLogGroup, @@ -37,11 +40,12 @@ describe("Metrics", () => { beforeAll(async () => { const deploymentDetails = getDeploymentDetails(); + const { targets } = getMockItClientConfig(); sqsClient = createSqsClient(deploymentDetails); cloudWatchClient = createCloudWatchLogsClient(deploymentDetails); callbackEventQueueUrl = buildInboundEventQueueUrl(deploymentDetails); - clientDlqQueueUrl = buildMockClientDlqQueueUrl(deploymentDetails); + clientDlqQueueUrl = buildMockClientDlqQueueUrl(deploymentDetails, targets); inboundEventDlqQueueUrl = buildInboundEventDlqQueueUrl(deploymentDetails); logGroupName = buildLambdaLogGroupName( deploymentDetails, diff --git a/tests/integration/package.json b/tests/integration/package.json index 7a4b6cbc..e3e6543d 100644 --- a/tests/integration/package.json +++ b/tests/integration/package.json @@ -13,6 +13,7 @@ "typecheck": "tsc --noEmit" }, "dependencies": { + "@aws-sdk/client-cloudwatch": "^3.1019.0", "@nhs-notify-client-callbacks/logger": "*", "@nhs-notify-client-callbacks/models": "*", "@aws-sdk/client-cloudwatch-logs": "^3.1020.0", From bf52fc52995bcbebfa6970ea205d24d1a28acd62 Mon Sep 17 00:00:00 2001 From: Mike Wild Date: Mon, 30 Mar 2026 16:40:34 +0100 Subject: [PATCH 02/15] Remove redundant infrastructure exists test - if the config bucket doesn't exist nothing would work --- .../integration/infrastructure-exists.test.ts | 30 ------------------- 1 file changed, 30 deletions(-) delete mode 100644 tests/integration/infrastructure-exists.test.ts diff --git a/tests/integration/infrastructure-exists.test.ts b/tests/integration/infrastructure-exists.test.ts deleted file mode 100644 index 37c88dc6..00000000 --- a/tests/integration/infrastructure-exists.test.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { HeadBucketCommand } from "@aws-sdk/client-s3"; -import type { S3Client } from "@aws-sdk/client-s3"; -import { - buildSubscriptionConfigBucketName, - createS3Client, - getDeploymentDetails, -} from "@nhs-notify-client-callbacks/test-support/helpers"; - -describe("Infrastructure exists", () => { - let s3Client: S3Client; - let bucketName: string; - - beforeAll(async () => { - const deploymentDetails = getDeploymentDetails(); - bucketName = buildSubscriptionConfigBucketName(deploymentDetails); - s3Client = createS3Client(deploymentDetails); - }); - - afterAll(() => { - s3Client?.destroy(); - }); - - it("should confirm the subscription config S3 bucket exists", async () => { - const response = await s3Client.send( - new HeadBucketCommand({ Bucket: bucketName }), - ); - - expect(response.$metadata.httpStatusCode).toBe(200); - }); -}); From 2c3e125b8e5bff91b867b50c5a51fa7663d2d406 Mon Sep 17 00:00:00 2001 From: Mike Wild Date: Mon, 30 Mar 2026 16:55:18 +0100 Subject: [PATCH 03/15] Target fan out test --- tests/integration/helpers/cloudwatch.ts | 48 ++++++++++++++ .../integration/helpers/mock-client-config.ts | 20 +++++- .../inbound-sqs-to-webhook.test.ts | 63 +++++++++++++++++-- 3 files changed, 123 insertions(+), 8 deletions(-) diff --git a/tests/integration/helpers/cloudwatch.ts b/tests/integration/helpers/cloudwatch.ts index b66aedb1..50815db0 100644 --- a/tests/integration/helpers/cloudwatch.ts +++ b/tests/integration/helpers/cloudwatch.ts @@ -144,6 +144,54 @@ export async function awaitSignedCallbacksFromWebhookLogGroup( return callbacks; } +export async function awaitSignedCallbacksByCountFromWebhookLogGroup( + client: CloudWatchLogsClient, + logGroupName: string, + messageId: string, + callbackType: CallbackItem["type"], + expectedCount: number, +): Promise { + logger.debug( + `Waiting for callbacks in webhook CloudWatch log group (messageId=${messageId}, callbackType=${callbackType}, expectedCount=${expectedCount}, logGroup=${logGroupName})`, + ); + + let callbacks: SignedCallback[] = []; + + try { + await waitUntil( + async () => { + callbacks = await querySignedCallbacksFromWebhookLogGroup( + client, + logGroupName, + messageId, + callbackType, + ); + return callbacks.length === expectedCount; + }, + { + timeout: CALLBACK_WAIT_TIMEOUT_MS, + intervalBetweenAttempts: POLL_INTERVAL_MS, + }, + ); + } catch (error) { + if (error instanceof TimeoutError) { + logger.warn( + `Timed out waiting for callbacks in webhook CloudWatch log group (messageId=${messageId}, callbackType=${callbackType}, expectedCount=${expectedCount}, timeoutMs=${CALLBACK_WAIT_TIMEOUT_MS})`, + ); + } else { + throw error; + } + } + + if (callbacks.length !== expectedCount) { + throw new Error( + `Expected exactly ${expectedCount} callbacks for messageId="${messageId}" callbackType="${callbackType}", but found ${callbacks.length}`, + ); + } + + return callbacks; +} + type EmfEntry = Record; function collectMetricNamesFromEvent( diff --git a/tests/integration/helpers/mock-client-config.ts b/tests/integration/helpers/mock-client-config.ts index 42bac438..ad047fd3 100644 --- a/tests/integration/helpers/mock-client-config.ts +++ b/tests/integration/helpers/mock-client-config.ts @@ -47,9 +47,25 @@ export function getMockItClient2Config(): MockItClientConfig { return getClientConfig("client2"); } +function buildWebhookTargetPaths(key: ClientFixtureKey): string[] { + const config = getClientConfig(key); + return config.targets.map(({ targetId }) => `/${targetId}`); +} + export function buildMockWebhookTargetPath( key: ClientFixtureKey = "client1", ): string { - const config = getClientConfig(key); - return `/${config.targets[0].targetId}`; + const paths = buildWebhookTargetPaths(key); + + if (paths.length === 0) { + throw new Error(`No webhook targets configured for fixture key: ${key}`); + } + + return paths[0]; +} + +export function buildMockWebhookTargetPaths( + key: ClientFixtureKey = "client1", +): string[] { + return buildWebhookTargetPaths(key); } diff --git a/tests/integration/inbound-sqs-to-webhook.test.ts b/tests/integration/inbound-sqs-to-webhook.test.ts index 7bf6c824..8f679420 100644 --- a/tests/integration/inbound-sqs-to-webhook.test.ts +++ b/tests/integration/inbound-sqs-to-webhook.test.ts @@ -1,5 +1,5 @@ -import { DeleteMessageCommand, SQSClient } from "@aws-sdk/client-sqs"; import { CloudWatchLogsClient } from "@aws-sdk/client-cloudwatch-logs"; +import { DeleteMessageCommand, SQSClient } from "@aws-sdk/client-sqs"; import { type ChannelStatusData, type MessageStatusData, @@ -13,11 +13,18 @@ import { createSqsClient, getDeploymentDetails, } from "@nhs-notify-client-callbacks/test-support/helpers"; -import { assertCallbackHeaders } from "./helpers/signature"; +import { awaitSignedCallbacksByCountFromWebhookLogGroup } from "./helpers/cloudwatch"; +import { + createChannelStatusPublishEvent, + createMessageStatusPublishEvent, +} from "./helpers/event-factories"; import { buildMockWebhookTargetPath, + buildMockWebhookTargetPaths, + getMockItClient2Config, getMockItClientConfig, } from "./helpers/mock-client-config"; +import { assertCallbackHeaders } from "./helpers/signature"; import { awaitQueueMessage, awaitQueueMessageByMessageId, @@ -26,10 +33,6 @@ import { purgeQueues, sendSqsEvent, } from "./helpers/sqs"; -import { - createChannelStatusPublishEvent, - createMessageStatusPublishEvent, -} from "./helpers/event-factories"; import { processChannelStatusEvent, processMessageStatusEvent, @@ -103,6 +106,54 @@ describe("SQS to Webhook Integration", () => { assertCallbackHeaders(callbacks[0]); }, 120_000); + + it("should fan out a message status event to subscription with multiple target endpoints", async () => { + const client2Config = getMockItClient2Config(); + const expectedPaths = buildMockWebhookTargetPaths("client2"); + + const messageStatusEvent: StatusPublishEvent = + createMessageStatusPublishEvent({ + data: { + clientId: client2Config.clientId, + }, + }); + + await sendSqsEvent(sqsClient, callbackEventQueueUrl, messageStatusEvent); + await ensureInboundQueueIsEmpty(sqsClient, callbackEventQueueUrl); + + const callbacks = await awaitSignedCallbacksByCountFromWebhookLogGroup( + cloudWatchClient, + webhookLogGroupName, + messageStatusEvent.data.messageId, + "MessageStatus", + expectedPaths.length, + ); + + expect(callbacks).toHaveLength(expectedPaths.length); + + const actualPaths = callbacks + .map((callback) => callback.path) + .toSorted((a, b) => a.localeCompare(b)); + expect(actualPaths).toEqual( + expectedPaths.toSorted((a, b) => a.localeCompare(b)), + ); + + for (const callback of callbacks) { + expect(callback.payload).toMatchObject({ + type: "MessageStatus", + attributes: expect.objectContaining({ + messageId: messageStatusEvent.data.messageId, + messageStatus: "delivered", + }), + }); + + assertCallbackHeaders( + callback, + client2Config.apiKeyVar, + client2Config.applicationIdVar, + ); + } + }, 120_000); }); describe("Channel Status Event Flow", () => { From 06d9d784d8a2b7f2416071c38060fad43cc43477 Mon Sep 17 00:00:00 2001 From: Mike Wild Date: Mon, 30 Mar 2026 17:32:41 +0100 Subject: [PATCH 04/15] Assert DLQ for every target --- tests/integration/dlq-alarms.test.ts | 15 ++------ tests/integration/dlq-redrive.test.ts | 38 +++++++++++++------ .../integration/helpers/mock-client-config.ts | 25 ++++++++++++ 3 files changed, 56 insertions(+), 22 deletions(-) diff --git a/tests/integration/dlq-alarms.test.ts b/tests/integration/dlq-alarms.test.ts index cb476f70..7f29f88f 100644 --- a/tests/integration/dlq-alarms.test.ts +++ b/tests/integration/dlq-alarms.test.ts @@ -5,10 +5,7 @@ import { } from "@aws-sdk/client-cloudwatch"; import type { DeploymentDetails } from "@nhs-notify-client-callbacks/test-support/helpers"; import { getDeploymentDetails } from "@nhs-notify-client-callbacks/test-support/helpers"; -import { - getMockItClient2Config, - getMockItClientConfig, -} from "./helpers/mock-client-config"; +import { getAllSubscriptionTargetIds } from "./helpers/mock-client-config"; import { buildMockClientDlqQueueUrl } from "./helpers/sqs"; function buildDlqDepthAlarmName( @@ -38,10 +35,7 @@ describe("DLQ alarms", () => { region: deploymentDetails.region, }); - targetIds = [ - ...getMockItClientConfig().targets.map(({ targetId }) => targetId), - ...getMockItClient2Config().targets.map(({ targetId }) => targetId), - ]; + targetIds = getAllSubscriptionTargetIds(); }); afterAll(() => { @@ -49,10 +43,9 @@ describe("DLQ alarms", () => { }); it("should create a DLQ depth alarm for every target DLQ", async () => { - const uniqueTargetIds = [...new Set(targetIds)]; - expect(uniqueTargetIds.length).toBeGreaterThan(0); + expect(targetIds.length).toBeGreaterThan(0); - for (const targetId of uniqueTargetIds) { + for (const targetId of targetIds) { const alarmName = buildDlqDepthAlarmName(deploymentDetails, targetId); const targetDlqQueueUrl = buildMockClientDlqQueueUrl(deploymentDetails, [ { targetId }, diff --git a/tests/integration/dlq-redrive.test.ts b/tests/integration/dlq-redrive.test.ts index 3847d9da..1398ae4a 100644 --- a/tests/integration/dlq-redrive.test.ts +++ b/tests/integration/dlq-redrive.test.ts @@ -20,6 +20,7 @@ import { } from "./helpers/sqs"; import { buildMockWebhookTargetPath, + getAllSubscriptionTargetIds, getMockItClientConfig, } from "./helpers/mock-client-config"; import { awaitSignedCallbacksFromWebhookLogGroup } from "./helpers/cloudwatch"; @@ -30,42 +31,57 @@ describe("DLQ Redrive", () => { let sqsClient: SQSClient; let cloudWatchClient: CloudWatchLogsClient; let dlqQueueUrl!: string; + let allTargetDlqQueueUrls: string[]; let inboundQueueUrl: string; let webhookLogGroupName: string; beforeAll(async () => { const deploymentDetails = getDeploymentDetails(); - const { targets } = getMockItClientConfig(); + const mockClient1 = getMockItClientConfig(); + + const allSubscriptionTargetIds = getAllSubscriptionTargetIds(); sqsClient = createSqsClient(deploymentDetails); cloudWatchClient = createCloudWatchLogsClient(deploymentDetails); inboundQueueUrl = buildInboundEventQueueUrl(deploymentDetails); - dlqQueueUrl = buildMockClientDlqQueueUrl(deploymentDetails, targets); + dlqQueueUrl = buildMockClientDlqQueueUrl( + deploymentDetails, + mockClient1.targets, + ); + allTargetDlqQueueUrls = allSubscriptionTargetIds.map((targetId) => + buildMockClientDlqQueueUrl(deploymentDetails, [{ targetId }]), + ); webhookLogGroupName = buildLambdaLogGroupName( deploymentDetails, "mock-webhook", ); - await purgeQueues(sqsClient, [inboundQueueUrl, dlqQueueUrl]); + await purgeQueues(sqsClient, [inboundQueueUrl, ...allTargetDlqQueueUrls]); }); afterAll(async () => { - await purgeQueues(sqsClient, [inboundQueueUrl, dlqQueueUrl]); + await purgeQueues(sqsClient, [inboundQueueUrl, ...allTargetDlqQueueUrls]); sqsClient.destroy(); cloudWatchClient.destroy(); }); describe("Infrastructure validation", () => { - it("should confirm the target DLQ is accessible", async () => { - const response = await sqsClient.send( - new GetQueueAttributesCommand({ - QueueUrl: dlqQueueUrl, - AttributeNames: ["QueueArn", "ApproximateNumberOfMessages"], - }), + it("should confirm a target DLQ is accessible for all configured subscription targets across mock-client-1 and mock-client-2", async () => { + const responses = await Promise.all( + allTargetDlqQueueUrls.map((queueUrl) => + sqsClient.send( + new GetQueueAttributesCommand({ + QueueUrl: queueUrl, + AttributeNames: ["QueueArn", "ApproximateNumberOfMessages"], + }), + ), + ), ); - expect(response.Attributes?.QueueArn).toBeDefined(); + for (const response of responses) { + expect(response.Attributes?.QueueArn).toBeDefined(); + } }); it("should confirm the inbound event queue exists and is accessible", async () => { diff --git a/tests/integration/helpers/mock-client-config.ts b/tests/integration/helpers/mock-client-config.ts index ad047fd3..c03002de 100644 --- a/tests/integration/helpers/mock-client-config.ts +++ b/tests/integration/helpers/mock-client-config.ts @@ -24,6 +24,16 @@ export const CLIENT_FIXTURES = { export type ClientFixtureKey = keyof typeof CLIENT_FIXTURES; +const ALL_CLIENT_FIXTURE_KEYS = Object.keys( + CLIENT_FIXTURES, +) as ClientFixtureKey[]; + +function dedupe(values: string[]): string[] { + return values.filter( + (value, index, allValues) => allValues.indexOf(value) === index, + ); +} + export function getClientConfig(key: ClientFixtureKey): MockItClientConfig { // eslint-disable-next-line security/detect-object-injection -- key is constrained to ClientFixtureKey, a keyof the hardcoded as-const CLIENT_FIXTURES object const { apiKeyVar, applicationIdVar, fixture } = CLIENT_FIXTURES[key]; @@ -69,3 +79,18 @@ export function buildMockWebhookTargetPaths( ): string[] { return buildWebhookTargetPaths(key); } + +export function getSubscriptionTargetIds( + key: ClientFixtureKey = "client1", +): string[] { + const config = getClientConfig(key); + return dedupe( + config.subscriptions.flatMap((subscription) => subscription.targetIds), + ); +} + +export function getAllSubscriptionTargetIds( + keys: ClientFixtureKey[] = ALL_CLIENT_FIXTURE_KEYS, +): string[] { + return dedupe(keys.flatMap((key) => getSubscriptionTargetIds(key))); +} From 5178047e1660f8146c29914015bacc6a475ed39f Mon Sep 17 00:00:00 2001 From: Mike Wild Date: Mon, 30 Mar 2026 17:40:06 +0100 Subject: [PATCH 05/15] fixup! Assert DLQ for every target --- tests/integration/dlq-redrive.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/dlq-redrive.test.ts b/tests/integration/dlq-redrive.test.ts index 1398ae4a..e88e4920 100644 --- a/tests/integration/dlq-redrive.test.ts +++ b/tests/integration/dlq-redrive.test.ts @@ -67,7 +67,7 @@ describe("DLQ Redrive", () => { }); describe("Infrastructure validation", () => { - it("should confirm a target DLQ is accessible for all configured subscription targets across mock-client-1 and mock-client-2", async () => { + it("should confirm a target DLQ is accessible for all configured subscription targets", async () => { const responses = await Promise.all( allTargetDlqQueueUrls.map((queueUrl) => sqsClient.send( From b6d51f90f9d240ffb13dba21a88ceaa1868a1bc1 Mon Sep 17 00:00:00 2001 From: Mike Wild Date: Tue, 31 Mar 2026 11:15:35 +0100 Subject: [PATCH 06/15] Add script to run integration tests locally --- scripts/tests/integration-local.sh | 45 ++++++++++++++++++++++++++++++ scripts/tests/integration.sh | 6 +++- scripts/tests/test.mk | 3 ++ 3 files changed, 53 insertions(+), 1 deletion(-) create mode 100755 scripts/tests/integration-local.sh diff --git a/scripts/tests/integration-local.sh b/scripts/tests/integration-local.sh new file mode 100755 index 00000000..8657f76b --- /dev/null +++ b/scripts/tests/integration-local.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +set -euo pipefail + +# Run integration tests against a live environment. +# +# Usage (via make): +# ENVIRONMENT= make test-integration-local +# ENVIRONMENT= make test-integration-local FILE=metrics +# ENVIRONMENT= make test-integration-local NAME="should emit processing metrics" +# ENVIRONMENT= make test-integration-local FILE=metrics NAME="should emit processing metrics" +# +# Optional overrides: +# AWS_REGION (default: eu-west-2) +# +# Required: +# AWS_PROFILE + +if [ -z "${ENVIRONMENT:-}" ]; then + echo "Error: ENVIRONMENT must be set before running this target." >&2 + echo "Example: ENVIRONMENT= make test-integration-local" >&2 + exit 1 +fi + +if [ -z "${AWS_PROFILE:-}" ]; then + echo "Error: AWS_PROFILE must be set before running this target." >&2 + exit 1 +fi + +AWS_REGION="${AWS_REGION:-eu-west-2}" +LOG_LEVEL="${LOG_LEVEL:-debug}" +NODE_OPTIONS="${NODE_OPTIONS:---experimental-vm-modules}" +COMPONENT="callbacks" +PROJECT="nhs" + +if ! aws sts get-caller-identity --profile "$AWS_PROFILE" >/dev/null 2>&1; then + echo "No active AWS SSO session for profile '$AWS_PROFILE'. Running aws sso login..." + aws sso login --profile "$AWS_PROFILE" +fi + +AWS_ACCOUNT_ID="$(aws sts get-caller-identity --profile "$AWS_PROFILE" --query Account --output text)" + +export AWS_PROFILE AWS_REGION LOG_LEVEL NODE_OPTIONS AWS_ACCOUNT_ID ENVIRONMENT PROJECT COMPONENT + +CI=true exec ./scripts/tests/integration.sh diff --git a/scripts/tests/integration.sh b/scripts/tests/integration.sh index fcc89389..9ba79472 100755 --- a/scripts/tests/integration.sh +++ b/scripts/tests/integration.sh @@ -8,4 +8,8 @@ npm ci source ./scripts/tests/integration-env.sh -npm run test:integration +JEST_ARGS=() +[ -n "${FILE:-}" ] && JEST_ARGS+=("$FILE") +[ -n "${NAME:-}" ] && JEST_ARGS+=(--testNamePattern "$NAME") + +npm run test:integration --workspace tests/integration -- "${JEST_ARGS[@]}" diff --git a/scripts/tests/test.mk b/scripts/tests/test.mk index 2615cfe3..c00a9b1c 100644 --- a/scripts/tests/test.mk +++ b/scripts/tests/test.mk @@ -29,6 +29,9 @@ test-contract: # Run your contract tests from scripts/test/contract @Testing test-integration: # Run your integration tests from scripts/test/integration @Testing make _test name="integration" +test-integration-local: # Run integration tests locally against a remoptely deployed environment (requires ENVIRONMENT) @Testing + make _test name="integration-local" + test-load: # Run all your load tests @Testing make \ test-capacity \ From 1b272c34f63aae00587fedbbc7ae01f71855fad5 Mon Sep 17 00:00:00 2001 From: Mike Wild Date: Tue, 31 Mar 2026 12:15:14 +0100 Subject: [PATCH 07/15] Add integration debug script and readme --- scripts/tests/integration-debug.sh | 283 +++++++++++++++++++++++++++++ scripts/tests/test.mk | 6 + tests/integration/README.md | 68 +++++++ 3 files changed, 357 insertions(+) create mode 100755 scripts/tests/integration-debug.sh create mode 100644 tests/integration/README.md diff --git a/scripts/tests/integration-debug.sh b/scripts/tests/integration-debug.sh new file mode 100755 index 00000000..6a684341 --- /dev/null +++ b/scripts/tests/integration-debug.sh @@ -0,0 +1,283 @@ +#!/bin/bash + +set -euo pipefail + +# Debug a live environment: inspect queues, tail logs, check pipe state. +# +# Usage (via make): +# ENVIRONMENT= AWS_PROFILE= make test-integration-debug ACTION=queue-status +# ENVIRONMENT= AWS_PROFILE= make test-integration-debug ACTION=tail-transform +# ENVIRONMENT= AWS_PROFILE= make test-integration-debug ACTION=tail-transform LOG_FILTER= +# +# Actions: +# queue-status Show SQS queue message counts +# queue-peek Peek one message from each SQS queue +# tail-transform Tail client-transform-filter lambda logs +# tail-webhook Tail mock-webhook lambda logs +# tail-pipe Tail EventBridge pipe log group +# pipe-state Show EventBridge pipe state and recent metrics +# +# Required: +# ENVIRONMENT +# AWS_PROFILE +# ACTION +# +# Optional: +# LOG_FILTER CloudWatch Logs filter pattern / text +# AWS_REGION (default: eu-west-2) + +if [ -z "${ENVIRONMENT:-}" ]; then + echo "Error: ENVIRONMENT must be set before running this target." >&2 + echo "Example: ENVIRONMENT= AWS_PROFILE= make test-integration-debug ACTION=queue-status" >&2 + exit 1 +fi + +if [ -z "${AWS_PROFILE:-}" ]; then + echo "Error: AWS_PROFILE must be set before running this target." >&2 + exit 1 +fi + +if [ -z "${ACTION:-}" ]; then + echo "Error: ACTION must be set before running this target." >&2 + echo "Actions: queue-status, queue-peek, tail-transform, tail-webhook, tail-pipe, pipe-state" >&2 + exit 1 +fi + +REGION="${AWS_REGION:-eu-west-2}" +LOG_FILTER="${LOG_FILTER:-}" +SUBSCRIPTION_FIXTURE_PATH="${SUBSCRIPTION_FIXTURE_PATH:-tests/integration/fixtures/subscriptions/mock-client-1.json}" + +if ! aws sts get-caller-identity --profile "$AWS_PROFILE" >/dev/null 2>&1; then + echo "No active AWS SSO session for profile '$AWS_PROFILE'. Running aws sso login..." + aws sso login --profile "$AWS_PROFILE" +fi + +ACCOUNT_ID="$(aws sts get-caller-identity --profile "$AWS_PROFILE" --query Account --output text)" + +PREFIX="nhs-${ENVIRONMENT}-callbacks" +PIPE_NAME="${PREFIX}-main" + +print_section() { + echo "" + echo "========================================" + echo "$1" + echo "========================================" +} + +queue_url() { + local queue_name="$1" + echo "https://sqs.${REGION}.amazonaws.com/${ACCOUNT_ID}/${queue_name}" +} + +target_dlq_queue_name() { + local target_id + + if [ ! -f "$SUBSCRIPTION_FIXTURE_PATH" ]; then + echo "Error: subscription fixture not found: $SUBSCRIPTION_FIXTURE_PATH" >&2 + exit 1 + fi + + target_id="$(jq -r '.targets[0].targetId // empty' "$SUBSCRIPTION_FIXTURE_PATH")" + if [ -z "$target_id" ]; then + echo "Error: unable to read targets[0].targetId from $SUBSCRIPTION_FIXTURE_PATH" >&2 + exit 1 + fi + + echo "${PREFIX}-${target_id}-dlq-queue" +} + +show_queue_counts() { + local title="$1" + local name="$2" + + print_section "$title" + aws sqs get-queue-attributes \ + --region "$REGION" \ + --profile "$AWS_PROFILE" \ + --no-cli-pager \ + --queue-url "$(queue_url "$name")" \ + --attribute-names ApproximateNumberOfMessages ApproximateNumberOfMessagesNotVisible \ + --query '{ApproximateNumberOfMessages: Attributes.ApproximateNumberOfMessages, ApproximateNumberOfMessagesNotVisible: Attributes.ApproximateNumberOfMessagesNotVisible}' +} + +action_queue_status() { + show_queue_counts "Mock Target DLQ - Queue Message Counts" "$(target_dlq_queue_name)" + show_queue_counts "Inbound Event Queue - Queue Message Counts" "${PREFIX}-inbound-event-queue" + show_queue_counts "Inbound Event DLQ - Queue Message Counts" "${PREFIX}-inbound-event-dlq" +} + +peek_queue_message() { + local title="$1" + local name="$2" + + print_section "$title" + aws sqs receive-message \ + --region "$REGION" \ + --profile "$AWS_PROFILE" \ + --no-cli-pager \ + --queue-url "$(queue_url "$name")" \ + --attribute-names All \ + --message-attribute-names All \ + --max-number-of-messages 1 \ + --visibility-timeout 0 \ + --output json \ + | jq 'if (.Messages // [] | length) == 0 + then {message: "No messages"} + else .Messages[0] | {Body, Attributes, MessageAttributes} + end' +} + +action_queue_peek() { + peek_queue_message "Mock Target DLQ - Message Peek" "$(target_dlq_queue_name)" + peek_queue_message "Inbound Event Queue - Message Peek" "${PREFIX}-inbound-event-queue" + peek_queue_message "Inbound Event DLQ - Message Peek" "${PREFIX}-inbound-event-dlq" +} + +log_filter_args() { + local -a args=() + local escaped_log_filter + if [[ -n "$LOG_FILTER" ]]; then + escaped_log_filter="${LOG_FILTER//\"/\\\"}" + # CloudWatch filter patterns treat quoted strings as exact phrases. + args+=(--filter-pattern "\"$escaped_log_filter\"") + fi + + printf '%s\n' "${args[@]}" +} + +action_tail_transform() { + local -a filter_args=() + mapfile -t filter_args < <(log_filter_args) + + print_section "Transform/Filter Lambda Logs" + aws logs tail \ + "/aws/lambda/${PREFIX}-client-transform-filter" \ + --region "$REGION" \ + --profile "$AWS_PROFILE" \ + --since 2h \ + --follow \ + --format short \ + "${filter_args[@]}" +} + +action_tail_webhook() { + local -a filter_args=() + mapfile -t filter_args < <(log_filter_args) + + print_section "Mock Webhook Lambda Logs" + aws logs tail \ + "/aws/lambda/${PREFIX}-mock-webhook" \ + --region "$REGION" \ + --profile "$AWS_PROFILE" \ + --since 2h \ + --follow \ + --format short \ + "${filter_args[@]}" +} + +action_tail_pipe() { + local pipe_log_group_arn + local pipe_log_group_name + local -a filter_args=() + + mapfile -t filter_args < <(log_filter_args) + + pipe_log_group_arn=$(aws pipes describe-pipe \ + --region "$REGION" \ + --profile "$AWS_PROFILE" \ + --name "$PIPE_NAME" \ + --no-cli-pager \ + --query "LogConfiguration.CloudwatchLogsLogDestination.LogGroupArn" \ + --output text) + + if [[ -z "$pipe_log_group_arn" || "$pipe_log_group_arn" == "None" ]]; then + echo "No CloudWatch log group configured for pipe: $PIPE_NAME" + return 1 + fi + + pipe_log_group_name="${pipe_log_group_arn#*:log-group:}" + + print_section "EventBridge Pipe Logs" + aws logs tail \ + "$pipe_log_group_name" \ + --region "$REGION" \ + --profile "$AWS_PROFILE" \ + --since 2h \ + --follow \ + --format short \ + "${filter_args[@]}" +} + +pipe_metric_sum() { + local metric_name="$1" + local start_time="$2" + local end_time="$3" + + aws cloudwatch get-metric-statistics \ + --region "$REGION" \ + --profile "$AWS_PROFILE" \ + --namespace "AWS/Pipes" \ + --metric-name "$metric_name" \ + --dimensions "Name=PipeName,Value=$PIPE_NAME" \ + --start-time "$start_time" \ + --end-time "$end_time" \ + --period 60 \ + --statistics Sum \ + --query "sum(Datapoints[].Sum)" \ + --output text \ + --no-cli-pager +} + +action_pipe_state() { + local start_time + local end_time + local execution_succeeded + local execution_failed + local dead_lettered_events + + start_time="$(date -u -d '10 minutes ago' +%Y-%m-%dT%H:%M:%SZ)" + end_time="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + + print_section "EventBridge Pipe State" + aws pipes describe-pipe \ + --region "$REGION" \ + --profile "$AWS_PROFILE" \ + --name "$PIPE_NAME" \ + --no-cli-pager \ + --query '{Name:Name,CurrentState:CurrentState,DesiredState:DesiredState,StateReason:StateReason,LogConfiguration:LogConfiguration}' + + execution_succeeded="$(pipe_metric_sum "ExecutionSucceeded" "$start_time" "$end_time")" + execution_failed="$(pipe_metric_sum "ExecutionFailed" "$start_time" "$end_time")" + dead_lettered_events="$(pipe_metric_sum "DeadLetteredEvents" "$start_time" "$end_time")" + + print_section "EventBridge Pipe Metrics (last 10 minutes)" + echo "ExecutionSucceeded: ${execution_succeeded}" + echo "ExecutionFailed: ${execution_failed}" + echo "DeadLetteredEvents: ${dead_lettered_events}" +} + +case "$ACTION" in + queue-status) + action_queue_status + ;; + queue-peek) + action_queue_peek + ;; + tail-transform) + action_tail_transform + ;; + tail-webhook) + action_tail_webhook + ;; + tail-pipe) + action_tail_pipe + ;; + pipe-state) + action_pipe_state + ;; + *) + echo "Unknown action: $ACTION" >&2 + echo "Actions: queue-status, queue-peek, tail-transform, tail-webhook, tail-pipe, pipe-state" >&2 + exit 1 + ;; +esac diff --git a/scripts/tests/test.mk b/scripts/tests/test.mk index c00a9b1c..d9303d92 100644 --- a/scripts/tests/test.mk +++ b/scripts/tests/test.mk @@ -32,6 +32,12 @@ test-integration: # Run your integration tests from scripts/test/integration @Te test-integration-local: # Run integration tests locally against a remoptely deployed environment (requires ENVIRONMENT) @Testing make _test name="integration-local" +test-integration-debug: # Debug a live environment - inspect queues, tail logs, check pipe state (requires ENVIRONMENT, AWS_PROFILE, ACTION) @Testing + make _test name="integration-debug" ACTION="$(or $(ACTION),$(word 2,$(MAKECMDGOALS)))" + +queue-status queue-peek tail-transform tail-webhook tail-pipe pipe-state: + @: + test-load: # Run all your load tests @Testing make \ test-capacity \ diff --git a/tests/integration/README.md b/tests/integration/README.md new file mode 100644 index 00000000..4be0a839 --- /dev/null +++ b/tests/integration/README.md @@ -0,0 +1,68 @@ +# Integration Tests + +This folder contains integration tests for the callbacks service. + +These instructions are for running integration tests locally against a remotely deployed environment. +In normal delivery flow, integration tests are triggered via the CI workflow. + +## Prerequisites + +- AWS CLI installed and authenticated +- Valid AWS SSO session for your chosen profile (the script will prompt for login if needed) +- Environment variables: + - `ENVIRONMENT` (required) + - `AWS_PROFILE` (required) + - `AWS_REGION` (optional, defaults to `eu-west-2`) + +## Run Integration Tests Locally + +Run all integration tests against a deployed environment: + +```sh +ENVIRONMENT= AWS_PROFILE= make test-integration-local +``` + +Run a single test file: + +```sh +ENVIRONMENT= AWS_PROFILE= make test-integration-local FILE=metrics +``` + +Run a single test name: + +```sh +ENVIRONMENT= AWS_PROFILE= make test-integration-local NAME="should emit processing metrics when a valid event is fully processed" +``` + +Combine file + test name: + +```sh +ENVIRONMENT= AWS_PROFILE= make test-integration-local FILE=metrics NAME="should emit processing metrics when a valid event is fully processed" +``` + +## Debug an Environment + +Use debug actions to inspect queues, tail logs, and check pipe state: + +```sh +ENVIRONMENT= AWS_PROFILE= make test-integration-debug queue-status +``` + +Tail webhook logs: + +```sh +ENVIRONMENT= AWS_PROFILE= make test-integration-debug tail-webhook +``` + +Tail webhook logs filtered to a specific message ID: + +```sh +ENVIRONMENT= AWS_PROFILE= LOG_FILTER=SOME-MESSAGE-ID make test-integration-debug tail-webhook +``` + +Other available actions: + +- `queue-peek` +- `tail-transform` +- `tail-pipe` +- `pipe-state` From a0c2d4e4e878dc25acb4d9a7d8cf6982f4f41cba Mon Sep 17 00:00:00 2001 From: Mike Wild Date: Tue, 31 Mar 2026 14:10:18 +0100 Subject: [PATCH 08/15] fixup! Add script to run integration tests locally --- scripts/tests/integration-local.sh | 6 +++--- scripts/tests/integration.sh | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/tests/integration-local.sh b/scripts/tests/integration-local.sh index 8657f76b..7d0e36c4 100755 --- a/scripts/tests/integration-local.sh +++ b/scripts/tests/integration-local.sh @@ -6,9 +6,9 @@ set -euo pipefail # # Usage (via make): # ENVIRONMENT= make test-integration-local -# ENVIRONMENT= make test-integration-local FILE=metrics -# ENVIRONMENT= make test-integration-local NAME="should emit processing metrics" -# ENVIRONMENT= make test-integration-local FILE=metrics NAME="should emit processing metrics" +# ENVIRONMENT= make test-integration-local TEST_FILE=metrics +# ENVIRONMENT= make test-integration-local TEST_NAME="should emit processing metrics" +# ENVIRONMENT= make test-integration-local TEST_FILE=metrics TEST_NAME="should emit processing metrics" # # Optional overrides: # AWS_REGION (default: eu-west-2) diff --git a/scripts/tests/integration.sh b/scripts/tests/integration.sh index 9ba79472..e3e4b975 100755 --- a/scripts/tests/integration.sh +++ b/scripts/tests/integration.sh @@ -9,7 +9,7 @@ npm ci source ./scripts/tests/integration-env.sh JEST_ARGS=() -[ -n "${FILE:-}" ] && JEST_ARGS+=("$FILE") -[ -n "${NAME:-}" ] && JEST_ARGS+=(--testNamePattern "$NAME") +[ -n "${TEST_FILE:-}" ] && JEST_ARGS+=("$TEST_FILE") +[ -n "${TEST_NAME:-}" ] && JEST_ARGS+=(--testNamePattern "$TEST_NAME") npm run test:integration --workspace tests/integration -- "${JEST_ARGS[@]}" From 6296bb19cf01d21fad4857cd275540ee227c9892 Mon Sep 17 00:00:00 2001 From: Mike Wild Date: Tue, 31 Mar 2026 14:10:34 +0100 Subject: [PATCH 09/15] fixup! Add integration debug script and readme --- tests/integration/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration/README.md b/tests/integration/README.md index 4be0a839..dcc67c49 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -25,19 +25,19 @@ ENVIRONMENT= AWS_PROFILE= make test-integration-local Run a single test file: ```sh -ENVIRONMENT= AWS_PROFILE= make test-integration-local FILE=metrics +ENVIRONMENT= AWS_PROFILE= make test-integration-local TEST_FILE=metrics ``` Run a single test name: ```sh -ENVIRONMENT= AWS_PROFILE= make test-integration-local NAME="should emit processing metrics when a valid event is fully processed" +ENVIRONMENT= AWS_PROFILE= make test-integration-local TEST_NAME="should emit processing metrics when a valid event is fully processed" ``` Combine file + test name: ```sh -ENVIRONMENT= AWS_PROFILE= make test-integration-local FILE=metrics NAME="should emit processing metrics when a valid event is fully processed" +ENVIRONMENT= AWS_PROFILE= make test-integration-local TEST_FILE=metrics TEST_NAME="should emit processing metrics when a valid event is fully processed" ``` ## Debug an Environment From eb83d46355a8d769b30d5b5b4380db12974bc544 Mon Sep 17 00:00:00 2001 From: Mike Wild Date: Tue, 31 Mar 2026 14:11:22 +0100 Subject: [PATCH 10/15] Filter logs by time to avoid pagination issue --- tests/integration/dlq-redrive.test.ts | 5 +++++ tests/integration/helpers/cloudwatch.ts | 3 +++ tests/integration/helpers/status-events.ts | 18 ++++++++++++------ .../integration/inbound-sqs-to-webhook.test.ts | 6 +++++- tests/integration/metrics.test.ts | 1 + 5 files changed, 26 insertions(+), 7 deletions(-) diff --git a/tests/integration/dlq-redrive.test.ts b/tests/integration/dlq-redrive.test.ts index e88e4920..7e63e88c 100644 --- a/tests/integration/dlq-redrive.test.ts +++ b/tests/integration/dlq-redrive.test.ts @@ -34,6 +34,7 @@ describe("DLQ Redrive", () => { let allTargetDlqQueueUrls: string[]; let inboundQueueUrl: string; let webhookLogGroupName: string; + let startTime: number; beforeAll(async () => { const deploymentDetails = getDeploymentDetails(); @@ -45,6 +46,7 @@ describe("DLQ Redrive", () => { cloudWatchClient = createCloudWatchLogsClient(deploymentDetails); inboundQueueUrl = buildInboundEventQueueUrl(deploymentDetails); + startTime = Date.now(); dlqQueueUrl = buildMockClientDlqQueueUrl( deploymentDetails, mockClient1.targets, @@ -118,6 +120,7 @@ describe("DLQ Redrive", () => { "MessageStatus", startTime, buildMockWebhookTargetPath(), + startTime, ); expect(callbacks.length).toBeGreaterThan(0); @@ -172,6 +175,7 @@ describe("DLQ Redrive", () => { "MessageStatus", startTime, buildMockWebhookTargetPath(), + startTime, ), awaitSignedCallbacksFromWebhookLogGroup( cloudWatchClient, @@ -180,6 +184,7 @@ describe("DLQ Redrive", () => { "MessageStatus", startTime, buildMockWebhookTargetPath(), + startTime, ), ]); diff --git a/tests/integration/helpers/cloudwatch.ts b/tests/integration/helpers/cloudwatch.ts index 50815db0..07384203 100644 --- a/tests/integration/helpers/cloudwatch.ts +++ b/tests/integration/helpers/cloudwatch.ts @@ -110,6 +110,7 @@ export async function awaitSignedCallbacksFromWebhookLogGroup( callbackType: CallbackItem["type"], startTime: number, path: string, + startTime: number, ): Promise { const queryStartTime = Math.max(0, startTime - CLOUDWATCH_QUERY_LOOKBACK_MS); logger.debug( @@ -150,6 +151,7 @@ export async function awaitSignedCallbacksByCountFromWebhookLogGroup( messageId: string, callbackType: CallbackItem["type"], expectedCount: number, + startTime: number, ): Promise { logger.debug( `Waiting for callbacks in webhook CloudWatch log group (messageId=${messageId}, callbackType=${callbackType}, expectedCount=${expectedCount}, logGroup=${logGroupName})`, @@ -165,6 +167,7 @@ export async function awaitSignedCallbacksByCountFromWebhookLogGroup( logGroupName, messageId, callbackType, + startTime, ); return callbacks.length === expectedCount; }, diff --git a/tests/integration/helpers/status-events.ts b/tests/integration/helpers/status-events.ts index b50b87da..0b708273 100644 --- a/tests/integration/helpers/status-events.ts +++ b/tests/integration/helpers/status-events.ts @@ -15,13 +15,16 @@ import { ensureInboundQueueIsEmpty, sendSqsEvent } from "./sqs"; async function processStatusEvent< T extends MessageStatusData | ChannelStatusData, >( - sqsClient: SQSClient, - cloudWatchClient: CloudWatchLogsClient, + { + cloudWatchClient, + sqsClient, + }: { cloudWatchClient: CloudWatchLogsClient; sqsClient: SQSClient }, callbackEventQueueUrl: string, webhookLogGroupName: string, event: StatusPublishEvent, callbackType: SignedCallback["payload"]["type"], webhookPath: string, + startTime: number, ): Promise { const startTime = Date.now(); const sendMessageResponse = await sendSqsEvent( @@ -43,6 +46,7 @@ async function processStatusEvent< callbackType, startTime, webhookPath, + startTime, ); } @@ -53,15 +57,16 @@ export async function processMessageStatusEvent( webhookLogGroupName: string, messageStatusEvent: StatusPublishEvent, webhookPath: string, + startTime: number, ): Promise { return processStatusEvent( - sqsClient, - cloudWatchClient, + { cloudWatchClient, sqsClient }, callbackEventQueueUrl, webhookLogGroupName, messageStatusEvent, "MessageStatus", webhookPath, + startTime, ); } @@ -72,14 +77,15 @@ export async function processChannelStatusEvent( webhookLogGroupName: string, channelStatusEvent: StatusPublishEvent, webhookPath: string, + startTime: number, ): Promise { return processStatusEvent( - sqsClient, - cloudWatchClient, + { cloudWatchClient, sqsClient }, callbackEventQueueUrl, webhookLogGroupName, channelStatusEvent, "ChannelStatus", webhookPath, + startTime, ); } diff --git a/tests/integration/inbound-sqs-to-webhook.test.ts b/tests/integration/inbound-sqs-to-webhook.test.ts index 8f679420..0a7b4451 100644 --- a/tests/integration/inbound-sqs-to-webhook.test.ts +++ b/tests/integration/inbound-sqs-to-webhook.test.ts @@ -46,6 +46,7 @@ describe("SQS to Webhook Integration", () => { let inboundEventDlqQueueUrl: string; let webhookLogGroupName: string; let webhookTargetPath: string; + let startTime: number; beforeAll(async () => { const deploymentDetails = getDeploymentDetails(); @@ -61,7 +62,7 @@ describe("SQS to Webhook Integration", () => { "mock-webhook", ); webhookTargetPath = buildMockWebhookTargetPath(); - + startTime = Date.now(); await purgeQueues(sqsClient, [ inboundEventDlqQueueUrl, clientDlqQueueUrl, @@ -92,6 +93,7 @@ describe("SQS to Webhook Integration", () => { webhookLogGroupName, messageStatusEvent, webhookTargetPath, + startTime, ); expect(callbacks).toHaveLength(1); @@ -127,6 +129,7 @@ describe("SQS to Webhook Integration", () => { messageStatusEvent.data.messageId, "MessageStatus", expectedPaths.length, + startTime, ); expect(callbacks).toHaveLength(expectedPaths.length); @@ -168,6 +171,7 @@ describe("SQS to Webhook Integration", () => { webhookLogGroupName, channelStatusEvent, webhookTargetPath, + startTime, ); expect(callbacks).toHaveLength(1); diff --git a/tests/integration/metrics.test.ts b/tests/integration/metrics.test.ts index 2f314f85..089778dc 100644 --- a/tests/integration/metrics.test.ts +++ b/tests/integration/metrics.test.ts @@ -90,6 +90,7 @@ describe("Metrics", () => { "MessageStatus", startTime, buildMockWebhookTargetPath(), + startTime, ); expect(callbacks.length).toBeGreaterThan(0); From 93e2134f74dba2c8617e7187a653b2cb1984e2d1 Mon Sep 17 00:00:00 2001 From: Mike Wild Date: Tue, 31 Mar 2026 14:20:29 +0100 Subject: [PATCH 11/15] fixup! Filter logs by time to avoid pagination issue --- tests/integration/helpers/status-events.ts | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/tests/integration/helpers/status-events.ts b/tests/integration/helpers/status-events.ts index 0b708273..7d51373c 100644 --- a/tests/integration/helpers/status-events.ts +++ b/tests/integration/helpers/status-events.ts @@ -15,10 +15,7 @@ import { ensureInboundQueueIsEmpty, sendSqsEvent } from "./sqs"; async function processStatusEvent< T extends MessageStatusData | ChannelStatusData, >( - { - cloudWatchClient, - sqsClient, - }: { cloudWatchClient: CloudWatchLogsClient; sqsClient: SQSClient }, + { CloudWatchLogsClient: cloudWatchClient, SQSClient: sqsClient }, callbackEventQueueUrl: string, webhookLogGroupName: string, event: StatusPublishEvent, @@ -60,7 +57,7 @@ export async function processMessageStatusEvent( startTime: number, ): Promise { return processStatusEvent( - { cloudWatchClient, sqsClient }, + { CloudWatchLogsClient: cloudWatchClient, SQSClient: sqsClient }, callbackEventQueueUrl, webhookLogGroupName, messageStatusEvent, @@ -80,7 +77,7 @@ export async function processChannelStatusEvent( startTime: number, ): Promise { return processStatusEvent( - { cloudWatchClient, sqsClient }, + { CloudWatchLogsClient: cloudWatchClient, SQSClient: sqsClient }, callbackEventQueueUrl, webhookLogGroupName, channelStatusEvent, From d8cff15312e3002b4b007e14c4072bbe430e2913 Mon Sep 17 00:00:00 2001 From: Mike Wild Date: Thu, 2 Apr 2026 15:44:20 +0100 Subject: [PATCH 12/15] Feedback: improve debug tool readme --- tests/integration/README.md | 89 ++++++++++++++++++++++++++++++++----- 1 file changed, 78 insertions(+), 11 deletions(-) diff --git a/tests/integration/README.md b/tests/integration/README.md index dcc67c49..a58531b8 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -42,27 +42,94 @@ ENVIRONMENT= AWS_PROFILE= make test-integration-local TEST_FILE=me ## Debug an Environment -Use debug actions to inspect queues, tail logs, and check pipe state: +The following debug tools are available for inspecting a deployed environment. +All are run via `make test-integration-debug ACTION=`. + +**Available actions:** + +- [`queue-status`](#queue-status) – SQS queue message counts +- [`queue-peek`](#queue-peek) – Peek at one message from each SQS queue +- [`tail-transform`](#tail-transform) – Tail the transform/filter Lambda logs +- [`tail-webhook`](#tail-webhook) – Tail the mock-webhook Lambda logs +- [`tail-pipe`](#tail-pipe) – Tail the EventBridge pipe logs +- [`pipe-state`](#pipe-state) – Show EventBridge pipe state and recent metrics + +All log-tailing actions (`tail-transform`, `tail-webhook`, `tail-pipe`) accept an optional `LOG_FILTER` to narrow output to a specific message ID or pattern. + +--- + +### `queue-status` + +Shows approximate message counts for the inbound event queue, inbound event DLQ, and mock target DLQ. + +```sh +ENVIRONMENT= AWS_PROFILE= make test-integration-debug ACTION=queue-status +``` + +--- + +### `queue-peek` + +Reads one message (without deleting it) from each of the same three queues, printing body, attributes, and message attributes. + +```sh +ENVIRONMENT= AWS_PROFILE= make test-integration-debug ACTION=queue-peek +``` + +--- + +### `tail-transform` + +Tails CloudWatch logs for the `client-transform-filter` Lambda, following from the last 30 minutes. + +```sh +ENVIRONMENT= AWS_PROFILE= make test-integration-debug ACTION=tail-transform +``` + +Filter to a specific message ID: ```sh -ENVIRONMENT= AWS_PROFILE= make test-integration-debug queue-status +ENVIRONMENT= AWS_PROFILE= LOG_FILTER=SOME-MESSAGE-ID make test-integration-debug ACTION=tail-transform ``` -Tail webhook logs: +--- + +### `tail-webhook` + +Tails CloudWatch logs for the `mock-webhook` Lambda, following from the last 30 minutes. ```sh -ENVIRONMENT= AWS_PROFILE= make test-integration-debug tail-webhook +ENVIRONMENT= AWS_PROFILE= make test-integration-debug ACTION=tail-webhook ``` -Tail webhook logs filtered to a specific message ID: +Filter to a specific message ID: ```sh -ENVIRONMENT= AWS_PROFILE= LOG_FILTER=SOME-MESSAGE-ID make test-integration-debug tail-webhook +ENVIRONMENT= AWS_PROFILE= LOG_FILTER=SOME-MESSAGE-ID make test-integration-debug ACTION=tail-webhook ``` -Other available actions: +--- + +### `tail-pipe` + +Tails the CloudWatch log group attached to the EventBridge pipe, following from the last 30 minutes. + +```sh +ENVIRONMENT= AWS_PROFILE= make test-integration-debug ACTION=tail-pipe +``` -- `queue-peek` -- `tail-transform` -- `tail-pipe` -- `pipe-state` +Filter to a specific message ID: + +```sh +ENVIRONMENT= AWS_PROFILE= LOG_FILTER=SOME-MESSAGE-ID make test-integration-debug ACTION=tail-pipe +``` + +--- + +### `pipe-state` + +Shows the current state (running/stopped), desired state, and last state reason for the EventBridge pipe, plus execution metrics (succeeded, failed, dead-lettered) for the last 30 minutes. + +```sh +ENVIRONMENT= AWS_PROFILE= make test-integration-debug ACTION=pipe-state +``` From 9d33b9d665d66d10608959a585789b281dc26315 Mon Sep 17 00:00:00 2001 From: Mike Wild Date: Thu, 2 Apr 2026 15:45:07 +0100 Subject: [PATCH 13/15] Make times in debug script consistent (last 30m) --- scripts/tests/integration-debug.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/tests/integration-debug.sh b/scripts/tests/integration-debug.sh index 6a684341..ac9fb905 100755 --- a/scripts/tests/integration-debug.sh +++ b/scripts/tests/integration-debug.sh @@ -154,7 +154,7 @@ action_tail_transform() { "/aws/lambda/${PREFIX}-client-transform-filter" \ --region "$REGION" \ --profile "$AWS_PROFILE" \ - --since 2h \ + --since 30m \ --follow \ --format short \ "${filter_args[@]}" @@ -169,7 +169,7 @@ action_tail_webhook() { "/aws/lambda/${PREFIX}-mock-webhook" \ --region "$REGION" \ --profile "$AWS_PROFILE" \ - --since 2h \ + --since 30m \ --follow \ --format short \ "${filter_args[@]}" @@ -202,7 +202,7 @@ action_tail_pipe() { "$pipe_log_group_name" \ --region "$REGION" \ --profile "$AWS_PROFILE" \ - --since 2h \ + --since 30m \ --follow \ --format short \ "${filter_args[@]}" @@ -235,7 +235,7 @@ action_pipe_state() { local execution_failed local dead_lettered_events - start_time="$(date -u -d '10 minutes ago' +%Y-%m-%dT%H:%M:%SZ)" + start_time="$(date -u -d '30 minutes ago' +%Y-%m-%dT%H:%M:%SZ)" end_time="$(date -u +%Y-%m-%dT%H:%M:%SZ)" print_section "EventBridge Pipe State" @@ -250,7 +250,7 @@ action_pipe_state() { execution_failed="$(pipe_metric_sum "ExecutionFailed" "$start_time" "$end_time")" dead_lettered_events="$(pipe_metric_sum "DeadLetteredEvents" "$start_time" "$end_time")" - print_section "EventBridge Pipe Metrics (last 10 minutes)" + print_section "EventBridge Pipe Metrics (last 30 minutes)" echo "ExecutionSucceeded: ${execution_succeeded}" echo "ExecutionFailed: ${execution_failed}" echo "DeadLetteredEvents: ${dead_lettered_events}" From 645dcb7a24add5a362d3126faef83f119b8797cf Mon Sep 17 00:00:00 2001 From: Mike Wild Date: Thu, 2 Apr 2026 16:12:46 +0100 Subject: [PATCH 14/15] Update package.lock following rebase --- package-lock.json | 80 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/package-lock.json b/package-lock.json index 8ec1428f..a346ae45 100644 --- a/package-lock.json +++ b/package-lock.json @@ -332,6 +332,58 @@ "node": ">=14.0.0" } }, + "node_modules/@aws-sdk/client-cloudwatch": { + "version": "3.1019.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-cloudwatch/-/client-cloudwatch-3.1019.0.tgz", + "integrity": "sha512-ojG1Ot0N1ucI+kqIH8lUCSyuzqXuMpLKA00Laip3m1hnAxJb0fjISZw6oh0NfBfEXJ407n450/As91NCFSLCHg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/core": "^3.973.25", + "@aws-sdk/credential-provider-node": "^3.972.27", + "@aws-sdk/middleware-host-header": "^3.972.8", + "@aws-sdk/middleware-logger": "^3.972.8", + "@aws-sdk/middleware-recursion-detection": "^3.972.9", + "@aws-sdk/middleware-user-agent": "^3.972.26", + "@aws-sdk/region-config-resolver": "^3.972.10", + "@aws-sdk/types": "^3.973.6", + "@aws-sdk/util-endpoints": "^3.996.5", + "@aws-sdk/util-user-agent-browser": "^3.972.8", + "@aws-sdk/util-user-agent-node": "^3.973.12", + "@smithy/config-resolver": "^4.4.13", + "@smithy/core": "^3.23.12", + "@smithy/fetch-http-handler": "^5.3.15", + "@smithy/hash-node": "^4.2.12", + "@smithy/invalid-dependency": "^4.2.12", + "@smithy/middleware-compression": "^4.3.41", + "@smithy/middleware-content-length": "^4.2.12", + "@smithy/middleware-endpoint": "^4.4.27", + "@smithy/middleware-retry": "^4.4.44", + "@smithy/middleware-serde": "^4.2.15", + "@smithy/middleware-stack": "^4.2.12", + "@smithy/node-config-provider": "^4.3.12", + "@smithy/node-http-handler": "^4.5.0", + "@smithy/protocol-http": "^5.3.12", + "@smithy/smithy-client": "^4.12.7", + "@smithy/types": "^4.13.1", + "@smithy/url-parser": "^4.2.12", + "@smithy/util-base64": "^4.3.2", + "@smithy/util-body-length-browser": "^4.2.2", + "@smithy/util-body-length-node": "^4.2.3", + "@smithy/util-defaults-mode-browser": "^4.3.43", + "@smithy/util-defaults-mode-node": "^4.2.47", + "@smithy/util-endpoints": "^3.3.3", + "@smithy/util-middleware": "^4.2.12", + "@smithy/util-retry": "^4.2.12", + "@smithy/util-utf8": "^4.2.2", + "@smithy/util-waiter": "^4.2.13", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, "node_modules/@aws-sdk/client-cloudwatch-logs": { "version": "3.1020.0", "resolved": "https://registry.npmjs.org/@aws-sdk/client-cloudwatch-logs/-/client-cloudwatch-logs-3.1020.0.tgz", @@ -4257,6 +4309,27 @@ "node": ">=18.0.0" } }, + "node_modules/@smithy/middleware-compression": { + "version": "4.3.41", + "resolved": "https://registry.npmjs.org/@smithy/middleware-compression/-/middleware-compression-4.3.41.tgz", + "integrity": "sha512-lJ/yTWaPQZfvT5GJUgGpjjmG4ZgNhlPmvAN+SfQKcsNBApY+CaW2vg/x7GhsD3g/liKlo7suMAAZNK5RVQ9OIQ==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.23.12", + "@smithy/is-array-buffer": "^4.2.2", + "@smithy/node-config-provider": "^4.3.12", + "@smithy/protocol-http": "^5.3.12", + "@smithy/types": "^4.13.1", + "@smithy/util-config-provider": "^4.2.2", + "@smithy/util-middleware": "^4.2.12", + "@smithy/util-utf8": "^4.2.2", + "fflate": "0.8.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@smithy/middleware-content-length": { "version": "4.2.12", "resolved": "https://registry.npmjs.org/@smithy/middleware-content-length/-/middleware-content-length-4.2.12.tgz", @@ -8652,6 +8725,12 @@ } } }, + "node_modules/fflate": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/fflate/-/fflate-0.8.1.tgz", + "integrity": "sha512-/exOvEuc+/iaUm105QIiOt4LpBdMTWsXxqR0HDF35vx3fmaKzw7354gTilCh5rkzEt8WYyG//ku3h3nRmd7CHQ==", + "license": "MIT" + }, "node_modules/file-entry-cache": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-8.0.0.tgz", @@ -16571,6 +16650,7 @@ "name": "nhs-notify-client-callbacks-integration-tests", "version": "0.0.1", "dependencies": { + "@aws-sdk/client-cloudwatch": "^3.1019.0", "@aws-sdk/client-cloudwatch-logs": "^3.1020.0", "@aws-sdk/client-s3": "^3.1020.0", "@aws-sdk/client-sqs": "^3.1020.0", From c2bc65127bc99513afa37a48473d41fdeda842a0 Mon Sep 17 00:00:00 2001 From: Mike Wild Date: Thu, 2 Apr 2026 16:30:48 +0100 Subject: [PATCH 15/15] fixup! fixup! Filter logs by time to avoid pagination issue --- tests/integration/dlq-redrive.test.ts | 5 ----- tests/integration/helpers/cloudwatch.ts | 1 - tests/integration/helpers/status-events.ts | 7 ++++--- tests/integration/metrics.test.ts | 1 - 4 files changed, 4 insertions(+), 10 deletions(-) diff --git a/tests/integration/dlq-redrive.test.ts b/tests/integration/dlq-redrive.test.ts index 7e63e88c..e88e4920 100644 --- a/tests/integration/dlq-redrive.test.ts +++ b/tests/integration/dlq-redrive.test.ts @@ -34,7 +34,6 @@ describe("DLQ Redrive", () => { let allTargetDlqQueueUrls: string[]; let inboundQueueUrl: string; let webhookLogGroupName: string; - let startTime: number; beforeAll(async () => { const deploymentDetails = getDeploymentDetails(); @@ -46,7 +45,6 @@ describe("DLQ Redrive", () => { cloudWatchClient = createCloudWatchLogsClient(deploymentDetails); inboundQueueUrl = buildInboundEventQueueUrl(deploymentDetails); - startTime = Date.now(); dlqQueueUrl = buildMockClientDlqQueueUrl( deploymentDetails, mockClient1.targets, @@ -120,7 +118,6 @@ describe("DLQ Redrive", () => { "MessageStatus", startTime, buildMockWebhookTargetPath(), - startTime, ); expect(callbacks.length).toBeGreaterThan(0); @@ -175,7 +172,6 @@ describe("DLQ Redrive", () => { "MessageStatus", startTime, buildMockWebhookTargetPath(), - startTime, ), awaitSignedCallbacksFromWebhookLogGroup( cloudWatchClient, @@ -184,7 +180,6 @@ describe("DLQ Redrive", () => { "MessageStatus", startTime, buildMockWebhookTargetPath(), - startTime, ), ]); diff --git a/tests/integration/helpers/cloudwatch.ts b/tests/integration/helpers/cloudwatch.ts index 07384203..9ee13739 100644 --- a/tests/integration/helpers/cloudwatch.ts +++ b/tests/integration/helpers/cloudwatch.ts @@ -110,7 +110,6 @@ export async function awaitSignedCallbacksFromWebhookLogGroup( callbackType: CallbackItem["type"], startTime: number, path: string, - startTime: number, ): Promise { const queryStartTime = Math.max(0, startTime - CLOUDWATCH_QUERY_LOOKBACK_MS); logger.debug( diff --git a/tests/integration/helpers/status-events.ts b/tests/integration/helpers/status-events.ts index 7d51373c..1bccf0bb 100644 --- a/tests/integration/helpers/status-events.ts +++ b/tests/integration/helpers/status-events.ts @@ -15,7 +15,10 @@ import { ensureInboundQueueIsEmpty, sendSqsEvent } from "./sqs"; async function processStatusEvent< T extends MessageStatusData | ChannelStatusData, >( - { CloudWatchLogsClient: cloudWatchClient, SQSClient: sqsClient }, + { + CloudWatchLogsClient: cloudWatchClient, + SQSClient: sqsClient, + }: { CloudWatchLogsClient: CloudWatchLogsClient; SQSClient: SQSClient }, callbackEventQueueUrl: string, webhookLogGroupName: string, event: StatusPublishEvent, @@ -23,7 +26,6 @@ async function processStatusEvent< webhookPath: string, startTime: number, ): Promise { - const startTime = Date.now(); const sendMessageResponse = await sendSqsEvent( sqsClient, callbackEventQueueUrl, @@ -43,7 +45,6 @@ async function processStatusEvent< callbackType, startTime, webhookPath, - startTime, ); } diff --git a/tests/integration/metrics.test.ts b/tests/integration/metrics.test.ts index 089778dc..2f314f85 100644 --- a/tests/integration/metrics.test.ts +++ b/tests/integration/metrics.test.ts @@ -90,7 +90,6 @@ describe("Metrics", () => { "MessageStatus", startTime, buildMockWebhookTargetPath(), - startTime, ); expect(callbacks.length).toBeGreaterThan(0);