diff --git a/package-lock.json b/package-lock.json index 8ec1428f..a346ae45 100644 --- a/package-lock.json +++ b/package-lock.json @@ -332,6 +332,58 @@ "node": ">=14.0.0" } }, + "node_modules/@aws-sdk/client-cloudwatch": { + "version": "3.1019.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-cloudwatch/-/client-cloudwatch-3.1019.0.tgz", + "integrity": "sha512-ojG1Ot0N1ucI+kqIH8lUCSyuzqXuMpLKA00Laip3m1hnAxJb0fjISZw6oh0NfBfEXJ407n450/As91NCFSLCHg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/core": "^3.973.25", + "@aws-sdk/credential-provider-node": "^3.972.27", + "@aws-sdk/middleware-host-header": "^3.972.8", + "@aws-sdk/middleware-logger": "^3.972.8", + "@aws-sdk/middleware-recursion-detection": "^3.972.9", + "@aws-sdk/middleware-user-agent": "^3.972.26", + "@aws-sdk/region-config-resolver": "^3.972.10", + "@aws-sdk/types": "^3.973.6", + "@aws-sdk/util-endpoints": "^3.996.5", + "@aws-sdk/util-user-agent-browser": "^3.972.8", + "@aws-sdk/util-user-agent-node": "^3.973.12", + "@smithy/config-resolver": "^4.4.13", + "@smithy/core": "^3.23.12", + "@smithy/fetch-http-handler": "^5.3.15", + "@smithy/hash-node": "^4.2.12", + "@smithy/invalid-dependency": "^4.2.12", + "@smithy/middleware-compression": "^4.3.41", + "@smithy/middleware-content-length": "^4.2.12", + "@smithy/middleware-endpoint": "^4.4.27", + "@smithy/middleware-retry": "^4.4.44", + "@smithy/middleware-serde": "^4.2.15", + "@smithy/middleware-stack": "^4.2.12", + "@smithy/node-config-provider": "^4.3.12", + "@smithy/node-http-handler": "^4.5.0", + "@smithy/protocol-http": "^5.3.12", + "@smithy/smithy-client": "^4.12.7", + "@smithy/types": "^4.13.1", + "@smithy/url-parser": "^4.2.12", + "@smithy/util-base64": "^4.3.2", + "@smithy/util-body-length-browser": "^4.2.2", + "@smithy/util-body-length-node": "^4.2.3", + "@smithy/util-defaults-mode-browser": "^4.3.43", + "@smithy/util-defaults-mode-node": "^4.2.47", + "@smithy/util-endpoints": "^3.3.3", + "@smithy/util-middleware": "^4.2.12", + "@smithy/util-retry": "^4.2.12", + "@smithy/util-utf8": "^4.2.2", + "@smithy/util-waiter": "^4.2.13", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, "node_modules/@aws-sdk/client-cloudwatch-logs": { "version": "3.1020.0", "resolved": "https://registry.npmjs.org/@aws-sdk/client-cloudwatch-logs/-/client-cloudwatch-logs-3.1020.0.tgz", @@ -4257,6 +4309,27 @@ "node": ">=18.0.0" } }, + "node_modules/@smithy/middleware-compression": { + "version": "4.3.41", + "resolved": "https://registry.npmjs.org/@smithy/middleware-compression/-/middleware-compression-4.3.41.tgz", + "integrity": "sha512-lJ/yTWaPQZfvT5GJUgGpjjmG4ZgNhlPmvAN+SfQKcsNBApY+CaW2vg/x7GhsD3g/liKlo7suMAAZNK5RVQ9OIQ==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.23.12", + "@smithy/is-array-buffer": "^4.2.2", + "@smithy/node-config-provider": "^4.3.12", + "@smithy/protocol-http": "^5.3.12", + "@smithy/types": "^4.13.1", + "@smithy/util-config-provider": "^4.2.2", + "@smithy/util-middleware": "^4.2.12", + "@smithy/util-utf8": "^4.2.2", + "fflate": "0.8.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@smithy/middleware-content-length": { "version": "4.2.12", "resolved": "https://registry.npmjs.org/@smithy/middleware-content-length/-/middleware-content-length-4.2.12.tgz", @@ -8652,6 +8725,12 @@ } } }, + "node_modules/fflate": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/fflate/-/fflate-0.8.1.tgz", + "integrity": "sha512-/exOvEuc+/iaUm105QIiOt4LpBdMTWsXxqR0HDF35vx3fmaKzw7354gTilCh5rkzEt8WYyG//ku3h3nRmd7CHQ==", + "license": "MIT" + }, "node_modules/file-entry-cache": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-8.0.0.tgz", @@ -16571,6 +16650,7 @@ "name": "nhs-notify-client-callbacks-integration-tests", "version": "0.0.1", "dependencies": { + "@aws-sdk/client-cloudwatch": "^3.1019.0", "@aws-sdk/client-cloudwatch-logs": "^3.1020.0", "@aws-sdk/client-s3": "^3.1020.0", "@aws-sdk/client-sqs": "^3.1020.0", diff --git a/scripts/tests/integration-debug.sh b/scripts/tests/integration-debug.sh new file mode 100755 index 00000000..ac9fb905 --- /dev/null +++ b/scripts/tests/integration-debug.sh @@ -0,0 +1,283 @@ +#!/bin/bash + +set -euo pipefail + +# Debug a live environment: inspect queues, tail logs, check pipe state. +# +# Usage (via make): +# ENVIRONMENT= AWS_PROFILE= make test-integration-debug ACTION=queue-status +# ENVIRONMENT= AWS_PROFILE= make test-integration-debug ACTION=tail-transform +# ENVIRONMENT= AWS_PROFILE= make test-integration-debug ACTION=tail-transform LOG_FILTER= +# +# Actions: +# queue-status Show SQS queue message counts +# queue-peek Peek one message from each SQS queue +# tail-transform Tail client-transform-filter lambda logs +# tail-webhook Tail mock-webhook lambda logs +# tail-pipe Tail EventBridge pipe log group +# pipe-state Show EventBridge pipe state and recent metrics +# +# Required: +# ENVIRONMENT +# AWS_PROFILE +# ACTION +# +# Optional: +# LOG_FILTER CloudWatch Logs filter pattern / text +# AWS_REGION (default: eu-west-2) + +if [ -z "${ENVIRONMENT:-}" ]; then + echo "Error: ENVIRONMENT must be set before running this target." >&2 + echo "Example: ENVIRONMENT= AWS_PROFILE= make test-integration-debug ACTION=queue-status" >&2 + exit 1 +fi + +if [ -z "${AWS_PROFILE:-}" ]; then + echo "Error: AWS_PROFILE must be set before running this target." >&2 + exit 1 +fi + +if [ -z "${ACTION:-}" ]; then + echo "Error: ACTION must be set before running this target." >&2 + echo "Actions: queue-status, queue-peek, tail-transform, tail-webhook, tail-pipe, pipe-state" >&2 + exit 1 +fi + +REGION="${AWS_REGION:-eu-west-2}" +LOG_FILTER="${LOG_FILTER:-}" +SUBSCRIPTION_FIXTURE_PATH="${SUBSCRIPTION_FIXTURE_PATH:-tests/integration/fixtures/subscriptions/mock-client-1.json}" + +if ! aws sts get-caller-identity --profile "$AWS_PROFILE" >/dev/null 2>&1; then + echo "No active AWS SSO session for profile '$AWS_PROFILE'. Running aws sso login..." + aws sso login --profile "$AWS_PROFILE" +fi + +ACCOUNT_ID="$(aws sts get-caller-identity --profile "$AWS_PROFILE" --query Account --output text)" + +PREFIX="nhs-${ENVIRONMENT}-callbacks" +PIPE_NAME="${PREFIX}-main" + +print_section() { + echo "" + echo "========================================" + echo "$1" + echo "========================================" +} + +queue_url() { + local queue_name="$1" + echo "https://sqs.${REGION}.amazonaws.com/${ACCOUNT_ID}/${queue_name}" +} + +target_dlq_queue_name() { + local target_id + + if [ ! -f "$SUBSCRIPTION_FIXTURE_PATH" ]; then + echo "Error: subscription fixture not found: $SUBSCRIPTION_FIXTURE_PATH" >&2 + exit 1 + fi + + target_id="$(jq -r '.targets[0].targetId // empty' "$SUBSCRIPTION_FIXTURE_PATH")" + if [ -z "$target_id" ]; then + echo "Error: unable to read targets[0].targetId from $SUBSCRIPTION_FIXTURE_PATH" >&2 + exit 1 + fi + + echo "${PREFIX}-${target_id}-dlq-queue" +} + +show_queue_counts() { + local title="$1" + local name="$2" + + print_section "$title" + aws sqs get-queue-attributes \ + --region "$REGION" \ + --profile "$AWS_PROFILE" \ + --no-cli-pager \ + --queue-url "$(queue_url "$name")" \ + --attribute-names ApproximateNumberOfMessages ApproximateNumberOfMessagesNotVisible \ + --query '{ApproximateNumberOfMessages: Attributes.ApproximateNumberOfMessages, ApproximateNumberOfMessagesNotVisible: Attributes.ApproximateNumberOfMessagesNotVisible}' +} + +action_queue_status() { + show_queue_counts "Mock Target DLQ - Queue Message Counts" "$(target_dlq_queue_name)" + show_queue_counts "Inbound Event Queue - Queue Message Counts" "${PREFIX}-inbound-event-queue" + show_queue_counts "Inbound Event DLQ - Queue Message Counts" "${PREFIX}-inbound-event-dlq" +} + +peek_queue_message() { + local title="$1" + local name="$2" + + print_section "$title" + aws sqs receive-message \ + --region "$REGION" \ + --profile "$AWS_PROFILE" \ + --no-cli-pager \ + --queue-url "$(queue_url "$name")" \ + --attribute-names All \ + --message-attribute-names All \ + --max-number-of-messages 1 \ + --visibility-timeout 0 \ + --output json \ + | jq 'if (.Messages // [] | length) == 0 + then {message: "No messages"} + else .Messages[0] | {Body, Attributes, MessageAttributes} + end' +} + +action_queue_peek() { + peek_queue_message "Mock Target DLQ - Message Peek" "$(target_dlq_queue_name)" + peek_queue_message "Inbound Event Queue - Message Peek" "${PREFIX}-inbound-event-queue" + peek_queue_message "Inbound Event DLQ - Message Peek" "${PREFIX}-inbound-event-dlq" +} + +log_filter_args() { + local -a args=() + local escaped_log_filter + if [[ -n "$LOG_FILTER" ]]; then + escaped_log_filter="${LOG_FILTER//\"/\\\"}" + # CloudWatch filter patterns treat quoted strings as exact phrases. + args+=(--filter-pattern "\"$escaped_log_filter\"") + fi + + printf '%s\n' "${args[@]}" +} + +action_tail_transform() { + local -a filter_args=() + mapfile -t filter_args < <(log_filter_args) + + print_section "Transform/Filter Lambda Logs" + aws logs tail \ + "/aws/lambda/${PREFIX}-client-transform-filter" \ + --region "$REGION" \ + --profile "$AWS_PROFILE" \ + --since 30m \ + --follow \ + --format short \ + "${filter_args[@]}" +} + +action_tail_webhook() { + local -a filter_args=() + mapfile -t filter_args < <(log_filter_args) + + print_section "Mock Webhook Lambda Logs" + aws logs tail \ + "/aws/lambda/${PREFIX}-mock-webhook" \ + --region "$REGION" \ + --profile "$AWS_PROFILE" \ + --since 30m \ + --follow \ + --format short \ + "${filter_args[@]}" +} + +action_tail_pipe() { + local pipe_log_group_arn + local pipe_log_group_name + local -a filter_args=() + + mapfile -t filter_args < <(log_filter_args) + + pipe_log_group_arn=$(aws pipes describe-pipe \ + --region "$REGION" \ + --profile "$AWS_PROFILE" \ + --name "$PIPE_NAME" \ + --no-cli-pager \ + --query "LogConfiguration.CloudwatchLogsLogDestination.LogGroupArn" \ + --output text) + + if [[ -z "$pipe_log_group_arn" || "$pipe_log_group_arn" == "None" ]]; then + echo "No CloudWatch log group configured for pipe: $PIPE_NAME" + return 1 + fi + + pipe_log_group_name="${pipe_log_group_arn#*:log-group:}" + + print_section "EventBridge Pipe Logs" + aws logs tail \ + "$pipe_log_group_name" \ + --region "$REGION" \ + --profile "$AWS_PROFILE" \ + --since 30m \ + --follow \ + --format short \ + "${filter_args[@]}" +} + +pipe_metric_sum() { + local metric_name="$1" + local start_time="$2" + local end_time="$3" + + aws cloudwatch get-metric-statistics \ + --region "$REGION" \ + --profile "$AWS_PROFILE" \ + --namespace "AWS/Pipes" \ + --metric-name "$metric_name" \ + --dimensions "Name=PipeName,Value=$PIPE_NAME" \ + --start-time "$start_time" \ + --end-time "$end_time" \ + --period 60 \ + --statistics Sum \ + --query "sum(Datapoints[].Sum)" \ + --output text \ + --no-cli-pager +} + +action_pipe_state() { + local start_time + local end_time + local execution_succeeded + local execution_failed + local dead_lettered_events + + start_time="$(date -u -d '30 minutes ago' +%Y-%m-%dT%H:%M:%SZ)" + end_time="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + + print_section "EventBridge Pipe State" + aws pipes describe-pipe \ + --region "$REGION" \ + --profile "$AWS_PROFILE" \ + --name "$PIPE_NAME" \ + --no-cli-pager \ + --query '{Name:Name,CurrentState:CurrentState,DesiredState:DesiredState,StateReason:StateReason,LogConfiguration:LogConfiguration}' + + execution_succeeded="$(pipe_metric_sum "ExecutionSucceeded" "$start_time" "$end_time")" + execution_failed="$(pipe_metric_sum "ExecutionFailed" "$start_time" "$end_time")" + dead_lettered_events="$(pipe_metric_sum "DeadLetteredEvents" "$start_time" "$end_time")" + + print_section "EventBridge Pipe Metrics (last 30 minutes)" + echo "ExecutionSucceeded: ${execution_succeeded}" + echo "ExecutionFailed: ${execution_failed}" + echo "DeadLetteredEvents: ${dead_lettered_events}" +} + +case "$ACTION" in + queue-status) + action_queue_status + ;; + queue-peek) + action_queue_peek + ;; + tail-transform) + action_tail_transform + ;; + tail-webhook) + action_tail_webhook + ;; + tail-pipe) + action_tail_pipe + ;; + pipe-state) + action_pipe_state + ;; + *) + echo "Unknown action: $ACTION" >&2 + echo "Actions: queue-status, queue-peek, tail-transform, tail-webhook, tail-pipe, pipe-state" >&2 + exit 1 + ;; +esac diff --git a/scripts/tests/integration-local.sh b/scripts/tests/integration-local.sh new file mode 100755 index 00000000..7d0e36c4 --- /dev/null +++ b/scripts/tests/integration-local.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +set -euo pipefail + +# Run integration tests against a live environment. +# +# Usage (via make): +# ENVIRONMENT= make test-integration-local +# ENVIRONMENT= make test-integration-local TEST_FILE=metrics +# ENVIRONMENT= make test-integration-local TEST_NAME="should emit processing metrics" +# ENVIRONMENT= make test-integration-local TEST_FILE=metrics TEST_NAME="should emit processing metrics" +# +# Optional overrides: +# AWS_REGION (default: eu-west-2) +# +# Required: +# AWS_PROFILE + +if [ -z "${ENVIRONMENT:-}" ]; then + echo "Error: ENVIRONMENT must be set before running this target." >&2 + echo "Example: ENVIRONMENT= make test-integration-local" >&2 + exit 1 +fi + +if [ -z "${AWS_PROFILE:-}" ]; then + echo "Error: AWS_PROFILE must be set before running this target." >&2 + exit 1 +fi + +AWS_REGION="${AWS_REGION:-eu-west-2}" +LOG_LEVEL="${LOG_LEVEL:-debug}" +NODE_OPTIONS="${NODE_OPTIONS:---experimental-vm-modules}" +COMPONENT="callbacks" +PROJECT="nhs" + +if ! aws sts get-caller-identity --profile "$AWS_PROFILE" >/dev/null 2>&1; then + echo "No active AWS SSO session for profile '$AWS_PROFILE'. Running aws sso login..." + aws sso login --profile "$AWS_PROFILE" +fi + +AWS_ACCOUNT_ID="$(aws sts get-caller-identity --profile "$AWS_PROFILE" --query Account --output text)" + +export AWS_PROFILE AWS_REGION LOG_LEVEL NODE_OPTIONS AWS_ACCOUNT_ID ENVIRONMENT PROJECT COMPONENT + +CI=true exec ./scripts/tests/integration.sh diff --git a/scripts/tests/integration.sh b/scripts/tests/integration.sh index fcc89389..e3e4b975 100755 --- a/scripts/tests/integration.sh +++ b/scripts/tests/integration.sh @@ -8,4 +8,8 @@ npm ci source ./scripts/tests/integration-env.sh -npm run test:integration +JEST_ARGS=() +[ -n "${TEST_FILE:-}" ] && JEST_ARGS+=("$TEST_FILE") +[ -n "${TEST_NAME:-}" ] && JEST_ARGS+=(--testNamePattern "$TEST_NAME") + +npm run test:integration --workspace tests/integration -- "${JEST_ARGS[@]}" diff --git a/scripts/tests/test.mk b/scripts/tests/test.mk index 2615cfe3..d9303d92 100644 --- a/scripts/tests/test.mk +++ b/scripts/tests/test.mk @@ -29,6 +29,15 @@ test-contract: # Run your contract tests from scripts/test/contract @Testing test-integration: # Run your integration tests from scripts/test/integration @Testing make _test name="integration" +test-integration-local: # Run integration tests locally against a remoptely deployed environment (requires ENVIRONMENT) @Testing + make _test name="integration-local" + +test-integration-debug: # Debug a live environment - inspect queues, tail logs, check pipe state (requires ENVIRONMENT, AWS_PROFILE, ACTION) @Testing + make _test name="integration-debug" ACTION="$(or $(ACTION),$(word 2,$(MAKECMDGOALS)))" + +queue-status queue-peek tail-transform tail-webhook tail-pipe pipe-state: + @: + test-load: # Run all your load tests @Testing make \ test-capacity \ diff --git a/tests/integration/README.md b/tests/integration/README.md new file mode 100644 index 00000000..a58531b8 --- /dev/null +++ b/tests/integration/README.md @@ -0,0 +1,135 @@ +# Integration Tests + +This folder contains integration tests for the callbacks service. + +These instructions are for running integration tests locally against a remotely deployed environment. +In normal delivery flow, integration tests are triggered via the CI workflow. + +## Prerequisites + +- AWS CLI installed and authenticated +- Valid AWS SSO session for your chosen profile (the script will prompt for login if needed) +- Environment variables: + - `ENVIRONMENT` (required) + - `AWS_PROFILE` (required) + - `AWS_REGION` (optional, defaults to `eu-west-2`) + +## Run Integration Tests Locally + +Run all integration tests against a deployed environment: + +```sh +ENVIRONMENT= AWS_PROFILE= make test-integration-local +``` + +Run a single test file: + +```sh +ENVIRONMENT= AWS_PROFILE= make test-integration-local TEST_FILE=metrics +``` + +Run a single test name: + +```sh +ENVIRONMENT= AWS_PROFILE= make test-integration-local TEST_NAME="should emit processing metrics when a valid event is fully processed" +``` + +Combine file + test name: + +```sh +ENVIRONMENT= AWS_PROFILE= make test-integration-local TEST_FILE=metrics TEST_NAME="should emit processing metrics when a valid event is fully processed" +``` + +## Debug an Environment + +The following debug tools are available for inspecting a deployed environment. +All are run via `make test-integration-debug ACTION=`. + +**Available actions:** + +- [`queue-status`](#queue-status) – SQS queue message counts +- [`queue-peek`](#queue-peek) – Peek at one message from each SQS queue +- [`tail-transform`](#tail-transform) – Tail the transform/filter Lambda logs +- [`tail-webhook`](#tail-webhook) – Tail the mock-webhook Lambda logs +- [`tail-pipe`](#tail-pipe) – Tail the EventBridge pipe logs +- [`pipe-state`](#pipe-state) – Show EventBridge pipe state and recent metrics + +All log-tailing actions (`tail-transform`, `tail-webhook`, `tail-pipe`) accept an optional `LOG_FILTER` to narrow output to a specific message ID or pattern. + +--- + +### `queue-status` + +Shows approximate message counts for the inbound event queue, inbound event DLQ, and mock target DLQ. + +```sh +ENVIRONMENT= AWS_PROFILE= make test-integration-debug ACTION=queue-status +``` + +--- + +### `queue-peek` + +Reads one message (without deleting it) from each of the same three queues, printing body, attributes, and message attributes. + +```sh +ENVIRONMENT= AWS_PROFILE= make test-integration-debug ACTION=queue-peek +``` + +--- + +### `tail-transform` + +Tails CloudWatch logs for the `client-transform-filter` Lambda, following from the last 30 minutes. + +```sh +ENVIRONMENT= AWS_PROFILE= make test-integration-debug ACTION=tail-transform +``` + +Filter to a specific message ID: + +```sh +ENVIRONMENT= AWS_PROFILE= LOG_FILTER=SOME-MESSAGE-ID make test-integration-debug ACTION=tail-transform +``` + +--- + +### `tail-webhook` + +Tails CloudWatch logs for the `mock-webhook` Lambda, following from the last 30 minutes. + +```sh +ENVIRONMENT= AWS_PROFILE= make test-integration-debug ACTION=tail-webhook +``` + +Filter to a specific message ID: + +```sh +ENVIRONMENT= AWS_PROFILE= LOG_FILTER=SOME-MESSAGE-ID make test-integration-debug ACTION=tail-webhook +``` + +--- + +### `tail-pipe` + +Tails the CloudWatch log group attached to the EventBridge pipe, following from the last 30 minutes. + +```sh +ENVIRONMENT= AWS_PROFILE= make test-integration-debug ACTION=tail-pipe +``` + +Filter to a specific message ID: + +```sh +ENVIRONMENT= AWS_PROFILE= LOG_FILTER=SOME-MESSAGE-ID make test-integration-debug ACTION=tail-pipe +``` + +--- + +### `pipe-state` + +Shows the current state (running/stopped), desired state, and last state reason for the EventBridge pipe, plus execution metrics (succeeded, failed, dead-lettered) for the last 30 minutes. + +```sh +ENVIRONMENT= AWS_PROFILE= make test-integration-debug ACTION=pipe-state +``` diff --git a/tests/integration/dlq-alarms.test.ts b/tests/integration/dlq-alarms.test.ts new file mode 100644 index 00000000..7f29f88f --- /dev/null +++ b/tests/integration/dlq-alarms.test.ts @@ -0,0 +1,76 @@ +import { + CloudWatchClient, + DescribeAlarmsCommand, + type MetricAlarm, +} from "@aws-sdk/client-cloudwatch"; +import type { DeploymentDetails } from "@nhs-notify-client-callbacks/test-support/helpers"; +import { getDeploymentDetails } from "@nhs-notify-client-callbacks/test-support/helpers"; +import { getAllSubscriptionTargetIds } from "./helpers/mock-client-config"; +import { buildMockClientDlqQueueUrl } from "./helpers/sqs"; + +function buildDlqDepthAlarmName( + { component, environment, project }: DeploymentDetails, + targetId: string, +): string { + return `${project}-${environment}-${component}-${targetId}-dlq-depth`; +} + +function getQueueNameFromUrl(queueUrl: string): string { + const queueName = queueUrl.split("/").pop(); + if (!queueName) { + throw new Error(`Unable to derive queue name from URL: ${queueUrl}`); + } + + return queueName; +} + +describe("DLQ alarms", () => { + let cloudWatchClient: CloudWatchClient; + let deploymentDetails: DeploymentDetails; + let targetIds: string[]; + + beforeAll(() => { + deploymentDetails = getDeploymentDetails(); + cloudWatchClient = new CloudWatchClient({ + region: deploymentDetails.region, + }); + + targetIds = getAllSubscriptionTargetIds(); + }); + + afterAll(() => { + cloudWatchClient.destroy(); + }); + + it("should create a DLQ depth alarm for every target DLQ", async () => { + expect(targetIds.length).toBeGreaterThan(0); + + for (const targetId of targetIds) { + const alarmName = buildDlqDepthAlarmName(deploymentDetails, targetId); + const targetDlqQueueUrl = buildMockClientDlqQueueUrl(deploymentDetails, [ + { targetId }, + ]); + const targetDlqQueueName = getQueueNameFromUrl(targetDlqQueueUrl); + const response = await cloudWatchClient.send( + new DescribeAlarmsCommand({ + AlarmNames: [alarmName], + }), + ); + + const alarm: MetricAlarm | undefined = response.MetricAlarms?.[0]; + + expect(alarm?.AlarmName).toBe(alarmName); + expect(alarm?.MetricName).toBe("ApproximateNumberOfMessagesVisible"); + expect(alarm?.Namespace).toBe("AWS/SQS"); + expect(alarm?.Threshold).toBe(0); + expect(alarm?.Dimensions).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + Name: "QueueName", + Value: targetDlqQueueName, + }), + ]), + ); + } + }, 120_000); +}); diff --git a/tests/integration/dlq-redrive.test.ts b/tests/integration/dlq-redrive.test.ts index 3360a792..e88e4920 100644 --- a/tests/integration/dlq-redrive.test.ts +++ b/tests/integration/dlq-redrive.test.ts @@ -18,7 +18,11 @@ import { purgeQueues, sendSqsEvent, } from "./helpers/sqs"; -import { buildMockWebhookTargetPath } from "./helpers/mock-client-config"; +import { + buildMockWebhookTargetPath, + getAllSubscriptionTargetIds, + getMockItClientConfig, +} from "./helpers/mock-client-config"; import { awaitSignedCallbacksFromWebhookLogGroup } from "./helpers/cloudwatch"; import { createMessageStatusPublishEvent } from "./helpers/event-factories"; import sendEventToDlqAndRedrive from "./helpers/redrive"; @@ -27,41 +31,57 @@ describe("DLQ Redrive", () => { let sqsClient: SQSClient; let cloudWatchClient: CloudWatchLogsClient; let dlqQueueUrl!: string; + let allTargetDlqQueueUrls: string[]; let inboundQueueUrl: string; let webhookLogGroupName: string; beforeAll(async () => { const deploymentDetails = getDeploymentDetails(); + const mockClient1 = getMockItClientConfig(); + + const allSubscriptionTargetIds = getAllSubscriptionTargetIds(); sqsClient = createSqsClient(deploymentDetails); cloudWatchClient = createCloudWatchLogsClient(deploymentDetails); inboundQueueUrl = buildInboundEventQueueUrl(deploymentDetails); - dlqQueueUrl = buildMockClientDlqQueueUrl(deploymentDetails); + dlqQueueUrl = buildMockClientDlqQueueUrl( + deploymentDetails, + mockClient1.targets, + ); + allTargetDlqQueueUrls = allSubscriptionTargetIds.map((targetId) => + buildMockClientDlqQueueUrl(deploymentDetails, [{ targetId }]), + ); webhookLogGroupName = buildLambdaLogGroupName( deploymentDetails, "mock-webhook", ); - await purgeQueues(sqsClient, [inboundQueueUrl, dlqQueueUrl]); + await purgeQueues(sqsClient, [inboundQueueUrl, ...allTargetDlqQueueUrls]); }); afterAll(async () => { - await purgeQueues(sqsClient, [inboundQueueUrl, dlqQueueUrl]); + await purgeQueues(sqsClient, [inboundQueueUrl, ...allTargetDlqQueueUrls]); sqsClient.destroy(); cloudWatchClient.destroy(); }); describe("Infrastructure validation", () => { - it("should confirm the target DLQ is accessible", async () => { - const response = await sqsClient.send( - new GetQueueAttributesCommand({ - QueueUrl: dlqQueueUrl, - AttributeNames: ["QueueArn", "ApproximateNumberOfMessages"], - }), + it("should confirm a target DLQ is accessible for all configured subscription targets", async () => { + const responses = await Promise.all( + allTargetDlqQueueUrls.map((queueUrl) => + sqsClient.send( + new GetQueueAttributesCommand({ + QueueUrl: queueUrl, + AttributeNames: ["QueueArn", "ApproximateNumberOfMessages"], + }), + ), + ), ); - expect(response.Attributes?.QueueArn).toBeDefined(); + for (const response of responses) { + expect(response.Attributes?.QueueArn).toBeDefined(); + } }); it("should confirm the inbound event queue exists and is accessible", async () => { diff --git a/tests/integration/helpers/cloudwatch.ts b/tests/integration/helpers/cloudwatch.ts index b66aedb1..9ee13739 100644 --- a/tests/integration/helpers/cloudwatch.ts +++ b/tests/integration/helpers/cloudwatch.ts @@ -144,6 +144,56 @@ export async function awaitSignedCallbacksFromWebhookLogGroup( return callbacks; } +export async function awaitSignedCallbacksByCountFromWebhookLogGroup( + client: CloudWatchLogsClient, + logGroupName: string, + messageId: string, + callbackType: CallbackItem["type"], + expectedCount: number, + startTime: number, +): Promise { + logger.debug( + `Waiting for callbacks in webhook CloudWatch log group (messageId=${messageId}, callbackType=${callbackType}, expectedCount=${expectedCount}, logGroup=${logGroupName})`, + ); + + let callbacks: SignedCallback[] = []; + + try { + await waitUntil( + async () => { + callbacks = await querySignedCallbacksFromWebhookLogGroup( + client, + logGroupName, + messageId, + callbackType, + startTime, + ); + return callbacks.length === expectedCount; + }, + { + timeout: CALLBACK_WAIT_TIMEOUT_MS, + intervalBetweenAttempts: POLL_INTERVAL_MS, + }, + ); + } catch (error) { + if (error instanceof TimeoutError) { + logger.warn( + `Timed out waiting for callbacks in webhook CloudWatch log group (messageId=${messageId}, callbackType=${callbackType}, expectedCount=${expectedCount}, timeoutMs=${CALLBACK_WAIT_TIMEOUT_MS})`, + ); + } else { + throw error; + } + } + + if (callbacks.length !== expectedCount) { + throw new Error( + `Expected exactly ${expectedCount} callbacks for messageId="${messageId}" callbackType="${callbackType}", but found ${callbacks.length}`, + ); + } + + return callbacks; +} + type EmfEntry = Record; function collectMetricNamesFromEvent( diff --git a/tests/integration/helpers/mock-client-config.ts b/tests/integration/helpers/mock-client-config.ts index 42bac438..c03002de 100644 --- a/tests/integration/helpers/mock-client-config.ts +++ b/tests/integration/helpers/mock-client-config.ts @@ -24,6 +24,16 @@ export const CLIENT_FIXTURES = { export type ClientFixtureKey = keyof typeof CLIENT_FIXTURES; +const ALL_CLIENT_FIXTURE_KEYS = Object.keys( + CLIENT_FIXTURES, +) as ClientFixtureKey[]; + +function dedupe(values: string[]): string[] { + return values.filter( + (value, index, allValues) => allValues.indexOf(value) === index, + ); +} + export function getClientConfig(key: ClientFixtureKey): MockItClientConfig { // eslint-disable-next-line security/detect-object-injection -- key is constrained to ClientFixtureKey, a keyof the hardcoded as-const CLIENT_FIXTURES object const { apiKeyVar, applicationIdVar, fixture } = CLIENT_FIXTURES[key]; @@ -47,9 +57,40 @@ export function getMockItClient2Config(): MockItClientConfig { return getClientConfig("client2"); } +function buildWebhookTargetPaths(key: ClientFixtureKey): string[] { + const config = getClientConfig(key); + return config.targets.map(({ targetId }) => `/${targetId}`); +} + export function buildMockWebhookTargetPath( key: ClientFixtureKey = "client1", ): string { + const paths = buildWebhookTargetPaths(key); + + if (paths.length === 0) { + throw new Error(`No webhook targets configured for fixture key: ${key}`); + } + + return paths[0]; +} + +export function buildMockWebhookTargetPaths( + key: ClientFixtureKey = "client1", +): string[] { + return buildWebhookTargetPaths(key); +} + +export function getSubscriptionTargetIds( + key: ClientFixtureKey = "client1", +): string[] { const config = getClientConfig(key); - return `/${config.targets[0].targetId}`; + return dedupe( + config.subscriptions.flatMap((subscription) => subscription.targetIds), + ); +} + +export function getAllSubscriptionTargetIds( + keys: ClientFixtureKey[] = ALL_CLIENT_FIXTURE_KEYS, +): string[] { + return dedupe(keys.flatMap((key) => getSubscriptionTargetIds(key))); } diff --git a/tests/integration/helpers/sqs.ts b/tests/integration/helpers/sqs.ts index 7df2bee9..c98ea4ca 100644 --- a/tests/integration/helpers/sqs.ts +++ b/tests/integration/helpers/sqs.ts @@ -12,8 +12,6 @@ import { logger } from "@nhs-notify-client-callbacks/logger"; import type { DeploymentDetails } from "@nhs-notify-client-callbacks/test-support/helpers/deployment"; import { waitUntil } from "async-wait-until"; -import { getMockItClientConfig } from "./mock-client-config"; - const QUEUE_WAIT_TIMEOUT_MS = 60_000; const POLL_INTERVAL_MS = 500; const SQS_MAX_NUMBER_OF_MESSAGES = 1; @@ -48,8 +46,8 @@ function buildQueueUrl( export function buildMockClientDlqQueueUrl( deploymentDetails: DeploymentDetails, + targets: { targetId: string }[], ): string { - const { targets } = getMockItClientConfig(); return buildQueueUrl(deploymentDetails, `${targets[0].targetId}-dlq`); } diff --git a/tests/integration/helpers/status-events.ts b/tests/integration/helpers/status-events.ts index b50b87da..1bccf0bb 100644 --- a/tests/integration/helpers/status-events.ts +++ b/tests/integration/helpers/status-events.ts @@ -15,15 +15,17 @@ import { ensureInboundQueueIsEmpty, sendSqsEvent } from "./sqs"; async function processStatusEvent< T extends MessageStatusData | ChannelStatusData, >( - sqsClient: SQSClient, - cloudWatchClient: CloudWatchLogsClient, + { + CloudWatchLogsClient: cloudWatchClient, + SQSClient: sqsClient, + }: { CloudWatchLogsClient: CloudWatchLogsClient; SQSClient: SQSClient }, callbackEventQueueUrl: string, webhookLogGroupName: string, event: StatusPublishEvent, callbackType: SignedCallback["payload"]["type"], webhookPath: string, + startTime: number, ): Promise { - const startTime = Date.now(); const sendMessageResponse = await sendSqsEvent( sqsClient, callbackEventQueueUrl, @@ -53,15 +55,16 @@ export async function processMessageStatusEvent( webhookLogGroupName: string, messageStatusEvent: StatusPublishEvent, webhookPath: string, + startTime: number, ): Promise { return processStatusEvent( - sqsClient, - cloudWatchClient, + { CloudWatchLogsClient: cloudWatchClient, SQSClient: sqsClient }, callbackEventQueueUrl, webhookLogGroupName, messageStatusEvent, "MessageStatus", webhookPath, + startTime, ); } @@ -72,14 +75,15 @@ export async function processChannelStatusEvent( webhookLogGroupName: string, channelStatusEvent: StatusPublishEvent, webhookPath: string, + startTime: number, ): Promise { return processStatusEvent( - sqsClient, - cloudWatchClient, + { CloudWatchLogsClient: cloudWatchClient, SQSClient: sqsClient }, callbackEventQueueUrl, webhookLogGroupName, channelStatusEvent, "ChannelStatus", webhookPath, + startTime, ); } diff --git a/tests/integration/inbound-sqs-to-webhook.test.ts b/tests/integration/inbound-sqs-to-webhook.test.ts index 3e61d090..0a7b4451 100644 --- a/tests/integration/inbound-sqs-to-webhook.test.ts +++ b/tests/integration/inbound-sqs-to-webhook.test.ts @@ -1,5 +1,5 @@ -import { DeleteMessageCommand, SQSClient } from "@aws-sdk/client-sqs"; import { CloudWatchLogsClient } from "@aws-sdk/client-cloudwatch-logs"; +import { DeleteMessageCommand, SQSClient } from "@aws-sdk/client-sqs"; import { type ChannelStatusData, type MessageStatusData, @@ -13,8 +13,18 @@ import { createSqsClient, getDeploymentDetails, } from "@nhs-notify-client-callbacks/test-support/helpers"; +import { awaitSignedCallbacksByCountFromWebhookLogGroup } from "./helpers/cloudwatch"; +import { + createChannelStatusPublishEvent, + createMessageStatusPublishEvent, +} from "./helpers/event-factories"; +import { + buildMockWebhookTargetPath, + buildMockWebhookTargetPaths, + getMockItClient2Config, + getMockItClientConfig, +} from "./helpers/mock-client-config"; import { assertCallbackHeaders } from "./helpers/signature"; -import { buildMockWebhookTargetPath } from "./helpers/mock-client-config"; import { awaitQueueMessage, awaitQueueMessageByMessageId, @@ -23,10 +33,6 @@ import { purgeQueues, sendSqsEvent, } from "./helpers/sqs"; -import { - createChannelStatusPublishEvent, - createMessageStatusPublishEvent, -} from "./helpers/event-factories"; import { processChannelStatusEvent, processMessageStatusEvent, @@ -40,21 +46,23 @@ describe("SQS to Webhook Integration", () => { let inboundEventDlqQueueUrl: string; let webhookLogGroupName: string; let webhookTargetPath: string; + let startTime: number; beforeAll(async () => { const deploymentDetails = getDeploymentDetails(); + const { targets } = getMockItClientConfig(); sqsClient = createSqsClient(deploymentDetails); cloudWatchClient = createCloudWatchLogsClient(deploymentDetails); callbackEventQueueUrl = buildInboundEventQueueUrl(deploymentDetails); - clientDlqQueueUrl = buildMockClientDlqQueueUrl(deploymentDetails); + clientDlqQueueUrl = buildMockClientDlqQueueUrl(deploymentDetails, targets); inboundEventDlqQueueUrl = buildInboundEventDlqQueueUrl(deploymentDetails); webhookLogGroupName = buildLambdaLogGroupName( deploymentDetails, "mock-webhook", ); webhookTargetPath = buildMockWebhookTargetPath(); - + startTime = Date.now(); await purgeQueues(sqsClient, [ inboundEventDlqQueueUrl, clientDlqQueueUrl, @@ -85,6 +93,7 @@ describe("SQS to Webhook Integration", () => { webhookLogGroupName, messageStatusEvent, webhookTargetPath, + startTime, ); expect(callbacks).toHaveLength(1); @@ -99,6 +108,55 @@ describe("SQS to Webhook Integration", () => { assertCallbackHeaders(callbacks[0]); }, 120_000); + + it("should fan out a message status event to subscription with multiple target endpoints", async () => { + const client2Config = getMockItClient2Config(); + const expectedPaths = buildMockWebhookTargetPaths("client2"); + + const messageStatusEvent: StatusPublishEvent = + createMessageStatusPublishEvent({ + data: { + clientId: client2Config.clientId, + }, + }); + + await sendSqsEvent(sqsClient, callbackEventQueueUrl, messageStatusEvent); + await ensureInboundQueueIsEmpty(sqsClient, callbackEventQueueUrl); + + const callbacks = await awaitSignedCallbacksByCountFromWebhookLogGroup( + cloudWatchClient, + webhookLogGroupName, + messageStatusEvent.data.messageId, + "MessageStatus", + expectedPaths.length, + startTime, + ); + + expect(callbacks).toHaveLength(expectedPaths.length); + + const actualPaths = callbacks + .map((callback) => callback.path) + .toSorted((a, b) => a.localeCompare(b)); + expect(actualPaths).toEqual( + expectedPaths.toSorted((a, b) => a.localeCompare(b)), + ); + + for (const callback of callbacks) { + expect(callback.payload).toMatchObject({ + type: "MessageStatus", + attributes: expect.objectContaining({ + messageId: messageStatusEvent.data.messageId, + messageStatus: "delivered", + }), + }); + + assertCallbackHeaders( + callback, + client2Config.apiKeyVar, + client2Config.applicationIdVar, + ); + } + }, 120_000); }); describe("Channel Status Event Flow", () => { @@ -113,6 +171,7 @@ describe("SQS to Webhook Integration", () => { webhookLogGroupName, channelStatusEvent, webhookTargetPath, + startTime, ); expect(callbacks).toHaveLength(1); diff --git a/tests/integration/infrastructure-exists.test.ts b/tests/integration/infrastructure-exists.test.ts deleted file mode 100644 index 37c88dc6..00000000 --- a/tests/integration/infrastructure-exists.test.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { HeadBucketCommand } from "@aws-sdk/client-s3"; -import type { S3Client } from "@aws-sdk/client-s3"; -import { - buildSubscriptionConfigBucketName, - createS3Client, - getDeploymentDetails, -} from "@nhs-notify-client-callbacks/test-support/helpers"; - -describe("Infrastructure exists", () => { - let s3Client: S3Client; - let bucketName: string; - - beforeAll(async () => { - const deploymentDetails = getDeploymentDetails(); - bucketName = buildSubscriptionConfigBucketName(deploymentDetails); - s3Client = createS3Client(deploymentDetails); - }); - - afterAll(() => { - s3Client?.destroy(); - }); - - it("should confirm the subscription config S3 bucket exists", async () => { - const response = await s3Client.send( - new HeadBucketCommand({ Bucket: bucketName }), - ); - - expect(response.$metadata.httpStatusCode).toBe(200); - }); -}); diff --git a/tests/integration/metrics.test.ts b/tests/integration/metrics.test.ts index 6391c093..2f314f85 100644 --- a/tests/integration/metrics.test.ts +++ b/tests/integration/metrics.test.ts @@ -19,7 +19,10 @@ import { purgeQueues, sendSqsEvent, } from "./helpers/sqs"; -import { buildMockWebhookTargetPath } from "./helpers/mock-client-config"; +import { + buildMockWebhookTargetPath, + getMockItClientConfig, +} from "./helpers/mock-client-config"; import { awaitAllEmfMetricsInLogGroup, awaitSignedCallbacksFromWebhookLogGroup, @@ -37,11 +40,12 @@ describe("Metrics", () => { beforeAll(async () => { const deploymentDetails = getDeploymentDetails(); + const { targets } = getMockItClientConfig(); sqsClient = createSqsClient(deploymentDetails); cloudWatchClient = createCloudWatchLogsClient(deploymentDetails); callbackEventQueueUrl = buildInboundEventQueueUrl(deploymentDetails); - clientDlqQueueUrl = buildMockClientDlqQueueUrl(deploymentDetails); + clientDlqQueueUrl = buildMockClientDlqQueueUrl(deploymentDetails, targets); inboundEventDlqQueueUrl = buildInboundEventDlqQueueUrl(deploymentDetails); logGroupName = buildLambdaLogGroupName( deploymentDetails, diff --git a/tests/integration/package.json b/tests/integration/package.json index 7a4b6cbc..e3e6543d 100644 --- a/tests/integration/package.json +++ b/tests/integration/package.json @@ -13,6 +13,7 @@ "typecheck": "tsc --noEmit" }, "dependencies": { + "@aws-sdk/client-cloudwatch": "^3.1019.0", "@nhs-notify-client-callbacks/logger": "*", "@nhs-notify-client-callbacks/models": "*", "@aws-sdk/client-cloudwatch-logs": "^3.1020.0",