From d2eebf1a4fa125b0b07df1f0fb2e9612f070d6ea Mon Sep 17 00:00:00 2001 From: Mark Ramsden Date: Fri, 8 May 2026 07:50:08 +0100 Subject: [PATCH 01/10] CCM-15212: add SQS into eventpub module --- .../eventpub/lambda/eventpub/src/index.js | 6 ++-- .../modules/eventpub/lambda_function.tf | 17 +++++++++- .../modules/eventpub/module_sqs_queue.tf | 33 +++++++++++++++++++ .../eventpub/sns_topic_subscription_lambda.tf | 5 --- .../eventpub/sns_topic_subscription_sqs.tf | 5 +++ 5 files changed, 57 insertions(+), 9 deletions(-) create mode 100644 infrastructure/terraform/modules/eventpub/module_sqs_queue.tf delete mode 100644 infrastructure/terraform/modules/eventpub/sns_topic_subscription_lambda.tf create mode 100644 infrastructure/terraform/modules/eventpub/sns_topic_subscription_sqs.tf diff --git a/infrastructure/terraform/modules/eventpub/lambda/eventpub/src/index.js b/infrastructure/terraform/modules/eventpub/lambda/eventpub/src/index.js index 6c305a5..4bfdc7f 100644 --- a/infrastructure/terraform/modules/eventpub/lambda/eventpub/src/index.js +++ b/infrastructure/terraform/modules/eventpub/lambda/eventpub/src/index.js @@ -95,15 +95,15 @@ async function sendToDLQ(events) { } } -exports.handler = async (snsEvent) => { - console.debug(`Received SNS event with ${snsEvent.Records.length} records.`); +exports.handler = async (sqsEvent) => { + console.debug(`Received SQS event with ${sqsEvent.Records.length} records.`); if (THROTTLE_DELAY_MS > 0) { console.info(`Throttling enabled. Delaying processing by ${THROTTLE_DELAY_MS}ms`); await new Promise(res => setTimeout(res, THROTTLE_DELAY_MS)); } - const records = snsEvent.Records.map(record => JSON.parse(record.Sns.Message)); + const records = sqsEvent.Records.map(record => JSON.parse(record.body)); const validEvents = records.filter(validateEvent); const invalidEvents = records.filter(event => !validateEvent(event)); diff --git a/infrastructure/terraform/modules/eventpub/lambda_function.tf b/infrastructure/terraform/modules/eventpub/lambda_function.tf index db7f925..497f877 100644 --- a/infrastructure/terraform/modules/eventpub/lambda_function.tf +++ b/infrastructure/terraform/modules/eventpub/lambda_function.tf @@ -6,7 +6,7 @@ resource "aws_lambda_function" "main" { handler = "index.handler" runtime = "nodejs22.x" publish = true - memory_size = 128 + memory_size = 512 timeout = 20 filename = data.archive_file.lambda.output_path @@ -28,3 +28,18 @@ resource "aws_lambda_function" "main" { } } } + +resource "aws_lambda_event_source_mapping" "sqs_to_lambda" { + event_source_arn = module.sqs_queue.sqs_queue_arn + function_name = aws_lambda_function.main.function_name + batch_size = 5000 + maximum_batching_window_in_seconds = 0 + function_response_types = [ + "ReportBatchItemFailures" + ] + + scaling_config { + maximum_concurrency = 20 + } +} + diff --git a/infrastructure/terraform/modules/eventpub/module_sqs_queue.tf b/infrastructure/terraform/modules/eventpub/module_sqs_queue.tf new file mode 100644 index 0000000..366fe49 --- /dev/null +++ b/infrastructure/terraform/modules/eventpub/module_sqs_queue.tf @@ -0,0 +1,33 @@ +module "sqs_queue" { + source = "../../sqs" + + aws_account_id = var.aws_account_id + component = var.component + environment = var.environment + project = var.project + region = var.region + name = local.csi + create_dlq = true + sqs_kms_key_arn = var.kms_key_arn + sqs_policy_overload = data.aws_iam_policy_document.allow_sns_send.json +} + +data "aws_iam_policy_document" "allow_sns_send" { + statement { + sid = "AllowSNSSendMessage" + effect = "Allow" + + principals { + type = "Service" + identifiers = ["sns.amazonaws.com"] + } + + actions = [ + "sqs:SendMessage", + ] + + resources = [ + module.sqs_queue.sqs_queue_arn, + ] + } +} \ No newline at end of file diff --git a/infrastructure/terraform/modules/eventpub/sns_topic_subscription_lambda.tf b/infrastructure/terraform/modules/eventpub/sns_topic_subscription_lambda.tf deleted file mode 100644 index ffee18a..0000000 --- a/infrastructure/terraform/modules/eventpub/sns_topic_subscription_lambda.tf +++ /dev/null @@ -1,5 +0,0 @@ -resource "aws_sns_topic_subscription" "lambda" { - topic_arn = aws_sns_topic.main.arn - protocol = "lambda" - endpoint = aws_lambda_function.main.arn -} diff --git a/infrastructure/terraform/modules/eventpub/sns_topic_subscription_sqs.tf b/infrastructure/terraform/modules/eventpub/sns_topic_subscription_sqs.tf new file mode 100644 index 0000000..46241c8 --- /dev/null +++ b/infrastructure/terraform/modules/eventpub/sns_topic_subscription_sqs.tf @@ -0,0 +1,5 @@ +resource "aws_sns_topic_subscription" "sqs" { + topic_arn = aws_sns_topic.main.arn + protocol = "sqs" + endpoint = module.sqs_queue.sqs_queue_arn +} From 733dc747e9e3d72e0550aae70c6aea70eea4d6f7 Mon Sep 17 00:00:00 2001 From: Mark Ramsden Date: Fri, 8 May 2026 08:19:22 +0100 Subject: [PATCH 02/10] CCM-15212: linting --- infrastructure/terraform/modules/eventpub/README.md | 1 + .../terraform/modules/eventpub/lambda/eventpub/src/index.js | 2 ++ infrastructure/terraform/modules/eventpub/module_sqs_queue.tf | 4 ++-- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/infrastructure/terraform/modules/eventpub/README.md b/infrastructure/terraform/modules/eventpub/README.md index 8cb9ce9..471ba58 100644 --- a/infrastructure/terraform/modules/eventpub/README.md +++ b/infrastructure/terraform/modules/eventpub/README.md @@ -43,6 +43,7 @@ | Name | Source | Version | |------|--------|---------| | [s3bucket\_event\_cache](#module\_s3bucket\_event\_cache) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.0.3/terraform-s3bucket.zip | n/a | +| [sqs\_queue](#module\_sqs\_queue) | ../sqs | n/a | ## Outputs | Name | Description | diff --git a/infrastructure/terraform/modules/eventpub/lambda/eventpub/src/index.js b/infrastructure/terraform/modules/eventpub/lambda/eventpub/src/index.js index 4bfdc7f..8eb4c6e 100644 --- a/infrastructure/terraform/modules/eventpub/lambda/eventpub/src/index.js +++ b/infrastructure/terraform/modules/eventpub/lambda/eventpub/src/index.js @@ -103,6 +103,8 @@ exports.handler = async (sqsEvent) => { await new Promise(res => setTimeout(res, THROTTLE_DELAY_MS)); } + console.log(record.body); + const records = sqsEvent.Records.map(record => JSON.parse(record.body)); const validEvents = records.filter(validateEvent); const invalidEvents = records.filter(event => !validateEvent(event)); diff --git a/infrastructure/terraform/modules/eventpub/module_sqs_queue.tf b/infrastructure/terraform/modules/eventpub/module_sqs_queue.tf index 366fe49..14ae078 100644 --- a/infrastructure/terraform/modules/eventpub/module_sqs_queue.tf +++ b/infrastructure/terraform/modules/eventpub/module_sqs_queue.tf @@ -1,5 +1,5 @@ module "sqs_queue" { - source = "../../sqs" + source = "../sqs" aws_account_id = var.aws_account_id component = var.component @@ -30,4 +30,4 @@ data "aws_iam_policy_document" "allow_sns_send" { module.sqs_queue.sqs_queue_arn, ] } -} \ No newline at end of file +} From 3e19e8f5cc3b8cb7e26036f60876f99d0c775f41 Mon Sep 17 00:00:00 2001 From: Mark Ramsden Date: Fri, 8 May 2026 08:43:28 +0100 Subject: [PATCH 03/10] CCM-15212: removed unused permission --- .../modules/eventpub/lambda_permissions_sns_event_cache.tf | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 infrastructure/terraform/modules/eventpub/lambda_permissions_sns_event_cache.tf diff --git a/infrastructure/terraform/modules/eventpub/lambda_permissions_sns_event_cache.tf b/infrastructure/terraform/modules/eventpub/lambda_permissions_sns_event_cache.tf deleted file mode 100644 index ad473e9..0000000 --- a/infrastructure/terraform/modules/eventpub/lambda_permissions_sns_event_cache.tf +++ /dev/null @@ -1,7 +0,0 @@ -resource "aws_lambda_permission" "sns_lambda" { - statement_id = "AllowExecutionFromSNS" - action = "lambda:InvokeFunction" - function_name = aws_lambda_function.main.function_name - principal = "sns.amazonaws.com" - source_arn = aws_sns_topic.main.arn -} From b5803bbdf4d8b191de363f2607e5cfb8dcb433dc Mon Sep 17 00:00:00 2001 From: Mark Ramsden Date: Fri, 8 May 2026 09:22:32 +0100 Subject: [PATCH 04/10] CCM-15212: use external sqs module reference --- infrastructure/terraform/modules/eventpub/README.md | 2 +- infrastructure/terraform/modules/eventpub/module_sqs_queue.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/infrastructure/terraform/modules/eventpub/README.md b/infrastructure/terraform/modules/eventpub/README.md index 471ba58..f46d061 100644 --- a/infrastructure/terraform/modules/eventpub/README.md +++ b/infrastructure/terraform/modules/eventpub/README.md @@ -43,7 +43,7 @@ | Name | Source | Version | |------|--------|---------| | [s3bucket\_event\_cache](#module\_s3bucket\_event\_cache) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.0.3/terraform-s3bucket.zip | n/a | -| [sqs\_queue](#module\_sqs\_queue) | ../sqs | n/a | +| [sqs\_queue](#module\_sqs\_queue) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.1.3/terraform-sqs.zip | n/a | ## Outputs | Name | Description | diff --git a/infrastructure/terraform/modules/eventpub/module_sqs_queue.tf b/infrastructure/terraform/modules/eventpub/module_sqs_queue.tf index 14ae078..0512d7d 100644 --- a/infrastructure/terraform/modules/eventpub/module_sqs_queue.tf +++ b/infrastructure/terraform/modules/eventpub/module_sqs_queue.tf @@ -1,5 +1,5 @@ module "sqs_queue" { - source = "../sqs" + source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.1.3/terraform-sqs.zip" aws_account_id = var.aws_account_id component = var.component From 83d82f7241d8e54c9aa419929a9cb04af06c8fb0 Mon Sep 17 00:00:00 2001 From: Mark Ramsden Date: Fri, 8 May 2026 09:56:57 +0100 Subject: [PATCH 05/10] CCM-15212: configure batching --- infrastructure/terraform/modules/eventpub/lambda_function.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/terraform/modules/eventpub/lambda_function.tf b/infrastructure/terraform/modules/eventpub/lambda_function.tf index 497f877..2610c38 100644 --- a/infrastructure/terraform/modules/eventpub/lambda_function.tf +++ b/infrastructure/terraform/modules/eventpub/lambda_function.tf @@ -33,7 +33,7 @@ resource "aws_lambda_event_source_mapping" "sqs_to_lambda" { event_source_arn = module.sqs_queue.sqs_queue_arn function_name = aws_lambda_function.main.function_name batch_size = 5000 - maximum_batching_window_in_seconds = 0 + maximum_batching_window_in_seconds = 1 function_response_types = [ "ReportBatchItemFailures" ] From 578fa0c4038a8331dac93e5f9d33c348a5bf8010 Mon Sep 17 00:00:00 2001 From: Mark Ramsden Date: Fri, 8 May 2026 10:37:03 +0100 Subject: [PATCH 06/10] CCM-15212: update permissions --- .../modules/eventpub/iam_role_lambda.tf | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/infrastructure/terraform/modules/eventpub/iam_role_lambda.tf b/infrastructure/terraform/modules/eventpub/iam_role_lambda.tf index d53bd25..c6d5d5f 100644 --- a/infrastructure/terraform/modules/eventpub/iam_role_lambda.tf +++ b/infrastructure/terraform/modules/eventpub/iam_role_lambda.tf @@ -81,6 +81,22 @@ data "aws_iam_policy_document" "lambda" { ] } + statement { + sid = "AllowSQSInput" + effect = "Allow" + + actions = [ + "sqs:ReceiveMessage", + "sqs:DeleteMessage", + "sqs:GetQueueAttributes", + "sqs:ChangeMessageVisibility", + ] + + resources = [ + module.sqs_queue.sqs_queue_arn, + ] + } + statement { sid = "KMSCloudwatchKeyAccess" effect = "Allow" From 9cc596917786e68bf9d9febb5f3a6cf9c2bc1e26 Mon Sep 17 00:00:00 2001 From: Mark Ramsden Date: Fri, 8 May 2026 11:51:37 +0100 Subject: [PATCH 07/10] CCM-15212: update eventpub module --- .../modules/eventpub/module_sqs_queue.tf | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/infrastructure/terraform/modules/eventpub/module_sqs_queue.tf b/infrastructure/terraform/modules/eventpub/module_sqs_queue.tf index 0512d7d..9f19a47 100644 --- a/infrastructure/terraform/modules/eventpub/module_sqs_queue.tf +++ b/infrastructure/terraform/modules/eventpub/module_sqs_queue.tf @@ -1,15 +1,16 @@ module "sqs_queue" { source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.1.3/terraform-sqs.zip" - aws_account_id = var.aws_account_id - component = var.component - environment = var.environment - project = var.project - region = var.region - name = local.csi - create_dlq = true - sqs_kms_key_arn = var.kms_key_arn - sqs_policy_overload = data.aws_iam_policy_document.allow_sns_send.json + aws_account_id = var.aws_account_id + component = var.component + environment = var.environment + project = var.project + region = var.region + name = local.csi + create_dlq = true + sqs_kms_key_arn = var.sqs_kms_key_arn + sqs_policy_overload = data.aws_iam_policy_document.allow_sns_send.json + message_retention_seconds = 1209600 # 14 days } data "aws_iam_policy_document" "allow_sns_send" { From 9b7ec718f599ba92e5d0017b0fa975b0828aed73 Mon Sep 17 00:00:00 2001 From: Mark Ramsden Date: Fri, 8 May 2026 11:56:13 +0100 Subject: [PATCH 08/10] CCM-15212: update eventpub module --- infrastructure/terraform/modules/eventpub/module_sqs_queue.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/terraform/modules/eventpub/module_sqs_queue.tf b/infrastructure/terraform/modules/eventpub/module_sqs_queue.tf index 9f19a47..5a6d1de 100644 --- a/infrastructure/terraform/modules/eventpub/module_sqs_queue.tf +++ b/infrastructure/terraform/modules/eventpub/module_sqs_queue.tf @@ -8,7 +8,7 @@ module "sqs_queue" { region = var.region name = local.csi create_dlq = true - sqs_kms_key_arn = var.sqs_kms_key_arn + sqs_kms_key_arn = var.kms_key_arn sqs_policy_overload = data.aws_iam_policy_document.allow_sns_send.json message_retention_seconds = 1209600 # 14 days } From 6d35e880be0a05596967e5fe0885155504a6380c Mon Sep 17 00:00:00 2001 From: Mark Ramsden Date: Fri, 8 May 2026 13:02:00 +0100 Subject: [PATCH 09/10] CCM-15212: update eventpub module --- .../modules/eventpub/lambda/eventpub/src/index.js | 8 +++++--- .../terraform/modules/eventpub/module_sqs_queue.tf | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/infrastructure/terraform/modules/eventpub/lambda/eventpub/src/index.js b/infrastructure/terraform/modules/eventpub/lambda/eventpub/src/index.js index 8eb4c6e..1ae58f0 100644 --- a/infrastructure/terraform/modules/eventpub/lambda/eventpub/src/index.js +++ b/infrastructure/terraform/modules/eventpub/lambda/eventpub/src/index.js @@ -103,9 +103,11 @@ exports.handler = async (sqsEvent) => { await new Promise(res => setTimeout(res, THROTTLE_DELAY_MS)); } - console.log(record.body); - - const records = sqsEvent.Records.map(record => JSON.parse(record.body)); + const records = sqsEvent.Records + .map(record => record.body) + .map(JSON.parse) + .map(record => record.Message) + .map(JSON.parse); const validEvents = records.filter(validateEvent); const invalidEvents = records.filter(event => !validateEvent(event)); diff --git a/infrastructure/terraform/modules/eventpub/module_sqs_queue.tf b/infrastructure/terraform/modules/eventpub/module_sqs_queue.tf index 5a6d1de..08f2945 100644 --- a/infrastructure/terraform/modules/eventpub/module_sqs_queue.tf +++ b/infrastructure/terraform/modules/eventpub/module_sqs_queue.tf @@ -1,5 +1,5 @@ module "sqs_queue" { - source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.1.3/terraform-sqs.zip" + source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.1.4/terraform-sqs.zip" aws_account_id = var.aws_account_id component = var.component From 42877694800499517c19b740107b7339e1598e1e Mon Sep 17 00:00:00 2001 From: Mark Ramsden Date: Fri, 8 May 2026 13:02:12 +0100 Subject: [PATCH 10/10] CCM-15212: update eventpub module --- infrastructure/terraform/modules/eventpub/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/terraform/modules/eventpub/README.md b/infrastructure/terraform/modules/eventpub/README.md index f46d061..8a88106 100644 --- a/infrastructure/terraform/modules/eventpub/README.md +++ b/infrastructure/terraform/modules/eventpub/README.md @@ -43,7 +43,7 @@ | Name | Source | Version | |------|--------|---------| | [s3bucket\_event\_cache](#module\_s3bucket\_event\_cache) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.0.3/terraform-s3bucket.zip | n/a | -| [sqs\_queue](#module\_sqs\_queue) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.1.3/terraform-sqs.zip | n/a | +| [sqs\_queue](#module\_sqs\_queue) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.1.4/terraform-sqs.zip | n/a | ## Outputs | Name | Description |