diff --git a/infrastructure/terraform/modules/eventpub/README.md b/infrastructure/terraform/modules/eventpub/README.md index 8cb9ce9..8a88106 100644 --- a/infrastructure/terraform/modules/eventpub/README.md +++ b/infrastructure/terraform/modules/eventpub/README.md @@ -43,6 +43,7 @@ | Name | Source | Version | |------|--------|---------| | [s3bucket\_event\_cache](#module\_s3bucket\_event\_cache) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.0.3/terraform-s3bucket.zip | n/a | +| [sqs\_queue](#module\_sqs\_queue) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.1.4/terraform-sqs.zip | n/a | ## Outputs | Name | Description | diff --git a/infrastructure/terraform/modules/eventpub/iam_role_lambda.tf b/infrastructure/terraform/modules/eventpub/iam_role_lambda.tf index d53bd25..c6d5d5f 100644 --- a/infrastructure/terraform/modules/eventpub/iam_role_lambda.tf +++ b/infrastructure/terraform/modules/eventpub/iam_role_lambda.tf @@ -81,6 +81,22 @@ data "aws_iam_policy_document" "lambda" { ] } + statement { + sid = "AllowSQSInput" + effect = "Allow" + + actions = [ + "sqs:ReceiveMessage", + "sqs:DeleteMessage", + "sqs:GetQueueAttributes", + "sqs:ChangeMessageVisibility", + ] + + resources = [ + module.sqs_queue.sqs_queue_arn, + ] + } + statement { sid = "KMSCloudwatchKeyAccess" effect = "Allow" diff --git a/infrastructure/terraform/modules/eventpub/lambda/eventpub/src/index.js b/infrastructure/terraform/modules/eventpub/lambda/eventpub/src/index.js index 6c305a5..1ae58f0 100644 --- a/infrastructure/terraform/modules/eventpub/lambda/eventpub/src/index.js +++ b/infrastructure/terraform/modules/eventpub/lambda/eventpub/src/index.js @@ -95,15 +95,19 @@ async function sendToDLQ(events) { } } -exports.handler = async (snsEvent) => { - console.debug(`Received SNS event with ${snsEvent.Records.length} records.`); +exports.handler = async (sqsEvent) => { + console.debug(`Received SQS event with ${sqsEvent.Records.length} records.`); if (THROTTLE_DELAY_MS > 0) { console.info(`Throttling enabled. Delaying processing by ${THROTTLE_DELAY_MS}ms`); await new Promise(res => setTimeout(res, THROTTLE_DELAY_MS)); } - const records = snsEvent.Records.map(record => JSON.parse(record.Sns.Message)); + const records = sqsEvent.Records + .map(record => record.body) + .map(JSON.parse) + .map(record => record.Message) + .map(JSON.parse); const validEvents = records.filter(validateEvent); const invalidEvents = records.filter(event => !validateEvent(event)); diff --git a/infrastructure/terraform/modules/eventpub/lambda_function.tf b/infrastructure/terraform/modules/eventpub/lambda_function.tf index db7f925..2610c38 100644 --- a/infrastructure/terraform/modules/eventpub/lambda_function.tf +++ b/infrastructure/terraform/modules/eventpub/lambda_function.tf @@ -6,7 +6,7 @@ resource "aws_lambda_function" "main" { handler = "index.handler" runtime = "nodejs22.x" publish = true - memory_size = 128 + memory_size = 512 timeout = 20 filename = data.archive_file.lambda.output_path @@ -28,3 +28,18 @@ resource "aws_lambda_function" "main" { } } } + +resource "aws_lambda_event_source_mapping" "sqs_to_lambda" { + event_source_arn = module.sqs_queue.sqs_queue_arn + function_name = aws_lambda_function.main.function_name + batch_size = 5000 + maximum_batching_window_in_seconds = 1 + function_response_types = [ + "ReportBatchItemFailures" + ] + + scaling_config { + maximum_concurrency = 20 + } +} + diff --git a/infrastructure/terraform/modules/eventpub/lambda_permissions_sns_event_cache.tf b/infrastructure/terraform/modules/eventpub/lambda_permissions_sns_event_cache.tf deleted file mode 100644 index ad473e9..0000000 --- a/infrastructure/terraform/modules/eventpub/lambda_permissions_sns_event_cache.tf +++ /dev/null @@ -1,7 +0,0 @@ -resource "aws_lambda_permission" "sns_lambda" { - statement_id = "AllowExecutionFromSNS" - action = "lambda:InvokeFunction" - function_name = aws_lambda_function.main.function_name - principal = "sns.amazonaws.com" - source_arn = aws_sns_topic.main.arn -} diff --git a/infrastructure/terraform/modules/eventpub/module_sqs_queue.tf b/infrastructure/terraform/modules/eventpub/module_sqs_queue.tf new file mode 100644 index 0000000..08f2945 --- /dev/null +++ b/infrastructure/terraform/modules/eventpub/module_sqs_queue.tf @@ -0,0 +1,34 @@ +module "sqs_queue" { + source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.1.4/terraform-sqs.zip" + + aws_account_id = var.aws_account_id + component = var.component + environment = var.environment + project = var.project + region = var.region + name = local.csi + create_dlq = true + sqs_kms_key_arn = var.kms_key_arn + sqs_policy_overload = data.aws_iam_policy_document.allow_sns_send.json + message_retention_seconds = 1209600 # 14 days +} + +data "aws_iam_policy_document" "allow_sns_send" { + statement { + sid = "AllowSNSSendMessage" + effect = "Allow" + + principals { + type = "Service" + identifiers = ["sns.amazonaws.com"] + } + + actions = [ + "sqs:SendMessage", + ] + + resources = [ + module.sqs_queue.sqs_queue_arn, + ] + } +} diff --git a/infrastructure/terraform/modules/eventpub/sns_topic_subscription_lambda.tf b/infrastructure/terraform/modules/eventpub/sns_topic_subscription_lambda.tf deleted file mode 100644 index ffee18a..0000000 --- a/infrastructure/terraform/modules/eventpub/sns_topic_subscription_lambda.tf +++ /dev/null @@ -1,5 +0,0 @@ -resource "aws_sns_topic_subscription" "lambda" { - topic_arn = aws_sns_topic.main.arn - protocol = "lambda" - endpoint = aws_lambda_function.main.arn -} diff --git a/infrastructure/terraform/modules/eventpub/sns_topic_subscription_sqs.tf b/infrastructure/terraform/modules/eventpub/sns_topic_subscription_sqs.tf new file mode 100644 index 0000000..46241c8 --- /dev/null +++ b/infrastructure/terraform/modules/eventpub/sns_topic_subscription_sqs.tf @@ -0,0 +1,5 @@ +resource "aws_sns_topic_subscription" "sqs" { + topic_arn = aws_sns_topic.main.arn + protocol = "sqs" + endpoint = module.sqs_queue.sqs_queue_arn +}