CCM-24041: standard letter breaches

.github/workflows/cicd-1-pull-request.yaml

@@ -8,8 +8,6 @@ on:
       - main
   pull_request:
     types: [opened, reopened, synchronize]
-    branches:
-      - main
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name }}

.github/workflows/stage-2-test.yaml

@@ -64,6 +64,14 @@ jobs:
     env:
       NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     steps:
+      # Pull testcontainers images in the background so the download overlaps
+      # the setup steps. Keep these references in sync with the images used in
+      # internal/datastore/src/__test__/db.ts and
+      # internal/sql-test-helpers/src/trino.ts.
+      - name: "Pre-pull test container images"
+        run: |
+          docker pull amazon/dynamodb-local > /dev/null 2>&1 &
+          docker pull trinodb/trino:476 > /dev/null 2>&1 &
       - name: "Checkout code"
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
       - name: "Repo setup"

infrastructure/terraform/components/api/README.md

@@ -92,6 +92,7 @@ No requirements.
 | <a name="module_populate_supplier_config_cache"></a> [populate\_supplier\_config\_cache](#module\_populate\_supplier\_config\_cache) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip | n/a |
 | <a name="module_post_letters"></a> [post\_letters](#module\_post\_letters) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip | n/a |
 | <a name="module_post_mi"></a> [post\_mi](#module\_post\_mi) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip | n/a |
+| <a name="module_s3bucket_athena_query_results"></a> [s3bucket\_athena\_query\_results](#module\_s3bucket\_athena\_query\_results) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.0.6/terraform-s3bucket.zip | n/a |
 | <a name="module_s3bucket_reporting"></a> [s3bucket\_reporting](#module\_s3bucket\_reporting) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.6/terraform-s3bucket.zip | n/a |
 | <a name="module_s3bucket_test_letters"></a> [s3bucket\_test\_letters](#module\_s3bucket\_test\_letters) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.26/terraform-s3bucket.zip | n/a |
 | <a name="module_sqs_alarms"></a> [sqs\_alarms](#module\_sqs\_alarms) | ../../modules/alarms-sqs | n/a |

infrastructure/terraform/components/api/athena_workgroup_letter_sla_breaches.tf

@@ -0,0 +1,19 @@
+resource "aws_athena_workgroup" "letter_sla_breaches" {
+  name          = "${local.csi}-letter-sla-breaches"
+  description   = "Athena workgroup for the letter SLA breaches state machine"
+  force_destroy = var.force_destroy
+
+  configuration {
+    enforce_workgroup_configuration    = true
+    publish_cloudwatch_metrics_enabled = true
+
+    result_configuration {
+      output_location = "s3://${module.s3bucket_athena_query_results.bucket}/letter-sla-breaches/"
+
+      encryption_configuration {
+        encryption_option = "SSE_KMS"
+        kms_key_arn       = module.kms.key_arn
+      }
+    }
+  }
+}

infrastructure/terraform/components/api/iam_role_policy_letter_sla_breaches.tf

@@ -1,5 +1,6 @@
 resource "aws_iam_role_policy" "letter_sla_breaches" {
-  count  = local.eventpub_parquet_enabled ? 1 : 0
+  count = local.eventpub_parquet_enabled ? 1 : 0
+
   name   = "${local.csi}-letter-sla-breaches-sfn"
   role   = aws_iam_role.letter_sla_breaches[0].id
   policy = data.aws_iam_policy_document.letter_sla_breaches_policy[0].json

infrastructure/terraform/components/api/locals.tf

@@ -78,6 +78,10 @@ locals {
 
   event_cache_bucket_name          = lookup(module.eventpub.s3_bucket_event_cache, "bucket", null)
   eventsub_event_cache_bucket_name = lookup(module.eventsub.s3_bucket_event_cache, "bucket", null)
+  reporting_bucket_name            = lookup(module.s3bucket_reporting, "bucket", null)
+
+  # The Glue crawler names tables after the crawled bucket, replacing characters that are invalid in table names
+  event_cache_glue_table_name = local.event_cache_bucket_name != null ? lower(replace(local.event_cache_bucket_name, "/[^a-zA-Z0-9_]/", "_")) : null
 
   csoc_api_gw_log_destination_arn = format("arn:aws:logs:%s:%s:destination:api_gateway_log_destination",
     var.region,

infrastructure/terraform/components/api/module_lambda_populate_standard_breaches.tf

@@ -1,5 +1,6 @@
 module "populate_standard_breaches" {
-  count  = local.eventpub_parquet_enabled ? 1 : 0
+  count = local.eventpub_parquet_enabled ? 1 : 0
+
   source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/6.0.5/terraform-lambda.zip"
 
   function_name = "populate_standard_breaches"
@@ -26,7 +27,7 @@ module "populate_standard_breaches" {
   handler_function_name   = "populateStandardBreachesHandler"
   runtime                 = "nodejs22.x"
   memory                  = 128
-  timeout                 = 29
+  timeout                 = 300
   log_level               = var.log_level
 
   force_lambda_code_deploy = var.force_lambda_code_deploy
@@ -37,9 +38,12 @@ module "populate_standard_breaches" {
   log_subscription_role_arn = local.acct.log_subscription_role_arn
 
   lambda_env_vars = {
-    ATHENA_WORKGROUP       = aws_athena_workgroup.reporting.name
-    ATHENA_DATABASE        = aws_glue_catalog_database.supplier.name
-    EVENT_CACHE_TABLE_NAME = aws_glue_catalog_table.eventpub_parquet[0].name
+    ATHENA_WORKGROUP              = aws_athena_workgroup.reporting.name
+    ATHENA_DATABASE               = aws_glue_catalog_database.supplier.name
+    EVENT_CACHE_TABLE_NAME        = aws_glue_catalog_table.eventpub_parquet[0].name
+    PACK_SPECIFICATION_TABLE_NAME = "${replace(local.csi, "-", "_")}_pack_specification"
+    DAYS_TO_LOOK_BACK             = "14"
+    STANDARD_SLA_WORKING_DAYS     = "2"
   }
 }
 

infrastructure/terraform/components/api/s3_athena_query_results.tf

@@ -0,0 +1,73 @@
+module "s3bucket_athena_query_results" {
+  source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.0.6/terraform-s3bucket.zip"
+
+  name = "athena-query-results"
+
+  aws_account_id = var.aws_account_id
+  region         = var.region
+  project        = var.project
+  environment    = var.environment
+  component      = var.component
+
+  acl           = "private"
+  force_destroy = var.force_destroy
+  versioning    = false
+  kms_key_arn   = module.kms.key_arn
+
+  lifecycle_rules = [
+    {
+      enabled = true
+
+      expiration = {
+        days = "7"
+      }
+
+      abort_incomplete_multipart_upload = {
+        days = "1"
+      }
+    }
+  ]
+
+  policy_documents = [
+    data.aws_iam_policy_document.s3bucket_athena_query_results.json
+  ]
+
+  bucket_logging_target = {
+    bucket = local.acct.s3_buckets["access_logs"]["id"]
+  }
+
+  public_access = {
+    block_public_acls       = true
+    block_public_policy     = true
+    ignore_public_acls      = true
+    restrict_public_buckets = true
+  }
+
+  default_tags = {
+    Name = "Athena Query Results"
+  }
+}
+
+data "aws_iam_policy_document" "s3bucket_athena_query_results" {
+  statement {
+    sid     = "DontAllowNonSecureConnection"
+    effect  = "Deny"
+    actions = ["s3:*"]
+
+    resources = [
+      module.s3bucket_athena_query_results.arn,
+      "${module.s3bucket_athena_query_results.arn}/*",
+    ]
+
+    principals {
+      type        = "AWS"
+      identifiers = ["*"]
+    }
+
+    condition {
+      test     = "Bool"
+      variable = "aws:SecureTransport"
+      values   = ["false"]
+    }
+  }
+}

infrastructure/terraform/components/api/sfn_state_machine_letter_sla_breaches.tf

@@ -1,5 +1,6 @@
 resource "aws_sfn_state_machine" "letter_sla_breaches" {
-  count    = local.eventpub_parquet_enabled ? 1 : 0
+  count = local.eventpub_parquet_enabled ? 1 : 0
+
   name     = local.letter_sla_breaches_state_machine_name
   role_arn = aws_iam_role.letter_sla_breaches[0].arn
 

infrastructure/terraform/modules/alarms-sla-breaches/README.md

@@ -0,0 +1,19 @@
+<!-- BEGIN_TF_DOCS -->
+<!-- markdownlint-disable -->
+<!-- vale off -->
+
+## Requirements
+
+No requirements.
+## Inputs
+
+No inputs.
+## Modules
+
+No modules.
+## Outputs
+
+No outputs.
+<!-- vale on -->
+<!-- markdownlint-enable -->
+<!-- END_TF_DOCS -->

infrastructure/terraform/modules/alarms-sla-breaches/cloudwatch_metric_alarm_errors.tf

@@ -0,0 +1,22 @@
+resource "aws_cloudwatch_metric_alarm" "errors" {
+  alarm_name        = "${var.alarm_prefix}-lambda-${var.function_name}-errors"
+  alarm_description = "ERROR: Lambda errors"
+
+  namespace   = "AWS/Lambda"
+  metric_name = "Errors"
+  statistic   = "Sum"
+  period      = var.period_seconds
+
+  evaluation_periods  = var.evaluation_periods
+  threshold           = var.errors_threshold
+  comparison_operator = "GreaterThanThreshold"
+  treat_missing_data  = "notBreaching"
+
+  dimensions = { FunctionName = var.function_name }
+
+  actions_enabled           = false
+  alarm_actions             = []
+  ok_actions                = []
+  insufficient_data_actions = []
+  tags                      = var.tags
+}

internal/datastore/src/__test__/db.ts

@@ -9,6 +9,7 @@ import { DynamoDBDocumentClient } from "@aws-sdk/lib-dynamodb";
 import { DatastoreConfig } from "../config";
 
 export async function setupDynamoDBContainer() {
+  // Pre-pulled in CI by .github/workflows/stage-2-test.yaml; keep in sync.
   const container = await new GenericContainer("amazon/dynamodb-local")
     .withExposedPorts(8000)
     .start();

internal/helpers/src/__tests__/metrics.test.ts

@@ -0,0 +1,151 @@
+import type { MetricsLogger } from "aws-embedded-metrics";
+import { Unit } from "aws-embedded-metrics";
+import {
+  MetricStatus,
+  buildEMFObject,
+  emitForSingleSupplier,
+} from "../metrics";
+
+describe("emitForSingleSupplier", () => {
+  const originalFunctionName = process.env.AWS_LAMBDA_FUNCTION_NAME;
+
+  afterEach(() => {
+    if (originalFunctionName === undefined) {
+      delete process.env.AWS_LAMBDA_FUNCTION_NAME;
+    } else {
+      process.env.AWS_LAMBDA_FUNCTION_NAME = originalFunctionName;
+    }
+  });
+
+  it("sets the function namespace, supplier dimensions, and count metric", () => {
+    delete process.env.AWS_LAMBDA_FUNCTION_NAME;
+    const metrics = {
+      setNamespace: jest.fn(),
+      putDimensions: jest.fn(),
+      putMetric: jest.fn(),
+    } as unknown as MetricsLogger;
+
+    emitForSingleSupplier(
+      metrics,
+      "process-letter",
+      "supplier-1",
+      3,
+      MetricStatus.Success,
+    );
+
+    expect(metrics.setNamespace).toHaveBeenCalledWith("process-letter");
+    expect(metrics.putDimensions).toHaveBeenCalledWith({
+      Supplier: "supplier-1",
+    });
+    expect(metrics.putMetric).toHaveBeenCalledWith(
+      MetricStatus.Success,
+      3,
+      Unit.Count,
+    );
+  });
+
+  it("uses the Lambda function name and preserves additional dimensions", () => {
+    process.env.AWS_LAMBDA_FUNCTION_NAME = "lambda-function";
+    const metrics = {
+      setNamespace: jest.fn(),
+      putDimensions: jest.fn(),
+      putMetric: jest.fn(),
+    } as unknown as MetricsLogger;
+
+    emitForSingleSupplier(
+      metrics,
+      "fallback-function",
+      "supplier-1",
+      2,
+      MetricStatus.Failure,
+      { Environment: "test", Supplier: "overridden-supplier" },
+    );
+
+    expect(metrics.setNamespace).toHaveBeenCalledWith("lambda-function");
+    expect(metrics.putDimensions).toHaveBeenCalledWith({
+      Environment: "test",
+      Supplier: "supplier-1",
+    });
+    expect(metrics.putMetric).toHaveBeenCalledWith(
+      MetricStatus.Failure,
+      2,
+      Unit.Count,
+    );
+  });
+});
+
+describe("buildEMFObject", () => {
+  const originalFunctionName = process.env.AWS_LAMBDA_FUNCTION_NAME;
+
+  afterEach(() => {
+    jest.restoreAllMocks();
+    if (originalFunctionName === undefined) {
+      delete process.env.AWS_LAMBDA_FUNCTION_NAME;
+    } else {
+      process.env.AWS_LAMBDA_FUNCTION_NAME = originalFunctionName;
+    }
+  });
+
+  it("builds an EMF object using the supplied function name", () => {
+    delete process.env.AWS_LAMBDA_FUNCTION_NAME;
+    jest.spyOn(Date, "now").mockReturnValue(123_456_789);
+
+    expect(
+      buildEMFObject(
+        "process-letter",
+        { Supplier: "supplier-1", Environment: "test" },
+        { key: MetricStatus.Success, value: 1, unit: Unit.Count },
+      ),
+    ).toEqual({
+      Supplier: "supplier-1",
+      Environment: "test",
+      _aws: {
+        Timestamp: 123_456_789,
+        CloudWatchMetrics: [
+          {
+            Namespace: "process-letter",
+            Dimensions: [["Supplier", "Environment"]],
+            Metrics: [
+              {
+                Name: MetricStatus.Success,
+                Value: 1,
+                Unit: Unit.Count,
+              },
+            ],
+          },
+        ],
+      },
+      [MetricStatus.Success]: 1,
+    });
+  });
+
+  it("prefers the Lambda function name for the EMF namespace", () => {
+    process.env.AWS_LAMBDA_FUNCTION_NAME = "lambda-function";
+
+    const emfObject = buildEMFObject(
+      "fallback-function",
+      { Supplier: "supplier-1" },
+      { key: "BreachCount", value: 4, unit: Unit.Count },
+    );
+
+    expect(emfObject._aws.CloudWatchMetrics[0].Namespace).toBe(
+      "lambda-function",
+    );
+  });
+
+  it("uses the optional dimension sets if provided", () => {
+    process.env.AWS_LAMBDA_FUNCTION_NAME = "lambda-function";
+
+    const emfObject = buildEMFObject(
+      "process-letter",
+      { Supplier: "supplier-1" },
+      { key: "BreachCount", value: 4, unit: Unit.Count },
+      [[], ["Supplier"]],
+    );
+
+    expect(emfObject._aws.CloudWatchMetrics[0].Dimensions).toEqual([
+      [],
+      ["Supplier"],
+    ]);
+  });
+});