diff --git a/database-monitoring/terraform/.module.toml b/database-monitoring/terraform/.module.toml index 2c37ec7..0b32659 100644 --- a/database-monitoring/terraform/.module.toml +++ b/database-monitoring/terraform/.module.toml @@ -1,6 +1,6 @@ [module] name = "aws-elastio-database-monitoring-agent" -version = "0.1.0" +version = "0.2.0" description = "Terraform module that runs the Elastio database monitoring agent on AWS Fargate" type = "terraform" diff --git a/database-monitoring/terraform/README.md b/database-monitoring/terraform/README.md index 2b36884..4aa5ab1 100644 --- a/database-monitoring/terraform/README.md +++ b/database-monitoring/terraform/README.md @@ -62,12 +62,22 @@ A security group with the usual allow-all egress already allows all three. The d - an access point that maps every client to uid/gid 65532 and roots it at `/elastio-dbmon`, created with mode 0700; - a task-role policy that allows `elasticfilesystem:ClientMount` and `ClientWrite` on the file system, only through that access point. - a file system policy that refuses anonymous clients and denies every mount that does not use that access point or TLS. Other roles in the account with their own EFS permissions can still mount through the access point if the network admits them, so keep `security_group_ids` for the agent only. -- A Fargate task definition of `task_cpu` and `task_memory` (by default 0.25 vCPU and 0.5 GB), ARM64, with a read-only root filesystem. The task runs as the image's non-root user, with `GOMEMLIMIT` set to 80% of `task_memory`. +- A Fargate task definition of `task_cpu` and `task_memory` (by default 0.25 vCPU and 0.5 GB), ARM64, with a read-only root filesystem. The task runs as the image's non-root user, with `GOMEMLIMIT` set to 80% of the agent's memory: `task_memory`, less 384 MiB with `content_analysis = true`. +- With `content_analysis = true`, a second container in the same task, running Elastio's encryption detector (`ed_image`) as the agent's non-root user. Its gRPC port is on loopback only. Its Prometheus metrics port, TCP 50052, listens on all interfaces of the task, so `security_group_ids` must not allow inbound traffic on it. The agent needs no inbound rules at all. It reads a task-local scratch volume at `/var/lib/elastio-dbmon-ed`, read-only; see **Content analysis** below. - A service with `desired_count = 1`. Its deployment policy (minimum healthy 0%, maximum 100%) stops the old task before it starts the new one. A replication slot allows only one consumer at a time, and the agent's state allows only one writer. Don't raise the maximum. +## Content analysis + +When rows are rewritten in place and no rewrite limit is declared for the table, the agent can't tell a migration from an attacker encrypting the data: both look the same in the change stream. With `content_analysis = true`, Elastio's encryption detector (ED) runs beside the agent and settles it. If the new values are ciphertext, the window becomes a high-severity finding that names the encrypted columns. + +- **Off by default.** It needs agent 0.1.7 or later and `task_memory` of at least 1024. ED takes 384 MiB of the task and the agent gets the rest, so with `task_memory = 1024` the agent has 640 MiB and a `GOMEMLIMIT` of 512 MiB. +- **What it does with your data.** The agent samples the rewritten values in memory, at most 4 MiB. When a window closes, it writes the sample to task-local storage for ED to read and removes it once ED answers. Values never reach the agent's EFS state, its log, or Elastio. Only the verdict does, as part of the finding. +- **Where ED comes from.** `ed_image` is built for ARM64 and x86-64 from the ED version the agent release was tested with. Each module version defaults to the matching pair. +- **If ED stops.** The agent keeps running. The windows it judges in the meantime get no verdict, which can raise a finding and never withholds one. ECS restarts ED. + ## Sizing -The default size, 0.25 vCPU and 512 MiB, is enough for one database's ordinary traffic. A larger task handles larger single transactions. A transaction larger than the task can handle doesn't stop the agent. +The default size, 0.25 vCPU and 512 MiB, is enough for one database's ordinary traffic. With `content_analysis = true`, start at `task_memory = 1024`; the agent's figures below then apply to its 640 MiB share. A larger task handles larger single transactions. A transaction larger than the task can handle doesn't stop the agent. **The Elastio UI tells you when to size up.** When a transaction was too large for the task, the database shows an "oversized transaction" observation with the smallest task memory that would have handled it. Set `task_memory` to that value or higher, with a `task_cpu` that allows it, and apply. @@ -180,23 +190,25 @@ No modules. ## Inputs -| Name | Description | Type | Default | Required | -| --------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------- | ------------------------------------------------------------------ | :------: | -| [api_key](#input_api_key) | Bearer token the agent presents to the server. Stored in Secrets Manager, never in the task definition. | `string` | n/a | yes | -| [assign_public_ip](#input_assign_public_ip) | Give the task a public IP. Needed only when the subnets have no NAT gateway and the server URL is on the internet. | `bool` | `false` | no | -| [database_url](#input_database_url) | postgres:// URL of the database to watch, for a role with REPLICATION. Stored in Secrets Manager, never in the task definition. | `string` | n/a | yes | -| [image](#input_image) | Agent image. The default is the agent release this module version was tested with. Pin a version; :latest moves. Needs 0.1.5 or later: older agents read only the QUELL\_\* variable names. | `string` | `"public.ecr.aws/elastio/elastio-database-monitoring-agent:0.1.6"` | no | -| [log_retention_days](#input_log_retention_days) | How long CloudWatch keeps the agent's log. | `number` | `7` | no | -| [name](#input_name) | What the product shows for this agent. Also the ECS cluster, service and task family name, sanitised to the characters ECS allows. | `string` | n/a | yes | -| [persistent_ledger](#input_persistent_ledger) | Keep the agent's state on an encrypted EFS file system so it persists across task replacement. When false, the state is on the task's ephemeral storage and every replacement starts it empty. | `bool` | `true` | no | -| [publication](#input_publication) | Name of the publication whose tables are streamed. | `string` | `"elastio_monitor"` | no | -| [security_group_ids](#input_security_group_ids) | Security groups attached to the task. The database's own security group must admit them on its port. | `list(string)` | n/a | yes | -| [server_url](#input_server_url) | Base URL of the server the agent reports to. | `string` | n/a | yes | -| [slot](#input_slot) | Name of the existing pgoutput replication slot. The agent never creates or drops one. | `string` | `"elastio_monitor"` | no | -| [subnet_ids](#input_subnet_ids) | Subnets the task runs in. They must reach the database and the server URL. | `list(string)` | n/a | yes | -| [tags](#input_tags) | Tags applied to every resource the module creates. | `map(string)` | `{}` | no | -| [task_cpu](#input_task_cpu) | CPU units for the Fargate task: 256 (0.25 vCPU), 512, 1024 or 2048. Together with task_memory it must be a size Fargate runs on ARM64; see Sizing in the README. | `number` | `256` | no | -| [task_memory](#input_task_memory) | Memory for the Fargate task, in MiB. With task_cpu 256: 512, 1024 or 2048. With 512: 1024 to 4096. With 1024: 2048 to 8192. With 2048: 4096 to 16384. Above 512, in steps of 1024. The agent's Go memory limit follows it. Raise it when the Elastio UI recommends a larger size. | `number` | `512` | no | +| Name | Description | Type | Default | Required | +| --------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------- | ------------------------------------------------------------------ | :------: | +| [api_key](#input_api_key) | Bearer token the agent presents to the server. Stored in Secrets Manager, never in the task definition. | `string` | n/a | yes | +| [assign_public_ip](#input_assign_public_ip) | Give the task a public IP. Needed only when the subnets have no NAT gateway and the server URL is on the internet. | `bool` | `false` | no | +| [content_analysis](#input_content_analysis) | Run Elastio's encryption detector beside the agent, so a table whose rows are overwritten with ciphertext is reported as a high-severity finding even where no rewrite limit is declared. The agent samples the rewritten values in memory and on task-local storage only, for the detector to read; only the verdict leaves the task. Needs agent 0.1.7 or later and task_memory of at least 1024. | `bool` | `false` | no | +| [database_url](#input_database_url) | postgres:// URL of the database to watch, for a role with REPLICATION. Stored in Secrets Manager, never in the task definition. | `string` | n/a | yes | +| [ed_image](#input_ed_image) | Encryption detector image, used when content_analysis is true: Elastio's ED gRPC server, built for amd64 and arm64 from the elastio/ed commit the agent release pins. The default is the one released with the default agent image. | `string` | `"public.ecr.aws/elastio/elastio-database-monitoring-ed:0.1.7"` | no | +| [image](#input_image) | Agent image. The default is the agent release this module version was tested with. Pin a version; :latest moves. Needs 0.1.5 or later: older agents read only the QUELL\_\* variable names. | `string` | `"public.ecr.aws/elastio/elastio-database-monitoring-agent:0.1.7"` | no | +| [log_retention_days](#input_log_retention_days) | How long CloudWatch keeps the agent's log. | `number` | `7` | no | +| [name](#input_name) | What the product shows for this agent. Also the ECS cluster, service and task family name, sanitised to the characters ECS allows. | `string` | n/a | yes | +| [persistent_ledger](#input_persistent_ledger) | Keep the agent's state on an encrypted EFS file system so it persists across task replacement. When false, the state is on the task's ephemeral storage and every replacement starts it empty. | `bool` | `true` | no | +| [publication](#input_publication) | Name of the publication whose tables are streamed. | `string` | `"elastio_monitor"` | no | +| [security_group_ids](#input_security_group_ids) | Security groups attached to the task. The database's own security group must admit them on its port. | `list(string)` | n/a | yes | +| [server_url](#input_server_url) | Base URL of the server the agent reports to. | `string` | n/a | yes | +| [slot](#input_slot) | Name of the existing pgoutput replication slot. The agent never creates or drops one. | `string` | `"elastio_monitor"` | no | +| [subnet_ids](#input_subnet_ids) | Subnets the task runs in. They must reach the database and the server URL. | `list(string)` | n/a | yes | +| [tags](#input_tags) | Tags applied to every resource the module creates. | `map(string)` | `{}` | no | +| [task_cpu](#input_task_cpu) | CPU units for the Fargate task: 256 (0.25 vCPU), 512, 1024 or 2048. Together with task_memory it must be a size Fargate runs on ARM64; see Sizing in the README. | `number` | `256` | no | +| [task_memory](#input_task_memory) | Memory for the Fargate task, in MiB. With task_cpu 256: 512, 1024 or 2048. With 512: 1024 to 4096. With 1024: 2048 to 8192. With 2048: 4096 to 16384. Above 512, in steps of 1024. The agent's Go memory limit follows it. Raise it when the Elastio UI recommends a larger size. | `number` | `512` | no | ## Outputs diff --git a/database-monitoring/terraform/main.tf b/database-monitoring/terraform/main.tf index cabd3fc..2bdf3dc 100644 --- a/database-monitoring/terraform/main.tf +++ b/database-monitoring/terraform/main.tf @@ -35,14 +35,140 @@ locals { task_cpu = var.task_cpu task_memory = var.task_memory - # The Go runtime's soft memory limit, 80% of the task. Go does not derive one from the + # Content analysis: Elastio's encryption detector (ED) as a second + # container. It takes ed_memory of the task, as a hard limit, and the agent + # the rest -- so the agent's Go memory limit below is 80% of the agent's + # share, not of the task. Measured on the arm64 image with one worker: + # 158 MiB idle, 231 MiB after judging a window. + content_analysis = var.content_analysis + ed_memory = 384 + ed_port = 50051 + ed_scratch_dir = "/var/lib/elastio-dbmon-ed" + agent_memory = local.task_memory - (local.content_analysis ? local.ed_memory : 0) + + # The Go runtime's soft memory limit, 80% of the agent's memory. Go does not derive one from the # container on its own, and without it a heap whose live size is half the # task is allowed to double before it is collected. The agent derives the # same number itself when this is unset (from 0.1.6 through the ECS task # metadata, because a Fargate task's limit is on the task and the # container's cgroup reads "max"); it is set here so the number in force # is visible in the task definition. - gomemlimit = "${floor(local.task_memory * 0.8)}MiB" + gomemlimit = "${floor(local.agent_memory * 0.8)}MiB" + + # The agent's container, and ED's beside it when content analysis is on. + agent_container = merge(local.agent_container_base, + # With content analysis on, the agent's container is limited to its + # share, so the task's memory is not what it reads as its own. Off, the + # key is absent and the task definition is what it always was. + local.content_analysis ? { memory = local.agent_memory } : {}, + ) + + agent_container_base = { + name = "${local.prefix}-agent" + image = var.image + essential = true + user = "${local.uid}:${local.gid}" + + readonlyRootFilesystem = true + + mountPoints = concat( + [ + { + sourceVolume = "ledger" + containerPath = local.ledger_dir + readOnly = false + }, + ], + local.content_analysis ? [ + { + sourceVolume = "ed-scratch" + containerPath = local.ed_scratch_dir + readOnly = false + }, + ] : [], + ) + + # The ELASTIO_DBMON_* names are the agent binary's configuration + # contract from 0.1.5. The agent still reads the QUELL_* names older + # deployments set, but an image older than 0.1.5 reads only those, so + # this module needs agent 0.1.5 or later. + environment = concat( + [ + { name = "ELASTIO_DBMON_SERVER_URL", value = var.server_url }, + { name = "ELASTIO_DBMON_SLOT", value = var.slot }, + { name = "ELASTIO_DBMON_PUBLICATION", value = var.publication }, + { name = "ELASTIO_DBMON_AGENT_NAME", value = var.name }, + { name = "ELASTIO_DBMON_LEDGER_PATH", value = "${local.ledger_dir}/ledger" }, + { name = "GOMEMLIMIT", value = local.gomemlimit }, + ], + local.content_analysis ? [ + { name = "ELASTIO_DBMON_ED_ADDR", value = "127.0.0.1:${local.ed_port}" }, + { name = "ELASTIO_DBMON_ED_SCRATCH_DIR", value = local.ed_scratch_dir }, + ] : [], + ) + + secrets = [ + { name = "ELASTIO_DBMON_API_KEY", valueFrom = aws_secretsmanager_secret.api_key.arn }, + { name = "ELASTIO_DBMON_DATABASE_URL", valueFrom = aws_secretsmanager_secret.database_url.arn }, + { name = "ELASTIO_DBMON_HASH_SECRET", valueFrom = aws_secretsmanager_secret.hash_secret.arn }, + ] + + logConfiguration = { + logDriver = "awslogs" + options = { + "awslogs-group" = aws_cloudwatch_log_group.this.name + "awslogs-region" = local.region + "awslogs-stream-prefix" = "agent" + } + } + } + + # ED reads each sample by its path, so it mounts the scratch volume at the + # same path, read-only. It runs as the agent's user: the agent writes + # samples readable by that user alone. Not essential: an ED that stops + # costs a window its verdict and nothing else -- the agent judges without + # one and says so -- and ECS restarts it. + # + # The gRPC port is bound to loopback, so only the agent reaches it. The + # Prometheus metrics port is NOT: ED's Python server starts it with + # start_http_server(port) and no address, so it listens on every interface + # of the task ENI whatever --host says (verified on the image, ED 624e66b; + # still so on ED master). It serves counters and timings, never values, + # and the task's security groups are what keep it in: they must not admit + # inbound TCP 50052. The agent needs no inbound rule at all. + ed_container = { + name = "${local.prefix}-ed" + image = var.ed_image + essential = false + user = "${local.uid}:${local.gid}" + memory = local.ed_memory + + command = [ + "--host", "127.0.0.1", + "--port", tostring(local.ed_port), + "--metrics-port", tostring(local.ed_port + 1), + "--workers", "1", + ] + + restartPolicy = { enabled = true } + + mountPoints = [ + { + sourceVolume = "ed-scratch" + containerPath = local.ed_scratch_dir + readOnly = true + }, + ] + + logConfiguration = { + logDriver = "awslogs" + options = { + "awslogs-group" = aws_cloudwatch_log_group.this.name + "awslogs-region" = local.region + "awslogs-stream-prefix" = "ed" + } + } + } } resource "aws_ecs_cluster" "this" { @@ -376,6 +502,19 @@ resource "aws_ecs_task_definition" "this" { size_in_gib = 21 # the Fargate minimum } + # A window's sample, written by the agent for ED to read and removed once + # it answers. Task-local storage, never the ledger's EFS: the values must + # not outlive the task, and the agent refuses a scratch directory inside + # the ledger's. Writable by the agent because its image owns a VOLUME at + # that path, as for the ledger below. + dynamic "volume" { + for_each = local.content_analysis ? [1] : [] + + content { + name = "ed-scratch" + } + } + volume { name = "ledger" @@ -394,52 +533,9 @@ resource "aws_ecs_task_definition" "this" { } } - container_definitions = jsonencode([ - { - name = "${local.prefix}-agent" - image = var.image - essential = true - user = "${local.uid}:${local.gid}" - - readonlyRootFilesystem = true - - mountPoints = [ - { - sourceVolume = "ledger" - containerPath = local.ledger_dir - readOnly = false - }, - ] - - # The ELASTIO_DBMON_* names are the agent binary's configuration - # contract from 0.1.5. The agent still reads the QUELL_* names older - # deployments set, but an image older than 0.1.5 reads only those, so - # this module needs agent 0.1.5 or later. - environment = [ - { name = "ELASTIO_DBMON_SERVER_URL", value = var.server_url }, - { name = "ELASTIO_DBMON_SLOT", value = var.slot }, - { name = "ELASTIO_DBMON_PUBLICATION", value = var.publication }, - { name = "ELASTIO_DBMON_AGENT_NAME", value = var.name }, - { name = "ELASTIO_DBMON_LEDGER_PATH", value = "${local.ledger_dir}/ledger" }, - { name = "GOMEMLIMIT", value = local.gomemlimit }, - ] - - secrets = [ - { name = "ELASTIO_DBMON_API_KEY", valueFrom = aws_secretsmanager_secret.api_key.arn }, - { name = "ELASTIO_DBMON_DATABASE_URL", valueFrom = aws_secretsmanager_secret.database_url.arn }, - { name = "ELASTIO_DBMON_HASH_SECRET", valueFrom = aws_secretsmanager_secret.hash_secret.arn }, - ] - - logConfiguration = { - logDriver = "awslogs" - options = { - "awslogs-group" = aws_cloudwatch_log_group.this.name - "awslogs-region" = local.region - "awslogs-stream-prefix" = "agent" - } - } - }, - ]) + container_definitions = jsonencode( + concat([local.agent_container], local.content_analysis ? [local.ed_container] : []) + ) tags = var.tags } diff --git a/database-monitoring/terraform/tests/module.tftest.hcl b/database-monitoring/terraform/tests/module.tftest.hcl index 487b608..49936e1 100644 --- a/database-monitoring/terraform/tests/module.tftest.hcl +++ b/database-monitoring/terraform/tests/module.tftest.hcl @@ -135,8 +135,8 @@ run "names_and_image" { error_message = "secrets must be named elastio-dbmon//..." } assert { - condition = strcontains(aws_ecs_task_definition.this.container_definitions, "\"image\":\"public.ecr.aws/elastio/elastio-database-monitoring-agent:0.1.6\"") - error_message = "the default image must be Elastio's public ECR image of agent 0.1.6, the first to report oversized transactions and read its limit on Fargate" + condition = strcontains(aws_ecs_task_definition.this.container_definitions, "\"image\":\"public.ecr.aws/elastio/elastio-database-monitoring-agent:0.1.7\"") + error_message = "the default image must be Elastio's public ECR image of agent 0.1.7, the first that can ask the encryption detector" } } @@ -270,3 +270,95 @@ run "ephemeral" { error_message = "ledger_file_system_id must be null without EFS" } } + +# Content analysis on: the encryption detector runs beside the agent, the two +# share a task-local scratch volume at the same path, and the agent's memory +# -- and so its Go memory limit -- is its share of the task, not the task. +run "content_analysis_runs_the_detector_beside_the_agent" { + command = plan + + variables { + content_analysis = true + task_cpu = 512 + task_memory = 1024 + } + + assert { + condition = length(jsondecode(aws_ecs_task_definition.this.container_definitions)) == 2 + error_message = "content analysis must add the detector as a second container" + } + assert { + condition = alltrue([ + for c in jsondecode(aws_ecs_task_definition.this.container_definitions) : + c.user == "65532:65532" + ]) + error_message = "the detector must run as the agent's user, which alone can read the samples" + } + assert { + condition = one([ + for c in jsondecode(aws_ecs_task_definition.this.container_definitions) : c + if c.name == "elastio-dbmon-ed" + ]).essential == false + error_message = "a stopped detector must cost a window its verdict, not stop the agent" + } + assert { + condition = one([ + for c in jsondecode(aws_ecs_task_definition.this.container_definitions) : c + if c.name == "elastio-dbmon-ed" + ]).mountPoints[0].readOnly == true + error_message = "the detector reads the scratch volume and never writes it" + } + assert { + condition = one([ + for c in jsondecode(aws_ecs_task_definition.this.container_definitions) : c + if c.name == "elastio-dbmon-agent" + ]).memory == 640 + error_message = "the agent's container must be limited to the task less the detector's 384 MiB" + } + assert { + condition = strcontains(aws_ecs_task_definition.this.container_definitions, "{\"name\":\"GOMEMLIMIT\",\"value\":\"512MiB\"}") + error_message = "GOMEMLIMIT must be 80% of the agent's share (640 MiB), not of the task" + } + assert { + condition = strcontains(aws_ecs_task_definition.this.container_definitions, "{\"name\":\"ELASTIO_DBMON_ED_ADDR\",\"value\":\"127.0.0.1:50051\"}") + error_message = "the agent must be told where the detector listens" + } + assert { + condition = length([for v in aws_ecs_task_definition.this.volume : v if v.name == "ed-scratch" && length(v.efs_volume_configuration) == 0]) == 1 + error_message = "the scratch volume must be task-local, never EFS" + } + assert { + condition = strcontains(aws_ecs_task_definition.this.container_definitions, "elastio-database-monitoring-ed:0.1.7") + error_message = "the detector defaults to the image released with the default agent" + } +} + +# Off, the default: exactly the task this module always made. +run "content_analysis_is_off_by_default" { + command = plan + + assert { + condition = length(jsondecode(aws_ecs_task_definition.this.container_definitions)) == 1 + error_message = "content analysis must be opted into" + } + assert { + condition = !strcontains(aws_ecs_task_definition.this.container_definitions, "ELASTIO_DBMON_ED_") + error_message = "with content analysis off the agent must not be pointed at a detector" + } + assert { + condition = length(aws_ecs_task_definition.this.volume) == 1 + error_message = "with content analysis off there is no scratch volume" + } +} + +# Too small a task for both is refused, not planned. +run "content_analysis_needs_room_for_the_detector" { + command = plan + + variables { + content_analysis = true + task_memory = 512 + } + + expect_failures = [var.content_analysis] +} diff --git a/database-monitoring/terraform/variables.tf b/database-monitoring/terraform/variables.tf index 8723f3a..0ef45c7 100644 --- a/database-monitoring/terraform/variables.tf +++ b/database-monitoring/terraform/variables.tf @@ -58,7 +58,26 @@ variable "security_group_ids" { variable "image" { description = "Agent image. The default is the agent release this module version was tested with. Pin a version; :latest moves. Needs 0.1.5 or later: older agents read only the QUELL_* variable names." type = string - default = "public.ecr.aws/elastio/elastio-database-monitoring-agent:0.1.6" + default = "public.ecr.aws/elastio/elastio-database-monitoring-agent:0.1.7" +} + +variable "content_analysis" { + description = "Run Elastio's encryption detector beside the agent, so a table whose rows are overwritten with ciphertext is reported as a high-severity finding even where no rewrite limit is declared. The agent samples the rewritten values in memory and on task-local storage only, for the detector to read; only the verdict leaves the task. Needs agent 0.1.7 or later and task_memory of at least 1024." + type = bool + default = false + + # The detector takes 384 MiB of the task. Below 1024 the agent's share is + # smaller than the smallest task it is sized for on its own. + validation { + condition = !var.content_analysis || var.task_memory >= 1024 + error_message = "content_analysis needs task_memory of at least 1024: the encryption detector takes 384 MiB of the task, and the agent needs the rest." + } +} + +variable "ed_image" { + description = "Encryption detector image, used when content_analysis is true: Elastio's ED gRPC server, built for amd64 and arm64 from the elastio/ed commit the agent release pins. The default is the one released with the default agent image." + type = string + default = "public.ecr.aws/elastio/elastio-database-monitoring-ed:0.1.7" } variable "task_cpu" {