From 35cbf5889fdc58c1fcba70f96dea75e3edeb3ae5 Mon Sep 17 00:00:00 2001 From: Jan Marco Schreier Date: Wed, 7 Oct 2026 15:52:41 +0200 Subject: [PATCH 01/38] example(iaas): scaffold the restricted egress example --- .../.terraform.lock.hcl | 25 ++++++++++++ .../010-provider.tf | 34 ++++++++++++++++ .../020-variables.tf | 40 +++++++++++++++++++ 3 files changed, 99 insertions(+) create mode 100644 examples/iaas-server-agent-restricted-egress/.terraform.lock.hcl create mode 100644 examples/iaas-server-agent-restricted-egress/010-provider.tf create mode 100644 examples/iaas-server-agent-restricted-egress/020-variables.tf diff --git a/examples/iaas-server-agent-restricted-egress/.terraform.lock.hcl b/examples/iaas-server-agent-restricted-egress/.terraform.lock.hcl new file mode 100644 index 0000000..b7639a6 --- /dev/null +++ b/examples/iaas-server-agent-restricted-egress/.terraform.lock.hcl @@ -0,0 +1,25 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/stackitcloud/stackit" { + version = "0.118.0" + constraints = ">= 0.118.0" + hashes = [ + "h1:S7Hibl5mtlfQKuqOSx8EUtEdEz8te8hb8BtmrEhuVp8=", + "h1:c0nVaHC+IEWQIaCvCdgn/aw+uBaWfzZAlycdK00oryM=", + "zh:0dde99e7b343fa01f8eefc378171fb8621bedb20f59157d6cc8e3d46c738105f", + "zh:1638670ce7afc90caec430293983bab382c65ba2f5a48f5e2f63d1ef607bc34b", + "zh:36a030d183ca2c14089ee9f6054a22ff2f8f38bed6905b09c5d77c63180129bb", + "zh:59dd0d53cd2ed975fbb87da23856ed43b61cc9db7e19d4dde2e4c95ab7a2859c", + "zh:69748f230b8b975fe42a843c8ff69c88688552cd6887eef7bbd02c5f699bad26", + "zh:6cd23fb5c1808088f778ce0711ab2f72f509c84d87c089fa150d60f0d9a73441", + "zh:6da2707e3a62a29efbfdddf96f3c3c54b882c6ee3a41f86ff8d50b07f3484a4e", + "zh:80891eab98b4d6e469e3469320cc91b1b3b3037ada4e532e0de98b766657d8be", + "zh:930a1b94a94ca26ff37ae9f09d13c61bfe50de6d01dd77f87177d7d2ddb0ae6f", + "zh:c046027ae6a69386c4c3a64c1dbf5e50a3620437d799b8d5f4681c027887322b", + "zh:d8c4973d1189ed9aaa3155637226f552da768c59f447ac578c9b56a92177c50d", + "zh:f20c73a8b9745437c2ac3a4f7feed8f2f855566db0edd4a6e4485fe5868c6929", + "zh:f289b696d8e040b6fab0365a04b1264aa7fec70de46fc70ac521e6c8f2e289f6", + "zh:f8ec5af29ccf825a1c02a05fa7af71b271798cae7226a29c4e09c1a4dd849606", + ] +} diff --git a/examples/iaas-server-agent-restricted-egress/010-provider.tf b/examples/iaas-server-agent-restricted-egress/010-provider.tf new file mode 100644 index 0000000..a302c55 --- /dev/null +++ b/examples/iaas-server-agent-restricted-egress/010-provider.tf @@ -0,0 +1,34 @@ +# Copyright 2026 Schwarz Digits Cloud GmbH & Co. KG +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + # Ephemeral resources, used for the access token in + # 070-default-egress-rules.tf, exist since Terraform 1.10. + required_version = ">= 1.10.0" + + required_providers { + stackit = { + source = "stackitcloud/stackit" + version = ">= 0.118.0" + } + } +} + +provider "stackit" { + default_region = var.stackit_region + service_account_key_path = var.stackit_service_account_key_path + # Required by the ephemeral resource stackit_access_token in + # 070-default-egress-rules.tf. + enable_beta_resources = true +} diff --git a/examples/iaas-server-agent-restricted-egress/020-variables.tf b/examples/iaas-server-agent-restricted-egress/020-variables.tf new file mode 100644 index 0000000..9aebcb2 --- /dev/null +++ b/examples/iaas-server-agent-restricted-egress/020-variables.tf @@ -0,0 +1,40 @@ +# Copyright 2026 Schwarz Digits Cloud GmbH & Co. KG +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "stackit_org_id" { + description = "The STACKIT organization ID in which the network area is created." + type = string +} + +variable "stackit_parent_container_id" { + description = "Existing folder ID the project is created under, or the organization ID to create it directly under the organization. This example does not create the folder." + type = string +} + +variable "stackit_admin_email" { + description = "An existing STACKIT user that becomes owner of the created project. A non-existent address fails with 409 \"subject is not found\"." + type = string +} + +variable "stackit_region" { + description = "The STACKIT region." + type = string + default = "eu01" +} + +variable "stackit_service_account_key_path" { + description = "Path to the service account key file. Unset falls back to the STACKIT_SERVICE_ACCOUNT_KEY_PATH environment variable, then to that entry in the credentials file ($HOME/.stackit/credentials.json, or STACKIT_CREDENTIALS_PATH)." + type = string + default = null +} From 968a8721896955d2ea16771a3c9b50277181e90e Mon Sep 17 00:00:00 2001 From: Jan Marco Schreier Date: Wed, 7 Oct 2026 16:19:08 +0200 Subject: [PATCH 02/38] example(iaas): add network area, project and network --- .../020-variables.tf | 51 +++++++++++++++++++ .../030-network-area.tf | 31 +++++++++++ .../040-project.tf | 34 +++++++++++++ .../050-network.tf | 24 +++++++++ 4 files changed, 140 insertions(+) create mode 100644 examples/iaas-server-agent-restricted-egress/030-network-area.tf create mode 100644 examples/iaas-server-agent-restricted-egress/040-project.tf create mode 100644 examples/iaas-server-agent-restricted-egress/050-network.tf diff --git a/examples/iaas-server-agent-restricted-egress/020-variables.tf b/examples/iaas-server-agent-restricted-egress/020-variables.tf index 9aebcb2..0fd51c6 100644 --- a/examples/iaas-server-agent-restricted-egress/020-variables.tf +++ b/examples/iaas-server-agent-restricted-egress/020-variables.tf @@ -38,3 +38,54 @@ variable "stackit_service_account_key_path" { type = string default = null } + +# --- naming -------------------------------------------------------------- + +variable "project_name" { + description = "Display name of the project this example creates." + type = string + default = "server-agent-egress" +} + +variable "network_area_name" { + description = "Name of the STACKIT Network Area this example creates." + type = string + default = "server-agent-egress" +} + +variable "name_prefix" { + description = "Prefix for the names of the network, the security groups, the network interfaces, the key pair and the two machines." + type = string + default = "agent-egress" +} + +# --- addressing ---------------------------------------------------------- + +variable "network_area_range" { + description = "IPv4 range of the network area." + type = string + default = "10.80.0.0/16" +} + +variable "network_area_transfer_network" { + description = "Transfer network of the network area. It must not overlap with network_area_range." + type = string + default = "172.30.0.0/16" +} + +variable "network_prefix" { + description = "Prefix of the network that holds both machines. It must sit inside network_area_range." + type = string + default = "10.80.1.0/24" +} + +variable "nameservers" { + description = "IPv4 nameservers of the network. The egress rules with ip_range = \"nameservers\" allow DNS to exactly these addresses." + type = list(string) + default = ["1.1.1.1", "9.9.9.9"] + + validation { + condition = length(var.nameservers) > 0 && alltrue([for ns in var.nameservers : !strcontains(ns, ":") && can(cidrhost("${ns}/32", 0))]) + error_message = "nameservers needs at least one entry, and every entry must be an IPv4 address without a prefix length." + } +} diff --git a/examples/iaas-server-agent-restricted-egress/030-network-area.tf b/examples/iaas-server-agent-restricted-egress/030-network-area.tf new file mode 100644 index 0000000..61e11e1 --- /dev/null +++ b/examples/iaas-server-agent-restricted-egress/030-network-area.tf @@ -0,0 +1,31 @@ +# Copyright 2026 Schwarz Digits Cloud GmbH & Co. KG +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +resource "stackit_network_area" "this" { + organization_id = var.stackit_org_id + name = var.network_area_name +} + +# The regional configuration holds the address ranges of the network area. +# Networks and projects of the area need it, so both depend on it. +resource "stackit_network_area_region" "this" { + organization_id = var.stackit_org_id + network_area_id = stackit_network_area.this.network_area_id + + ipv4 = { + transfer_network = var.network_area_transfer_network + network_ranges = [{ prefix = var.network_area_range }] + default_nameservers = var.nameservers + } +} diff --git a/examples/iaas-server-agent-restricted-egress/040-project.tf b/examples/iaas-server-agent-restricted-egress/040-project.tf new file mode 100644 index 0000000..a21d2b3 --- /dev/null +++ b/examples/iaas-server-agent-restricted-egress/040-project.tf @@ -0,0 +1,34 @@ +# Copyright 2026 Schwarz Digits Cloud GmbH & Co. KG +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +resource "stackit_resourcemanager_project" "this" { + parent_container_id = var.stackit_parent_container_id + name = var.project_name + owner_email = var.stackit_admin_email + + # The label attaches the project to the network area. It cannot be changed + # after the project exists. + labels = { + networkArea = stackit_network_area.this.network_area_id + } + + # Terraform destroys in reverse order. Without this dependency it deletes the + # regional configuration and the project in parallel, and the API rejects the + # deletion of a regional configuration that still has active projects. + depends_on = [stackit_network_area_region.this] +} + +locals { + project_id = stackit_resourcemanager_project.this.project_id +} diff --git a/examples/iaas-server-agent-restricted-egress/050-network.tf b/examples/iaas-server-agent-restricted-egress/050-network.tf new file mode 100644 index 0000000..227048a --- /dev/null +++ b/examples/iaas-server-agent-restricted-egress/050-network.tf @@ -0,0 +1,24 @@ +# Copyright 2026 Schwarz Digits Cloud GmbH & Co. KG +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# One network holds the jump host and the target. The two machines are +# separated by their security groups, not by the network. +resource "stackit_network" "this" { + project_id = local.project_id + name = var.name_prefix + ipv4_prefix = var.network_prefix + ipv4_nameservers = var.nameservers + + depends_on = [stackit_network_area_region.this] +} From 8e1c02677c96b12ce730f32f5aef442e05145bc3 Mon Sep 17 00:00:00 2001 From: Jan Marco Schreier Date: Wed, 7 Oct 2026 16:47:33 +0200 Subject: [PATCH 03/38] example(iaas): add security groups and the egress allow-list --- .../020-variables.tf | 79 +++++++++++++ .../060-security-groups.tf | 106 ++++++++++++++++++ 2 files changed, 185 insertions(+) create mode 100644 examples/iaas-server-agent-restricted-egress/060-security-groups.tf diff --git a/examples/iaas-server-agent-restricted-egress/020-variables.tf b/examples/iaas-server-agent-restricted-egress/020-variables.tf index 0fd51c6..c29ce60 100644 --- a/examples/iaas-server-agent-restricted-egress/020-variables.tf +++ b/examples/iaas-server-agent-restricted-egress/020-variables.tf @@ -89,3 +89,82 @@ variable "nameservers" { error_message = "nameservers needs at least one entry, and every entry must be an IPv4 address without a prefix length." } } + +# --- access -------------------------------------------------------------- + +variable "admin_cidr" { + description = "IPv4 source range that reaches the jump host on TCP 2222, for example your public IP with a /32 suffix." + type = string + + validation { + condition = !strcontains(var.admin_cidr, ":") && can(cidrhost(var.admin_cidr, 0)) + error_message = "admin_cidr must be an IPv4 range in CIDR notation, for example 198.51.100.10/32." + } +} + +# --- outbound traffic of the target -------------------------------------- + +variable "egress_rules" { + description = "Outbound allow-list of the target's security group. Each entry becomes one IPv4 egress rule. ip_range takes a CIDR or one of two names: \"nameservers\" stands for every address in var.nameservers, \"stackit\" for every IPv4 range in data.stackit_public_ip_ranges. Leave port_min and port_max unset for a protocol without ports, such as icmp. The default holds the destinations the STACKIT documentation names." + type = map(object({ + description = string + protocol = string + port_min = optional(number) + port_max = optional(number) + ip_range = string + })) + default = { + metadata = { + description = "Metadata service: cloud-init reads user data, metadata and vendor data; download of the Server Agent package" + protocol = "tcp" + port_min = 80 + port_max = 80 + ip_range = "169.254.169.254/32" + } + dns-udp = { + description = "DNS: resolves the host names of the HTTPS destinations" + protocol = "udp" + port_min = 53 + port_max = 53 + ip_range = "nameservers" + } + dns-tcp = { + description = "DNS over TCP: answers that do not fit into a UDP datagram" + protocol = "tcp" + port_min = 53 + port_max = 53 + ip_range = "nameservers" + } + https-stackit = { + description = "HTTPS to the STACKIT public IP ranges: signature check of the Server Agent package on Object Storage" + protocol = "tcp" + port_min = 443 + port_max = 443 + ip_range = "stackit" + } + } + + validation { + condition = alltrue([ + for rule in values(var.egress_rules) : + contains(["nameservers", "stackit"], rule.ip_range) || (!strcontains(rule.ip_range, ":") && can(cidrhost(rule.ip_range, 0))) + ]) + error_message = "ip_range of every egress rule must be an IPv4 range in CIDR notation, \"nameservers\" or \"stackit\"." + } + + validation { + condition = alltrue([ + for rule in values(var.egress_rules) : + length(rule.description) >= 1 && length(rule.description) <= 127 + ]) + error_message = "description of every egress rule must have 1 to 127 characters. The STACKIT provider rejects longer rule descriptions." + } + + validation { + condition = alltrue([ + for rule in values(var.egress_rules) : + (rule.port_min == null) == (rule.port_max == null) + ]) + error_message = "Every egress rule sets both port_min and port_max, or neither." + } +} diff --git a/examples/iaas-server-agent-restricted-egress/060-security-groups.tf b/examples/iaas-server-agent-restricted-egress/060-security-groups.tf new file mode 100644 index 0000000..30b113b --- /dev/null +++ b/examples/iaas-server-agent-restricted-egress/060-security-groups.tf @@ -0,0 +1,106 @@ +# Copyright 2026 Schwarz Digits Cloud GmbH & Co. KG +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +locals { + # sshd on the jump host listens on this port. The security group rule, the + # sshd configuration and the outputs read it from here. + jump_ssh_port = 2222 +} + +# --- jump host ----------------------------------------------------------- +# +# A security group only allows. The group has one ingress rule, so the jump +# host accepts new connections on TCP 2222 from admin_cidr and nothing else. +# The group is stateful: answers to connections the jump host opened itself +# pass without a rule. +resource "stackit_security_group" "jump" { + project_id = local.project_id + name = "${var.name_prefix}-jump" + description = "Jump host: inbound SSH on TCP ${local.jump_ssh_port} from the admin range only" + stateful = true +} + +resource "stackit_security_group_rule" "jump_ssh" { + project_id = local.project_id + security_group_id = stackit_security_group.jump.security_group_id + description = "SSH from the admin range to the port sshd listens on" + direction = "ingress" + ether_type = "IPv4" + protocol = { name = "tcp" } + port_range = { min = local.jump_ssh_port, max = local.jump_ssh_port } + ip_range = var.admin_cidr +} + +# The jump host is not the subject of the egress test. Its group keeps the two +# default egress rules that STACKIT adds to every new security group. + +# --- target -------------------------------------------------------------- +# +# The target has no public IP. The only inbound rule names the jump host's +# security group as the source, so SSH reaches the target from the jump host +# and from nowhere else. +resource "stackit_security_group" "target" { + project_id = local.project_id + name = "${var.name_prefix}-target" + description = "Target: inbound SSH from the jump host's group, outbound traffic from an allow-list" + stateful = true +} + +resource "stackit_security_group_rule" "target_ssh" { + project_id = local.project_id + security_group_id = stackit_security_group.target.security_group_id + description = "SSH from members of the jump host's security group" + direction = "ingress" + ether_type = "IPv4" + protocol = { name = "tcp" } + port_range = { min = 22, max = 22 } + remote_security_group_id = stackit_security_group.jump.security_group_id +} + +# --- outbound allow-list of the target ----------------------------------- + +# The public IPv4 and IPv6 ranges STACKIT uses. The data source needs no +# project, so Terraform reads it while planning. +data "stackit_public_ip_ranges" "this" {} + +locals { + # The destinations an egress rule can name instead of a CIDR. The rules are + # IPv4 rules, so IPv6 ranges are left out. + egress_named_destinations = { + nameservers = tolist([for nameserver in var.nameservers : "${nameserver}/32"]) + stackit = tolist([for cidr in data.stackit_public_ip_ranges.this.cidr_list : cidr if !strcontains(cidr, ":")]) + } + + # A security group rule takes one address range. An entry of egress_rules + # that names a group of destinations becomes one rule per range. + target_egress_rules = merge([ + for name, rule in var.egress_rules : { + for cidr in lookup(local.egress_named_destinations, rule.ip_range, tolist([rule.ip_range])) : + (contains(keys(local.egress_named_destinations), rule.ip_range) ? "${name}-${cidr}" : name) => merge(rule, { ip_range = cidr }) + } + ]...) +} + +resource "stackit_security_group_rule" "target_egress" { + for_each = local.target_egress_rules + + project_id = local.project_id + security_group_id = stackit_security_group.target.security_group_id + description = each.value.description + direction = "egress" + ether_type = "IPv4" + protocol = { name = each.value.protocol } + port_range = each.value.port_min == null ? null : { min = each.value.port_min, max = each.value.port_max } + ip_range = each.value.ip_range +} From 5c2545467c8eb7f73b7f06367b8e86a274f07cb0 Mon Sep 17 00:00:00 2001 From: Jan Marco Schreier Date: Wed, 7 Oct 2026 17:26:15 +0200 Subject: [PATCH 04/38] example(iaas): remove the default egress rules of the target --- .../020-variables.tf | 11 ++ .../060-security-groups.tf | 6 + .../070-default-egress-rules.tf | 59 ++++++++ .../scripts/remove-default-egress-rules.sh | 130 ++++++++++++++++++ 4 files changed, 206 insertions(+) create mode 100644 examples/iaas-server-agent-restricted-egress/070-default-egress-rules.tf create mode 100755 examples/iaas-server-agent-restricted-egress/scripts/remove-default-egress-rules.sh diff --git a/examples/iaas-server-agent-restricted-egress/020-variables.tf b/examples/iaas-server-agent-restricted-egress/020-variables.tf index c29ce60..39da020 100644 --- a/examples/iaas-server-agent-restricted-egress/020-variables.tf +++ b/examples/iaas-server-agent-restricted-egress/020-variables.tf @@ -104,6 +104,17 @@ variable "admin_cidr" { # --- outbound traffic of the target -------------------------------------- +variable "egress_mode" { + description = "\"restricted\" removes the two default egress rules from the target's security group, so only egress_rules allows outbound traffic. \"open\" keeps the default rules, so all outbound traffic passes; use it to record what the machine contacts. A change replaces the target's security group, network interface and server." + type = string + default = "restricted" + + validation { + condition = contains(["restricted", "open"], var.egress_mode) + error_message = "egress_mode must be \"restricted\" or \"open\"." + } +} + variable "egress_rules" { description = "Outbound allow-list of the target's security group. Each entry becomes one IPv4 egress rule. ip_range takes a CIDR or one of two names: \"nameservers\" stands for every address in var.nameservers, \"stackit\" for every IPv4 range in data.stackit_public_ip_ranges. Leave port_min and port_max unset for a protocol without ports, such as icmp. The default holds the destinations the STACKIT documentation names." type = map(object({ diff --git a/examples/iaas-server-agent-restricted-egress/060-security-groups.tf b/examples/iaas-server-agent-restricted-egress/060-security-groups.tf index 30b113b..2f69a97 100644 --- a/examples/iaas-server-agent-restricted-egress/060-security-groups.tf +++ b/examples/iaas-server-agent-restricted-egress/060-security-groups.tf @@ -55,6 +55,12 @@ resource "stackit_security_group" "target" { name = "${var.name_prefix}-target" description = "Target: inbound SSH from the jump host's group, outbound traffic from an allow-list" stateful = true + + # A deleted default egress rule does not come back. A change of egress_mode + # therefore creates a new group, which starts with its two default rules. + lifecycle { + replace_triggered_by = [terraform_data.egress_mode] + } } resource "stackit_security_group_rule" "target_ssh" { diff --git a/examples/iaas-server-agent-restricted-egress/070-default-egress-rules.tf b/examples/iaas-server-agent-restricted-egress/070-default-egress-rules.tf new file mode 100644 index 0000000..64af77d --- /dev/null +++ b/examples/iaas-server-agent-restricted-egress/070-default-egress-rules.tf @@ -0,0 +1,59 @@ +# Copyright 2026 Schwarz Digits Cloud GmbH & Co. KG +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# STACKIT adds two egress rules to every new security group. One allows all +# outbound IPv4 traffic, the other all outbound IPv6 traffic. Rules only +# allow, so the allow-list in 060-security-groups.tf restricts nothing while +# these two rules exist. +# +# The provider cannot remove them: stackit_security_group has no argument for +# its rules, and the data source does not list them. This file removes them +# with two calls against the IaaS API. + +# Holds the mode, so that other resources can react to a change of it. The +# target's security group, network interface and server are replaced when it +# changes: the new group starts with its default rules again, and the new +# server boots for the first time under the rule set of the new mode. +resource "terraform_data" "egress_mode" { + input = var.egress_mode +} + +# The token of the service account the provider is configured with. An +# ephemeral resource is not written to the plan or the state. +ephemeral "stackit_access_token" "iaas" {} + +# Runs once per security group, in mode "restricted" only. The script deletes +# every egress rule that has no protocol, no port range, no IP range and no +# remote security group, and fails unless the group is then free of egress +# rules that allow all traffic. It needs bash, curl and jq on the machine that +# runs Terraform. +# +# Terraform hides the output of the script, because its environment holds the +# token. When the script fails, the error message of Terraform contains the +# output. +resource "terraform_data" "remove_default_egress_rules" { + count = var.egress_mode == "restricted" ? 1 : 0 + + triggers_replace = stackit_security_group.target.security_group_id + + provisioner "local-exec" { + interpreter = ["bash"] + command = "${path.module}/scripts/remove-default-egress-rules.sh" + + environment = { + SECURITY_GROUP_URL = "https://iaas.api.stackit.cloud/v2/projects/${stackit_security_group.target.project_id}/regions/${stackit_security_group.target.region}/security-groups/${stackit_security_group.target.security_group_id}" + STACKIT_ACCESS_TOKEN = ephemeral.stackit_access_token.iaas.access_token + } + } +} diff --git a/examples/iaas-server-agent-restricted-egress/scripts/remove-default-egress-rules.sh b/examples/iaas-server-agent-restricted-egress/scripts/remove-default-egress-rules.sh new file mode 100755 index 0000000..377f26e --- /dev/null +++ b/examples/iaas-server-agent-restricted-egress/scripts/remove-default-egress-rules.sh @@ -0,0 +1,130 @@ +#!/usr/bin/env bash +# Copyright 2026 Schwarz Digits Cloud GmbH & Co. KG +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Removes the default egress rules from one STACKIT security group. +# +# STACKIT adds two rules to every new security group: one allows all outbound +# IPv4 traffic, the other all outbound IPv6 traffic. The rules of a security +# group only allow, so an allow-list has no effect while these two exist. The +# Terraform provider neither creates nor shows them. This script reads the +# group through the IaaS API, deletes the two rules and then checks that no +# egress rule is left that allows all traffic. +# +# A default rule is an egress rule without protocol, port range, IP range and +# remote security group. The script deletes nothing else and never touches an +# ingress rule. A second run finds no default rule and only checks. +# +# Environment: +# SECURITY_GROUP_URL URL of the group in the IaaS API: +# https://iaas.api.stackit.cloud/v2/projects//regions//security-groups/ +# STACKIT_ACCESS_TOKEN access token of an identity that is allowed to edit the group +# VERIFY_ATTEMPTS optional: how often the final check reads the group (default 12) +# VERIFY_DELAY_SECONDS optional: pause between two checks (default 5) +# +# Requires bash, jq and curl 7.55.0 or later. +# Exit 0: no egress rule that allows all traffic is left. Exit 1: anything else. + +set -euo pipefail + +fail() { + echo "error: $*" >&2 + exit 1 +} + +[[ -n "${SECURITY_GROUP_URL:-}" ]] || fail "SECURITY_GROUP_URL is not set" +[[ -n "${STACKIT_ACCESS_TOKEN:-}" ]] || fail "STACKIT_ACCESS_TOKEN is not set" +verify_attempts="${VERIFY_ATTEMPTS:-12}" +verify_delay="${VERIFY_DELAY_SECONDS:-5}" + +for tool in curl jq; do + command -v "$tool" >/dev/null 2>&1 || fail "$tool is required but not installed" +done + +# An egress rule without protocol, port range and remote security group matches +# every packet to its IP range. +readonly unrestricted_egress=' + .rules[] + | select(.direction == "egress") + | select((.protocol // {}) | (.name == null and .number == null)) + | select(.portRange == null and .icmpParameters == null) + | select(.remoteSecurityGroupId == null)' + +# The default rules have no IP range at all. +readonly default_rule_ids="$unrestricted_egress | select(.ipRange == null) | .id" + +# A rule to 0.0.0.0/0 or ::/0 has the same effect, but STACKIT did not create +# it. The script reports it instead of deleting it. +readonly foreign_rule_ids="$unrestricted_egress | select(.ipRange == \"0.0.0.0/0\" or .ipRange == \"::/0\") | .id" + +# api METHOD URL [CURL_OPTION...] +# curl reads the Authorization header from a file descriptor, so the token does +# not show up in the process list. curl repeats a request on a timeout and on +# the status codes 408, 429, 500, 502, 503 and 504. +api() { + local method="$1" url="$2" + shift 2 + curl --silent --show-error --fail \ + --retry 5 --retry-delay 5 --max-time 60 \ + --request "$method" \ + --header @<(printf 'Authorization: Bearer %s\n' "$STACKIT_ACCESS_TOKEN") \ + --header "Accept: application/json" \ + "$@" "$url" +} + +read_group() { + local group + group=$(api GET "$SECURITY_GROUP_URL") || fail "cannot read the security group at $SECURITY_GROUP_URL" + jq -e '(.rules | type) == "array"' >/dev/null 2>&1 <<<"$group" || + fail "the answer of the IaaS API contains no list of rules" + printf '%s' "$group" +} + +group=$(read_group) +to_delete=$(jq -r "$default_rule_ids" <<<"$group") + +deleted=0 +while IFS= read -r rule_id; do + [[ -n "$rule_id" ]] || continue + # The API answers 404 when the rule is already gone, for example after curl + # repeated a request whose first answer was lost. + status=$(api DELETE "$SECURITY_GROUP_URL/rules/$rule_id" --output /dev/null --write-out '%{http_code}') || + [[ "$status" == "404" ]] || + fail "cannot delete the default egress rule $rule_id (HTTP status $status)" + deleted=$((deleted + 1)) +done <<<"$to_delete" + +# The API accepts a deletion before the rule is gone, so the check reads the +# group until no default rule is listed any more. +attempt=1 +while true; do + group=$(read_group) + + foreign=$(jq -r "[$foreign_rule_ids] | join(\" \")" <<<"$group") + [[ -z "$foreign" ]] || + fail "the group has an egress rule that allows all traffic and is not a default rule: $foreign. Delete it, then run again." + + pending=$(jq -r "[$default_rule_ids] | join(\" \")" <<<"$group") + if [[ -z "$pending" ]]; then + remaining=$(jq -r '[.rules[] | select(.direction == "egress")] | length' <<<"$group") + echo "deleted $deleted default egress rule(s); no egress rule that allows all traffic is left, $remaining egress rule(s) remain" + exit 0 + fi + + if ((attempt >= verify_attempts)); then + fail "default egress rules are still listed after $verify_attempts checks: $pending" + fi + attempt=$((attempt + 1)) + sleep "$verify_delay" +done From e0609020121261a137153d22a17d2798f89d707c Mon Sep 17 00:00:00 2001 From: Jan Marco Schreier Date: Wed, 7 Oct 2026 17:58:49 +0200 Subject: [PATCH 05/38] example(iaas): add cloud-init for the jump host and the probe --- .../080-cloud-init.tf | 69 ++++++++++++ .../files/egress-probe-boothook.sh | 102 ++++++++++++++++++ 2 files changed, 171 insertions(+) create mode 100644 examples/iaas-server-agent-restricted-egress/080-cloud-init.tf create mode 100755 examples/iaas-server-agent-restricted-egress/files/egress-probe-boothook.sh diff --git a/examples/iaas-server-agent-restricted-egress/080-cloud-init.tf b/examples/iaas-server-agent-restricted-egress/080-cloud-init.tf new file mode 100644 index 0000000..25f579f --- /dev/null +++ b/examples/iaas-server-agent-restricted-egress/080-cloud-init.tf @@ -0,0 +1,69 @@ +# Copyright 2026 Schwarz Digits Cloud GmbH & Co. KG +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +locals { + # --- jump host --------------------------------------------------------- + # + # sshd reads the files in /etc/ssh/sshd_config.d before the rest of its + # configuration. The main file sets no port, so this file alone decides + # where sshd listens. + jump_sshd_config = <<-EOT + # Jump host: sshd listens on TCP ${local.jump_ssh_port} only and accepts public keys only. + Port ${local.jump_ssh_port} + PermitRootLogin no + PubkeyAuthentication yes + PasswordAuthentication no + KbdInteractiveAuthentication no + AuthenticationMethods publickey + # ProxyJump opens a forwarded TCP connection to the target. + AllowTcpForwarding yes + AllowAgentForwarding no + X11Forwarding no + EOT + + # Ubuntu 24.04 starts sshd through socket activation: systemd holds the + # listening socket in ssh.socket, and a generator derives its port from the + # sshd configuration. "systemctl daemon-reload" runs the generator again, and + # the restart of ssh.socket closes port 22 and opens the new port. The other + # branch covers an image that runs sshd as the plain service ssh.service. + jump_user_data = join("\n", [ + "#cloud-config", + yamlencode({ + write_files = [{ + path = "/etc/ssh/sshd_config.d/00-jump-host.conf" + owner = "root:root" + permissions = "0644" + content = local.jump_sshd_config + }] + runcmd = [ + "systemctl daemon-reload", + "if systemctl is-enabled --quiet ssh.socket; then systemctl restart ssh.socket; else systemctl restart ssh.service; fi", + ] + }), + ]) + + # --- target ------------------------------------------------------------ + # + # The user data of the target is a single boothook and no cloud-config. + # cloud-init merges the cloud-config of the user over the cloud-config of + # the vendor data: where both set a key, the value of the user wins. The + # platform installs the Server Agent through vendor data. A boothook is a + # script of its own and merges with nothing, so this user data overrides + # nothing the platform delivers. A boothook is also the earliest point at + # which user data runs. + target_user_data = join("\n", [ + "#cloud-boothook", + file("${path.module}/files/egress-probe-boothook.sh"), + ]) +} diff --git a/examples/iaas-server-agent-restricted-egress/files/egress-probe-boothook.sh b/examples/iaas-server-agent-restricted-egress/files/egress-probe-boothook.sh new file mode 100755 index 0000000..c676af3 --- /dev/null +++ b/examples/iaas-server-agent-restricted-egress/files/egress-probe-boothook.sh @@ -0,0 +1,102 @@ +#!/bin/sh +# Copyright 2026 Schwarz Digits Cloud GmbH & Co. KG +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Egress probe of the target. Terraform passes this script to the server as +# cloud-init user data of the type cloud-boothook. cloud-init runs a boothook +# on every boot, during its network stage and before any of its modules. +# +# The probe writes one line to the kernel log for every new outbound +# connection: destination address, protocol, destination port and the user ID +# of the socket. It changes no verdict and drops nothing. +# +# The script writes two files and loads the first one: +# /etc/egress-probe.nft nftables rule set of the probe +# /usr/local/bin/egress-probe-report prints the distinct destinations +# +# cloud-init reads metadata, user data and vendor data from the metadata +# service before it runs this script. The probe does not see those requests. +# +# The script always ends with exit status 0, so a machine without nft still +# finishes cloud-init without an error. The report tells whether the rule set +# is loaded. + +ruleset=/etc/egress-probe.nft +report=/usr/local/bin/egress-probe-report + +cat >"$ruleset" <<'RULESET' +# Egress probe: logs every new outbound connection that leaves the machine. +# The chain has no rule with a verdict and its policy is accept. + +# Declare the table, then delete it: loading this file again replaces the +# table instead of adding the rule a second time. +table inet egress_probe +delete table inet egress_probe + +table inet egress_probe { + chain output { + type filter hook output priority filter; policy accept; + oifname != "lo" ct state new log prefix "egress-probe: " level info flags skuid + } +} +RULESET + +cat >"$report" <<'REPORT' +#!/bin/sh +# Prints one line for every distinct destination in the log of the egress +# probe: number of new connections, destination address, protocol, destination +# port and the user that owns the socket. +# +# Arguments are passed to journalctl. Without arguments the report covers all +# boots; "egress-probe-report --boot" covers the current boot. + +if [ "$(id -u)" -ne 0 ]; then + echo "error: run as root: sudo egress-probe-report" >&2 + exit 1 +fi + +if ! nft list table inet egress_probe >/dev/null 2>&1; then + echo "warning: the rule set /etc/egress-probe.nft is not loaded, the probe logs nothing" >&2 +fi + +journalctl --no-pager --output=cat _TRANSPORT=kernel "$@" | awk ' + /egress-probe: / { + destination = "-"; protocol = "-"; port = "-"; uid = "-" + for (i = 1; i <= NF; i++) { + if ($i ~ /^DST=/) destination = substr($i, 5) + else if ($i ~ /^PROTO=/) protocol = substr($i, 7) + else if ($i ~ /^DPT=/) port = substr($i, 5) + else if ($i ~ /^UID=/) uid = substr($i, 5) + } + connections[destination " " protocol " " port " " uid]++ + } + END { + for (key in connections) print connections[key], key + }' | sort -k2,2 -k3,3 -k4,4n -k5,5 | { + printf '%-6s %-40s %-6s %-6s %s\n' COUNT DESTINATION PROTO PORT USER + while read -r count destination protocol port uid; do + user=$(getent passwd "$uid" | cut -d: -f1) + printf '%-6s %-40s %-6s %-6s %s\n' "$count" "$destination" "$protocol" "$port" "${user:-$uid}" + done +} +REPORT +chmod 0755 "$report" + +if nft -f "$ruleset"; then + echo "egress-probe: rule set loaded" +else + echo "egress-probe: cannot load $ruleset" >&2 +fi + +exit 0 From 08568a136a6dbf208048994d33dd282a7e981cd7 Mon Sep 17 00:00:00 2001 From: Jan Marco Schreier Date: Wed, 7 Oct 2026 18:24:02 +0200 Subject: [PATCH 06/38] example(iaas): add the jump host and the target server --- .../020-variables.tf | 47 +++++++ .../090-servers.tf | 119 ++++++++++++++++++ 2 files changed, 166 insertions(+) create mode 100644 examples/iaas-server-agent-restricted-egress/090-servers.tf diff --git a/examples/iaas-server-agent-restricted-egress/020-variables.tf b/examples/iaas-server-agent-restricted-egress/020-variables.tf index 39da020..0238840 100644 --- a/examples/iaas-server-agent-restricted-egress/020-variables.tf +++ b/examples/iaas-server-agent-restricted-egress/020-variables.tf @@ -179,3 +179,50 @@ variable "egress_rules" { error_message = "Every egress rule sets both port_min and port_max, or neither." } } + +# --- machines ------------------------------------------------------------ + +variable "ssh_public_key" { + description = "Public SSH key installed on both machines for the default user of the image. Never the private key." + type = string +} + +variable "ssh_user" { + description = "User that receives ssh_public_key. STACKIT images create one default user per distribution; on Ubuntu it is \"ubuntu\". The outputs use the name in their SSH commands." + type = string + default = "ubuntu" +} + +variable "image_id" { + description = "Image UUID of both machines. The cloud-init configuration is written for the public image \"Ubuntu 24.04\" (amd64): it relies on nftables and on sshd socket activation. Look the ID up with: stackit image list --all --project-id " + type = string + + validation { + condition = can(regex("^[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}$", var.image_id)) && var.image_id != "00000000-0000-0000-0000-000000000000" + error_message = "image_id must be the UUID of an image, not the placeholder from terraform.tfvars.example." + } +} + +variable "availability_zone" { + description = "Availability zone of both machines." + type = string + default = "eu01-1" +} + +variable "jump_machine_type" { + description = "Machine type of the jump host. t3i.1 has 1 vCPU and 1 GB RAM." + type = string + default = "t3i.1" +} + +variable "target_machine_type" { + description = "Machine type of the target. c3i.1 has 1 vCPU and 2 GB RAM." + type = string + default = "c3i.1" +} + +variable "boot_volume_size" { + description = "Size of each boot volume in GB." + type = number + default = 20 +} diff --git a/examples/iaas-server-agent-restricted-egress/090-servers.tf b/examples/iaas-server-agent-restricted-egress/090-servers.tf new file mode 100644 index 0000000..7ee281e --- /dev/null +++ b/examples/iaas-server-agent-restricted-egress/090-servers.tf @@ -0,0 +1,119 @@ +# Copyright 2026 Schwarz Digits Cloud GmbH & Co. KG +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# cloud-init installs the key of the key pair for the default user of the +# image. Both machines use it. +resource "stackit_key_pair" "admin" { + name = "${var.name_prefix}-admin" + public_key = chomp(var.ssh_public_key) +} + +# --- jump host ----------------------------------------------------------- +# +# The only machine with a public IP. Its network interface carries the jump +# host's security group and no other. +resource "stackit_network_interface" "jump" { + project_id = local.project_id + network_id = stackit_network.this.network_id + name = "${var.name_prefix}-jump" + security = true + security_group_ids = [stackit_security_group.jump.security_group_id] +} + +resource "stackit_public_ip" "jump" { + project_id = local.project_id + network_interface_id = stackit_network_interface.jump.network_interface_id +} + +resource "stackit_server" "jump" { + project_id = local.project_id + name = "${var.name_prefix}-jump" + availability_zone = var.availability_zone + machine_type = var.jump_machine_type + keypair_name = stackit_key_pair.admin.name + network_interfaces = [stackit_network_interface.jump.network_interface_id] + user_data = local.jump_user_data + + boot_volume = { + source_type = "image" + source_id = var.image_id + size = var.boot_volume_size + delete_on_termination = true + } + + # The jump host only forwards SSH. It gets no Server Agent. + agent = { + provisioning_policy = "NEVER" + } +} + +# --- target -------------------------------------------------------------- +# +# The network interface is created after the rule set of its security group +# is complete: the default egress rules are gone and every rule of the +# allow-list exists. The server needs the interface, so its first boot already +# runs under the final rule set. cloud-init and the installation of the Server +# Agent happen during that first boot. +resource "stackit_network_interface" "target" { + project_id = local.project_id + network_id = stackit_network.this.network_id + name = "${var.name_prefix}-target" + security = true + security_group_ids = [stackit_security_group.target.security_group_id] + + depends_on = [ + terraform_data.remove_default_egress_rules, + stackit_security_group_rule.target_egress, + stackit_security_group_rule.target_ssh, + ] + + # Terraform deletes the interface before the security group it belongs to. + lifecycle { + replace_triggered_by = [terraform_data.egress_mode] + } +} + +resource "stackit_server" "target" { + project_id = local.project_id + name = "${var.name_prefix}-target" + availability_zone = var.availability_zone + machine_type = var.target_machine_type + keypair_name = stackit_key_pair.admin.name + network_interfaces = [stackit_network_interface.target.network_interface_id] + user_data = local.target_user_data + + # A server created from an image gets a new boot volume. A replaced server + # therefore starts from the unchanged image and runs its first boot again. + boot_volume = { + source_type = "image" + source_id = var.image_id + size = var.boot_volume_size + delete_on_termination = true + } + + # The platform installs the STACKIT Server Agent during the first boot. + agent = { + provisioning_policy = "ALWAYS" + } + + depends_on = [ + terraform_data.remove_default_egress_rules, + stackit_security_group_rule.target_egress, + stackit_security_group_rule.target_ssh, + ] + + lifecycle { + replace_triggered_by = [terraform_data.egress_mode] + } +} From b7325d3e17190f030e2034a00460720924d2771c Mon Sep 17 00:00:00 2001 From: Jan Marco Schreier Date: Wed, 7 Oct 2026 18:39:27 +0200 Subject: [PATCH 07/38] example(iaas): add outputs with SSH and verify commands --- .../100-outputs.tf | 95 +++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 examples/iaas-server-agent-restricted-egress/100-outputs.tf diff --git a/examples/iaas-server-agent-restricted-egress/100-outputs.tf b/examples/iaas-server-agent-restricted-egress/100-outputs.tf new file mode 100644 index 0000000..cc8287b --- /dev/null +++ b/examples/iaas-server-agent-restricted-egress/100-outputs.tf @@ -0,0 +1,95 @@ +# Copyright 2026 Schwarz Digits Cloud GmbH & Co. KG +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +locals { + jump_public_ip = stackit_public_ip.jump.ip + target_private_ip = stackit_network_interface.target.ipv4 + + # ProxyJump: the SSH client opens the connection to the target through the + # jump host. The private key stays on the machine that runs the command. + ssh_jump = "ssh -p ${local.jump_ssh_port} ${var.ssh_user}@${local.jump_public_ip}" + ssh_target = "ssh -J ${var.ssh_user}@${local.jump_public_ip}:${local.jump_ssh_port} ${var.ssh_user}@${local.target_private_ip}" +} + +output "project_id" { + description = "ID of the project this example created." + value = local.project_id +} + +output "egress_mode" { + description = "Rule set the target booted with: \"restricted\" or \"open\"." + value = var.egress_mode +} + +output "jump_server_id" { + description = "Server ID of the jump host." + value = stackit_server.jump.server_id +} + +output "target_server_id" { + description = "Server ID of the target." + value = stackit_server.target.server_id +} + +output "jump_public_ip" { + description = "Public IP of the jump host. sshd listens there on TCP 2222." + value = local.jump_public_ip +} + +output "target_private_ip" { + description = "Private IP of the target. Only the jump host reaches it." + value = local.target_private_ip +} + +output "jump_security_group_id" { + description = "ID of the jump host's security group." + value = stackit_security_group.jump.security_group_id +} + +output "target_security_group_id" { + description = "ID of the target's security group." + value = stackit_security_group.target.security_group_id +} + +output "ssh_command" { + description = "Opens a shell on the target through the jump host." + value = local.ssh_target +} + +output "verify_commands" { + description = "Commands that check the deployment, one per line. Print them with: terraform output -raw verify_commands" + value = <<-EOT + # Jump host: the only listener outside the loopback addresses is TCP ${local.jump_ssh_port}. + ${local.ssh_jump} 'ss -Htln' + + # Target: cloud-init finished its first boot without an error. + ${local.ssh_target} 'cloud-init status --long' + + # Target: the unit of the STACKIT Server Agent is active. + ${local.ssh_target} 'systemctl is-active stackit-server-agent.service' + + # Target: DNS and HTTPS reach the endpoint of the Server Agent signature. Prints the HTTP status. + ${local.ssh_target} 'curl -sS -o /dev/null -w "%%{http_code}\n" https://stackit-server-agent.object.storage.${var.stackit_region}.onstackit.cloud/stackit-server-agent.gpg' + + # Target: destinations of all outbound connections since the first boot. + ${local.ssh_target} 'sudo egress-probe-report' + + # Rules of both security groups as the IaaS API reports them. + stackit security-group rule list --project-id ${local.project_id} --security-group-id ${stackit_security_group.jump.security_group_id} + stackit security-group rule list --project-id ${local.project_id} --security-group-id ${stackit_security_group.target.security_group_id} + + # Run Command: the platform reaches the Server Agent on the target. + stackit server command create --project-id ${local.project_id} --server-id ${stackit_server.target.server_id} --template-name RunShellScript --params script='systemctl is-active stackit-server-agent.service' + EOT +} From 7285e628ece362e8a5abdb778e824983a547b9f3 Mon Sep 17 00:00:00 2001 From: Jan Marco Schreier Date: Wed, 7 Oct 2026 18:46:11 +0200 Subject: [PATCH 08/38] example(iaas): add tfvars example and reject its placeholders --- .../020-variables.tf | 12 +++++++- .../terraform.tfvars.example | 28 +++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 examples/iaas-server-agent-restricted-egress/terraform.tfvars.example diff --git a/examples/iaas-server-agent-restricted-egress/020-variables.tf b/examples/iaas-server-agent-restricted-egress/020-variables.tf index 0238840..6ee6ae7 100644 --- a/examples/iaas-server-agent-restricted-egress/020-variables.tf +++ b/examples/iaas-server-agent-restricted-egress/020-variables.tf @@ -15,11 +15,21 @@ variable "stackit_org_id" { description = "The STACKIT organization ID in which the network area is created." type = string + + validation { + condition = can(regex("^[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}$", var.stackit_org_id)) + error_message = "stackit_org_id must be the UUID of an organization, not the placeholder from terraform.tfvars.example." + } } variable "stackit_parent_container_id" { description = "Existing folder ID the project is created under, or the organization ID to create it directly under the organization. This example does not create the folder." type = string + + validation { + condition = !startswith(var.stackit_parent_container_id, "xxxxxxxx-") + error_message = "stackit_parent_container_id is still the placeholder from terraform.tfvars.example." + } } variable "stackit_admin_email" { @@ -198,7 +208,7 @@ variable "image_id" { type = string validation { - condition = can(regex("^[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}$", var.image_id)) && var.image_id != "00000000-0000-0000-0000-000000000000" + condition = can(regex("^[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}$", var.image_id)) error_message = "image_id must be the UUID of an image, not the placeholder from terraform.tfvars.example." } } diff --git a/examples/iaas-server-agent-restricted-egress/terraform.tfvars.example b/examples/iaas-server-agent-restricted-egress/terraform.tfvars.example new file mode 100644 index 0000000..d3f5779 --- /dev/null +++ b/examples/iaas-server-agent-restricted-egress/terraform.tfvars.example @@ -0,0 +1,28 @@ +# STACKIT organization the network area is created in. +stackit_org_id = "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" + +# Existing folder the project is created under, or the organization ID to +# create it directly under the organization. Terraform does not create it. +stackit_parent_container_id = "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" + +# An existing STACKIT user. A non-existent address fails with +# 409 Conflict "subject is not found". +stackit_admin_email = "firstname.lastname@example.com" + +# Path to the service account key file. Without it the provider reads the +# STACKIT_SERVICE_ACCOUNT_KEY_PATH environment variable or the credentials file. +# stackit_service_account_key_path = "/path/to/stackit-sa.json" + +# Only this IPv4 range reaches the jump host, on TCP 2222. +admin_cidr = "198.51.100.10/32" + +# Public half of your SSH key - never the private key. +ssh_public_key = "ssh-ed25519 AAAA... you@example.com" + +# UUID of the public image "Ubuntu 24.04" (amd64): +# stackit image list --all --project-id +image_id = "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" + +# "restricted" (default) removes the default egress rules of the target. +# "open" keeps them, see "Measure what a VM needs" in the README. +# egress_mode = "open" From a8a3face83a90d44f1bd1d57c60c22d25307b4b6 Mon Sep 17 00:00:00 2001 From: Jan Marco Schreier Date: Wed, 7 Oct 2026 19:28:54 +0200 Subject: [PATCH 09/38] docs(iaas): document the restricted egress example --- AGENTS.md | 2 + .../MAINTAINERS.md | 11 + .../README.md | 285 ++++++++++++++++++ 3 files changed, 298 insertions(+) create mode 100644 examples/iaas-server-agent-restricted-egress/MAINTAINERS.md create mode 100644 examples/iaas-server-agent-restricted-egress/README.md diff --git a/AGENTS.md b/AGENTS.md index 79463b2..d4ebdd2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -118,6 +118,8 @@ Fetch only the files relevant to the task. A typical example contains Upload a custom VM image to STACKIT using Terraform - **`iaas-k3s-self-managed`** `[iaas, k3s, kubernetes, self-managed]` Deploys a 6-node k3s + Cilium cluster on STACKIT VMs and validates: L4 NLB, L7 ALB, CSI block storage, Cilium NetworkPolicy, and egress connectivity +- **`iaas-server-agent-restricted-egress`** `[iaas, compute, networking, sna, security-group, egress, cloud-init, server-agent, jump-host, ssh]` + Deploys a Linux VM whose security group allows outbound traffic to an allow-list only, with the STACKIT Server Agent switched on, and a jump host as the only way to reach it over SSH. The example lists the outbound rules the STACKIT documentation names for cloud-init and the Server Agent, removes the default egress rules that Terraform cannot manage, and records every destination the VM contacts, so you can check and extend the list - **`iaas-volume-encryption`** `[iaas, volume, encryption, kms, block-storage]` KMS & IaaS Resources to deploy a encrypted Block Storage Volume - **`iaas-windows-byol-stackit-migration`** `[iaas, windows, byol, migration, compute]` diff --git a/examples/iaas-server-agent-restricted-egress/MAINTAINERS.md b/examples/iaas-server-agent-restricted-egress/MAINTAINERS.md new file mode 100644 index 0000000..d8ad666 --- /dev/null +++ b/examples/iaas-server-agent-restricted-egress/MAINTAINERS.md @@ -0,0 +1,11 @@ +# Maintainers + +General maintainers: + +- Jan Marco Schreier (jan_marco.schreier_ext@external.digits.schwarz) + +This example is actively maintained. The owner is responsible for reviewing +and updating dependencies and functionalities on a monthly basis. +For questions, issues, or feature requests, please email general maintainers. +Please include the BP name and version in your request. We will track your +request as an issue. diff --git a/examples/iaas-server-agent-restricted-egress/README.md b/examples/iaas-server-agent-restricted-egress/README.md new file mode 100644 index 0000000..8725da1 --- /dev/null +++ b/examples/iaas-server-agent-restricted-egress/README.md @@ -0,0 +1,285 @@ + + +# Restricted Egress for a VM with cloud-init and the STACKIT Server Agent + +Deploys a Linux VM whose security group allows outbound traffic to an allow-list only, with the STACKIT Server Agent switched on, and a jump host as the only way to reach it over SSH. The example lists the outbound rules the STACKIT documentation names for cloud-init and the Server Agent, removes the default egress rules that Terraform cannot manage, and records every destination the VM contacts, so you can check and extend the list. + +## What the example shows + +A security group rule only allows. Every new security group starts with two rules that allow all outbound traffic, so an egress rule of your own restricts nothing until those two are gone. This example removes them before the VM boots for the first time. The first boot is the one that matters: cloud-init runs then, and the platform installs the Server Agent then. + +**cloud-init and the Server Agent are two different things.** [cloud-init](https://docs.cloud-init.io/en/latest/explanation/boot.html) is part of the image. It runs at boot and reads metadata, user data and vendor data from the metadata service at `http://169.254.169.254`. The [STACKIT Server Agent](https://docs.stackit.cloud/products/compute-engine/server-agent/basics/introduction-to-server-agent/) is a service of its own. The platform [installs it during the first boot through vendor data](https://docs.stackit.cloud/products/compute-engine/server-agent/getting-started/installing-server-agent-api/), and [Run Command](https://docs.stackit.cloud/products/compute-engine/run-command/) and Server Update Management use it afterwards. + +The example creates: + +- a STACKIT Network Area (SNA) with its regional configuration, a project attached to it and one network +- a **jump host** with a public IP, whose security group has one ingress rule and whose sshd listens on TCP 2222 +- a **target** without a public IP, with the Server Agent, an egress allow-list and a probe that logs every outbound connection +- a step that deletes the two default egress rules of the target's security group through the IaaS API + +## Architecture + +```mermaid +flowchart LR + admin(["Admin\nadmin_cidr"]) + + subgraph net["STACKIT Network Area, project, network 10.80.1.0/24"] + direction TB + jump["Jump host\npublic IP\nsshd on TCP 2222\nno Server Agent"] + target["Target\nno public IP\nServer Agent\negress probe"] + end + + metadata(["Metadata service\n169.254.169.254"]) + dns(["Nameservers\n1.1.1.1, 9.9.9.9"]) + stackit(["STACKIT public IP ranges\nObject Storage"]) + + admin -->|"TCP 2222"| jump + jump -->|"TCP 22"| target + target -->|"TCP 80"| metadata + target -->|"UDP 53, TCP 53"| dns + target -->|"TCP 443"| stackit +``` + +Each machine has its own security group, and its network interface carries that group and no other. The two groups have one ingress rule each: + +| Security group | Ingress rule | Source | +| -------------- | ------------ | ----------------------------------------- | +| jump host | TCP 2222 | `admin_cidr` | +| target | TCP 22 | members of the jump host's security group | + +Nothing else reaches the jump host from outside, and nothing but the jump host reaches the target. Both groups are stateful: the answers to an allowed connection pass without a rule of their own. The target therefore needs no egress rule for the SSH sessions that arrive from the jump host. + +sshd on the jump host listens on port 2222 itself. No port forwarding sits in front of it, and port 22 is closed. It accepts public keys only. TCP forwarding stays on, because `ssh -J` opens the connection to the target through it. The private key never leaves your machine. + +The jump host is not the subject of the egress test. Its security group keeps the two default egress rules. + +## Egress rules of the target + +`egress_rules` is the complete outbound allow-list. Its default holds the destinations the STACKIT documentation names: + +| Key | Destination | Protocol and port | What needs it | Source | +| -------------------- | ---------------------------------------------- | ----------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `metadata` | `169.254.169.254/32` | TCP 80 | cloud-init reads metadata, user data and vendor data. The Server Agent package comes from `http://169.254.169.254/stackit/v1/server-agent/amd64/deb`. | [cloud-init boot stages](https://docs.cloud-init.io/en/latest/explanation/boot.html), [Install the Server Agent on a running server](https://docs.stackit.cloud/products/compute-engine/server-agent/getting-started/installing-server-agent-existing-server/) | +| `dns-udp`, `dns-tcp` | every address in `nameservers` | UDP 53, TCP 53 | Name resolution for the HTTPS endpoint below. | [Windows Server FAQ](https://docs.stackit.cloud/products/compute-engine/windows-server/faq/), which lists the metadata service and DNS as outbound rules of a custom security group | +| `https-stackit` | every IPv4 range of `stackit_public_ip_ranges` | TCP 443 | Signature check of the Server Agent package. Key and signature come from `https://stackit-server-agent.object.storage..onstackit.cloud`. | [Install the Server Agent on a running server](https://docs.stackit.cloud/products/compute-engine/server-agent/getting-started/installing-server-agent-existing-server/) | + +`ip_range` of a rule takes a CIDR or one of two names. `nameservers` stands for every address in the variable `nameservers`, which also configures the network. `stackit` stands for every IPv4 range the data source `stackit_public_ip_ranges` returns. A security group rule takes one range, so Terraform creates one rule per range. + +The Object Storage endpoint of the signature check lies inside the STACKIT public IP ranges. The API hosts `iaas.api.stackit.cloud` and `run-command.api.stackit.cloud` resolve to an address outside them, so the rule `https-stackit` does not cover them. [`scripts/check-stackit-ip.sh`](../../scripts/check-stackit-ip.sh) tells for any address whether it belongs to the ranges. + +The list has no rule for NTP (UDP 123) and none for the package mirrors of the image. It has no IPv6 rule; the network has no IPv6 prefix. Add what your machine needs through `egress_rules`. The section [Measure what a VM needs](#measure-what-a-vm-needs) shows how to find it. + +## How the default egress rules are removed + +[Every new security group contains a rule that allows all egress traffic](https://docs.stackit.cloud/products/network/core-networking/security-groups/basics/concepts/). The [IaaS API shows two of them](https://docs.stackit.cloud/products/iaas-api/how-tos/manage-security-groups-and-rules-via-iaas-api/), one for IPv4 and one for IPv6. They have a direction and an ethertype and nothing else: no protocol, no port range, no IP range, no remote security group. + +The provider cannot remove them. `stackit_security_group` has no argument for its rules. The data source `stackit_security_group` does not list rules. `stackit_security_group_rule` manages the rules it created and takes over an existing rule only by an import with the rule ID, which only the API returns. + +`070-default-egress-rules.tf` therefore calls the API directly: + +1. `ephemeral "stackit_access_token"` issues a token for the service account the provider uses. Terraform writes an ephemeral value neither to the plan nor to the state. +2. `terraform_data.remove_default_egress_rules` runs [`scripts/remove-default-egress-rules.sh`](scripts/remove-default-egress-rules.sh) once per security group. The token and the URL of the group reach the script through its environment. +3. The script reads the group, deletes every egress rule that has no protocol, no port range, no IP range and no remote security group, and reads the group again until no such rule is listed. +4. The script fails, and `terraform apply` with it, when a call fails or when an egress rule that allows all traffic is left. + +The network interface of the target and the target itself depend on this step and on every rule of the allow-list. Terraform creates them last, so the first boot already runs under the final rule set. + +Terraform hides the output of the script, because its environment holds the token. When the script fails, the error message of Terraform contains the output. A second run changes nothing: the script finds no default rule and only checks. + +### Two modes + +`egress_mode` selects the rule set the target boots with: + +| `egress_mode` | Default egress rules | Effect | +| ---------------------- | -------------------- | ------------------------------------------------------------------------- | +| `restricted` (default) | removed | Only `egress_rules` allows outbound traffic. | +| `open` | kept | All outbound traffic passes. The probe records what the machine contacts. | + +A deleted default rule does not come back. A change of `egress_mode` therefore replaces the target's security group, its network interface and the server. The new group starts with its default rules, and the new server boots for the first time under the rule set of the new mode. The jump host stays as it is. + +## Files + +| File | Contents | +| ---------------------------------------- | --------------------------------------------------------------------- | +| `010-provider.tf` | provider, with `enable_beta_resources` for the access token | +| `020-variables.tf` | all inputs, including `egress_rules` and `egress_mode` | +| `030-network-area.tf` | network area and its regional configuration | +| `040-project.tf` | project with the `networkArea` label | +| `050-network.tf` | the network of both machines | +| `060-security-groups.tf` | both security groups, the two ingress rules and the egress allow-list | +| `070-default-egress-rules.tf` | the mode switch and the removal of the default egress rules | +| `080-cloud-init.tf` | user data of both machines | +| `090-servers.tf` | key pair, network interfaces, public IP and both servers | +| `100-outputs.tf` | IDs, addresses, the SSH command and the verify commands | +| `scripts/remove-default-egress-rules.sh` | runs on the machine that runs Terraform | +| `files/egress-probe-boothook.sh` | runs on the target at every boot and installs the probe | + +## Usage + +### Prerequisites + +- Terraform 1.10 or later. Ephemeral resources exist since that version. +- Provider `stackitcloud/stackit` 0.118.0 or later. +- A STACKIT service account key that is allowed to create network areas in `stackit_org_id` and projects under `stackit_parent_container_id`. `stackit_access_token` works with a service account key only. +- `bash`, `jq` and `curl` 7.55.0 or later on the machine that runs Terraform. +- An SSH key pair and an OpenSSH client with `-J` (ProxyJump). +- The ID of the public image `Ubuntu 24.04` (amd64). The user data relies on two properties of that image: it ships `nft`, and it starts sshd through socket activation. +- An authenticated [`stackit` CLI](https://github.com/stackitcloud/stackit-cli) for the image lookup and for the checks that read the API. + +STACKIT replaces its public images with new builds and [deletes a released image after one year](https://docs.stackit.cloud/products/compute-engine/linux-server/faq/), so `image_id` has no default: + +```bash +stackit image list --all --project-id +``` + +### 1. Configure + +```bash +cp terraform.tfvars.example terraform.tfvars +``` + +Replace every placeholder. These variables have no default: + +- `stackit_org_id`: organization the network area is created in +- `stackit_parent_container_id`: existing folder for the project, or the organization ID +- `stackit_admin_email`: existing STACKIT user that owns the project +- `admin_cidr`: IPv4 range that reaches the jump host, for example your public IP with a `/32` suffix +- `ssh_public_key`: your complete public SSH key +- `image_id`: see above + +`stackit_service_account_key_path` is optional. Without it the provider reads `STACKIT_SERVICE_ACCOUNT_KEY_PATH` or the credentials file. + +### 2. Deploy + +```bash +terraform init +terraform validate +terraform plan +terraform apply +``` + +### 3. Connect + +```bash +$(terraform output -raw ssh_command) +``` + +The command has the form `ssh -J ubuntu@:2222 ubuntu@`. Both machines get `ssh_public_key` for the default user of the image through a key pair. `ssh_user` changes the user name in the outputs for an image with another default user. + +## Verify + +`terraform output -raw verify_commands` prints the following commands with the addresses and IDs of your deployment. + +**Jump host.** `ss -Htln` lists the listening TCP sockets. Port 2222 is the only one that is not bound to a loopback address: + +```bash +ssh -p 2222 ubuntu@ 'ss -Htln' +``` + +**cloud-init on the target.** `status: done` means the first boot finished. `cloud-init status` exits with a status other than 0 when a stage reported an error. `cloud-init status --wait` blocks until cloud-init finished: + +```bash +ssh -J ubuntu@:2222 ubuntu@ 'cloud-init status --long' +``` + +**Server Agent on the target.** The [installation guide](https://docs.stackit.cloud/products/compute-engine/server-agent/getting-started/installing-server-agent-api/) names the unit: + +```bash +ssh -J ubuntu@:2222 ubuntu@ 'systemctl is-active stackit-server-agent.service' +``` + +**DNS and HTTPS from the target.** The request goes to the endpoint of the signature check. An HTTP status in the output shows that the DNS rules and the HTTPS rule work. A timeout shows a missing rule: + +```bash +ssh -J ubuntu@:2222 ubuntu@ \ + 'curl -sS -o /dev/null -w "%{http_code}\n" https://stackit-server-agent.object.storage.eu01.onstackit.cloud/stackit-server-agent.gpg' +``` + +**Rules of both security groups.** The target's group lists its ingress rule and the allow-list, and no egress rule without a protocol. The jump host's group lists its ingress rule and the two default egress rules: + +```bash +stackit security-group rule list --project-id --security-group-id +``` + +**Run Command.** Run Command is the test for the running Server Agent: the platform sends a script, and the agent on the VM runs it. [Enable the service once per project](https://docs.stackit.cloud/products/compute-engine/run-command/how-tos/api-enable-the-run-command-service/), in the portal under _Server > Commands > Templates > Enable Commands_ or through the API: + +```bash +stackit curl -X POST \ + "https://service-enablement.api.eu01.stackit.cloud/v1/projects//services/cloud.stackit.run-command" +``` + +Then send a command and read its result. `create` prints the ID of the command: + +```bash +stackit server command create --project-id --server-id \ + --template-name RunShellScript --params script='systemctl is-active stackit-server-agent.service' + +stackit server command describe --project-id --server-id +``` + +The command reached the agent when its status is `completed` and its exit code is 0. + +## Measure what a VM needs + +The probe on the target is an nftables rule set that logs every new outbound connection to the kernel log. It changes no verdict and drops nothing. It runs inside the VM, in front of the security group, so it also logs connections the security group drops afterwards. cloud-init loads it as a boothook, the earliest point at which user data runs. cloud-init reads metadata, user data and vendor data before that point, and the probe does not see those requests. + +`egress-probe-report` prints one line per distinct destination: + +```bash +ssh -J ubuntu@:2222 ubuntu@ 'sudo egress-probe-report' +``` + +```text +COUNT DESTINATION PROTO PORT USER +``` + +`COUNT` is the number of new connections, `USER` the owner of the socket. Arguments go to `journalctl`: `sudo egress-probe-report --boot` covers the current boot only. + +The user data of the target is this boothook and no cloud-config. cloud-init [merges the cloud-config of the user over the cloud-config of the vendor data](https://docs.cloud-init.io/en/latest/explanation/vendordata.html). A boothook merges with nothing, so the user data of this example overrides nothing the platform delivers as vendor data. + +To find the rules a machine needs: + +1. Boot the target without restriction: + + ```bash + terraform apply -var egress_mode=open + ``` + +2. Wait until `cloud-init status --wait` returns on the target. Then use what you rely on: send a Run Command, start an update, let the machine run for a while. +3. Read the report. Every destination in it is a candidate for a rule. `sudo ss -tunp` names the process behind a connection that is open at that moment. `sudo cat /var/lib/cloud/instance/vendor-data.txt` shows what the platform passes to cloud-init as vendor data. +4. Add the rules you decide on to `egress_rules`. Setting the variable replaces the default, so keep the default entries: + + ```hcl + egress_rules = { + # ... the four default entries from 020-variables.tf ... + ntp = { + description = "NTP: time synchronisation" + protocol = "udp" + port_min = 123 + port_max = 123 + ip_range = "0.0.0.0/0" + } + } + ``` + +5. Boot the target with the allow-list and repeat the checks from [Verify](#verify): + + ```bash + terraform apply + ``` + +The report of a restricted target lists the connections the security group dropped as well. A destination that appears there and is missing in `egress_rules` is one the machine tried and did not reach. + +Removing an entry from `egress_rules` shows what stops working without it. A change of `egress_rules` alone adds or deletes rules on the running machine. To see a first boot under the changed list, replace the server: + +```bash +terraform apply -replace=stackit_server.target +``` + +## Clean up + +```bash +terraform destroy +``` + +`terraform destroy` removes everything the example created, including the project and the network area. From d8756b36a26223d11a4d0d019b3b6000eeb31bb4 Mon Sep 17 00:00:00 2001 From: Jan Marco Schreier Date: Thu, 15 Oct 2026 09:31:22 +0200 Subject: [PATCH 10/38] fix(iaas): read the rules from the list endpoint of the IaaS API --- .../scripts/remove-default-egress-rules.sh | 39 ++++++++++--------- 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/examples/iaas-server-agent-restricted-egress/scripts/remove-default-egress-rules.sh b/examples/iaas-server-agent-restricted-egress/scripts/remove-default-egress-rules.sh index 377f26e..7ccaf43 100755 --- a/examples/iaas-server-agent-restricted-egress/scripts/remove-default-egress-rules.sh +++ b/examples/iaas-server-agent-restricted-egress/scripts/remove-default-egress-rules.sh @@ -18,9 +18,9 @@ # STACKIT adds two rules to every new security group: one allows all outbound # IPv4 traffic, the other all outbound IPv6 traffic. The rules of a security # group only allow, so an allow-list has no effect while these two exist. The -# Terraform provider neither creates nor shows them. This script reads the -# group through the IaaS API, deletes the two rules and then checks that no -# egress rule is left that allows all traffic. +# Terraform provider neither creates nor shows them. This script lists the +# rules of the group through the IaaS API, deletes the two rules and then +# checks that no egress rule is left that allows all traffic. # # A default rule is an egress rule without protocol, port range, IP range and # remote security group. The script deletes nothing else and never touches an @@ -30,7 +30,7 @@ # SECURITY_GROUP_URL URL of the group in the IaaS API: # https://iaas.api.stackit.cloud/v2/projects//regions//security-groups/ # STACKIT_ACCESS_TOKEN access token of an identity that is allowed to edit the group -# VERIFY_ATTEMPTS optional: how often the final check reads the group (default 12) +# VERIFY_ATTEMPTS optional: how often the final check lists the rules (default 12) # VERIFY_DELAY_SECONDS optional: pause between two checks (default 5) # # Requires bash, jq and curl 7.55.0 or later. @@ -55,7 +55,7 @@ done # An egress rule without protocol, port range and remote security group matches # every packet to its IP range. readonly unrestricted_egress=' - .rules[] + .items[] | select(.direction == "egress") | select((.protocol // {}) | (.name == null and .number == null)) | select(.portRange == null and .icmpParameters == null) @@ -83,16 +83,19 @@ api() { "$@" "$url" } -read_group() { - local group - group=$(api GET "$SECURITY_GROUP_URL") || fail "cannot read the security group at $SECURITY_GROUP_URL" - jq -e '(.rules | type) == "array"' >/dev/null 2>&1 <<<"$group" || +# The rules have an endpoint of their own. The API specification requires the +# field "items" in its answer. In the group object the field "rules" is +# optional, so a group without rules can come without it. +read_rules() { + local rules + rules=$(api GET "$SECURITY_GROUP_URL/rules") || fail "cannot read the rules of the security group at $SECURITY_GROUP_URL" + jq -e '(.items | type) == "array"' >/dev/null 2>&1 <<<"$rules" || fail "the answer of the IaaS API contains no list of rules" - printf '%s' "$group" + printf '%s' "$rules" } -group=$(read_group) -to_delete=$(jq -r "$default_rule_ids" <<<"$group") +rules=$(read_rules) +to_delete=$(jq -r "$default_rule_ids" <<<"$rules") deleted=0 while IFS= read -r rule_id; do @@ -105,19 +108,19 @@ while IFS= read -r rule_id; do deleted=$((deleted + 1)) done <<<"$to_delete" -# The API accepts a deletion before the rule is gone, so the check reads the -# group until no default rule is listed any more. +# The API accepts a deletion before the rule is gone, so the check lists the +# rules until no default rule is among them any more. attempt=1 while true; do - group=$(read_group) + rules=$(read_rules) - foreign=$(jq -r "[$foreign_rule_ids] | join(\" \")" <<<"$group") + foreign=$(jq -r "[$foreign_rule_ids] | join(\" \")" <<<"$rules") [[ -z "$foreign" ]] || fail "the group has an egress rule that allows all traffic and is not a default rule: $foreign. Delete it, then run again." - pending=$(jq -r "[$default_rule_ids] | join(\" \")" <<<"$group") + pending=$(jq -r "[$default_rule_ids] | join(\" \")" <<<"$rules") if [[ -z "$pending" ]]; then - remaining=$(jq -r '[.rules[] | select(.direction == "egress")] | length' <<<"$group") + remaining=$(jq -r '[.items[] | select(.direction == "egress")] | length' <<<"$rules") echo "deleted $deleted default egress rule(s); no egress rule that allows all traffic is left, $remaining egress rule(s) remain" exit 0 fi From 6ec4f3719dbc3a1f81da37af823a6603fe94c861 Mon Sep 17 00:00:00 2001 From: Jan Marco Schreier Date: Thu, 15 Oct 2026 09:37:05 +0200 Subject: [PATCH 11/38] docs(iaas): describe the rule listing of the removal script --- examples/iaas-server-agent-restricted-egress/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/iaas-server-agent-restricted-egress/README.md b/examples/iaas-server-agent-restricted-egress/README.md index 8725da1..6b88173 100644 --- a/examples/iaas-server-agent-restricted-egress/README.md +++ b/examples/iaas-server-agent-restricted-egress/README.md @@ -79,7 +79,7 @@ The provider cannot remove them. `stackit_security_group` has no argument for it 1. `ephemeral "stackit_access_token"` issues a token for the service account the provider uses. Terraform writes an ephemeral value neither to the plan nor to the state. 2. `terraform_data.remove_default_egress_rules` runs [`scripts/remove-default-egress-rules.sh`](scripts/remove-default-egress-rules.sh) once per security group. The token and the URL of the group reach the script through its environment. -3. The script reads the group, deletes every egress rule that has no protocol, no port range, no IP range and no remote security group, and reads the group again until no such rule is listed. +3. The script lists the rules of the group, deletes every egress rule that has no protocol, no port range, no IP range and no remote security group, and lists the rules again until no such rule is among them. 4. The script fails, and `terraform apply` with it, when a call fails or when an egress rule that allows all traffic is left. The network interface of the target and the target itself depend on this step and on every rule of the allow-list. Terraform creates them last, so the first boot already runs under the final rule set. From c0472cd56ac4bb30e901bd19c45f7a6280e34546 Mon Sep 17 00:00:00 2001 From: Jan Marco Schreier Date: Thu, 15 Oct 2026 09:52:47 +0200 Subject: [PATCH 12/38] fix(iaas): limit nameservers to three distinct addresses --- .../020-variables.tf | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/examples/iaas-server-agent-restricted-egress/020-variables.tf b/examples/iaas-server-agent-restricted-egress/020-variables.tf index 6ee6ae7..7c40ae1 100644 --- a/examples/iaas-server-agent-restricted-egress/020-variables.tf +++ b/examples/iaas-server-agent-restricted-egress/020-variables.tf @@ -90,13 +90,19 @@ variable "network_prefix" { } variable "nameservers" { - description = "IPv4 nameservers of the network. The egress rules with ip_range = \"nameservers\" allow DNS to exactly these addresses." + description = "One to three IPv4 nameservers of the network. The egress rules with ip_range = \"nameservers\" allow DNS to exactly these addresses." type = list(string) default = ["1.1.1.1", "9.9.9.9"] + # The IaaS API takes at most three nameservers. Every address becomes part + # of a for_each key in 060-security-groups.tf, so no address repeats. validation { - condition = length(var.nameservers) > 0 && alltrue([for ns in var.nameservers : !strcontains(ns, ":") && can(cidrhost("${ns}/32", 0))]) - error_message = "nameservers needs at least one entry, and every entry must be an IPv4 address without a prefix length." + condition = ( + length(var.nameservers) >= 1 && length(var.nameservers) <= 3 && + length(distinct(var.nameservers)) == length(var.nameservers) && + alltrue([for ns in var.nameservers : !strcontains(ns, ":") && can(cidrhost("${ns}/32", 0))]) + ) + error_message = "nameservers needs one to three distinct entries, and every entry must be an IPv4 address without a prefix length." } } From 08b4831bcc5429d7f647d72a98c1e2cf02c7a975 Mon Sep 17 00:00:00 2001 From: Jan Marco Schreier Date: Thu, 15 Oct 2026 09:59:30 +0200 Subject: [PATCH 13/38] docs(iaas): say that an egress entry becomes one rule per range --- examples/iaas-server-agent-restricted-egress/020-variables.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/iaas-server-agent-restricted-egress/020-variables.tf b/examples/iaas-server-agent-restricted-egress/020-variables.tf index 7c40ae1..55f0b88 100644 --- a/examples/iaas-server-agent-restricted-egress/020-variables.tf +++ b/examples/iaas-server-agent-restricted-egress/020-variables.tf @@ -132,7 +132,7 @@ variable "egress_mode" { } variable "egress_rules" { - description = "Outbound allow-list of the target's security group. Each entry becomes one IPv4 egress rule. ip_range takes a CIDR or one of two names: \"nameservers\" stands for every address in var.nameservers, \"stackit\" for every IPv4 range in data.stackit_public_ip_ranges. Leave port_min and port_max unset for a protocol without ports, such as icmp. The default holds the destinations the STACKIT documentation names." + description = "Outbound allow-list of the target's security group. Each entry becomes one IPv4 egress rule per address range. ip_range takes a CIDR or one of two names: \"nameservers\" stands for every address in var.nameservers, \"stackit\" for every IPv4 range in data.stackit_public_ip_ranges. Leave port_min and port_max unset for a protocol without ports, such as icmp. The default holds the destinations the STACKIT documentation names." type = map(object({ description = string protocol = string From c76babd40ed1d900aa34f3f89ab4d023a212f82d Mon Sep 17 00:00:00 2001 From: Jan Marco Schreier Date: Thu, 15 Oct 2026 10:05:12 +0200 Subject: [PATCH 14/38] docs(iaas): drop the completeness claim about egress_rules --- examples/iaas-server-agent-restricted-egress/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/iaas-server-agent-restricted-egress/README.md b/examples/iaas-server-agent-restricted-egress/README.md index 6b88173..dd8f4d5 100644 --- a/examples/iaas-server-agent-restricted-egress/README.md +++ b/examples/iaas-server-agent-restricted-egress/README.md @@ -55,7 +55,7 @@ The jump host is not the subject of the egress test. Its security group keeps th ## Egress rules of the target -`egress_rules` is the complete outbound allow-list. Its default holds the destinations the STACKIT documentation names: +`egress_rules` holds every egress rule this example adds to the target's security group. Its default holds the destinations the STACKIT documentation names: | Key | Destination | Protocol and port | What needs it | Source | | -------------------- | ---------------------------------------------- | ----------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | From 80792dfe10423a3c2d4ff19a3046da4f02304843 Mon Sep 17 00:00:00 2001 From: Jan Marco Schreier Date: Thu, 15 Oct 2026 10:16:56 +0200 Subject: [PATCH 15/38] docs(iaas): name the provider as the one that refuses the delete --- examples/iaas-server-agent-restricted-egress/040-project.tf | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/examples/iaas-server-agent-restricted-egress/040-project.tf b/examples/iaas-server-agent-restricted-egress/040-project.tf index a21d2b3..2df89e6 100644 --- a/examples/iaas-server-agent-restricted-egress/040-project.tf +++ b/examples/iaas-server-agent-restricted-egress/040-project.tf @@ -24,8 +24,9 @@ resource "stackit_resourcemanager_project" "this" { } # Terraform destroys in reverse order. Without this dependency it deletes the - # regional configuration and the project in parallel, and the API rejects the - # deletion of a regional configuration that still has active projects. + # regional configuration and the project in parallel, and the provider + # refuses to delete a regional configuration while the network area still + # has active projects. depends_on = [stackit_network_area_region.this] } From 9e4e438da047c14173665267f997915268c80eb8 Mon Sep 17 00:00:00 2001 From: Jan Marco Schreier Date: Thu, 15 Oct 2026 10:27:08 +0200 Subject: [PATCH 16/38] docs(iaas): note that the key pair name is unique per account --- examples/iaas-server-agent-restricted-egress/020-variables.tf | 2 +- examples/iaas-server-agent-restricted-egress/README.md | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/iaas-server-agent-restricted-egress/020-variables.tf b/examples/iaas-server-agent-restricted-egress/020-variables.tf index 55f0b88..f9e76de 100644 --- a/examples/iaas-server-agent-restricted-egress/020-variables.tf +++ b/examples/iaas-server-agent-restricted-egress/020-variables.tf @@ -64,7 +64,7 @@ variable "network_area_name" { } variable "name_prefix" { - description = "Prefix for the names of the network, the security groups, the network interfaces, the key pair and the two machines." + description = "Prefix for the names of the network, the security groups, the network interfaces, the key pair and the two machines. The key pair belongs to the service account that runs Terraform, not to the project. Its name is unique for that account, so a second deployment with the same service account needs another prefix." type = string default = "agent-egress" } diff --git a/examples/iaas-server-agent-restricted-egress/README.md b/examples/iaas-server-agent-restricted-egress/README.md index dd8f4d5..0bcd89c 100644 --- a/examples/iaas-server-agent-restricted-egress/README.md +++ b/examples/iaas-server-agent-restricted-egress/README.md @@ -166,6 +166,8 @@ $(terraform output -raw ssh_command) The command has the form `ssh -J ubuntu@:2222 ubuntu@`. Both machines get `ssh_public_key` for the default user of the image through a key pair. `ssh_user` changes the user name in the outputs for an image with another default user. +The key pair belongs to the service account that runs Terraform, not to the project. Its name is unique for that account, so a second deployment with the same service account needs another `name_prefix`. + ## Verify `terraform output -raw verify_commands` prints the following commands with the addresses and IDs of your deployment. From 115c6140d7027354d215af250a5559210b79039d Mon Sep 17 00:00:00 2001 From: Jan Marco Schreier Date: Thu, 15 Oct 2026 10:43:43 +0200 Subject: [PATCH 17/38] docs(iaas): state the ingress rules Terraform adds to each group --- .../060-security-groups.tf | 13 ++++++------- .../iaas-server-agent-restricted-egress/README.md | 8 +++++--- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/examples/iaas-server-agent-restricted-egress/060-security-groups.tf b/examples/iaas-server-agent-restricted-egress/060-security-groups.tf index 2f69a97..ce0b8bb 100644 --- a/examples/iaas-server-agent-restricted-egress/060-security-groups.tf +++ b/examples/iaas-server-agent-restricted-egress/060-security-groups.tf @@ -20,10 +20,9 @@ locals { # --- jump host ----------------------------------------------------------- # -# A security group only allows. The group has one ingress rule, so the jump -# host accepts new connections on TCP 2222 from admin_cidr and nothing else. -# The group is stateful: answers to connections the jump host opened itself -# pass without a rule. +# A security group only allows. Terraform adds one ingress rule to the group: +# new connections on TCP 2222 from admin_cidr. The group is stateful: answers +# to connections the jump host opened itself pass without a rule. resource "stackit_security_group" "jump" { project_id = local.project_id name = "${var.name_prefix}-jump" @@ -47,9 +46,9 @@ resource "stackit_security_group_rule" "jump_ssh" { # --- target -------------------------------------------------------------- # -# The target has no public IP. The only inbound rule names the jump host's -# security group as the source, so SSH reaches the target from the jump host -# and from nowhere else. +# The target has no public IP. The ingress rule Terraform adds names the jump +# host's security group as the source, so SSH reaches the target from the +# jump host and from nowhere else. resource "stackit_security_group" "target" { project_id = local.project_id name = "${var.name_prefix}-target" diff --git a/examples/iaas-server-agent-restricted-egress/README.md b/examples/iaas-server-agent-restricted-egress/README.md index 0bcd89c..8b424bd 100644 --- a/examples/iaas-server-agent-restricted-egress/README.md +++ b/examples/iaas-server-agent-restricted-egress/README.md @@ -13,7 +13,7 @@ A security group rule only allows. Every new security group starts with two rule The example creates: - a STACKIT Network Area (SNA) with its regional configuration, a project attached to it and one network -- a **jump host** with a public IP, whose security group has one ingress rule and whose sshd listens on TCP 2222 +- a **jump host** with a public IP, whose security group admits TCP 2222 from `admin_cidr` and whose sshd listens on that port - a **target** without a public IP, with the Server Agent, an egress allow-list and a probe that logs every outbound connection - a step that deletes the two default egress rules of the target's security group through the IaaS API @@ -40,7 +40,7 @@ flowchart LR target -->|"TCP 443"| stackit ``` -Each machine has its own security group, and its network interface carries that group and no other. The two groups have one ingress rule each: +Each machine has its own security group, and its network interface carries that group and no other. Terraform adds one ingress rule to each group: | Security group | Ingress rule | Source | | -------------- | ------------ | ----------------------------------------- | @@ -197,12 +197,14 @@ ssh -J ubuntu@:2222 ubuntu@ \ 'curl -sS -o /dev/null -w "%{http_code}\n" https://stackit-server-agent.object.storage.eu01.onstackit.cloud/stackit-server-agent.gpg' ``` -**Rules of both security groups.** The target's group lists its ingress rule and the allow-list, and no egress rule without a protocol. The jump host's group lists its ingress rule and the two default egress rules: +**Rules of both security groups.** The target's group lists the rule for TCP 22 from the jump host's group and the allow-list, and no egress rule without a protocol. The jump host's group lists the rule for TCP 2222 from `admin_cidr` and the two default egress rules: ```bash stackit security-group rule list --project-id --security-group-id ``` +If a list also shows an ingress rule whose remote security group is the group itself, the platform added it. The [security group concepts](https://docs.stackit.cloud/products/network/core-networking/security-groups/basics/concepts/) describe such a rule for a new group. It admits only members of that group, and each group has one member. + **Run Command.** Run Command is the test for the running Server Agent: the platform sends a script, and the agent on the VM runs it. [Enable the service once per project](https://docs.stackit.cloud/products/compute-engine/run-command/how-tos/api-enable-the-run-command-service/), in the portal under _Server > Commands > Templates > Enable Commands_ or through the API: ```bash From 2b942c3e1a6c88ab565deba9f2efd28236d95fa8 Mon Sep 17 00:00:00 2001 From: Jan Marco Schreier Date: Thu, 15 Oct 2026 10:56:19 +0200 Subject: [PATCH 18/38] docs(iaas): say from which point on the probe logs connections --- AGENTS.md | 2 +- examples/iaas-server-agent-restricted-egress/100-outputs.tf | 2 +- examples/iaas-server-agent-restricted-egress/README.md | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index d4ebdd2..ed1f0c0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -119,7 +119,7 @@ Fetch only the files relevant to the task. A typical example contains - **`iaas-k3s-self-managed`** `[iaas, k3s, kubernetes, self-managed]` Deploys a 6-node k3s + Cilium cluster on STACKIT VMs and validates: L4 NLB, L7 ALB, CSI block storage, Cilium NetworkPolicy, and egress connectivity - **`iaas-server-agent-restricted-egress`** `[iaas, compute, networking, sna, security-group, egress, cloud-init, server-agent, jump-host, ssh]` - Deploys a Linux VM whose security group allows outbound traffic to an allow-list only, with the STACKIT Server Agent switched on, and a jump host as the only way to reach it over SSH. The example lists the outbound rules the STACKIT documentation names for cloud-init and the Server Agent, removes the default egress rules that Terraform cannot manage, and records every destination the VM contacts, so you can check and extend the list + Deploys a Linux VM whose security group allows outbound traffic to an allow-list only, with the STACKIT Server Agent switched on, and a jump host as the only way to reach it over SSH. The example lists the outbound rules the STACKIT documentation names for cloud-init and the Server Agent, removes the default egress rules that Terraform cannot manage, and adds a probe that logs the destination of every new outbound connection once cloud-init has loaded it, so you can check and extend the list - **`iaas-volume-encryption`** `[iaas, volume, encryption, kms, block-storage]` KMS & IaaS Resources to deploy a encrypted Block Storage Volume - **`iaas-windows-byol-stackit-migration`** `[iaas, windows, byol, migration, compute]` diff --git a/examples/iaas-server-agent-restricted-egress/100-outputs.tf b/examples/iaas-server-agent-restricted-egress/100-outputs.tf index cc8287b..65b59cd 100644 --- a/examples/iaas-server-agent-restricted-egress/100-outputs.tf +++ b/examples/iaas-server-agent-restricted-egress/100-outputs.tf @@ -82,7 +82,7 @@ output "verify_commands" { # Target: DNS and HTTPS reach the endpoint of the Server Agent signature. Prints the HTTP status. ${local.ssh_target} 'curl -sS -o /dev/null -w "%%{http_code}\n" https://stackit-server-agent.object.storage.${var.stackit_region}.onstackit.cloud/stackit-server-agent.gpg' - # Target: destinations of all outbound connections since the first boot. + # Target: destinations of the outbound connections the probe logged, over all boots. ${local.ssh_target} 'sudo egress-probe-report' # Rules of both security groups as the IaaS API reports them. diff --git a/examples/iaas-server-agent-restricted-egress/README.md b/examples/iaas-server-agent-restricted-egress/README.md index 8b424bd..03467b0 100644 --- a/examples/iaas-server-agent-restricted-egress/README.md +++ b/examples/iaas-server-agent-restricted-egress/README.md @@ -2,7 +2,7 @@ # Restricted Egress for a VM with cloud-init and the STACKIT Server Agent -Deploys a Linux VM whose security group allows outbound traffic to an allow-list only, with the STACKIT Server Agent switched on, and a jump host as the only way to reach it over SSH. The example lists the outbound rules the STACKIT documentation names for cloud-init and the Server Agent, removes the default egress rules that Terraform cannot manage, and records every destination the VM contacts, so you can check and extend the list. +Deploys a Linux VM whose security group allows outbound traffic to an allow-list only, with the STACKIT Server Agent switched on, and a jump host as the only way to reach it over SSH. The example lists the outbound rules the STACKIT documentation names for cloud-init and the Server Agent, removes the default egress rules that Terraform cannot manage, and adds a probe that logs the destination of every new outbound connection once cloud-init has loaded it, so you can check and extend the list. ## What the example shows @@ -14,7 +14,7 @@ The example creates: - a STACKIT Network Area (SNA) with its regional configuration, a project attached to it and one network - a **jump host** with a public IP, whose security group admits TCP 2222 from `admin_cidr` and whose sshd listens on that port -- a **target** without a public IP, with the Server Agent, an egress allow-list and a probe that logs every outbound connection +- a **target** without a public IP, with the Server Agent, an egress allow-list and a probe that logs every new outbound connection once cloud-init has loaded it - a step that deletes the two default egress rules of the target's security group through the IaaS API ## Architecture From 2d26848f3f31e21113942b51bb6a910d922bb76e Mon Sep 17 00:00:00 2001 From: Jan Marco Schreier Date: Thu, 15 Oct 2026 14:21:51 +0200 Subject: [PATCH 19/38] docs(iaas): cite the sources of the metadata service address --- .../iaas-server-agent-restricted-egress/README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/examples/iaas-server-agent-restricted-egress/README.md b/examples/iaas-server-agent-restricted-egress/README.md index 03467b0..0c39ca7 100644 --- a/examples/iaas-server-agent-restricted-egress/README.md +++ b/examples/iaas-server-agent-restricted-egress/README.md @@ -8,7 +8,7 @@ Deploys a Linux VM whose security group allows outbound traffic to an allow-list A security group rule only allows. Every new security group starts with two rules that allow all outbound traffic, so an egress rule of your own restricts nothing until those two are gone. This example removes them before the VM boots for the first time. The first boot is the one that matters: cloud-init runs then, and the platform installs the Server Agent then. -**cloud-init and the Server Agent are two different things.** [cloud-init](https://docs.cloud-init.io/en/latest/explanation/boot.html) is part of the image. It runs at boot and reads metadata, user data and vendor data from the metadata service at `http://169.254.169.254`. The [STACKIT Server Agent](https://docs.stackit.cloud/products/compute-engine/server-agent/basics/introduction-to-server-agent/) is a service of its own. The platform [installs it during the first boot through vendor data](https://docs.stackit.cloud/products/compute-engine/server-agent/getting-started/installing-server-agent-api/), and [Run Command](https://docs.stackit.cloud/products/compute-engine/run-command/) and Server Update Management use it afterwards. +**cloud-init and the Server Agent are two different things.** [cloud-init](https://docs.cloud-init.io/en/latest/explanation/boot.html) is part of the image. It runs at boot and reads metadata, user data and vendor data from the [metadata service](https://docs.stackit.cloud/products/compute-engine/server/basics/maintenance/) at `http://169.254.169.254`. The [STACKIT Server Agent](https://docs.stackit.cloud/products/compute-engine/server-agent/basics/introduction-to-server-agent/) is a service of its own. The platform [installs it during the first boot through vendor data](https://docs.stackit.cloud/products/compute-engine/server-agent/getting-started/installing-server-agent-api/), and [Run Command](https://docs.stackit.cloud/products/compute-engine/run-command/) and Server Update Management use it afterwards. The example creates: @@ -57,11 +57,11 @@ The jump host is not the subject of the egress test. Its security group keeps th `egress_rules` holds every egress rule this example adds to the target's security group. Its default holds the destinations the STACKIT documentation names: -| Key | Destination | Protocol and port | What needs it | Source | -| -------------------- | ---------------------------------------------- | ----------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `metadata` | `169.254.169.254/32` | TCP 80 | cloud-init reads metadata, user data and vendor data. The Server Agent package comes from `http://169.254.169.254/stackit/v1/server-agent/amd64/deb`. | [cloud-init boot stages](https://docs.cloud-init.io/en/latest/explanation/boot.html), [Install the Server Agent on a running server](https://docs.stackit.cloud/products/compute-engine/server-agent/getting-started/installing-server-agent-existing-server/) | -| `dns-udp`, `dns-tcp` | every address in `nameservers` | UDP 53, TCP 53 | Name resolution for the HTTPS endpoint below. | [Windows Server FAQ](https://docs.stackit.cloud/products/compute-engine/windows-server/faq/), which lists the metadata service and DNS as outbound rules of a custom security group | -| `https-stackit` | every IPv4 range of `stackit_public_ip_ranges` | TCP 443 | Signature check of the Server Agent package. Key and signature come from `https://stackit-server-agent.object.storage..onstackit.cloud`. | [Install the Server Agent on a running server](https://docs.stackit.cloud/products/compute-engine/server-agent/getting-started/installing-server-agent-existing-server/) | +| Key | Destination | Protocol and port | What needs it | Source | +| -------------------- | ---------------------------------------------- | ----------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `metadata` | `169.254.169.254/32` | TCP 80 | cloud-init reads metadata, user data and vendor data. The Server Agent package comes from `http://169.254.169.254/stackit/v1/server-agent/amd64/deb`. | [cloud-init boot stages](https://docs.cloud-init.io/en/latest/explanation/boot.html) for the stages, [Windows Server FAQ](https://docs.stackit.cloud/products/compute-engine/windows-server/faq/) for the address as an outbound rule of a custom security group, [Install the Server Agent on a running server](https://docs.stackit.cloud/products/compute-engine/server-agent/getting-started/installing-server-agent-existing-server/) for the package URL | +| `dns-udp`, `dns-tcp` | every address in `nameservers` | UDP 53, TCP 53 | Name resolution for the HTTPS endpoint below. | [Windows Server FAQ](https://docs.stackit.cloud/products/compute-engine/windows-server/faq/), which lists the metadata service and DNS as outbound rules of a custom security group | +| `https-stackit` | every IPv4 range of `stackit_public_ip_ranges` | TCP 443 | Signature check of the Server Agent package. Key and signature come from `https://stackit-server-agent.object.storage..onstackit.cloud`. | [Install the Server Agent on a running server](https://docs.stackit.cloud/products/compute-engine/server-agent/getting-started/installing-server-agent-existing-server/) | `ip_range` of a rule takes a CIDR or one of two names. `nameservers` stands for every address in the variable `nameservers`, which also configures the network. `stackit` stands for every IPv4 range the data source `stackit_public_ip_ranges` returns. A security group rule takes one range, so Terraform creates one rule per range. From 8592e367a9e4ec925021685827ea49fa00c00c7b Mon Sep 17 00:00:00 2001 From: Jan Marco Schreier Date: Thu, 15 Oct 2026 14:29:17 +0200 Subject: [PATCH 20/38] docs(iaas): link the portal guide for enabling Run Command --- examples/iaas-server-agent-restricted-egress/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/iaas-server-agent-restricted-egress/README.md b/examples/iaas-server-agent-restricted-egress/README.md index 0c39ca7..e26c437 100644 --- a/examples/iaas-server-agent-restricted-egress/README.md +++ b/examples/iaas-server-agent-restricted-egress/README.md @@ -205,7 +205,7 @@ stackit security-group rule list --project-id --security-group-id < If a list also shows an ingress rule whose remote security group is the group itself, the platform added it. The [security group concepts](https://docs.stackit.cloud/products/network/core-networking/security-groups/basics/concepts/) describe such a rule for a new group. It admits only members of that group, and each group has one member. -**Run Command.** Run Command is the test for the running Server Agent: the platform sends a script, and the agent on the VM runs it. [Enable the service once per project](https://docs.stackit.cloud/products/compute-engine/run-command/how-tos/api-enable-the-run-command-service/), in the portal under _Server > Commands > Templates > Enable Commands_ or through the API: +**Run Command.** Run Command is the test for the running Server Agent: the platform sends a script, and the agent on the VM runs it. Enable the service once per project, [in the portal](https://docs.stackit.cloud/products/compute-engine/run-command/getting-started/enable-run-command-service/) under _Computing > Server > your server > Command > Templates > Enable Commands_ or [through the API](https://docs.stackit.cloud/products/compute-engine/run-command/how-tos/api-enable-the-run-command-service/): ```bash stackit curl -X POST \ From 1ced2ef44c80d25ecc066adcd65fb6d4b1bc333a Mon Sep 17 00:00:00 2001 From: Jan Marco Schreier Date: Thu, 15 Oct 2026 14:44:34 +0200 Subject: [PATCH 21/38] example(iaas): add a check for a destination outside the allow-list --- .../iaas-server-agent-restricted-egress/100-outputs.tf | 3 +++ examples/iaas-server-agent-restricted-egress/README.md | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/examples/iaas-server-agent-restricted-egress/100-outputs.tf b/examples/iaas-server-agent-restricted-egress/100-outputs.tf index 65b59cd..6f5e2fb 100644 --- a/examples/iaas-server-agent-restricted-egress/100-outputs.tf +++ b/examples/iaas-server-agent-restricted-egress/100-outputs.tf @@ -82,6 +82,9 @@ output "verify_commands" { # Target: DNS and HTTPS reach the endpoint of the Server Agent signature. Prints the HTTP status. ${local.ssh_target} 'curl -sS -o /dev/null -w "%%{http_code}\n" https://stackit-server-agent.object.storage.${var.stackit_region}.onstackit.cloud/stackit-server-agent.gpg' + # Target: a destination outside the default allow-list. "restricted": curl gets no connection and prints 000. "open": curl prints the HTTP status. + ${local.ssh_target} 'curl -sS -m 10 -o /dev/null -w "%%{http_code}\n" https://example.com' + # Target: destinations of the outbound connections the probe logged, over all boots. ${local.ssh_target} 'sudo egress-probe-report' diff --git a/examples/iaas-server-agent-restricted-egress/README.md b/examples/iaas-server-agent-restricted-egress/README.md index e26c437..6500528 100644 --- a/examples/iaas-server-agent-restricted-egress/README.md +++ b/examples/iaas-server-agent-restricted-egress/README.md @@ -197,6 +197,13 @@ ssh -J ubuntu@:2222 ubuntu@ \ 'curl -sS -o /dev/null -w "%{http_code}\n" https://stackit-server-agent.object.storage.eu01.onstackit.cloud/stackit-server-agent.gpg' ``` +**A destination outside the allow-list.** This check shows that the restriction is in effect. The name `example.com` resolves through the DNS rules, and its addresses lie outside the STACKIT public IP ranges, so no rule of the default allow-list covers the request. With `egress_mode = "restricted"` curl gets no connection: it prints `000` and ends with an error after at most 10 seconds. With `egress_mode = "open"` it prints an HTTP status. An HTTP status on a restricted target shows that a rule allows the connection, for example a default egress rule that is still in place: + +```bash +ssh -J ubuntu@:2222 ubuntu@ \ + 'curl -sS -m 10 -o /dev/null -w "%{http_code}\n" https://example.com' +``` + **Rules of both security groups.** The target's group lists the rule for TCP 22 from the jump host's group and the allow-list, and no egress rule without a protocol. The jump host's group lists the rule for TCP 2222 from `admin_cidr` and the two default egress rules: ```bash From 5337e8491879a6a50139ad77b27c2f9946f89670 Mon Sep 17 00:00:00 2001 From: Jan Marco Schreier Date: Thu, 15 Oct 2026 14:51:02 +0200 Subject: [PATCH 22/38] docs(iaas): quote the concepts page on same-group traffic --- examples/iaas-server-agent-restricted-egress/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/iaas-server-agent-restricted-egress/README.md b/examples/iaas-server-agent-restricted-egress/README.md index 6500528..34c411c 100644 --- a/examples/iaas-server-agent-restricted-egress/README.md +++ b/examples/iaas-server-agent-restricted-egress/README.md @@ -210,7 +210,7 @@ ssh -J ubuntu@:2222 ubuntu@ \ stackit security-group rule list --project-id --security-group-id ``` -If a list also shows an ingress rule whose remote security group is the group itself, the platform added it. The [security group concepts](https://docs.stackit.cloud/products/network/core-networking/security-groups/basics/concepts/) describe such a rule for a new group. It admits only members of that group, and each group has one member. +If a list also shows an ingress rule whose remote security group is the group itself, the platform added it. The [security group concepts](https://docs.stackit.cloud/products/network/core-networking/security-groups/basics/concepts/) state that a new group allows traffic from members of the same group. Such a rule admits only members of that group, and each group has one member. **Run Command.** Run Command is the test for the running Server Agent: the platform sends a script, and the agent on the VM runs it. Enable the service once per project, [in the portal](https://docs.stackit.cloud/products/compute-engine/run-command/getting-started/enable-run-command-service/) under _Computing > Server > your server > Command > Templates > Enable Commands_ or [through the API](https://docs.stackit.cloud/products/compute-engine/run-command/how-tos/api-enable-the-run-command-service/): From 25eb98ef9d0e5e2a48f490040520872a231c8a68 Mon Sep 17 00:00:00 2001 From: Jan Marco Schreier Date: Thu, 15 Oct 2026 15:01:40 +0200 Subject: [PATCH 23/38] docs(iaas): tell a blocked connection from a failed DNS lookup --- examples/iaas-server-agent-restricted-egress/100-outputs.tf | 1 + examples/iaas-server-agent-restricted-egress/README.md | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/iaas-server-agent-restricted-egress/100-outputs.tf b/examples/iaas-server-agent-restricted-egress/100-outputs.tf index 6f5e2fb..499d54c 100644 --- a/examples/iaas-server-agent-restricted-egress/100-outputs.tf +++ b/examples/iaas-server-agent-restricted-egress/100-outputs.tf @@ -83,6 +83,7 @@ output "verify_commands" { ${local.ssh_target} 'curl -sS -o /dev/null -w "%%{http_code}\n" https://stackit-server-agent.object.storage.${var.stackit_region}.onstackit.cloud/stackit-server-agent.gpg' # Target: a destination outside the default allow-list. "restricted": curl gets no connection and prints 000. "open": curl prints the HTTP status. + # curl error 28 is the timeout of a dropped connection. Error 6 means the name did not resolve; the check is then void. ${local.ssh_target} 'curl -sS -m 10 -o /dev/null -w "%%{http_code}\n" https://example.com' # Target: destinations of the outbound connections the probe logged, over all boots. diff --git a/examples/iaas-server-agent-restricted-egress/README.md b/examples/iaas-server-agent-restricted-egress/README.md index 34c411c..4fb4cee 100644 --- a/examples/iaas-server-agent-restricted-egress/README.md +++ b/examples/iaas-server-agent-restricted-egress/README.md @@ -197,7 +197,7 @@ ssh -J ubuntu@:2222 ubuntu@ \ 'curl -sS -o /dev/null -w "%{http_code}\n" https://stackit-server-agent.object.storage.eu01.onstackit.cloud/stackit-server-agent.gpg' ``` -**A destination outside the allow-list.** This check shows that the restriction is in effect. The name `example.com` resolves through the DNS rules, and its addresses lie outside the STACKIT public IP ranges, so no rule of the default allow-list covers the request. With `egress_mode = "restricted"` curl gets no connection: it prints `000` and ends with an error after at most 10 seconds. With `egress_mode = "open"` it prints an HTTP status. An HTTP status on a restricted target shows that a rule allows the connection, for example a default egress rule that is still in place: +**A destination outside the allow-list.** This check shows that the restriction is in effect. The name `example.com` resolves through the DNS rules, and its addresses lie outside the STACKIT public IP ranges, so no rule of the default allow-list covers the request. With `egress_mode = "restricted"` curl gets no connection: it prints `000` and ends with an error after at most 10 seconds. `curl: (28)` is the timeout of a connection that got no answer. `curl: (6)` means that the name did not resolve, and the check then says nothing about the restriction. With `egress_mode = "open"` curl prints an HTTP status. An HTTP status on a restricted target shows that a rule allows the connection, for example a default egress rule that is still in place: ```bash ssh -J ubuntu@:2222 ubuntu@ \ From 3c8bbe6b72026deac90c8c30fc763b6c625eb882 Mon Sep 17 00:00:00 2001 From: Jan Marco Schreier Date: Thu, 15 Oct 2026 15:44:46 +0200 Subject: [PATCH 24/38] example(iaas): give a service account the role run-command.editor --- .../010-provider.tf | 3 ++ .../020-variables.tf | 26 +++++++++++ .../045-run-command-access.tf | 46 +++++++++++++++++++ .../100-outputs.tf | 9 +++- .../terraform.tfvars.example | 5 ++ 5 files changed, 87 insertions(+), 2 deletions(-) create mode 100644 examples/iaas-server-agent-restricted-egress/045-run-command-access.tf diff --git a/examples/iaas-server-agent-restricted-egress/010-provider.tf b/examples/iaas-server-agent-restricted-egress/010-provider.tf index a302c55..54983b0 100644 --- a/examples/iaas-server-agent-restricted-egress/010-provider.tf +++ b/examples/iaas-server-agent-restricted-egress/010-provider.tf @@ -31,4 +31,7 @@ provider "stackit" { # Required by the ephemeral resource stackit_access_token in # 070-default-egress-rules.tf. enable_beta_resources = true + # Required by stackit_authorization_project_role_assignment in + # 045-run-command-access.tf. + experiments = ["iam"] } diff --git a/examples/iaas-server-agent-restricted-egress/020-variables.tf b/examples/iaas-server-agent-restricted-egress/020-variables.tf index f9e76de..f094c6f 100644 --- a/examples/iaas-server-agent-restricted-egress/020-variables.tf +++ b/examples/iaas-server-agent-restricted-egress/020-variables.tf @@ -118,6 +118,32 @@ variable "admin_cidr" { } } +# --- Run Command --------------------------------------------------------- + +variable "run_command_service_account_email" { + description = "Email of an existing service account that sends Run Command to the target. Terraform gives it the role run-command.editor on the project. Unset, Terraform creates a service account in the project and gives the role to that one." + type = string + default = null + + # The provider rejects a subject with upper-case letters, but only at plan. + validation { + condition = var.run_command_service_account_email == null || can(regex("^[a-z0-9._-]+@[a-z0-9.-]+$", var.run_command_service_account_email)) + error_message = "run_command_service_account_email must be a service account email in lower case, for example name-1234567@sa.stackit.cloud." + } +} + +variable "run_command_service_account_name" { + description = "Name of the service account Terraform creates when run_command_service_account_email is unset. STACKIT appends a random suffix to form the email." + type = string + default = "run-command" + + # The provider checks these rules only at plan. + validation { + condition = length(var.run_command_service_account_name) <= 20 && can(regex("^[a-z](-?[a-z0-9]+)*$", var.run_command_service_account_name)) + error_message = "run_command_service_account_name has at most 20 characters, starts with a lower-case letter and holds lower-case letters, digits and single dashes, but no dash at the end." + } +} + # --- outbound traffic of the target -------------------------------------- variable "egress_mode" { diff --git a/examples/iaas-server-agent-restricted-egress/045-run-command-access.tf b/examples/iaas-server-agent-restricted-egress/045-run-command-access.tf new file mode 100644 index 0000000..b7c45bd --- /dev/null +++ b/examples/iaas-server-agent-restricted-egress/045-run-command-access.tf @@ -0,0 +1,46 @@ +# Copyright 2026 Schwarz Digits Cloud GmbH & Co. KG +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Run Command checks the permission run-command.command.create on the project. +# The role owner on a folder carries no run-command permission, so a service +# account that created the project through that role gets 403 Forbidden. This +# file gives one service account the role run-command.editor on the project: +# an existing one, or one that Terraform creates. + +# Created only when run_command_service_account_email is unset. Terraform +# creates no key for it, because the private key would be written to the +# state. The README shows how to create one with the CLI. +resource "stackit_service_account" "run_command" { + count = var.run_command_service_account_email == null ? 1 : 0 + + project_id = local.project_id + name = var.run_command_service_account_name +} + +locals { + run_command_service_account_email = ( + var.run_command_service_account_email == null + ? stackit_service_account.run_command[0].email + : var.run_command_service_account_email + ) +} + +# run-command.editor holds the permissions to read the command templates and +# the agent, and to create, list and read commands. It cannot enable the Run +# Command service. +resource "stackit_authorization_project_role_assignment" "run_command" { + resource_id = local.project_id + role = "run-command.editor" + subject = local.run_command_service_account_email +} diff --git a/examples/iaas-server-agent-restricted-egress/100-outputs.tf b/examples/iaas-server-agent-restricted-egress/100-outputs.tf index 499d54c..d066a68 100644 --- a/examples/iaas-server-agent-restricted-egress/100-outputs.tf +++ b/examples/iaas-server-agent-restricted-egress/100-outputs.tf @@ -62,6 +62,11 @@ output "target_security_group_id" { value = stackit_security_group.target.security_group_id } +output "run_command_service_account_email" { + description = "Service account with the role run-command.editor on the project. Send Run Command as this account." + value = local.run_command_service_account_email +} + output "ssh_command" { description = "Opens a shell on the target through the jump host." value = local.ssh_target @@ -93,7 +98,7 @@ output "verify_commands" { stackit security-group rule list --project-id ${local.project_id} --security-group-id ${stackit_security_group.jump.security_group_id} stackit security-group rule list --project-id ${local.project_id} --security-group-id ${stackit_security_group.target.security_group_id} - # Run Command: the platform reaches the Server Agent on the target. - stackit server command create --project-id ${local.project_id} --server-id ${stackit_server.target.server_id} --template-name RunShellScript --params script='systemctl is-active stackit-server-agent.service' + # Run Command: the platform reaches the Server Agent on the target. The CLI profile run-command holds a key of ${local.run_command_service_account_email}, see "Verify" in the README. + STACKIT_CLI_PROFILE=run-command stackit server command create --project-id ${local.project_id} --server-id ${stackit_server.target.server_id} --template-name RunShellScript --params script='systemctl is-active stackit-server-agent.service' EOT } diff --git a/examples/iaas-server-agent-restricted-egress/terraform.tfvars.example b/examples/iaas-server-agent-restricted-egress/terraform.tfvars.example index d3f5779..794a5ca 100644 --- a/examples/iaas-server-agent-restricted-egress/terraform.tfvars.example +++ b/examples/iaas-server-agent-restricted-egress/terraform.tfvars.example @@ -26,3 +26,8 @@ image_id = "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" # "restricted" (default) removes the default egress rules of the target. # "open" keeps them, see "Measure what a VM needs" in the README. # egress_mode = "open" + +# Existing service account that sends Run Command. Terraform gives it the role +# run-command.editor on the project. Unset, Terraform creates one in the +# project. Lower case only. +# run_command_service_account_email = "name-1234567@sa.stackit.cloud" From 19c71d287833251c9251e08a36486b44935df4bf Mon Sep 17 00:00:00 2001 From: Jan Marco Schreier Date: Thu, 15 Oct 2026 15:44:46 +0200 Subject: [PATCH 25/38] docs(iaas): explain who may send Run Command and how to switch to it --- .../README.md | 85 ++++++++++++++----- 1 file changed, 65 insertions(+), 20 deletions(-) diff --git a/examples/iaas-server-agent-restricted-egress/README.md b/examples/iaas-server-agent-restricted-egress/README.md index 4fb4cee..b1753f6 100644 --- a/examples/iaas-server-agent-restricted-egress/README.md +++ b/examples/iaas-server-agent-restricted-egress/README.md @@ -16,6 +16,7 @@ The example creates: - a **jump host** with a public IP, whose security group admits TCP 2222 from `admin_cidr` and whose sshd listens on that port - a **target** without a public IP, with the Server Agent, an egress allow-list and a probe that logs every new outbound connection once cloud-init has loaded it - a step that deletes the two default egress rules of the target's security group through the IaaS API +- the role `run-command.editor` on the project for a service account, either an existing one or one that Terraform creates, so that this account can send Run Command to the target ## Architecture @@ -97,22 +98,45 @@ Terraform hides the output of the script, because its environment holds the toke A deleted default rule does not come back. A change of `egress_mode` therefore replaces the target's security group, its network interface and the server. The new group starts with its default rules, and the new server boots for the first time under the rule set of the new mode. The jump host stays as it is. +## Who may send Run Command + +The Run Command API checks the permission `run-command.command.create` on the project. It answers `403 Forbidden` to an identity without that permission, even when that identity created the project and every resource in it. + +Terraform makes `stackit_admin_email` the owner of the project. The service account that runs Terraform does not become a member of the project; it acts through its roles on `stackit_parent_container_id` or above. A role on a folder carries other permissions than the role of the same name on a project. On a folder, the role `owner` contains no `run-command` permission. On a project, it contains all five. The authorization API lists the roles of a resource with their permissions: + +```bash +stackit curl https://authorization.api.stackit.cloud/v2/project//roles +stackit curl https://authorization.api.stackit.cloud/v2/folder//roles +``` + +`045-run-command-access.tf` therefore gives one service account the role `run-command.editor` on the project. Of the predefined roles that contain `run-command.command.create`, it has the fewest permissions: it reads the agent and the command templates, creates commands and lists and reads their results. It cannot enable the Run Command service. That needs `service-enablement.service-state.edit`, which roles such as `editor` and `owner` contain. + +`run_command_service_account_email` selects the service account: + +| `run_command_service_account_email` | Service account that gets the role | +| ----------------------------------- | ------------------------------------------------------------------------------------- | +| unset (default) | Terraform creates `run_command_service_account_name` in the project. | +| an email in lower case | the existing account. It can belong to another project. Terraform does not change it. | + +The output `run_command_service_account_email` names the account in both cases. Terraform creates no key for it, because `stackit_service_account_key` keeps the key JSON, private key included, in the state. [Verify](#verify) shows how to create a key with the CLI and send the command as that account. + ## Files -| File | Contents | -| ---------------------------------------- | --------------------------------------------------------------------- | -| `010-provider.tf` | provider, with `enable_beta_resources` for the access token | -| `020-variables.tf` | all inputs, including `egress_rules` and `egress_mode` | -| `030-network-area.tf` | network area and its regional configuration | -| `040-project.tf` | project with the `networkArea` label | -| `050-network.tf` | the network of both machines | -| `060-security-groups.tf` | both security groups, the two ingress rules and the egress allow-list | -| `070-default-egress-rules.tf` | the mode switch and the removal of the default egress rules | -| `080-cloud-init.tf` | user data of both machines | -| `090-servers.tf` | key pair, network interfaces, public IP and both servers | -| `100-outputs.tf` | IDs, addresses, the SSH command and the verify commands | -| `scripts/remove-default-egress-rules.sh` | runs on the machine that runs Terraform | -| `files/egress-probe-boothook.sh` | runs on the target at every boot and installs the probe | +| File | Contents | +| ---------------------------------------- | ------------------------------------------------------------------------------------------------------------ | +| `010-provider.tf` | provider, with `enable_beta_resources` for the access token and the experiment `iam` for the role assignment | +| `020-variables.tf` | all inputs, including `egress_rules` and `egress_mode` | +| `030-network-area.tf` | network area and its regional configuration | +| `040-project.tf` | project with the `networkArea` label | +| `045-run-command-access.tf` | the service account for Run Command and its role on the project | +| `050-network.tf` | the network of both machines | +| `060-security-groups.tf` | both security groups, the two ingress rules and the egress allow-list | +| `070-default-egress-rules.tf` | the mode switch and the removal of the default egress rules | +| `080-cloud-init.tf` | user data of both machines | +| `090-servers.tf` | key pair, network interfaces, public IP and both servers | +| `100-outputs.tf` | IDs, addresses, the SSH command and the verify commands | +| `scripts/remove-default-egress-rules.sh` | runs on the machine that runs Terraform | +| `files/egress-probe-boothook.sh` | runs on the target at every boot and installs the probe | ## Usage @@ -120,7 +144,8 @@ A deleted default rule does not come back. A change of `egress_mode` therefore r - Terraform 1.10 or later. Ephemeral resources exist since that version. - Provider `stackitcloud/stackit` 0.118.0 or later. -- A STACKIT service account key that is allowed to create network areas in `stackit_org_id` and projects under `stackit_parent_container_id`. `stackit_access_token` works with a service account key only. +- A STACKIT service account key that is allowed to create network areas in `stackit_org_id` and projects under `stackit_parent_container_id`. In the new project it also adds a member (`iam.member.add`) and, unless `run_command_service_account_email` is set, creates a service account (`iam.service-account.create`). `stackit_access_token` works with a service account key only. +- For the Run Command check: an identity that may enable the Run Command service on the project, and a key of the service account that holds `run-command.editor`. See [Who may send Run Command](#who-may-send-run-command). - `bash`, `jq` and `curl` 7.55.0 or later on the machine that runs Terraform. - An SSH key pair and an OpenSSH client with `-J` (ProxyJump). - The ID of the public image `Ubuntu 24.04` (amd64). The user data relies on two properties of that image: it ships `nft`, and it starts sshd through socket activation. @@ -149,6 +174,8 @@ Replace every placeholder. These variables have no default: `stackit_service_account_key_path` is optional. Without it the provider reads `STACKIT_SERVICE_ACCOUNT_KEY_PATH` or the credentials file. +`run_command_service_account_email` is optional. Set it to give an existing service account the role for Run Command. Without it Terraform creates a service account named `run_command_service_account_name` (default `run-command`) in the project. + ### 2. Deploy ```bash @@ -212,23 +239,36 @@ stackit security-group rule list --project-id --security-group-id < If a list also shows an ingress rule whose remote security group is the group itself, the platform added it. The [security group concepts](https://docs.stackit.cloud/products/network/core-networking/security-groups/basics/concepts/) state that a new group allows traffic from members of the same group. Such a rule admits only members of that group, and each group has one member. -**Run Command.** Run Command is the test for the running Server Agent: the platform sends a script, and the agent on the VM runs it. Enable the service once per project, [in the portal](https://docs.stackit.cloud/products/compute-engine/run-command/getting-started/enable-run-command-service/) under _Computing > Server > your server > Command > Templates > Enable Commands_ or [through the API](https://docs.stackit.cloud/products/compute-engine/run-command/how-tos/api-enable-the-run-command-service/): +**Run Command.** Run Command is the test for the running Server Agent: the platform sends a script, and the agent on the VM runs it. Enable the service once per project, [in the portal](https://docs.stackit.cloud/products/compute-engine/run-command/getting-started/enable-run-command-service/) under _Computing > Server > your server > Command > Templates > Enable Commands_ or [through the API](https://docs.stackit.cloud/products/compute-engine/run-command/how-tos/api-enable-the-run-command-service/). The role `run-command.editor` does not allow this step. Run it as the project owner `stackit_admin_email` or as another identity with `service-enablement.service-state.edit` on the project: ```bash stackit curl -X POST \ "https://service-enablement.api.eu01.stackit.cloud/v1/projects//services/cloud.stackit.run-command" ``` +Send the command as the service account that `terraform output -raw run_command_service_account_email` prints. A CLI profile of its own keeps the login of your default profile. For the account Terraform created, create a key and activate it in that profile. The key file holds the private key; `umask 077` makes it readable for you only: + +```bash +(umask 077; stackit service-account key create --email --project-id \ + --output-format json --assume-yes > ~/.stackit/run-command-key.json) + +stackit config profile create run-command --empty --no-set +STACKIT_CLI_PROFILE=run-command stackit auth activate-service-account \ + --service-account-key-path ~/.stackit/run-command-key.json +``` + +For an existing account, activate a key you already have. To create one, pass the project the account belongs to as `--project-id`. + Then send a command and read its result. `create` prints the ID of the command: ```bash -stackit server command create --project-id --server-id \ +STACKIT_CLI_PROFILE=run-command stackit server command create --project-id --server-id \ --template-name RunShellScript --params script='systemctl is-active stackit-server-agent.service' -stackit server command describe --project-id --server-id +STACKIT_CLI_PROFILE=run-command stackit server command describe --project-id --server-id ``` -The command reached the agent when its status is `completed` and its exit code is 0. +The command reached the agent when its status is `completed` and its exit code is 0. `403 Forbidden` from `create` means that the active identity lacks `run-command.command.create` on the project. ## Measure what a VM needs @@ -293,4 +333,9 @@ terraform apply -replace=stackit_server.target terraform destroy ``` -`terraform destroy` removes everything the example created, including the project and the network area. +`terraform destroy` removes everything the example created, including the project, the network area and the service account for Run Command if Terraform created it. An existing account loses its role on the project and keeps its keys. Delete the key file you created and the CLI profile yourself: + +```bash +rm ~/.stackit/run-command-key.json +stackit config profile delete run-command +``` From 229f13c5b628f5a153806b1afb47cc56c7892c33 Mon Sep 17 00:00:00 2001 From: Jan Marco Schreier Date: Thu, 15 Oct 2026 15:47:09 +0200 Subject: [PATCH 26/38] docs(iaas): list the Run Command account among the outputs --- examples/iaas-server-agent-restricted-egress/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/iaas-server-agent-restricted-egress/README.md b/examples/iaas-server-agent-restricted-egress/README.md index b1753f6..00c72e6 100644 --- a/examples/iaas-server-agent-restricted-egress/README.md +++ b/examples/iaas-server-agent-restricted-egress/README.md @@ -134,7 +134,7 @@ The output `run_command_service_account_email` names the account in both cases. | `070-default-egress-rules.tf` | the mode switch and the removal of the default egress rules | | `080-cloud-init.tf` | user data of both machines | | `090-servers.tf` | key pair, network interfaces, public IP and both servers | -| `100-outputs.tf` | IDs, addresses, the SSH command and the verify commands | +| `100-outputs.tf` | IDs, addresses, the Run Command service account, the SSH command and the verify commands | | `scripts/remove-default-egress-rules.sh` | runs on the machine that runs Terraform | | `files/egress-probe-boothook.sh` | runs on the target at every boot and installs the probe | From 8d6bd56f350a07746e5834c96d7e16d93a877b31 Mon Sep 17 00:00:00 2001 From: Jan Marco Schreier Date: Thu, 15 Oct 2026 17:03:38 +0200 Subject: [PATCH 27/38] fix(iaas): allow DHCP and drop DNS and HTTPS from the egress default --- AGENTS.md | 2 +- .../020-variables.tf | 32 ++++-------- .../100-outputs.tf | 11 ++-- .../README.md | 51 +++++++++---------- 4 files changed, 38 insertions(+), 58 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index ed1f0c0..e239129 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -119,7 +119,7 @@ Fetch only the files relevant to the task. A typical example contains - **`iaas-k3s-self-managed`** `[iaas, k3s, kubernetes, self-managed]` Deploys a 6-node k3s + Cilium cluster on STACKIT VMs and validates: L4 NLB, L7 ALB, CSI block storage, Cilium NetworkPolicy, and egress connectivity - **`iaas-server-agent-restricted-egress`** `[iaas, compute, networking, sna, security-group, egress, cloud-init, server-agent, jump-host, ssh]` - Deploys a Linux VM whose security group allows outbound traffic to an allow-list only, with the STACKIT Server Agent switched on, and a jump host as the only way to reach it over SSH. The example lists the outbound rules the STACKIT documentation names for cloud-init and the Server Agent, removes the default egress rules that Terraform cannot manage, and adds a probe that logs the destination of every new outbound connection once cloud-init has loaded it, so you can check and extend the list + Deploys a Linux VM whose security group allows outbound traffic to an allow-list only, with the STACKIT Server Agent switched on, and a jump host as the only way to reach it over SSH. The example allows the two outbound destinations that cloud-init, the Server Agent and Run Command need, removes the default egress rules that Terraform cannot manage, and adds a probe that logs the destination of every new outbound connection once cloud-init has loaded it, so you can check and extend the list - **`iaas-volume-encryption`** `[iaas, volume, encryption, kms, block-storage]` KMS & IaaS Resources to deploy a encrypted Block Storage Volume - **`iaas-windows-byol-stackit-migration`** `[iaas, windows, byol, migration, compute]` diff --git a/examples/iaas-server-agent-restricted-egress/020-variables.tf b/examples/iaas-server-agent-restricted-egress/020-variables.tf index f094c6f..8cfd512 100644 --- a/examples/iaas-server-agent-restricted-egress/020-variables.tf +++ b/examples/iaas-server-agent-restricted-egress/020-variables.tf @@ -158,7 +158,7 @@ variable "egress_mode" { } variable "egress_rules" { - description = "Outbound allow-list of the target's security group. Each entry becomes one IPv4 egress rule per address range. ip_range takes a CIDR or one of two names: \"nameservers\" stands for every address in var.nameservers, \"stackit\" for every IPv4 range in data.stackit_public_ip_ranges. Leave port_min and port_max unset for a protocol without ports, such as icmp. The default holds the destinations the STACKIT documentation names." + description = "Outbound allow-list of the target's security group. Each entry becomes one IPv4 egress rule per address range. ip_range takes a CIDR or one of two names: \"nameservers\" stands for every address in var.nameservers, \"stackit\" for every IPv4 range in data.stackit_public_ip_ranges. Leave port_min and port_max unset for a protocol without ports, such as icmp. The default holds the two rules cloud-init, the Server Agent and Run Command need: DHCP and the metadata service." type = map(object({ description = string protocol = string @@ -167,34 +167,20 @@ variable "egress_rules" { ip_range = string })) default = { + dhcp = { + description = "DHCP: lease and renewal of the IPv4 address of the interface" + protocol = "udp" + port_min = 67 + port_max = 67 + ip_range = "0.0.0.0/0" + } metadata = { - description = "Metadata service: cloud-init reads user data, metadata and vendor data; download of the Server Agent package" + description = "Metadata service: cloud-init data, Server Agent package, traffic of the Server Agent and the monitoring agent" protocol = "tcp" port_min = 80 port_max = 80 ip_range = "169.254.169.254/32" } - dns-udp = { - description = "DNS: resolves the host names of the HTTPS destinations" - protocol = "udp" - port_min = 53 - port_max = 53 - ip_range = "nameservers" - } - dns-tcp = { - description = "DNS over TCP: answers that do not fit into a UDP datagram" - protocol = "tcp" - port_min = 53 - port_max = 53 - ip_range = "nameservers" - } - https-stackit = { - description = "HTTPS to the STACKIT public IP ranges: signature check of the Server Agent package on Object Storage" - protocol = "tcp" - port_min = 443 - port_max = 443 - ip_range = "stackit" - } } validation { diff --git a/examples/iaas-server-agent-restricted-egress/100-outputs.tf b/examples/iaas-server-agent-restricted-egress/100-outputs.tf index d066a68..dbd7ce3 100644 --- a/examples/iaas-server-agent-restricted-egress/100-outputs.tf +++ b/examples/iaas-server-agent-restricted-egress/100-outputs.tf @@ -78,18 +78,15 @@ output "verify_commands" { # Jump host: the only listener outside the loopback addresses is TCP ${local.jump_ssh_port}. ${local.ssh_jump} 'ss -Htln' - # Target: cloud-init finished its first boot without an error. + # Target: cloud-init finished its first boot without an error. "detail" names DataSourceOpenStackLocal; DataSourceNone means the VM reached no metadata service. ${local.ssh_target} 'cloud-init status --long' # Target: the unit of the STACKIT Server Agent is active. ${local.ssh_target} 'systemctl is-active stackit-server-agent.service' - # Target: DNS and HTTPS reach the endpoint of the Server Agent signature. Prints the HTTP status. - ${local.ssh_target} 'curl -sS -o /dev/null -w "%%{http_code}\n" https://stackit-server-agent.object.storage.${var.stackit_region}.onstackit.cloud/stackit-server-agent.gpg' - - # Target: a destination outside the default allow-list. "restricted": curl gets no connection and prints 000. "open": curl prints the HTTP status. - # curl error 28 is the timeout of a dropped connection. Error 6 means the name did not resolve; the check is then void. - ${local.ssh_target} 'curl -sS -m 10 -o /dev/null -w "%%{http_code}\n" https://example.com' + # Target: a destination outside the default allow-list, given as an address so the check needs no DNS. + # "restricted": curl gets no connection, prints 000 and ends with curl error 28 after 10 seconds. "open": curl prints the HTTP status. + ${local.ssh_target} 'curl -sS -m 10 -o /dev/null -w "%%{http_code}\n" https://1.1.1.1' # Target: destinations of the outbound connections the probe logged, over all boots. ${local.ssh_target} 'sudo egress-probe-report' diff --git a/examples/iaas-server-agent-restricted-egress/README.md b/examples/iaas-server-agent-restricted-egress/README.md index 00c72e6..5adc748 100644 --- a/examples/iaas-server-agent-restricted-egress/README.md +++ b/examples/iaas-server-agent-restricted-egress/README.md @@ -2,11 +2,11 @@ # Restricted Egress for a VM with cloud-init and the STACKIT Server Agent -Deploys a Linux VM whose security group allows outbound traffic to an allow-list only, with the STACKIT Server Agent switched on, and a jump host as the only way to reach it over SSH. The example lists the outbound rules the STACKIT documentation names for cloud-init and the Server Agent, removes the default egress rules that Terraform cannot manage, and adds a probe that logs the destination of every new outbound connection once cloud-init has loaded it, so you can check and extend the list. +Deploys a Linux VM whose security group allows outbound traffic to an allow-list only, with the STACKIT Server Agent switched on, and a jump host as the only way to reach it over SSH. The example allows the two outbound destinations that cloud-init, the Server Agent and Run Command need, removes the default egress rules that Terraform cannot manage, and adds a probe that logs the destination of every new outbound connection once cloud-init has loaded it, so you can check and extend the list. ## What the example shows -A security group rule only allows. Every new security group starts with two rules that allow all outbound traffic, so an egress rule of your own restricts nothing until those two are gone. This example removes them before the VM boots for the first time. The first boot is the one that matters: cloud-init runs then, and the platform installs the Server Agent then. +A security group rule only allows. Every new security group starts with two rules that allow all outbound traffic, so an egress rule of your own restricts nothing until those two are gone. This example removes them before the VM boots for the first time. Without them the group also drops DHCP, so the allow-list has to allow it. The first boot is the one that matters: cloud-init runs then, and the platform installs the Server Agent then. **cloud-init and the Server Agent are two different things.** [cloud-init](https://docs.cloud-init.io/en/latest/explanation/boot.html) is part of the image. It runs at boot and reads metadata, user data and vendor data from the [metadata service](https://docs.stackit.cloud/products/compute-engine/server/basics/maintenance/) at `http://169.254.169.254`. The [STACKIT Server Agent](https://docs.stackit.cloud/products/compute-engine/server-agent/basics/introduction-to-server-agent/) is a service of its own. The platform [installs it during the first boot through vendor data](https://docs.stackit.cloud/products/compute-engine/server-agent/getting-started/installing-server-agent-api/), and [Run Command](https://docs.stackit.cloud/products/compute-engine/run-command/) and Server Update Management use it afterwards. @@ -30,15 +30,13 @@ flowchart LR target["Target\nno public IP\nServer Agent\negress probe"] end + dhcp(["DHCP server\nof the network"]) metadata(["Metadata service\n169.254.169.254"]) - dns(["Nameservers\n1.1.1.1, 9.9.9.9"]) - stackit(["STACKIT public IP ranges\nObject Storage"]) admin -->|"TCP 2222"| jump jump -->|"TCP 22"| target + target -->|"UDP 67"| dhcp target -->|"TCP 80"| metadata - target -->|"UDP 53, TCP 53"| dns - target -->|"TCP 443"| stackit ``` Each machine has its own security group, and its network interface carries that group and no other. Terraform adds one ingress rule to each group: @@ -56,19 +54,18 @@ The jump host is not the subject of the egress test. Its security group keeps th ## Egress rules of the target -`egress_rules` holds every egress rule this example adds to the target's security group. Its default holds the destinations the STACKIT documentation names: +`egress_rules` holds every egress rule this example adds to the target's security group. Its default holds the two rules that cloud-init, the Server Agent and Run Command need: -| Key | Destination | Protocol and port | What needs it | Source | -| -------------------- | ---------------------------------------------- | ----------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `metadata` | `169.254.169.254/32` | TCP 80 | cloud-init reads metadata, user data and vendor data. The Server Agent package comes from `http://169.254.169.254/stackit/v1/server-agent/amd64/deb`. | [cloud-init boot stages](https://docs.cloud-init.io/en/latest/explanation/boot.html) for the stages, [Windows Server FAQ](https://docs.stackit.cloud/products/compute-engine/windows-server/faq/) for the address as an outbound rule of a custom security group, [Install the Server Agent on a running server](https://docs.stackit.cloud/products/compute-engine/server-agent/getting-started/installing-server-agent-existing-server/) for the package URL | -| `dns-udp`, `dns-tcp` | every address in `nameservers` | UDP 53, TCP 53 | Name resolution for the HTTPS endpoint below. | [Windows Server FAQ](https://docs.stackit.cloud/products/compute-engine/windows-server/faq/), which lists the metadata service and DNS as outbound rules of a custom security group | -| `https-stackit` | every IPv4 range of `stackit_public_ip_ranges` | TCP 443 | Signature check of the Server Agent package. Key and signature come from `https://stackit-server-agent.object.storage..onstackit.cloud`. | [Install the Server Agent on a running server](https://docs.stackit.cloud/products/compute-engine/server-agent/getting-started/installing-server-agent-existing-server/) | +| Key | Destination | Protocol and port | What needs it | +| ---------- | -------------------- | ----------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `dhcp` | `0.0.0.0/0` | UDP 67 | The interface gets its IPv4 address, the gateway and the nameservers through DHCP. The first request goes to the broadcast address, the renewal of the lease to the DHCP server of the network. | +| `metadata` | `169.254.169.254/32` | TCP 80 | [cloud-init reads metadata, user data and vendor data](https://docs.cloud-init.io/en/latest/explanation/boot.html). The vendor data installs the Server Agent from `http://169.254.169.254/stackit/v1/server-agent/amd64/deb` ([Install the Server Agent on a running server](https://docs.stackit.cloud/products/compute-engine/server-agent/getting-started/installing-server-agent-existing-server/)). The Server Agent and `stackit-server-monitoring-agent`, which comes with it, open connections to this address only. Run Command needs no further rule: the agent receives its commands through the metadata service. | -`ip_range` of a rule takes a CIDR or one of two names. `nameservers` stands for every address in the variable `nameservers`, which also configures the network. `stackit` stands for every IPv4 range the data source `stackit_public_ip_ranges` returns. A security group rule takes one range, so Terraform creates one rule per range. +The default egress rules of a new security group allow DHCP like all other traffic. Once they are gone, the group drops DHCP as well, unless a rule allows it. Without the rule `dhcp` the interface gets no IPv4 address. cloud-init then finds no metadata service and falls back to `DataSourceNone`, the platform does not install the Server Agent, and Run Command answers `404 agent not found`. -The Object Storage endpoint of the signature check lies inside the STACKIT public IP ranges. The API hosts `iaas.api.stackit.cloud` and `run-command.api.stackit.cloud` resolve to an address outside them, so the rule `https-stackit` does not cover them. [`scripts/check-stackit-ip.sh`](../../scripts/check-stackit-ip.sh) tells for any address whether it belongs to the ranges. +`ip_range` of a rule takes a CIDR or one of two names. `nameservers` stands for every address in the variable `nameservers`, which also configures the network. `stackit` stands for every IPv4 range the data source `stackit_public_ip_ranges` returns. A security group rule takes one range, so Terraform creates one rule per range. [`scripts/check-stackit-ip.sh`](../../scripts/check-stackit-ip.sh) tells for any address whether it belongs to the ranges. -The list has no rule for NTP (UDP 123) and none for the package mirrors of the image. It has no IPv6 rule; the network has no IPv6 prefix. Add what your machine needs through `egress_rules`. The section [Measure what a VM needs](#measure-what-a-vm-needs) shows how to find it. +The default has no rule for DNS, NTP or the package mirrors of the image. Name resolution, time synchronisation and package updates of the operating system fail on the target until you add rules for them; `systemd-timesyncd` does not synchronise the clock. The list has no IPv6 rule; the network has no IPv6 prefix. Add what your machine needs through `egress_rules`. The section [Measure what a VM needs](#measure-what-a-vm-needs) shows how to find it. ## How the default egress rules are removed @@ -205,7 +202,7 @@ The key pair belongs to the service account that runs Terraform, not to the proj ssh -p 2222 ubuntu@ 'ss -Htln' ``` -**cloud-init on the target.** `status: done` means the first boot finished. `cloud-init status` exits with a status other than 0 when a stage reported an error. `cloud-init status --wait` blocks until cloud-init finished: +**cloud-init on the target.** `status: done` means the first boot finished. `cloud-init status` exits with a status other than 0 when a stage reported an error. `cloud-init status --wait` blocks until cloud-init finished. The line `detail` names `DataSourceOpenStackLocal`. `DataSourceNone` there means that cloud-init reached no metadata service, for example because the rule `dhcp` is missing: ```bash ssh -J ubuntu@:2222 ubuntu@ 'cloud-init status --long' @@ -217,18 +214,11 @@ ssh -J ubuntu@:2222 ubuntu@ 'cloud-init status --long' ssh -J ubuntu@:2222 ubuntu@ 'systemctl is-active stackit-server-agent.service' ``` -**DNS and HTTPS from the target.** The request goes to the endpoint of the signature check. An HTTP status in the output shows that the DNS rules and the HTTPS rule work. A timeout shows a missing rule: +**A destination outside the allow-list.** This check shows that the restriction is in effect. The request goes to the address `1.1.1.1` directly, so it needs no DNS, and no rule of the default allow-list covers it. With `egress_mode = "restricted"` curl gets no connection: it prints `000` and ends with `curl: (28)`, the timeout of a connection that got no answer, after 10 seconds. With `egress_mode = "open"` curl prints an HTTP status. An HTTP status on a restricted target shows that a rule allows the connection, for example a default egress rule that is still in place: ```bash ssh -J ubuntu@:2222 ubuntu@ \ - 'curl -sS -o /dev/null -w "%{http_code}\n" https://stackit-server-agent.object.storage.eu01.onstackit.cloud/stackit-server-agent.gpg' -``` - -**A destination outside the allow-list.** This check shows that the restriction is in effect. The name `example.com` resolves through the DNS rules, and its addresses lie outside the STACKIT public IP ranges, so no rule of the default allow-list covers the request. With `egress_mode = "restricted"` curl gets no connection: it prints `000` and ends with an error after at most 10 seconds. `curl: (28)` is the timeout of a connection that got no answer. `curl: (6)` means that the name did not resolve, and the check then says nothing about the restriction. With `egress_mode = "open"` curl prints an HTTP status. An HTTP status on a restricted target shows that a rule allows the connection, for example a default egress rule that is still in place: - -```bash -ssh -J ubuntu@:2222 ubuntu@ \ - 'curl -sS -m 10 -o /dev/null -w "%{http_code}\n" https://example.com' + 'curl -sS -m 10 -o /dev/null -w "%{http_code}\n" https://1.1.1.1' ``` **Rules of both security groups.** The target's group lists the rule for TCP 22 from the jump host's group and the allow-list, and no egress rule without a protocol. The jump host's group lists the rule for TCP 2222 from `admin_cidr` and the two default egress rules: @@ -284,7 +274,7 @@ ssh -J ubuntu@:2222 ubuntu@ 'sudo egress-probe-report' COUNT DESTINATION PROTO PORT USER ``` -`COUNT` is the number of new connections, `USER` the owner of the socket. Arguments go to `journalctl`: `sudo egress-probe-report --boot` covers the current boot only. +`COUNT` is the number of new connections, `USER` the owner of the socket. Arguments go to `journalctl`: `sudo egress-probe-report --boot` covers the current boot only. The report lists neither DHCP nor the requests cloud-init sends before the probe loads. The rules `dhcp` and `metadata` cover both. The user data of the target is this boothook and no cloud-config. cloud-init [merges the cloud-config of the user over the cloud-config of the vendor data](https://docs.cloud-init.io/en/latest/explanation/vendordata.html). A boothook merges with nothing, so the user data of this example overrides nothing the platform delivers as vendor data. @@ -302,7 +292,14 @@ To find the rules a machine needs: ```hcl egress_rules = { - # ... the four default entries from 020-variables.tf ... + # ... the two default entries from 020-variables.tf ... + dns-udp = { + description = "DNS: name resolution through the nameservers of the network" + protocol = "udp" + port_min = 53 + port_max = 53 + ip_range = "nameservers" + } ntp = { description = "NTP: time synchronisation" protocol = "udp" From e1c79c24a9135b075b2f32d39689b62730f0bddb Mon Sep 17 00:00:00 2001 From: Jan Marco Schreier Date: Thu, 15 Oct 2026 17:03:38 +0200 Subject: [PATCH 28/38] docs(iaas): explain how admin_cidr can miss the address STACKIT sees --- examples/iaas-server-agent-restricted-egress/020-variables.tf | 2 +- examples/iaas-server-agent-restricted-egress/README.md | 4 +++- .../terraform.tfvars.example | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/examples/iaas-server-agent-restricted-egress/020-variables.tf b/examples/iaas-server-agent-restricted-egress/020-variables.tf index 8cfd512..3be628d 100644 --- a/examples/iaas-server-agent-restricted-egress/020-variables.tf +++ b/examples/iaas-server-agent-restricted-egress/020-variables.tf @@ -109,7 +109,7 @@ variable "nameservers" { # --- access -------------------------------------------------------------- variable "admin_cidr" { - description = "IPv4 source range that reaches the jump host on TCP 2222, for example your public IP with a /32 suffix." + description = "IPv4 source range that reaches the jump host on TCP 2222, for example your public IP with a /32 suffix. It must hold the address STACKIT sees; a connection with several public IPv4 addresses can show another one to STACKIT than to an IP lookup service." type = string validation { diff --git a/examples/iaas-server-agent-restricted-egress/README.md b/examples/iaas-server-agent-restricted-egress/README.md index 5adc748..a4a4dad 100644 --- a/examples/iaas-server-agent-restricted-egress/README.md +++ b/examples/iaas-server-agent-restricted-egress/README.md @@ -169,6 +169,8 @@ Replace every placeholder. These variables have no default: - `ssh_public_key`: your complete public SSH key - `image_id`: see above +`admin_cidr` must hold the address STACKIT sees as the source of your SSH connection. Some internet connections leave through more than one public IPv4 address, depending on the destination, and an IP lookup service then shows another address than the one STACKIT sees. Compare two services, for example `curl -4 https://api.ipify.org` and `curl -4 https://ifconfig.me`. If they differ and SSH to the jump host times out, set the other address. A changed rule admits the first connection after about 90 seconds. + `stackit_service_account_key_path` is optional. Without it the provider reads `STACKIT_SERVICE_ACCOUNT_KEY_PATH` or the credentials file. `run_command_service_account_email` is optional. Set it to give an existing service account the role for Run Command. Without it Terraform creates a service account named `run_command_service_account_name` (default `run-command`) in the project. @@ -188,7 +190,7 @@ terraform apply $(terraform output -raw ssh_command) ``` -The command has the form `ssh -J ubuntu@:2222 ubuntu@`. Both machines get `ssh_public_key` for the default user of the image through a key pair. `ssh_user` changes the user name in the outputs for an image with another default user. +The command has the form `ssh -J ubuntu@:2222 ubuntu@`. A timeout on port 2222 means that the security group of the jump host dropped the connection; see `admin_cidr` in [Configure](#1-configure). Both machines get `ssh_public_key` for the default user of the image through a key pair. `ssh_user` changes the user name in the outputs for an image with another default user. The key pair belongs to the service account that runs Terraform, not to the project. Its name is unique for that account, so a second deployment with the same service account needs another `name_prefix`. diff --git a/examples/iaas-server-agent-restricted-egress/terraform.tfvars.example b/examples/iaas-server-agent-restricted-egress/terraform.tfvars.example index 794a5ca..c6aed3b 100644 --- a/examples/iaas-server-agent-restricted-egress/terraform.tfvars.example +++ b/examples/iaas-server-agent-restricted-egress/terraform.tfvars.example @@ -13,7 +13,8 @@ stackit_admin_email = "firstname.lastname@example.com" # STACKIT_SERVICE_ACCOUNT_KEY_PATH environment variable or the credentials file. # stackit_service_account_key_path = "/path/to/stackit-sa.json" -# Only this IPv4 range reaches the jump host, on TCP 2222. +# Only this IPv4 range reaches the jump host, on TCP 2222. It must hold the +# address STACKIT sees. If SSH times out, see admin_cidr in the README. admin_cidr = "198.51.100.10/32" # Public half of your SSH key - never the private key. From 11d8463ac7541d0172f232cea63d96f50c8fd60f Mon Sep 17 00:00:00 2001 From: Jan Marco Schreier Date: Thu, 15 Oct 2026 17:20:25 +0200 Subject: [PATCH 29/38] docs(iaas): check the data source without SSH and tighten the wording --- AGENTS.md | 2 +- .../020-variables.tf | 2 +- .../100-outputs.tf | 7 +++- .../README.md | 37 ++++++++++++------- 4 files changed, 30 insertions(+), 18 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index e239129..9af977b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -119,7 +119,7 @@ Fetch only the files relevant to the task. A typical example contains - **`iaas-k3s-self-managed`** `[iaas, k3s, kubernetes, self-managed]` Deploys a 6-node k3s + Cilium cluster on STACKIT VMs and validates: L4 NLB, L7 ALB, CSI block storage, Cilium NetworkPolicy, and egress connectivity - **`iaas-server-agent-restricted-egress`** `[iaas, compute, networking, sna, security-group, egress, cloud-init, server-agent, jump-host, ssh]` - Deploys a Linux VM whose security group allows outbound traffic to an allow-list only, with the STACKIT Server Agent switched on, and a jump host as the only way to reach it over SSH. The example allows the two outbound destinations that cloud-init, the Server Agent and Run Command need, removes the default egress rules that Terraform cannot manage, and adds a probe that logs the destination of every new outbound connection once cloud-init has loaded it, so you can check and extend the list + Deploys a Linux VM whose security group allows outbound traffic to an allow-list only, with the STACKIT Server Agent switched on, and a jump host as the only way to reach it over SSH. The example adds the two egress rules that cloud-init, the Server Agent and Run Command need: DHCP and the metadata service. It removes the default egress rules, which Terraform cannot manage. From the moment cloud-init loads it, a probe logs the destination of every new outbound connection, so you can check and extend the list - **`iaas-volume-encryption`** `[iaas, volume, encryption, kms, block-storage]` KMS & IaaS Resources to deploy a encrypted Block Storage Volume - **`iaas-windows-byol-stackit-migration`** `[iaas, windows, byol, migration, compute]` diff --git a/examples/iaas-server-agent-restricted-egress/020-variables.tf b/examples/iaas-server-agent-restricted-egress/020-variables.tf index 3be628d..03dcd4d 100644 --- a/examples/iaas-server-agent-restricted-egress/020-variables.tf +++ b/examples/iaas-server-agent-restricted-egress/020-variables.tf @@ -109,7 +109,7 @@ variable "nameservers" { # --- access -------------------------------------------------------------- variable "admin_cidr" { - description = "IPv4 source range that reaches the jump host on TCP 2222, for example your public IP with a /32 suffix. It must hold the address STACKIT sees; a connection with several public IPv4 addresses can show another one to STACKIT than to an IP lookup service." + description = "IPv4 source range that reaches the jump host on TCP 2222, for example your public IP with a /32 suffix. It must hold the address that STACKIT sees. An internet line with several public IPv4 addresses can show STACKIT a different address from the one an IP lookup service shows." type = string validation { diff --git a/examples/iaas-server-agent-restricted-egress/100-outputs.tf b/examples/iaas-server-agent-restricted-egress/100-outputs.tf index dbd7ce3..968ffd5 100644 --- a/examples/iaas-server-agent-restricted-egress/100-outputs.tf +++ b/examples/iaas-server-agent-restricted-egress/100-outputs.tf @@ -78,14 +78,17 @@ output "verify_commands" { # Jump host: the only listener outside the loopback addresses is TCP ${local.jump_ssh_port}. ${local.ssh_jump} 'ss -Htln' - # Target: cloud-init finished its first boot without an error. "detail" names DataSourceOpenStackLocal; DataSourceNone means the VM reached no metadata service. + # Target: cloud-init finished its first boot without an error. "detail" names DataSourceOpenStackLocal. ${local.ssh_target} 'cloud-init status --long' + # Target, without SSH: the serial console names the data source of the first boot. DataSourceNone means that cloud-init reached no metadata service. + stackit server log ${stackit_server.target.server_id} --project-id ${local.project_id} --length 5000 | grep -a -E 'No active metadata service found|Datasource DataSource' + # Target: the unit of the STACKIT Server Agent is active. ${local.ssh_target} 'systemctl is-active stackit-server-agent.service' # Target: a destination outside the default allow-list, given as an address so the check needs no DNS. - # "restricted": curl gets no connection, prints 000 and ends with curl error 28 after 10 seconds. "open": curl prints the HTTP status. + # "restricted": curl gets no connection, reports curl error 28 after 10 seconds and then prints 000. "open": curl prints the HTTP status. ${local.ssh_target} 'curl -sS -m 10 -o /dev/null -w "%%{http_code}\n" https://1.1.1.1' # Target: destinations of the outbound connections the probe logged, over all boots. diff --git a/examples/iaas-server-agent-restricted-egress/README.md b/examples/iaas-server-agent-restricted-egress/README.md index a4a4dad..9f80d4f 100644 --- a/examples/iaas-server-agent-restricted-egress/README.md +++ b/examples/iaas-server-agent-restricted-egress/README.md @@ -2,7 +2,7 @@ # Restricted Egress for a VM with cloud-init and the STACKIT Server Agent -Deploys a Linux VM whose security group allows outbound traffic to an allow-list only, with the STACKIT Server Agent switched on, and a jump host as the only way to reach it over SSH. The example allows the two outbound destinations that cloud-init, the Server Agent and Run Command need, removes the default egress rules that Terraform cannot manage, and adds a probe that logs the destination of every new outbound connection once cloud-init has loaded it, so you can check and extend the list. +Deploys a Linux VM whose security group allows outbound traffic to an allow-list only, with the STACKIT Server Agent switched on, and a jump host as the only way to reach it over SSH. The example adds the two egress rules that cloud-init, the Server Agent and Run Command need: DHCP and the metadata service. It removes the default egress rules, which Terraform cannot manage. From the moment cloud-init loads it, a probe logs the destination of every new outbound connection, so you can check and extend the list. ## What the example shows @@ -56,16 +56,16 @@ The jump host is not the subject of the egress test. Its security group keeps th `egress_rules` holds every egress rule this example adds to the target's security group. Its default holds the two rules that cloud-init, the Server Agent and Run Command need: -| Key | Destination | Protocol and port | What needs it | -| ---------- | -------------------- | ----------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `dhcp` | `0.0.0.0/0` | UDP 67 | The interface gets its IPv4 address, the gateway and the nameservers through DHCP. The first request goes to the broadcast address, the renewal of the lease to the DHCP server of the network. | -| `metadata` | `169.254.169.254/32` | TCP 80 | [cloud-init reads metadata, user data and vendor data](https://docs.cloud-init.io/en/latest/explanation/boot.html). The vendor data installs the Server Agent from `http://169.254.169.254/stackit/v1/server-agent/amd64/deb` ([Install the Server Agent on a running server](https://docs.stackit.cloud/products/compute-engine/server-agent/getting-started/installing-server-agent-existing-server/)). The Server Agent and `stackit-server-monitoring-agent`, which comes with it, open connections to this address only. Run Command needs no further rule: the agent receives its commands through the metadata service. | +| Key | Destination | Protocol and port | What needs it | +| ---------- | -------------------- | ----------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `dhcp` | `0.0.0.0/0` | UDP 67 | The interface gets its IPv4 address, the gateway and the nameservers through DHCP. The VM sends its first request to the broadcast address and renews the lease at the DHCP server of the network, both to UDP port 67 ([RFC 2131](https://www.rfc-editor.org/rfc/rfc2131), sections 3.1, 4.1 and 4.4.5). The rule allows UDP 67 to every IPv4 address, so a process on the target can send datagrams to port 67 of any host on the internet as well. | +| `metadata` | `169.254.169.254/32` | TCP 80 | [cloud-init reads metadata, user data and vendor data](https://docs.cloud-init.io/en/latest/explanation/boot.html). The vendor data installs the Server Agent at the first boot. The [guide for a running server](https://docs.stackit.cloud/products/compute-engine/server-agent/getting-started/installing-server-agent-existing-server/) loads the same package from `http://169.254.169.254/stackit/v1/server-agent/amd64/deb`. The Server Agent connects to this address only. Run Command needs no further rule: the Server Agent receives its commands through the metadata service. `stackit-server-monitoring-agent`, which comes with the Server Agent, sends its metrics to `http://169.254.169.254/stackit/server-management-service/v1/agent/monitoring/metrics`. | -The default egress rules of a new security group allow DHCP like all other traffic. Once they are gone, the group drops DHCP as well, unless a rule allows it. Without the rule `dhcp` the interface gets no IPv4 address. cloud-init then finds no metadata service and falls back to `DataSourceNone`, the platform does not install the Server Agent, and Run Command answers `404 agent not found`. +The default egress rules of a new security group allow DHCP like all other traffic. Once they are gone, the group drops DHCP unless a rule allows it. Without the rule `dhcp` the interface gets no IPv4 address. cloud-init then reaches no metadata service and falls back to `DataSourceNone`. The platform does not install the Server Agent, and `stackit server command create` fails with `404 Not Found` and the message `agent not found`. `ip_range` of a rule takes a CIDR or one of two names. `nameservers` stands for every address in the variable `nameservers`, which also configures the network. `stackit` stands for every IPv4 range the data source `stackit_public_ip_ranges` returns. A security group rule takes one range, so Terraform creates one rule per range. [`scripts/check-stackit-ip.sh`](../../scripts/check-stackit-ip.sh) tells for any address whether it belongs to the ranges. -The default has no rule for DNS, NTP or the package mirrors of the image. Name resolution, time synchronisation and package updates of the operating system fail on the target until you add rules for them; `systemd-timesyncd` does not synchronise the clock. The list has no IPv6 rule; the network has no IPv6 prefix. Add what your machine needs through `egress_rules`. The section [Measure what a VM needs](#measure-what-a-vm-needs) shows how to find it. +The default has no rule for DNS, NTP or the package mirrors of the image. Until you add rules for them, the target resolves no names, does not synchronise its clock and installs no package updates of the operating system. The list has no IPv6 rule; the network has no IPv6 prefix. Add what your machine needs through `egress_rules`. The section [Measure what a VM needs](#measure-what-a-vm-needs) shows how to find it. ## How the default egress rules are removed @@ -169,7 +169,7 @@ Replace every placeholder. These variables have no default: - `ssh_public_key`: your complete public SSH key - `image_id`: see above -`admin_cidr` must hold the address STACKIT sees as the source of your SSH connection. Some internet connections leave through more than one public IPv4 address, depending on the destination, and an IP lookup service then shows another address than the one STACKIT sees. Compare two services, for example `curl -4 https://api.ipify.org` and `curl -4 https://ifconfig.me`. If they differ and SSH to the jump host times out, set the other address. A changed rule admits the first connection after about 90 seconds. +`admin_cidr` must hold the address that STACKIT sees as the source of your SSH connection. Some internet lines use more than one public IPv4 address and pick one per destination. An IP lookup service then shows a different address from the one STACKIT sees. Compare two services, for example `curl -4 https://api.ipify.org` and `curl -4 https://ifconfig.me`. If they differ and SSH to the jump host times out, set the other address. A changed rule admits connections about 85 seconds after `terraform apply` has finished. `stackit_service_account_key_path` is optional. Without it the provider reads `STACKIT_SERVICE_ACCOUNT_KEY_PATH` or the credentials file. @@ -190,13 +190,13 @@ terraform apply $(terraform output -raw ssh_command) ``` -The command has the form `ssh -J ubuntu@:2222 ubuntu@`. A timeout on port 2222 means that the security group of the jump host dropped the connection; see `admin_cidr` in [Configure](#1-configure). Both machines get `ssh_public_key` for the default user of the image through a key pair. `ssh_user` changes the user name in the outputs for an image with another default user. +The command has the form `ssh -J ubuntu@:2222 ubuntu@`. If SSH to port 2222 times out, check that `admin_cidr` holds the address STACKIT sees, as described in [Configure](#1-configure). Both machines get `ssh_public_key` for the default user of the image through a key pair. `ssh_user` changes the user name in the outputs for an image with another default user. The key pair belongs to the service account that runs Terraform, not to the project. Its name is unique for that account, so a second deployment with the same service account needs another `name_prefix`. ## Verify -`terraform output -raw verify_commands` prints the following commands with the addresses and IDs of your deployment. +`terraform output -raw verify_commands` prints the commands of this section and the probe report from [Measure what a VM needs](#measure-what-a-vm-needs) with the addresses and IDs of your deployment. **Jump host.** `ss -Htln` lists the listening TCP sockets. Port 2222 is the only one that is not bound to a loopback address: @@ -204,19 +204,28 @@ The key pair belongs to the service account that runs Terraform, not to the proj ssh -p 2222 ubuntu@ 'ss -Htln' ``` -**cloud-init on the target.** `status: done` means the first boot finished. `cloud-init status` exits with a status other than 0 when a stage reported an error. `cloud-init status --wait` blocks until cloud-init finished. The line `detail` names `DataSourceOpenStackLocal`. `DataSourceNone` there means that cloud-init reached no metadata service, for example because the rule `dhcp` is missing: +**cloud-init on the target.** `status: done` means the first boot finished. `cloud-init status` exits with a status other than 0 when a stage reported an error. `cloud-init status --wait` blocks until cloud-init finished. The line `detail` names `DataSourceOpenStackLocal`: ```bash ssh -J ubuntu@:2222 ubuntu@ 'cloud-init status --long' ``` +**cloud-init on the target, without SSH.** Without the rule `dhcp` the target gets neither an IPv4 address nor the SSH key, and SSH to it fails. The serial console of the target shows the data source in every case: + +```bash +stackit server log --project-id --length 5000 \ + | grep -a -E 'No active metadata service found|Datasource DataSource' +``` + +A working target prints `Datasource DataSourceOpenStackLocal`. `No active metadata service found` followed by `Datasource DataSourceNone` means that cloud-init reached no metadata service. + **Server Agent on the target.** The [installation guide](https://docs.stackit.cloud/products/compute-engine/server-agent/getting-started/installing-server-agent-api/) names the unit: ```bash ssh -J ubuntu@:2222 ubuntu@ 'systemctl is-active stackit-server-agent.service' ``` -**A destination outside the allow-list.** This check shows that the restriction is in effect. The request goes to the address `1.1.1.1` directly, so it needs no DNS, and no rule of the default allow-list covers it. With `egress_mode = "restricted"` curl gets no connection: it prints `000` and ends with `curl: (28)`, the timeout of a connection that got no answer, after 10 seconds. With `egress_mode = "open"` curl prints an HTTP status. An HTTP status on a restricted target shows that a rule allows the connection, for example a default egress rule that is still in place: +**A destination outside the allow-list.** This check shows that the restriction is in effect. The request goes to the address `1.1.1.1` directly and needs no DNS. No rule of the default allow-list covers it. With `egress_mode = "restricted"` curl gets no connection. After 10 seconds it reports `curl: (28)`, the timeout of a connection that got no answer, and prints `000`. With `egress_mode = "open"` curl prints an HTTP status. An HTTP status on a restricted target shows that a rule allows the connection, for example a default egress rule that is still in place: ```bash ssh -J ubuntu@:2222 ubuntu@ \ @@ -260,7 +269,7 @@ STACKIT_CLI_PROFILE=run-command stackit server command create --project-id --project-id --server-id ``` -The command reached the agent when its status is `completed` and its exit code is 0. `403 Forbidden` from `create` means that the active identity lacks `run-command.command.create` on the project. +The command reached the agent when its status is `completed` and its exit code is 0. `403 Forbidden` from `create` means that the active identity lacks `run-command.command.create` on the project. `404 Not Found` with the message `agent not found` means that no Server Agent has registered for the target. A new target gives this answer until its Server Agent registers, about two minutes after the server is created. If the answer stays, the serial console shows whether cloud-init reached the metadata service. ## Measure what a VM needs @@ -276,7 +285,7 @@ ssh -J ubuntu@:2222 ubuntu@ 'sudo egress-probe-report' COUNT DESTINATION PROTO PORT USER ``` -`COUNT` is the number of new connections, `USER` the owner of the socket. Arguments go to `journalctl`: `sudo egress-probe-report --boot` covers the current boot only. The report lists neither DHCP nor the requests cloud-init sends before the probe loads. The rules `dhcp` and `metadata` cover both. +`COUNT` is the number of new connections, `USER` the owner of the socket. Arguments go to `journalctl`: `sudo egress-probe-report --boot` covers the current boot only. During boot the report lists neither the DHCP exchange nor the requests cloud-init sends before the probe loads. The rules `dhcp` and `metadata` cover both. The user data of the target is this boothook and no cloud-config. cloud-init [merges the cloud-config of the user over the cloud-config of the vendor data](https://docs.cloud-init.io/en/latest/explanation/vendordata.html). A boothook merges with nothing, so the user data of this example overrides nothing the platform delivers as vendor data. From 3b1e0b8b8a35102ff838297ba3eb27ce8f030d7d Mon Sep 17 00:00:00 2001 From: Jan Marco Schreier Date: Fri, 16 Oct 2026 11:51:58 +0200 Subject: [PATCH 30/38] refactor(iaas): drop the egress probe and the open mode --- AGENTS.md | 2 +- .../020-variables.tf | 13 +- .../060-security-groups.tf | 6 - .../070-default-egress-rules.tf | 12 +- .../080-cloud-init.tf | 14 -- .../090-servers.tf | 10 -- .../100-outputs.tf | 10 +- .../README.md | 124 ++++++------------ .../files/egress-probe-boothook.sh | 102 -------------- .../terraform.tfvars.example | 4 - 10 files changed, 41 insertions(+), 256 deletions(-) delete mode 100755 examples/iaas-server-agent-restricted-egress/files/egress-probe-boothook.sh diff --git a/AGENTS.md b/AGENTS.md index 9af977b..a5b369c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -119,7 +119,7 @@ Fetch only the files relevant to the task. A typical example contains - **`iaas-k3s-self-managed`** `[iaas, k3s, kubernetes, self-managed]` Deploys a 6-node k3s + Cilium cluster on STACKIT VMs and validates: L4 NLB, L7 ALB, CSI block storage, Cilium NetworkPolicy, and egress connectivity - **`iaas-server-agent-restricted-egress`** `[iaas, compute, networking, sna, security-group, egress, cloud-init, server-agent, jump-host, ssh]` - Deploys a Linux VM whose security group allows outbound traffic to an allow-list only, with the STACKIT Server Agent switched on, and a jump host as the only way to reach it over SSH. The example adds the two egress rules that cloud-init, the Server Agent and Run Command need: DHCP and the metadata service. It removes the default egress rules, which Terraform cannot manage. From the moment cloud-init loads it, a probe logs the destination of every new outbound connection, so you can check and extend the list + Deploys a Linux VM whose security group allows outbound traffic to an allow-list only, with the STACKIT Server Agent switched on, and a jump host as the only way to reach it over SSH. The example adds the two egress rules that cloud-init, the Server Agent and Run Command need: DHCP and the metadata service. It removes the default egress rules, which Terraform cannot manage - **`iaas-volume-encryption`** `[iaas, volume, encryption, kms, block-storage]` KMS & IaaS Resources to deploy a encrypted Block Storage Volume - **`iaas-windows-byol-stackit-migration`** `[iaas, windows, byol, migration, compute]` diff --git a/examples/iaas-server-agent-restricted-egress/020-variables.tf b/examples/iaas-server-agent-restricted-egress/020-variables.tf index 03dcd4d..fa5b157 100644 --- a/examples/iaas-server-agent-restricted-egress/020-variables.tf +++ b/examples/iaas-server-agent-restricted-egress/020-variables.tf @@ -146,17 +146,6 @@ variable "run_command_service_account_name" { # --- outbound traffic of the target -------------------------------------- -variable "egress_mode" { - description = "\"restricted\" removes the two default egress rules from the target's security group, so only egress_rules allows outbound traffic. \"open\" keeps the default rules, so all outbound traffic passes; use it to record what the machine contacts. A change replaces the target's security group, network interface and server." - type = string - default = "restricted" - - validation { - condition = contains(["restricted", "open"], var.egress_mode) - error_message = "egress_mode must be \"restricted\" or \"open\"." - } -} - variable "egress_rules" { description = "Outbound allow-list of the target's security group. Each entry becomes one IPv4 egress rule per address range. ip_range takes a CIDR or one of two names: \"nameservers\" stands for every address in var.nameservers, \"stackit\" for every IPv4 range in data.stackit_public_ip_ranges. Leave port_min and port_max unset for a protocol without ports, such as icmp. The default holds the two rules cloud-init, the Server Agent and Run Command need: DHCP and the metadata service." type = map(object({ @@ -222,7 +211,7 @@ variable "ssh_user" { } variable "image_id" { - description = "Image UUID of both machines. The cloud-init configuration is written for the public image \"Ubuntu 24.04\" (amd64): it relies on nftables and on sshd socket activation. Look the ID up with: stackit image list --all --project-id " + description = "Image UUID of both machines. The cloud-init configuration is written for the public image \"Ubuntu 24.04\" (amd64): the jump host relies on sshd socket activation. Look the ID up with: stackit image list --all --project-id " type = string validation { diff --git a/examples/iaas-server-agent-restricted-egress/060-security-groups.tf b/examples/iaas-server-agent-restricted-egress/060-security-groups.tf index ce0b8bb..983fcf5 100644 --- a/examples/iaas-server-agent-restricted-egress/060-security-groups.tf +++ b/examples/iaas-server-agent-restricted-egress/060-security-groups.tf @@ -54,12 +54,6 @@ resource "stackit_security_group" "target" { name = "${var.name_prefix}-target" description = "Target: inbound SSH from the jump host's group, outbound traffic from an allow-list" stateful = true - - # A deleted default egress rule does not come back. A change of egress_mode - # therefore creates a new group, which starts with its two default rules. - lifecycle { - replace_triggered_by = [terraform_data.egress_mode] - } } resource "stackit_security_group_rule" "target_ssh" { diff --git a/examples/iaas-server-agent-restricted-egress/070-default-egress-rules.tf b/examples/iaas-server-agent-restricted-egress/070-default-egress-rules.tf index 64af77d..8eed1fa 100644 --- a/examples/iaas-server-agent-restricted-egress/070-default-egress-rules.tf +++ b/examples/iaas-server-agent-restricted-egress/070-default-egress-rules.tf @@ -21,19 +21,11 @@ # its rules, and the data source does not list them. This file removes them # with two calls against the IaaS API. -# Holds the mode, so that other resources can react to a change of it. The -# target's security group, network interface and server are replaced when it -# changes: the new group starts with its default rules again, and the new -# server boots for the first time under the rule set of the new mode. -resource "terraform_data" "egress_mode" { - input = var.egress_mode -} - # The token of the service account the provider is configured with. An # ephemeral resource is not written to the plan or the state. ephemeral "stackit_access_token" "iaas" {} -# Runs once per security group, in mode "restricted" only. The script deletes +# Runs once per security group. The script deletes # every egress rule that has no protocol, no port range, no IP range and no # remote security group, and fails unless the group is then free of egress # rules that allow all traffic. It needs bash, curl and jq on the machine that @@ -43,8 +35,6 @@ ephemeral "stackit_access_token" "iaas" {} # token. When the script fails, the error message of Terraform contains the # output. resource "terraform_data" "remove_default_egress_rules" { - count = var.egress_mode == "restricted" ? 1 : 0 - triggers_replace = stackit_security_group.target.security_group_id provisioner "local-exec" { diff --git a/examples/iaas-server-agent-restricted-egress/080-cloud-init.tf b/examples/iaas-server-agent-restricted-egress/080-cloud-init.tf index 25f579f..cdfeda3 100644 --- a/examples/iaas-server-agent-restricted-egress/080-cloud-init.tf +++ b/examples/iaas-server-agent-restricted-egress/080-cloud-init.tf @@ -52,18 +52,4 @@ locals { ] }), ]) - - # --- target ------------------------------------------------------------ - # - # The user data of the target is a single boothook and no cloud-config. - # cloud-init merges the cloud-config of the user over the cloud-config of - # the vendor data: where both set a key, the value of the user wins. The - # platform installs the Server Agent through vendor data. A boothook is a - # script of its own and merges with nothing, so this user data overrides - # nothing the platform delivers. A boothook is also the earliest point at - # which user data runs. - target_user_data = join("\n", [ - "#cloud-boothook", - file("${path.module}/files/egress-probe-boothook.sh"), - ]) } diff --git a/examples/iaas-server-agent-restricted-egress/090-servers.tf b/examples/iaas-server-agent-restricted-egress/090-servers.tf index 7ee281e..73854b9 100644 --- a/examples/iaas-server-agent-restricted-egress/090-servers.tf +++ b/examples/iaas-server-agent-restricted-egress/090-servers.tf @@ -77,11 +77,6 @@ resource "stackit_network_interface" "target" { stackit_security_group_rule.target_egress, stackit_security_group_rule.target_ssh, ] - - # Terraform deletes the interface before the security group it belongs to. - lifecycle { - replace_triggered_by = [terraform_data.egress_mode] - } } resource "stackit_server" "target" { @@ -91,7 +86,6 @@ resource "stackit_server" "target" { machine_type = var.target_machine_type keypair_name = stackit_key_pair.admin.name network_interfaces = [stackit_network_interface.target.network_interface_id] - user_data = local.target_user_data # A server created from an image gets a new boot volume. A replaced server # therefore starts from the unchanged image and runs its first boot again. @@ -112,8 +106,4 @@ resource "stackit_server" "target" { stackit_security_group_rule.target_egress, stackit_security_group_rule.target_ssh, ] - - lifecycle { - replace_triggered_by = [terraform_data.egress_mode] - } } diff --git a/examples/iaas-server-agent-restricted-egress/100-outputs.tf b/examples/iaas-server-agent-restricted-egress/100-outputs.tf index 968ffd5..d587c41 100644 --- a/examples/iaas-server-agent-restricted-egress/100-outputs.tf +++ b/examples/iaas-server-agent-restricted-egress/100-outputs.tf @@ -27,11 +27,6 @@ output "project_id" { value = local.project_id } -output "egress_mode" { - description = "Rule set the target booted with: \"restricted\" or \"open\"." - value = var.egress_mode -} - output "jump_server_id" { description = "Server ID of the jump host." value = stackit_server.jump.server_id @@ -88,12 +83,9 @@ output "verify_commands" { ${local.ssh_target} 'systemctl is-active stackit-server-agent.service' # Target: a destination outside the default allow-list, given as an address so the check needs no DNS. - # "restricted": curl gets no connection, reports curl error 28 after 10 seconds and then prints 000. "open": curl prints the HTTP status. + # curl gets no connection, reports curl error 28 after 10 seconds and then prints 000. ${local.ssh_target} 'curl -sS -m 10 -o /dev/null -w "%%{http_code}\n" https://1.1.1.1' - # Target: destinations of the outbound connections the probe logged, over all boots. - ${local.ssh_target} 'sudo egress-probe-report' - # Rules of both security groups as the IaaS API reports them. stackit security-group rule list --project-id ${local.project_id} --security-group-id ${stackit_security_group.jump.security_group_id} stackit security-group rule list --project-id ${local.project_id} --security-group-id ${stackit_security_group.target.security_group_id} diff --git a/examples/iaas-server-agent-restricted-egress/README.md b/examples/iaas-server-agent-restricted-egress/README.md index 9f80d4f..6a6c308 100644 --- a/examples/iaas-server-agent-restricted-egress/README.md +++ b/examples/iaas-server-agent-restricted-egress/README.md @@ -2,7 +2,7 @@ # Restricted Egress for a VM with cloud-init and the STACKIT Server Agent -Deploys a Linux VM whose security group allows outbound traffic to an allow-list only, with the STACKIT Server Agent switched on, and a jump host as the only way to reach it over SSH. The example adds the two egress rules that cloud-init, the Server Agent and Run Command need: DHCP and the metadata service. It removes the default egress rules, which Terraform cannot manage. From the moment cloud-init loads it, a probe logs the destination of every new outbound connection, so you can check and extend the list. +Deploys a Linux VM whose security group allows outbound traffic to an allow-list only, with the STACKIT Server Agent switched on, and a jump host as the only way to reach it over SSH. The example adds the two egress rules that cloud-init, the Server Agent and Run Command need: DHCP and the metadata service. It removes the default egress rules, which Terraform cannot manage. ## What the example shows @@ -14,7 +14,7 @@ The example creates: - a STACKIT Network Area (SNA) with its regional configuration, a project attached to it and one network - a **jump host** with a public IP, whose security group admits TCP 2222 from `admin_cidr` and whose sshd listens on that port -- a **target** without a public IP, with the Server Agent, an egress allow-list and a probe that logs every new outbound connection once cloud-init has loaded it +- a **target** without a public IP, with the Server Agent and an egress allow-list - a step that deletes the two default egress rules of the target's security group through the IaaS API - the role `run-command.editor` on the project for a service account, either an existing one or one that Terraform creates, so that this account can send Run Command to the target @@ -27,7 +27,7 @@ flowchart LR subgraph net["STACKIT Network Area, project, network 10.80.1.0/24"] direction TB jump["Jump host\npublic IP\nsshd on TCP 2222\nno Server Agent"] - target["Target\nno public IP\nServer Agent\negress probe"] + target["Target\nno public IP\nServer Agent"] end dhcp(["DHCP server\nof the network"]) @@ -50,7 +50,7 @@ Nothing else reaches the jump host from outside, and nothing but the jump host r sshd on the jump host listens on port 2222 itself. No port forwarding sits in front of it, and port 22 is closed. It accepts public keys only. TCP forwarding stays on, because `ssh -J` opens the connection to the target through it. The private key never leaves your machine. -The jump host is not the subject of the egress test. Its security group keeps the two default egress rules. +The allow-list applies to the target only. The security group of the jump host keeps the two default egress rules. ## Egress rules of the target @@ -65,7 +65,33 @@ The default egress rules of a new security group allow DHCP like all other traff `ip_range` of a rule takes a CIDR or one of two names. `nameservers` stands for every address in the variable `nameservers`, which also configures the network. `stackit` stands for every IPv4 range the data source `stackit_public_ip_ranges` returns. A security group rule takes one range, so Terraform creates one rule per range. [`scripts/check-stackit-ip.sh`](../../scripts/check-stackit-ip.sh) tells for any address whether it belongs to the ranges. -The default has no rule for DNS, NTP or the package mirrors of the image. Until you add rules for them, the target resolves no names, does not synchronise its clock and installs no package updates of the operating system. The list has no IPv6 rule; the network has no IPv6 prefix. Add what your machine needs through `egress_rules`. The section [Measure what a VM needs](#measure-what-a-vm-needs) shows how to find it. +The default has no rule for DNS, NTP or the package mirrors of the image. Until you add rules for them, the target resolves no names, does not synchronise its clock and installs no package updates of the operating system. The list has no IPv6 rule; the network has no IPv6 prefix. Add what your machine needs through `egress_rules`. Setting the variable replaces the default, so keep the two default entries: + +```hcl +egress_rules = { + # ... the two default entries from 020-variables.tf ... + dns-udp = { + description = "DNS: name resolution through the nameservers of the network" + protocol = "udp" + port_min = 53 + port_max = 53 + ip_range = "nameservers" + } + ntp = { + description = "NTP: time synchronisation" + protocol = "udp" + port_min = 123 + port_max = 123 + ip_range = "0.0.0.0/0" + } +} +``` + +A change of `egress_rules` adds or deletes rules on the running machine. To boot the target for the first time under the changed list, replace the server: + +```bash +terraform apply -replace=stackit_server.target +``` ## How the default egress rules are removed @@ -84,17 +110,6 @@ The network interface of the target and the target itself depend on this step an Terraform hides the output of the script, because its environment holds the token. When the script fails, the error message of Terraform contains the output. A second run changes nothing: the script finds no default rule and only checks. -### Two modes - -`egress_mode` selects the rule set the target boots with: - -| `egress_mode` | Default egress rules | Effect | -| ---------------------- | -------------------- | ------------------------------------------------------------------------- | -| `restricted` (default) | removed | Only `egress_rules` allows outbound traffic. | -| `open` | kept | All outbound traffic passes. The probe records what the machine contacts. | - -A deleted default rule does not come back. A change of `egress_mode` therefore replaces the target's security group, its network interface and the server. The new group starts with its default rules, and the new server boots for the first time under the rule set of the new mode. The jump host stays as it is. - ## Who may send Run Command The Run Command API checks the permission `run-command.command.create` on the project. It answers `403 Forbidden` to an identity without that permission, even when that identity created the project and every resource in it. @@ -122,18 +137,17 @@ The output `run_command_service_account_email` names the account in both cases. | File | Contents | | ---------------------------------------- | ------------------------------------------------------------------------------------------------------------ | | `010-provider.tf` | provider, with `enable_beta_resources` for the access token and the experiment `iam` for the role assignment | -| `020-variables.tf` | all inputs, including `egress_rules` and `egress_mode` | +| `020-variables.tf` | all inputs, including `egress_rules` | | `030-network-area.tf` | network area and its regional configuration | | `040-project.tf` | project with the `networkArea` label | | `045-run-command-access.tf` | the service account for Run Command and its role on the project | | `050-network.tf` | the network of both machines | | `060-security-groups.tf` | both security groups, the two ingress rules and the egress allow-list | -| `070-default-egress-rules.tf` | the mode switch and the removal of the default egress rules | -| `080-cloud-init.tf` | user data of both machines | +| `070-default-egress-rules.tf` | the removal of the default egress rules | +| `080-cloud-init.tf` | user data of the jump host | | `090-servers.tf` | key pair, network interfaces, public IP and both servers | | `100-outputs.tf` | IDs, addresses, the Run Command service account, the SSH command and the verify commands | | `scripts/remove-default-egress-rules.sh` | runs on the machine that runs Terraform | -| `files/egress-probe-boothook.sh` | runs on the target at every boot and installs the probe | ## Usage @@ -145,7 +159,7 @@ The output `run_command_service_account_email` names the account in both cases. - For the Run Command check: an identity that may enable the Run Command service on the project, and a key of the service account that holds `run-command.editor`. See [Who may send Run Command](#who-may-send-run-command). - `bash`, `jq` and `curl` 7.55.0 or later on the machine that runs Terraform. - An SSH key pair and an OpenSSH client with `-J` (ProxyJump). -- The ID of the public image `Ubuntu 24.04` (amd64). The user data relies on two properties of that image: it ships `nft`, and it starts sshd through socket activation. +- The ID of the public image `Ubuntu 24.04` (amd64). The user data of the jump host relies on one property of that image: it starts sshd through socket activation. - An authenticated [`stackit` CLI](https://github.com/stackitcloud/stackit-cli) for the image lookup and for the checks that read the API. STACKIT replaces its public images with new builds and [deletes a released image after one year](https://docs.stackit.cloud/products/compute-engine/linux-server/faq/), so `image_id` has no default: @@ -196,7 +210,7 @@ The key pair belongs to the service account that runs Terraform, not to the proj ## Verify -`terraform output -raw verify_commands` prints the commands of this section and the probe report from [Measure what a VM needs](#measure-what-a-vm-needs) with the addresses and IDs of your deployment. +`terraform output -raw verify_commands` prints the commands of this section with the addresses and IDs of your deployment. **Jump host.** `ss -Htln` lists the listening TCP sockets. Port 2222 is the only one that is not bound to a loopback address: @@ -225,7 +239,7 @@ A working target prints `Datasource DataSourceOpenStackLocal`. `No active metada ssh -J ubuntu@:2222 ubuntu@ 'systemctl is-active stackit-server-agent.service' ``` -**A destination outside the allow-list.** This check shows that the restriction is in effect. The request goes to the address `1.1.1.1` directly and needs no DNS. No rule of the default allow-list covers it. With `egress_mode = "restricted"` curl gets no connection. After 10 seconds it reports `curl: (28)`, the timeout of a connection that got no answer, and prints `000`. With `egress_mode = "open"` curl prints an HTTP status. An HTTP status on a restricted target shows that a rule allows the connection, for example a default egress rule that is still in place: +**A destination outside the allow-list.** This check shows that the restriction is in effect. The request goes to the address `1.1.1.1` directly and needs no DNS. No rule of the default allow-list covers it, so curl gets no connection. After 10 seconds it reports `curl: (28)`, the timeout of a connection that got no answer, and prints `000`. An HTTP status shows that a rule allows the connection, for example a default egress rule that is still in place: ```bash ssh -J ubuntu@:2222 ubuntu@ \ @@ -271,70 +285,6 @@ STACKIT_CLI_PROFILE=run-command stackit server command describe --p The command reached the agent when its status is `completed` and its exit code is 0. `403 Forbidden` from `create` means that the active identity lacks `run-command.command.create` on the project. `404 Not Found` with the message `agent not found` means that no Server Agent has registered for the target. A new target gives this answer until its Server Agent registers, about two minutes after the server is created. If the answer stays, the serial console shows whether cloud-init reached the metadata service. -## Measure what a VM needs - -The probe on the target is an nftables rule set that logs every new outbound connection to the kernel log. It changes no verdict and drops nothing. It runs inside the VM, in front of the security group, so it also logs connections the security group drops afterwards. cloud-init loads it as a boothook, the earliest point at which user data runs. cloud-init reads metadata, user data and vendor data before that point, and the probe does not see those requests. - -`egress-probe-report` prints one line per distinct destination: - -```bash -ssh -J ubuntu@:2222 ubuntu@ 'sudo egress-probe-report' -``` - -```text -COUNT DESTINATION PROTO PORT USER -``` - -`COUNT` is the number of new connections, `USER` the owner of the socket. Arguments go to `journalctl`: `sudo egress-probe-report --boot` covers the current boot only. During boot the report lists neither the DHCP exchange nor the requests cloud-init sends before the probe loads. The rules `dhcp` and `metadata` cover both. - -The user data of the target is this boothook and no cloud-config. cloud-init [merges the cloud-config of the user over the cloud-config of the vendor data](https://docs.cloud-init.io/en/latest/explanation/vendordata.html). A boothook merges with nothing, so the user data of this example overrides nothing the platform delivers as vendor data. - -To find the rules a machine needs: - -1. Boot the target without restriction: - - ```bash - terraform apply -var egress_mode=open - ``` - -2. Wait until `cloud-init status --wait` returns on the target. Then use what you rely on: send a Run Command, start an update, let the machine run for a while. -3. Read the report. Every destination in it is a candidate for a rule. `sudo ss -tunp` names the process behind a connection that is open at that moment. `sudo cat /var/lib/cloud/instance/vendor-data.txt` shows what the platform passes to cloud-init as vendor data. -4. Add the rules you decide on to `egress_rules`. Setting the variable replaces the default, so keep the default entries: - - ```hcl - egress_rules = { - # ... the two default entries from 020-variables.tf ... - dns-udp = { - description = "DNS: name resolution through the nameservers of the network" - protocol = "udp" - port_min = 53 - port_max = 53 - ip_range = "nameservers" - } - ntp = { - description = "NTP: time synchronisation" - protocol = "udp" - port_min = 123 - port_max = 123 - ip_range = "0.0.0.0/0" - } - } - ``` - -5. Boot the target with the allow-list and repeat the checks from [Verify](#verify): - - ```bash - terraform apply - ``` - -The report of a restricted target lists the connections the security group dropped as well. A destination that appears there and is missing in `egress_rules` is one the machine tried and did not reach. - -Removing an entry from `egress_rules` shows what stops working without it. A change of `egress_rules` alone adds or deletes rules on the running machine. To see a first boot under the changed list, replace the server: - -```bash -terraform apply -replace=stackit_server.target -``` - ## Clean up ```bash diff --git a/examples/iaas-server-agent-restricted-egress/files/egress-probe-boothook.sh b/examples/iaas-server-agent-restricted-egress/files/egress-probe-boothook.sh deleted file mode 100755 index c676af3..0000000 --- a/examples/iaas-server-agent-restricted-egress/files/egress-probe-boothook.sh +++ /dev/null @@ -1,102 +0,0 @@ -#!/bin/sh -# Copyright 2026 Schwarz Digits Cloud GmbH & Co. KG -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Egress probe of the target. Terraform passes this script to the server as -# cloud-init user data of the type cloud-boothook. cloud-init runs a boothook -# on every boot, during its network stage and before any of its modules. -# -# The probe writes one line to the kernel log for every new outbound -# connection: destination address, protocol, destination port and the user ID -# of the socket. It changes no verdict and drops nothing. -# -# The script writes two files and loads the first one: -# /etc/egress-probe.nft nftables rule set of the probe -# /usr/local/bin/egress-probe-report prints the distinct destinations -# -# cloud-init reads metadata, user data and vendor data from the metadata -# service before it runs this script. The probe does not see those requests. -# -# The script always ends with exit status 0, so a machine without nft still -# finishes cloud-init without an error. The report tells whether the rule set -# is loaded. - -ruleset=/etc/egress-probe.nft -report=/usr/local/bin/egress-probe-report - -cat >"$ruleset" <<'RULESET' -# Egress probe: logs every new outbound connection that leaves the machine. -# The chain has no rule with a verdict and its policy is accept. - -# Declare the table, then delete it: loading this file again replaces the -# table instead of adding the rule a second time. -table inet egress_probe -delete table inet egress_probe - -table inet egress_probe { - chain output { - type filter hook output priority filter; policy accept; - oifname != "lo" ct state new log prefix "egress-probe: " level info flags skuid - } -} -RULESET - -cat >"$report" <<'REPORT' -#!/bin/sh -# Prints one line for every distinct destination in the log of the egress -# probe: number of new connections, destination address, protocol, destination -# port and the user that owns the socket. -# -# Arguments are passed to journalctl. Without arguments the report covers all -# boots; "egress-probe-report --boot" covers the current boot. - -if [ "$(id -u)" -ne 0 ]; then - echo "error: run as root: sudo egress-probe-report" >&2 - exit 1 -fi - -if ! nft list table inet egress_probe >/dev/null 2>&1; then - echo "warning: the rule set /etc/egress-probe.nft is not loaded, the probe logs nothing" >&2 -fi - -journalctl --no-pager --output=cat _TRANSPORT=kernel "$@" | awk ' - /egress-probe: / { - destination = "-"; protocol = "-"; port = "-"; uid = "-" - for (i = 1; i <= NF; i++) { - if ($i ~ /^DST=/) destination = substr($i, 5) - else if ($i ~ /^PROTO=/) protocol = substr($i, 7) - else if ($i ~ /^DPT=/) port = substr($i, 5) - else if ($i ~ /^UID=/) uid = substr($i, 5) - } - connections[destination " " protocol " " port " " uid]++ - } - END { - for (key in connections) print connections[key], key - }' | sort -k2,2 -k3,3 -k4,4n -k5,5 | { - printf '%-6s %-40s %-6s %-6s %s\n' COUNT DESTINATION PROTO PORT USER - while read -r count destination protocol port uid; do - user=$(getent passwd "$uid" | cut -d: -f1) - printf '%-6s %-40s %-6s %-6s %s\n' "$count" "$destination" "$protocol" "$port" "${user:-$uid}" - done -} -REPORT -chmod 0755 "$report" - -if nft -f "$ruleset"; then - echo "egress-probe: rule set loaded" -else - echo "egress-probe: cannot load $ruleset" >&2 -fi - -exit 0 diff --git a/examples/iaas-server-agent-restricted-egress/terraform.tfvars.example b/examples/iaas-server-agent-restricted-egress/terraform.tfvars.example index c6aed3b..9cd0938 100644 --- a/examples/iaas-server-agent-restricted-egress/terraform.tfvars.example +++ b/examples/iaas-server-agent-restricted-egress/terraform.tfvars.example @@ -24,10 +24,6 @@ ssh_public_key = "ssh-ed25519 AAAA... you@example.com" # stackit image list --all --project-id image_id = "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -# "restricted" (default) removes the default egress rules of the target. -# "open" keeps them, see "Measure what a VM needs" in the README. -# egress_mode = "open" - # Existing service account that sends Run Command. Terraform gives it the role # run-command.editor on the project. Unset, Terraform creates one in the # project. Lower case only. From 254f8f0a28178fb3372cfcc067b13bd0988e37e2 Mon Sep 17 00:00:00 2001 From: Jan Marco Schreier Date: Fri, 16 Oct 2026 12:04:28 +0200 Subject: [PATCH 31/38] docs(iaas): align comments and README with the trimmed example --- AGENTS.md | 2 +- .../020-variables.tf | 2 +- .../060-security-groups.tf | 5 +++-- .../070-default-egress-rules.tf | 14 ++++++------- .../README.md | 20 +++++++++---------- 5 files changed, 22 insertions(+), 21 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index a5b369c..76eca97 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -119,7 +119,7 @@ Fetch only the files relevant to the task. A typical example contains - **`iaas-k3s-self-managed`** `[iaas, k3s, kubernetes, self-managed]` Deploys a 6-node k3s + Cilium cluster on STACKIT VMs and validates: L4 NLB, L7 ALB, CSI block storage, Cilium NetworkPolicy, and egress connectivity - **`iaas-server-agent-restricted-egress`** `[iaas, compute, networking, sna, security-group, egress, cloud-init, server-agent, jump-host, ssh]` - Deploys a Linux VM whose security group allows outbound traffic to an allow-list only, with the STACKIT Server Agent switched on, and a jump host as the only way to reach it over SSH. The example adds the two egress rules that cloud-init, the Server Agent and Run Command need: DHCP and the metadata service. It removes the default egress rules, which Terraform cannot manage + Deploys a Linux VM, the target, whose security group allows outbound traffic to an allow-list only. The platform installs the STACKIT Server Agent on the target. A jump host is the only way to reach the target over SSH. The example adds the two egress rules that cloud-init, the Server Agent and Run Command need: DHCP and the metadata service. Every new security group also carries two default egress rules that allow all outbound traffic. The STACKIT provider cannot remove them, so Terraform runs a script that deletes them through the IaaS API - **`iaas-volume-encryption`** `[iaas, volume, encryption, kms, block-storage]` KMS & IaaS Resources to deploy a encrypted Block Storage Volume - **`iaas-windows-byol-stackit-migration`** `[iaas, windows, byol, migration, compute]` diff --git a/examples/iaas-server-agent-restricted-egress/020-variables.tf b/examples/iaas-server-agent-restricted-egress/020-variables.tf index fa5b157..58561a8 100644 --- a/examples/iaas-server-agent-restricted-egress/020-variables.tf +++ b/examples/iaas-server-agent-restricted-egress/020-variables.tf @@ -211,7 +211,7 @@ variable "ssh_user" { } variable "image_id" { - description = "Image UUID of both machines. The cloud-init configuration is written for the public image \"Ubuntu 24.04\" (amd64): the jump host relies on sshd socket activation. Look the ID up with: stackit image list --all --project-id " + description = "Image UUID of both machines. The user data of the jump host expects the public image \"Ubuntu 24.04\" (amd64), whose main sshd configuration reads /etc/ssh/sshd_config.d and sets no port. Look the ID up with: stackit image list --all --project-id " type = string validation { diff --git a/examples/iaas-server-agent-restricted-egress/060-security-groups.tf b/examples/iaas-server-agent-restricted-egress/060-security-groups.tf index 983fcf5..de49049 100644 --- a/examples/iaas-server-agent-restricted-egress/060-security-groups.tf +++ b/examples/iaas-server-agent-restricted-egress/060-security-groups.tf @@ -41,8 +41,9 @@ resource "stackit_security_group_rule" "jump_ssh" { ip_range = var.admin_cidr } -# The jump host is not the subject of the egress test. Its group keeps the two -# default egress rules that STACKIT adds to every new security group. +# The allow-list applies to the target only. The jump host's security group +# keeps the two default egress rules that STACKIT adds to every new security +# group. # --- target -------------------------------------------------------------- # diff --git a/examples/iaas-server-agent-restricted-egress/070-default-egress-rules.tf b/examples/iaas-server-agent-restricted-egress/070-default-egress-rules.tf index 8eed1fa..654b373 100644 --- a/examples/iaas-server-agent-restricted-egress/070-default-egress-rules.tf +++ b/examples/iaas-server-agent-restricted-egress/070-default-egress-rules.tf @@ -18,18 +18,18 @@ # these two rules exist. # # The provider cannot remove them: stackit_security_group has no argument for -# its rules, and the data source does not list them. This file removes them -# with two calls against the IaaS API. +# its rules, and the data source does not list them. This file deletes them +# through the IaaS API with scripts/remove-default-egress-rules.sh. # The token of the service account the provider is configured with. An # ephemeral resource is not written to the plan or the state. ephemeral "stackit_access_token" "iaas" {} -# Runs once per security group. The script deletes -# every egress rule that has no protocol, no port range, no IP range and no -# remote security group, and fails unless the group is then free of egress -# rules that allow all traffic. It needs bash, curl and jq on the machine that -# runs Terraform. +# Runs for the target's security group, and again when Terraform replaces +# that group. The script deletes every egress rule that has no protocol, no +# port range, no IP range and no remote security group, and fails unless the +# group is then free of egress rules that allow all traffic. It needs bash, +# curl and jq on the machine that runs Terraform. # # Terraform hides the output of the script, because its environment holds the # token. When the script fails, the error message of Terraform contains the diff --git a/examples/iaas-server-agent-restricted-egress/README.md b/examples/iaas-server-agent-restricted-egress/README.md index 6a6c308..caf6d13 100644 --- a/examples/iaas-server-agent-restricted-egress/README.md +++ b/examples/iaas-server-agent-restricted-egress/README.md @@ -2,7 +2,7 @@ # Restricted Egress for a VM with cloud-init and the STACKIT Server Agent -Deploys a Linux VM whose security group allows outbound traffic to an allow-list only, with the STACKIT Server Agent switched on, and a jump host as the only way to reach it over SSH. The example adds the two egress rules that cloud-init, the Server Agent and Run Command need: DHCP and the metadata service. It removes the default egress rules, which Terraform cannot manage. +Deploys a Linux VM, the target, whose security group allows outbound traffic to an allow-list only. The platform installs the STACKIT Server Agent on the target. A jump host is the only way to reach the target over SSH. The example adds the two egress rules that cloud-init, the Server Agent and Run Command need: DHCP and the metadata service. Every new security group also carries two default egress rules that allow all outbound traffic. The STACKIT provider cannot remove them, so Terraform runs a script that deletes them through the IaaS API. ## What the example shows @@ -65,11 +65,11 @@ The default egress rules of a new security group allow DHCP like all other traff `ip_range` of a rule takes a CIDR or one of two names. `nameservers` stands for every address in the variable `nameservers`, which also configures the network. `stackit` stands for every IPv4 range the data source `stackit_public_ip_ranges` returns. A security group rule takes one range, so Terraform creates one rule per range. [`scripts/check-stackit-ip.sh`](../../scripts/check-stackit-ip.sh) tells for any address whether it belongs to the ranges. -The default has no rule for DNS, NTP or the package mirrors of the image. Until you add rules for them, the target resolves no names, does not synchronise its clock and installs no package updates of the operating system. The list has no IPv6 rule; the network has no IPv6 prefix. Add what your machine needs through `egress_rules`. Setting the variable replaces the default, so keep the two default entries: +The default value of `egress_rules` has no rule for DNS, NTP or the package mirrors of the image. Until you add rules for them, the target resolves no names, does not synchronise its clock and installs no package updates of the operating system. The list has no IPv6 rule; the network has no IPv6 prefix. Add the rules the target needs through `egress_rules`. Terraform does not merge your value with the default value, so copy the rules `dhcp` and `metadata` from `020-variables.tf` into it: ```hcl egress_rules = { - # ... the two default entries from 020-variables.tf ... + # ... dhcp and metadata, copied from the default value in 020-variables.tf ... dns-udp = { description = "DNS: name resolution through the nameservers of the network" protocol = "udp" @@ -87,7 +87,7 @@ egress_rules = { } ``` -A change of `egress_rules` adds or deletes rules on the running machine. To boot the target for the first time under the changed list, replace the server: +When `egress_rules` changes, Terraform adds and deletes rules in the target's security group. They apply to the running target without a reboot. cloud-init and the installation of the Server Agent run at the first boot only. To run that boot under the changed rules, replace the target: ```bash terraform apply -replace=stackit_server.target @@ -102,7 +102,7 @@ The provider cannot remove them. `stackit_security_group` has no argument for it `070-default-egress-rules.tf` therefore calls the API directly: 1. `ephemeral "stackit_access_token"` issues a token for the service account the provider uses. Terraform writes an ephemeral value neither to the plan nor to the state. -2. `terraform_data.remove_default_egress_rules` runs [`scripts/remove-default-egress-rules.sh`](scripts/remove-default-egress-rules.sh) once per security group. The token and the URL of the group reach the script through its environment. +2. `terraform_data.remove_default_egress_rules` runs [`scripts/remove-default-egress-rules.sh`](scripts/remove-default-egress-rules.sh) for the target's security group, and again when Terraform replaces that group. The token and the URL of the group reach the script through its environment. 3. The script lists the rules of the group, deletes every egress rule that has no protocol, no port range, no IP range and no remote security group, and lists the rules again until no such rule is among them. 4. The script fails, and `terraform apply` with it, when a call fails or when an egress rule that allows all traffic is left. @@ -143,11 +143,11 @@ The output `run_command_service_account_email` names the account in both cases. | `045-run-command-access.tf` | the service account for Run Command and its role on the project | | `050-network.tf` | the network of both machines | | `060-security-groups.tf` | both security groups, the two ingress rules and the egress allow-list | -| `070-default-egress-rules.tf` | the removal of the default egress rules | +| `070-default-egress-rules.tf` | the access token and the step that runs the script below | | `080-cloud-init.tf` | user data of the jump host | | `090-servers.tf` | key pair, network interfaces, public IP and both servers | | `100-outputs.tf` | IDs, addresses, the Run Command service account, the SSH command and the verify commands | -| `scripts/remove-default-egress-rules.sh` | runs on the machine that runs Terraform | +| `scripts/remove-default-egress-rules.sh` | deletes the default egress rules through the IaaS API, on the machine that runs Terraform | ## Usage @@ -159,7 +159,7 @@ The output `run_command_service_account_email` names the account in both cases. - For the Run Command check: an identity that may enable the Run Command service on the project, and a key of the service account that holds `run-command.editor`. See [Who may send Run Command](#who-may-send-run-command). - `bash`, `jq` and `curl` 7.55.0 or later on the machine that runs Terraform. - An SSH key pair and an OpenSSH client with `-J` (ProxyJump). -- The ID of the public image `Ubuntu 24.04` (amd64). The user data of the jump host relies on one property of that image: it starts sshd through socket activation. +- The ID of the public image `Ubuntu 24.04` (amd64). The user data of the jump host expects the sshd layout of Debian and Ubuntu: the main sshd configuration reads `/etc/ssh/sshd_config.d` first and sets no port, and systemd runs sshd as `ssh.socket` or `ssh.service`. - An authenticated [`stackit` CLI](https://github.com/stackitcloud/stackit-cli) for the image lookup and for the checks that read the API. STACKIT replaces its public images with new builds and [deletes a released image after one year](https://docs.stackit.cloud/products/compute-engine/linux-server/faq/), so `image_id` has no default: @@ -210,7 +210,7 @@ The key pair belongs to the service account that runs Terraform, not to the proj ## Verify -`terraform output -raw verify_commands` prints the commands of this section with the addresses and IDs of your deployment. +`terraform output -raw verify_commands` prints the checks of this section with the addresses and IDs of your deployment. It leaves out the setup for Run Command and the command that reads the result of a Run Command. **Jump host.** `ss -Htln` lists the listening TCP sockets. Port 2222 is the only one that is not bound to a loopback address: @@ -239,7 +239,7 @@ A working target prints `Datasource DataSourceOpenStackLocal`. `No active metada ssh -J ubuntu@:2222 ubuntu@ 'systemctl is-active stackit-server-agent.service' ``` -**A destination outside the allow-list.** This check shows that the restriction is in effect. The request goes to the address `1.1.1.1` directly and needs no DNS. No rule of the default allow-list covers it, so curl gets no connection. After 10 seconds it reports `curl: (28)`, the timeout of a connection that got no answer, and prints `000`. An HTTP status shows that a rule allows the connection, for example a default egress rule that is still in place: +**A destination outside the allow-list.** This check shows that the restriction is in effect. The request goes to the address `1.1.1.1` directly and needs no DNS. Neither the rule `dhcp` nor the rule `metadata` covers it, so curl gets no connection. After 10 seconds it reports `curl: (28)`, the timeout of a connection that got no answer, and prints `000`. An HTTP status shows that a rule allows the connection, for example a default egress rule that is still in place: ```bash ssh -J ubuntu@:2222 ubuntu@ \ From ab30f3c4028302a7af6f76b2b11362de9815d84b Mon Sep 17 00:00:00 2001 From: Jan Marco Schreier Date: Fri, 16 Oct 2026 12:04:56 +0200 Subject: [PATCH 32/38] example(iaas): reject the remaining placeholders of the tfvars example --- .../020-variables.tf | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/examples/iaas-server-agent-restricted-egress/020-variables.tf b/examples/iaas-server-agent-restricted-egress/020-variables.tf index 58561a8..7d0890f 100644 --- a/examples/iaas-server-agent-restricted-egress/020-variables.tf +++ b/examples/iaas-server-agent-restricted-egress/020-variables.tf @@ -35,6 +35,11 @@ variable "stackit_parent_container_id" { variable "stackit_admin_email" { description = "An existing STACKIT user that becomes owner of the created project. A non-existent address fails with 409 \"subject is not found\"." type = string + + validation { + condition = !endswith(lower(var.stackit_admin_email), "@example.com") + error_message = "stackit_admin_email is still the placeholder from terraform.tfvars.example." + } } variable "stackit_region" { @@ -114,7 +119,12 @@ variable "admin_cidr" { validation { condition = !strcontains(var.admin_cidr, ":") && can(cidrhost(var.admin_cidr, 0)) - error_message = "admin_cidr must be an IPv4 range in CIDR notation, for example 198.51.100.10/32." + error_message = "admin_cidr must be an IPv4 range in CIDR notation, for example your public IPv4 address with a /32 suffix." + } + + validation { + condition = !startswith(var.admin_cidr, "198.51.100.") + error_message = "admin_cidr is still the placeholder from terraform.tfvars.example." } } @@ -202,6 +212,11 @@ variable "egress_rules" { variable "ssh_public_key" { description = "Public SSH key installed on both machines for the default user of the image. Never the private key." type = string + + validation { + condition = !strcontains(var.ssh_public_key, "AAAA...") + error_message = "ssh_public_key is still the placeholder from terraform.tfvars.example." + } } variable "ssh_user" { From e586a17d64c8a7c2a18a6044ee521c9408213e7a Mon Sep 17 00:00:00 2001 From: Jan Marco Schreier Date: Fri, 16 Oct 2026 14:39:18 +0200 Subject: [PATCH 33/38] docs(iaas): shorten the README of the egress example --- AGENTS.md | 2 +- .../README.md | 210 ++++-------------- 2 files changed, 46 insertions(+), 166 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 76eca97..6b4f08e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -119,7 +119,7 @@ Fetch only the files relevant to the task. A typical example contains - **`iaas-k3s-self-managed`** `[iaas, k3s, kubernetes, self-managed]` Deploys a 6-node k3s + Cilium cluster on STACKIT VMs and validates: L4 NLB, L7 ALB, CSI block storage, Cilium NetworkPolicy, and egress connectivity - **`iaas-server-agent-restricted-egress`** `[iaas, compute, networking, sna, security-group, egress, cloud-init, server-agent, jump-host, ssh]` - Deploys a Linux VM, the target, whose security group allows outbound traffic to an allow-list only. The platform installs the STACKIT Server Agent on the target. A jump host is the only way to reach the target over SSH. The example adds the two egress rules that cloud-init, the Server Agent and Run Command need: DHCP and the metadata service. Every new security group also carries two default egress rules that allow all outbound traffic. The STACKIT provider cannot remove them, so Terraform runs a script that deletes them through the IaaS API + Deploys a Linux VM, the target, whose security group allows outbound traffic to an allow-list only. The platform installs the STACKIT Server Agent on the target. A jump host is the only way to reach the target over SSH. The example adds the two egress rules that cloud-init, the Server Agent and Run Command need: DHCP and the metadata service. Every new security group also carries two default egress rules that allow all outbound traffic. The STACKIT provider cannot remove them, so Terraform deletes them through the IaaS API - **`iaas-volume-encryption`** `[iaas, volume, encryption, kms, block-storage]` KMS & IaaS Resources to deploy a encrypted Block Storage Volume - **`iaas-windows-byol-stackit-migration`** `[iaas, windows, byol, migration, compute]` diff --git a/examples/iaas-server-agent-restricted-egress/README.md b/examples/iaas-server-agent-restricted-egress/README.md index caf6d13..cd449c5 100644 --- a/examples/iaas-server-agent-restricted-egress/README.md +++ b/examples/iaas-server-agent-restricted-egress/README.md @@ -2,21 +2,21 @@ # Restricted Egress for a VM with cloud-init and the STACKIT Server Agent -Deploys a Linux VM, the target, whose security group allows outbound traffic to an allow-list only. The platform installs the STACKIT Server Agent on the target. A jump host is the only way to reach the target over SSH. The example adds the two egress rules that cloud-init, the Server Agent and Run Command need: DHCP and the metadata service. Every new security group also carries two default egress rules that allow all outbound traffic. The STACKIT provider cannot remove them, so Terraform runs a script that deletes them through the IaaS API. +Deploys a Linux VM, the target, whose security group allows outbound traffic to an allow-list only. The platform installs the STACKIT Server Agent on the target. A jump host is the only way to reach the target over SSH. The example adds the two egress rules that cloud-init, the Server Agent and Run Command need: DHCP and the metadata service. Every new security group also carries two default egress rules that allow all outbound traffic. The STACKIT provider cannot remove them, so Terraform deletes them through the IaaS API. ## What the example shows -A security group rule only allows. Every new security group starts with two rules that allow all outbound traffic, so an egress rule of your own restricts nothing until those two are gone. This example removes them before the VM boots for the first time. Without them the group also drops DHCP, so the allow-list has to allow it. The first boot is the one that matters: cloud-init runs then, and the platform installs the Server Agent then. +An egress rule of your own restricts nothing while the two default egress rules exist. This example deletes them before the target boots for the first time. That first boot matters: cloud-init runs then, and the platform installs the Server Agent then. -**cloud-init and the Server Agent are two different things.** [cloud-init](https://docs.cloud-init.io/en/latest/explanation/boot.html) is part of the image. It runs at boot and reads metadata, user data and vendor data from the [metadata service](https://docs.stackit.cloud/products/compute-engine/server/basics/maintenance/) at `http://169.254.169.254`. The [STACKIT Server Agent](https://docs.stackit.cloud/products/compute-engine/server-agent/basics/introduction-to-server-agent/) is a service of its own. The platform [installs it during the first boot through vendor data](https://docs.stackit.cloud/products/compute-engine/server-agent/getting-started/installing-server-agent-api/), and [Run Command](https://docs.stackit.cloud/products/compute-engine/run-command/) and Server Update Management use it afterwards. +[cloud-init](https://docs.cloud-init.io/en/latest/explanation/boot.html) is part of the image and reads metadata, user data and vendor data from the [metadata service](https://docs.stackit.cloud/products/compute-engine/server/basics/maintenance/) at `http://169.254.169.254`. The [STACKIT Server Agent](https://docs.stackit.cloud/products/compute-engine/server-agent/basics/introduction-to-server-agent/) is a service of its own. The platform [installs it through vendor data](https://docs.stackit.cloud/products/compute-engine/server-agent/getting-started/installing-server-agent-api/), and [Run Command](https://docs.stackit.cloud/products/compute-engine/run-command/) uses it afterwards. The example creates: - a STACKIT Network Area (SNA) with its regional configuration, a project attached to it and one network -- a **jump host** with a public IP, whose security group admits TCP 2222 from `admin_cidr` and whose sshd listens on that port -- a **target** without a public IP, with the Server Agent and an egress allow-list -- a step that deletes the two default egress rules of the target's security group through the IaaS API -- the role `run-command.editor` on the project for a service account, either an existing one or one that Terraform creates, so that this account can send Run Command to the target +- a **jump host** with a public IP, whose sshd listens on TCP 2222 +- a **target** without a public IP, with the Server Agent and the egress allow-list +- a step that deletes the two default egress rules of the target's security group +- the role `run-command.editor` on the project for a service account, an existing one or one that Terraform creates ## Architecture @@ -39,33 +39,20 @@ flowchart LR target -->|"TCP 80"| metadata ``` -Each machine has its own security group, and its network interface carries that group and no other. Terraform adds one ingress rule to each group: - -| Security group | Ingress rule | Source | -| -------------- | ------------ | ----------------------------------------- | -| jump host | TCP 2222 | `admin_cidr` | -| target | TCP 22 | members of the jump host's security group | - -Nothing else reaches the jump host from outside, and nothing but the jump host reaches the target. Both groups are stateful: the answers to an allowed connection pass without a rule of their own. The target therefore needs no egress rule for the SSH sessions that arrive from the jump host. - -sshd on the jump host listens on port 2222 itself. No port forwarding sits in front of it, and port 22 is closed. It accepts public keys only. TCP forwarding stays on, because `ssh -J` opens the connection to the target through it. The private key never leaves your machine. - -The allow-list applies to the target only. The security group of the jump host keeps the two default egress rules. +Each machine has its own security group. The jump host's group admits TCP 2222 from `admin_cidr`. The target's group admits TCP 22 from members of the jump host's group. Both groups are stateful, so answers to an allowed connection need no rule of their own. The jump host's group keeps its two default egress rules. ## Egress rules of the target -`egress_rules` holds every egress rule this example adds to the target's security group. Its default holds the two rules that cloud-init, the Server Agent and Run Command need: +The default value of `egress_rules` holds the two rules that cloud-init, the Server Agent and Run Command need: -| Key | Destination | Protocol and port | What needs it | -| ---------- | -------------------- | ----------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `dhcp` | `0.0.0.0/0` | UDP 67 | The interface gets its IPv4 address, the gateway and the nameservers through DHCP. The VM sends its first request to the broadcast address and renews the lease at the DHCP server of the network, both to UDP port 67 ([RFC 2131](https://www.rfc-editor.org/rfc/rfc2131), sections 3.1, 4.1 and 4.4.5). The rule allows UDP 67 to every IPv4 address, so a process on the target can send datagrams to port 67 of any host on the internet as well. | -| `metadata` | `169.254.169.254/32` | TCP 80 | [cloud-init reads metadata, user data and vendor data](https://docs.cloud-init.io/en/latest/explanation/boot.html). The vendor data installs the Server Agent at the first boot. The [guide for a running server](https://docs.stackit.cloud/products/compute-engine/server-agent/getting-started/installing-server-agent-existing-server/) loads the same package from `http://169.254.169.254/stackit/v1/server-agent/amd64/deb`. The Server Agent connects to this address only. Run Command needs no further rule: the Server Agent receives its commands through the metadata service. `stackit-server-monitoring-agent`, which comes with the Server Agent, sends its metrics to `http://169.254.169.254/stackit/server-management-service/v1/agent/monitoring/metrics`. | +| Key | Destination | Protocol and port | What needs it | +| ---------- | -------------------- | ----------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `dhcp` | `0.0.0.0/0` | UDP 67 | The interface gets its IPv4 address through DHCP. The target sends its first request to the broadcast address and renews the lease at the DHCP server of the network ([RFC 2131](https://www.rfc-editor.org/rfc/rfc2131), sections 3.1 and 4.4.5). | +| `metadata` | `169.254.169.254/32` | TCP 80 | cloud-init reads its data from the metadata service, and the vendor data installs the Server Agent from there. The Server Agent connects to this address only and receives its Run Command jobs through it. | -The default egress rules of a new security group allow DHCP like all other traffic. Once they are gone, the group drops DHCP unless a rule allows it. Without the rule `dhcp` the interface gets no IPv4 address. cloud-init then reaches no metadata service and falls back to `DataSourceNone`. The platform does not install the Server Agent, and `stackit server command create` fails with `404 Not Found` and the message `agent not found`. +Without the rule `dhcp` the interface gets no IPv4 address. cloud-init then reaches no metadata service, and the platform does not install the Server Agent. The rule allows UDP 67 to every IPv4 address, the internet included. -`ip_range` of a rule takes a CIDR or one of two names. `nameservers` stands for every address in the variable `nameservers`, which also configures the network. `stackit` stands for every IPv4 range the data source `stackit_public_ip_ranges` returns. A security group rule takes one range, so Terraform creates one rule per range. [`scripts/check-stackit-ip.sh`](../../scripts/check-stackit-ip.sh) tells for any address whether it belongs to the ranges. - -The default value of `egress_rules` has no rule for DNS, NTP or the package mirrors of the image. Until you add rules for them, the target resolves no names, does not synchronise its clock and installs no package updates of the operating system. The list has no IPv6 rule; the network has no IPv6 prefix. Add the rules the target needs through `egress_rules`. Terraform does not merge your value with the default value, so copy the rules `dhcp` and `metadata` from `020-variables.tf` into it: +The default value has no rule for DNS, NTP or package mirrors. Until you add them, the target resolves no names, does not synchronise its clock and installs no updates. Your value of `egress_rules` replaces the default value, so copy `dhcp` and `metadata` into it. `ip_range` takes a CIDR or one of two names: `nameservers` for the nameservers of the network, `stackit` for every IPv4 range of the data source `stackit_public_ip_ranges`. Terraform creates one rule per address range. ```hcl egress_rules = { @@ -87,86 +74,34 @@ egress_rules = { } ``` -When `egress_rules` changes, Terraform adds and deletes rules in the target's security group. They apply to the running target without a reboot. cloud-init and the installation of the Server Agent run at the first boot only. To run that boot under the changed rules, replace the target: - -```bash -terraform apply -replace=stackit_server.target -``` +Changed rules apply to the running target. To run the first boot again under them, replace the target with `terraform apply -replace=stackit_server.target`. ## How the default egress rules are removed -[Every new security group contains a rule that allows all egress traffic](https://docs.stackit.cloud/products/network/core-networking/security-groups/basics/concepts/). The [IaaS API shows two of them](https://docs.stackit.cloud/products/iaas-api/how-tos/manage-security-groups-and-rules-via-iaas-api/), one for IPv4 and one for IPv6. They have a direction and an ethertype and nothing else: no protocol, no port range, no IP range, no remote security group. +[Every new security group contains a rule that allows all egress traffic](https://docs.stackit.cloud/products/network/core-networking/security-groups/basics/concepts/). The IaaS API lists two of them, one for IPv4 and one for IPv6, without protocol, port range or IP range. The provider cannot remove them, because `stackit_security_group` has no argument for its rules. -The provider cannot remove them. `stackit_security_group` has no argument for its rules. The data source `stackit_security_group` does not list rules. `stackit_security_group_rule` manages the rules it created and takes over an existing rule only by an import with the rule ID, which only the API returns. - -`070-default-egress-rules.tf` therefore calls the API directly: - -1. `ephemeral "stackit_access_token"` issues a token for the service account the provider uses. Terraform writes an ephemeral value neither to the plan nor to the state. -2. `terraform_data.remove_default_egress_rules` runs [`scripts/remove-default-egress-rules.sh`](scripts/remove-default-egress-rules.sh) for the target's security group, and again when Terraform replaces that group. The token and the URL of the group reach the script through its environment. -3. The script lists the rules of the group, deletes every egress rule that has no protocol, no port range, no IP range and no remote security group, and lists the rules again until no such rule is among them. -4. The script fails, and `terraform apply` with it, when a call fails or when an egress rule that allows all traffic is left. - -The network interface of the target and the target itself depend on this step and on every rule of the allow-list. Terraform creates them last, so the first boot already runs under the final rule set. - -Terraform hides the output of the script, because its environment holds the token. When the script fails, the error message of Terraform contains the output. A second run changes nothing: the script finds no default rule and only checks. +`070-default-egress-rules.tf` therefore deletes them with `curl` and `jq` through the IaaS API. The token comes from the ephemeral resource `stackit_access_token`, which Terraform writes neither to the plan nor to the state. The network interface and the server of the target wait for this step, so the first boot already runs under the final rule set. ## Who may send Run Command -The Run Command API checks the permission `run-command.command.create` on the project. It answers `403 Forbidden` to an identity without that permission, even when that identity created the project and every resource in it. - -Terraform makes `stackit_admin_email` the owner of the project. The service account that runs Terraform does not become a member of the project; it acts through its roles on `stackit_parent_container_id` or above. A role on a folder carries other permissions than the role of the same name on a project. On a folder, the role `owner` contains no `run-command` permission. On a project, it contains all five. The authorization API lists the roles of a resource with their permissions: - -```bash -stackit curl https://authorization.api.stackit.cloud/v2/project//roles -stackit curl https://authorization.api.stackit.cloud/v2/folder//roles -``` - -`045-run-command-access.tf` therefore gives one service account the role `run-command.editor` on the project. Of the predefined roles that contain `run-command.command.create`, it has the fewest permissions: it reads the agent and the command templates, creates commands and lists and reads their results. It cannot enable the Run Command service. That needs `service-enablement.service-state.edit`, which roles such as `editor` and `owner` contain. - -`run_command_service_account_email` selects the service account: - -| `run_command_service_account_email` | Service account that gets the role | -| ----------------------------------- | ------------------------------------------------------------------------------------- | -| unset (default) | Terraform creates `run_command_service_account_name` in the project. | -| an email in lower case | the existing account. It can belong to another project. Terraform does not change it. | - -The output `run_command_service_account_email` names the account in both cases. Terraform creates no key for it, because `stackit_service_account_key` keeps the key JSON, private key included, in the state. [Verify](#verify) shows how to create a key with the CLI and send the command as that account. +The Run Command API answers `403 Forbidden` to an identity without the permission `run-command.command.create` on the project. The service account that runs Terraform does not become a member of the new project. It acts through its role on `stackit_parent_container_id`, and on a folder the role `owner` contains no `run-command` permission. -## Files - -| File | Contents | -| ---------------------------------------- | ------------------------------------------------------------------------------------------------------------ | -| `010-provider.tf` | provider, with `enable_beta_resources` for the access token and the experiment `iam` for the role assignment | -| `020-variables.tf` | all inputs, including `egress_rules` | -| `030-network-area.tf` | network area and its regional configuration | -| `040-project.tf` | project with the `networkArea` label | -| `045-run-command-access.tf` | the service account for Run Command and its role on the project | -| `050-network.tf` | the network of both machines | -| `060-security-groups.tf` | both security groups, the two ingress rules and the egress allow-list | -| `070-default-egress-rules.tf` | the access token and the step that runs the script below | -| `080-cloud-init.tf` | user data of the jump host | -| `090-servers.tf` | key pair, network interfaces, public IP and both servers | -| `100-outputs.tf` | IDs, addresses, the Run Command service account, the SSH command and the verify commands | -| `scripts/remove-default-egress-rules.sh` | deletes the default egress rules through the IaaS API, on the machine that runs Terraform | +`045-run-command-access.tf` therefore gives a service account the role `run-command.editor` on the project. Of the predefined roles that allow Run Command, it holds the fewest permissions. `run_command_service_account_email` names an existing account. Without it, Terraform creates one in the project. Terraform creates no key, because `stackit_service_account_key` keeps the private key in the state. ## Usage ### Prerequisites -- Terraform 1.10 or later. Ephemeral resources exist since that version. -- Provider `stackitcloud/stackit` 0.118.0 or later. -- A STACKIT service account key that is allowed to create network areas in `stackit_org_id` and projects under `stackit_parent_container_id`. In the new project it also adds a member (`iam.member.add`) and, unless `run_command_service_account_email` is set, creates a service account (`iam.service-account.create`). `stackit_access_token` works with a service account key only. -- For the Run Command check: an identity that may enable the Run Command service on the project, and a key of the service account that holds `run-command.editor`. See [Who may send Run Command](#who-may-send-run-command). +- Terraform 1.10 or later and provider `stackitcloud/stackit` 0.118.0 or later. +- A STACKIT service account key that may create network areas in `stackit_org_id`, and projects, project members and service accounts under `stackit_parent_container_id`. `stackit_access_token` works with a service account key only. - `bash`, `jq` and `curl` 7.55.0 or later on the machine that runs Terraform. - An SSH key pair and an OpenSSH client with `-J` (ProxyJump). -- The ID of the public image `Ubuntu 24.04` (amd64). The user data of the jump host expects the sshd layout of Debian and Ubuntu: the main sshd configuration reads `/etc/ssh/sshd_config.d` first and sets no port, and systemd runs sshd as `ssh.socket` or `ssh.service`. -- An authenticated [`stackit` CLI](https://github.com/stackitcloud/stackit-cli) for the image lookup and for the checks that read the API. - -STACKIT replaces its public images with new builds and [deletes a released image after one year](https://docs.stackit.cloud/products/compute-engine/linux-server/faq/), so `image_id` has no default: +- An authenticated [`stackit` CLI](https://github.com/stackitcloud/stackit-cli) for the image lookup and the checks. +- The ID of the public image `Ubuntu 24.04` (amd64). The jump host's user data expects its sshd layout: `/etc/ssh/sshd_config.d` without a port in the main configuration, and `ssh.socket` or `ssh.service`. STACKIT [deletes a released image after one year](https://docs.stackit.cloud/products/compute-engine/linux-server/faq/), so `image_id` has no default: -```bash -stackit image list --all --project-id -``` + ```bash + stackit image list --all --project-id + ``` ### 1. Configure @@ -174,20 +109,11 @@ stackit image list --all --project-id cp terraform.tfvars.example terraform.tfvars ``` -Replace every placeholder. These variables have no default: +Replace every placeholder: `stackit_org_id`, `stackit_parent_container_id`, `stackit_admin_email` (owner of the project), `admin_cidr`, `ssh_public_key` and `image_id`. -- `stackit_org_id`: organization the network area is created in -- `stackit_parent_container_id`: existing folder for the project, or the organization ID -- `stackit_admin_email`: existing STACKIT user that owns the project -- `admin_cidr`: IPv4 range that reaches the jump host, for example your public IP with a `/32` suffix -- `ssh_public_key`: your complete public SSH key -- `image_id`: see above +`admin_cidr` must hold the address that STACKIT sees as the source of your SSH connection. Some internet lines use several public IPv4 addresses and pick one per destination. Compare `curl -4 https://api.ipify.org` with `curl -4 https://ifconfig.me`. If they differ and SSH times out, set the other address. A changed rule admits connections about 85 seconds after `terraform apply` has finished. -`admin_cidr` must hold the address that STACKIT sees as the source of your SSH connection. Some internet lines use more than one public IPv4 address and pick one per destination. An IP lookup service then shows a different address from the one STACKIT sees. Compare two services, for example `curl -4 https://api.ipify.org` and `curl -4 https://ifconfig.me`. If they differ and SSH to the jump host times out, set the other address. A changed rule admits connections about 85 seconds after `terraform apply` has finished. - -`stackit_service_account_key_path` is optional. Without it the provider reads `STACKIT_SERVICE_ACCOUNT_KEY_PATH` or the credentials file. - -`run_command_service_account_email` is optional. Set it to give an existing service account the role for Run Command. Without it Terraform creates a service account named `run_command_service_account_name` (default `run-command`) in the project. +`run_command_service_account_email` and `stackit_service_account_key_path` are optional. ### 2. Deploy @@ -204,96 +130,50 @@ terraform apply $(terraform output -raw ssh_command) ``` -The command has the form `ssh -J ubuntu@:2222 ubuntu@`. If SSH to port 2222 times out, check that `admin_cidr` holds the address STACKIT sees, as described in [Configure](#1-configure). Both machines get `ssh_public_key` for the default user of the image through a key pair. `ssh_user` changes the user name in the outputs for an image with another default user. - -The key pair belongs to the service account that runs Terraform, not to the project. Its name is unique for that account, so a second deployment with the same service account needs another `name_prefix`. +The command has the form `ssh -J ubuntu@:2222 ubuntu@`. The key pair belongs to the service account that runs Terraform. Its name is unique for that account, so a second deployment with the same service account needs another `name_prefix`. ## Verify -`terraform output -raw verify_commands` prints the checks of this section with the addresses and IDs of your deployment. It leaves out the setup for Run Command and the command that reads the result of a Run Command. - -**Jump host.** `ss -Htln` lists the listening TCP sockets. Port 2222 is the only one that is not bound to a loopback address: +`terraform output -raw verify_commands` prints these checks with the addresses and IDs of your deployment: -```bash -ssh -p 2222 ubuntu@ 'ss -Htln' -``` +| Check | Expected result | +| -------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `ss -Htln` on the jump host | Port 2222 is the only listener outside the loopback addresses. | +| `cloud-init status --long` on the target | `status: done`, and `detail` names `DataSourceOpenStackLocal`. | +| `stackit server log` of the target, filtered | `Datasource DataSourceOpenStackLocal`. `No active metadata service found` and `DataSourceNone` mean that cloud-init reached no metadata service, for example without the rule `dhcp`. SSH then fails too. | +| `systemctl is-active stackit-server-agent.service` | `active` | +| `curl -m 10 https://1.1.1.1` on the target | After 10 seconds `curl: (28)` and `000`. An HTTP status means that a rule allows the connection. | +| `stackit security-group rule list` for both groups | The target's group holds TCP 22, `dhcp` and `metadata`, and no egress rule without protocol. The jump host's group holds TCP 2222 and its default rules. | -**cloud-init on the target.** `status: done` means the first boot finished. `cloud-init status` exits with a status other than 0 when a stage reported an error. `cloud-init status --wait` blocks until cloud-init finished. The line `detail` names `DataSourceOpenStackLocal`: - -```bash -ssh -J ubuntu@:2222 ubuntu@ 'cloud-init status --long' -``` - -**cloud-init on the target, without SSH.** Without the rule `dhcp` the target gets neither an IPv4 address nor the SSH key, and SSH to it fails. The serial console of the target shows the data source in every case: - -```bash -stackit server log --project-id --length 5000 \ - | grep -a -E 'No active metadata service found|Datasource DataSource' -``` - -A working target prints `Datasource DataSourceOpenStackLocal`. `No active metadata service found` followed by `Datasource DataSourceNone` means that cloud-init reached no metadata service. - -**Server Agent on the target.** The [installation guide](https://docs.stackit.cloud/products/compute-engine/server-agent/getting-started/installing-server-agent-api/) names the unit: - -```bash -ssh -J ubuntu@:2222 ubuntu@ 'systemctl is-active stackit-server-agent.service' -``` - -**A destination outside the allow-list.** This check shows that the restriction is in effect. The request goes to the address `1.1.1.1` directly and needs no DNS. Neither the rule `dhcp` nor the rule `metadata` covers it, so curl gets no connection. After 10 seconds it reports `curl: (28)`, the timeout of a connection that got no answer, and prints `000`. An HTTP status shows that a rule allows the connection, for example a default egress rule that is still in place: - -```bash -ssh -J ubuntu@:2222 ubuntu@ \ - 'curl -sS -m 10 -o /dev/null -w "%{http_code}\n" https://1.1.1.1' -``` - -**Rules of both security groups.** The target's group lists the rule for TCP 22 from the jump host's group and the allow-list, and no egress rule without a protocol. The jump host's group lists the rule for TCP 2222 from `admin_cidr` and the two default egress rules: - -```bash -stackit security-group rule list --project-id --security-group-id -``` - -If a list also shows an ingress rule whose remote security group is the group itself, the platform added it. The [security group concepts](https://docs.stackit.cloud/products/network/core-networking/security-groups/basics/concepts/) state that a new group allows traffic from members of the same group. Such a rule admits only members of that group, and each group has one member. - -**Run Command.** Run Command is the test for the running Server Agent: the platform sends a script, and the agent on the VM runs it. Enable the service once per project, [in the portal](https://docs.stackit.cloud/products/compute-engine/run-command/getting-started/enable-run-command-service/) under _Computing > Server > your server > Command > Templates > Enable Commands_ or [through the API](https://docs.stackit.cloud/products/compute-engine/run-command/how-tos/api-enable-the-run-command-service/). The role `run-command.editor` does not allow this step. Run it as the project owner `stackit_admin_email` or as another identity with `service-enablement.service-state.edit` on the project: +**Run Command** proves that the Server Agent works. Enable the service once per project as the project owner, because `run-command.editor` cannot: ```bash stackit curl -X POST \ "https://service-enablement.api.eu01.stackit.cloud/v1/projects//services/cloud.stackit.run-command" ``` -Send the command as the service account that `terraform output -raw run_command_service_account_email` prints. A CLI profile of its own keeps the login of your default profile. For the account Terraform created, create a key and activate it in that profile. The key file holds the private key; `umask 077` makes it readable for you only: +Create a key for the account from `terraform output -raw run_command_service_account_email`, activate it in a CLI profile of its own, and send a command: ```bash (umask 077; stackit service-account key create --email --project-id \ --output-format json --assume-yes > ~/.stackit/run-command-key.json) - stackit config profile create run-command --empty --no-set STACKIT_CLI_PROFILE=run-command stackit auth activate-service-account \ --service-account-key-path ~/.stackit/run-command-key.json -``` - -For an existing account, activate a key you already have. To create one, pass the project the account belongs to as `--project-id`. - -Then send a command and read its result. `create` prints the ID of the command: -```bash STACKIT_CLI_PROFILE=run-command stackit server command create --project-id --server-id \ --template-name RunShellScript --params script='systemctl is-active stackit-server-agent.service' - STACKIT_CLI_PROFILE=run-command stackit server command describe --project-id --server-id ``` -The command reached the agent when its status is `completed` and its exit code is 0. `403 Forbidden` from `create` means that the active identity lacks `run-command.command.create` on the project. `404 Not Found` with the message `agent not found` means that no Server Agent has registered for the target. A new target gives this answer until its Server Agent registers, about two minutes after the server is created. If the answer stays, the serial console shows whether cloud-init reached the metadata service. +Status `completed` with exit code 0 shows a working Server Agent. `403 Forbidden` means that the identity lacks `run-command.command.create`. `404 Not Found` with `agent not found` means that no Server Agent has registered yet; a new target needs about two minutes. ## Clean up ```bash terraform destroy -``` - -`terraform destroy` removes everything the example created, including the project, the network area and the service account for Run Command if Terraform created it. An existing account loses its role on the project and keeps its keys. Delete the key file you created and the CLI profile yourself: - -```bash rm ~/.stackit/run-command-key.json stackit config profile delete run-command ``` + +`terraform destroy` removes the project, the network area and the service account Terraform created. An existing account loses its role and keeps its keys. From 3b7e2605fb211cfee37e1398f43b2453dc8e9543 Mon Sep 17 00:00:00 2001 From: Jan Marco Schreier Date: Fri, 16 Oct 2026 14:39:18 +0200 Subject: [PATCH 34/38] refactor(iaas): delete the default egress rules inline, without scripts/ --- .../070-default-egress-rules.tf | 48 ++++--- .../scripts/remove-default-egress-rules.sh | 133 ------------------ 2 files changed, 27 insertions(+), 154 deletions(-) delete mode 100755 examples/iaas-server-agent-restricted-egress/scripts/remove-default-egress-rules.sh diff --git a/examples/iaas-server-agent-restricted-egress/070-default-egress-rules.tf b/examples/iaas-server-agent-restricted-egress/070-default-egress-rules.tf index 654b373..305a79a 100644 --- a/examples/iaas-server-agent-restricted-egress/070-default-egress-rules.tf +++ b/examples/iaas-server-agent-restricted-egress/070-default-egress-rules.tf @@ -12,34 +12,40 @@ # See the License for the specific language governing permissions and # limitations under the License. -# STACKIT adds two egress rules to every new security group. One allows all -# outbound IPv4 traffic, the other all outbound IPv6 traffic. Rules only -# allow, so the allow-list in 060-security-groups.tf restricts nothing while -# these two rules exist. -# -# The provider cannot remove them: stackit_security_group has no argument for -# its rules, and the data source does not list them. This file deletes them -# through the IaaS API with scripts/remove-default-egress-rules.sh. +# STACKIT adds two egress rules to every new security group, one for IPv4 and +# one for IPv6. Both allow all outbound traffic, so the allow-list in +# 060-security-groups.tf restricts nothing while they exist. The provider +# cannot remove them, so this file deletes them through the IaaS API. -# The token of the service account the provider is configured with. An -# ephemeral resource is not written to the plan or the state. +# The token of the service account the provider is configured with. Terraform +# writes an ephemeral value neither to the plan nor to the state. ephemeral "stackit_access_token" "iaas" {} -# Runs for the target's security group, and again when Terraform replaces -# that group. The script deletes every egress rule that has no protocol, no -# port range, no IP range and no remote security group, and fails unless the -# group is then free of egress rules that allow all traffic. It needs bash, -# curl and jq on the machine that runs Terraform. -# -# Terraform hides the output of the script, because its environment holds the -# token. When the script fails, the error message of Terraform contains the -# output. +# Deletes every egress rule of the target's group that has no protocol, no +# port range, no IP range and no remote security group: the two default rules. +# It then lists the rules again until none of them is left, and fails after +# 30 seconds. Terraform runs it again when it replaces the group. It needs +# bash, curl 7.55.0 or later and jq on the machine that runs Terraform. curl +# reads the token from a file descriptor, so it stays out of the process list. resource "terraform_data" "remove_default_egress_rules" { triggers_replace = stackit_security_group.target.security_group_id provisioner "local-exec" { - interpreter = ["bash"] - command = "${path.module}/scripts/remove-default-egress-rules.sh" + interpreter = ["bash", "-c"] + command = <<-EOT + set -euo pipefail + api() { curl -sSf --retry 3 -H @<(printf 'Authorization: Bearer %s\n' "$STACKIT_ACCESS_TOKEN") "$@"; } + default_rules='.items[] | select(.direction == "egress" and .protocol == null and .portRange == null and .ipRange == null and .remoteSecurityGroupId == null) | .id' + ids=$(api "$SECURITY_GROUP_URL/rules" | jq -r "$default_rules") + for id in $ids; do api -X DELETE "$SECURITY_GROUP_URL/rules/$id" >/dev/null; done + for _ in 1 2 3 4 5 6; do + left=$(api "$SECURITY_GROUP_URL/rules" | jq -r "$default_rules") + [ -z "$left" ] && exit 0 + sleep 5 + done + echo "default egress rules still present: $left" >&2 + exit 1 + EOT environment = { SECURITY_GROUP_URL = "https://iaas.api.stackit.cloud/v2/projects/${stackit_security_group.target.project_id}/regions/${stackit_security_group.target.region}/security-groups/${stackit_security_group.target.security_group_id}" diff --git a/examples/iaas-server-agent-restricted-egress/scripts/remove-default-egress-rules.sh b/examples/iaas-server-agent-restricted-egress/scripts/remove-default-egress-rules.sh deleted file mode 100755 index 7ccaf43..0000000 --- a/examples/iaas-server-agent-restricted-egress/scripts/remove-default-egress-rules.sh +++ /dev/null @@ -1,133 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2026 Schwarz Digits Cloud GmbH & Co. KG -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Removes the default egress rules from one STACKIT security group. -# -# STACKIT adds two rules to every new security group: one allows all outbound -# IPv4 traffic, the other all outbound IPv6 traffic. The rules of a security -# group only allow, so an allow-list has no effect while these two exist. The -# Terraform provider neither creates nor shows them. This script lists the -# rules of the group through the IaaS API, deletes the two rules and then -# checks that no egress rule is left that allows all traffic. -# -# A default rule is an egress rule without protocol, port range, IP range and -# remote security group. The script deletes nothing else and never touches an -# ingress rule. A second run finds no default rule and only checks. -# -# Environment: -# SECURITY_GROUP_URL URL of the group in the IaaS API: -# https://iaas.api.stackit.cloud/v2/projects//regions//security-groups/ -# STACKIT_ACCESS_TOKEN access token of an identity that is allowed to edit the group -# VERIFY_ATTEMPTS optional: how often the final check lists the rules (default 12) -# VERIFY_DELAY_SECONDS optional: pause between two checks (default 5) -# -# Requires bash, jq and curl 7.55.0 or later. -# Exit 0: no egress rule that allows all traffic is left. Exit 1: anything else. - -set -euo pipefail - -fail() { - echo "error: $*" >&2 - exit 1 -} - -[[ -n "${SECURITY_GROUP_URL:-}" ]] || fail "SECURITY_GROUP_URL is not set" -[[ -n "${STACKIT_ACCESS_TOKEN:-}" ]] || fail "STACKIT_ACCESS_TOKEN is not set" -verify_attempts="${VERIFY_ATTEMPTS:-12}" -verify_delay="${VERIFY_DELAY_SECONDS:-5}" - -for tool in curl jq; do - command -v "$tool" >/dev/null 2>&1 || fail "$tool is required but not installed" -done - -# An egress rule without protocol, port range and remote security group matches -# every packet to its IP range. -readonly unrestricted_egress=' - .items[] - | select(.direction == "egress") - | select((.protocol // {}) | (.name == null and .number == null)) - | select(.portRange == null and .icmpParameters == null) - | select(.remoteSecurityGroupId == null)' - -# The default rules have no IP range at all. -readonly default_rule_ids="$unrestricted_egress | select(.ipRange == null) | .id" - -# A rule to 0.0.0.0/0 or ::/0 has the same effect, but STACKIT did not create -# it. The script reports it instead of deleting it. -readonly foreign_rule_ids="$unrestricted_egress | select(.ipRange == \"0.0.0.0/0\" or .ipRange == \"::/0\") | .id" - -# api METHOD URL [CURL_OPTION...] -# curl reads the Authorization header from a file descriptor, so the token does -# not show up in the process list. curl repeats a request on a timeout and on -# the status codes 408, 429, 500, 502, 503 and 504. -api() { - local method="$1" url="$2" - shift 2 - curl --silent --show-error --fail \ - --retry 5 --retry-delay 5 --max-time 60 \ - --request "$method" \ - --header @<(printf 'Authorization: Bearer %s\n' "$STACKIT_ACCESS_TOKEN") \ - --header "Accept: application/json" \ - "$@" "$url" -} - -# The rules have an endpoint of their own. The API specification requires the -# field "items" in its answer. In the group object the field "rules" is -# optional, so a group without rules can come without it. -read_rules() { - local rules - rules=$(api GET "$SECURITY_GROUP_URL/rules") || fail "cannot read the rules of the security group at $SECURITY_GROUP_URL" - jq -e '(.items | type) == "array"' >/dev/null 2>&1 <<<"$rules" || - fail "the answer of the IaaS API contains no list of rules" - printf '%s' "$rules" -} - -rules=$(read_rules) -to_delete=$(jq -r "$default_rule_ids" <<<"$rules") - -deleted=0 -while IFS= read -r rule_id; do - [[ -n "$rule_id" ]] || continue - # The API answers 404 when the rule is already gone, for example after curl - # repeated a request whose first answer was lost. - status=$(api DELETE "$SECURITY_GROUP_URL/rules/$rule_id" --output /dev/null --write-out '%{http_code}') || - [[ "$status" == "404" ]] || - fail "cannot delete the default egress rule $rule_id (HTTP status $status)" - deleted=$((deleted + 1)) -done <<<"$to_delete" - -# The API accepts a deletion before the rule is gone, so the check lists the -# rules until no default rule is among them any more. -attempt=1 -while true; do - rules=$(read_rules) - - foreign=$(jq -r "[$foreign_rule_ids] | join(\" \")" <<<"$rules") - [[ -z "$foreign" ]] || - fail "the group has an egress rule that allows all traffic and is not a default rule: $foreign. Delete it, then run again." - - pending=$(jq -r "[$default_rule_ids] | join(\" \")" <<<"$rules") - if [[ -z "$pending" ]]; then - remaining=$(jq -r '[.items[] | select(.direction == "egress")] | length' <<<"$rules") - echo "deleted $deleted default egress rule(s); no egress rule that allows all traffic is left, $remaining egress rule(s) remain" - exit 0 - fi - - if ((attempt >= verify_attempts)); then - fail "default egress rules are still listed after $verify_attempts checks: $pending" - fi - attempt=$((attempt + 1)) - sleep "$verify_delay" -done From d017c529575e7788c6ed48913e0646f3a495e95f Mon Sep 17 00:00:00 2001 From: Jan Marco Schreier Date: Fri, 16 Oct 2026 15:01:06 +0200 Subject: [PATCH 35/38] fix(iaas): bound the curl calls and accept an already deleted rule --- .../070-default-egress-rules.tf | 26 ++++++++++++------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/examples/iaas-server-agent-restricted-egress/070-default-egress-rules.tf b/examples/iaas-server-agent-restricted-egress/070-default-egress-rules.tf index 305a79a..412cfe5 100644 --- a/examples/iaas-server-agent-restricted-egress/070-default-egress-rules.tf +++ b/examples/iaas-server-agent-restricted-egress/070-default-egress-rules.tf @@ -17,16 +17,20 @@ # 060-security-groups.tf restricts nothing while they exist. The provider # cannot remove them, so this file deletes them through the IaaS API. -# The token of the service account the provider is configured with. Terraform -# writes an ephemeral value neither to the plan nor to the state. +# This ephemeral resource returns a token for the service account that the +# provider uses. Terraform writes it neither to the plan nor to the state. ephemeral "stackit_access_token" "iaas" {} -# Deletes every egress rule of the target's group that has no protocol, no -# port range, no IP range and no remote security group: the two default rules. -# It then lists the rules again until none of them is left, and fails after -# 30 seconds. Terraform runs it again when it replaces the group. It needs -# bash, curl 7.55.0 or later and jq on the machine that runs Terraform. curl -# reads the token from a file descriptor, so it stays out of the process list. +# This resource deletes every egress rule of the target's security group that +# has no protocol, no port range, no IP range and no remote security group. +# These are the two default egress rules. A rule that is already gone answers +# 404, for example after curl repeated a request whose answer was lost. The +# block then lists the rules every 5 seconds until no default egress rule is +# left, and fails after 30 seconds. curl gives up on a request after 60 +# seconds and repeats it up to five times. Terraform runs the block again +# when it replaces the group. It needs bash, curl 7.55.0 or later and jq on +# the machine that runs Terraform. curl reads the header from a file +# descriptor, so the token does not appear in the command line of curl. resource "terraform_data" "remove_default_egress_rules" { triggers_replace = stackit_security_group.target.security_group_id @@ -34,10 +38,12 @@ resource "terraform_data" "remove_default_egress_rules" { interpreter = ["bash", "-c"] command = <<-EOT set -euo pipefail - api() { curl -sSf --retry 3 -H @<(printf 'Authorization: Bearer %s\n' "$STACKIT_ACCESS_TOKEN") "$@"; } + api() { curl -sSf --retry 5 --retry-delay 5 --max-time 60 -H @<(printf 'Authorization: Bearer %s\n' "$STACKIT_ACCESS_TOKEN") "$@"; } default_rules='.items[] | select(.direction == "egress" and .protocol == null and .portRange == null and .ipRange == null and .remoteSecurityGroupId == null) | .id' ids=$(api "$SECURITY_GROUP_URL/rules" | jq -r "$default_rules") - for id in $ids; do api -X DELETE "$SECURITY_GROUP_URL/rules/$id" >/dev/null; done + for id in $ids; do + code=$(api -X DELETE -o /dev/null -w '%%{http_code}' "$SECURITY_GROUP_URL/rules/$id") || [ "$code" = 404 ] + done for _ in 1 2 3 4 5 6; do left=$(api "$SECURITY_GROUP_URL/rules" | jq -r "$default_rules") [ -z "$left" ] && exit 0 From 9aebc88f9dadf1646352fd378b62188f5f490127 Mon Sep 17 00:00:00 2001 From: Jan Marco Schreier Date: Fri, 16 Oct 2026 15:01:06 +0200 Subject: [PATCH 36/38] docs(iaas): add the region to CLI calls and close gaps in the README --- AGENTS.md | 2 +- .../100-outputs.tf | 13 +++-- .../README.md | 48 +++++++++---------- 3 files changed, 31 insertions(+), 32 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 6b4f08e..cc2aec9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -119,7 +119,7 @@ Fetch only the files relevant to the task. A typical example contains - **`iaas-k3s-self-managed`** `[iaas, k3s, kubernetes, self-managed]` Deploys a 6-node k3s + Cilium cluster on STACKIT VMs and validates: L4 NLB, L7 ALB, CSI block storage, Cilium NetworkPolicy, and egress connectivity - **`iaas-server-agent-restricted-egress`** `[iaas, compute, networking, sna, security-group, egress, cloud-init, server-agent, jump-host, ssh]` - Deploys a Linux VM, the target, whose security group allows outbound traffic to an allow-list only. The platform installs the STACKIT Server Agent on the target. A jump host is the only way to reach the target over SSH. The example adds the two egress rules that cloud-init, the Server Agent and Run Command need: DHCP and the metadata service. Every new security group also carries two default egress rules that allow all outbound traffic. The STACKIT provider cannot remove them, so Terraform deletes them through the IaaS API + This example deploys a Linux VM, the target. Its security group allows outbound traffic only to an allow-list. The platform installs the STACKIT Server Agent on the target. A jump host is the only way to reach the target over SSH. The example adds the two egress rules that cloud-init, the Server Agent and Run Command need: DHCP and the metadata service. Every new security group also carries two default egress rules that allow all outbound traffic. The STACKIT provider cannot remove them, so Terraform deletes them through the IaaS API - **`iaas-volume-encryption`** `[iaas, volume, encryption, kms, block-storage]` KMS & IaaS Resources to deploy a encrypted Block Storage Volume - **`iaas-windows-byol-stackit-migration`** `[iaas, windows, byol, migration, compute]` diff --git a/examples/iaas-server-agent-restricted-egress/100-outputs.tf b/examples/iaas-server-agent-restricted-egress/100-outputs.tf index d587c41..26eed4f 100644 --- a/examples/iaas-server-agent-restricted-egress/100-outputs.tf +++ b/examples/iaas-server-agent-restricted-egress/100-outputs.tf @@ -77,7 +77,7 @@ output "verify_commands" { ${local.ssh_target} 'cloud-init status --long' # Target, without SSH: the serial console names the data source of the first boot. DataSourceNone means that cloud-init reached no metadata service. - stackit server log ${stackit_server.target.server_id} --project-id ${local.project_id} --length 5000 | grep -a -E 'No active metadata service found|Datasource DataSource' + stackit server log ${stackit_server.target.server_id} --project-id ${local.project_id} --region ${var.stackit_region} --length 5000 | grep -a -E 'No active metadata service found|Datasource DataSource' # Target: the unit of the STACKIT Server Agent is active. ${local.ssh_target} 'systemctl is-active stackit-server-agent.service' @@ -87,10 +87,13 @@ output "verify_commands" { ${local.ssh_target} 'curl -sS -m 10 -o /dev/null -w "%%{http_code}\n" https://1.1.1.1' # Rules of both security groups as the IaaS API reports them. - stackit security-group rule list --project-id ${local.project_id} --security-group-id ${stackit_security_group.jump.security_group_id} - stackit security-group rule list --project-id ${local.project_id} --security-group-id ${stackit_security_group.target.security_group_id} + stackit security-group rule list --project-id ${local.project_id} --region ${var.stackit_region} --security-group-id ${stackit_security_group.jump.security_group_id} + stackit security-group rule list --project-id ${local.project_id} --region ${var.stackit_region} --security-group-id ${stackit_security_group.target.security_group_id} - # Run Command: the platform reaches the Server Agent on the target. The CLI profile run-command holds a key of ${local.run_command_service_account_email}, see "Verify" in the README. - STACKIT_CLI_PROFILE=run-command stackit server command create --project-id ${local.project_id} --server-id ${stackit_server.target.server_id} --template-name RunShellScript --params script='systemctl is-active stackit-server-agent.service' + # Run Command, once per project and as the project owner: enable the service. The role run-command.editor cannot do this. + stackit curl -X POST "https://service-enablement.api.${var.stackit_region}.stackit.cloud/v1/projects/${local.project_id}/services/cloud.stackit.run-command" + + # Run Command: send a command to the Server Agent on the target. The CLI profile run-command holds a key of ${local.run_command_service_account_email}, see "Verify" in the README. + STACKIT_CLI_PROFILE=run-command stackit server command create --project-id ${local.project_id} --region ${var.stackit_region} --server-id ${stackit_server.target.server_id} --template-name RunShellScript --params script='systemctl is-active stackit-server-agent.service' EOT } diff --git a/examples/iaas-server-agent-restricted-egress/README.md b/examples/iaas-server-agent-restricted-egress/README.md index cd449c5..57ad8ff 100644 --- a/examples/iaas-server-agent-restricted-egress/README.md +++ b/examples/iaas-server-agent-restricted-egress/README.md @@ -2,13 +2,13 @@ # Restricted Egress for a VM with cloud-init and the STACKIT Server Agent -Deploys a Linux VM, the target, whose security group allows outbound traffic to an allow-list only. The platform installs the STACKIT Server Agent on the target. A jump host is the only way to reach the target over SSH. The example adds the two egress rules that cloud-init, the Server Agent and Run Command need: DHCP and the metadata service. Every new security group also carries two default egress rules that allow all outbound traffic. The STACKIT provider cannot remove them, so Terraform deletes them through the IaaS API. +This example deploys a Linux VM, the target. Its security group allows outbound traffic only to an allow-list. The platform installs the STACKIT Server Agent on the target. A jump host is the only way to reach the target over SSH. The example adds the two egress rules that cloud-init, the Server Agent and Run Command need: DHCP and the metadata service. Every new security group also carries two default egress rules that allow all outbound traffic. The STACKIT provider cannot remove them, so Terraform deletes them through the IaaS API. ## What the example shows -An egress rule of your own restricts nothing while the two default egress rules exist. This example deletes them before the target boots for the first time. That first boot matters: cloud-init runs then, and the platform installs the Server Agent then. +Security group rules only allow traffic. While the two default egress rules exist, the allow-list restricts nothing. This example deletes them before the target boots for the first time. cloud-init runs at the first boot only, and the platform installs the Server Agent during the same boot. -[cloud-init](https://docs.cloud-init.io/en/latest/explanation/boot.html) is part of the image and reads metadata, user data and vendor data from the [metadata service](https://docs.stackit.cloud/products/compute-engine/server/basics/maintenance/) at `http://169.254.169.254`. The [STACKIT Server Agent](https://docs.stackit.cloud/products/compute-engine/server-agent/basics/introduction-to-server-agent/) is a service of its own. The platform [installs it through vendor data](https://docs.stackit.cloud/products/compute-engine/server-agent/getting-started/installing-server-agent-api/), and [Run Command](https://docs.stackit.cloud/products/compute-engine/run-command/) uses it afterwards. +[cloud-init](https://docs.cloud-init.io/en/latest/explanation/boot.html) is part of the image and reads metadata, user data and vendor data from the [metadata service](https://docs.stackit.cloud/products/compute-engine/server/basics/maintenance/) at `http://169.254.169.254`. The [STACKIT Server Agent](https://docs.stackit.cloud/products/compute-engine/server-agent/basics/introduction-to-server-agent/) is a service of its own. The platform [installs it through vendor data](https://docs.stackit.cloud/products/compute-engine/server-agent/getting-started/installing-server-agent-api/). [Run Command](https://docs.stackit.cloud/products/compute-engine/run-command/) sends a script to the target, and the Server Agent runs it. The example creates: @@ -39,7 +39,7 @@ flowchart LR target -->|"TCP 80"| metadata ``` -Each machine has its own security group. The jump host's group admits TCP 2222 from `admin_cidr`. The target's group admits TCP 22 from members of the jump host's group. Both groups are stateful, so answers to an allowed connection need no rule of their own. The jump host's group keeps its two default egress rules. +The jump host and the target each have their own security group. The jump host's group admits TCP 2222 from `admin_cidr`. The target's group admits TCP 22 from members of the jump host's group. Both groups are stateful, so answers to an allowed connection need no rule of their own. The jump host's group keeps its two default egress rules. ## Egress rules of the target @@ -48,7 +48,7 @@ The default value of `egress_rules` holds the two rules that cloud-init, the Ser | Key | Destination | Protocol and port | What needs it | | ---------- | -------------------- | ----------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `dhcp` | `0.0.0.0/0` | UDP 67 | The interface gets its IPv4 address through DHCP. The target sends its first request to the broadcast address and renews the lease at the DHCP server of the network ([RFC 2131](https://www.rfc-editor.org/rfc/rfc2131), sections 3.1 and 4.4.5). | -| `metadata` | `169.254.169.254/32` | TCP 80 | cloud-init reads its data from the metadata service, and the vendor data installs the Server Agent from there. The Server Agent connects to this address only and receives its Run Command jobs through it. | +| `metadata` | `169.254.169.254/32` | TCP 80 | cloud-init reads its data, the vendor data included, from the metadata service. The platform installs the Server Agent through this vendor data. The Server Agent connects to this address only and receives its Run Command jobs through it. | Without the rule `dhcp` the interface gets no IPv4 address. cloud-init then reaches no metadata service, and the platform does not install the Server Agent. The rule allows UDP 67 to every IPv4 address, the internet included. @@ -74,17 +74,17 @@ egress_rules = { } ``` -Changed rules apply to the running target. To run the first boot again under them, replace the target with `terraform apply -replace=stackit_server.target`. +Changed rules apply to the running target. cloud-init and the installation of the Server Agent run at the first boot only. To run both under the changed rules, replace the target with `terraform apply -replace=stackit_server.target`. -## How the default egress rules are removed +## How Terraform deletes the default egress rules -[Every new security group contains a rule that allows all egress traffic](https://docs.stackit.cloud/products/network/core-networking/security-groups/basics/concepts/). The IaaS API lists two of them, one for IPv4 and one for IPv6, without protocol, port range or IP range. The provider cannot remove them, because `stackit_security_group` has no argument for its rules. +[Every new security group allows all egress traffic](https://docs.stackit.cloud/products/network/core-networking/security-groups/basics/concepts/). The IaaS API lists two default egress rules for this: one for IPv4 and one for IPv6. Neither has a protocol, a port range or an IP range. The provider cannot remove them, because `stackit_security_group` has no argument for its rules. -`070-default-egress-rules.tf` therefore deletes them with `curl` and `jq` through the IaaS API. The token comes from the ephemeral resource `stackit_access_token`, which Terraform writes neither to the plan nor to the state. The network interface and the server of the target wait for this step, so the first boot already runs under the final rule set. +`070-default-egress-rules.tf` therefore deletes them with `curl` and `jq` through the IaaS API. The token comes from the ephemeral resource `stackit_access_token`, which Terraform writes neither to the plan nor to the state. The network interface and the server of the target wait for this step. The target therefore boots for the first time with the allow-list and without the default egress rules. ## Who may send Run Command -The Run Command API answers `403 Forbidden` to an identity without the permission `run-command.command.create` on the project. The service account that runs Terraform does not become a member of the new project. It acts through its role on `stackit_parent_container_id`, and on a folder the role `owner` contains no `run-command` permission. +The Run Command API answers `403 Forbidden` to an identity without the permission `run-command.command.create` on the project. The service account that runs Terraform does not become a member of the new project. It acts through its role on `stackit_parent_container_id`. On a folder, the role `owner` contains no `run-command` permission. `045-run-command-access.tf` therefore gives a service account the role `run-command.editor` on the project. Of the predefined roles that allow Run Command, it holds the fewest permissions. `run_command_service_account_email` names an existing account. Without it, Terraform creates one in the project. Terraform creates no key, because `stackit_service_account_key` keeps the private key in the state. @@ -111,7 +111,7 @@ cp terraform.tfvars.example terraform.tfvars Replace every placeholder: `stackit_org_id`, `stackit_parent_container_id`, `stackit_admin_email` (owner of the project), `admin_cidr`, `ssh_public_key` and `image_id`. -`admin_cidr` must hold the address that STACKIT sees as the source of your SSH connection. Some internet lines use several public IPv4 addresses and pick one per destination. Compare `curl -4 https://api.ipify.org` with `curl -4 https://ifconfig.me`. If they differ and SSH times out, set the other address. A changed rule admits connections about 85 seconds after `terraform apply` has finished. +`admin_cidr` must hold the address that STACKIT sees as the source of your SSH connection. Some internet lines use several public IPv4 addresses and pick one per destination. Compare `curl -4 https://api.ipify.org` with `curl -4 https://ifconfig.me`. If the two addresses differ and SSH times out, set `admin_cidr` to the other one. A changed rule admits connections about 85 seconds after `terraform apply` has finished. `run_command_service_account_email` and `stackit_service_account_key_path` are optional. @@ -130,7 +130,7 @@ terraform apply $(terraform output -raw ssh_command) ``` -The command has the form `ssh -J ubuntu@:2222 ubuntu@`. The key pair belongs to the service account that runs Terraform. Its name is unique for that account, so a second deployment with the same service account needs another `name_prefix`. +The command has the form `ssh -J ubuntu@:2222 ubuntu@`. Terraform registers `ssh_public_key` as a STACKIT key pair and installs it on both machines. The key pair belongs to the service account that runs Terraform, not to the project. Its name must be unique for that account, so a second deployment with the same service account needs another `name_prefix`. ## Verify @@ -143,16 +143,9 @@ The command has the form `ssh -J ubuntu@:2222 ubuntu@`. The k | `stackit server log` of the target, filtered | `Datasource DataSourceOpenStackLocal`. `No active metadata service found` and `DataSourceNone` mean that cloud-init reached no metadata service, for example without the rule `dhcp`. SSH then fails too. | | `systemctl is-active stackit-server-agent.service` | `active` | | `curl -m 10 https://1.1.1.1` on the target | After 10 seconds `curl: (28)` and `000`. An HTTP status means that a rule allows the connection. | -| `stackit security-group rule list` for both groups | The target's group holds TCP 22, `dhcp` and `metadata`, and no egress rule without protocol. The jump host's group holds TCP 2222 and its default rules. | +| `stackit security-group rule list` for both groups | The target's group holds TCP 22, `dhcp` and `metadata`, and no egress rule without protocol. The jump host's group holds TCP 2222 and its two default egress rules. | -**Run Command** proves that the Server Agent works. Enable the service once per project as the project owner, because `run-command.editor` cannot: - -```bash -stackit curl -X POST \ - "https://service-enablement.api.eu01.stackit.cloud/v1/projects//services/cloud.stackit.run-command" -``` - -Create a key for the account from `terraform output -raw run_command_service_account_email`, activate it in a CLI profile of its own, and send a command: +**Run Command** proves that the Server Agent works. `verify_commands` prints the call that enables the service. Run it once per project as the project owner, because `run-command.editor` cannot. Then create a key for the account from `terraform output -raw run_command_service_account_email` and activate it in a CLI profile of its own: ```bash (umask 077; stackit service-account key create --email --project-id \ @@ -160,13 +153,16 @@ Create a key for the account from `terraform output -raw run_command_service_acc stackit config profile create run-command --empty --no-set STACKIT_CLI_PROFILE=run-command stackit auth activate-service-account \ --service-account-key-path ~/.stackit/run-command-key.json +``` + +For an existing account, activate a key you already have. To create one, pass the project the account belongs to as `--project-id`. Then send the command that `verify_commands` prints, and read its result: -STACKIT_CLI_PROFILE=run-command stackit server command create --project-id --server-id \ - --template-name RunShellScript --params script='systemctl is-active stackit-server-agent.service' -STACKIT_CLI_PROFILE=run-command stackit server command describe --project-id --server-id +```bash +STACKIT_CLI_PROFILE=run-command stackit server command describe \ + --project-id --region --server-id ``` -Status `completed` with exit code 0 shows a working Server Agent. `403 Forbidden` means that the identity lacks `run-command.command.create`. `404 Not Found` with `agent not found` means that no Server Agent has registered yet; a new target needs about two minutes. +Status `completed` with exit code 0 shows a working Server Agent. `403 Forbidden` means that the identity lacks `run-command.command.create`. `404 Not Found` with `agent not found` means that no Server Agent has registered yet. On a new target, the Server Agent registers about two minutes after Terraform creates the server. ## Clean up @@ -176,4 +172,4 @@ rm ~/.stackit/run-command-key.json stackit config profile delete run-command ``` -`terraform destroy` removes the project, the network area and the service account Terraform created. An existing account loses its role and keeps its keys. +`terraform destroy` removes the project, the network area and the service account Terraform created. An existing account loses its role and keeps its keys. Delete a key you created for it with `stackit service-account key delete --email --project-id `. From 59e1fea378aca6de25154dd557f1a9110b4ffe39 Mon Sep 17 00:00:00 2001 From: Jan Marco Schreier Date: Mon, 19 Oct 2026 14:36:14 +0200 Subject: [PATCH 37/38] refactor(iaas): move the jump host's cloud-config into a template --- .../080-cloud-init.tf | 55 ------------------- .../090-servers.tf | 2 +- .../jump-host-cloud-init.yaml.tftpl | 29 ++++++++++ 3 files changed, 30 insertions(+), 56 deletions(-) delete mode 100644 examples/iaas-server-agent-restricted-egress/080-cloud-init.tf create mode 100644 examples/iaas-server-agent-restricted-egress/jump-host-cloud-init.yaml.tftpl diff --git a/examples/iaas-server-agent-restricted-egress/080-cloud-init.tf b/examples/iaas-server-agent-restricted-egress/080-cloud-init.tf deleted file mode 100644 index cdfeda3..0000000 --- a/examples/iaas-server-agent-restricted-egress/080-cloud-init.tf +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright 2026 Schwarz Digits Cloud GmbH & Co. KG -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -locals { - # --- jump host --------------------------------------------------------- - # - # sshd reads the files in /etc/ssh/sshd_config.d before the rest of its - # configuration. The main file sets no port, so this file alone decides - # where sshd listens. - jump_sshd_config = <<-EOT - # Jump host: sshd listens on TCP ${local.jump_ssh_port} only and accepts public keys only. - Port ${local.jump_ssh_port} - PermitRootLogin no - PubkeyAuthentication yes - PasswordAuthentication no - KbdInteractiveAuthentication no - AuthenticationMethods publickey - # ProxyJump opens a forwarded TCP connection to the target. - AllowTcpForwarding yes - AllowAgentForwarding no - X11Forwarding no - EOT - - # Ubuntu 24.04 starts sshd through socket activation: systemd holds the - # listening socket in ssh.socket, and a generator derives its port from the - # sshd configuration. "systemctl daemon-reload" runs the generator again, and - # the restart of ssh.socket closes port 22 and opens the new port. The other - # branch covers an image that runs sshd as the plain service ssh.service. - jump_user_data = join("\n", [ - "#cloud-config", - yamlencode({ - write_files = [{ - path = "/etc/ssh/sshd_config.d/00-jump-host.conf" - owner = "root:root" - permissions = "0644" - content = local.jump_sshd_config - }] - runcmd = [ - "systemctl daemon-reload", - "if systemctl is-enabled --quiet ssh.socket; then systemctl restart ssh.socket; else systemctl restart ssh.service; fi", - ] - }), - ]) -} diff --git a/examples/iaas-server-agent-restricted-egress/090-servers.tf b/examples/iaas-server-agent-restricted-egress/090-servers.tf index 73854b9..299d716 100644 --- a/examples/iaas-server-agent-restricted-egress/090-servers.tf +++ b/examples/iaas-server-agent-restricted-egress/090-servers.tf @@ -43,7 +43,7 @@ resource "stackit_server" "jump" { machine_type = var.jump_machine_type keypair_name = stackit_key_pair.admin.name network_interfaces = [stackit_network_interface.jump.network_interface_id] - user_data = local.jump_user_data + user_data = templatefile("${path.module}/jump-host-cloud-init.yaml.tftpl", { ssh_port = local.jump_ssh_port }) boot_volume = { source_type = "image" diff --git a/examples/iaas-server-agent-restricted-egress/jump-host-cloud-init.yaml.tftpl b/examples/iaas-server-agent-restricted-egress/jump-host-cloud-init.yaml.tftpl new file mode 100644 index 0000000..f2995d4 --- /dev/null +++ b/examples/iaas-server-agent-restricted-egress/jump-host-cloud-init.yaml.tftpl @@ -0,0 +1,29 @@ +#cloud-config +# sshd reads the files in /etc/ssh/sshd_config.d before the rest of its +# configuration. The main file sets no port, so this file alone decides where +# sshd listens. +write_files: + - path: /etc/ssh/sshd_config.d/00-jump-host.conf + owner: root:root + permissions: "0644" + content: | + # Jump host: sshd listens on TCP ${ssh_port} only and accepts public keys only. + Port ${ssh_port} + PermitRootLogin no + PubkeyAuthentication yes + PasswordAuthentication no + KbdInteractiveAuthentication no + AuthenticationMethods publickey + # ProxyJump opens a forwarded TCP connection to the target. + AllowTcpForwarding yes + AllowAgentForwarding no + X11Forwarding no + +# Ubuntu 24.04 starts sshd through socket activation: systemd holds the +# listening socket in ssh.socket, and a generator derives its port from the +# sshd configuration. "systemctl daemon-reload" runs the generator again, and +# the restart of ssh.socket closes port 22 and opens the new port. The other +# branch covers an image that runs sshd as the plain service ssh.service. +runcmd: + - systemctl daemon-reload + - "if systemctl is-enabled --quiet ssh.socket; then systemctl restart ssh.socket; else systemctl restart ssh.service; fi" From c7c58d1669ebd12712d6f147fdd188da95aa01a7 Mon Sep 17 00:00:00 2001 From: Jan Marco Schreier Date: Mon, 19 Oct 2026 15:07:31 +0200 Subject: [PATCH 38/38] refactor(iaas): load the default-rule deletion from a shell script --- .../070-default-egress-rules.tf | 32 +++----------- .../README.md | 2 +- .../remove-default-egress-rules.sh | 43 +++++++++++++++++++ 3 files changed, 49 insertions(+), 28 deletions(-) create mode 100755 examples/iaas-server-agent-restricted-egress/remove-default-egress-rules.sh diff --git a/examples/iaas-server-agent-restricted-egress/070-default-egress-rules.tf b/examples/iaas-server-agent-restricted-egress/070-default-egress-rules.tf index 412cfe5..4babd06 100644 --- a/examples/iaas-server-agent-restricted-egress/070-default-egress-rules.tf +++ b/examples/iaas-server-agent-restricted-egress/070-default-egress-rules.tf @@ -21,37 +21,15 @@ # provider uses. Terraform writes it neither to the plan nor to the state. ephemeral "stackit_access_token" "iaas" {} -# This resource deletes every egress rule of the target's security group that -# has no protocol, no port range, no IP range and no remote security group. -# These are the two default egress rules. A rule that is already gone answers -# 404, for example after curl repeated a request whose answer was lost. The -# block then lists the rules every 5 seconds until no default egress rule is -# left, and fails after 30 seconds. curl gives up on a request after 60 -# seconds and repeats it up to five times. Terraform runs the block again -# when it replaces the group. It needs bash, curl 7.55.0 or later and jq on -# the machine that runs Terraform. curl reads the header from a file -# descriptor, so the token does not appear in the command line of curl. +# Runs remove-default-egress-rules.sh for the target's security group, and +# again when Terraform replaces that group. The script needs bash, curl 7.55.0 +# or later and jq on the machine that runs Terraform. resource "terraform_data" "remove_default_egress_rules" { triggers_replace = stackit_security_group.target.security_group_id provisioner "local-exec" { - interpreter = ["bash", "-c"] - command = <<-EOT - set -euo pipefail - api() { curl -sSf --retry 5 --retry-delay 5 --max-time 60 -H @<(printf 'Authorization: Bearer %s\n' "$STACKIT_ACCESS_TOKEN") "$@"; } - default_rules='.items[] | select(.direction == "egress" and .protocol == null and .portRange == null and .ipRange == null and .remoteSecurityGroupId == null) | .id' - ids=$(api "$SECURITY_GROUP_URL/rules" | jq -r "$default_rules") - for id in $ids; do - code=$(api -X DELETE -o /dev/null -w '%%{http_code}' "$SECURITY_GROUP_URL/rules/$id") || [ "$code" = 404 ] - done - for _ in 1 2 3 4 5 6; do - left=$(api "$SECURITY_GROUP_URL/rules" | jq -r "$default_rules") - [ -z "$left" ] && exit 0 - sleep 5 - done - echo "default egress rules still present: $left" >&2 - exit 1 - EOT + interpreter = ["bash"] + command = "${path.module}/remove-default-egress-rules.sh" environment = { SECURITY_GROUP_URL = "https://iaas.api.stackit.cloud/v2/projects/${stackit_security_group.target.project_id}/regions/${stackit_security_group.target.region}/security-groups/${stackit_security_group.target.security_group_id}" diff --git a/examples/iaas-server-agent-restricted-egress/README.md b/examples/iaas-server-agent-restricted-egress/README.md index 57ad8ff..937b4f8 100644 --- a/examples/iaas-server-agent-restricted-egress/README.md +++ b/examples/iaas-server-agent-restricted-egress/README.md @@ -80,7 +80,7 @@ Changed rules apply to the running target. cloud-init and the installation of th [Every new security group allows all egress traffic](https://docs.stackit.cloud/products/network/core-networking/security-groups/basics/concepts/). The IaaS API lists two default egress rules for this: one for IPv4 and one for IPv6. Neither has a protocol, a port range or an IP range. The provider cannot remove them, because `stackit_security_group` has no argument for its rules. -`070-default-egress-rules.tf` therefore deletes them with `curl` and `jq` through the IaaS API. The token comes from the ephemeral resource `stackit_access_token`, which Terraform writes neither to the plan nor to the state. The network interface and the server of the target wait for this step. The target therefore boots for the first time with the allow-list and without the default egress rules. +`070-default-egress-rules.tf` therefore runs [`remove-default-egress-rules.sh`](remove-default-egress-rules.sh), which deletes them with `curl` and `jq` through the IaaS API. The token comes from the ephemeral resource `stackit_access_token`, which Terraform writes neither to the plan nor to the state. The network interface and the server of the target wait for this step. The target therefore boots for the first time with the allow-list and without the default egress rules. ## Who may send Run Command diff --git a/examples/iaas-server-agent-restricted-egress/remove-default-egress-rules.sh b/examples/iaas-server-agent-restricted-egress/remove-default-egress-rules.sh new file mode 100755 index 0000000..4dada05 --- /dev/null +++ b/examples/iaas-server-agent-restricted-egress/remove-default-egress-rules.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# Copyright 2026 Schwarz Digits Cloud GmbH & Co. KG +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Deletes the two default egress rules of one STACKIT security group: every +# egress rule without protocol, port range, IP range and remote security group. +# A rule that is already gone answers 404, for example after curl repeated a +# request whose answer was lost. The script then lists the rules every 5 +# seconds until no default egress rule is left, and fails after 30 seconds. +# curl gives up on a request after 60 seconds and repeats it up to five times. +# +# Environment: SECURITY_GROUP_URL (the group in the IaaS API) and +# STACKIT_ACCESS_TOKEN. curl reads the header from a file descriptor, so the +# token does not appear in the command line of curl. +# Requires bash, curl 7.55.0 or later and jq. +set -euo pipefail + +api() { curl -sSf --retry 5 --retry-delay 5 --max-time 60 -H @<(printf 'Authorization: Bearer %s\n' "$STACKIT_ACCESS_TOKEN") "$@"; } +default_rules='.items[] | select(.direction == "egress" and .protocol == null and .portRange == null and .ipRange == null and .remoteSecurityGroupId == null) | .id' + +ids=$(api "$SECURITY_GROUP_URL/rules" | jq -r "$default_rules") +for id in $ids; do + code=$(api -X DELETE -o /dev/null -w '%{http_code}' "$SECURITY_GROUP_URL/rules/$id") || [ "$code" = 404 ] +done + +for _ in 1 2 3 4 5 6; do + left=$(api "$SECURITY_GROUP_URL/rules" | jq -r "$default_rules") + [ -z "$left" ] && exit 0 + sleep 5 +done +echo "default egress rules still present: $left" >&2 +exit 1