diff --git a/.github/workflows/run-samples.yml b/.github/workflows/run-samples.yml index dbf3c65..7d0720c 100644 --- a/.github/workflows/run-samples.yml +++ b/.github/workflows/run-samples.yml @@ -4,7 +4,9 @@ name: Test samples on LocalStack for Azure # app with its numbered scripts, one job per app and platform, and checks the app end to end with # tests/test-vacation-planner.sh. As in the Samples CI of localstack-azure-samples, a pull request or a # push runs the apps it changes (every app when a file all of them use changes), and a manual run can -# choose to run every app, on Linux, on Windows or on both. +# choose to run every app, on Linux, on Windows or on both. Two more jobs create the cluster with bicep/ +# and with terraform/ instead of the script, and deploy and check one app on each. Every job checks the +# diagnostic settings of the cluster's resources with tests/test-diagnostic-settings.sh. on: push: @@ -76,6 +78,8 @@ jobs: # Every app a job can deploy, as samples//, so a new sample needs no change here apps=$(printf '%s\n' samples/*/*/scripts/05-deploy-app.sh | sed 's|/scripts/05-deploy-app\.sh$||') selected=$apps + # The Bicep and Terraform deployments of the cluster, each checked with one app + deployments="bicep terraform" if [ "$RUN_MODE" = changed ]; then case $EVENT_NAME in @@ -87,14 +91,18 @@ jobs: changed=$(git diff --name-only "$base...HEAD") echo "Files changed since $base:" echo "$changed" - # Every job creates the cluster with the same script and checks the app with the same test, so a - # change to either, or to this workflow, runs every app; otherwise an app runs when its folder changes - shared='^(scripts/01-system-assigned-managed-identity\.sh|tests/test-vacation-planner\.sh|\.github/workflows/run-samples\.yml)$' + # Every job creates the cluster with the same script and checks it and the app with the same tests, so + # a change to any of them, or to this workflow, runs every app; otherwise an app runs when its folder changes + shared='^(scripts/01-system-assigned-managed-identity\.sh|tests/test-vacation-planner\.sh|tests/test-diagnostic-settings\.sh|\.github/workflows/run-samples\.yml)$' if ! grep -q -E "$shared" <<< "$changed"; then selected=$(while read -r app; do if grep -q "^$app/" <<< "$changed"; then echo "$app"; fi done <<< "$apps") fi + # A deployment runs when its folder, one of the tests or this workflow changes + deployments=$(for deployment in bicep terraform; do + if grep -q -E "^($deployment/|tests/|\.github/workflows/run-samples\.yml$)" <<< "$changed"; then echo "$deployment"; fi + done) else # A push that creates the branch or rewrites its history leaves no base commit to compare with echo "Base commit '$base' not found: running every sample" @@ -106,15 +114,19 @@ jobs: *) echo "::error::Unknown platform '$PLATFORM' (expected Linux, Windows or all)"; exit 1 ;; esac - # One job per app and platform: the entry names the runner and the job, so a run lists only the jobs - # it runs, under their own names - matrix=$(jq -n -R -c --arg platform "$PLATFORM" ' - [inputs | select(length > 0) | split("/") | {sample: .[1], language: .[2]}] as $apps + # One job per app and platform, and one per deployment and platform: the entry names the runner and the + # job, so a run lists only the jobs it runs, under their own names. The script creates the cluster of the + # app jobs; each deployment job creates it with bicep/ or terraform/ and checks web-app-managed-identity + matrix=$(jq -n -R -c --arg platform "$PLATFORM" --arg deployments "$deployments" ' + ([inputs | select(length > 0) | split("/") | {sample: .[1], language: .[2], provisioning: "script", cluster: ""}] + + [$deployments | splits("\\s+") | select(length > 0) + | {sample: "web-app-managed-identity", language: "python", provisioning: ., + cluster: (if . == "bicep" then "Bicep cluster: " else "Terraform cluster: " end)}]) as $jobs | {include: ( (if $platform == "Windows" then [] else - [$apps[] | . + {os: "linux", runner: "ubuntu-latest", name: "\(.sample) (\(.language))"}] end) + [$jobs[] | . + {os: "linux", runner: "ubuntu-latest", name: "\(.cluster)\(.sample) (\(.language))"}] end) + (if $platform == "Linux" then [] else - [$apps[] | . + {os: "windows", runner: "windows-latest", name: "Windows (Git Bash) \(.sample) (\(.language))"}] end) + [$jobs[] | . + {os: "windows", runner: "windows-latest", name: "Windows (Git Bash) \(.cluster)\(.sample) (\(.language))"}] end) )}' <<< "$selected") echo "Jobs to run: $(jq -r '[.include[].name] | if length > 0 then join(", ") else "none" end' <<< "$matrix")" { @@ -147,6 +159,7 @@ jobs: env: SAMPLE: ${{ matrix.sample }} LANGUAGE: ${{ matrix.language }} + PROVISIONING: ${{ matrix.provisioning }} steps: - name: Check out repo @@ -347,11 +360,26 @@ jobs: fi lstk az start-interception - # The script prompts for the network configuration and both node pool sizes: - # pick option 1 (Azure network plugin and policy) and keep the default sizes + - name: Set up Terraform + if: matrix.provisioning == 'terraform' + uses: hashicorp/setup-terraform@dfe3c3f87815947d99a8997f908cb6525fc44e9e # v4.0.1 + with: + # deploy.sh runs terraform itself: the wrapper only serves steps that read terraform's outputs + terraform_wrapper: false + + # The script prompts for the network configuration and both node pool sizes: pick option 1 (Azure network + # plugin and policy) and keep the default sizes. bicep/deploy.sh and terraform/deploy.sh default to the same. - name: Create the AKS cluster - working-directory: scripts - run: printf '1\n\n\n' | ./01-system-assigned-managed-identity.sh + run: | + case $PROVISIONING in + script) (cd scripts && printf '1\n\n\n' | ./01-system-assigned-managed-identity.sh) ;; + bicep) bicep/deploy.sh ;; + terraform) terraform/deploy.sh ;; + *) echo "::error::Unknown provisioning '$PROVISIONING'"; exit 1 ;; + esac + + - name: Check the diagnostic settings of the cluster's resources + run: tests/test-diagnostic-settings.sh - name: Deploy the sample working-directory: samples/${{ matrix.sample }}/${{ matrix.language }}/scripts diff --git a/README.md b/README.md index 55d9a2a..8d8490d 100644 --- a/README.md +++ b/README.md @@ -142,9 +142,17 @@ Before deploying any sample you must first provision an AKS cluster. Four script [scripts/README.md](scripts/README.md) details what each script provisions, which samples need which feature, and the variables of the configurable script. +The same full cluster can also be deployed as infrastructure as code. [bicep/](bicep/) and [terraform/](terraform/) create the cluster of `01-system-assigned-managed-identity.sh`, or with a user-assigned identity the one of `01-user-assigned-managed-identity.sh`, with the same names and defaults, so every sample runs on it: + +| Deployment | Cluster identity | How to run it | +| ---------- | ---------------- | ------------- | +| [bicep/](bicep/README.md) | System-assigned, or user-assigned with `IDENTITY_TYPE=UserAssigned` | `cd bicep && ./deploy.sh` | +| [terraform/](terraform/README.md) | System-assigned, or user-assigned with `IDENTITY_TYPE=UserAssigned` | `cd terraform && ./deploy.sh` | + All four scripts are idempotent: they check whether each resource already exists before creating it, so they can be safely re-run. The two managed-identity scripts, and the configurable one with every default, provision the following building blocks: - A resource group and a Log Analytics workspace (wired to the cluster through the monitoring add-on / Azure Monitor for containers). +- A diagnostic setting called `default` on the cluster, the virtual network, the Bastion host and its public IP address, the container registry and the key vault, which sends their logs and metrics to the workspace. See [Monitoring](scripts/README.md#monitoring). - A dedicated virtual network (`10.0.0.0/8`) with three subnets: `SystemSubnet` (system node pool), `UserSubnet` (user node pool), and `AzureBastionSubnet`. - An [Azure Bastion](https://learn.microsoft.com/en-us/azure/bastion/bastion-overview) host in `AzureBastionSubnet`, with the Standard SKU static public IP it requires, for reaching the nodes without exposing SSH to the internet. Creating it takes several minutes. - An [Azure Container Registry (ACR)](https://learn.microsoft.com/en-us/azure/container-registry/container-registry-intro) that is attached to the cluster (via `--attach-acr`) so nodes can pull images without extra credentials. @@ -239,10 +247,11 @@ Alternatively, you can use a terminal UI such as [k9s](https://k9scli.io/) to se The [Test samples on LocalStack for Azure](.github/workflows/run-samples.yml) workflow runs this against the emulator, one job per sample app, in Python and .NET. Each job creates the cluster with `scripts/01-system-assigned-managed-identity.sh`, deploys the app with its numbered scripts, and runs the test. A pull request or a push to `main` runs the apps whose folders it changes, and all 20 when it changes the cluster script, the test or the workflow. A manual run tests the changed apps by default, or all 20 with `run_mode: all`. It also picks where they run with `platform`: `Linux` (the default), `Windows`, from Git Bash, or `all`, which runs both at once, in parallel jobs of the same run. +Two more jobs create the cluster with [bicep/](bicep/) and with [terraform/](terraform/) and deploy and test one app on each, `web-app-managed-identity` in Python. They run when their folder, the test scripts or the workflow change, and in every manual run with `run_mode: all`. Every job also checks the diagnostic settings of the cluster's resources with [tests/test-diagnostic-settings.sh](tests/test-diagnostic-settings.sh). ## Tutorials -Beyond the Vacation Planner samples, the repository includes standalone tutorials that exercise specific AKS capabilities. Unlike the samples above, they do not deploy the web app. +Beyond the Vacation Planner samples, the repository includes standalone tutorials that exercise specific AKS capabilities. Unlike the samples above, they do not deploy the web app. The Bicep and Terraform tutorials build their own feature-rich cluster, with other names and settings; to deploy the cluster the samples run on with Bicep or Terraform, use [bicep/](bicep/) and [terraform/](terraform/) instead. | Tutorial | Description | | ------ | ----------- | diff --git a/bicep/README.md b/bicep/README.md new file mode 100644 index 0000000..cf8bb09 --- /dev/null +++ b/bicep/README.md @@ -0,0 +1,129 @@ +# Bicep Deployment + +This directory contains the Bicep template and a deployment script that create the same Azure Kubernetes Service (AKS) cluster as [scripts/01-system-assigned-managed-identity.sh](../scripts/01-system-assigned-managed-identity.sh) or, with `identityType` set to `UserAssigned`, as [scripts/01-user-assigned-managed-identity.sh](../scripts/01-user-assigned-managed-identity.sh): the same resources, with the same names and defaults, including the [diagnostic settings](../scripts/README.md#monitoring). Every [sample](../samples/) runs on it. For the samples and the other ways to create the cluster, see the [README](../README.md) at the root of the repository. + +## Prerequisites + +- [LocalStack for Azure](https://docs.localstack.cloud/azure/), or an Azure subscription +- [Docker](https://docs.docker.com/get-docker/) +- [Azure CLI](https://learn.microsoft.com/en-us/cli/azure/install-azure-cli), which installs Bicep on first use, and the [lstk CLI](https://docs.localstack.cloud/aws/developer-tools/running-localstack/lstk/) +- [kubectl](https://kubernetes.io/docs/tasks/tools/), and on Azure [kubelogin](https://learn.microsoft.com/en-us/azure/aks/kubelogin-authentication) (`az aks install-cli` installs both) +- [Visual Studio Code](https://code.visualstudio.com/) + [Bicep extension](https://marketplace.visualstudio.com/items?itemName=ms-azuretools.vscode-bicep), optional + +```bash +brew install localstack/tap/lstk # or: npm install -g @localstack/lstk +``` + +## Architecture Overview + +The [`deploy.sh`](deploy.sh) script creates the `local-rg` resource group, while the Bicep modules create: + +1. [Azure Log Analytics Workspace](https://learn.microsoft.com/azure/azure-monitor/logs/log-analytics-overview) (`log-analytics.bicep`) `local-log-analytics-test`, `PerGB2018` pricing tier, with a system-assigned identity. +2. [Azure Key Vault](https://learn.microsoft.com/en-us/azure/key-vault/general/overview) (`key-vault.bicep`) `local-kv-test` with Azure RBAC authorization. +3. [Azure Container Registry](https://learn.microsoft.com/en-us/azure/container-registry/container-registry-intro) (`container-registry.bicep`) `localacrtest`, Basic SKU, with the admin user enabled. Every sample pushes its image to it. +4. [Azure Virtual Network](https://learn.microsoft.com/azure/virtual-network/virtual-networks-overview) (`virtual-network.bicep`) `local-vnet-test`, `10.0.0.0/8`, with three subnets: + - *SystemSubnet* (`10.240.0.0/16`) and *UserSubnet* (`10.241.0.0/16`) for the two node pools, both with the `Microsoft.Storage` service endpoint that the NFS volumes of the Azure Files CSI driver need. + - *AzureBastionSubnet* (`10.242.2.0/24`) for the Bastion host. +5. [Azure Bastion](https://learn.microsoft.com/en-us/azure/bastion/bastion-overview) (`bastion-host.bicep`) `local-bastion-test`, Basic SKU, with its Standard, static public IP address `local-bastion-ip-test`. +6. With `identityType` set to `UserAssigned`, the [user-assigned managed identity](https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/overview) `local-aks-identity-test` of the cluster (`managed-identity.bicep`). +7. The AKS cluster `local-aks-test` (`aks-cluster.bicep`): + - The latest GA Kubernetes version in the region, which `deploy.sh` looks up as the scripts do. + - The `system` node pool on *SystemSubnet*: `Standard_D4ds_v5` nodes with Azure Linux and a 50 GB ephemeral OS disk, one node with the cluster autoscaler between 1 and 3, 100 pods per node. + - Azure CNI Overlay, with the network policy and dataplane of the scripts' menu option (`networkOption`), pod CIDR `192.168.0.0/16`, service CIDR `172.16.0.0/16`, DNS service IP `172.16.0.10`, and [Advanced Container Networking Services](https://learn.microsoft.com/en-us/azure/aks/advanced-container-networking-services-overview). + - Microsoft Entra ID with Azure RBAC, the OIDC issuer and [Microsoft Entra Workload ID](https://learn.microsoft.com/en-us/azure/aks/workload-identity-overview). + - The `monitoring` add-on (Container Insights) with its `MSCI--local-aks-test` data collection rule and association, and the Key Vault secrets provider add-on. + - The managed Gateway API CRDs, KEDA, the Azure Files and Azure Disk CSI drivers and the CSI snapshot controller. +8. The `user` node pool (`aks-node-pool.bicep`) on *UserSubnet*, with the same node settings as the system pool, spread across availability zones 1, 2 and 3. +9. The role assignments of the scripts: Network Contributor on the virtual network and on *SystemSubnet* for the cluster identity (the second is what `az aks create --vnet-subnet-id` grants), Key Vault Administrator on the key vault for the identity of the Key Vault secrets provider add-on, and AcrPull on the registry for the kubelet identity (what `az aks create --attach-acr` grants). AKS itself grants the cluster identity the Contributor role on the node resource group, which the user-assigned script checks for. +10. A diagnostic setting called `default` on the cluster, the virtual network, the public IP address, the Bastion host, the registry and the key vault, inside each resource's module, which sends its logs and metrics to the workspace. See [Monitoring](../scripts/README.md#monitoring) for the log categories. + +## Configuration + +[`main.bicepparam`](main.bicepparam) holds the scripts' defaults: + +```bicep +using 'main.bicep' + +param prefix = 'local' +param suffix = 'test' +param identityType = 'SystemAssigned' +param networkOption = 1 +param systemNodeCount = 1 +param userNodeCount = 1 +param nodeVmSize = 'Standard_D4ds_v5' +param osSku = 'AzureLinux' +param osDiskSizeGB = 50 +param osDiskType = 'Ephemeral' +param maxPods = 100 +param windowsAdminUsername = 'azadmin' + +param windowsAdminPassword = readEnvironmentVariable('WINDOWS_ADMIN_PASSWORD') +``` + +`deploy.sh` passes the parameters below on the command line, from environment variables that default to the scripts' values. The scripts ask for the network option and the node counts; here they come from the environment: + +| Environment variable | Parameter | Default | Script variable | +| -------------------- | --------- | ------- | --------------- | +| `PREFIX` | `prefix` | `local` | `prefix` | +| `SUFFIX` | `suffix` | `test` | `suffix` | +| `LOCATION` | `location` | `italynorth` | `location` | +| `IDENTITY_TYPE` | `identityType` | `SystemAssigned` | The choice of script | +| `NETWORK_OPTION` | `networkOption` | `1` | The network plugin and policy menu: `1` Azure CNI Overlay with the Azure network policy, `2` with the Cilium dataplane and network policy, `3` with the Calico network policy | +| `SYSTEM_NODE_COUNT` | `systemNodeCount` | `1` | `node_count`, asked by the scripts | +| `USER_NODE_COUNT` | `userNodeCount` | `1` | `node_pool_node_count`, asked by the scripts | +| `AAD_ADMIN_GROUP_OBJECT_IDS` | `aadAdminGroupObjectIds` | `` | `aad_profile_admin_group_object_ids` | +| `SSH_PUBLIC_KEY_FILE` | `sshPublicKey`, the file's content | `~/.ssh/id_rsa.pub` | `ssh_key_value` | +| `WINDOWS_ADMIN_PASSWORD` | `windowsAdminPassword` | The scripts' password | `windows_admin_password` | +| | `kubernetesVersion` | The latest GA version in the region | `kubernetes_version` | + +The autoscaler range of each node pool is 1 to 3 nodes, widened to include the node count, as in the scripts. The admin group placeholder works on the emulator; on Azure, set `AAD_ADMIN_GROUP_OBJECT_IDS` to the comma-separated object IDs of real Microsoft Entra ID groups. The registry and key vault names are globally unique on Azure, so export your own `SUFFIX` there. + +## Deployment + +```bash +# the cluster of scripts/01-system-assigned-managed-identity.sh +./deploy.sh + +# the cluster of scripts/01-user-assigned-managed-identity.sh, with the Cilium dataplane and network policy +IDENTITY_TYPE=UserAssigned NETWORK_OPTION=2 ./deploy.sh + +# preview the changes of a re-deployment instead of validating the template +USE_WHAT_IF=1 ./deploy.sh +``` + +The script will: + +1. Look up the latest GA Kubernetes version in the region. +2. Ensure the resource group exists. +3. Validate `main.bicep` (or run a what-if with `USE_WHAT_IF=1`; `VALIDATE_TEMPLATE=0` skips both). +4. Deploy the template, passing the parameters above. +5. Configure `kubectl` for the cluster with `az aks get-credentials`. +6. Print the cluster's storage profile and storage classes, as the scripts do. + +Then deploy the samples as their READMEs describe, as on a cluster created by the scripts. + +## Verification + +```bash +# the diagnostic settings of the cluster's resources +../tests/test-diagnostic-settings.sh + +# the cluster and its node pools +az aks show --name local-aks-test --resource-group local-rg \ + --query "{identity: identity.type, network: networkProfile.networkPolicy, gatewayApi: ingressProfile}" --output json +az aks nodepool list --cluster-name local-aks-test --resource-group local-rg \ + --query "[].{name: name, mode: mode, count: count, zones: availabilityZones}" --output table +kubectl get nodes +``` + +## LocalStack emulator notes + +The template deploys on the emulator as it is, and the cluster it creates there matches the one the scripts create: the same resources, cluster settings, node pools, role assignments and diagnostic settings. + +## Related Documentation + +- [Microsoft.ContainerService managedClusters (Bicep)](https://learn.microsoft.com/en-us/azure/templates/microsoft.containerservice/managedclusters) +- [Microsoft.Insights diagnosticSettings (Bicep)](https://learn.microsoft.com/en-us/azure/templates/microsoft.insights/diagnosticsettings) +- [Monitor AKS: resource logs](https://learn.microsoft.com/en-us/azure/aks/monitor-aks#azure-monitor-resource-logs) +- [LocalStack for Azure Documentation](https://docs.localstack.cloud/azure/) +- [lstk CLI](https://docs.localstack.cloud/aws/developer-tools/running-localstack/lstk/) diff --git a/bicep/deploy.sh b/bicep/deploy.sh new file mode 100755 index 0000000..f489da7 --- /dev/null +++ b/bicep/deploy.sh @@ -0,0 +1,198 @@ +#!/bin/bash + +# Deploy with Bicep the AKS cluster that scripts/01-system-assigned-managed-identity.sh creates or, with +# IDENTITY_TYPE=UserAssigned, the one of scripts/01-user-assigned-managed-identity.sh: the same resources, names +# and defaults, including the diagnostic settings. It works the same way against Azure and against the LocalStack +# for Azure emulator: see "Run against LocalStack" in the README at the root of the repository. + +# Variables +# Every default below can be overridden through the environment, for example +# IDENTITY_TYPE=UserAssigned NETWORK_OPTION=2 ./deploy.sh +# The container registry and the key vault have globally unique names on Azure, so a run against a real +# subscription normally needs its own SUFFIX (or PREFIX). +PREFIX="${PREFIX:-local}" +SUFFIX="${SUFFIX:-test}" +LOCATION="${LOCATION:-italynorth}" +RESOURCE_GROUP_NAME="${RESOURCE_GROUP_NAME:-${PREFIX}-rg}" +AKS_CLUSTER_NAME="${PREFIX}-aks-${SUFFIX}" +IDENTITY_TYPE="${IDENTITY_TYPE:-SystemAssigned}" +NETWORK_OPTION="${NETWORK_OPTION:-1}" +SYSTEM_NODE_COUNT="${SYSTEM_NODE_COUNT:-1}" +USER_NODE_COUNT="${USER_NODE_COUNT:-1}" +AAD_ADMIN_GROUP_OBJECT_IDS="${AAD_ADMIN_GROUP_OBJECT_IDS:-}" +SSH_PUBLIC_KEY_FILE="${SSH_PUBLIC_KEY_FILE:-$HOME/.ssh/id_rsa.pub}" +# main.bicepparam reads it from the environment +export WINDOWS_ADMIN_PASSWORD="${WINDOWS_ADMIN_PASSWORD:-Trustno123456!}" +TEMPLATE="main.bicep" +PARAMETERS="main.bicepparam" +VALIDATE_TEMPLATE="${VALIDATE_TEMPLATE:-1}" +USE_WHAT_IF="${USE_WHAT_IF:-0}" +CURRENT_DIR="$(cd "$(dirname "$0")" && pwd)" + +# Check prerequisites +for tool in az kubectl; do + if ! command -v "$tool" >/dev/null 2>&1; then + echo "Error: required tool '$tool' is not installed or not in PATH." >&2 + exit 1 + fi +done + +# Change the current directory to the script's directory +cd "$CURRENT_DIR" || exit + +SUBSCRIPTION_NAME=$(az account show --query name --output tsv --only-show-errors) + +# Read the SSH public key of the Linux nodes, as the scripts pass ~/.ssh/id_rsa.pub +if [[ -f $SSH_PUBLIC_KEY_FILE ]]; then + SSH_PUBLIC_KEY=$(tr -d '\r\n' <"$SSH_PUBLIC_KEY_FILE") +else + echo "No SSH public key at [$SSH_PUBLIC_KEY_FILE]: the cluster is created without one" + SSH_PUBLIC_KEY="" +fi + +# The comma-separated object ids of the admin groups, as a JSON array +AAD_ADMIN_GROUP_OBJECT_IDS_JSON="[\"${AAD_ADMIN_GROUP_OBJECT_IDS//,/\",\"}\"]" + +# Keep the Kubernetes version of an existing cluster: like the scripts, use the latest version only to create one, +# so that a re-deployment does not upgrade the cluster +KUBERNETES_VERSION=$(az aks show \ + --name "$AKS_CLUSTER_NAME" \ + --resource-group "$RESOURCE_GROUP_NAME" \ + --query kubernetesVersion \ + --output tsv \ + --only-show-errors 2>/dev/null | tr -d '\r') + +if [[ -n $KUBERNETES_VERSION ]]; then + echo "The [$AKS_CLUSTER_NAME] AKS cluster exists and runs Kubernetes [$KUBERNETES_VERSION]: keeping that version" +else + # Get the last Kubernetes version available in the region, as the scripts do + KUBERNETES_VERSION=$(az aks get-versions \ + --location "$LOCATION" \ + --query "values[?isPreview==null].version | sort(@) | [-1]" \ + --output tsv \ + --only-show-errors 2>/dev/null | tr -d '\r') + + if [[ -n $KUBERNETES_VERSION ]]; then + echo "The latest Kubernetes version available in the [$LOCATION] region is [$KUBERNETES_VERSION]" + else + echo "Failed to retrieve the latest Kubernetes version available in the [$LOCATION] region: AKS picks its default" + fi +fi + +# The parameters that override main.bicepparam +parameters=( + location="$LOCATION" + prefix="$PREFIX" + suffix="$SUFFIX" + identityType="$IDENTITY_TYPE" + networkOption="$NETWORK_OPTION" + systemNodeCount="$SYSTEM_NODE_COUNT" + userNodeCount="$USER_NODE_COUNT" + kubernetesVersion="$KUBERNETES_VERSION" + sshPublicKey="$SSH_PUBLIC_KEY" + aadAdminGroupObjectIds="$AAD_ADMIN_GROUP_OBJECT_IDS_JSON" +) + +# Validates if the resource group exists in the subscription, if not creates it +echo "Checking if resource group [$RESOURCE_GROUP_NAME] exists in the subscription [$SUBSCRIPTION_NAME]..." +az group show --name "$RESOURCE_GROUP_NAME" &>/dev/null + +if [[ $? != 0 ]]; then + echo "No resource group [$RESOURCE_GROUP_NAME] exists in the subscription [$SUBSCRIPTION_NAME]" + echo "Creating resource group [$RESOURCE_GROUP_NAME] in the subscription [$SUBSCRIPTION_NAME]..." + + # Create the resource group + az group create \ + --name "$RESOURCE_GROUP_NAME" \ + --location "$LOCATION" \ + --only-show-errors 1>/dev/null + + if [[ $? == 0 ]]; then + echo "Resource group [$RESOURCE_GROUP_NAME] successfully created in the subscription [$SUBSCRIPTION_NAME]" + else + echo "Failed to create resource group [$RESOURCE_GROUP_NAME] in the subscription [$SUBSCRIPTION_NAME]" + exit 1 + fi +else + echo "Resource group [$RESOURCE_GROUP_NAME] already exists in the subscription [$SUBSCRIPTION_NAME]" +fi + +# Validates the Bicep template +if [[ $VALIDATE_TEMPLATE == 1 ]]; then + if [[ $USE_WHAT_IF == 1 ]]; then + # Execute a deployment What-If operation at resource group scope + echo "Previewing changes deployed by Bicep template [$TEMPLATE]..." + az deployment group what-if \ + --resource-group "$RESOURCE_GROUP_NAME" \ + --template-file "$TEMPLATE" \ + --parameters "$PARAMETERS" \ + --parameters "${parameters[@]}" \ + --only-show-errors + + if [[ $? == 0 ]]; then + echo "Bicep template [$TEMPLATE] validation succeeded" + else + echo "Failed to validate Bicep template [$TEMPLATE]" + exit 1 + fi + else + # Validate the Bicep template + echo "Validating Bicep template [$TEMPLATE]..." + output=$(az deployment group validate \ + --resource-group "$RESOURCE_GROUP_NAME" \ + --template-file "$TEMPLATE" \ + --parameters "$PARAMETERS" \ + --parameters "${parameters[@]}" \ + --only-show-errors) + + if [[ $? == 0 ]]; then + echo "Bicep template [$TEMPLATE] validation succeeded" + else + echo "Failed to validate Bicep template [$TEMPLATE]" + echo "$output" + exit 1 + fi + fi +fi + +# Deploy the Bicep template +echo "Deploying Bicep template [$TEMPLATE]..." +az deployment group create \ + --resource-group "$RESOURCE_GROUP_NAME" \ + --template-file "$TEMPLATE" \ + --parameters "$PARAMETERS" \ + --parameters "${parameters[@]}" \ + --only-show-errors 1>/dev/null + +if [[ $? == 0 ]]; then + echo "Bicep template [$TEMPLATE] deployed successfully" +else + echo "Failed to deploy Bicep template [$TEMPLATE]" + exit 1 +fi + +# Use the following command to configure kubectl to connect to the new Kubernetes cluster +echo "Getting access credentials configure kubectl to connect to the [$AKS_CLUSTER_NAME] AKS cluster..." +az aks get-credentials \ + --name "$AKS_CLUSTER_NAME" \ + --resource-group "$RESOURCE_GROUP_NAME" \ + --overwrite-existing \ + --only-show-errors + +if [[ $? == 0 ]]; then + echo "Credentials for the [$AKS_CLUSTER_NAME] cluster successfully retrieved" +else + echo "Failed to retrieve the credentials for the [$AKS_CLUSTER_NAME] cluster" + exit 1 +fi + +# Print the storage drivers enabled on the cluster, followed by the storage classes they provide, as the scripts do +echo "Storage profile of the [$AKS_CLUSTER_NAME] AKS cluster:" +az aks show \ + --name "$AKS_CLUSTER_NAME" \ + --resource-group "$RESOURCE_GROUP_NAME" \ + --query storageProfile \ + --output yaml \ + --only-show-errors +echo "Storage classes of the [$AKS_CLUSTER_NAME] AKS cluster:" +kubectl get storageclass diff --git a/bicep/main.bicep b/bicep/main.bicep new file mode 100644 index 0000000..bbb8c03 --- /dev/null +++ b/bicep/main.bicep @@ -0,0 +1,332 @@ +// The AKS cluster of scripts/01-system-assigned-managed-identity.sh and, with identityType set to UserAssigned, of +// scripts/01-user-assigned-managed-identity.sh: the same resources, names and defaults, including the diagnostic +// settings that send the logs and metrics of every resource to the Log Analytics workspace. + +@description('Specifies the prefix of every resource name, as in the scripts.') +@minLength(1) +param prefix string = 'local' + +@description('Specifies the suffix of every resource name, as in the scripts.') +@minLength(1) +param suffix string = 'test' + +@description('Specifies the location of every resource.') +param location string = resourceGroup().location + +@description('Specifies the cluster identity: SystemAssigned as in scripts/01-system-assigned-managed-identity.sh, UserAssigned as in scripts/01-user-assigned-managed-identity.sh.') +@allowed([ + 'SystemAssigned' + 'UserAssigned' +]) +param identityType string = 'SystemAssigned' + +@description('Specifies the network plugin and policy, as the scripts\' menu: 1 Azure CNI Overlay with the Azure network policy, 2 with the Cilium dataplane and network policy, 3 with the Calico network policy.') +@allowed([ + 1 + 2 + 3 +]) +param networkOption int = 1 + +@description('Specifies the Kubernetes version. Empty lets AKS pick its default; deploy.sh passes the latest GA version in the region, as the scripts do.') +param kubernetesVersion string = '' + +@description('Specifies the number of nodes of the system node pool. The autoscaler range, 1 to 3, is widened to include it.') +@minValue(1) +param systemNodeCount int = 1 + +@description('Specifies the number of nodes of the user node pool. The autoscaler range, 1 to 3, is widened to include it.') +@minValue(1) +param userNodeCount int = 1 + +@description('Specifies the VM size of the nodes of both node pools.') +param nodeVmSize string = 'Standard_D4ds_v5' + +@description('Specifies the OS SKU of the nodes.') +param osSku string = 'AzureLinux' + +@description('Specifies the OS disk size in GB of the nodes.') +param osDiskSizeGB int = 50 + +@description('Specifies the OS disk type of the nodes.') +@allowed([ + 'Ephemeral' + 'Managed' +]) +param osDiskType string = 'Ephemeral' + +@description('Specifies the maximum number of pods per node.') +param maxPods int = 100 + +@description('Specifies the SSH public key of the Linux nodes. deploy.sh reads ~/.ssh/id_rsa.pub, as the scripts do.') +param sshPublicKey string = '' + +@description('Specifies the administrator user name of Windows nodes.') +param windowsAdminUsername string = 'azadmin' + +@description('Specifies the administrator password of Windows nodes.') +@secure() +param windowsAdminPassword string + +@description('Specifies the object ids of the Microsoft Entra ID groups that administer the cluster. The placeholder works on the emulator; on Azure, pass real group object ids.') +param aadAdminGroupObjectIds array = [ + '' +] + +@description('Specifies the SKU of the container registry.') +param acrSku string = 'Basic' + +@description('Specifies the pricing tier of the Log Analytics workspace.') +param logAnalyticsSku string = 'PerGB2018' + +@description('Specifies the name of the diagnostic settings of every resource.') +param diagnosticSettingsName string = 'default' + +// Names, as in the scripts +var aksClusterName = '${prefix}-aks-${suffix}' +var aksManagedIdentityName = '${prefix}-aks-identity-${suffix}' +var keyVaultName = '${prefix}-kv-${suffix}' +var acrName = '${prefix}acr${suffix}' +var logAnalyticsName = '${prefix}-log-analytics-${suffix}' +var virtualNetworkName = '${prefix}-vnet-${suffix}' +var bastionPublicIpName = '${prefix}-bastion-ip-${suffix}' +var bastionHostName = '${prefix}-bastion-${suffix}' +var systemSubnetName = 'SystemSubnet' + +// Network policy and dataplane of each option of the scripts' menu +var networkOptions = { + '1': { + policy: 'azure' + dataplane: 'azure' + } + '2': { + policy: 'cilium' + dataplane: 'cilium' + } + '3': { + policy: 'calico' + dataplane: 'azure' + } +} + +// The node pools' tags and labels the scripts set +var nodeLabels = { + os_disk_type: osDiskType + os_type: 'Linux' +} +var provisioningTags = { + provisioning: 'bicep' +} + +// Built-in roles +var networkContributorRoleId = '4d97b98b-1d4f-4787-a291-c67834d212e7' +var keyVaultAdministratorRoleId = '00482a5a-887f-4fb3-b363-3b7fe8e74483' +var acrPullRoleId = '7f951dda-4ed3-4680-a7ca-43fe172d538d' + +module logAnalytics 'modules/log-analytics.bicep' = { + name: 'logAnalytics' + params: { + name: logAnalyticsName + location: location + sku: logAnalyticsSku + } +} + +module keyVault 'modules/key-vault.bicep' = { + name: 'keyVault' + params: { + name: keyVaultName + location: location + workspaceId: logAnalytics.outputs.id + diagnosticSettingsName: diagnosticSettingsName + } +} + +module containerRegistry 'modules/container-registry.bicep' = { + name: 'containerRegistry' + params: { + name: acrName + location: location + sku: acrSku + workspaceId: logAnalytics.outputs.id + diagnosticSettingsName: diagnosticSettingsName + } +} + +module virtualNetwork 'modules/virtual-network.bicep' = { + name: 'virtualNetwork' + params: { + name: virtualNetworkName + location: location + systemSubnetName: systemSubnetName + workspaceId: logAnalytics.outputs.id + diagnosticSettingsName: diagnosticSettingsName + } +} + +module bastionHost 'modules/bastion-host.bicep' = { + name: 'bastionHost' + params: { + name: bastionHostName + publicIpAddressName: bastionPublicIpName + location: location + subnetId: virtualNetwork.outputs.bastionSubnetId + workspaceId: logAnalytics.outputs.id + diagnosticSettingsName: diagnosticSettingsName + } +} + +module managedIdentity 'modules/managed-identity.bicep' = if (identityType == 'UserAssigned') { + name: 'managedIdentity' + params: { + name: aksManagedIdentityName + location: location + } +} + +module aksCluster 'modules/aks-cluster.bicep' = { + name: 'aksCluster' + dependsOn: [ + userAssignedSystemSubnetNetworkContributorRoleAssignment + ] + params: { + name: aksClusterName + location: location + tags: provisioningTags + kubernetesVersion: kubernetesVersion + identityType: identityType + userAssignedIdentityId: identityType == 'UserAssigned' ? managedIdentity!.outputs.id : '' + systemSubnetId: virtualNetwork.outputs.systemSubnetId + systemNodePoolVmSize: nodeVmSize + systemNodePoolNodeCount: systemNodeCount + systemNodePoolMinCount: min(1, systemNodeCount) + systemNodePoolMaxCount: max(3, systemNodeCount) + systemNodePoolMaxPods: maxPods + systemNodePoolOsDiskSizeGB: osDiskSizeGB + systemNodePoolOsDiskType: osDiskType + systemNodePoolOsSku: osSku + systemNodePoolTags: nodeLabels + systemNodePoolNodeLabels: nodeLabels + sshPublicKey: sshPublicKey + windowsAdminUsername: windowsAdminUsername + windowsAdminPassword: windowsAdminPassword + networkPolicy: networkOptions[string(networkOption)].policy + networkDataplane: networkOptions[string(networkOption)].dataplane + aadAdminGroupObjectIds: aadAdminGroupObjectIds + workspaceId: logAnalytics.outputs.id + diagnosticSettingsName: diagnosticSettingsName + } +} + +// Added after the cluster identity's Network Contributor role assignments, as the scripts add it +module userNodePool 'modules/aks-node-pool.bicep' = { + name: 'userNodePool' + dependsOn: [ + networkContributorRoleAssignment + systemAssignedSystemSubnetNetworkContributorRoleAssignment + ] + params: { + clusterName: aksCluster.outputs.name + subnetId: virtualNetwork.outputs.userSubnetId + vmSize: nodeVmSize + nodeCount: userNodeCount + minCount: min(1, userNodeCount) + maxCount: max(3, userNodeCount) + maxPods: maxPods + osDiskSizeGB: osDiskSizeGB + osDiskType: osDiskType + osSku: osSku + tags: provisioningTags + nodeLabels: nodeLabels + } +} + +resource existingVirtualNetwork 'Microsoft.Network/virtualNetworks@2024-05-01' existing = { + name: virtualNetworkName +} + +resource existingSystemSubnet 'Microsoft.Network/virtualNetworks/subnets@2024-05-01' existing = { + parent: existingVirtualNetwork + name: systemSubnetName +} + +resource existingKeyVault 'Microsoft.KeyVault/vaults@2024-04-01-preview' existing = { + name: keyVaultName +} + +resource existingContainerRegistry 'Microsoft.ContainerRegistry/registries@2024-11-01-preview' existing = { + name: acrName +} + +// Network Contributor on the virtual network for the cluster identity +resource networkContributorRoleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(existingVirtualNetwork.id, aksClusterName, networkContributorRoleId) + scope: existingVirtualNetwork + properties: { + roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', networkContributorRoleId) + principalId: identityType == 'UserAssigned' ? managedIdentity!.outputs.principalId : aksCluster.outputs.principalId + principalType: 'ServicePrincipal' + } +} + +// Network Contributor on the subnet of the system node pool for the cluster identity, which az aks create +// --vnet-subnet-id grants as well: before it creates the cluster for a user-assigned identity, after it for a +// system-assigned one, which exists only once the cluster does +resource userAssignedSystemSubnetNetworkContributorRoleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = if (identityType == 'UserAssigned') { + name: guid(existingSystemSubnet.id, aksManagedIdentityName, networkContributorRoleId) + scope: existingSystemSubnet + dependsOn: [ + virtualNetwork + ] + properties: { + roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', networkContributorRoleId) + principalId: managedIdentity!.outputs.principalId + principalType: 'ServicePrincipal' + } +} + +resource systemAssignedSystemSubnetNetworkContributorRoleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = if (identityType == 'SystemAssigned') { + name: guid(existingSystemSubnet.id, aksClusterName, networkContributorRoleId) + scope: existingSystemSubnet + properties: { + roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', networkContributorRoleId) + principalId: aksCluster.outputs.principalId + principalType: 'ServicePrincipal' + } +} + +// AKS itself grants the cluster identity, system-assigned or user-assigned, the Contributor role on the node resource +// group: scripts/01-user-assigned-managed-identity.sh finds that assignment, and a second one would conflict with it + +// Key Vault Administrator on the key vault for the identity of the Key Vault secrets provider add-on +resource keyVaultAdministratorRoleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(existingKeyVault.id, aksClusterName, keyVaultAdministratorRoleId) + scope: existingKeyVault + dependsOn: [ + keyVault + ] + properties: { + roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', keyVaultAdministratorRoleId) + principalId: aksCluster.outputs.keyVaultSecretsProviderIdentityObjectId + principalType: 'ServicePrincipal' + } +} + +// AcrPull on the container registry for the kubelet identity, which is what az aks create --attach-acr grants +resource acrPullRoleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(existingContainerRegistry.id, aksClusterName, acrPullRoleId) + scope: existingContainerRegistry + dependsOn: [ + containerRegistry + ] + properties: { + roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', acrPullRoleId) + principalId: aksCluster.outputs.kubeletIdentityObjectId + principalType: 'ServicePrincipal' + } +} + +output aksClusterName string = aksCluster.outputs.name +output acrName string = containerRegistry.outputs.name +output keyVaultName string = keyVault.outputs.name +output logAnalyticsName string = logAnalytics.outputs.name diff --git a/bicep/main.bicepparam b/bicep/main.bicepparam new file mode 100644 index 0000000..af78779 --- /dev/null +++ b/bicep/main.bicepparam @@ -0,0 +1,18 @@ +using 'main.bicep' + +// The defaults of the cluster scripts. deploy.sh overrides each of them from its environment variables. +param prefix = 'local' +param suffix = 'test' +param identityType = 'SystemAssigned' +param networkOption = 1 +param systemNodeCount = 1 +param userNodeCount = 1 +param nodeVmSize = 'Standard_D4ds_v5' +param osSku = 'AzureLinux' +param osDiskSizeGB = 50 +param osDiskType = 'Ephemeral' +param maxPods = 100 +param windowsAdminUsername = 'azadmin' + +// Read when the parameters are built: deploy.sh exports it, with the scripts' default +param windowsAdminPassword = readEnvironmentVariable('WINDOWS_ADMIN_PASSWORD') diff --git a/bicep/modules/aks-cluster.bicep b/bicep/modules/aks-cluster.bicep new file mode 100644 index 0000000..569f9ab --- /dev/null +++ b/bicep/modules/aks-cluster.bicep @@ -0,0 +1,340 @@ +// AKS cluster of the cluster scripts: the system node pool on SystemSubnet, Microsoft Entra ID with Azure RBAC, +// Workload ID, the monitoring (Container Insights) and Key Vault secrets provider add-ons, ACNS, the managed Gateway +// API, KEDA, the Azure Files CSI driver and the snapshot controller, and the diagnostic settings that send the +// control plane logs and the metrics to the Log Analytics workspace. The user node pool is in aks-node-pool.bicep. + +@description('Specifies the name of the AKS cluster.') +param name string + +@description('Specifies the location of the AKS cluster.') +param location string = resourceGroup().location + +@description('Specifies the tags of the AKS cluster.') +param tags object = {} + +@description('Specifies the Kubernetes version. Empty lets AKS pick its default.') +param kubernetesVersion string = '' + +@description('Specifies the type of the cluster identity.') +@allowed([ + 'SystemAssigned' + 'UserAssigned' +]) +param identityType string = 'SystemAssigned' + +@description('Specifies the resource id of the user-assigned identity of the cluster, when identityType is UserAssigned.') +param userAssignedIdentityId string = '' + +@description('Specifies the name of the system node pool.') +param systemNodePoolName string = 'system' + +@description('Specifies the resource id of the subnet of the system node pool.') +param systemSubnetId string + +@description('Specifies the VM size of the system node pool.') +param systemNodePoolVmSize string = 'Standard_D4ds_v5' + +@description('Specifies the number of nodes of the system node pool.') +param systemNodePoolNodeCount int = 1 + +@description('Specifies the minimum number of nodes of the system node pool for the cluster autoscaler.') +param systemNodePoolMinCount int = 1 + +@description('Specifies the maximum number of nodes of the system node pool for the cluster autoscaler.') +param systemNodePoolMaxCount int = 3 + +@description('Specifies the maximum number of pods per node of the system node pool.') +param systemNodePoolMaxPods int = 100 + +@description('Specifies the OS disk size in GB of the system node pool.') +param systemNodePoolOsDiskSizeGB int = 50 + +@description('Specifies the OS disk type of the system node pool.') +@allowed([ + 'Ephemeral' + 'Managed' +]) +param systemNodePoolOsDiskType string = 'Ephemeral' + +@description('Specifies the OS SKU of the system node pool.') +param systemNodePoolOsSku string = 'AzureLinux' + +@description('Specifies the tags of the system node pool.') +param systemNodePoolTags object = {} + +@description('Specifies the node labels of the system node pool.') +param systemNodePoolNodeLabels object = {} + +@description('Specifies the administrator user name of the Linux nodes.') +param linuxAdminUsername string = 'azureuser' + +@description('Specifies the SSH public key of the Linux nodes. Empty omits the Linux profile.') +param sshPublicKey string = '' + +@description('Specifies the administrator user name of Windows nodes.') +param windowsAdminUsername string = 'azadmin' + +@description('Specifies the administrator password of Windows nodes.') +@secure() +param windowsAdminPassword string + +@description('Specifies the network policy.') +@allowed([ + 'azure' + 'calico' + 'cilium' +]) +param networkPolicy string = 'azure' + +@description('Specifies the network dataplane.') +@allowed([ + 'azure' + 'cilium' +]) +param networkDataplane string = 'azure' + +@description('Specifies the CIDR of the pods.') +param podCidr string = '192.168.0.0/16' + +@description('Specifies the CIDR of the services.') +param serviceCidr string = '172.16.0.0/16' + +@description('Specifies the IP address of the Kubernetes DNS service, within serviceCidr.') +param dnsServiceIP string = '172.16.0.10' + +@description('Specifies the object ids of the Microsoft Entra ID groups that administer the cluster.') +param aadAdminGroupObjectIds array = [] + +@description('Specifies the resource id of the Log Analytics workspace.') +param workspaceId string + +@description('Specifies the name of the diagnostic settings.') +param diagnosticSettingsName string = 'default' + +// The control plane logs the diagnostic settings collect. kube-audit is left out on purpose: it adds every get and +// list request to what kube-audit-admin collects, which can be costly on Azure. +var logCategories = [ + 'kube-apiserver' + 'kube-audit-admin' + 'kube-controller-manager' + 'kube-scheduler' + 'cluster-autoscaler' + 'cloud-controller-manager' + 'guard' + 'csi-azurefile-controller' + 'csi-snapshot-controller' +] + +// The stream that the Container Insights data collection rule sends to the workspace +var containerInsightsStreams = [ + 'Microsoft-ContainerInsights-Group-Default' +] + +resource aksCluster 'Microsoft.ContainerService/managedClusters@2026-04-02-preview' = { + name: name + location: location + tags: tags + sku: { + name: 'Base' + tier: 'Free' + } + identity: identityType == 'UserAssigned' + ? { + type: 'UserAssigned' + userAssignedIdentities: { + '${userAssignedIdentityId}': {} + } + } + : { + type: 'SystemAssigned' + } + properties: { + kubernetesVersion: empty(kubernetesVersion) ? null : kubernetesVersion + // The DNS prefix az aks create generates: the first 10 characters of the cluster name, the first 16 of the + // resource group name and the first 6 of the subscription id + dnsPrefix: '${take(name, 10)}-${take(resourceGroup().name, 16)}-${take(subscription().subscriptionId, 6)}' + enableRBAC: true + agentPoolProfiles: [ + { + name: systemNodePoolName + mode: 'System' + count: systemNodePoolNodeCount + vmSize: systemNodePoolVmSize + osType: 'Linux' + osSKU: systemNodePoolOsSku + osDiskSizeGB: systemNodePoolOsDiskSizeGB + osDiskType: systemNodePoolOsDiskType + vnetSubnetID: systemSubnetId + maxPods: systemNodePoolMaxPods + enableAutoScaling: true + minCount: systemNodePoolMinCount + maxCount: systemNodePoolMaxCount + tags: systemNodePoolTags + nodeLabels: systemNodePoolNodeLabels + type: 'VirtualMachineScaleSets' + } + ] + linuxProfile: empty(sshPublicKey) + ? null + : { + adminUsername: linuxAdminUsername + ssh: { + publicKeys: [ + { + keyData: sshPublicKey + } + ] + } + } + windowsProfile: { + adminUsername: windowsAdminUsername + adminPassword: windowsAdminPassword + } + addonProfiles: { + omsagent: { + enabled: true + config: { + logAnalyticsWorkspaceResourceID: workspaceId + useAADAuth: 'true' + } + } + azureKeyvaultSecretsProvider: { + enabled: true + config: { + enableSecretRotation: 'false' + rotationPollInterval: '2m' + } + } + } + oidcIssuerProfile: { + enabled: true + } + securityProfile: { + workloadIdentity: { + enabled: true + } + } + aadProfile: { + managed: true + enableAzureRBAC: true + adminGroupObjectIDs: aadAdminGroupObjectIds + tenantID: subscription().tenantId + } + ingressProfile: { + gatewayAPI: { + installation: 'Standard' + } + } + workloadAutoScalerProfile: { + keda: { + enabled: true + } + } + networkProfile: { + networkPlugin: 'azure' + networkPluginMode: 'overlay' + networkPolicy: networkPolicy + networkDataplane: networkDataplane + podCidr: podCidr + serviceCidr: serviceCidr + dnsServiceIP: dnsServiceIP + outboundType: 'loadBalancer' + loadBalancerSku: 'standard' + // Advanced Container Networking Services, exactly as az aks create --enable-acns requests them: AKS turns on + // observability, and the security features (FQDN filtering) only with the Cilium dataplane, which they need + advancedNetworking: { + enabled: true + } + } + storageProfile: { + diskCSIDriver: { + enabled: true + } + fileCSIDriver: { + enabled: true + } + snapshotController: { + enabled: true + } + } + } +} + +// Container Insights collects through a data collection rule associated with the cluster, which az aks create +// --enable-addons monitoring creates with these names +resource dataCollectionRule 'Microsoft.Insights/dataCollectionRules@2023-03-11' = { + name: 'MSCI-${location}-${name}' + location: location + kind: 'Linux' + properties: { + dataSources: { + extensions: [ + { + name: 'ContainerInsightsExtension' + extensionName: 'ContainerInsights' + streams: containerInsightsStreams + extensionSettings: { + dataCollectionSettings: { + interval: '1m' + namespaceFilteringMode: 'Off' + enableContainerLogV2: true + } + } + } + ] + } + destinations: { + logAnalytics: [ + { + name: 'la-workspace' + workspaceResourceId: workspaceId + } + ] + } + dataFlows: [ + { + streams: containerInsightsStreams + destinations: [ + 'la-workspace' + ] + } + ] + } +} + +resource dataCollectionRuleAssociation 'Microsoft.Insights/dataCollectionRuleAssociations@2023-03-11' = { + name: 'ContainerInsightsExtension' + scope: aksCluster + properties: { + dataCollectionRuleId: dataCollectionRule.id + description: 'Association of data collection rule. Deleting this association will break the data collection for this AKS Cluster.' + } +} + +// The control plane logs go to the resource-specific tables of the workspace (AKSAuditAdmin, AKSControlPlane) +resource diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { + name: diagnosticSettingsName + scope: aksCluster + properties: { + workspaceId: workspaceId + logAnalyticsDestinationType: 'Dedicated' + logs: [ + for category in logCategories: { + category: category + enabled: true + } + ] + metrics: [ + { + category: 'AllMetrics' + enabled: true + } + ] + } +} + +output id string = aksCluster.id +output name string = aksCluster.name +output principalId string = identityType == 'SystemAssigned' ? aksCluster.identity.principalId : '' +output kubeletIdentityObjectId string = aksCluster.properties.identityProfile.kubeletidentity.objectId +output keyVaultSecretsProviderIdentityObjectId string = aksCluster.properties.addonProfiles.azureKeyvaultSecretsProvider.identity.objectId diff --git a/bicep/modules/aks-node-pool.bicep b/bicep/modules/aks-node-pool.bicep new file mode 100644 index 0000000..3351c67 --- /dev/null +++ b/bicep/modules/aks-node-pool.bicep @@ -0,0 +1,80 @@ +// User node pool of the AKS cluster, as the cluster scripts add it with az aks nodepool add + +@description('Specifies the name of the AKS cluster.') +param clusterName string + +@description('Specifies the name of the node pool.') +param name string = 'user' + +@description('Specifies the resource id of the subnet of the node pool.') +param subnetId string + +@description('Specifies the VM size of the nodes.') +param vmSize string = 'Standard_D4ds_v5' + +@description('Specifies the number of nodes.') +param nodeCount int = 1 + +@description('Specifies the minimum number of nodes for the cluster autoscaler.') +param minCount int = 1 + +@description('Specifies the maximum number of nodes for the cluster autoscaler.') +param maxCount int = 3 + +@description('Specifies the maximum number of pods per node.') +param maxPods int = 100 + +@description('Specifies the OS disk size in GB.') +param osDiskSizeGB int = 50 + +@description('Specifies the OS disk type.') +@allowed([ + 'Ephemeral' + 'Managed' +]) +param osDiskType string = 'Ephemeral' + +@description('Specifies the OS SKU.') +param osSku string = 'AzureLinux' + +@description('Specifies the availability zones the nodes are spread across.') +param availabilityZones array = [ + '1' + '2' + '3' +] + +@description('Specifies the tags of the node pool.') +param tags object = {} + +@description('Specifies the node labels of the node pool.') +param nodeLabels object = {} + +resource aksCluster 'Microsoft.ContainerService/managedClusters@2026-04-02-preview' existing = { + name: clusterName +} + +resource nodePool 'Microsoft.ContainerService/managedClusters/agentPools@2026-04-02-preview' = { + parent: aksCluster + name: name + properties: { + mode: 'User' + count: nodeCount + vmSize: vmSize + osType: 'Linux' + osSKU: osSku + osDiskSizeGB: osDiskSizeGB + osDiskType: osDiskType + vnetSubnetID: subnetId + maxPods: maxPods + enableAutoScaling: true + minCount: minCount + maxCount: maxCount + availabilityZones: availabilityZones + tags: tags + nodeLabels: nodeLabels + type: 'VirtualMachineScaleSets' + } +} + +output name string = nodePool.name diff --git a/bicep/modules/bastion-host.bicep b/bicep/modules/bastion-host.bicep new file mode 100644 index 0000000..2fab1cb --- /dev/null +++ b/bicep/modules/bastion-host.bicep @@ -0,0 +1,114 @@ +// Bastion host and its public IP address, and the diagnostic settings that send their logs and metrics to the +// Log Analytics workspace + +@description('Specifies the name of the Bastion host.') +param name string + +@description('Specifies the name of the public IP address of the Bastion host.') +param publicIpAddressName string + +@description('Specifies the location of the Bastion host and its public IP address.') +param location string = resourceGroup().location + +@description('Specifies the SKU of the Bastion host.') +@allowed([ + 'Basic' + 'Standard' +]) +param sku string = 'Basic' + +@description('Specifies the resource id of the AzureBastionSubnet subnet.') +param subnetId string + +@description('Specifies the resource id of the Log Analytics workspace.') +param workspaceId string + +@description('Specifies the name of the diagnostic settings.') +param diagnosticSettingsName string = 'default' + +// Azure Bastion requires a Standard SKU public IP address with a static allocation method +resource publicIpAddress 'Microsoft.Network/publicIPAddresses@2024-05-01' = { + name: publicIpAddressName + location: location + sku: { + name: 'Standard' + } + properties: { + publicIPAllocationMethod: 'Static' + } +} + +resource bastionHost 'Microsoft.Network/bastionHosts@2024-05-01' = { + name: name + location: location + sku: { + name: sku + } + properties: { + ipConfigurations: [ + { + name: 'ipconfig' + properties: { + subnet: { + id: subnetId + } + publicIPAddress: { + id: publicIpAddress.id + } + } + } + ] + } +} + +// The DDoS categories only receive data while Azure DDoS Protection mitigates an attack on the address +resource publicIpAddressDiagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { + name: diagnosticSettingsName + scope: publicIpAddress + properties: { + workspaceId: workspaceId + logs: [ + { + category: 'DDoSProtectionNotifications' + enabled: true + } + { + category: 'DDoSMitigationFlowLogs' + enabled: true + } + { + category: 'DDoSMitigationReports' + enabled: true + } + ] + metrics: [ + { + category: 'AllMetrics' + enabled: true + } + ] + } +} + +resource bastionHostDiagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { + name: diagnosticSettingsName + scope: bastionHost + properties: { + workspaceId: workspaceId + logs: [ + { + category: 'BastionAuditLogs' + enabled: true + } + ] + metrics: [ + { + category: 'AllMetrics' + enabled: true + } + ] + } +} + +output id string = bastionHost.id +output publicIpAddressId string = publicIpAddress.id diff --git a/bicep/modules/container-registry.bicep b/bicep/modules/container-registry.bicep new file mode 100644 index 0000000..f921ccb --- /dev/null +++ b/bicep/modules/container-registry.bicep @@ -0,0 +1,61 @@ +// Container registry that every sample pushes its image to, and the diagnostic settings that send its logs and +// metrics to the Log Analytics workspace + +@description('Specifies the name of the container registry.') +param name string + +@description('Specifies the location of the container registry.') +param location string = resourceGroup().location + +@description('Specifies the SKU of the container registry.') +@allowed([ + 'Basic' + 'Standard' + 'Premium' +]) +param sku string = 'Basic' + +@description('Specifies the resource id of the Log Analytics workspace.') +param workspaceId string + +@description('Specifies the name of the diagnostic settings.') +param diagnosticSettingsName string = 'default' + +resource containerRegistry 'Microsoft.ContainerRegistry/registries@2024-11-01-preview' = { + name: name + location: location + sku: { + name: sku + } + properties: { + // The scripts create the registry with --admin-enabled true + adminUserEnabled: true + } +} + +resource diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { + name: diagnosticSettingsName + scope: containerRegistry + properties: { + workspaceId: workspaceId + logs: [ + { + category: 'ContainerRegistryRepositoryEvents' + enabled: true + } + { + category: 'ContainerRegistryLoginEvents' + enabled: true + } + ] + metrics: [ + { + category: 'AllMetrics' + enabled: true + } + ] + } +} + +output id string = containerRegistry.id +output name string = containerRegistry.name diff --git a/bicep/modules/key-vault.bicep b/bicep/modules/key-vault.bicep new file mode 100644 index 0000000..e3d7935 --- /dev/null +++ b/bicep/modules/key-vault.bicep @@ -0,0 +1,54 @@ +// Key vault with Azure RBAC authorization, and the diagnostic settings that send its logs and metrics to the +// Log Analytics workspace + +@description('Specifies the name of the key vault.') +param name string + +@description('Specifies the location of the key vault.') +param location string = resourceGroup().location + +@description('Specifies the resource id of the Log Analytics workspace.') +param workspaceId string + +@description('Specifies the name of the diagnostic settings.') +param diagnosticSettingsName string = 'default' + +resource keyVault 'Microsoft.KeyVault/vaults@2024-04-01-preview' = { + name: name + location: location + properties: { + tenantId: subscription().tenantId + sku: { + family: 'A' + name: 'standard' + } + enableRbacAuthorization: true + } +} + +resource diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { + name: diagnosticSettingsName + scope: keyVault + properties: { + workspaceId: workspaceId + logs: [ + { + category: 'AuditEvent' + enabled: true + } + { + category: 'AzurePolicyEvaluationDetails' + enabled: true + } + ] + metrics: [ + { + category: 'AllMetrics' + enabled: true + } + ] + } +} + +output id string = keyVault.id +output name string = keyVault.name diff --git a/bicep/modules/log-analytics.bicep b/bicep/modules/log-analytics.bicep new file mode 100644 index 0000000..cdb3a7b --- /dev/null +++ b/bicep/modules/log-analytics.bicep @@ -0,0 +1,27 @@ +// Log Analytics workspace that Container Insights and the diagnostic settings of every resource send their data to + +@description('Specifies the name of the Log Analytics workspace.') +param name string + +@description('Specifies the location of the Log Analytics workspace.') +param location string = resourceGroup().location + +@description('Specifies the pricing tier of the Log Analytics workspace.') +param sku string = 'PerGB2018' + +resource workspace 'Microsoft.OperationalInsights/workspaces@2023-09-01' = { + name: name + location: location + // The scripts create the workspace with --identity-type SystemAssigned + identity: { + type: 'SystemAssigned' + } + properties: { + sku: { + name: sku + } + } +} + +output id string = workspace.id +output name string = workspace.name diff --git a/bicep/modules/managed-identity.bicep b/bicep/modules/managed-identity.bicep new file mode 100644 index 0000000..129a438 --- /dev/null +++ b/bicep/modules/managed-identity.bicep @@ -0,0 +1,16 @@ +// User-assigned managed identity of the AKS cluster, as scripts/01-user-assigned-managed-identity.sh creates it + +@description('Specifies the name of the user-assigned managed identity.') +param name string + +@description('Specifies the location of the user-assigned managed identity.') +param location string = resourceGroup().location + +resource managedIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2024-11-30' = { + name: name + location: location +} + +output id string = managedIdentity.id +output name string = managedIdentity.name +output principalId string = managedIdentity.properties.principalId diff --git a/bicep/modules/virtual-network.bicep b/bicep/modules/virtual-network.bicep new file mode 100644 index 0000000..687c94f --- /dev/null +++ b/bicep/modules/virtual-network.bicep @@ -0,0 +1,101 @@ +// Virtual network with the subnets of the two node pools and of the Bastion host, and the diagnostic settings that +// send its logs and metrics to the Log Analytics workspace + +@description('Specifies the name of the virtual network.') +param name string + +@description('Specifies the location of the virtual network.') +param location string = resourceGroup().location + +@description('Specifies the address prefix of the virtual network.') +param addressPrefix string = '10.0.0.0/8' + +@description('Specifies the name of the subnet of the system node pool.') +param systemSubnetName string = 'SystemSubnet' + +@description('Specifies the address prefix of the subnet of the system node pool.') +param systemSubnetPrefix string = '10.240.0.0/16' + +@description('Specifies the name of the subnet of the user node pool.') +param userSubnetName string = 'UserSubnet' + +@description('Specifies the address prefix of the subnet of the user node pool.') +param userSubnetPrefix string = '10.241.0.0/16' + +@description('Specifies the address prefix of the AzureBastionSubnet subnet.') +param bastionSubnetPrefix string = '10.242.2.0/24' + +@description('Specifies the resource id of the Log Analytics workspace.') +param workspaceId string + +@description('Specifies the name of the diagnostic settings.') +param diagnosticSettingsName string = 'default' + +// An Azure Files NFS share can only be reached from a virtual network, so the Azure Files CSI driver needs the +// Microsoft.Storage service endpoint on the node subnets to mount an NFS volume +var storageServiceEndpoints = [ + { + service: 'Microsoft.Storage' + } +] + +resource virtualNetwork 'Microsoft.Network/virtualNetworks@2024-05-01' = { + name: name + location: location + properties: { + addressSpace: { + addressPrefixes: [ + addressPrefix + ] + } + subnets: [ + { + name: systemSubnetName + properties: { + addressPrefix: systemSubnetPrefix + serviceEndpoints: storageServiceEndpoints + } + } + { + name: userSubnetName + properties: { + addressPrefix: userSubnetPrefix + serviceEndpoints: storageServiceEndpoints + } + } + { + // Azure Bastion looks this subnet up by name + name: 'AzureBastionSubnet' + properties: { + addressPrefix: bastionSubnetPrefix + } + } + ] + } +} + +resource diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { + name: diagnosticSettingsName + scope: virtualNetwork + properties: { + workspaceId: workspaceId + logs: [ + { + category: 'VMProtectionAlerts' + enabled: true + } + ] + metrics: [ + { + category: 'AllMetrics' + enabled: true + } + ] + } +} + +output id string = virtualNetwork.id +output name string = virtualNetwork.name +output systemSubnetId string = resourceId('Microsoft.Network/virtualNetworks/subnets', name, systemSubnetName) +output userSubnetId string = resourceId('Microsoft.Network/virtualNetworks/subnets', name, userSubnetName) +output bastionSubnetId string = resourceId('Microsoft.Network/virtualNetworks/subnets', name, 'AzureBastionSubnet') diff --git a/scripts/01-configurable-cluster.sh b/scripts/01-configurable-cluster.sh index 1830afd..903761d 100755 --- a/scripts/01-configurable-cluster.sh +++ b/scripts/01-configurable-cluster.sh @@ -75,6 +75,9 @@ minimum_gateway_api_cli_version="2.86.0" log_analytics_name="$prefix-log-analytics-$suffix" log_analytics_sku="PerGB2018" +# Diagnostic settings that send the logs and metrics of the cluster's resources to the Log Analytics workspace +diagnostic_settings_name="default" + # Node size and ssh key location for AKS nodes. Without the key file the cluster is created with --no-ssh-key: # az aks create refuses to run with neither. node_size="Standard_D4ds_v5" @@ -377,7 +380,7 @@ ask_questions() { ask_bool ENABLE_CLUSTER_AUTOSCALER "Enable the cluster autoscaler on the node pools?" true ask_bool ENABLE_WORKLOAD_IDENTITY "Enable Microsoft Entra Workload ID?" true ask_bool CREATE_KEY_VAULT "Create a Key Vault with the CSI secrets provider?" true - ask_bool ENABLE_MONITORING "Enable Container Insights with Log Analytics?" true + ask_bool ENABLE_MONITORING "Enable Container Insights and diagnostic settings with Log Analytics?" true ask_bool ENABLE_FILE_CSI_DRIVER "Enable the Azure Files CSI driver?" true ask_bool ENABLE_DISK_CSI_DRIVER "Enable the Azure Disk CSI driver?" true @@ -478,7 +481,7 @@ print_resources() { echo " - Managed identity [$aks_managed_identity_name]" fi if [[ $ENABLE_MONITORING == true ]]; then - echo " - Log Analytics workspace [$log_analytics_name]" + echo " - Log Analytics workspace [$log_analytics_name], and [$diagnostic_settings_name] diagnostic settings that send the logs and metrics of these resources to it" fi if [[ $CREATE_KEY_VAULT == true ]]; then echo " - Key Vault [$key_vault_name]" @@ -536,7 +539,7 @@ print_summary() { summary_line "Cluster autoscaler" "$(yes_no "$ENABLE_CLUSTER_AUTOSCALER")" summary_line "Workload ID" "$(yes_no "$ENABLE_WORKLOAD_IDENTITY")" summary_line "Key Vault + CSI secrets provider" "$(yes_no "$CREATE_KEY_VAULT")" - summary_line "Monitoring (Container Insights)" "$(yes_no "$ENABLE_MONITORING")" + summary_line "Monitoring + diagnostic settings" "$(yes_no "$ENABLE_MONITORING")" summary_line "Azure Files CSI driver" "$(yes_no "$ENABLE_FILE_CSI_DRIVER")" summary_line "Azure Disk CSI driver" "$(yes_no "$ENABLE_DISK_CSI_DRIVER")" summary_line "Custom virtual network" "$(yes_no "$CREATE_VNET")" @@ -1345,6 +1348,106 @@ ensure_role_assignments() { fi } +# ensure_diagnostic_setting DESCRIPTION RESOURCE_ID LOGS METRICS [ARGS...]: create the diagnostic settings that send +# a resource's logs and metrics to the Log Analytics workspace, unless they exist +ensure_diagnostic_setting() { + local description=$1 resource_id=$2 logs=$3 metrics=$4 + shift 4 + log "Checking if [$diagnostic_settings_name] diagnostic settings for the $description already exist..." + if exists az monitor diagnostic-settings show --name "$diagnostic_settings_name" --resource "$resource_id" --only-show-errors; then + log "[$diagnostic_settings_name] diagnostic settings for the $description already exist" + return + fi + log "Creating [$diagnostic_settings_name] diagnostic settings for the $description..." + run az monitor diagnostic-settings create \ + --name "$diagnostic_settings_name" \ + --resource "$resource_id" \ + --workspace "$workspace_resource_id" \ + --logs "$logs" \ + --metrics "$metrics" \ + "$@" \ + --only-show-errors >/dev/null || + die "Failed to create [$diagnostic_settings_name] diagnostic settings for the $description" + log "[$diagnostic_settings_name] diagnostic settings for the $description successfully created" +} + +# Create the diagnostic settings of every resource the answers provisioned. The AKS control plane logs go to the +# resource-specific tables of the workspace (AKSAuditAdmin, AKSControlPlane). kube-audit is left out on purpose: +# it adds every get and list request to what kube-audit-admin collects, which can be costly on Azure. +ensure_diagnostic_settings() { + local all_metrics='[{"category": "AllMetrics", "enabled": true}]' aks_cluster_id acr_id bastion_public_ip_id bastion_host_id + aks_cluster_id=$(query "" az aks show \ + --name "$aks_cluster_name" \ + --resource-group "$resource_group_name" \ + --query id \ + --output tsv \ + --only-show-errors) + [[ -n $aks_cluster_id ]] || die "Failed to retrieve the id for the [$aks_cluster_name] AKS cluster" + ensure_diagnostic_setting "[$aks_cluster_name] AKS cluster" "$aks_cluster_id" '[ + {"category": "kube-apiserver", "enabled": true}, + {"category": "kube-audit-admin", "enabled": true}, + {"category": "kube-controller-manager", "enabled": true}, + {"category": "kube-scheduler", "enabled": true}, + {"category": "cluster-autoscaler", "enabled": true}, + {"category": "cloud-controller-manager", "enabled": true}, + {"category": "guard", "enabled": true}, + {"category": "csi-azurefile-controller", "enabled": true}, + {"category": "csi-snapshot-controller", "enabled": true} + ]' "$all_metrics" --export-to-resource-specific true + + acr_id=$(query "" az acr show \ + --name "$acr_name" \ + --resource-group "$resource_group_name" \ + --query id \ + --output tsv \ + --only-show-errors) + [[ -n $acr_id ]] || die "Failed to retrieve the id for the [$acr_name] container registry" + ensure_diagnostic_setting "[$acr_name] container registry" "$acr_id" '[ + {"category": "ContainerRegistryRepositoryEvents", "enabled": true}, + {"category": "ContainerRegistryLoginEvents", "enabled": true} + ]' "$all_metrics" + + if [[ $CREATE_VNET == true ]]; then + ensure_diagnostic_setting "[$virtual_network_name] virtual network" "$virtual_network_id" '[ + {"category": "VMProtectionAlerts", "enabled": true} + ]' "$all_metrics" + fi + + if [[ $CREATE_BASTION == true ]]; then + bastion_public_ip_id=$(query "" az network public-ip show \ + --name "$bastion_public_ip_name" \ + --resource-group "$resource_group_name" \ + --query id \ + --output tsv \ + --only-show-errors) + [[ -n $bastion_public_ip_id ]] || die "Failed to retrieve the id for the [$bastion_public_ip_name] public IP address" + # The DDoS categories only receive data while Azure DDoS Protection mitigates an attack on the address + ensure_diagnostic_setting "[$bastion_public_ip_name] public IP address" "$bastion_public_ip_id" '[ + {"category": "DDoSProtectionNotifications", "enabled": true}, + {"category": "DDoSMitigationFlowLogs", "enabled": true}, + {"category": "DDoSMitigationReports", "enabled": true} + ]' "$all_metrics" + + bastion_host_id=$(query "" az network bastion show \ + --name "$bastion_host_name" \ + --resource-group "$resource_group_name" \ + --query id \ + --output tsv \ + --only-show-errors) + [[ -n $bastion_host_id ]] || die "Failed to retrieve the id for the [$bastion_host_name] bastion host" + ensure_diagnostic_setting "[$bastion_host_name] bastion host" "$bastion_host_id" '[ + {"category": "BastionAuditLogs", "enabled": true} + ]' "$all_metrics" + fi + + if [[ $CREATE_KEY_VAULT == true ]]; then + ensure_diagnostic_setting "[$key_vault_name] key vault" "$key_vault_id" '[ + {"category": "AuditEvent", "enabled": true}, + {"category": "AzurePolicyEvaluationDetails", "enabled": true} + ]' "$all_metrics" + fi +} + ensure_user_node_pool() { local args log "Checking if [$aks_cluster_name] aks cluster actually has a user node pool..." @@ -1527,6 +1630,9 @@ main() { install_gateway_controllers fi wait_job bastion + if [[ $ENABLE_MONITORING == true ]]; then + ensure_diagnostic_settings + fi print_storage if [[ $DRY_RUN != true ]]; then diff --git a/scripts/01-system-assigned-managed-identity.sh b/scripts/01-system-assigned-managed-identity.sh index 81d766e..d15bbcc 100755 --- a/scripts/01-system-assigned-managed-identity.sh +++ b/scripts/01-system-assigned-managed-identity.sh @@ -75,6 +75,9 @@ aad_profile_admin_group_object_ids="" log_analytics_name="$prefix-log-analytics-$suffix" log_analytics_sku="PerGB2018" +# Diagnostic settings that send the logs and metrics of the cluster's resources to the Log Analytics workspace +diagnostic_settings_name="default" + # Node count, node size, and ssh key location for AKS nodes node_size="Standard_D4ds_v5" # Standard_D4ds_v4 ssh_key_value="$(native_path "$HOME/.ssh/id_rsa.pub")" @@ -968,6 +971,286 @@ else fi fi +# Retrieve the container registry id +acr_id=$(az acr show \ + --name $acr_name \ + --resource-group $resource_group_name \ + --query id \ + --output tsv \ + --only-show-errors 2>/dev/null) + +if [[ -n $acr_id ]]; then + echo "Successfully retrieved the id for the [$acr_name] container registry" +else + echo "Failed to retrieve the id for the [$acr_name] container registry" + exit 1 +fi + +# Retrieve the id of the public IP address of the bastion host +bastion_public_ip_id=$(az network public-ip show \ + --name $bastion_public_ip_name \ + --resource-group $resource_group_name \ + --query id \ + --output tsv \ + --only-show-errors 2>/dev/null) + +if [[ -n $bastion_public_ip_id ]]; then + echo "Successfully retrieved the id for the [$bastion_public_ip_name] public IP address" +else + echo "Failed to retrieve the id for the [$bastion_public_ip_name] public IP address" + exit 1 +fi + +# Retrieve the bastion host id +bastion_host_id=$(az network bastion show \ + --name $bastion_host_name \ + --resource-group $resource_group_name \ + --query id \ + --output tsv \ + --only-show-errors 2>/dev/null) + +if [[ -n $bastion_host_id ]]; then + echo "Successfully retrieved the id for the [$bastion_host_name] bastion host" +else + echo "Failed to retrieve the id for the [$bastion_host_name] bastion host" + exit 1 +fi + +# Retrieve the AKS cluster id +aks_cluster_id=$(az aks show \ + --name $aks_cluster_name \ + --resource-group $resource_group_name \ + --query id \ + --output tsv \ + --only-show-errors 2>/dev/null) + +if [[ -n $aks_cluster_id ]]; then + echo "Successfully retrieved the id for the [$aks_cluster_name] AKS cluster" +else + echo "Failed to retrieve the id for the [$aks_cluster_name] AKS cluster" + exit 1 +fi + +# Check whether the diagnostic settings for the AKS cluster already exist +echo "Checking if [$diagnostic_settings_name] diagnostic settings for the [$aks_cluster_name] AKS cluster already exist..." +az monitor diagnostic-settings show \ + --name "$diagnostic_settings_name" \ + --resource "$aks_cluster_id" \ + --only-show-errors &>/dev/null + +if [[ $? != 0 ]]; then + echo "No [$diagnostic_settings_name] diagnostic settings for the [$aks_cluster_name] AKS cluster actually exist" + echo "Creating [$diagnostic_settings_name] diagnostic settings for the [$aks_cluster_name] AKS cluster..." + + # Create the diagnostic settings for the AKS cluster to send the control plane logs to the resource-specific + # tables of the Log Analytics workspace (AKSAuditAdmin, AKSControlPlane). kube-audit is left out on purpose: + # it adds every get and list request to what kube-audit-admin collects, which can be costly on Azure. + az monitor diagnostic-settings create \ + --name "$diagnostic_settings_name" \ + --resource "$aks_cluster_id" \ + --workspace "$workspace_resource_id" \ + --export-to-resource-specific true \ + --logs '[ + {"category": "kube-apiserver", "enabled": true}, + {"category": "kube-audit-admin", "enabled": true}, + {"category": "kube-controller-manager", "enabled": true}, + {"category": "kube-scheduler", "enabled": true}, + {"category": "cluster-autoscaler", "enabled": true}, + {"category": "cloud-controller-manager", "enabled": true}, + {"category": "guard", "enabled": true}, + {"category": "csi-azurefile-controller", "enabled": true}, + {"category": "csi-snapshot-controller", "enabled": true} + ]' \ + --metrics '[ + {"category": "AllMetrics", "enabled": true} + ]' \ + --only-show-errors 1>/dev/null + + if [[ $? == 0 ]]; then + echo "[$diagnostic_settings_name] diagnostic settings for the [$aks_cluster_name] AKS cluster successfully created" + else + echo "Failed to create [$diagnostic_settings_name] diagnostic settings for the [$aks_cluster_name] AKS cluster" + exit 1 + fi +else + echo "[$diagnostic_settings_name] diagnostic settings for the [$aks_cluster_name] AKS cluster already exist" +fi + +# Check whether the diagnostic settings for the virtual network already exist +echo "Checking if [$diagnostic_settings_name] diagnostic settings for the [$virtual_network_name] virtual network already exist..." +az monitor diagnostic-settings show \ + --name "$diagnostic_settings_name" \ + --resource "$virtual_network_id" \ + --only-show-errors &>/dev/null + +if [[ $? != 0 ]]; then + echo "No [$diagnostic_settings_name] diagnostic settings for the [$virtual_network_name] virtual network actually exist" + echo "Creating [$diagnostic_settings_name] diagnostic settings for the [$virtual_network_name] virtual network..." + + # Create the diagnostic settings for the virtual network to send logs and metrics to the Log Analytics workspace + az monitor diagnostic-settings create \ + --name "$diagnostic_settings_name" \ + --resource "$virtual_network_id" \ + --workspace "$workspace_resource_id" \ + --logs '[ + {"category": "VMProtectionAlerts", "enabled": true} + ]' \ + --metrics '[ + {"category": "AllMetrics", "enabled": true} + ]' \ + --only-show-errors 1>/dev/null + + if [[ $? == 0 ]]; then + echo "[$diagnostic_settings_name] diagnostic settings for the [$virtual_network_name] virtual network successfully created" + else + echo "Failed to create [$diagnostic_settings_name] diagnostic settings for the [$virtual_network_name] virtual network" + exit 1 + fi +else + echo "[$diagnostic_settings_name] diagnostic settings for the [$virtual_network_name] virtual network already exist" +fi + +# Check whether the diagnostic settings for the public IP address of the bastion host already exist +echo "Checking if [$diagnostic_settings_name] diagnostic settings for the [$bastion_public_ip_name] public IP address already exist..." +az monitor diagnostic-settings show \ + --name "$diagnostic_settings_name" \ + --resource "$bastion_public_ip_id" \ + --only-show-errors &>/dev/null + +if [[ $? != 0 ]]; then + echo "No [$diagnostic_settings_name] diagnostic settings for the [$bastion_public_ip_name] public IP address actually exist" + echo "Creating [$diagnostic_settings_name] diagnostic settings for the [$bastion_public_ip_name] public IP address..." + + # Create the diagnostic settings for the public IP address to send logs and metrics to the Log Analytics workspace. + # The DDoS categories only receive data while Azure DDoS Protection mitigates an attack on the address. + az monitor diagnostic-settings create \ + --name "$diagnostic_settings_name" \ + --resource "$bastion_public_ip_id" \ + --workspace "$workspace_resource_id" \ + --logs '[ + {"category": "DDoSProtectionNotifications", "enabled": true}, + {"category": "DDoSMitigationFlowLogs", "enabled": true}, + {"category": "DDoSMitigationReports", "enabled": true} + ]' \ + --metrics '[ + {"category": "AllMetrics", "enabled": true} + ]' \ + --only-show-errors 1>/dev/null + + if [[ $? == 0 ]]; then + echo "[$diagnostic_settings_name] diagnostic settings for the [$bastion_public_ip_name] public IP address successfully created" + else + echo "Failed to create [$diagnostic_settings_name] diagnostic settings for the [$bastion_public_ip_name] public IP address" + exit 1 + fi +else + echo "[$diagnostic_settings_name] diagnostic settings for the [$bastion_public_ip_name] public IP address already exist" +fi + +# Check whether the diagnostic settings for the bastion host already exist +echo "Checking if [$diagnostic_settings_name] diagnostic settings for the [$bastion_host_name] bastion host already exist..." +az monitor diagnostic-settings show \ + --name "$diagnostic_settings_name" \ + --resource "$bastion_host_id" \ + --only-show-errors &>/dev/null + +if [[ $? != 0 ]]; then + echo "No [$diagnostic_settings_name] diagnostic settings for the [$bastion_host_name] bastion host actually exist" + echo "Creating [$diagnostic_settings_name] diagnostic settings for the [$bastion_host_name] bastion host..." + + # Create the diagnostic settings for the bastion host to send logs and metrics to the Log Analytics workspace + az monitor diagnostic-settings create \ + --name "$diagnostic_settings_name" \ + --resource "$bastion_host_id" \ + --workspace "$workspace_resource_id" \ + --logs '[ + {"category": "BastionAuditLogs", "enabled": true} + ]' \ + --metrics '[ + {"category": "AllMetrics", "enabled": true} + ]' \ + --only-show-errors 1>/dev/null + + if [[ $? == 0 ]]; then + echo "[$diagnostic_settings_name] diagnostic settings for the [$bastion_host_name] bastion host successfully created" + else + echo "Failed to create [$diagnostic_settings_name] diagnostic settings for the [$bastion_host_name] bastion host" + exit 1 + fi +else + echo "[$diagnostic_settings_name] diagnostic settings for the [$bastion_host_name] bastion host already exist" +fi + +# Check whether the diagnostic settings for the container registry already exist +echo "Checking if [$diagnostic_settings_name] diagnostic settings for the [$acr_name] container registry already exist..." +az monitor diagnostic-settings show \ + --name "$diagnostic_settings_name" \ + --resource "$acr_id" \ + --only-show-errors &>/dev/null + +if [[ $? != 0 ]]; then + echo "No [$diagnostic_settings_name] diagnostic settings for the [$acr_name] container registry actually exist" + echo "Creating [$diagnostic_settings_name] diagnostic settings for the [$acr_name] container registry..." + + # Create the diagnostic settings for the container registry to send logs and metrics to the Log Analytics workspace + az monitor diagnostic-settings create \ + --name "$diagnostic_settings_name" \ + --resource "$acr_id" \ + --workspace "$workspace_resource_id" \ + --logs '[ + {"category": "ContainerRegistryRepositoryEvents", "enabled": true}, + {"category": "ContainerRegistryLoginEvents", "enabled": true} + ]' \ + --metrics '[ + {"category": "AllMetrics", "enabled": true} + ]' \ + --only-show-errors 1>/dev/null + + if [[ $? == 0 ]]; then + echo "[$diagnostic_settings_name] diagnostic settings for the [$acr_name] container registry successfully created" + else + echo "Failed to create [$diagnostic_settings_name] diagnostic settings for the [$acr_name] container registry" + exit 1 + fi +else + echo "[$diagnostic_settings_name] diagnostic settings for the [$acr_name] container registry already exist" +fi + +# Check whether the diagnostic settings for the key vault already exist +echo "Checking if [$diagnostic_settings_name] diagnostic settings for the [$key_vault_name] key vault already exist..." +az monitor diagnostic-settings show \ + --name "$diagnostic_settings_name" \ + --resource "$key_vault_id" \ + --only-show-errors &>/dev/null + +if [[ $? != 0 ]]; then + echo "No [$diagnostic_settings_name] diagnostic settings for the [$key_vault_name] key vault actually exist" + echo "Creating [$diagnostic_settings_name] diagnostic settings for the [$key_vault_name] key vault..." + + # Create the diagnostic settings for the key vault to send logs and metrics to the Log Analytics workspace + az monitor diagnostic-settings create \ + --name "$diagnostic_settings_name" \ + --resource "$key_vault_id" \ + --workspace "$workspace_resource_id" \ + --logs '[ + {"category": "AuditEvent", "enabled": true}, + {"category": "AzurePolicyEvaluationDetails", "enabled": true} + ]' \ + --metrics '[ + {"category": "AllMetrics", "enabled": true} + ]' \ + --only-show-errors 1>/dev/null + + if [[ $? == 0 ]]; then + echo "[$diagnostic_settings_name] diagnostic settings for the [$key_vault_name] key vault successfully created" + else + echo "Failed to create [$diagnostic_settings_name] diagnostic settings for the [$key_vault_name] key vault" + exit 1 + fi +else + echo "[$diagnostic_settings_name] diagnostic settings for the [$key_vault_name] key vault already exist" +fi + # Use the following command to configure kubectl to connect to the new Kubernetes cluster echo "Getting access credentials configure kubectl to connect to the [$aks_cluster_name] AKS cluster..." az aks get-credentials \ diff --git a/scripts/01-user-assigned-managed-identity.sh b/scripts/01-user-assigned-managed-identity.sh index 86be2be..7922c35 100755 --- a/scripts/01-user-assigned-managed-identity.sh +++ b/scripts/01-user-assigned-managed-identity.sh @@ -76,6 +76,9 @@ aad_profile_admin_group_object_ids="" log_analytics_name="$prefix-log-analytics-$suffix" log_analytics_sku="PerGB2018" +# Diagnostic settings that send the logs and metrics of the cluster's resources to the Log Analytics workspace +diagnostic_settings_name="default" + # Node count, node size, and ssh key location for AKS nodes node_size="Standard_D4ds_v5" # Standard_D4ds_v4 ssh_key_value="$(native_path "$HOME/.ssh/id_rsa.pub")" @@ -1078,6 +1081,286 @@ else fi fi +# Retrieve the container registry id +acr_id=$(az acr show \ + --name $acr_name \ + --resource-group $resource_group_name \ + --query id \ + --output tsv \ + --only-show-errors 2>/dev/null) + +if [[ -n $acr_id ]]; then + echo "Successfully retrieved the id for the [$acr_name] container registry" +else + echo "Failed to retrieve the id for the [$acr_name] container registry" + exit 1 +fi + +# Retrieve the id of the public IP address of the bastion host +bastion_public_ip_id=$(az network public-ip show \ + --name $bastion_public_ip_name \ + --resource-group $resource_group_name \ + --query id \ + --output tsv \ + --only-show-errors 2>/dev/null) + +if [[ -n $bastion_public_ip_id ]]; then + echo "Successfully retrieved the id for the [$bastion_public_ip_name] public IP address" +else + echo "Failed to retrieve the id for the [$bastion_public_ip_name] public IP address" + exit 1 +fi + +# Retrieve the bastion host id +bastion_host_id=$(az network bastion show \ + --name $bastion_host_name \ + --resource-group $resource_group_name \ + --query id \ + --output tsv \ + --only-show-errors 2>/dev/null) + +if [[ -n $bastion_host_id ]]; then + echo "Successfully retrieved the id for the [$bastion_host_name] bastion host" +else + echo "Failed to retrieve the id for the [$bastion_host_name] bastion host" + exit 1 +fi + +# Retrieve the AKS cluster id +aks_cluster_id=$(az aks show \ + --name $aks_cluster_name \ + --resource-group $resource_group_name \ + --query id \ + --output tsv \ + --only-show-errors 2>/dev/null) + +if [[ -n $aks_cluster_id ]]; then + echo "Successfully retrieved the id for the [$aks_cluster_name] AKS cluster" +else + echo "Failed to retrieve the id for the [$aks_cluster_name] AKS cluster" + exit 1 +fi + +# Check whether the diagnostic settings for the AKS cluster already exist +echo "Checking if [$diagnostic_settings_name] diagnostic settings for the [$aks_cluster_name] AKS cluster already exist..." +az monitor diagnostic-settings show \ + --name "$diagnostic_settings_name" \ + --resource "$aks_cluster_id" \ + --only-show-errors &>/dev/null + +if [[ $? != 0 ]]; then + echo "No [$diagnostic_settings_name] diagnostic settings for the [$aks_cluster_name] AKS cluster actually exist" + echo "Creating [$diagnostic_settings_name] diagnostic settings for the [$aks_cluster_name] AKS cluster..." + + # Create the diagnostic settings for the AKS cluster to send the control plane logs to the resource-specific + # tables of the Log Analytics workspace (AKSAuditAdmin, AKSControlPlane). kube-audit is left out on purpose: + # it adds every get and list request to what kube-audit-admin collects, which can be costly on Azure. + az monitor diagnostic-settings create \ + --name "$diagnostic_settings_name" \ + --resource "$aks_cluster_id" \ + --workspace "$workspace_resource_id" \ + --export-to-resource-specific true \ + --logs '[ + {"category": "kube-apiserver", "enabled": true}, + {"category": "kube-audit-admin", "enabled": true}, + {"category": "kube-controller-manager", "enabled": true}, + {"category": "kube-scheduler", "enabled": true}, + {"category": "cluster-autoscaler", "enabled": true}, + {"category": "cloud-controller-manager", "enabled": true}, + {"category": "guard", "enabled": true}, + {"category": "csi-azurefile-controller", "enabled": true}, + {"category": "csi-snapshot-controller", "enabled": true} + ]' \ + --metrics '[ + {"category": "AllMetrics", "enabled": true} + ]' \ + --only-show-errors 1>/dev/null + + if [[ $? == 0 ]]; then + echo "[$diagnostic_settings_name] diagnostic settings for the [$aks_cluster_name] AKS cluster successfully created" + else + echo "Failed to create [$diagnostic_settings_name] diagnostic settings for the [$aks_cluster_name] AKS cluster" + exit 1 + fi +else + echo "[$diagnostic_settings_name] diagnostic settings for the [$aks_cluster_name] AKS cluster already exist" +fi + +# Check whether the diagnostic settings for the virtual network already exist +echo "Checking if [$diagnostic_settings_name] diagnostic settings for the [$virtual_network_name] virtual network already exist..." +az monitor diagnostic-settings show \ + --name "$diagnostic_settings_name" \ + --resource "$virtual_network_id" \ + --only-show-errors &>/dev/null + +if [[ $? != 0 ]]; then + echo "No [$diagnostic_settings_name] diagnostic settings for the [$virtual_network_name] virtual network actually exist" + echo "Creating [$diagnostic_settings_name] diagnostic settings for the [$virtual_network_name] virtual network..." + + # Create the diagnostic settings for the virtual network to send logs and metrics to the Log Analytics workspace + az monitor diagnostic-settings create \ + --name "$diagnostic_settings_name" \ + --resource "$virtual_network_id" \ + --workspace "$workspace_resource_id" \ + --logs '[ + {"category": "VMProtectionAlerts", "enabled": true} + ]' \ + --metrics '[ + {"category": "AllMetrics", "enabled": true} + ]' \ + --only-show-errors 1>/dev/null + + if [[ $? == 0 ]]; then + echo "[$diagnostic_settings_name] diagnostic settings for the [$virtual_network_name] virtual network successfully created" + else + echo "Failed to create [$diagnostic_settings_name] diagnostic settings for the [$virtual_network_name] virtual network" + exit 1 + fi +else + echo "[$diagnostic_settings_name] diagnostic settings for the [$virtual_network_name] virtual network already exist" +fi + +# Check whether the diagnostic settings for the public IP address of the bastion host already exist +echo "Checking if [$diagnostic_settings_name] diagnostic settings for the [$bastion_public_ip_name] public IP address already exist..." +az monitor diagnostic-settings show \ + --name "$diagnostic_settings_name" \ + --resource "$bastion_public_ip_id" \ + --only-show-errors &>/dev/null + +if [[ $? != 0 ]]; then + echo "No [$diagnostic_settings_name] diagnostic settings for the [$bastion_public_ip_name] public IP address actually exist" + echo "Creating [$diagnostic_settings_name] diagnostic settings for the [$bastion_public_ip_name] public IP address..." + + # Create the diagnostic settings for the public IP address to send logs and metrics to the Log Analytics workspace. + # The DDoS categories only receive data while Azure DDoS Protection mitigates an attack on the address. + az monitor diagnostic-settings create \ + --name "$diagnostic_settings_name" \ + --resource "$bastion_public_ip_id" \ + --workspace "$workspace_resource_id" \ + --logs '[ + {"category": "DDoSProtectionNotifications", "enabled": true}, + {"category": "DDoSMitigationFlowLogs", "enabled": true}, + {"category": "DDoSMitigationReports", "enabled": true} + ]' \ + --metrics '[ + {"category": "AllMetrics", "enabled": true} + ]' \ + --only-show-errors 1>/dev/null + + if [[ $? == 0 ]]; then + echo "[$diagnostic_settings_name] diagnostic settings for the [$bastion_public_ip_name] public IP address successfully created" + else + echo "Failed to create [$diagnostic_settings_name] diagnostic settings for the [$bastion_public_ip_name] public IP address" + exit 1 + fi +else + echo "[$diagnostic_settings_name] diagnostic settings for the [$bastion_public_ip_name] public IP address already exist" +fi + +# Check whether the diagnostic settings for the bastion host already exist +echo "Checking if [$diagnostic_settings_name] diagnostic settings for the [$bastion_host_name] bastion host already exist..." +az monitor diagnostic-settings show \ + --name "$diagnostic_settings_name" \ + --resource "$bastion_host_id" \ + --only-show-errors &>/dev/null + +if [[ $? != 0 ]]; then + echo "No [$diagnostic_settings_name] diagnostic settings for the [$bastion_host_name] bastion host actually exist" + echo "Creating [$diagnostic_settings_name] diagnostic settings for the [$bastion_host_name] bastion host..." + + # Create the diagnostic settings for the bastion host to send logs and metrics to the Log Analytics workspace + az monitor diagnostic-settings create \ + --name "$diagnostic_settings_name" \ + --resource "$bastion_host_id" \ + --workspace "$workspace_resource_id" \ + --logs '[ + {"category": "BastionAuditLogs", "enabled": true} + ]' \ + --metrics '[ + {"category": "AllMetrics", "enabled": true} + ]' \ + --only-show-errors 1>/dev/null + + if [[ $? == 0 ]]; then + echo "[$diagnostic_settings_name] diagnostic settings for the [$bastion_host_name] bastion host successfully created" + else + echo "Failed to create [$diagnostic_settings_name] diagnostic settings for the [$bastion_host_name] bastion host" + exit 1 + fi +else + echo "[$diagnostic_settings_name] diagnostic settings for the [$bastion_host_name] bastion host already exist" +fi + +# Check whether the diagnostic settings for the container registry already exist +echo "Checking if [$diagnostic_settings_name] diagnostic settings for the [$acr_name] container registry already exist..." +az monitor diagnostic-settings show \ + --name "$diagnostic_settings_name" \ + --resource "$acr_id" \ + --only-show-errors &>/dev/null + +if [[ $? != 0 ]]; then + echo "No [$diagnostic_settings_name] diagnostic settings for the [$acr_name] container registry actually exist" + echo "Creating [$diagnostic_settings_name] diagnostic settings for the [$acr_name] container registry..." + + # Create the diagnostic settings for the container registry to send logs and metrics to the Log Analytics workspace + az monitor diagnostic-settings create \ + --name "$diagnostic_settings_name" \ + --resource "$acr_id" \ + --workspace "$workspace_resource_id" \ + --logs '[ + {"category": "ContainerRegistryRepositoryEvents", "enabled": true}, + {"category": "ContainerRegistryLoginEvents", "enabled": true} + ]' \ + --metrics '[ + {"category": "AllMetrics", "enabled": true} + ]' \ + --only-show-errors 1>/dev/null + + if [[ $? == 0 ]]; then + echo "[$diagnostic_settings_name] diagnostic settings for the [$acr_name] container registry successfully created" + else + echo "Failed to create [$diagnostic_settings_name] diagnostic settings for the [$acr_name] container registry" + exit 1 + fi +else + echo "[$diagnostic_settings_name] diagnostic settings for the [$acr_name] container registry already exist" +fi + +# Check whether the diagnostic settings for the key vault already exist +echo "Checking if [$diagnostic_settings_name] diagnostic settings for the [$key_vault_name] key vault already exist..." +az monitor diagnostic-settings show \ + --name "$diagnostic_settings_name" \ + --resource "$key_vault_id" \ + --only-show-errors &>/dev/null + +if [[ $? != 0 ]]; then + echo "No [$diagnostic_settings_name] diagnostic settings for the [$key_vault_name] key vault actually exist" + echo "Creating [$diagnostic_settings_name] diagnostic settings for the [$key_vault_name] key vault..." + + # Create the diagnostic settings for the key vault to send logs and metrics to the Log Analytics workspace + az monitor diagnostic-settings create \ + --name "$diagnostic_settings_name" \ + --resource "$key_vault_id" \ + --workspace "$workspace_resource_id" \ + --logs '[ + {"category": "AuditEvent", "enabled": true}, + {"category": "AzurePolicyEvaluationDetails", "enabled": true} + ]' \ + --metrics '[ + {"category": "AllMetrics", "enabled": true} + ]' \ + --only-show-errors 1>/dev/null + + if [[ $? == 0 ]]; then + echo "[$diagnostic_settings_name] diagnostic settings for the [$key_vault_name] key vault successfully created" + else + echo "Failed to create [$diagnostic_settings_name] diagnostic settings for the [$key_vault_name] key vault" + exit 1 + fi +else + echo "[$diagnostic_settings_name] diagnostic settings for the [$key_vault_name] key vault already exist" +fi + # Use the following command to configure kubectl to connect to the new Kubernetes cluster echo "Getting access credentials configure kubectl to connect to the [$aks_cluster_name] AKS cluster..." az aks get-credentials \ diff --git a/scripts/README.md b/scripts/README.md index abe84bb..1800c9d 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -10,7 +10,7 @@ Every cluster script creates the `local-aks-test` cluster in the `local-rg` reso | ------ | ----------- | -------- | | [01-basic-cluster.sh](01-basic-cluster.sh) | No questions: the smallest and fastest cluster to provision, for demos. To save time it turns off the Azure Files and Azure Disk CSI drivers and the CSI snapshot controller, so it runs every sample except [web-app-file-storage](../samples/web-app-file-storage/) and [web-app-in-cluster-postgresql](../samples/web-app-in-cluster-postgresql/). | System-assigned identity, Azure CNI Overlay with the Azure network policy, one node with the cluster autoscaler (1 to 3 nodes), Microsoft Entra Workload ID, Entra ID authentication with Azure RBAC (admin groups from `AAD_ADMIN_GROUP_OBJECT_IDS`), Advanced Container Networking Services (ACNS), container registry. Azure Files and Azure Disk CSI drivers and CSI snapshot controller off. | | [01-configurable-cluster.sh](01-configurable-cluster.sh) | Asks which optional features to provision. Every question has a default and an environment variable, so the script also runs unattended. Pressing Enter at every question provisions the same features as the two scripts below. | Each feature of the scripts below can be turned off, plus the choice of cluster identity and network policy. Optionally installs NGINX Gateway Fabric and cert-manager. | -| [01-system-assigned-managed-identity.sh](01-system-assigned-managed-identity.sh) | The full, production-like cluster, with a system-assigned identity. The samples' CI uses it. | Custom virtual network with Azure Bastion, Key Vault with the CSI secrets provider, Container Insights, a user node pool across availability zones, cluster autoscaler, Workload ID, KEDA, managed Gateway API CRDs, Entra ID with Azure RBAC, ACNS, container registry. | +| [01-system-assigned-managed-identity.sh](01-system-assigned-managed-identity.sh) | The full, production-like cluster, with a system-assigned identity. The samples' CI uses it. | Custom virtual network with Azure Bastion, Key Vault with the CSI secrets provider, Container Insights, [diagnostic settings](#monitoring) on every resource, a user node pool across availability zones, cluster autoscaler, Workload ID, KEDA, managed Gateway API CRDs, Entra ID with Azure RBAC, ACNS, container registry. | | [01-user-assigned-managed-identity.sh](01-user-assigned-managed-identity.sh) | The same full cluster with a pre-created user-assigned identity, which also gets the Contributor role on the node resource group. | As above, with a user-assigned identity. | What the samples and tutorials need from the cluster: @@ -37,7 +37,7 @@ Each question is skipped when its variable is set. Yes/no variables accept `true | `ENABLE_CLUSTER_AUTOSCALER` | `true` | The cluster autoscaler on both node pools, between 1 and 3 nodes (widened to include the node count). | | `ENABLE_WORKLOAD_IDENTITY` | `true` | The OIDC issuer and Microsoft Entra Workload ID. | | `CREATE_KEY_VAULT` | `true` | The `local-kv-test` vault, the `azure-keyvault-secrets-provider` add-on, and the Key Vault Administrator role for the add-on identity. | -| `ENABLE_MONITORING` | `true` | The `local-log-analytics-test` workspace and the `monitoring` add-on (Container Insights). | +| `ENABLE_MONITORING` | `true` | The `local-log-analytics-test` workspace, the `monitoring` add-on (Container Insights), and the `default` [diagnostic settings](#monitoring) of the cluster's resources. | | `ENABLE_FILE_CSI_DRIVER` | `true` | The Azure Files CSI driver and its `azurefile*` storage classes. `false` passes `--disable-file-driver`. | | `ENABLE_DISK_CSI_DRIVER` | `true` | The Azure Disk CSI driver and its `managed-csi*` storage classes. `false` passes `--disable-disk-driver`. | | `CREATE_VNET` | `true` | The `local-vnet-test` virtual network with `SystemSubnet`, plus `UserSubnet` when there is a user node pool, the Microsoft.Storage service endpoint on the node subnets (only with the Azure Files CSI driver, whose NFS volumes need it), and the Network Contributor role for the cluster identity. Without it, AKS creates and manages the virtual network. | @@ -91,6 +91,30 @@ With every default, the configurable script creates the same resources as the or - A re-run keeps the Helm releases of NGINX Gateway Fabric and cert-manager that are already deployed instead of upgrading them. - When the cluster already exists but was built with different answers, the script stops before creating anything and lists the differences, because most of them cannot be changed on an existing cluster. +## Monitoring + +`01-system-assigned-managed-identity.sh`, `01-user-assigned-managed-identity.sh` and `01-configurable-cluster.sh` (with `ENABLE_MONITORING=true`, the default) create a diagnostic setting called `default` on each resource below. It sends the resource's platform metrics (`AllMetrics`) and these log categories to the `local-log-analytics-test` workspace. The configurable script skips the resources its answers leave out, and `01-basic-cluster.sh` creates no workspace and no diagnostic settings. + +| Resource | Log categories | +| -------- | -------------- | +| AKS cluster `local-aks-test` | `kube-apiserver`, `kube-audit-admin`, `kube-controller-manager`, `kube-scheduler`, `cluster-autoscaler`, `cloud-controller-manager`, `guard`, `csi-azurefile-controller`, `csi-snapshot-controller` | +| Virtual network `local-vnet-test` | `VMProtectionAlerts` | +| Public IP address of the Bastion host `local-bastion-ip-test` | `DDoSProtectionNotifications`, `DDoSMitigationFlowLogs`, `DDoSMitigationReports` | +| Bastion host `local-bastion-test` | `BastionAuditLogs` | +| Container registry `localacrtest` | `ContainerRegistryRepositoryEvents`, `ContainerRegistryLoginEvents` | +| Key vault `local-kv-test` | `AuditEvent`, `AzurePolicyEvaluationDetails` | + +The AKS logs go to the resource-specific tables, such as `AKSAuditAdmin` and `AKSControlPlane`, instead of `AzureDiagnostics`. `kube-audit` is left out on purpose: it adds every `get` and `list` request to what `kube-audit-admin` collects, and Microsoft warns that it [can incur substantial cost](https://learn.microsoft.com/en-us/azure/aks/monitor-aks#azure-monitor-resource-logs). To collect it anyway, add `{"category": "kube-audit", "enabled": true}` to the AKS cluster's `--logs`. + +To list the diagnostic settings of a resource, such as the cluster: + +```bash +az monitor diagnostic-settings list \ + --resource "$(az aks show --name local-aks-test --resource-group local-rg --query id --output tsv)" +``` + +[tests/test-diagnostic-settings.sh](../tests/test-diagnostic-settings.sh) checks that every resource of the cluster has its `default` diagnostic setting. + ## Add-on installers Run these once the cluster is up. diff --git a/terraform/README.md b/terraform/README.md new file mode 100644 index 0000000..4f4a7f0 --- /dev/null +++ b/terraform/README.md @@ -0,0 +1,123 @@ +# Terraform Deployment + +This directory contains Terraform modules and a deployment script that create the same Azure Kubernetes Service (AKS) cluster as [scripts/01-system-assigned-managed-identity.sh](../scripts/01-system-assigned-managed-identity.sh) or, with `identity_type` set to `UserAssigned`, as [scripts/01-user-assigned-managed-identity.sh](../scripts/01-user-assigned-managed-identity.sh): the same resources, with the same names and defaults, including the [diagnostic settings](../scripts/README.md#monitoring). Every [sample](../samples/) runs on it. For the samples and the other ways to create the cluster, see the [README](../README.md) at the root of the repository. + +## Prerequisites + +- [LocalStack for Azure](https://docs.localstack.cloud/azure/), or an Azure subscription +- [Terraform](https://developer.hashicorp.com/terraform/downloads) (1.9+) +- [Docker](https://docs.docker.com/get-docker/) +- [Azure CLI](https://learn.microsoft.com/en-us/cli/azure/install-azure-cli) + [lstk CLI](https://docs.localstack.cloud/aws/developer-tools/running-localstack/lstk/) +- [kubectl](https://kubernetes.io/docs/tasks/tools/), and on Azure [kubelogin](https://learn.microsoft.com/en-us/azure/aks/kubelogin-authentication) (`az aks install-cli` installs both) + +```bash +brew install localstack/tap/lstk # or: npm install -g @localstack/lstk +``` + +## Architecture Overview + +The Terraform configuration uses two providers: [azurerm](https://registry.terraform.io/providers/hashicorp/azurerm/latest) 5.10.0 for every Azure resource, and [AzAPI](https://registry.terraform.io/providers/Azure/azapi/latest) 2.13.0 for the managed Gateway API only, which azurerm does not expose. It provisions: + +1. [Azure Resource Group](https://learn.microsoft.com/en-us/azure/azure-resource-manager/management/manage-resource-groups-cli) `local-rg`. +2. [Azure Log Analytics Workspace](https://learn.microsoft.com/azure/azure-monitor/logs/log-analytics-overview) `local-log-analytics-test` (module `log_analytics`), `PerGB2018` pricing tier, with a system-assigned identity. +3. [Azure Key Vault](https://learn.microsoft.com/en-us/azure/key-vault/general/overview) `local-kv-test` (module `key_vault`) with Azure RBAC authorization. +4. [Azure Container Registry](https://learn.microsoft.com/en-us/azure/container-registry/container-registry-intro) `localacrtest` (module `container_registry`), Basic SKU, with the admin user enabled. Every sample pushes its image to it. +5. [Azure Virtual Network](https://learn.microsoft.com/azure/virtual-network/virtual-networks-overview) `local-vnet-test` (module `virtual_network`), `10.0.0.0/8`, with three subnets: + - *SystemSubnet* (`10.240.0.0/16`) and *UserSubnet* (`10.241.0.0/16`) for the two node pools, both with the `Microsoft.Storage` service endpoint that the NFS volumes of the Azure Files CSI driver need. + - *AzureBastionSubnet* (`10.242.2.0/24`) for the Bastion host. +6. [Azure Bastion](https://learn.microsoft.com/en-us/azure/bastion/bastion-overview) `local-bastion-test` (module `bastion_host`), Basic SKU, with its Standard, static public IP address `local-bastion-ip-test`. +7. With `identity_type` set to `UserAssigned`, the [user-assigned managed identity](https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/overview) `local-aks-identity-test` of the cluster. +8. The AKS cluster `local-aks-test` (module `aks_cluster`): + - The latest GA Kubernetes version in the region, which `deploy.sh` looks up as the scripts do. + - The `system` node pool on *SystemSubnet*: `Standard_D4ds_v5` nodes with Azure Linux and a 50 GB ephemeral OS disk, one node with the cluster autoscaler between 1 and 3, 100 pods per node. + - Azure CNI Overlay, with the network policy and dataplane of the scripts' menu option (`network_option`), pod CIDR `192.168.0.0/16`, service CIDR `172.16.0.0/16`, DNS service IP `172.16.0.10`, and [Advanced Container Networking Services](https://learn.microsoft.com/en-us/azure/aks/advanced-container-networking-services-overview). + - Microsoft Entra ID with Azure RBAC, the OIDC issuer and [Microsoft Entra Workload ID](https://learn.microsoft.com/en-us/azure/aks/workload-identity-overview). + - The `monitoring` add-on (Container Insights) with its `MSCI--local-aks-test` data collection rule and association, and the Key Vault secrets provider add-on. + - KEDA, the Azure Files and Azure Disk CSI drivers and the CSI snapshot controller. +9. The `user` node pool (module `aks_node_pool`) on *UserSubnet*, with the same node settings as the system pool, spread across availability zones 1, 2 and 3. +10. The managed Gateway API CRDs (`azapi_update_resource`), ordered after the user node pool because AKS serializes the operations on a cluster. +11. The role assignments of the scripts: Network Contributor on the virtual network and on *SystemSubnet* for the cluster identity (the second is what `az aks create --vnet-subnet-id` grants), Key Vault Administrator on the key vault for the identity of the Key Vault secrets provider add-on, and AcrPull on the registry for the kubelet identity (what `az aks create --attach-acr` grants). AKS itself grants the cluster identity the Contributor role on the node resource group, which the user-assigned script checks for. +12. A diagnostic setting called `default` on the cluster, the virtual network, the public IP address, the Bastion host, the registry and the key vault, inside each resource's module, which sends its logs and metrics to the workspace. See [Monitoring](../scripts/README.md#monitoring) for the log categories. + +### Provider configuration: emulator and Azure + +[`providers.tf`](providers.tf) hardcodes no endpoint and no subscription: every setting is a variable that defaults to `null`, which targets Azure with the Azure CLI credentials. `deploy.sh` checks which cloud the Azure CLI targets and, when it is the emulator that `lstk az start-interception` configures, passes: + +| Variable | Value | +| -------- | ----- | +| `metadata_host` | `localhost.localstack.cloud:4566`, from which azurerm discovers every endpoint | +| `resource_manager_endpoint` | `az cloud show --query endpoints.resourceManager`, for AzAPI, which has no metadata discovery | +| `subscription_id`, `tenant_id`, `client_id`, `client_secret` | The emulator's placeholder credentials | +| `gateway_api_enabled` | `false`, see [LocalStack emulator notes](#localstack-emulator-notes) | + +Against Azure, `deploy.sh` passes only `subscription_id`, from `az account show`, and both providers authenticate with the Azure CLI session. + +## Provisioning Script + +[`deploy.sh`](deploy.sh) performs: + +1. Detection of the target, the emulator or Azure, from the active Azure CLI cloud. +2. Lookup of the latest GA Kubernetes version in the region. +3. `terraform init`, `terraform validate`, `terraform plan` and `terraform apply`, with the variables below. +4. On the emulator only, `az aks update --enable-gateway-api`. +5. `az aks get-credentials`, to configure `kubectl` for the cluster. +6. Printing the cluster's storage profile and storage classes, as the scripts do. + +```bash +# the cluster of scripts/01-system-assigned-managed-identity.sh +./deploy.sh + +# the cluster of scripts/01-user-assigned-managed-identity.sh, with the Cilium dataplane and network policy +IDENTITY_TYPE=UserAssigned NETWORK_OPTION=2 ./deploy.sh +``` + +Then deploy the samples as their READMEs describe, as on a cluster created by the scripts. + +## Variables + +[`terraform.tfvars`](terraform.tfvars) holds the scripts' defaults. `deploy.sh` passes the variables below with `-var`, from environment variables that default to the scripts' values. The scripts ask for the network option and the node counts; here they come from the environment: + +| Environment variable | Terraform variable | Default | Script variable | +| -------------------- | ------------------ | ------- | --------------- | +| `PREFIX` | `prefix` | `local` | `prefix` | +| `SUFFIX` | `suffix` | `test` | `suffix` | +| `LOCATION` | `location` | `italynorth` | `location` | +| `IDENTITY_TYPE` | `identity_type` | `SystemAssigned` | The choice of script | +| `NETWORK_OPTION` | `network_option` | `1` | The network plugin and policy menu: `1` Azure CNI Overlay with the Azure network policy, `2` with the Cilium dataplane and network policy, `3` with the Calico network policy | +| `SYSTEM_NODE_COUNT` | `system_node_count` | `1` | `node_count`, asked by the scripts | +| `USER_NODE_COUNT` | `user_node_count` | `1` | `node_pool_node_count`, asked by the scripts | +| `AAD_ADMIN_GROUP_OBJECT_IDS` | `aad_admin_group_object_ids` | None | `aad_profile_admin_group_object_ids` | +| `SSH_PUBLIC_KEY_FILE` | `ssh_public_key`, the file's content | `~/.ssh/id_rsa.pub` | `ssh_key_value` | +| `WINDOWS_ADMIN_PASSWORD` | `windows_admin_password` | The scripts' password | `windows_admin_password` | +| | `kubernetes_version` | The latest GA version in the region | `kubernetes_version` | + +The cluster differs from the scripts' in what azurerm adds: it sends the Azure Policy add-on as disabled, and adds the `enableRetinaNetworkFlags` setting, `false`, to the monitoring add-on. It requests the observability of Advanced Container Networking Services and, only with the Cilium dataplane (network option 2), their security features, which is what AKS turns on for `az aks create --enable-acns`. + +The autoscaler range of each node pool is 1 to 3 nodes, widened to include the node count, as in the scripts. azurerm accepts only GUIDs as admin group object IDs, so where the scripts pass the placeholder ``, the default here is no admin group; on Azure, set `AAD_ADMIN_GROUP_OBJECT_IDS` to the comma-separated object IDs of real Microsoft Entra ID groups. The registry and key vault names are globally unique on Azure, so export your own `SUFFIX` there. + +## Verification + +```bash +# the diagnostic settings of the cluster's resources +../tests/test-diagnostic-settings.sh + +# the cluster and its node pools +terraform output +az aks nodepool list --cluster-name local-aks-test --resource-group local-rg \ + --query "[].{name: name, mode: mode, count: count, zones: availabilityZones}" --output table +kubectl get nodes +``` + +## LocalStack emulator notes + +- The emulator does not persist the managed Gateway API from AzAPI's PUT, so on the emulator `deploy.sh` passes `gateway_api_enabled=false` and enables the Gateway API with `az aks update --enable-gateway-api` after the apply. +- Earlier versions of the emulator do not return the Windows profile, and azurerm replaces a cluster to change one, so the configuration ignores changes to the Windows profile, as it does to the node counts the cluster autoscaler manages. +- On the emulator, a cluster the scripts create reports the security features of Advanced Container Networking Services as enabled even with the Azure dataplane, which those features do not support; the cluster Terraform creates reports them as disabled. + +## Related Documentation + +- [azurerm_kubernetes_cluster](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/kubernetes_cluster) +- [azurerm_monitor_diagnostic_setting](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/monitor_diagnostic_setting) +- [Monitor AKS: resource logs](https://learn.microsoft.com/en-us/azure/aks/monitor-aks#azure-monitor-resource-logs) +- [LocalStack for Azure Documentation](https://docs.localstack.cloud/azure/) +- [lstk CLI](https://docs.localstack.cloud/aws/developer-tools/running-localstack/lstk/) diff --git a/terraform/deploy.sh b/terraform/deploy.sh new file mode 100755 index 0000000..e2e9e93 --- /dev/null +++ b/terraform/deploy.sh @@ -0,0 +1,210 @@ +#!/bin/bash + +# Deploy with Terraform the AKS cluster that scripts/01-system-assigned-managed-identity.sh creates or, with +# IDENTITY_TYPE=UserAssigned, the one of scripts/01-user-assigned-managed-identity.sh: the same resources, names +# and defaults, including the diagnostic settings. It works the same way against Azure and against the LocalStack +# for Azure emulator, whichever the Azure CLI targets: see "Run against LocalStack" in the README at the root of the +# repository. + +# Variables +# Every default below can be overridden through the environment, for example +# IDENTITY_TYPE=UserAssigned NETWORK_OPTION=2 ./deploy.sh +# The container registry and the key vault have globally unique names on Azure, so a run against a real +# subscription normally needs its own SUFFIX (or PREFIX). +PREFIX="${PREFIX:-local}" +SUFFIX="${SUFFIX:-test}" +LOCATION="${LOCATION:-italynorth}" +RESOURCE_GROUP_NAME="${PREFIX}-rg" +AKS_CLUSTER_NAME="${PREFIX}-aks-${SUFFIX}" +IDENTITY_TYPE="${IDENTITY_TYPE:-SystemAssigned}" +NETWORK_OPTION="${NETWORK_OPTION:-1}" +SYSTEM_NODE_COUNT="${SYSTEM_NODE_COUNT:-1}" +USER_NODE_COUNT="${USER_NODE_COUNT:-1}" +# Comma-separated object ids of the Entra ID admin groups; azurerm accepts only GUIDs, so the default is none +AAD_ADMIN_GROUP_OBJECT_IDS="${AAD_ADMIN_GROUP_OBJECT_IDS:-}" +SSH_PUBLIC_KEY_FILE="${SSH_PUBLIC_KEY_FILE:-$HOME/.ssh/id_rsa.pub}" +# The windows_admin_password variable reads it from the environment +export TF_VAR_windows_admin_password="${WINDOWS_ADMIN_PASSWORD:-Trustno123456!}" +EMULATOR_GUID="00000000-0000-0000-0000-000000000000" +EMULATOR_METADATA_HOST="localhost.localstack.cloud:4566" +CURRENT_DIR="$(cd "$(dirname "$0")" && pwd)" + +# Check prerequisites +for tool in az kubectl terraform; do + if ! command -v "$tool" >/dev/null 2>&1; then + echo "Error: required tool '$tool' is not installed or not in PATH." >&2 + exit 1 + fi +done + +# Change the current directory to the script's directory +cd "$CURRENT_DIR" || exit + +# Target the emulator when the Azure CLI does, as lstk az start-interception configures it +RESOURCE_MANAGER_ENDPOINT=$(az cloud show --query endpoints.resourceManager --output tsv --only-show-errors | tr -d '\r') +if [[ $RESOURCE_MANAGER_ENDPOINT == *localhost* || $RESOURCE_MANAGER_ENDPOINT == *localstack* ]]; then + IS_EMULATOR=1 + echo "The Azure CLI targets the LocalStack for Azure emulator at [$RESOURCE_MANAGER_ENDPOINT]" +else + IS_EMULATOR=0 + echo "The Azure CLI targets Azure at [$RESOURCE_MANAGER_ENDPOINT]" +fi + +# Keep the Kubernetes version of an existing cluster: like the scripts, use the latest version only to create one, +# so that a re-deployment does not upgrade the cluster +KUBERNETES_VERSION=$(az aks show \ + --name "$AKS_CLUSTER_NAME" \ + --resource-group "$RESOURCE_GROUP_NAME" \ + --query kubernetesVersion \ + --output tsv \ + --only-show-errors 2>/dev/null | tr -d '\r') + +if [[ -n $KUBERNETES_VERSION ]]; then + echo "The [$AKS_CLUSTER_NAME] AKS cluster exists and runs Kubernetes [$KUBERNETES_VERSION]: keeping that version" +else + # Get the last Kubernetes version available in the region, as the scripts do + KUBERNETES_VERSION=$(az aks get-versions \ + --location "$LOCATION" \ + --query "values[?isPreview==null].version | sort(@) | [-1]" \ + --output tsv \ + --only-show-errors 2>/dev/null | tr -d '\r') + + if [[ -n $KUBERNETES_VERSION ]]; then + echo "The latest Kubernetes version available in the [$LOCATION] region is [$KUBERNETES_VERSION]" + else + echo "Failed to retrieve the latest Kubernetes version available in the [$LOCATION] region: AKS picks its default" + fi +fi + +# Read the SSH public key of the Linux nodes, as the scripts pass ~/.ssh/id_rsa.pub +if [[ -f $SSH_PUBLIC_KEY_FILE ]]; then + SSH_PUBLIC_KEY=$(tr -d '\r\n' <"$SSH_PUBLIC_KEY_FILE") +else + echo "No SSH public key at [$SSH_PUBLIC_KEY_FILE]: the cluster is created without one" + SSH_PUBLIC_KEY="" +fi + +# The comma-separated object ids of the admin groups, as a list +if [[ -n $AAD_ADMIN_GROUP_OBJECT_IDS ]]; then + AAD_ADMIN_GROUP_OBJECT_IDS_LIST="[\"${AAD_ADMIN_GROUP_OBJECT_IDS//,/\",\"}\"]" +else + AAD_ADMIN_GROUP_OBJECT_IDS_LIST="[]" +fi + +# The variables that override terraform.tfvars +tf_vars=( + -var "prefix=$PREFIX" + -var "suffix=$SUFFIX" + -var "location=$LOCATION" + -var "identity_type=$IDENTITY_TYPE" + -var "network_option=$NETWORK_OPTION" + -var "system_node_count=$SYSTEM_NODE_COUNT" + -var "user_node_count=$USER_NODE_COUNT" + -var "aad_admin_group_object_ids=$AAD_ADMIN_GROUP_OBJECT_IDS_LIST" + -var "ssh_public_key=$SSH_PUBLIC_KEY" +) +if [[ -n $KUBERNETES_VERSION ]]; then + tf_vars+=(-var "kubernetes_version=$KUBERNETES_VERSION") +fi +if [[ $IS_EMULATOR == 1 ]]; then + # The emulator's endpoints and placeholder credentials. The Gateway API is enabled with az aks update after the + # apply instead of azapi, because the emulator does not persist it from azapi's PUT. + tf_vars+=( + -var "metadata_host=$EMULATOR_METADATA_HOST" + -var "resource_manager_endpoint=$RESOURCE_MANAGER_ENDPOINT" + -var "subscription_id=$EMULATOR_GUID" + -var "tenant_id=$EMULATOR_GUID" + -var "client_id=$EMULATOR_GUID" + -var "client_secret=fake-secret" + -var "gateway_api_enabled=false" + ) +else + tf_vars+=(-var "subscription_id=$(az account show --query id --output tsv --only-show-errors | tr -d '\r')") +fi + +# Initialize Terraform +echo "Initializing Terraform..." +terraform init -upgrade -input=false 1>/dev/null + +if [[ $? == 0 ]]; then + echo "Terraform successfully initialized" +else + echo "Failed to initialize Terraform" + exit 1 +fi + +# Validate the Terraform configuration +echo "Validating the Terraform configuration..." +terraform validate 1>/dev/null + +if [[ $? == 0 ]]; then + echo "Terraform configuration is valid" +else + echo "Terraform configuration is not valid" + exit 1 +fi + +# Run terraform plan and check for errors +echo "Planning the Terraform deployment..." +terraform plan -input=false -out=tfplan "${tf_vars[@]}" 1>/dev/null + +if [[ $? == 0 ]]; then + echo "Terraform plan successfully created" +else + echo "Failed to create the Terraform plan" + exit 1 +fi + +# Apply the Terraform plan +echo "Applying the Terraform plan..." +terraform apply -input=false -auto-approve tfplan + +if [[ $? == 0 ]]; then + echo "Terraform plan successfully applied" +else + echo "Failed to apply the Terraform plan" + exit 1 +fi + +# Enable the managed Gateway API CRDs on the emulator, which does not persist them from azapi's PUT +if [[ $IS_EMULATOR == 1 ]]; then + echo "Enabling the managed Gateway API CRDs on the [$AKS_CLUSTER_NAME] AKS cluster..." + az aks update \ + --name "$AKS_CLUSTER_NAME" \ + --resource-group "$RESOURCE_GROUP_NAME" \ + --enable-gateway-api \ + --only-show-errors 1>/dev/null + + if [[ $? == 0 ]]; then + echo "The managed Gateway API CRDs were successfully enabled on the [$AKS_CLUSTER_NAME] AKS cluster" + else + echo "Failed to enable the managed Gateway API CRDs on the [$AKS_CLUSTER_NAME] AKS cluster" + exit 1 + fi +fi + +# Use the following command to configure kubectl to connect to the new Kubernetes cluster +echo "Getting access credentials configure kubectl to connect to the [$AKS_CLUSTER_NAME] AKS cluster..." +az aks get-credentials \ + --name "$AKS_CLUSTER_NAME" \ + --resource-group "$RESOURCE_GROUP_NAME" \ + --overwrite-existing \ + --only-show-errors + +if [[ $? == 0 ]]; then + echo "Credentials for the [$AKS_CLUSTER_NAME] cluster successfully retrieved" +else + echo "Failed to retrieve the credentials for the [$AKS_CLUSTER_NAME] cluster" + exit 1 +fi + +# Print the storage drivers enabled on the cluster, followed by the storage classes they provide, as the scripts do +echo "Storage profile of the [$AKS_CLUSTER_NAME] AKS cluster:" +az aks show \ + --name "$AKS_CLUSTER_NAME" \ + --resource-group "$RESOURCE_GROUP_NAME" \ + --query storageProfile \ + --output yaml \ + --only-show-errors +echo "Storage classes of the [$AKS_CLUSTER_NAME] AKS cluster:" +kubectl get storageclass diff --git a/terraform/main.tf b/terraform/main.tf new file mode 100644 index 0000000..25c1fc4 --- /dev/null +++ b/terraform/main.tf @@ -0,0 +1,230 @@ +# The AKS cluster of scripts/01-system-assigned-managed-identity.sh and, with identity_type set to UserAssigned, of +# scripts/01-user-assigned-managed-identity.sh: the same resources, names and defaults, including the diagnostic +# settings that send the logs and metrics of every resource to the Log Analytics workspace. + +data "azurerm_client_config" "current" {} + +locals { + # Names, as in the scripts + resource_group_name = "${var.prefix}-rg" + aks_cluster_name = "${var.prefix}-aks-${var.suffix}" + aks_managed_identity_name = "${var.prefix}-aks-identity-${var.suffix}" + key_vault_name = "${var.prefix}-kv-${var.suffix}" + acr_name = "${var.prefix}acr${var.suffix}" + log_analytics_name = "${var.prefix}-log-analytics-${var.suffix}" + virtual_network_name = "${var.prefix}-vnet-${var.suffix}" + bastion_public_ip_name = "${var.prefix}-bastion-ip-${var.suffix}" + bastion_host_name = "${var.prefix}-bastion-${var.suffix}" + + # Network policy and dataplane of each option of the scripts' menu + network_options = { + "1" = { policy = "azure", data_plane = "azure" } + "2" = { policy = "cilium", data_plane = "cilium" } + "3" = { policy = "calico", data_plane = "azure" } + } + network = local.network_options[tostring(var.network_option)] + + # The node pools' tags and labels the scripts set + node_labels = { + os_disk_type = var.os_disk_type + os_type = "Linux" + } + provisioning_tags = { + provisioning = "terraform" + } +} + +resource "azurerm_resource_group" "resource_group" { + name = local.resource_group_name + location = var.location +} + +module "log_analytics" { + source = "./modules/log_analytics" + + name = local.log_analytics_name + location = var.location + resource_group_name = azurerm_resource_group.resource_group.name + sku = var.log_analytics_sku +} + +module "key_vault" { + source = "./modules/key_vault" + + name = local.key_vault_name + location = var.location + resource_group_name = azurerm_resource_group.resource_group.name + tenant_id = data.azurerm_client_config.current.tenant_id + log_analytics_workspace_id = module.log_analytics.id + diagnostic_settings_name = var.diagnostic_settings_name +} + +module "container_registry" { + source = "./modules/container_registry" + + name = local.acr_name + location = var.location + resource_group_name = azurerm_resource_group.resource_group.name + sku = var.acr_sku + log_analytics_workspace_id = module.log_analytics.id + diagnostic_settings_name = var.diagnostic_settings_name +} + +module "virtual_network" { + source = "./modules/virtual_network" + + name = local.virtual_network_name + location = var.location + resource_group_name = azurerm_resource_group.resource_group.name + log_analytics_workspace_id = module.log_analytics.id + diagnostic_settings_name = var.diagnostic_settings_name +} + +module "bastion_host" { + source = "./modules/bastion_host" + + name = local.bastion_host_name + public_ip_name = local.bastion_public_ip_name + location = var.location + resource_group_name = azurerm_resource_group.resource_group.name + subnet_id = module.virtual_network.bastion_subnet_id + log_analytics_workspace_id = module.log_analytics.id + diagnostic_settings_name = var.diagnostic_settings_name +} + +# The user-assigned identity of the cluster, as scripts/01-user-assigned-managed-identity.sh creates it +resource "azurerm_user_assigned_identity" "aks" { + count = var.identity_type == "UserAssigned" ? 1 : 0 + + name = local.aks_managed_identity_name + location = var.location + resource_group_name = azurerm_resource_group.resource_group.name +} + +module "aks_cluster" { + source = "./modules/aks_cluster" + + name = local.aks_cluster_name + location = var.location + resource_group_name = azurerm_resource_group.resource_group.name + subscription_id = data.azurerm_client_config.current.subscription_id + tenant_id = data.azurerm_client_config.current.tenant_id + tags = local.provisioning_tags + kubernetes_version = var.kubernetes_version + identity_type = var.identity_type + user_assigned_identity_id = var.identity_type == "UserAssigned" ? azurerm_user_assigned_identity.aks[0].id : "" + system_subnet_id = module.virtual_network.system_subnet_id + vm_size = var.node_vm_size + node_count = var.system_node_count + min_count = min(1, var.system_node_count) + max_count = max(3, var.system_node_count) + max_pods = var.max_pods + os_disk_size_gb = var.os_disk_size_gb + os_disk_type = var.os_disk_type + os_sku = var.os_sku + node_labels = local.node_labels + node_pool_tags = local.node_labels + ssh_public_key = var.ssh_public_key + windows_admin_username = var.windows_admin_username + windows_admin_password = var.windows_admin_password + network_policy = local.network.policy + network_data_plane = local.network.data_plane + aad_admin_group_object_ids = var.aad_admin_group_object_ids + log_analytics_workspace_id = module.log_analytics.id + diagnostic_settings_name = var.diagnostic_settings_name + + depends_on = [azurerm_role_assignment.user_assigned_system_subnet_network_contributor] +} + +# Added after the cluster identity's Network Contributor role assignments, as the scripts add it +module "aks_node_pool" { + source = "./modules/aks_node_pool" + + kubernetes_cluster_id = module.aks_cluster.id + subnet_id = module.virtual_network.user_subnet_id + vm_size = var.node_vm_size + node_count = var.user_node_count + min_count = min(1, var.user_node_count) + max_count = max(3, var.user_node_count) + max_pods = var.max_pods + os_disk_size_gb = var.os_disk_size_gb + os_disk_type = var.os_disk_type + os_sku = var.os_sku + node_labels = local.node_labels + tags = local.provisioning_tags + + depends_on = [ + azurerm_role_assignment.network_contributor, + azurerm_role_assignment.system_assigned_system_subnet_network_contributor, + ] +} + +# The managed Gateway API CRDs (azurerm does not expose ingressProfile.gatewayAPI). Ordered after the user node pool: +# AKS serializes the operations on a cluster. deploy.sh turns it off on the emulator, which does not persist it from +# this PUT, and enables the Gateway API there with az aks update instead. +resource "azapi_update_resource" "gateway_api" { + count = var.gateway_api_enabled ? 1 : 0 + + type = "Microsoft.ContainerService/managedClusters@2026-03-02-preview" + resource_id = module.aks_cluster.id + + body = { + properties = { + ingressProfile = { + gatewayAPI = { + installation = "Standard" + } + } + } + } + + depends_on = [module.aks_node_pool] +} + +# Network Contributor on the virtual network for the cluster identity +resource "azurerm_role_assignment" "network_contributor" { + scope = module.virtual_network.id + role_definition_name = "Network Contributor" + principal_id = var.identity_type == "UserAssigned" ? azurerm_user_assigned_identity.aks[0].principal_id : module.aks_cluster.principal_id + skip_service_principal_aad_check = true +} + +# Network Contributor on the subnet of the system node pool for the cluster identity, which az aks create +# --vnet-subnet-id grants as well: before it creates the cluster for a user-assigned identity, after it for a +# system-assigned one, which exists only once the cluster does +resource "azurerm_role_assignment" "user_assigned_system_subnet_network_contributor" { + count = var.identity_type == "UserAssigned" ? 1 : 0 + + scope = module.virtual_network.system_subnet_id + role_definition_name = "Network Contributor" + principal_id = azurerm_user_assigned_identity.aks[0].principal_id + skip_service_principal_aad_check = true +} + +resource "azurerm_role_assignment" "system_assigned_system_subnet_network_contributor" { + count = var.identity_type == "SystemAssigned" ? 1 : 0 + + scope = module.virtual_network.system_subnet_id + role_definition_name = "Network Contributor" + principal_id = module.aks_cluster.principal_id + skip_service_principal_aad_check = true +} + +# AKS itself grants the cluster identity, system-assigned or user-assigned, the Contributor role on the node resource +# group: scripts/01-user-assigned-managed-identity.sh finds that assignment, and a second one would conflict with it + +# Key Vault Administrator on the key vault for the identity of the Key Vault secrets provider add-on +resource "azurerm_role_assignment" "key_vault_administrator" { + scope = module.key_vault.id + role_definition_name = "Key Vault Administrator" + principal_id = module.aks_cluster.key_vault_secrets_provider_identity_object_id + skip_service_principal_aad_check = true +} + +# AcrPull on the container registry for the kubelet identity, which is what az aks create --attach-acr grants +resource "azurerm_role_assignment" "acr_pull" { + scope = module.container_registry.id + role_definition_name = "AcrPull" + principal_id = module.aks_cluster.kubelet_identity_object_id + skip_service_principal_aad_check = true +} diff --git a/terraform/modules/aks_cluster/main.tf b/terraform/modules/aks_cluster/main.tf new file mode 100644 index 0000000..345cdf2 --- /dev/null +++ b/terraform/modules/aks_cluster/main.tf @@ -0,0 +1,202 @@ +# AKS cluster of the cluster scripts: the system node pool on SystemSubnet, Microsoft Entra ID with Azure RBAC, +# Workload ID, the monitoring (Container Insights) and Key Vault secrets provider add-ons, ACNS, KEDA, the Azure Files +# CSI driver and the snapshot controller, and the diagnostic settings that send the control plane logs and the +# metrics to the Log Analytics workspace. The managed Gateway API is installed from the root module with azapi, +# because azurerm does not expose it, and the user node pool is in the aks_node_pool module. + +locals { + # The control plane logs the diagnostic settings collect. kube-audit is left out on purpose: it adds every get and + # list request to what kube-audit-admin collects, which can be costly on Azure. + log_categories = [ + "kube-apiserver", + "kube-audit-admin", + "kube-controller-manager", + "kube-scheduler", + "cluster-autoscaler", + "cloud-controller-manager", + "guard", + "csi-azurefile-controller", + "csi-snapshot-controller", + ] + + # The stream that the Container Insights data collection rule sends to the workspace + container_insights_streams = ["Microsoft-ContainerInsights-Group-Default"] +} + +resource "azurerm_kubernetes_cluster" "aks_cluster" { + name = var.name + location = var.location + resource_group_name = var.resource_group_name + # The DNS prefix az aks create generates: the first 10 characters of the cluster name, the first 16 of the resource + # group name and the first 6 of the subscription id + dns_prefix = "${substr(var.name, 0, 10)}-${substr(var.resource_group_name, 0, 16)}-${substr(var.subscription_id, 0, 6)}" + kubernetes_version = var.kubernetes_version + sku_tier = "Free" + tags = var.tags + + role_based_access_control_enabled = true + oidc_issuer_enabled = true + workload_identity_enabled = true + + # Required by azurerm 5: Manual is the AKS default, the node pools below rather than node auto-provisioning + node_provisioning_profile { + mode = "Manual" + } + + identity { + type = var.identity_type + identity_ids = var.identity_type == "UserAssigned" ? [var.user_assigned_identity_id] : null + } + + default_node_pool { + name = var.system_node_pool_name + vm_size = var.vm_size + vnet_subnet_id = var.system_subnet_id + node_count = var.node_count + auto_scaling_enabled = true + min_count = var.min_count + max_count = var.max_count + max_pods = var.max_pods + os_disk_size_gb = var.os_disk_size_gb + os_disk_type = var.os_disk_type + os_sku = var.os_sku + node_labels = var.node_labels + tags = var.node_pool_tags + temporary_name_for_rotation = "systemtemp" + + upgrade_settings { + max_surge = "10%" + } + } + + # Emitted only when an SSH public key is passed + dynamic "linux_profile" { + for_each = var.ssh_public_key == "" ? [] : [1] + content { + admin_username = var.linux_admin_username + ssh_key { + key_data = var.ssh_public_key + } + } + } + + windows_profile { + admin_username = var.windows_admin_username + admin_password = var.windows_admin_password + } + + network_profile { + network_plugin = "azure" + network_plugin_mode = "overlay" + network_policy = var.network_policy + network_data_plane = var.network_data_plane + pod_cidr = var.pod_cidr + service_cidr = var.service_cidr + dns_service_ip = var.dns_service_ip + outbound_type = "loadBalancer" + load_balancer_sku = "standard" + + # Advanced Container Networking Services, as --enable-acns turns them on. azurerm allows the security features + # (FQDN filtering) only with the Cilium dataplane, so with the Azure dataplane only observability is on. + advanced_networking { + observability_enabled = true + security_enabled = var.network_data_plane == "cilium" + } + } + + oms_agent { + log_analytics_workspace_id = var.log_analytics_workspace_id + msi_auth_for_monitoring_enabled = true + } + + key_vault_secrets_provider { + secret_rotation_enabled = false + } + + azure_active_directory_role_based_access_control { + tenant_id = var.tenant_id + admin_group_object_ids = var.aad_admin_group_object_ids + azure_rbac_enabled = true + } + + workload_autoscaler_profile { + keda_enabled = true + } + + storage_profile { + disk_driver_enabled = true + file_driver_enabled = true + snapshot_controller_enabled = true + } + + lifecycle { + ignore_changes = [ + # The cluster autoscaler changes the node count + default_node_pool[0].node_count, + # Earlier versions of the emulator do not return the Windows profile, and a change to it replaces the cluster + windows_profile, + ] + } +} + +# Container Insights collects through a data collection rule associated with the cluster, which az aks create +# --enable-addons monitoring creates with these names +resource "azurerm_monitor_data_collection_rule" "container_insights" { + name = "MSCI-${var.location}-${var.name}" + location = var.location + resource_group_name = var.resource_group_name + kind = "Linux" + + destinations { + log_analytics { + name = "la-workspace" + workspace_resource_id = var.log_analytics_workspace_id + } + } + + data_flow { + streams = local.container_insights_streams + destinations = ["la-workspace"] + } + + data_sources { + extension { + name = "ContainerInsightsExtension" + extension_name = "ContainerInsights" + streams = local.container_insights_streams + extension_json = jsonencode({ + dataCollectionSettings = { + interval = "1m" + namespaceFilteringMode = "Off" + enableContainerLogV2 = true + } + }) + } + } +} + +resource "azurerm_monitor_data_collection_rule_association" "container_insights" { + name = "ContainerInsightsExtension" + target_resource_id = azurerm_kubernetes_cluster.aks_cluster.id + data_collection_rule_id = azurerm_monitor_data_collection_rule.container_insights.id + description = "Association of data collection rule. Deleting this association will break the data collection for this AKS Cluster." +} + +# The control plane logs go to the resource-specific tables of the workspace (AKSAuditAdmin, AKSControlPlane) +resource "azurerm_monitor_diagnostic_setting" "diagnostic_setting" { + name = var.diagnostic_settings_name + target_resource_id = azurerm_kubernetes_cluster.aks_cluster.id + log_analytics_workspace_id = var.log_analytics_workspace_id + log_analytics_destination_type = "Dedicated" + + dynamic "enabled_log" { + for_each = local.log_categories + content { + category = enabled_log.value + } + } + + enabled_metric { + category = "AllMetrics" + } +} diff --git a/terraform/modules/aks_cluster/outputs.tf b/terraform/modules/aks_cluster/outputs.tf new file mode 100644 index 0000000..c7ddb51 --- /dev/null +++ b/terraform/modules/aks_cluster/outputs.tf @@ -0,0 +1,19 @@ +output "id" { + value = azurerm_kubernetes_cluster.aks_cluster.id +} + +output "name" { + value = azurerm_kubernetes_cluster.aks_cluster.name +} + +output "principal_id" { + value = var.identity_type == "SystemAssigned" ? azurerm_kubernetes_cluster.aks_cluster.identity[0].principal_id : "" +} + +output "kubelet_identity_object_id" { + value = azurerm_kubernetes_cluster.aks_cluster.kubelet_identity[0].object_id +} + +output "key_vault_secrets_provider_identity_object_id" { + value = azurerm_kubernetes_cluster.aks_cluster.key_vault_secrets_provider[0].secret_identity[0].object_id +} diff --git a/terraform/modules/aks_cluster/variables.tf b/terraform/modules/aks_cluster/variables.tf new file mode 100644 index 0000000..5eb6133 --- /dev/null +++ b/terraform/modules/aks_cluster/variables.tf @@ -0,0 +1,190 @@ +variable "name" { + description = "Specifies the name of the AKS cluster." + type = string +} + +variable "location" { + description = "Specifies the location of the AKS cluster." + type = string +} + +variable "resource_group_name" { + description = "Specifies the name of the resource group." + type = string +} + +variable "subscription_id" { + description = "Specifies the subscription id, which the DNS prefix includes as az aks create does." + type = string +} + +variable "tenant_id" { + description = "Specifies the tenant id of Microsoft Entra ID." + type = string +} + +variable "tags" { + description = "Specifies the tags of the AKS cluster." + type = map(string) + default = {} +} + +variable "kubernetes_version" { + description = "Specifies the Kubernetes version. null lets AKS pick its default." + type = string + default = null +} + +variable "identity_type" { + description = "Specifies the type of the cluster identity: SystemAssigned or UserAssigned." + type = string + default = "SystemAssigned" +} + +variable "user_assigned_identity_id" { + description = "Specifies the resource id of the user-assigned identity of the cluster, when identity_type is UserAssigned." + type = string + default = "" +} + +variable "system_node_pool_name" { + description = "Specifies the name of the system node pool." + type = string + default = "system" +} + +variable "system_subnet_id" { + description = "Specifies the resource id of the subnet of the system node pool." + type = string +} + +variable "vm_size" { + description = "Specifies the VM size of the system node pool." + type = string + default = "Standard_D4ds_v5" +} + +variable "node_count" { + description = "Specifies the number of nodes of the system node pool." + type = number + default = 1 +} + +variable "min_count" { + description = "Specifies the minimum number of nodes of the system node pool for the cluster autoscaler." + type = number + default = 1 +} + +variable "max_count" { + description = "Specifies the maximum number of nodes of the system node pool for the cluster autoscaler." + type = number + default = 3 +} + +variable "max_pods" { + description = "Specifies the maximum number of pods per node of the system node pool." + type = number + default = 100 +} + +variable "os_disk_size_gb" { + description = "Specifies the OS disk size in GB of the system node pool." + type = number + default = 50 +} + +variable "os_disk_type" { + description = "Specifies the OS disk type of the system node pool." + type = string + default = "Ephemeral" +} + +variable "os_sku" { + description = "Specifies the OS SKU of the system node pool." + type = string + default = "AzureLinux" +} + +variable "node_labels" { + description = "Specifies the node labels of the system node pool." + type = map(string) + default = {} +} + +variable "node_pool_tags" { + description = "Specifies the tags of the system node pool." + type = map(string) + default = {} +} + +variable "linux_admin_username" { + description = "Specifies the administrator user name of the Linux nodes." + type = string + default = "azureuser" +} + +variable "ssh_public_key" { + description = "Specifies the SSH public key of the Linux nodes. Empty omits the Linux profile." + type = string + default = "" +} + +variable "windows_admin_username" { + description = "Specifies the administrator user name of Windows nodes." + type = string + default = "azadmin" +} + +variable "windows_admin_password" { + description = "Specifies the administrator password of Windows nodes." + type = string + sensitive = true +} + +variable "network_policy" { + description = "Specifies the network policy: azure, calico or cilium." + type = string + default = "azure" +} + +variable "network_data_plane" { + description = "Specifies the network dataplane: azure or cilium." + type = string + default = "azure" +} + +variable "pod_cidr" { + description = "Specifies the CIDR of the pods." + type = string + default = "192.168.0.0/16" +} + +variable "service_cidr" { + description = "Specifies the CIDR of the services." + type = string + default = "172.16.0.0/16" +} + +variable "dns_service_ip" { + description = "Specifies the IP address of the Kubernetes DNS service, within service_cidr." + type = string + default = "172.16.0.10" +} + +variable "aad_admin_group_object_ids" { + description = "Specifies the object ids of the Microsoft Entra ID groups that administer the cluster." + type = list(string) + default = [] +} + +variable "log_analytics_workspace_id" { + description = "Specifies the resource id of the Log Analytics workspace." + type = string +} + +variable "diagnostic_settings_name" { + description = "Specifies the name of the diagnostic settings." + type = string + default = "default" +} diff --git a/terraform/modules/aks_node_pool/main.tf b/terraform/modules/aks_node_pool/main.tf new file mode 100644 index 0000000..e77321a --- /dev/null +++ b/terraform/modules/aks_node_pool/main.tf @@ -0,0 +1,29 @@ +# User node pool of the AKS cluster, as the cluster scripts add it with az aks nodepool add +resource "azurerm_kubernetes_cluster_node_pool" "node_pool" { + name = var.name + kubernetes_cluster_id = var.kubernetes_cluster_id + mode = "User" + vm_size = var.vm_size + vnet_subnet_id = var.subnet_id + node_count = var.node_count + auto_scaling_enabled = true + min_count = var.min_count + max_count = var.max_count + max_pods = var.max_pods + os_type = "Linux" + os_sku = var.os_sku + os_disk_size_gb = var.os_disk_size_gb + os_disk_type = var.os_disk_type + zones = var.availability_zones + node_labels = var.node_labels + tags = var.tags + + upgrade_settings { + max_surge = "10%" + } + + lifecycle { + # The cluster autoscaler changes the node count + ignore_changes = [node_count] + } +} diff --git a/terraform/modules/aks_node_pool/outputs.tf b/terraform/modules/aks_node_pool/outputs.tf new file mode 100644 index 0000000..45ef89a --- /dev/null +++ b/terraform/modules/aks_node_pool/outputs.tf @@ -0,0 +1,7 @@ +output "id" { + value = azurerm_kubernetes_cluster_node_pool.node_pool.id +} + +output "name" { + value = azurerm_kubernetes_cluster_node_pool.node_pool.name +} diff --git a/terraform/modules/aks_node_pool/variables.tf b/terraform/modules/aks_node_pool/variables.tf new file mode 100644 index 0000000..b4376d6 --- /dev/null +++ b/terraform/modules/aks_node_pool/variables.tf @@ -0,0 +1,81 @@ +variable "name" { + description = "Specifies the name of the node pool." + type = string + default = "user" +} + +variable "kubernetes_cluster_id" { + description = "Specifies the resource id of the AKS cluster." + type = string +} + +variable "subnet_id" { + description = "Specifies the resource id of the subnet of the node pool." + type = string +} + +variable "vm_size" { + description = "Specifies the VM size of the nodes." + type = string + default = "Standard_D4ds_v5" +} + +variable "node_count" { + description = "Specifies the number of nodes." + type = number + default = 1 +} + +variable "min_count" { + description = "Specifies the minimum number of nodes for the cluster autoscaler." + type = number + default = 1 +} + +variable "max_count" { + description = "Specifies the maximum number of nodes for the cluster autoscaler." + type = number + default = 3 +} + +variable "max_pods" { + description = "Specifies the maximum number of pods per node." + type = number + default = 100 +} + +variable "os_disk_size_gb" { + description = "Specifies the OS disk size in GB." + type = number + default = 50 +} + +variable "os_disk_type" { + description = "Specifies the OS disk type." + type = string + default = "Ephemeral" +} + +variable "os_sku" { + description = "Specifies the OS SKU." + type = string + default = "AzureLinux" +} + +variable "availability_zones" { + description = "Specifies the availability zones the nodes are spread across." + type = list(string) + default = ["1", "2", "3"] +} + +variable "node_labels" { + description = "Specifies the node labels of the node pool." + type = map(string) + default = {} +} + +variable "tags" { + description = "Specifies the tags of the node pool." + type = map(string) + default = {} +} diff --git a/terraform/modules/bastion_host/main.tf b/terraform/modules/bastion_host/main.tf new file mode 100644 index 0000000..2f09e7f --- /dev/null +++ b/terraform/modules/bastion_host/main.tf @@ -0,0 +1,61 @@ +# Bastion host and its public IP address, and the diagnostic settings that send their logs and metrics to the +# Log Analytics workspace + +# Azure Bastion requires a Standard SKU public IP address with a static allocation method +resource "azurerm_public_ip" "public_ip" { + name = var.public_ip_name + location = var.location + resource_group_name = var.resource_group_name + sku = "Standard" + allocation_method = "Static" +} + +resource "azurerm_bastion_host" "bastion_host" { + name = var.name + location = var.location + resource_group_name = var.resource_group_name + sku = var.sku + + ip_configuration { + name = "ipconfig" + subnet_id = var.subnet_id + public_ip_address_id = azurerm_public_ip.public_ip.id + } +} + +# The DDoS categories only receive data while Azure DDoS Protection mitigates an attack on the address +resource "azurerm_monitor_diagnostic_setting" "public_ip_diagnostic_setting" { + name = var.diagnostic_settings_name + target_resource_id = azurerm_public_ip.public_ip.id + log_analytics_workspace_id = var.log_analytics_workspace_id + + enabled_log { + category = "DDoSProtectionNotifications" + } + + enabled_log { + category = "DDoSMitigationFlowLogs" + } + + enabled_log { + category = "DDoSMitigationReports" + } + + enabled_metric { + category = "AllMetrics" + } +} + +resource "azurerm_monitor_diagnostic_setting" "bastion_host_diagnostic_setting" { + name = var.diagnostic_settings_name + target_resource_id = azurerm_bastion_host.bastion_host.id + log_analytics_workspace_id = var.log_analytics_workspace_id + + enabled_log { + category = "BastionAuditLogs" + } + + enabled_metric { + category = "AllMetrics" + } +} diff --git a/terraform/modules/bastion_host/outputs.tf b/terraform/modules/bastion_host/outputs.tf new file mode 100644 index 0000000..eba23da --- /dev/null +++ b/terraform/modules/bastion_host/outputs.tf @@ -0,0 +1,7 @@ +output "id" { + value = azurerm_bastion_host.bastion_host.id +} + +output "public_ip_id" { + value = azurerm_public_ip.public_ip.id +} diff --git a/terraform/modules/bastion_host/variables.tf b/terraform/modules/bastion_host/variables.tf new file mode 100644 index 0000000..e321f50 --- /dev/null +++ b/terraform/modules/bastion_host/variables.tf @@ -0,0 +1,41 @@ +variable "name" { + description = "Specifies the name of the Bastion host." + type = string +} + +variable "public_ip_name" { + description = "Specifies the name of the public IP address of the Bastion host." + type = string +} + +variable "location" { + description = "Specifies the location of the Bastion host and its public IP address." + type = string +} + +variable "resource_group_name" { + description = "Specifies the name of the resource group." + type = string +} + +variable "sku" { + description = "Specifies the SKU of the Bastion host." + type = string + default = "Basic" +} + +variable "subnet_id" { + description = "Specifies the resource id of the AzureBastionSubnet subnet." + type = string +} + +variable "log_analytics_workspace_id" { + description = "Specifies the resource id of the Log Analytics workspace." + type = string +} + +variable "diagnostic_settings_name" { + description = "Specifies the name of the diagnostic settings." + type = string + default = "default" +} diff --git a/terraform/modules/container_registry/main.tf b/terraform/modules/container_registry/main.tf new file mode 100644 index 0000000..38c3bee --- /dev/null +++ b/terraform/modules/container_registry/main.tf @@ -0,0 +1,29 @@ +# Container registry that every sample pushes its image to, and the diagnostic settings that send its logs and +# metrics to the Log Analytics workspace +resource "azurerm_container_registry" "registry" { + name = var.name + location = var.location + resource_group_name = var.resource_group_name + sku = var.sku + + # The scripts create the registry with --admin-enabled true + admin_enabled = true +} + +resource "azurerm_monitor_diagnostic_setting" "diagnostic_setting" { + name = var.diagnostic_settings_name + target_resource_id = azurerm_container_registry.registry.id + log_analytics_workspace_id = var.log_analytics_workspace_id + + enabled_log { + category = "ContainerRegistryRepositoryEvents" + } + + enabled_log { + category = "ContainerRegistryLoginEvents" + } + + enabled_metric { + category = "AllMetrics" + } +} diff --git a/terraform/modules/container_registry/outputs.tf b/terraform/modules/container_registry/outputs.tf new file mode 100644 index 0000000..bb85eee --- /dev/null +++ b/terraform/modules/container_registry/outputs.tf @@ -0,0 +1,7 @@ +output "id" { + value = azurerm_container_registry.registry.id +} + +output "name" { + value = azurerm_container_registry.registry.name +} diff --git a/terraform/modules/container_registry/variables.tf b/terraform/modules/container_registry/variables.tf new file mode 100644 index 0000000..478ef74 --- /dev/null +++ b/terraform/modules/container_registry/variables.tf @@ -0,0 +1,31 @@ +variable "name" { + description = "Specifies the name of the container registry." + type = string +} + +variable "location" { + description = "Specifies the location of the container registry." + type = string +} + +variable "resource_group_name" { + description = "Specifies the name of the resource group." + type = string +} + +variable "sku" { + description = "Specifies the SKU of the container registry." + type = string + default = "Basic" +} + +variable "log_analytics_workspace_id" { + description = "Specifies the resource id of the Log Analytics workspace." + type = string +} + +variable "diagnostic_settings_name" { + description = "Specifies the name of the diagnostic settings." + type = string + default = "default" +} diff --git a/terraform/modules/key_vault/main.tf b/terraform/modules/key_vault/main.tf new file mode 100644 index 0000000..0396d69 --- /dev/null +++ b/terraform/modules/key_vault/main.tf @@ -0,0 +1,28 @@ +# Key vault with Azure RBAC authorization, and the diagnostic settings that send its logs and metrics to the +# Log Analytics workspace +resource "azurerm_key_vault" "key_vault" { + name = var.name + location = var.location + resource_group_name = var.resource_group_name + tenant_id = var.tenant_id + sku_name = "standard" + rbac_authorization_enabled = true +} + +resource "azurerm_monitor_diagnostic_setting" "diagnostic_setting" { + name = var.diagnostic_settings_name + target_resource_id = azurerm_key_vault.key_vault.id + log_analytics_workspace_id = var.log_analytics_workspace_id + + enabled_log { + category = "AuditEvent" + } + + enabled_log { + category = "AzurePolicyEvaluationDetails" + } + + enabled_metric { + category = "AllMetrics" + } +} diff --git a/terraform/modules/key_vault/outputs.tf b/terraform/modules/key_vault/outputs.tf new file mode 100644 index 0000000..0a9f578 --- /dev/null +++ b/terraform/modules/key_vault/outputs.tf @@ -0,0 +1,7 @@ +output "id" { + value = azurerm_key_vault.key_vault.id +} + +output "name" { + value = azurerm_key_vault.key_vault.name +} diff --git a/terraform/modules/key_vault/variables.tf b/terraform/modules/key_vault/variables.tf new file mode 100644 index 0000000..6667377 --- /dev/null +++ b/terraform/modules/key_vault/variables.tf @@ -0,0 +1,30 @@ +variable "name" { + description = "Specifies the name of the key vault." + type = string +} + +variable "location" { + description = "Specifies the location of the key vault." + type = string +} + +variable "resource_group_name" { + description = "Specifies the name of the resource group." + type = string +} + +variable "tenant_id" { + description = "Specifies the tenant id of the key vault." + type = string +} + +variable "log_analytics_workspace_id" { + description = "Specifies the resource id of the Log Analytics workspace." + type = string +} + +variable "diagnostic_settings_name" { + description = "Specifies the name of the diagnostic settings." + type = string + default = "default" +} diff --git a/terraform/modules/log_analytics/main.tf b/terraform/modules/log_analytics/main.tf new file mode 100644 index 0000000..cc90bdf --- /dev/null +++ b/terraform/modules/log_analytics/main.tf @@ -0,0 +1,12 @@ +# Log Analytics workspace that Container Insights and the diagnostic settings of every resource send their data to +resource "azurerm_log_analytics_workspace" "workspace" { + name = var.name + location = var.location + resource_group_name = var.resource_group_name + sku = var.sku + + # The scripts create the workspace with --identity-type SystemAssigned + identity { + type = "SystemAssigned" + } +} diff --git a/terraform/modules/log_analytics/outputs.tf b/terraform/modules/log_analytics/outputs.tf new file mode 100644 index 0000000..1b1c104 --- /dev/null +++ b/terraform/modules/log_analytics/outputs.tf @@ -0,0 +1,7 @@ +output "id" { + value = azurerm_log_analytics_workspace.workspace.id +} + +output "name" { + value = azurerm_log_analytics_workspace.workspace.name +} diff --git a/terraform/modules/log_analytics/variables.tf b/terraform/modules/log_analytics/variables.tf new file mode 100644 index 0000000..dac5acc --- /dev/null +++ b/terraform/modules/log_analytics/variables.tf @@ -0,0 +1,20 @@ +variable "name" { + description = "Specifies the name of the Log Analytics workspace." + type = string +} + +variable "location" { + description = "Specifies the location of the Log Analytics workspace." + type = string +} + +variable "resource_group_name" { + description = "Specifies the name of the resource group." + type = string +} + +variable "sku" { + description = "Specifies the pricing tier of the Log Analytics workspace." + type = string + default = "PerGB2018" +} diff --git a/terraform/modules/virtual_network/main.tf b/terraform/modules/virtual_network/main.tf new file mode 100644 index 0000000..d47b2ca --- /dev/null +++ b/terraform/modules/virtual_network/main.tf @@ -0,0 +1,59 @@ +# Virtual network with the subnets of the two node pools and of the Bastion host, and the diagnostic settings that +# send its logs and metrics to the Log Analytics workspace +resource "azurerm_virtual_network" "virtual_network" { + name = var.name + location = var.location + resource_group_name = var.resource_group_name + address_space = [var.address_prefix] +} + +# An Azure Files NFS share can only be reached from a virtual network, so the Azure Files CSI driver needs the +# Microsoft.Storage service endpoint on the node subnets to mount an NFS volume +resource "azurerm_subnet" "system_subnet" { + name = var.system_subnet_name + resource_group_name = var.resource_group_name + virtual_network_name = azurerm_virtual_network.virtual_network.name + address_prefixes = [var.system_subnet_prefix] + + service_endpoint { + service = "Microsoft.Storage" + } +} + +# The subnets are created one at a time: Azure rejects concurrent changes to the same virtual network +resource "azurerm_subnet" "user_subnet" { + name = var.user_subnet_name + resource_group_name = var.resource_group_name + virtual_network_name = azurerm_virtual_network.virtual_network.name + address_prefixes = [var.user_subnet_prefix] + + service_endpoint { + service = "Microsoft.Storage" + } + + depends_on = [azurerm_subnet.system_subnet] +} + +# Azure Bastion looks this subnet up by name +resource "azurerm_subnet" "bastion_subnet" { + name = "AzureBastionSubnet" + resource_group_name = var.resource_group_name + virtual_network_name = azurerm_virtual_network.virtual_network.name + address_prefixes = [var.bastion_subnet_prefix] + + depends_on = [azurerm_subnet.user_subnet] +} + +resource "azurerm_monitor_diagnostic_setting" "diagnostic_setting" { + name = var.diagnostic_settings_name + target_resource_id = azurerm_virtual_network.virtual_network.id + log_analytics_workspace_id = var.log_analytics_workspace_id + + enabled_log { + category = "VMProtectionAlerts" + } + + enabled_metric { + category = "AllMetrics" + } +} diff --git a/terraform/modules/virtual_network/outputs.tf b/terraform/modules/virtual_network/outputs.tf new file mode 100644 index 0000000..657b7da --- /dev/null +++ b/terraform/modules/virtual_network/outputs.tf @@ -0,0 +1,19 @@ +output "id" { + value = azurerm_virtual_network.virtual_network.id +} + +output "name" { + value = azurerm_virtual_network.virtual_network.name +} + +output "system_subnet_id" { + value = azurerm_subnet.system_subnet.id +} + +output "user_subnet_id" { + value = azurerm_subnet.user_subnet.id +} + +output "bastion_subnet_id" { + value = azurerm_subnet.bastion_subnet.id +} diff --git a/terraform/modules/virtual_network/variables.tf b/terraform/modules/virtual_network/variables.tf new file mode 100644 index 0000000..eec5402 --- /dev/null +++ b/terraform/modules/virtual_network/variables.tf @@ -0,0 +1,61 @@ +variable "name" { + description = "Specifies the name of the virtual network." + type = string +} + +variable "location" { + description = "Specifies the location of the virtual network." + type = string +} + +variable "resource_group_name" { + description = "Specifies the name of the resource group." + type = string +} + +variable "address_prefix" { + description = "Specifies the address prefix of the virtual network." + type = string + default = "10.0.0.0/8" +} + +variable "system_subnet_name" { + description = "Specifies the name of the subnet of the system node pool." + type = string + default = "SystemSubnet" +} + +variable "system_subnet_prefix" { + description = "Specifies the address prefix of the subnet of the system node pool." + type = string + default = "10.240.0.0/16" +} + +variable "user_subnet_name" { + description = "Specifies the name of the subnet of the user node pool." + type = string + default = "UserSubnet" +} + +variable "user_subnet_prefix" { + description = "Specifies the address prefix of the subnet of the user node pool." + type = string + default = "10.241.0.0/16" +} + +variable "bastion_subnet_prefix" { + description = "Specifies the address prefix of the AzureBastionSubnet subnet." + type = string + default = "10.242.2.0/24" +} + +variable "log_analytics_workspace_id" { + description = "Specifies the resource id of the Log Analytics workspace." + type = string +} + +variable "diagnostic_settings_name" { + description = "Specifies the name of the diagnostic settings." + type = string + default = "default" +} diff --git a/terraform/outputs.tf b/terraform/outputs.tf new file mode 100644 index 0000000..446b3ae --- /dev/null +++ b/terraform/outputs.tf @@ -0,0 +1,19 @@ +output "resource_group_name" { + value = azurerm_resource_group.resource_group.name +} + +output "aks_cluster_name" { + value = module.aks_cluster.name +} + +output "acr_name" { + value = module.container_registry.name +} + +output "key_vault_name" { + value = module.key_vault.name +} + +output "log_analytics_name" { + value = module.log_analytics.name +} diff --git a/terraform/providers.tf b/terraform/providers.tf new file mode 100644 index 0000000..23a94ea --- /dev/null +++ b/terraform/providers.tf @@ -0,0 +1,65 @@ +terraform { + required_version = ">= 1.9.0" + + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "=5.10.0" + } + # azapi is needed only for the Managed Gateway API installation, which azurerm + # does not expose yet (hashicorp/terraform-provider-azurerm#31710). + azapi = { + source = "azure/azapi" + version = "=2.13.0" + } + } +} + +# Dual-target configuration: every identity field defaults to null (Azure CLI auth +# against real Azure). deploy.sh passes the LocalStack values when targeting the +# emulator: metadata_host=localhost.localstack.cloud:4566, all-zero GUIDs for +# subscription/tenant/client, and the placeholder client secret. +provider "azurerm" { + features { + resource_group { + prevent_deletion_if_contains_resources = false + } + key_vault { + # Auto-purge on real Azure (prevents the soft-delete global-name lock on + # re-runs); skipped on the emulator, which keeps no soft-deleted vaults and + # 404s the purge call (DeletedVaultNotFound), failing the destroy. + purge_soft_delete_on_destroy = var.metadata_host == null + } + } + + metadata_host = var.metadata_host + subscription_id = var.subscription_id + tenant_id = var.tenant_id + client_id = var.client_id + client_secret = var.client_secret + + # The emulator has no resource-provider registration surface. On Azure, "legacy" keeps + # registering the resource providers the stack needs, which azurerm 5.0 no longer does + # by default ("none"). + resource_provider_registrations = var.metadata_host == null ? "legacy" : "none" +} + +# azapi does not use azurerm's metadata_host discovery: when targeting the emulator +# deploy.sh passes resource_manager_endpoint explicitly and instance discovery is +# disabled (the emulator authority is not known to the Entra metadata service, the +# same reason `lstk az start-interception` sets core.instance_discovery=false for +# the az CLI). +provider "azapi" { + subscription_id = var.subscription_id + tenant_id = var.tenant_id + client_id = var.client_id + client_secret = var.client_secret + + disable_instance_discovery = var.resource_manager_endpoint == null ? null : true + + endpoint = var.resource_manager_endpoint == null ? null : [{ + resource_manager_endpoint = var.resource_manager_endpoint + active_directory_authority_host = var.resource_manager_endpoint + resource_manager_audience = var.resource_manager_endpoint + }] +} diff --git a/terraform/terraform.tfvars b/terraform/terraform.tfvars new file mode 100644 index 0000000..f08082a --- /dev/null +++ b/terraform/terraform.tfvars @@ -0,0 +1,13 @@ +# The defaults of the cluster scripts. deploy.sh overrides each of them from its environment variables. +prefix = "local" +suffix = "test" +location = "italynorth" +identity_type = "SystemAssigned" +network_option = 1 +system_node_count = 1 +user_node_count = 1 +node_vm_size = "Standard_D4ds_v5" +os_sku = "AzureLinux" +os_disk_size_gb = 50 +os_disk_type = "Ephemeral" +max_pods = 100 diff --git a/terraform/variables.tf b/terraform/variables.tf new file mode 100644 index 0000000..f0c6a57 --- /dev/null +++ b/terraform/variables.tf @@ -0,0 +1,174 @@ +variable "prefix" { + description = "Specifies the prefix of every resource name, as in the scripts." + type = string + default = "local" +} + +variable "suffix" { + description = "Specifies the suffix of every resource name, as in the scripts." + type = string + default = "test" +} + +variable "location" { + description = "Specifies the location of every resource." + type = string + default = "italynorth" +} + +variable "identity_type" { + description = "Specifies the cluster identity: SystemAssigned as in scripts/01-system-assigned-managed-identity.sh, UserAssigned as in scripts/01-user-assigned-managed-identity.sh." + type = string + default = "SystemAssigned" + + validation { + condition = contains(["SystemAssigned", "UserAssigned"], var.identity_type) + error_message = "identity_type must be SystemAssigned or UserAssigned." + } +} + +variable "network_option" { + description = "Specifies the network plugin and policy, as the scripts' menu: 1 Azure CNI Overlay with the Azure network policy, 2 with the Cilium dataplane and network policy, 3 with the Calico network policy." + type = number + default = 1 + + validation { + condition = contains([1, 2, 3], var.network_option) + error_message = "network_option must be 1, 2 or 3." + } +} + +variable "kubernetes_version" { + description = "Specifies the Kubernetes version. null lets AKS pick its default; deploy.sh passes the latest GA version in the region, as the scripts do." + type = string + default = null +} + +variable "system_node_count" { + description = "Specifies the number of nodes of the system node pool. The autoscaler range, 1 to 3, is widened to include it." + type = number + default = 1 +} + +variable "user_node_count" { + description = "Specifies the number of nodes of the user node pool. The autoscaler range, 1 to 3, is widened to include it." + type = number + default = 1 +} + +variable "node_vm_size" { + description = "Specifies the VM size of the nodes of both node pools." + type = string + default = "Standard_D4ds_v5" +} + +variable "os_sku" { + description = "Specifies the OS SKU of the nodes." + type = string + default = "AzureLinux" +} + +variable "os_disk_size_gb" { + description = "Specifies the OS disk size in GB of the nodes." + type = number + default = 50 +} + +variable "os_disk_type" { + description = "Specifies the OS disk type of the nodes." + type = string + default = "Ephemeral" +} + +variable "max_pods" { + description = "Specifies the maximum number of pods per node." + type = number + default = 100 +} + +variable "ssh_public_key" { + description = "Specifies the SSH public key of the Linux nodes. On Azure, deploy.sh reads ~/.ssh/id_rsa.pub, as the scripts do." + type = string + default = "" +} + +variable "windows_admin_username" { + description = "Specifies the administrator user name of Windows nodes." + type = string + default = "azadmin" +} + +variable "windows_admin_password" { + description = "Specifies the administrator password of Windows nodes. deploy.sh passes it as TF_VAR_windows_admin_password." + type = string + sensitive = true +} + +variable "aad_admin_group_object_ids" { + description = "Specifies the object ids of the Microsoft Entra ID groups that administer the cluster. On Azure, pass real group object ids." + type = list(string) + default = [] +} + +variable "acr_sku" { + description = "Specifies the SKU of the container registry." + type = string + default = "Basic" +} + +variable "log_analytics_sku" { + description = "Specifies the pricing tier of the Log Analytics workspace." + type = string + default = "PerGB2018" +} + +variable "diagnostic_settings_name" { + description = "Specifies the name of the diagnostic settings of every resource." + type = string + default = "default" +} + +variable "gateway_api_enabled" { + description = "Specifies whether to install the managed Gateway API CRDs with azapi. deploy.sh passes false on the emulator." + type = bool + default = true +} + +# Provider settings: null targets Azure with the Azure CLI credentials; deploy.sh passes the LocalStack values when +# the Azure CLI targets the emulator +variable "metadata_host" { + description = "Specifies the metadata host of the azurerm provider (the emulator's, or null for Azure)." + type = string + default = null +} + +variable "resource_manager_endpoint" { + description = "Specifies the Resource Manager endpoint of the azapi provider (the emulator's, or null for Azure)." + type = string + default = null +} + +variable "subscription_id" { + description = "Specifies the subscription id." + type = string + default = null +} + +variable "tenant_id" { + description = "Specifies the tenant id." + type = string + default = null +} + +variable "client_id" { + description = "Specifies the client id (the emulator's placeholder, or null for the Azure CLI credentials)." + type = string + default = null +} + +variable "client_secret" { + description = "Specifies the client secret (the emulator's placeholder, or null for the Azure CLI credentials)." + type = string + default = null + sensitive = true +} diff --git a/tests/test-diagnostic-settings.sh b/tests/test-diagnostic-settings.sh new file mode 100755 index 0000000..fa5762d --- /dev/null +++ b/tests/test-diagnostic-settings.sh @@ -0,0 +1,72 @@ +#!/bin/bash +# Check that every resource the cluster scripts, bicep/ and terraform/ create has a diagnostic setting called +# "default". A resource the deployment left out (scripts/01-configurable-cluster.sh can skip the virtual network, +# the Bastion host and the key vault) is reported as skipped. The resources are looked up by the names the scripts +# give them, not by type, because the samples add resources of their own to the same resource group. +# +# Usage: tests/test-diagnostic-settings.sh +# Override PREFIX, SUFFIX or RESOURCE_GROUP_NAME when the cluster was deployed with other names. +set -euo pipefail + +PREFIX=${PREFIX:-local} +SUFFIX=${SUFFIX:-test} +RESOURCE_GROUP_NAME=${RESOURCE_GROUP_NAME:-$PREFIX-rg} +DIAGNOSTIC_SETTINGS_NAME=${DIAGNOSTIC_SETTINGS_NAME:-default} + +aks_cluster_name="$PREFIX-aks-$SUFFIX" + +# Name and resource type of every resource that gets the diagnostic settings +expected=( + "$aks_cluster_name|Microsoft.ContainerService/managedClusters" + "$PREFIX-vnet-$SUFFIX|Microsoft.Network/virtualNetworks" + "$PREFIX-bastion-ip-$SUFFIX|Microsoft.Network/publicIPAddresses" + "$PREFIX-bastion-$SUFFIX|Microsoft.Network/bastionHosts" + "${PREFIX,,}acr${SUFFIX,,}|Microsoft.ContainerRegistry/registries" + "$PREFIX-kv-$SUFFIX|Microsoft.KeyVault/vaults" +) + +# Name, type and ID of every resource in the resource group, in one call. The carriage returns that the +# Windows Azure CLI writes are removed, so the lines compare equal on every platform. +if ! resources=$(az resource list \ + --resource-group "$RESOURCE_GROUP_NAME" \ + --query "[].[name, type, id]" \ + --output tsv \ + --only-show-errors | tr -d '\r'); then + echo "FAIL: cannot list the resources of the [$RESOURCE_GROUP_NAME] resource group" + exit 1 +fi + +# Print the ID of the resource with the given name and type, or nothing when the group has none +resource_id() { + awk -F '\t' -v name="${1,,}" -v type="${2,,}" \ + 'tolower($1) == name && tolower($2) == type { print $3; exit }' <<<"$resources" +} + +if [[ -z $(resource_id "$aks_cluster_name" Microsoft.ContainerService/managedClusters) ]]; then + echo "FAIL: no [$aks_cluster_name] AKS cluster in the [$RESOURCE_GROUP_NAME] resource group" + exit 1 +fi + +checked=0 +failures=0 +for entry in "${expected[@]}"; do + IFS='|' read -r name type <<<"$entry" + id=$(resource_id "$name" "$type") + if [[ -z $id ]]; then + echo "SKIP: no [$name] $type in the [$RESOURCE_GROUP_NAME] resource group" + continue + fi + checked=$((checked + 1)) + if az monitor diagnostic-settings show \ + --name "$DIAGNOSTIC_SETTINGS_NAME" \ + --resource "$id" \ + --only-show-errors &>/dev/null; then + echo "PASS: [$name] has the [$DIAGNOSTIC_SETTINGS_NAME] diagnostic settings" + else + echo "FAIL: [$name] has no [$DIAGNOSTIC_SETTINGS_NAME] diagnostic settings" + failures=$((failures + 1)) + fi +done + +echo "Checked the [$DIAGNOSTIC_SETTINGS_NAME] diagnostic settings of $checked resources in the [$RESOURCE_GROUP_NAME] resource group: $failures missing" +[[ $failures -eq 0 ]]