diff --git a/README.md b/README.md index d550e2f..857d45a 100644 --- a/README.md +++ b/README.md @@ -103,7 +103,7 @@ $HOME/fivetran-proxy-agent/ ├── proxy-agent-manager.sh --> Management script ├── config/ │ └── config.json --> Agent configuration (permissions: 600) -├── logs/ --> Agent and manager logs +├── logs/ --> Agent, manager and doctor logs └── version --> Pinned agent version (for example 1.2.9) # Windows @@ -111,7 +111,7 @@ $HOME/fivetran-proxy-agent/ ├── proxy-agent-manager.ps1 --> Management script ├── config\ │ └── config.json --> Agent configuration (owner read/write; SYSTEM and Administrators read) -├── logs\ --> Agent and manager logs +├── logs\ --> Agent, manager and doctor logs └── version --> Pinned agent version (for example 1.2.9) ``` @@ -122,7 +122,7 @@ $HOME/fivetran-proxy-agent/ Use `proxy-agent-manager.sh` to control the agent: ```bash -./proxy-agent-manager.sh {start|stop|restart|upgrade|status|logs} +./proxy-agent-manager.sh {start|stop|restart|upgrade|status|logs|doctor [options]} ``` ### Windows @@ -130,7 +130,7 @@ Use `proxy-agent-manager.sh` to control the agent: Use `proxy-agent-manager.ps1` to control the agent: ```powershell -& "$env:USERPROFILE\fivetran-proxy-agent\proxy-agent-manager.ps1" {start|stop|restart|upgrade|status|logs} +& "$env:USERPROFILE\fivetran-proxy-agent\proxy-agent-manager.ps1" {start|stop|restart|upgrade|status|logs|doctor [options]} ``` | Command | Description | @@ -141,6 +141,7 @@ Use `proxy-agent-manager.ps1` to control the agent: | `upgrade` | Pull and start the latest version, with auto-rollback on failure | | `status` | Show container name, image, and health status | | `logs` | Stream live container logs | +| `doctor` | Check the installation and connectivity to Fivetran in a separate container. The doctor's log and archive are saved to `logs/`, or its output if the doctor saves no log, as in 1.2.10. Options are passed to the agent's `--doctor`, e.g. `doctor --region us-east-1` | ## Troubleshooting @@ -163,7 +164,7 @@ CI (`.github/workflows/ci.yml`) runs on every pull request: - `shellcheck` on the shell scripts and PSScriptAnalyzer (settings in `PSScriptAnalyzerSettings.psd1`) on the PowerShell scripts. - Unit tests for the management scripts and the Windows installer: `tests/bash/manager.bats` (bats), and `tests/pwsh/Manager.Tests.ps1` and `tests/pwsh/Install.Tests.ps1` (Pester 5, under both Windows PowerShell 5.1 and PowerShell 7). They run the scripts with stubbed `docker`, registry and host calls, so they need no Docker daemon. -- Integration tests that run the installer and management script against a real Docker daemon: `tests/integration/linux.sh` on Ubuntu (amd64 and arm64), and `tests/integration/windows.ps1` on Windows Server 2022 and 2025 with Windows containers (on 2025 also under PowerShell 7). They use a stub agent image (`tests/stub/`) that only writes the heartbeat file, and a local stand-in for the registry, the script download and the configure endpoint (`tests/integration/registry_server.py`), so they need no Fivetran credentials. They cover installing from a config file, from a `TOKEN`, and from a legacy WebSocket config, then rollback, upgrade, restart and stop. +- Integration tests that run the installer and management script against a real Docker daemon: `tests/integration/linux.sh` on Ubuntu (amd64 and arm64), and `tests/integration/windows.ps1` on Windows Server 2022 and 2025 with Windows containers (on 2025 also under PowerShell 7). They use a stub agent image (`tests/stub/`) that only writes the heartbeat file, or stands in for the doctor, and a local stand-in for the registry, the script download and the configure endpoint (`tests/integration/registry_server.py`), so they need no Fivetran credentials. They cover installing from a config file, from a `TOKEN`, and from a legacy WebSocket config, then rollback, upgrade, doctor, restart and stop. To run them locally you need Docker for the integration tests, `python3` (`python` on Windows) for the registry stand-in, [bats-core](https://github.com/bats-core/bats-core) for the bash tests and [Pester 5](https://pester.dev) for the PowerShell tests. The stub images pull `alpine` and `servercore` base images, so the integration tests need network access for that. diff --git a/proxy-agent-manager.ps1 b/proxy-agent-manager.ps1 index af04653..5c4050e 100644 --- a/proxy-agent-manager.ps1 +++ b/proxy-agent-manager.ps1 @@ -1,7 +1,10 @@ #Requires -Version 5.1 param( [Parameter(Position = 0)] - [string]$Command + [string]$Command, + # Passed to --doctor by the doctor command, e.g. --region us-east-1 + [Parameter(ValueFromRemainingArguments)] + [string[]]$DoctorArgs ) $ErrorActionPreference = 'Stop' @@ -29,9 +32,9 @@ if (-not $WINDOWS_VERSION) { $WINDOWS_VERSION = $null } # ── Startup validation ─────────────────────────────────────────────────────── -$validCommands = @('start', 'stop', 'restart', 'upgrade', 'status', 'logs') +$validCommands = @('start', 'stop', 'restart', 'upgrade', 'status', 'logs', 'doctor') if (-not $Command -or $Command -notin $validCommands) { - Write-Host "Usage: $($MyInvocation.MyCommand.Name) {start|stop|restart|upgrade|status|logs}" + Write-Host "Usage: $($MyInvocation.MyCommand.Name) {start|stop|restart|upgrade|status|logs|doctor [options]}" exit 1 } @@ -299,6 +302,46 @@ function Assert-MemoryAllocation { } } +# Returns the config path inside the container. +function Get-ContainerConfigPath { + param([Parameter(Mandatory)][ValidateSet('linux', 'windows')][string]$ContainerOsType) + if ($ContainerOsType -eq 'windows') { return 'C:/config/config.json' } + return '/config/config.json' +} + +# Returns the log folder inside the container. +function Get-ContainerLogDir { + param([Parameter(Mandatory)][ValidateSet('linux', 'windows')][string]$ContainerOsType) + if ($ContainerOsType -eq 'windows') { return 'C:/app/logs' } + return $CONTAINER_LOG_DIR +} + +# Returns the docker run arguments shared by the agent and doctor containers, +# so the doctor checks the agent's config, memory limit and log folder. +function Get-AgentRunArgs { + param([Parameter(Mandatory)][ValidateSet('linux', 'windows')][string]$ContainerOsType) + + $null = New-Item -ItemType Directory -Force -Path (Join-Path $BASE_DIR 'logs') + + # Docker Desktop for Windows requires forward-slash paths in volume mounts. + $logsMount = (Join-Path $BASE_DIR 'logs') -replace '\\', '/' + if ($ContainerOsType -eq 'windows') { + # Windows containers can only mount folders. + $configMount = "$((Join-Path $BASE_DIR 'config') -replace '\\', '/'):C:/config:ro" + } else { + $configMount = "$((Join-Path $BASE_DIR 'config\config.json') -replace '\\', '/'):/config/config.json:ro" + } + $containerLogDir = Get-ContainerLogDir -ContainerOsType $ContainerOsType + + return @( + "--memory=${MEMORY_ALLOCATION_MB}m", + '--env', 'IS_DOCKER=true', + '--env', "LOG_FOLDER_PATH=$containerLogDir", + '-v', $configMount, + '-v', "${logsMount}:$containerLogDir" + ) +} + function Stop-ProxyAgent { try { docker inspect $CONTAINER_NAME 2>&1 | Out-Null } catch {} if ($LASTEXITCODE -eq 0) { @@ -317,22 +360,13 @@ function Start-ProxyAgent { Stop-ProxyAgent - $null = New-Item -ItemType Directory -Force -Path (Join-Path $BASE_DIR 'logs') - - # Docker Desktop for Windows requires forward-slash paths in volume mounts. - $configFileMount = (Join-Path $BASE_DIR 'config\config.json') -replace '\\', '/' - $configDirectoryMount = (Join-Path $BASE_DIR 'config') -replace '\\', '/' - $logsMount = (Join-Path $BASE_DIR 'logs') -replace '\\', '/' - # Build argument list to avoid PowerShell variable expansion inside the health check. $dockerArgs = @( 'run', '-d', '--name', $CONTAINER_NAME, '--restart', 'unless-stopped', - "--memory=${MEMORY_ALLOCATION_MB}m", '--label', 'fivetran=proxy-agent', '--label', "proxy_agent_id=$AGENT_ID", - '--env', 'IS_DOCKER=true', '--env', 'HEARTBEAT_EXPIRY_SECONDS=30', '--health-interval', '10s', '--health-retries', '3', @@ -340,8 +374,6 @@ function Start-ProxyAgent { ) if ($ContainerOsType -eq 'windows') { - $containerConfigPath = 'C:/config/config.json' - $containerLogDir = 'C:/app/logs' $containerHeartbeatPath = 'C:/app/logs/proxy-agent-heartbeat.txt' $windowsHealthCheckScript = @( '$path = $env:HEARTBEAT_PATH' @@ -352,29 +384,19 @@ function Start-ProxyAgent { $windowsHealthCheckEncoded = [Convert]::ToBase64String([System.Text.Encoding]::Unicode.GetBytes($windowsHealthCheckScript)) $windowsHealthCheck = "powershell.exe -NoProfile -EncodedCommand $windowsHealthCheckEncoded" $dockerArgs += @( - '--env', "LOG_FOLDER_PATH=$containerLogDir", '--env', "HEARTBEAT_PATH=$containerHeartbeatPath", '--health-cmd', $windowsHealthCheck, - '--health-timeout', '15s', - '-v', "${configDirectoryMount}:C:/config:ro", - '-v', "${logsMount}:$containerLogDir", - "${IMAGE}:${ImageTag}", - '-i', $containerConfigPath + '--health-timeout', '15s' ) } else { - $containerConfigPath = '/config/config.json' - $containerLogDir = $CONTAINER_LOG_DIR $dockerArgs += @( - '--env', "LOG_FOLDER_PATH=$containerLogDir", '--env', 'HEARTBEAT_PATH=/tmp/proxy-agent-heartbeat.txt', '--health-cmd', '[ ! -f $HEARTBEAT_PATH ] || { . $HEARTBEAT_PATH && [ $(date +%s) -lt $HEARTBEAT_EXPIRE_AT ]; }', - '--health-timeout', '3s', - '-v', "${configFileMount}:/config/config.json:ro", - '-v', "${logsMount}:$containerLogDir", - "${IMAGE}:${ImageTag}", - '-i', $containerConfigPath + '--health-timeout', '3s' ) } + $dockerArgs += Get-AgentRunArgs -ContainerOsType $ContainerOsType + $dockerArgs += @("${IMAGE}:${ImageTag}", '-i', (Get-ContainerConfigPath -ContainerOsType $ContainerOsType)) & docker @dockerArgs | Out-Host if ($LASTEXITCODE -ne 0) { @@ -461,6 +483,59 @@ function Invoke-Upgrade { } } +# Runs the doctor in a throwaway container beside the agent's, passing +# $DoctorArgs to --doctor (e.g. --region us-east-1), and exits with its exit +# code. Newer doctors save their log and an archive to the mounted log folder, +# so they outlive the container. Otherwise, as with older doctors (e.g. +# 1.2.10), which only print their report, or when the doctor fails before +# saving its log, its output is saved there instead. +function Invoke-Doctor { + param([string[]]$DoctorArgs) + + $containerOsType = Get-ContainerOsType + $windowsVersion = Resolve-WindowsVersion -ContainerOsType $containerOsType + $imageTag = Get-ImageTag -Version $CURRENT_VERSION -ContainerOsType $containerOsType -WindowsVersion $windowsVersion + + # No terminal (-t), which would merge the doctor's progress line into the + # saved output. The working folder is where newer doctors save their log if + # they can't read the config. + $dockerArgs = @('run', '--rm', '--workdir', (Get-ContainerLogDir -ContainerOsType $containerOsType)) + $dockerArgs += Get-AgentRunArgs -ContainerOsType $containerOsType + $dockerArgs += @("${IMAGE}:${imageTag}", (Get-ContainerConfigPath -ContainerOsType $containerOsType), '--doctor') + if ($DoctorArgs) { $dockerArgs += $DoctorArgs } + + $logsDir = Join-Path $BASE_DIR 'logs' + $timestamp = [DateTime]::UtcNow.ToString("yyyy-MM-dd'T'HH-mm-ss") + $outputFile = Join-Path $logsDir "proxy-agent-manager-doctor-$timestamp-$([Guid]::NewGuid().ToString('N')).log" + $earlierDoctorLogs = @(Get-ChildItem -LiteralPath $logsDir -Filter 'doctor-*.log' | ForEach-Object Name) + # Tee-Object can't choose the encoding on Windows PowerShell 5.1. + $outputStream = [System.IO.File]::Open($outputFile, [System.IO.FileMode]::CreateNew, [System.IO.FileAccess]::Write, [System.IO.FileShare]::Read) + $writer = [System.IO.StreamWriter]::new($outputStream, [System.Text.UTF8Encoding]::new($false)) + # Under 'Stop', Windows PowerShell 5.1 throws on the first line docker + # writes to stderr once it is redirected. + $ErrorActionPreference = 'Continue' + try { + # Stderr lines arrive as error records; "$_" is their text. + & docker @dockerArgs 2>&1 | ForEach-Object { + $line = "$_" + Write-Host $line + $writer.WriteLine($line) + } + $status = $LASTEXITCODE + } finally { + $writer.Dispose() + } + $newDoctorLogs = Get-ChildItem -LiteralPath $logsDir -Filter 'doctor-*.log' | Where-Object { $_.Name -notin $earlierDoctorLogs } + if ($newDoctorLogs) { + Remove-Item -LiteralPath $outputFile + # The doctor printed paths inside the container. + Write-Host "$(Get-ContainerLogDir -ContainerOsType $containerOsType) in the container is $logsDir on this host." + } else { + Write-Host "The doctor's output was saved to $outputFile" + } + exit $status +} + # ── Commands ───────────────────────────────────────────────────────────────── switch ($Command) { @@ -491,4 +566,9 @@ switch ($Command) { 'logs' { docker logs -f $CONTAINER_NAME } + 'doctor' { + $CURRENT_VERSION = Get-ReleaseVersion -Tag $PINNED_VERSION + Assert-MemoryAllocation + Invoke-Doctor -DoctorArgs $DoctorArgs + } } diff --git a/proxy-agent-manager.sh b/proxy-agent-manager.sh index c28c14a..6692f77 100755 --- a/proxy-agent-manager.sh +++ b/proxy-agent-manager.sh @@ -2,7 +2,7 @@ # # Management script for Fivetran Proxy Agent (Docker) # -# Usage: ./proxy-agent-manager.sh {start|stop|restart|upgrade|status|logs} +# Usage: ./proxy-agent-manager.sh {start|stop|restart|upgrade|status|logs|doctor [options]} # set -euo pipefail @@ -49,6 +49,7 @@ fi LOGFILE="$BASE_DIR/logs/proxy-agent-manager.log" CONTAINER_LOG_DIR="/app/logs" +CONTAINER_CONFIG_FILE="/config/config.json" # ── Functions ──────────────────────────────────────────────────────────────── @@ -73,6 +74,16 @@ check_memory_allocation() { fi } +# Shared by the agent and doctor containers, so the doctor checks the agent's +# config, memory limit and log folder. +AGENT_RUN_ARGS=( + --memory="${MEMORY_ALLOCATION_MB}m" + --env IS_DOCKER=true + --env LOG_FOLDER_PATH="$CONTAINER_LOG_DIR" + -v "$BASE_DIR/config/config.json:$CONTAINER_CONFIG_FILE:ro" + -v "$BASE_DIR/logs:$CONTAINER_LOG_DIR" +) + get_latest_version() { local registry_host="${IMAGE%%/*}" local repository_path="${IMAGE#*/}" @@ -155,11 +166,9 @@ start_agent() { docker run -d \ --name "${CONTAINER_NAME}" \ --restart unless-stopped \ - --memory="${MEMORY_ALLOCATION_MB}m" \ --label fivetran=proxy-agent \ --label proxy_agent_id="$AGENT_ID" \ - --env IS_DOCKER=true \ - --env LOG_FOLDER_PATH="$CONTAINER_LOG_DIR" \ + "${AGENT_RUN_ARGS[@]}" \ --env HEARTBEAT_PATH=/tmp/proxy-agent-heartbeat.txt \ --env HEARTBEAT_EXPIRY_SECONDS=30 \ --health-cmd '[ ! -f $HEARTBEAT_PATH ] || { . $HEARTBEAT_PATH && [ $(date +%s) -lt $HEARTBEAT_EXPIRE_AT ]; }' \ @@ -167,10 +176,8 @@ start_agent() { --health-timeout 3s \ --health-retries 3 \ --health-start-period 30s \ - -v "$BASE_DIR/config/config.json:/config/config.json:ro" \ - -v "$BASE_DIR/logs:$CONTAINER_LOG_DIR" \ "$(image_tag "$version")" \ - -i /config/config.json + -i "$CONTAINER_CONFIG_FILE" local timeout=60 log "Waiting for ${CONTAINER_NAME} to become healthy (timeout: ${timeout}s)..." @@ -199,6 +206,56 @@ start_agent() { fi } +# Lists the logs doctors saved in the log folder, sorted for comm. +list_doctor_logs() { + find "$BASE_DIR/logs" -maxdepth 1 -name 'doctor-*.log' | LC_ALL=C sort +} + +# Runs the doctor in a throwaway container beside the agent's, passing "$@" to +# --doctor (e.g. --region us-east-1), and exits with its exit code. Newer +# doctors save their log and an archive to the mounted log folder, so they +# outlive the container. Otherwise, as with older doctors (e.g. 1.2.10), which +# only print their report, or when the doctor fails before saving its log, its +# output is saved there instead. +run_doctor() { + mkdir -p "$BASE_DIR/logs" + local output_file temporary_output_file earlier_doctor_logs timestamp + timestamp=$(date -u +'%Y-%m-%dT%H-%M-%S') + temporary_output_file=$(mktemp "$BASE_DIR/logs/proxy-agent-manager-doctor-${timestamp}.XXXXXXXXXX") + output_file="${temporary_output_file}.log" + # macOS mktemp requires Xs at the end, so add the discoverable .log suffix + # with a hard link. ln creates it atomically and won't overwrite a report. + if ! ln "$temporary_output_file" "$output_file"; then + rm -f "$temporary_output_file" + return 1 + fi + rm -f "$temporary_output_file" + earlier_doctor_logs=$(list_doctor_logs) + + # No terminal (-t), which would merge the doctor's progress line into the + # saved output. The working folder is where newer doctors save their log if + # they can't read the config. + # set +e so a failing doctor doesn't exit the script, and PIPESTATUS keeps + # the doctor's exit code even if tee fails, e.g. on a full disk. + set +e + docker run --rm \ + --workdir "$CONTAINER_LOG_DIR" \ + "${AGENT_RUN_ARGS[@]}" \ + "$(image_tag "$CURRENT_VERSION")" \ + "$CONTAINER_CONFIG_FILE" --doctor "$@" 2>&1 \ + | tee "$output_file" + local status="${PIPESTATUS[0]}" + set -e + if [ -n "$(LC_ALL=C comm -13 <(echo "$earlier_doctor_logs") <(list_doctor_logs))" ]; then + rm -f "$output_file" + # The doctor printed paths inside the container. + echo "$CONTAINER_LOG_DIR in the container is $BASE_DIR/logs on this host." + else + echo "The doctor's output was saved to $output_file" + fi + exit "$status" +} + # ── Commands ───────────────────────────────────────────────────────────────── case "${1:-}" in @@ -237,8 +294,15 @@ case "${1:-}" in docker logs -f "${CONTAINER_NAME}" ;; + doctor) + CURRENT_VERSION=$(get_release_version "$PINNED_VERSION") || exit 1 + check_memory_allocation + shift + run_doctor "$@" + ;; + *) - echo "Usage: $0 {start|stop|restart|upgrade|status|logs}" + echo "Usage: $0 {start|stop|restart|upgrade|status|logs|doctor [options]}" exit 1 ;; esac diff --git a/tests/bash/manager.bats b/tests/bash/manager.bats index 8fa0bde..b993a13 100644 --- a/tests/bash/manager.bats +++ b/tests/bash/manager.bats @@ -102,13 +102,13 @@ version_file() { [ "$(version_file)" = "2.0" ] } -@test "restart and upgrade reject an invalid MEMORY_ALLOCATION_MB without touching the agent" { +@test "restart, upgrade and doctor reject an invalid MEMORY_ALLOCATION_MB without touching the agent" { manager start echo "MEMORY_ALLOCATION_MB=4GB" > "$AGENT_DIR/settings.sh" local image version image=$(last_image) version=$(cat "$AGENT_DIR/version") - for command in restart upgrade; do + for command in restart upgrade doctor; do run manager "$command" [ "$status" -eq 1 ] [[ "$output" == *"Invalid MEMORY_ALLOCATION_MB"* ]] @@ -238,6 +238,91 @@ version_file() { grep -q "^rm proxy-agent-abc123" "$DOCKER_LOG" } +@test "doctor runs a throwaway container with the agent's config, logs and memory limit" { + run manager doctor + [ "$status" -eq 0 ] + grep -q -- "^run --rm " "$DOCKER_LOG" + grep -q -- "--memory=5120m" "$DOCKER_LOG" + grep -q -- "--env LOG_FOLDER_PATH=/app/logs" "$DOCKER_LOG" + grep -q -- "$AGENT_DIR/config/config.json:/config/config.json:ro" "$DOCKER_LOG" + grep -q -- "$AGENT_DIR/logs:/app/logs" "$DOCKER_LOG" + grep -q -- "--workdir /app/logs" "$DOCKER_LOG" + grep -q -- "$IMAGE_REPO:1.2.9-ubuntu-26.04 /config/config.json --doctor$" "$DOCKER_LOG" +} + +@test "doctor passes its options to the agent" { + run manager doctor --region us-east-1 --tcp-attempts 3 + [ "$status" -eq 0 ] + grep -q -- "/config/config.json --doctor --region us-east-1 --tcp-attempts 3$" "$DOCKER_LOG" +} + +@test "doctor exits with the doctor's exit code" { + export DOCKER_DOCTOR_EXIT_CODE=3 + run manager doctor + [ "$status" -eq 3 ] +} + +@test "doctor leaves the running agent alone" { + manager start + run manager doctor + [ "$status" -eq 0 ] + [ "$(last_image)" = "$IMAGE_REPO:1.2.9-ubuntu-26.04" ] + [ -z "$(grep -E "^(stop|rm)" "$DOCKER_LOG")" ] +} + +@test "doctor saves its output, stderr included, when the doctor saves no log" { + run manager doctor + [ "$status" -eq 0 ] + local saved + saved=$(ls "$AGENT_DIR"/logs/proxy-agent-manager-doctor-*.log) + grep -qx "stub doctor report" "$saved" + grep -qx "stub doctor log" "$saved" + [[ "$output" == *"stub doctor report"* ]] + [[ "$output" == *"output was saved to $saved"* ]] +} + +@test "doctor preserves fallback reports from runs in the same second" { + mkdir -p "$AGENT_DIR/bin" + cat > "$AGENT_DIR/bin/date" <<'EOF' +#!/usr/bin/env bash +printf '%s\n' '2020-01-02T03-04-05' +EOF + chmod +x "$AGENT_DIR/bin/date" + export PATH="$AGENT_DIR/bin:$PATH" + + export DOCKER_DOCTOR_REPORT="first doctor report" + run manager doctor + [ "$status" -eq 0 ] + + export DOCKER_DOCTOR_REPORT="second doctor report" + run manager doctor + [ "$status" -eq 0 ] + + local saved_reports=("$AGENT_DIR"/logs/proxy-agent-manager-doctor-*.log) + [ "${#saved_reports[@]}" -eq 2 ] + grep -qx "first doctor report" "${saved_reports[@]}" + grep -qx "second doctor report" "${saved_reports[@]}" +} + +@test "doctor saves its output when only an earlier run's doctor log exists" { + mkdir -p "$AGENT_DIR/logs" + touch "$AGENT_DIR/logs/doctor-2020-01-01T00-00-00.log" + run manager doctor + [ "$status" -eq 0 ] + ls "$AGENT_DIR"/logs/proxy-agent-manager-doctor-*.log +} + +@test "doctor keeps only its own log when the doctor saves one beside an earlier run's" { + mkdir -p "$AGENT_DIR/logs" + touch "$AGENT_DIR/logs/doctor-2020-01-01T00-00-00.log" + export DOCKER_DOCTOR_LOG_DIR="$AGENT_DIR/logs" + run manager doctor + [ "$status" -eq 0 ] + [ -f "$AGENT_DIR/logs/doctor-stub.log" ] + [ -z "$(find "$AGENT_DIR/logs" -name 'proxy-agent-manager-doctor-*.log')" ] + [[ "$output" == *"/app/logs in the container is $AGENT_DIR/logs on this host."* ]] +} + @test "unknown command prints usage" { run manager bogus [ "$status" -eq 1 ] diff --git a/tests/bash/stubs/docker b/tests/bash/stubs/docker index 8c49f18..d6302c7 100755 --- a/tests/bash/stubs/docker +++ b/tests/bash/stubs/docker @@ -4,11 +4,23 @@ # $DOCKER_LOG and keeps one container's worth of state: $DOCKER_LAST_IMAGE # exists while a container "runs" and holds the image it was started from. # Nothing is pulled or run. If $DOCKER_UNHEALTHY_TAG is set and the running -# image contains it, the health check reports unhealthy. +# image contains it, the health check reports unhealthy. A `run` of the +# doctor (with `--doctor`) leaves that state alone, prints a line to stdout +# and one to stderr, and exits with $DOCKER_DOCTOR_EXIT_CODE. If +# $DOCKER_DOCTOR_LOG_DIR is set, it also saves a log there, like newer +# doctors. # echo "$*" >> "$DOCKER_LOG" case "$1" in run) + if [[ " $* " == *" --doctor "* ]]; then + echo "${DOCKER_DOCTOR_REPORT:-stub doctor report}" + echo "stub doctor log" >&2 + if [ -n "${DOCKER_DOCTOR_LOG_DIR:-}" ]; then + echo "stub doctor log" > "$DOCKER_DOCTOR_LOG_DIR/doctor-stub.log" + fi + exit "${DOCKER_DOCTOR_EXIT_CODE:-0}" + fi for arg in "$@"; do case "$arg" in *proxy-agent:*) echo "$arg" > "$DOCKER_LAST_IMAGE" ;; esac done diff --git a/tests/integration/linux.sh b/tests/integration/linux.sh index 28728ca..6cce5ae 100755 --- a/tests/integration/linux.sh +++ b/tests/integration/linux.sh @@ -133,6 +133,17 @@ echo "== restart" "$MANAGER" restart assert_healthy +echo "== doctor saves its log and archive to the host's log folder" +doctor_status=0 +doctor_output=$("$MANAGER" doctor --region us-east-1) || doctor_status=$? +[[ "$doctor_output" == *"stub doctor stderr"* ]] || fail "doctor did not show the doctor's stderr: $doctor_output" +[ "$doctor_status" -eq 1 ] || fail "expected the doctor's exit code 1, got $doctor_status" +[[ "$doctor_output" == *"stub doctor report for options: --region us-east-1"* ]] || fail "doctor did not print its report: $doctor_output" +[ -f "$INSTALL_DIR/logs/doctor-stub.log" ] || fail "the doctor's log is not in the host's log folder" +[ -f "$INSTALL_DIR/logs/doctor-stub.zip" ] || fail "the doctor's archive is not in the host's log folder" +[ -z "$(find "$INSTALL_DIR/logs" -name 'proxy-agent-manager-doctor-*.log')" ] || fail "the manager kept a copy of the output of a doctor that saved its own log" +assert_healthy + echo "== stop" "$MANAGER" stop if docker inspect "$CONTAINER" >/dev/null 2>&1; then diff --git a/tests/integration/windows.ps1 b/tests/integration/windows.ps1 index e40bbbb..4df68d9 100644 --- a/tests/integration/windows.ps1 +++ b/tests/integration/windows.ps1 @@ -170,6 +170,17 @@ try { $null = Invoke-Manager restart Assert-Healthy + Write-Host "== doctor saves its log and archive to the host's log folder" + $output = & (Join-Path $InstallDir 'proxy-agent-manager.ps1') doctor --region us-east-1 *>&1 | Out-String + Write-Host $output + if ($LASTEXITCODE -ne 1) { Fail "expected the doctor's exit code 1, got $LASTEXITCODE" } + if ($output -notmatch 'stub doctor report for options: --region us-east-1') { Fail 'doctor did not print its report' } + if ($output -notmatch 'stub doctor stderr') { Fail "doctor did not show the doctor's stderr" } + if (-not (Test-Path -LiteralPath (Join-Path $InstallDir 'logs\doctor-stub.log'))) { Fail "the doctor's log is not in the host's log folder" } + if (-not (Test-Path -LiteralPath (Join-Path $InstallDir 'logs\doctor-stub.zip'))) { Fail "the doctor's archive is not in the host's log folder" } + if (Get-ChildItem -LiteralPath (Join-Path $InstallDir 'logs') -Filter 'proxy-agent-manager-doctor-*.log') { Fail 'the manager kept a copy of the output of a doctor that saved its own log' } + Assert-Healthy + Write-Host '== stop' $null = Invoke-Manager stop $remaining = docker ps -a --filter "name=$Container" --format '{{.Names}}' diff --git a/tests/pwsh/Manager.Tests.ps1 b/tests/pwsh/Manager.Tests.ps1 index 2058cf6..07b35cc 100644 --- a/tests/pwsh/Manager.Tests.ps1 +++ b/tests/pwsh/Manager.Tests.ps1 @@ -74,7 +74,10 @@ BeforeAll { # Stand-in for the docker CLI. Records every call and keeps one container's # worth of state: LastImage is $null while no container "runs" and - # otherwise holds the image it was started from. + # otherwise holds the image it was started from. A `run` of the doctor (with + # `--doctor`) leaves that state alone, outputs a line and an error, standing + # in for stderr, and exits with DoctorExitCode. If DoctorLogDir is set, it + # also saves a log there, like newer doctors. function docker { $stub = $global:ProxyAgentStub $stub.DockerCalls += ,@($args) @@ -82,6 +85,15 @@ BeforeAll { switch ($args[0]) { 'info' { return $stub.OsType } 'run' { + if ($args -contains '--doctor') { + if ($stub.DoctorReport) { $stub.DoctorReport } else { 'stub doctor report' } + Write-Error 'stub doctor log' + if ($stub.DoctorLogDir) { + Set-Content -LiteralPath (Join-Path $stub.DoctorLogDir 'doctor-stub.log') -Value 'stub doctor log' -Encoding ascii + } + $global:LASTEXITCODE = $stub.DoctorExitCode + return + } $stub.LastImage = $args | Where-Object { $_ -like "*proxy-agent:*" } | Select-Object -First 1 return 'container-id' } @@ -133,6 +145,9 @@ Describe 'proxy-agent-manager.ps1' { OsType = 'linux' LastImage = $null UnhealthyTag = $null + DoctorExitCode = 0 + DoctorReport = $null + DoctorLogDir = $null DockerCalls = @() RegistryUrl = $null RegistryMethod = $null @@ -210,15 +225,16 @@ Describe 'proxy-agent-manager.ps1' { } } - It 'rejects an invalid MEMORY_ALLOCATION_MB on <_> without touching the agent' -ForEach @('restart', 'upgrade') { + It 'rejects an invalid MEMORY_ALLOCATION_MB on <_> without touching the agent' -ForEach @('restart', 'upgrade', 'doctor') { $manager = New-AgentDir $null = Invoke-Manager $manager start Set-Content -LiteralPath (Join-Path $script:AgentDir 'settings.ps1') -Value "`$MEMORY_ALLOCATION_MB = '4G'" -Encoding ascii $version = Get-VersionFile + $callsBefore = $global:ProxyAgentStub.DockerCalls.Count $output = Invoke-Manager $manager $_ $LASTEXITCODE | Should -Be 1 $output | Should -Match 'Invalid \$MEMORY_ALLOCATION_MB' - Get-DockerCall 'stop' | Should -BeNullOrEmpty + $global:ProxyAgentStub.DockerCalls.Count | Should -Be $callsBefore Get-VersionFile | Should -Be $version } @@ -325,6 +341,99 @@ Describe 'proxy-agent-manager.ps1' { $global:ProxyAgentStub.LastImage | Should -BeNullOrEmpty Get-DockerCall 'rm' | Should -Contain 'proxy-agent-abc123' } + + It "runs the doctor in a throwaway container with the agent's config, logs and memory limit" { + $manager = New-AgentDir + $null = Invoke-Manager $manager doctor + $LASTEXITCODE | Should -Be 0 + $runArgs = Get-DockerRunArgs + $runArgs[1] | Should -Be '--rm' + $runArgs | Should -Contain '--memory=5120m' + $runArgs | Should -Contain 'LOG_FOLDER_PATH=/app/logs' + ($runArgs | Where-Object { $_ -like '*/config/config.json:/config/config.json:ro' }) | Should -Not -BeNullOrEmpty + ($runArgs | Where-Object { $_ -like '*/logs:/app/logs' }) | Should -Not -BeNullOrEmpty + $runArgs[2..3] | Should -Be @('--workdir', '/app/logs') + $runArgs[-3..-1] | Should -Be @("$($script:ImageRepo):1.2.9-ubuntu-26.04", '/config/config.json', '--doctor') + } + + It 'passes the doctor its options' { + $manager = New-AgentDir + $null = & $manager doctor --region us-east-1 --tcp-attempts 3 *>&1 + (Get-DockerRunArgs)[-5..-1] | Should -Be @('--doctor', '--region', 'us-east-1', '--tcp-attempts', '3') + } + + It 'consumes PowerShell flags while forwarding doctor options' { + $manager = New-AgentDir + $null = & $manager doctor -Verbose --region us-east-1 --tcp-attempts 3 *>&1 + $LASTEXITCODE | Should -Be 0 + $runArgs = Get-DockerRunArgs + $runArgs | Should -Not -Contain '-Verbose' + $runArgs[-5..-1] | Should -Be @('--doctor', '--region', 'us-east-1', '--tcp-attempts', '3') + } + + It "exits with the doctor's exit code" { + $global:ProxyAgentStub.DoctorExitCode = 1 + $manager = New-AgentDir + $null = Invoke-Manager $manager doctor + $LASTEXITCODE | Should -Be 1 + } + + It "saves the doctor's output, errors included, when the doctor saves no log" { + $manager = New-AgentDir + $output = Invoke-Manager $manager doctor + $LASTEXITCODE | Should -Be 0 + $saved = @(Get-ChildItem -LiteralPath (Join-Path $script:AgentDir 'logs') -Filter 'proxy-agent-manager-doctor-*.log') + $saved.Count | Should -Be 1 + Get-Content -LiteralPath $saved[0].FullName | Should -Be @('stub doctor report', 'stub doctor log') + $output | Should -Match 'stub doctor report' + $output | Should -Match ([regex]::Escape("output was saved to $($saved[0].FullName)")) + } + + It 'preserves fallback reports from repeated doctor runs' { + $manager = New-AgentDir + $global:ProxyAgentStub.DoctorReport = 'first doctor report' + $null = Invoke-Manager $manager doctor + $global:ProxyAgentStub.DoctorReport = 'second doctor report' + $null = Invoke-Manager $manager doctor + $LASTEXITCODE | Should -Be 0 + + $saved = @(Get-ChildItem -LiteralPath (Join-Path $script:AgentDir 'logs') -Filter 'proxy-agent-manager-doctor-*.log') + $saved.Count | Should -Be 2 + $contents = @($saved | ForEach-Object { Get-Content -LiteralPath $_.FullName -Raw }) -join "`n" + $contents | Should -Match 'first doctor report' + $contents | Should -Match 'second doctor report' + } + + It "saves the doctor's output when only an earlier run's doctor log exists" { + $manager = New-AgentDir + $logsDir = Join-Path $script:AgentDir 'logs' + $null = New-Item -ItemType Directory -Force -Path $logsDir + $null = New-Item -ItemType File -Path (Join-Path $logsDir 'doctor-2020-01-01T00-00-00.log') + $null = Invoke-Manager $manager doctor + Get-ChildItem -LiteralPath $logsDir -Filter 'proxy-agent-manager-doctor-*.log' | Should -Not -BeNullOrEmpty + } + + It "keeps only its own log when the doctor saves one beside an earlier run's" { + $manager = New-AgentDir + $logsDir = Join-Path $script:AgentDir 'logs' + $null = New-Item -ItemType Directory -Force -Path $logsDir + $null = New-Item -ItemType File -Path (Join-Path $logsDir 'doctor-2020-01-01T00-00-00.log') + $global:ProxyAgentStub.DoctorLogDir = $logsDir + $output = Invoke-Manager $manager doctor + $LASTEXITCODE | Should -Be 0 + Join-Path $logsDir 'doctor-stub.log' | Should -Exist + Get-ChildItem -LiteralPath $logsDir -Filter 'proxy-agent-manager-doctor-*.log' | Should -BeNullOrEmpty + $output | Should -Match ([regex]::Escape("/app/logs in the container is $logsDir on this host.")) + } + + It 'leaves the running agent alone when running the doctor' { + $manager = New-AgentDir + $null = Invoke-Manager $manager start + $null = Invoke-Manager $manager doctor + $global:ProxyAgentStub.LastImage | Should -Be "$($script:ImageRepo):1.2.9-ubuntu-26.04" + Get-DockerCall 'stop' | Should -BeNullOrEmpty + Get-DockerCall 'rm' | Should -BeNullOrEmpty + } } Context 'with Windows containers' { @@ -458,6 +567,18 @@ Describe 'proxy-agent-manager.ps1' { $decoded = [System.Text.Encoding]::Unicode.GetString([Convert]::FromBase64String($encoded)) $decoded | Should -Match 'HEARTBEAT_EXPIRE_AT' } + + It 'runs the doctor with the Windows image, config folder and log folder' { + $manager = New-AgentDir -Version '1.2.9' + $null = Invoke-Manager $manager doctor + $LASTEXITCODE | Should -Be 0 + $runArgs = Get-DockerRunArgs + ($runArgs | Where-Object { $_ -like '*:C:/config:ro' }) | Should -Not -BeNullOrEmpty + $runArgs | Should -Contain 'LOG_FOLDER_PATH=C:/app/logs' + $runArgs[2..3] | Should -Be @('--workdir', 'C:/app/logs') + $runArgs[-3..-1] | Should -Be @("$($script:ImageRepo):1.2.9-windows-ltsc2022", 'C:/config/config.json', '--doctor') + Get-VersionFile | Should -Be '1.2.9' + } } # The manager applies HTTPS_PROXY/HTTP_PROXY/NO_PROXY to the process-wide diff --git a/tests/stub/heartbeat.ps1 b/tests/stub/heartbeat.ps1 index 0748558..18727e9 100644 --- a/tests/stub/heartbeat.ps1 +++ b/tests/stub/heartbeat.ps1 @@ -1,7 +1,19 @@ # Stub proxy agent: writes the heartbeat file the management script's health # check reads, then idles. It expects to be invoked like the real agent, with # `-i ` pointing at a readable file, and exits otherwise so the -# container never becomes healthy and the test fails. +# container never becomes healthy and the test fails. Invoked like the doctor, +# with ` --doctor [options]`, it instead acts like a newer doctor: +# it saves doctor-stub.log and doctor-stub.zip to $env:LOG_FOLDER_PATH, prints +# a report naming its options and a log line on stderr, and exits 1, as a +# doctor whose checks failed. +if ($args.Count -ge 2 -and $args[1] -eq '--doctor') { + if (-not $env:LOG_FOLDER_PATH) { throw 'LOG_FOLDER_PATH must be set' } + [Console]::Error.WriteLine('stub doctor stderr') + Set-Content -LiteralPath (Join-Path $env:LOG_FOLDER_PATH 'doctor-stub.log') -Value 'stub doctor log' -Encoding ascii + Set-Content -LiteralPath (Join-Path $env:LOG_FOLDER_PATH 'doctor-stub.zip') -Value 'stub doctor archive' -Encoding ascii + Write-Host "stub doctor report for options: $($args[2..($args.Count - 1)] -join ' ')" + exit 1 +} if ($args.Count -lt 2 -or $args[0] -ne '-i') { Write-Error "stub proxy agent: expected '-i ', got: $args" exit 1 diff --git a/tests/stub/heartbeat.sh b/tests/stub/heartbeat.sh index 44dfced..ee2420e 100755 --- a/tests/stub/heartbeat.sh +++ b/tests/stub/heartbeat.sh @@ -3,10 +3,23 @@ # Stub proxy agent: writes the heartbeat file the management script's health # check reads, then idles. It expects to be invoked like the real agent, with # `-i ` pointing at a readable file, and exits otherwise so the -# container never becomes healthy and the test fails. +# container never becomes healthy and the test fails. Invoked like the +# doctor, with ` --doctor [options]`, it instead acts like a +# newer doctor: it saves doctor-stub.log and doctor-stub.zip to +# $LOG_FOLDER_PATH, prints a report naming its options and a log line on +# stderr, and exits 1, as a doctor whose checks failed. # set -eu +if [ "${2:-}" = "--doctor" ]; then + echo "stub doctor stderr" >&2 + echo "stub doctor log" > "${LOG_FOLDER_PATH:?LOG_FOLDER_PATH must be set}/doctor-stub.log" + echo "stub doctor archive" > "$LOG_FOLDER_PATH/doctor-stub.zip" + shift 2 + echo "stub doctor report for options: $*" + exit 1 +fi + if [ "${1:-}" != "-i" ] || [ -z "${2:-}" ]; then echo "stub proxy agent: expected '-i ', got: $*" >&2 exit 1