diff --git a/README.md b/README.md index f02f740..475e16c 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,26 @@ Windows Server hosts use an exact LTSC mapping: Server 2019 selects `ltsc2019`, Linux releases continue to use `proxy-agent:` for backward compatibility and are also published as `proxy-agent:-ubuntu-26.04`. Both tags reference the same Linux multi-architecture image. Windows releases use the explicit `proxy-agent:-windows-ltsc2019`, `proxy-agent:-windows-ltsc2022`, or `proxy-agent:-windows-ltsc2025` tags. +### Installing behind a network proxy + +The installers and the `upgrade` command use a proxy for their web requests: downloading the management script, listing image tags, and fetching the agent configuration from Fivetran. + +On Linux, the shell scripts use `curl`. Set `HTTPS_PROXY` or `https_proxy` for HTTPS URLs, which the scripts use by default. Set lowercase `http_proxy` for HTTP URLs; curl ignores uppercase `HTTP_PROXY`. `NO_PROXY` or `no_proxy` excludes hosts. + +On Windows, the PowerShell scripts use `Invoke-WebRequest` and `Invoke-RestMethod`. They apply `HTTPS_PROXY` (or `HTTP_PROXY` as a fallback) and `NO_PROXY` themselves because Windows PowerShell 5.1 does not read these variables for web requests. The proxy URL may include credentials: `http://user:pass@proxy.example.com:3128`. + +```powershell +$env:HTTPS_PROXY = 'http://proxy.example.com:3128' +$env:NO_PROXY = 'localhost,127.0.0.1,.example.com' +$env:RUNTIME = 'docker'; $env:TOKEN = 'YOUR_AGENT_TOKEN'; & .\install.ps1 +``` + +On Windows, the variables take precedence over the proxy in Internet Options; without them, the scripts use the Internet Options proxy as any PowerShell web request would, and print it when one is in effect. In both cases a proxy that asks for NTLM or Kerberos authentication is answered with the current Windows identity, unless the URL carries its own credentials. + +On Windows, download `install.ps1` first, as shown above, rather than fetching it inline with `Invoke-RestMethod`: the download itself runs before the script can apply the proxy settings. + +These variables configure the installer and management scripts, not Docker. Configure [Docker's daemon proxy](https://docs.docker.com/engine/daemon/proxy/) for image pulls. For the running agent on either platform, set `proxy_host` and `proxy_port` in `config.json` as described in the [Linux](https://fivetran.com/docs/destinations/connection-options/proxy-agent/setup-guide-docker#optionalconfigureproxyagenttouseproxyserver) and [Windows](https://fivetran.com/docs/destinations/connection-options/proxy-agent/setup-guide-docker#optionalconfigureproxyagentonwindowstouseproxyserver) setup guides. Docker client proxy settings containing credentials are stored in the container's environment and can be seen with `docker inspect`. + The installer will: - Check prerequisites - Create the installation directory diff --git a/install.ps1 b/install.ps1 index 1b55549..700b450 100644 --- a/install.ps1 +++ b/install.ps1 @@ -50,10 +50,81 @@ Usage: Options: -InstallDir Installation directory (default: %USERPROFILE%\fivetran-proxy-agent) -WindowsVersion Windows LTSC image variant for Windows containers; detected from the host when omitted + +Environment: + HTTPS_PROXY / HTTP_PROXY Proxy URL for the installer's web requests (http://[user:pass@]host:port) + NO_PROXY Comma-separated hosts that bypass the proxy '@ exit 1 } +# Windows PowerShell 5.1 uses WebRequest.DefaultWebProxy; PowerShell 7 uses +# HttpClient.DefaultProxy. Apply proxy settings to the one used by each version. +# Without environment variables, keep the system proxy and its credentials. +# Docker has separate proxy settings; see the README. +function Set-WebProxyFromEnvironment { + param([Parameter(Mandatory)][Uri]$ProbeUri) + + $proxyUrl = if ($env:HTTPS_PROXY) { $env:HTTPS_PROXY } elseif ($env:HTTP_PROXY) { $env:HTTP_PROXY } else { $null } + if (-not $proxyUrl) { + $systemProxy = if ($PSVersionTable.PSVersion.Major -ge 7) { + [System.Net.Http.HttpClient]::DefaultProxy + } else { + [System.Net.WebRequest]::DefaultWebProxy + } + if ($systemProxy) { + if (-not $systemProxy.Credentials) { + $systemProxy.Credentials = [System.Net.CredentialCache]::DefaultCredentials + } + $effectiveProxy = $systemProxy.GetProxy($ProbeUri) + # Direct connections come back as the probe URI on .NET Framework and as null on .NET Core. + if ($effectiveProxy -and $effectiveProxy -ne $ProbeUri) { + Write-Host "Using system proxy $($effectiveProxy.Scheme)://$($effectiveProxy.Authority)" + } + } + return + } + + try { + $proxyUri = [Uri]$proxyUrl + } catch { + Write-Host "ERROR: Invalid proxy URL in HTTPS_PROXY/HTTP_PROXY" -ForegroundColor Red + exit 1 + } + + # .NET bypass entries are regular expressions matched against the request URL. As with curl, + # each entry matches the host and its subdomains, with or without a leading dot. CIDR ranges aren't supported. + $bypassList = @() + if ($env:NO_PROXY) { + foreach ($entry in ($env:NO_PROXY -split ',')) { + $bypassHost = $entry.Trim().TrimStart('.') + if (-not $bypassHost) { continue } + if ($bypassHost -eq '*') { + $bypassList = @('.*') + break + } + $bypassList += '^https?://([^/]+\.)?' + [regex]::Escape($bypassHost) + '(:\d+)?(/|$)' + } + } + + # Keep credentials out of the proxy address; pass them separately. + $proxyAddress = [Uri]"$($proxyUri.Scheme)://$($proxyUri.Authority)" + $webProxy = New-Object System.Net.WebProxy($proxyAddress, $false, [string[]]$bypassList) + if ($proxyUri.UserInfo) { + $userInfo = $proxyUri.UserInfo.Split(':', 2) + $proxyUser = [Uri]::UnescapeDataString($userInfo[0]) + $proxyPassword = if ($userInfo.Length -gt 1) { [Uri]::UnescapeDataString($userInfo[1]) } else { '' } + $webProxy.Credentials = New-Object System.Net.NetworkCredential($proxyUser, $proxyPassword) + } else { + $webProxy.UseDefaultCredentials = $true + } + [System.Net.WebRequest]::DefaultWebProxy = $webProxy + if ($PSVersionTable.PSVersion.Major -ge 7) { + [System.Net.Http.HttpClient]::DefaultProxy = $webProxy + } + Write-Host "Using proxy $proxyAddress from the environment" +} + # ── Validation ─────────────────────────────────────────────────────────────── function Test-DockerVersion { @@ -269,6 +340,8 @@ if ($env:RUNTIME -ne 'docker') { exit 1 } +Set-WebProxyFromEnvironment -ProbeUri ([Uri]$REGISTRY_TAGS_URL) + $currentPrincipal = [Security.Principal.WindowsPrincipal][Security.Principal.WindowsIdentity]::GetCurrent() if ($currentPrincipal.IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator)) { Write-Warning "Running as Administrator is not recommended. Consider running as a regular user." diff --git a/proxy-agent-manager.ps1 b/proxy-agent-manager.ps1 index 6eb00a8..f068aee 100644 --- a/proxy-agent-manager.ps1 +++ b/proxy-agent-manager.ps1 @@ -71,6 +71,73 @@ function Write-Log { Add-Content -Path $LOGFILE -Value "$timestamp UTC - $Message" -Encoding ascii } +# Windows PowerShell 5.1 uses WebRequest.DefaultWebProxy; PowerShell 7 uses +# HttpClient.DefaultProxy. Apply proxy settings to the one used by each version. +# Without environment variables, keep the system proxy and its credentials. +# Docker has separate proxy settings; see the README. +function Set-WebProxyFromEnvironment { + param([Parameter(Mandatory)][Uri]$ProbeUri) + + $proxyUrl = if ($env:HTTPS_PROXY) { $env:HTTPS_PROXY } elseif ($env:HTTP_PROXY) { $env:HTTP_PROXY } else { $null } + if (-not $proxyUrl) { + $systemProxy = if ($PSVersionTable.PSVersion.Major -ge 7) { + [System.Net.Http.HttpClient]::DefaultProxy + } else { + [System.Net.WebRequest]::DefaultWebProxy + } + if ($systemProxy) { + if (-not $systemProxy.Credentials) { + $systemProxy.Credentials = [System.Net.CredentialCache]::DefaultCredentials + } + $effectiveProxy = $systemProxy.GetProxy($ProbeUri) + # Direct connections come back as the probe URI on .NET Framework and as null on .NET Core. + if ($effectiveProxy -and $effectiveProxy -ne $ProbeUri) { + Write-Log "Using system proxy $($effectiveProxy.Scheme)://$($effectiveProxy.Authority)" + } + } + return + } + + try { + $proxyUri = [Uri]$proxyUrl + } catch { + Write-Host "ERROR: Invalid proxy URL in HTTPS_PROXY/HTTP_PROXY" -ForegroundColor Red + exit 1 + } + + # .NET bypass entries are regular expressions matched against the request URL. As with curl, + # each entry matches the host and its subdomains, with or without a leading dot. CIDR ranges aren't supported. + $bypassList = @() + if ($env:NO_PROXY) { + foreach ($entry in ($env:NO_PROXY -split ',')) { + $bypassHost = $entry.Trim().TrimStart('.') + if (-not $bypassHost) { continue } + if ($bypassHost -eq '*') { + $bypassList = @('.*') + break + } + $bypassList += '^https?://([^/]+\.)?' + [regex]::Escape($bypassHost) + '(:\d+)?(/|$)' + } + } + + # Keep credentials out of the proxy address; pass them separately. + $proxyAddress = [Uri]"$($proxyUri.Scheme)://$($proxyUri.Authority)" + $webProxy = New-Object System.Net.WebProxy($proxyAddress, $false, [string[]]$bypassList) + if ($proxyUri.UserInfo) { + $userInfo = $proxyUri.UserInfo.Split(':', 2) + $proxyUser = [Uri]::UnescapeDataString($userInfo[0]) + $proxyPassword = if ($userInfo.Length -gt 1) { [Uri]::UnescapeDataString($userInfo[1]) } else { '' } + $webProxy.Credentials = New-Object System.Net.NetworkCredential($proxyUser, $proxyPassword) + } else { + $webProxy.UseDefaultCredentials = $true + } + [System.Net.WebRequest]::DefaultWebProxy = $webProxy + if ($PSVersionTable.PSVersion.Major -ge 7) { + [System.Net.Http.HttpClient]::DefaultProxy = $webProxy + } + Write-Log "Using proxy $proxyAddress from the environment" +} + function Get-ContainerOsType { $osTypeOutput = docker info --format '{{.OSType}}' 2>$null if ($LASTEXITCODE -ne 0) { @@ -420,6 +487,7 @@ switch ($Command) { } 'upgrade' { Assert-MemoryAllocation + Set-WebProxyFromEnvironment -ProbeUri ([Uri]"https://$($IMAGE.Split('/')[0])/") Invoke-Upgrade } 'status' { diff --git a/tests/pwsh/Manager.Tests.ps1 b/tests/pwsh/Manager.Tests.ps1 index 005ec0f..da11742 100644 --- a/tests/pwsh/Manager.Tests.ps1 +++ b/tests/pwsh/Manager.Tests.ps1 @@ -117,8 +117,14 @@ AfterAll { Describe 'proxy-agent-manager.ps1' { BeforeEach { - # A developer's environment must not leak into the default-URL assertion. - Remove-Item Env:\PROXY_AGENT_REGISTRY_TAGS_URL -ErrorAction SilentlyContinue + # A developer's environment must not leak into the default-URL or proxy assertions. + Remove-Item Env:\PROXY_AGENT_REGISTRY_TAGS_URL, Env:\HTTPS_PROXY, Env:\HTTP_PROXY, Env:\NO_PROXY -ErrorAction SilentlyContinue + $script:OriginalWebProxy = [System.Net.WebRequest]::DefaultWebProxy + [System.Net.WebRequest]::DefaultWebProxy = New-Object System.Net.WebProxy + if ($PSVersionTable.PSVersion.Major -ge 7) { + $script:OriginalHttpClientProxy = [System.Net.Http.HttpClient]::DefaultProxy + [System.Net.Http.HttpClient]::DefaultProxy = New-Object System.Net.WebProxy + } $global:ProxyAgentStub = @{ OsType = 'linux' LastImage = $null @@ -133,7 +139,11 @@ Describe 'proxy-agent-manager.ps1' { } AfterEach { - Remove-Item Env:\PROXY_AGENT_REGISTRY_TAGS_URL -ErrorAction SilentlyContinue + Remove-Item Env:\PROXY_AGENT_REGISTRY_TAGS_URL, Env:\HTTPS_PROXY, Env:\HTTP_PROXY, Env:\NO_PROXY -ErrorAction SilentlyContinue + [System.Net.WebRequest]::DefaultWebProxy = $script:OriginalWebProxy + if ($PSVersionTable.PSVersion.Major -ge 7) { + [System.Net.Http.HttpClient]::DefaultProxy = $script:OriginalHttpClientProxy + } if ($script:AgentDir -and (Test-Path -LiteralPath $script:AgentDir)) { Remove-Item -LiteralPath $script:AgentDir -Recurse -Force } @@ -360,4 +370,134 @@ Describe 'proxy-agent-manager.ps1' { $decoded | Should -Match 'HEARTBEAT_EXPIRE_AT' } } + + # The manager applies HTTPS_PROXY/HTTP_PROXY/NO_PROXY to the process-wide + # proxy used by the current PowerShell version before the registry lookup. + # The script runs in this process, so the result is observable here. + Context 'behind a network proxy' { + It 'applies HTTPS_PROXY with credentials and NO_PROXY on upgrade' { + $env:HTTPS_PROXY = 'http://us%40er:p%3Ass@proxy.example.com:3128' + $env:NO_PROXY = 'localhost, .corp.example.com' + $manager = New-AgentDir + $output = Invoke-Manager $manager upgrade + $LASTEXITCODE | Should -Be 0 + $output | Should -Match 'Using proxy http://proxy.example.com:3128/' + + $proxy = [System.Net.WebRequest]::DefaultWebProxy + $proxy.Address | Should -Be 'http://proxy.example.com:3128/' + $proxy.Credentials.UserName | Should -Be 'us@er' + $proxy.Credentials.Password | Should -Be 'p:ss' + if ($PSVersionTable.PSVersion.Major -ge 7) { + [object]::ReferenceEquals($proxy, [System.Net.Http.HttpClient]::DefaultProxy) | Should -BeTrue + } + $proxy.IsBypassed([Uri]'https://us-docker.pkg.dev/v2/tags/list') | Should -BeFalse + $proxy.IsBypassed([Uri]'http://localhost:8080/') | Should -BeTrue + $proxy.IsBypassed([Uri]'https://corp.example.com/') | Should -BeTrue + $proxy.IsBypassed([Uri]'https://db.corp.example.com:5432/') | Should -BeTrue + $proxy.IsBypassed([Uri]'https://notcorp.example.com/') | Should -BeFalse + } + + It 'falls back to HTTP_PROXY and uses Windows credentials when the URL has none' { + $env:HTTP_PROXY = 'http://proxy.example.com:8080' + $manager = New-AgentDir + $null = Invoke-Manager $manager upgrade + $LASTEXITCODE | Should -Be 0 + $proxy = [System.Net.WebRequest]::DefaultWebProxy + $proxy.Address | Should -Be 'http://proxy.example.com:8080/' + $proxy.UseDefaultCredentials | Should -BeTrue + } + + It 'bypasses every destination when NO_PROXY is an asterisk' { + $env:HTTPS_PROXY = 'http://proxy.example.com:3128' + $env:NO_PROXY = '*' + $manager = New-AgentDir + $null = Invoke-Manager $manager upgrade + $LASTEXITCODE | Should -Be 0 + $proxy = [System.Net.WebRequest]::DefaultWebProxy + $proxy.IsBypassed([Uri]'https://us-docker.pkg.dev/v2/tags/list') | Should -BeTrue + if ($PSVersionTable.PSVersion.Major -ge 7) { + [object]::ReferenceEquals($proxy, [System.Net.Http.HttpClient]::DefaultProxy) | Should -BeTrue + } + } + + It 'keeps the system proxy, reports it and lets it use Windows credentials when no variables are set' { + # Stands in for an Internet Options proxy, which .NET exposes as the default. + $systemProxy = New-Object System.Net.WebProxy('http://system-proxy.example.com:8080') + [System.Net.WebRequest]::DefaultWebProxy = $systemProxy + if ($PSVersionTable.PSVersion.Major -ge 7) { + [System.Net.Http.HttpClient]::DefaultProxy = $systemProxy + } + $manager = New-AgentDir + $output = Invoke-Manager $manager upgrade + $LASTEXITCODE | Should -Be 0 + [object]::ReferenceEquals($systemProxy, [System.Net.WebRequest]::DefaultWebProxy) | Should -BeTrue + if ($PSVersionTable.PSVersion.Major -ge 7) { + [object]::ReferenceEquals($systemProxy, [System.Net.Http.HttpClient]::DefaultProxy) | Should -BeTrue + } + $systemProxy.Credentials | Should -Be ([System.Net.CredentialCache]::DefaultCredentials) + $output | Should -Match 'Using system proxy http://system-proxy.example.com:8080' + } + + It 'preserves credentials already configured on the system proxy' { + $systemProxy = New-Object System.Net.WebProxy('http://system-proxy.example.com:8080') + $credentials = New-Object System.Net.NetworkCredential('proxy-user', 'proxy-password') + $systemProxy.Credentials = $credentials + [System.Net.WebRequest]::DefaultWebProxy = $systemProxy + if ($PSVersionTable.PSVersion.Major -ge 7) { + [System.Net.Http.HttpClient]::DefaultProxy = $systemProxy + } + $manager = New-AgentDir + $null = Invoke-Manager $manager upgrade + $LASTEXITCODE | Should -Be 0 + [object]::ReferenceEquals($credentials, $systemProxy.Credentials) | Should -BeTrue + } + + It 'does not print credentials embedded in the system proxy URL' { + [System.Net.WebRequest]::DefaultWebProxy = New-Object System.Net.WebProxy('http://user:secret@system-proxy.example.com:8080') + if ($PSVersionTable.PSVersion.Major -ge 7) { + [System.Net.Http.HttpClient]::DefaultProxy = [System.Net.WebRequest]::DefaultWebProxy + } + $manager = New-AgentDir + $output = Invoke-Manager $manager upgrade + $LASTEXITCODE | Should -Be 0 + $output | Should -Match 'Using system proxy http://system-proxy.example.com:8080' + $output | Should -Not -Match 'secret' + } + + It 'reports nothing when the system proxy is a direct connection' { + [System.Net.WebRequest]::DefaultWebProxy = New-Object System.Net.WebProxy + if ($PSVersionTable.PSVersion.Major -ge 7) { + [System.Net.Http.HttpClient]::DefaultProxy = [System.Net.WebRequest]::DefaultWebProxy + } + $manager = New-AgentDir + $output = Invoke-Manager $manager upgrade + $LASTEXITCODE | Should -Be 0 + $output | Should -Not -Match 'Using (system )?proxy' + } + + It 'rejects an invalid proxy URL before contacting the registry' { + $env:HTTPS_PROXY = 'http://user:secret@bad host:' + $manager = New-AgentDir + $output = Invoke-Manager $manager upgrade + $LASTEXITCODE | Should -Be 1 + $output | Should -Match 'Invalid proxy URL' + $output | Should -Not -Match 'secret' + $global:ProxyAgentStub.RegistryUrl | Should -BeNullOrEmpty + } + + It 'shares its proxy handling with install.ps1' { + # The two scripts are downloaded separately, so the function is duplicated + # rather than shared. This keeps the copies from drifting apart; the only + # allowed difference is that the manager logs through Write-Log. + function Get-ProxyFunctionBody { + param([string]$ScriptName) + $tokens = $null; $errors = $null + $ast = [System.Management.Automation.Language.Parser]::ParseFile((Join-Path $script:RepoRoot $ScriptName), [ref]$tokens, [ref]$errors) + $function = $ast.Find({ param($node) $node -is [System.Management.Automation.Language.FunctionDefinitionAst] -and $node.Name -eq 'Set-WebProxyFromEnvironment' }, $false) + $function | Should -Not -BeNullOrEmpty -Because "$ScriptName must define Set-WebProxyFromEnvironment" + return $function.Body.Extent.Text -replace 'Write-Log ', 'Write-Host ' + } + Get-ProxyFunctionBody 'proxy-agent-manager.ps1' | Should -Be (Get-ProxyFunctionBody 'install.ps1') + } + } }