From ac8f1d59974c6c8589c18efaa9b7b9ae12f06ef6 Mon Sep 17 00:00:00 2001 From: Paul Harper Date: Fri, 30 Oct 2026 11:24:11 -0700 Subject: [PATCH 1/6] feat(windows): honor HTTPS_PROXY, HTTP_PROXY and NO_PROXY in PowerShell scripts Windows PowerShell 5.1 ignores the proxy environment variables that curl reads on Linux, so install.ps1 and the manager's upgrade command could not reach GitHub, the image registry or the Fivetran API from a proxied host unless the user set a .NET proxy in the same session. Both scripts now build a System.Net.WebProxy from HTTPS_PROXY (or HTTP_PROXY), including URL credentials, and translate NO_PROXY entries into bypass rules with curl's host-and-subdomains semantics. Co-Authored-By: Claude Fable 5.1 --- README.md | 14 +++++++++++++ install.ps1 | 45 +++++++++++++++++++++++++++++++++++++++++ proxy-agent-manager.ps1 | 39 +++++++++++++++++++++++++++++++++++ 3 files changed, 98 insertions(+) diff --git a/README.md b/README.md index f02f740..921421c 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,20 @@ Windows Server hosts use an exact LTSC mapping: Server 2019 selects `ltsc2019`, Linux releases continue to use `proxy-agent:` for backward compatibility and are also published as `proxy-agent:-ubuntu-26.04`. Both tags reference the same Linux multi-architecture image. Windows releases use the explicit `proxy-agent:-windows-ltsc2019`, `proxy-agent:-windows-ltsc2022`, or `proxy-agent:-windows-ltsc2025` tags. +### Installing behind a network proxy + +Both installers and the `upgrade` command honor `HTTPS_PROXY` (or `HTTP_PROXY`) and `NO_PROXY` for the web requests they make: downloading the management script, listing image tags, and fetching the agent configuration from Fivetran. On Linux, `curl` reads these variables natively. On Windows, Windows PowerShell 5.1 does not, so `install.ps1` and `proxy-agent-manager.ps1` apply them themselves. The URL may include credentials: `http://user:pass@proxy.example.com:3128`. + +```powershell +$env:HTTPS_PROXY = 'http://proxy.example.com:3128' +$env:NO_PROXY = 'localhost,127.0.0.1,.example.com' +$env:RUNTIME = 'docker'; $env:TOKEN = 'YOUR_AGENT_TOKEN'; & .\install.ps1 +``` + +On Windows, download `install.ps1` first, as shown above, rather than fetching it inline with `Invoke-RestMethod`: the download itself runs before the script can apply the proxy settings. + +These variables do not configure Docker. The Docker daemon needs its own proxy settings to pull the agent image, and the agent needs `proxy_host` and `proxy_port` in `config.json` to reach Fivetran through a proxy at runtime. See the [Proxy Agent documentation](https://fivetran.com/docs/destinations/connection-options/proxy-agent) for both. + The installer will: - Check prerequisites - Create the installation directory diff --git a/install.ps1 b/install.ps1 index 1b55549..f73add6 100644 --- a/install.ps1 +++ b/install.ps1 @@ -50,10 +50,53 @@ Usage: Options: -InstallDir Installation directory (default: %USERPROFILE%\fivetran-proxy-agent) -WindowsVersion Windows LTSC image variant for Windows containers; detected from the host when omitted + +Environment: + HTTPS_PROXY / HTTP_PROXY Proxy URL for the installer's web requests (http://[user:pass@]host:port) + NO_PROXY Comma-separated hosts that bypass the proxy '@ exit 1 } +# Windows PowerShell 5.1 ignores HTTP_PROXY/HTTPS_PROXY/NO_PROXY, so apply them to this +# script's web requests (GitHub, the image registry, the Fivetran API). curl honors the +# same variables, so install.sh and install.ps1 behave alike behind a proxy. Docker itself +# reads its own proxy settings; see the README. +function Set-WebProxyFromEnvironment { + $proxyUrl = if ($env:HTTPS_PROXY) { $env:HTTPS_PROXY } elseif ($env:HTTP_PROXY) { $env:HTTP_PROXY } else { $null } + if (-not $proxyUrl) { return } + + try { + $proxyUri = [Uri]$proxyUrl + } catch { + Write-Host "ERROR: Invalid proxy URL in HTTPS_PROXY/HTTP_PROXY: $proxyUrl" -ForegroundColor Red + exit 1 + } + + # .NET bypass entries are regular expressions matched against the request URL. As with curl, + # each entry matches the host and its subdomains, with or without a leading dot. CIDR ranges aren't supported. + $bypassList = @() + if ($env:NO_PROXY) { + foreach ($entry in ($env:NO_PROXY -split ',')) { + $bypassHost = $entry.Trim().TrimStart('.') + if (-not $bypassHost) { continue } + $bypassList += '^https?://([^/]+\.)?' + [regex]::Escape($bypassHost) + '(:\d+)?(/|$)' + } + } + + # Keep credentials out of the proxy address; pass them separately. + $proxyAddress = [Uri]"$($proxyUri.Scheme)://$($proxyUri.Authority)" + $webProxy = New-Object System.Net.WebProxy($proxyAddress, $false, [string[]]$bypassList) + if ($proxyUri.UserInfo) { + $userInfo = $proxyUri.UserInfo.Split(':', 2) + $proxyUser = [Uri]::UnescapeDataString($userInfo[0]) + $proxyPassword = if ($userInfo.Length -gt 1) { [Uri]::UnescapeDataString($userInfo[1]) } else { '' } + $webProxy.Credentials = New-Object System.Net.NetworkCredential($proxyUser, $proxyPassword) + } + [System.Net.WebRequest]::DefaultWebProxy = $webProxy + Write-Host "Using proxy $proxyAddress from the environment" +} + # ── Validation ─────────────────────────────────────────────────────────────── function Test-DockerVersion { @@ -269,6 +312,8 @@ if ($env:RUNTIME -ne 'docker') { exit 1 } +Set-WebProxyFromEnvironment + $currentPrincipal = [Security.Principal.WindowsPrincipal][Security.Principal.WindowsIdentity]::GetCurrent() if ($currentPrincipal.IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator)) { Write-Warning "Running as Administrator is not recommended. Consider running as a regular user." diff --git a/proxy-agent-manager.ps1 b/proxy-agent-manager.ps1 index 6eb00a8..22b7213 100644 --- a/proxy-agent-manager.ps1 +++ b/proxy-agent-manager.ps1 @@ -71,6 +71,44 @@ function Write-Log { Add-Content -Path $LOGFILE -Value "$timestamp UTC - $Message" -Encoding ascii } +# Windows PowerShell 5.1 ignores HTTP_PROXY/HTTPS_PROXY/NO_PROXY, so apply them to this +# script's web requests (the image registry lookup during upgrade). Kept in sync with +# install.ps1. Docker itself reads its own proxy settings; see the README. +function Set-WebProxyFromEnvironment { + $proxyUrl = if ($env:HTTPS_PROXY) { $env:HTTPS_PROXY } elseif ($env:HTTP_PROXY) { $env:HTTP_PROXY } else { $null } + if (-not $proxyUrl) { return } + + try { + $proxyUri = [Uri]$proxyUrl + } catch { + Write-Host "ERROR: Invalid proxy URL in HTTPS_PROXY/HTTP_PROXY: $proxyUrl" -ForegroundColor Red + exit 1 + } + + # .NET bypass entries are regular expressions matched against the request URL. As with curl, + # each entry matches the host and its subdomains, with or without a leading dot. CIDR ranges aren't supported. + $bypassList = @() + if ($env:NO_PROXY) { + foreach ($entry in ($env:NO_PROXY -split ',')) { + $bypassHost = $entry.Trim().TrimStart('.') + if (-not $bypassHost) { continue } + $bypassList += '^https?://([^/]+\.)?' + [regex]::Escape($bypassHost) + '(:\d+)?(/|$)' + } + } + + # Keep credentials out of the proxy address; pass them separately. + $proxyAddress = [Uri]"$($proxyUri.Scheme)://$($proxyUri.Authority)" + $webProxy = New-Object System.Net.WebProxy($proxyAddress, $false, [string[]]$bypassList) + if ($proxyUri.UserInfo) { + $userInfo = $proxyUri.UserInfo.Split(':', 2) + $proxyUser = [Uri]::UnescapeDataString($userInfo[0]) + $proxyPassword = if ($userInfo.Length -gt 1) { [Uri]::UnescapeDataString($userInfo[1]) } else { '' } + $webProxy.Credentials = New-Object System.Net.NetworkCredential($proxyUser, $proxyPassword) + } + [System.Net.WebRequest]::DefaultWebProxy = $webProxy + Write-Log "Using proxy $proxyAddress from the environment" +} + function Get-ContainerOsType { $osTypeOutput = docker info --format '{{.OSType}}' 2>$null if ($LASTEXITCODE -ne 0) { @@ -420,6 +458,7 @@ switch ($Command) { } 'upgrade' { Assert-MemoryAllocation + Set-WebProxyFromEnvironment Invoke-Upgrade } 'status' { From ab0e7c1b39065212daf5d967070fefb546100059 Mon Sep 17 00:00:00 2001 From: Paul Harper Date: Fri, 30 Oct 2026 11:31:43 -0700 Subject: [PATCH 2/6] test(windows): cover proxy environment handling in the manager Exercises HTTPS_PROXY with credentials and NO_PROXY, the HTTP_PROXY fallback, the no-op path, and an invalid proxy URL through the upgrade command, and asserts that install.ps1 carries the same function body so the two copies cannot drift apart. Co-Authored-By: Claude Fable 5.1 --- tests/pwsh/Manager.Tests.ps1 | 73 ++++++++++++++++++++++++++++++++++-- 1 file changed, 70 insertions(+), 3 deletions(-) diff --git a/tests/pwsh/Manager.Tests.ps1 b/tests/pwsh/Manager.Tests.ps1 index 005ec0f..4f8d0e6 100644 --- a/tests/pwsh/Manager.Tests.ps1 +++ b/tests/pwsh/Manager.Tests.ps1 @@ -117,8 +117,9 @@ AfterAll { Describe 'proxy-agent-manager.ps1' { BeforeEach { - # A developer's environment must not leak into the default-URL assertion. - Remove-Item Env:\PROXY_AGENT_REGISTRY_TAGS_URL -ErrorAction SilentlyContinue + # A developer's environment must not leak into the default-URL or proxy assertions. + Remove-Item Env:\PROXY_AGENT_REGISTRY_TAGS_URL, Env:\HTTPS_PROXY, Env:\HTTP_PROXY, Env:\NO_PROXY -ErrorAction SilentlyContinue + $script:OriginalWebProxy = [System.Net.WebRequest]::DefaultWebProxy $global:ProxyAgentStub = @{ OsType = 'linux' LastImage = $null @@ -133,7 +134,8 @@ Describe 'proxy-agent-manager.ps1' { } AfterEach { - Remove-Item Env:\PROXY_AGENT_REGISTRY_TAGS_URL -ErrorAction SilentlyContinue + Remove-Item Env:\PROXY_AGENT_REGISTRY_TAGS_URL, Env:\HTTPS_PROXY, Env:\HTTP_PROXY, Env:\NO_PROXY -ErrorAction SilentlyContinue + [System.Net.WebRequest]::DefaultWebProxy = $script:OriginalWebProxy if ($script:AgentDir -and (Test-Path -LiteralPath $script:AgentDir)) { Remove-Item -LiteralPath $script:AgentDir -Recurse -Force } @@ -360,4 +362,69 @@ Describe 'proxy-agent-manager.ps1' { $decoded | Should -Match 'HEARTBEAT_EXPIRE_AT' } } + + # The manager applies HTTPS_PROXY/HTTP_PROXY/NO_PROXY to the process-wide + # default proxy before the registry lookup, since Windows PowerShell 5.1 + # ignores those variables. The script runs in this process, so the result + # is observable here; AfterEach restores the original default proxy. + Context 'behind a network proxy' { + It 'applies HTTPS_PROXY with credentials and NO_PROXY on upgrade' { + $env:HTTPS_PROXY = 'http://us%40er:p%3Ass@proxy.example.com:3128' + $env:NO_PROXY = 'localhost, .corp.example.com' + $manager = New-AgentDir + $output = Invoke-Manager $manager upgrade + $LASTEXITCODE | Should -Be 0 + $output | Should -Match 'Using proxy http://proxy.example.com:3128/' + + $proxy = [System.Net.WebRequest]::DefaultWebProxy + $proxy.Address | Should -Be 'http://proxy.example.com:3128/' + $proxy.Credentials.UserName | Should -Be 'us@er' + $proxy.Credentials.Password | Should -Be 'p:ss' + $proxy.IsBypassed([Uri]'https://us-docker.pkg.dev/v2/tags/list') | Should -BeFalse + $proxy.IsBypassed([Uri]'http://localhost:8080/') | Should -BeTrue + $proxy.IsBypassed([Uri]'https://corp.example.com/') | Should -BeTrue + $proxy.IsBypassed([Uri]'https://db.corp.example.com:5432/') | Should -BeTrue + $proxy.IsBypassed([Uri]'https://notcorp.example.com/') | Should -BeFalse + } + + It 'falls back to HTTP_PROXY without credentials' { + $env:HTTP_PROXY = 'http://proxy.example.com:8080' + $manager = New-AgentDir + $null = Invoke-Manager $manager upgrade + $LASTEXITCODE | Should -Be 0 + $proxy = [System.Net.WebRequest]::DefaultWebProxy + $proxy.Address | Should -Be 'http://proxy.example.com:8080/' + $proxy.Credentials | Should -BeNullOrEmpty + } + + It 'leaves the default proxy alone when no proxy variables are set' { + $manager = New-AgentDir + $null = Invoke-Manager $manager upgrade + [object]::ReferenceEquals($script:OriginalWebProxy, [System.Net.WebRequest]::DefaultWebProxy) | Should -BeTrue + } + + It 'rejects an invalid proxy URL before contacting the registry' { + $env:HTTPS_PROXY = 'http://bad host:' + $manager = New-AgentDir + $output = Invoke-Manager $manager upgrade + $LASTEXITCODE | Should -Be 1 + $output | Should -Match 'Invalid proxy URL' + $global:ProxyAgentStub.RegistryUrl | Should -BeNullOrEmpty + } + + It 'shares its proxy handling with install.ps1' { + # The two scripts are downloaded separately, so the function is duplicated + # rather than shared. This keeps the copies from drifting apart; the only + # allowed difference is that the manager logs through Write-Log. + function Get-ProxyFunctionBody { + param([string]$ScriptName) + $tokens = $null; $errors = $null + $ast = [System.Management.Automation.Language.Parser]::ParseFile((Join-Path $script:RepoRoot $ScriptName), [ref]$tokens, [ref]$errors) + $function = $ast.Find({ param($node) $node -is [System.Management.Automation.Language.FunctionDefinitionAst] -and $node.Name -eq 'Set-WebProxyFromEnvironment' }, $false) + $function | Should -Not -BeNullOrEmpty -Because "$ScriptName must define Set-WebProxyFromEnvironment" + return $function.Body.Extent.Text -replace 'Write-Log ', 'Write-Host ' + } + Get-ProxyFunctionBody 'proxy-agent-manager.ps1' | Should -Be (Get-ProxyFunctionBody 'install.ps1') + } + } } From f3d1920e81fd186c6675c54451c5b8c3aecd72b6 Mon Sep 17 00:00:00 2001 From: Paul Harper Date: Fri, 30 Oct 2026 11:47:19 -0700 Subject: [PATCH 3/6] feat(windows): use Windows credentials for the proxy and report the system proxy Corporate proxies on Windows commonly require NTLM or Kerberos, which .NET only answers when told to. Both the proxy built from the environment (when its URL carries no credentials) and the Internet Options default now use the current Windows identity. When no proxy variables are set, the scripts also print the system proxy in effect for the registry host, so a stalled install shows what it is going through. Co-Authored-By: Claude Fable 5.1 --- README.md | 2 ++ install.ps1 | 27 ++++++++++++++++++++++----- proxy-agent-manager.ps1 | 26 ++++++++++++++++++++++---- tests/pwsh/Manager.Tests.ps1 | 24 +++++++++++++++++++----- 4 files changed, 65 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 921421c..107cce9 100644 --- a/README.md +++ b/README.md @@ -78,6 +78,8 @@ $env:NO_PROXY = 'localhost,127.0.0.1,.example.com' $env:RUNTIME = 'docker'; $env:TOKEN = 'YOUR_AGENT_TOKEN'; & .\install.ps1 ``` +On Windows, the variables take precedence over the proxy in Internet Options; without them, the scripts use the Internet Options proxy as any PowerShell web request would, and print it when one is in effect. In both cases a proxy that asks for NTLM or Kerberos authentication is answered with the current Windows identity, unless the URL carries its own credentials. + On Windows, download `install.ps1` first, as shown above, rather than fetching it inline with `Invoke-RestMethod`: the download itself runs before the script can apply the proxy settings. These variables do not configure Docker. The Docker daemon needs its own proxy settings to pull the agent image, and the agent needs `proxy_host` and `proxy_port` in `config.json` to reach Fivetran through a proxy at runtime. See the [Proxy Agent documentation](https://fivetran.com/docs/destinations/connection-options/proxy-agent) for both. diff --git a/install.ps1 b/install.ps1 index f73add6..c84f832 100644 --- a/install.ps1 +++ b/install.ps1 @@ -59,12 +59,27 @@ Environment: } # Windows PowerShell 5.1 ignores HTTP_PROXY/HTTPS_PROXY/NO_PROXY, so apply them to this -# script's web requests (GitHub, the image registry, the Fivetran API). curl honors the -# same variables, so install.sh and install.ps1 behave alike behind a proxy. Docker itself -# reads its own proxy settings; see the README. +# script's web requests. curl honors the same variables, so install.sh and install.ps1 +# behave alike behind a proxy. Without them, .NET's default proxy is the user's Internet +# Options proxy, which is kept. Either way the proxy may answer NTLM/Kerberos challenges +# with the current Windows identity, which .NET doesn't do on its own. Docker itself reads +# its own proxy settings; see the README. function Set-WebProxyFromEnvironment { + param([Parameter(Mandatory)][Uri]$ProbeUri) + $proxyUrl = if ($env:HTTPS_PROXY) { $env:HTTPS_PROXY } elseif ($env:HTTP_PROXY) { $env:HTTP_PROXY } else { $null } - if (-not $proxyUrl) { return } + if (-not $proxyUrl) { + $systemProxy = [System.Net.WebRequest]::DefaultWebProxy + if ($systemProxy) { + $systemProxy.Credentials = [System.Net.CredentialCache]::DefaultCredentials + $effectiveProxy = $systemProxy.GetProxy($ProbeUri) + # Direct connections come back as the probe URI on .NET Framework and as null on .NET Core. + if ($effectiveProxy -and $effectiveProxy -ne $ProbeUri) { + Write-Host "Using system proxy $effectiveProxy" + } + } + return + } try { $proxyUri = [Uri]$proxyUrl @@ -92,6 +107,8 @@ function Set-WebProxyFromEnvironment { $proxyUser = [Uri]::UnescapeDataString($userInfo[0]) $proxyPassword = if ($userInfo.Length -gt 1) { [Uri]::UnescapeDataString($userInfo[1]) } else { '' } $webProxy.Credentials = New-Object System.Net.NetworkCredential($proxyUser, $proxyPassword) + } else { + $webProxy.UseDefaultCredentials = $true } [System.Net.WebRequest]::DefaultWebProxy = $webProxy Write-Host "Using proxy $proxyAddress from the environment" @@ -312,7 +329,7 @@ if ($env:RUNTIME -ne 'docker') { exit 1 } -Set-WebProxyFromEnvironment +Set-WebProxyFromEnvironment -ProbeUri ([Uri]$REGISTRY_TAGS_URL) $currentPrincipal = [Security.Principal.WindowsPrincipal][Security.Principal.WindowsIdentity]::GetCurrent() if ($currentPrincipal.IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator)) { diff --git a/proxy-agent-manager.ps1 b/proxy-agent-manager.ps1 index 22b7213..e9f5ae8 100644 --- a/proxy-agent-manager.ps1 +++ b/proxy-agent-manager.ps1 @@ -72,11 +72,27 @@ function Write-Log { } # Windows PowerShell 5.1 ignores HTTP_PROXY/HTTPS_PROXY/NO_PROXY, so apply them to this -# script's web requests (the image registry lookup during upgrade). Kept in sync with -# install.ps1. Docker itself reads its own proxy settings; see the README. +# script's web requests during upgrade. curl honors the same variables, so install.sh and install.ps1 +# behave alike behind a proxy. Without them, .NET's default proxy is the user's Internet +# Options proxy, which is kept. Either way the proxy may answer NTLM/Kerberos challenges +# with the current Windows identity, which .NET doesn't do on its own. Docker itself reads +# its own proxy settings; see the README. function Set-WebProxyFromEnvironment { + param([Parameter(Mandatory)][Uri]$ProbeUri) + $proxyUrl = if ($env:HTTPS_PROXY) { $env:HTTPS_PROXY } elseif ($env:HTTP_PROXY) { $env:HTTP_PROXY } else { $null } - if (-not $proxyUrl) { return } + if (-not $proxyUrl) { + $systemProxy = [System.Net.WebRequest]::DefaultWebProxy + if ($systemProxy) { + $systemProxy.Credentials = [System.Net.CredentialCache]::DefaultCredentials + $effectiveProxy = $systemProxy.GetProxy($ProbeUri) + # Direct connections come back as the probe URI on .NET Framework and as null on .NET Core. + if ($effectiveProxy -and $effectiveProxy -ne $ProbeUri) { + Write-Log "Using system proxy $effectiveProxy" + } + } + return + } try { $proxyUri = [Uri]$proxyUrl @@ -104,6 +120,8 @@ function Set-WebProxyFromEnvironment { $proxyUser = [Uri]::UnescapeDataString($userInfo[0]) $proxyPassword = if ($userInfo.Length -gt 1) { [Uri]::UnescapeDataString($userInfo[1]) } else { '' } $webProxy.Credentials = New-Object System.Net.NetworkCredential($proxyUser, $proxyPassword) + } else { + $webProxy.UseDefaultCredentials = $true } [System.Net.WebRequest]::DefaultWebProxy = $webProxy Write-Log "Using proxy $proxyAddress from the environment" @@ -458,7 +476,7 @@ switch ($Command) { } 'upgrade' { Assert-MemoryAllocation - Set-WebProxyFromEnvironment + Set-WebProxyFromEnvironment -ProbeUri ([Uri]"https://$($IMAGE.Split('/')[0])/") Invoke-Upgrade } 'status' { diff --git a/tests/pwsh/Manager.Tests.ps1 b/tests/pwsh/Manager.Tests.ps1 index 4f8d0e6..cb6a5c6 100644 --- a/tests/pwsh/Manager.Tests.ps1 +++ b/tests/pwsh/Manager.Tests.ps1 @@ -387,20 +387,34 @@ Describe 'proxy-agent-manager.ps1' { $proxy.IsBypassed([Uri]'https://notcorp.example.com/') | Should -BeFalse } - It 'falls back to HTTP_PROXY without credentials' { + It 'falls back to HTTP_PROXY and uses Windows credentials when the URL has none' { $env:HTTP_PROXY = 'http://proxy.example.com:8080' $manager = New-AgentDir $null = Invoke-Manager $manager upgrade $LASTEXITCODE | Should -Be 0 $proxy = [System.Net.WebRequest]::DefaultWebProxy $proxy.Address | Should -Be 'http://proxy.example.com:8080/' - $proxy.Credentials | Should -BeNullOrEmpty + $proxy.UseDefaultCredentials | Should -BeTrue } - It 'leaves the default proxy alone when no proxy variables are set' { + It 'keeps the system proxy, reports it and lets it use Windows credentials when no variables are set' { + # Stands in for an Internet Options proxy, which .NET exposes as the default. + $systemProxy = New-Object System.Net.WebProxy('http://system-proxy.example.com:8080') + [System.Net.WebRequest]::DefaultWebProxy = $systemProxy $manager = New-AgentDir - $null = Invoke-Manager $manager upgrade - [object]::ReferenceEquals($script:OriginalWebProxy, [System.Net.WebRequest]::DefaultWebProxy) | Should -BeTrue + $output = Invoke-Manager $manager upgrade + $LASTEXITCODE | Should -Be 0 + [object]::ReferenceEquals($systemProxy, [System.Net.WebRequest]::DefaultWebProxy) | Should -BeTrue + $systemProxy.Credentials | Should -Be ([System.Net.CredentialCache]::DefaultCredentials) + $output | Should -Match 'Using system proxy http://system-proxy.example.com:8080/' + } + + It 'reports nothing when the system proxy is a direct connection' { + [System.Net.WebRequest]::DefaultWebProxy = New-Object System.Net.WebProxy + $manager = New-AgentDir + $output = Invoke-Manager $manager upgrade + $LASTEXITCODE | Should -Be 0 + $output | Should -Not -Match 'Using (system )?proxy' } It 'rejects an invalid proxy URL before contacting the registry' { From e46e786445425a309a0df484af1d81d272a5402d Mon Sep 17 00:00:00 2001 From: Paul Harper Date: Wed, 4 Nov 2026 11:49:56 -0800 Subject: [PATCH 4/6] fix(proxy): clarify curl variables and redact invalid URLs --- README.md | 6 +++++- install.ps1 | 5 ++--- proxy-agent-manager.ps1 | 5 ++--- tests/pwsh/Manager.Tests.ps1 | 3 ++- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 107cce9..fba1a36 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,11 @@ Linux releases continue to use `proxy-agent:` for backward compatibilit ### Installing behind a network proxy -Both installers and the `upgrade` command honor `HTTPS_PROXY` (or `HTTP_PROXY`) and `NO_PROXY` for the web requests they make: downloading the management script, listing image tags, and fetching the agent configuration from Fivetran. On Linux, `curl` reads these variables natively. On Windows, Windows PowerShell 5.1 does not, so `install.ps1` and `proxy-agent-manager.ps1` apply them themselves. The URL may include credentials: `http://user:pass@proxy.example.com:3128`. +The installers and the `upgrade` command use a proxy for their web requests: downloading the management script, listing image tags, and fetching the agent configuration from Fivetran. + +On Linux, the shell scripts use `curl`. Set `HTTPS_PROXY` or `https_proxy` for HTTPS URLs, which the scripts use by default. Set lowercase `http_proxy` for HTTP URLs; curl ignores uppercase `HTTP_PROXY`. `NO_PROXY` or `no_proxy` excludes hosts. + +On Windows, the PowerShell scripts use `Invoke-WebRequest` and `Invoke-RestMethod`. They apply `HTTPS_PROXY` (or `HTTP_PROXY` as a fallback) and `NO_PROXY` themselves because Windows PowerShell 5.1 does not read these variables for web requests. The proxy URL may include credentials: `http://user:pass@proxy.example.com:3128`. ```powershell $env:HTTPS_PROXY = 'http://proxy.example.com:3128' diff --git a/install.ps1 b/install.ps1 index c84f832..c2f3d2f 100644 --- a/install.ps1 +++ b/install.ps1 @@ -59,8 +59,7 @@ Environment: } # Windows PowerShell 5.1 ignores HTTP_PROXY/HTTPS_PROXY/NO_PROXY, so apply them to this -# script's web requests. curl honors the same variables, so install.sh and install.ps1 -# behave alike behind a proxy. Without them, .NET's default proxy is the user's Internet +# script's web requests. Without them, .NET's default proxy is the user's Internet # Options proxy, which is kept. Either way the proxy may answer NTLM/Kerberos challenges # with the current Windows identity, which .NET doesn't do on its own. Docker itself reads # its own proxy settings; see the README. @@ -84,7 +83,7 @@ function Set-WebProxyFromEnvironment { try { $proxyUri = [Uri]$proxyUrl } catch { - Write-Host "ERROR: Invalid proxy URL in HTTPS_PROXY/HTTP_PROXY: $proxyUrl" -ForegroundColor Red + Write-Host "ERROR: Invalid proxy URL in HTTPS_PROXY/HTTP_PROXY" -ForegroundColor Red exit 1 } diff --git a/proxy-agent-manager.ps1 b/proxy-agent-manager.ps1 index e9f5ae8..6f830b7 100644 --- a/proxy-agent-manager.ps1 +++ b/proxy-agent-manager.ps1 @@ -72,8 +72,7 @@ function Write-Log { } # Windows PowerShell 5.1 ignores HTTP_PROXY/HTTPS_PROXY/NO_PROXY, so apply them to this -# script's web requests during upgrade. curl honors the same variables, so install.sh and install.ps1 -# behave alike behind a proxy. Without them, .NET's default proxy is the user's Internet +# script's web requests during upgrade. Without them, .NET's default proxy is the user's Internet # Options proxy, which is kept. Either way the proxy may answer NTLM/Kerberos challenges # with the current Windows identity, which .NET doesn't do on its own. Docker itself reads # its own proxy settings; see the README. @@ -97,7 +96,7 @@ function Set-WebProxyFromEnvironment { try { $proxyUri = [Uri]$proxyUrl } catch { - Write-Host "ERROR: Invalid proxy URL in HTTPS_PROXY/HTTP_PROXY: $proxyUrl" -ForegroundColor Red + Write-Host "ERROR: Invalid proxy URL in HTTPS_PROXY/HTTP_PROXY" -ForegroundColor Red exit 1 } diff --git a/tests/pwsh/Manager.Tests.ps1 b/tests/pwsh/Manager.Tests.ps1 index cb6a5c6..00f1e13 100644 --- a/tests/pwsh/Manager.Tests.ps1 +++ b/tests/pwsh/Manager.Tests.ps1 @@ -418,11 +418,12 @@ Describe 'proxy-agent-manager.ps1' { } It 'rejects an invalid proxy URL before contacting the registry' { - $env:HTTPS_PROXY = 'http://bad host:' + $env:HTTPS_PROXY = 'http://user:secret@bad host:' $manager = New-AgentDir $output = Invoke-Manager $manager upgrade $LASTEXITCODE | Should -Be 1 $output | Should -Match 'Invalid proxy URL' + $output | Should -Not -Match 'secret' $global:ProxyAgentStub.RegistryUrl | Should -BeNullOrEmpty } From 2f436e0e80dd674c4eebaca9e259bb8f7313b811 Mon Sep 17 00:00:00 2001 From: Paul Harper Date: Wed, 4 Nov 2026 11:59:34 -0800 Subject: [PATCH 5/6] fix(proxy): redact system proxy address in logs --- README.md | 2 +- install.ps1 | 2 +- proxy-agent-manager.ps1 | 2 +- tests/pwsh/Manager.Tests.ps1 | 11 ++++++++++- 4 files changed, 13 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index fba1a36..475e16c 100644 --- a/README.md +++ b/README.md @@ -86,7 +86,7 @@ On Windows, the variables take precedence over the proxy in Internet Options; wi On Windows, download `install.ps1` first, as shown above, rather than fetching it inline with `Invoke-RestMethod`: the download itself runs before the script can apply the proxy settings. -These variables do not configure Docker. The Docker daemon needs its own proxy settings to pull the agent image, and the agent needs `proxy_host` and `proxy_port` in `config.json` to reach Fivetran through a proxy at runtime. See the [Proxy Agent documentation](https://fivetran.com/docs/destinations/connection-options/proxy-agent) for both. +These variables configure the installer and management scripts, not Docker. Configure [Docker's daemon proxy](https://docs.docker.com/engine/daemon/proxy/) for image pulls. For the running agent on either platform, set `proxy_host` and `proxy_port` in `config.json` as described in the [Linux](https://fivetran.com/docs/destinations/connection-options/proxy-agent/setup-guide-docker#optionalconfigureproxyagenttouseproxyserver) and [Windows](https://fivetran.com/docs/destinations/connection-options/proxy-agent/setup-guide-docker#optionalconfigureproxyagentonwindowstouseproxyserver) setup guides. Docker client proxy settings containing credentials are stored in the container's environment and can be seen with `docker inspect`. The installer will: - Check prerequisites diff --git a/install.ps1 b/install.ps1 index c2f3d2f..0ada7fa 100644 --- a/install.ps1 +++ b/install.ps1 @@ -74,7 +74,7 @@ function Set-WebProxyFromEnvironment { $effectiveProxy = $systemProxy.GetProxy($ProbeUri) # Direct connections come back as the probe URI on .NET Framework and as null on .NET Core. if ($effectiveProxy -and $effectiveProxy -ne $ProbeUri) { - Write-Host "Using system proxy $effectiveProxy" + Write-Host "Using system proxy $($effectiveProxy.Scheme)://$($effectiveProxy.Authority)" } } return diff --git a/proxy-agent-manager.ps1 b/proxy-agent-manager.ps1 index 6f830b7..a695e8c 100644 --- a/proxy-agent-manager.ps1 +++ b/proxy-agent-manager.ps1 @@ -87,7 +87,7 @@ function Set-WebProxyFromEnvironment { $effectiveProxy = $systemProxy.GetProxy($ProbeUri) # Direct connections come back as the probe URI on .NET Framework and as null on .NET Core. if ($effectiveProxy -and $effectiveProxy -ne $ProbeUri) { - Write-Log "Using system proxy $effectiveProxy" + Write-Log "Using system proxy $($effectiveProxy.Scheme)://$($effectiveProxy.Authority)" } } return diff --git a/tests/pwsh/Manager.Tests.ps1 b/tests/pwsh/Manager.Tests.ps1 index 00f1e13..e539dcd 100644 --- a/tests/pwsh/Manager.Tests.ps1 +++ b/tests/pwsh/Manager.Tests.ps1 @@ -406,7 +406,16 @@ Describe 'proxy-agent-manager.ps1' { $LASTEXITCODE | Should -Be 0 [object]::ReferenceEquals($systemProxy, [System.Net.WebRequest]::DefaultWebProxy) | Should -BeTrue $systemProxy.Credentials | Should -Be ([System.Net.CredentialCache]::DefaultCredentials) - $output | Should -Match 'Using system proxy http://system-proxy.example.com:8080/' + $output | Should -Match 'Using system proxy http://system-proxy.example.com:8080' + } + + It 'does not print credentials embedded in the system proxy URL' { + [System.Net.WebRequest]::DefaultWebProxy = New-Object System.Net.WebProxy('http://user:secret@system-proxy.example.com:8080') + $manager = New-AgentDir + $output = Invoke-Manager $manager upgrade + $LASTEXITCODE | Should -Be 0 + $output | Should -Match 'Using system proxy http://system-proxy.example.com:8080' + $output | Should -Not -Match 'secret' } It 'reports nothing when the system proxy is a direct connection' { From c426a55cac9b47a84cf5f60e161914fe443c7c7c Mon Sep 17 00:00:00 2001 From: Paul Harper Date: Wed, 4 Nov 2026 12:50:40 -0800 Subject: [PATCH 6/6] Handle system proxy credentials and PowerShell 7 proxy settings --- install.ps1 | 26 ++++++++++++----- proxy-agent-manager.ps1 | 26 ++++++++++++----- tests/pwsh/Manager.Tests.ps1 | 55 ++++++++++++++++++++++++++++++++++-- 3 files changed, 90 insertions(+), 17 deletions(-) diff --git a/install.ps1 b/install.ps1 index 0ada7fa..700b450 100644 --- a/install.ps1 +++ b/install.ps1 @@ -58,19 +58,24 @@ Environment: exit 1 } -# Windows PowerShell 5.1 ignores HTTP_PROXY/HTTPS_PROXY/NO_PROXY, so apply them to this -# script's web requests. Without them, .NET's default proxy is the user's Internet -# Options proxy, which is kept. Either way the proxy may answer NTLM/Kerberos challenges -# with the current Windows identity, which .NET doesn't do on its own. Docker itself reads -# its own proxy settings; see the README. +# Windows PowerShell 5.1 uses WebRequest.DefaultWebProxy; PowerShell 7 uses +# HttpClient.DefaultProxy. Apply proxy settings to the one used by each version. +# Without environment variables, keep the system proxy and its credentials. +# Docker has separate proxy settings; see the README. function Set-WebProxyFromEnvironment { param([Parameter(Mandatory)][Uri]$ProbeUri) $proxyUrl = if ($env:HTTPS_PROXY) { $env:HTTPS_PROXY } elseif ($env:HTTP_PROXY) { $env:HTTP_PROXY } else { $null } if (-not $proxyUrl) { - $systemProxy = [System.Net.WebRequest]::DefaultWebProxy + $systemProxy = if ($PSVersionTable.PSVersion.Major -ge 7) { + [System.Net.Http.HttpClient]::DefaultProxy + } else { + [System.Net.WebRequest]::DefaultWebProxy + } if ($systemProxy) { - $systemProxy.Credentials = [System.Net.CredentialCache]::DefaultCredentials + if (-not $systemProxy.Credentials) { + $systemProxy.Credentials = [System.Net.CredentialCache]::DefaultCredentials + } $effectiveProxy = $systemProxy.GetProxy($ProbeUri) # Direct connections come back as the probe URI on .NET Framework and as null on .NET Core. if ($effectiveProxy -and $effectiveProxy -ne $ProbeUri) { @@ -94,6 +99,10 @@ function Set-WebProxyFromEnvironment { foreach ($entry in ($env:NO_PROXY -split ',')) { $bypassHost = $entry.Trim().TrimStart('.') if (-not $bypassHost) { continue } + if ($bypassHost -eq '*') { + $bypassList = @('.*') + break + } $bypassList += '^https?://([^/]+\.)?' + [regex]::Escape($bypassHost) + '(:\d+)?(/|$)' } } @@ -110,6 +119,9 @@ function Set-WebProxyFromEnvironment { $webProxy.UseDefaultCredentials = $true } [System.Net.WebRequest]::DefaultWebProxy = $webProxy + if ($PSVersionTable.PSVersion.Major -ge 7) { + [System.Net.Http.HttpClient]::DefaultProxy = $webProxy + } Write-Host "Using proxy $proxyAddress from the environment" } diff --git a/proxy-agent-manager.ps1 b/proxy-agent-manager.ps1 index a695e8c..f068aee 100644 --- a/proxy-agent-manager.ps1 +++ b/proxy-agent-manager.ps1 @@ -71,19 +71,24 @@ function Write-Log { Add-Content -Path $LOGFILE -Value "$timestamp UTC - $Message" -Encoding ascii } -# Windows PowerShell 5.1 ignores HTTP_PROXY/HTTPS_PROXY/NO_PROXY, so apply them to this -# script's web requests during upgrade. Without them, .NET's default proxy is the user's Internet -# Options proxy, which is kept. Either way the proxy may answer NTLM/Kerberos challenges -# with the current Windows identity, which .NET doesn't do on its own. Docker itself reads -# its own proxy settings; see the README. +# Windows PowerShell 5.1 uses WebRequest.DefaultWebProxy; PowerShell 7 uses +# HttpClient.DefaultProxy. Apply proxy settings to the one used by each version. +# Without environment variables, keep the system proxy and its credentials. +# Docker has separate proxy settings; see the README. function Set-WebProxyFromEnvironment { param([Parameter(Mandatory)][Uri]$ProbeUri) $proxyUrl = if ($env:HTTPS_PROXY) { $env:HTTPS_PROXY } elseif ($env:HTTP_PROXY) { $env:HTTP_PROXY } else { $null } if (-not $proxyUrl) { - $systemProxy = [System.Net.WebRequest]::DefaultWebProxy + $systemProxy = if ($PSVersionTable.PSVersion.Major -ge 7) { + [System.Net.Http.HttpClient]::DefaultProxy + } else { + [System.Net.WebRequest]::DefaultWebProxy + } if ($systemProxy) { - $systemProxy.Credentials = [System.Net.CredentialCache]::DefaultCredentials + if (-not $systemProxy.Credentials) { + $systemProxy.Credentials = [System.Net.CredentialCache]::DefaultCredentials + } $effectiveProxy = $systemProxy.GetProxy($ProbeUri) # Direct connections come back as the probe URI on .NET Framework and as null on .NET Core. if ($effectiveProxy -and $effectiveProxy -ne $ProbeUri) { @@ -107,6 +112,10 @@ function Set-WebProxyFromEnvironment { foreach ($entry in ($env:NO_PROXY -split ',')) { $bypassHost = $entry.Trim().TrimStart('.') if (-not $bypassHost) { continue } + if ($bypassHost -eq '*') { + $bypassList = @('.*') + break + } $bypassList += '^https?://([^/]+\.)?' + [regex]::Escape($bypassHost) + '(:\d+)?(/|$)' } } @@ -123,6 +132,9 @@ function Set-WebProxyFromEnvironment { $webProxy.UseDefaultCredentials = $true } [System.Net.WebRequest]::DefaultWebProxy = $webProxy + if ($PSVersionTable.PSVersion.Major -ge 7) { + [System.Net.Http.HttpClient]::DefaultProxy = $webProxy + } Write-Log "Using proxy $proxyAddress from the environment" } diff --git a/tests/pwsh/Manager.Tests.ps1 b/tests/pwsh/Manager.Tests.ps1 index e539dcd..da11742 100644 --- a/tests/pwsh/Manager.Tests.ps1 +++ b/tests/pwsh/Manager.Tests.ps1 @@ -120,6 +120,11 @@ Describe 'proxy-agent-manager.ps1' { # A developer's environment must not leak into the default-URL or proxy assertions. Remove-Item Env:\PROXY_AGENT_REGISTRY_TAGS_URL, Env:\HTTPS_PROXY, Env:\HTTP_PROXY, Env:\NO_PROXY -ErrorAction SilentlyContinue $script:OriginalWebProxy = [System.Net.WebRequest]::DefaultWebProxy + [System.Net.WebRequest]::DefaultWebProxy = New-Object System.Net.WebProxy + if ($PSVersionTable.PSVersion.Major -ge 7) { + $script:OriginalHttpClientProxy = [System.Net.Http.HttpClient]::DefaultProxy + [System.Net.Http.HttpClient]::DefaultProxy = New-Object System.Net.WebProxy + } $global:ProxyAgentStub = @{ OsType = 'linux' LastImage = $null @@ -136,6 +141,9 @@ Describe 'proxy-agent-manager.ps1' { AfterEach { Remove-Item Env:\PROXY_AGENT_REGISTRY_TAGS_URL, Env:\HTTPS_PROXY, Env:\HTTP_PROXY, Env:\NO_PROXY -ErrorAction SilentlyContinue [System.Net.WebRequest]::DefaultWebProxy = $script:OriginalWebProxy + if ($PSVersionTable.PSVersion.Major -ge 7) { + [System.Net.Http.HttpClient]::DefaultProxy = $script:OriginalHttpClientProxy + } if ($script:AgentDir -and (Test-Path -LiteralPath $script:AgentDir)) { Remove-Item -LiteralPath $script:AgentDir -Recurse -Force } @@ -364,9 +372,8 @@ Describe 'proxy-agent-manager.ps1' { } # The manager applies HTTPS_PROXY/HTTP_PROXY/NO_PROXY to the process-wide - # default proxy before the registry lookup, since Windows PowerShell 5.1 - # ignores those variables. The script runs in this process, so the result - # is observable here; AfterEach restores the original default proxy. + # proxy used by the current PowerShell version before the registry lookup. + # The script runs in this process, so the result is observable here. Context 'behind a network proxy' { It 'applies HTTPS_PROXY with credentials and NO_PROXY on upgrade' { $env:HTTPS_PROXY = 'http://us%40er:p%3Ass@proxy.example.com:3128' @@ -380,6 +387,9 @@ Describe 'proxy-agent-manager.ps1' { $proxy.Address | Should -Be 'http://proxy.example.com:3128/' $proxy.Credentials.UserName | Should -Be 'us@er' $proxy.Credentials.Password | Should -Be 'p:ss' + if ($PSVersionTable.PSVersion.Major -ge 7) { + [object]::ReferenceEquals($proxy, [System.Net.Http.HttpClient]::DefaultProxy) | Should -BeTrue + } $proxy.IsBypassed([Uri]'https://us-docker.pkg.dev/v2/tags/list') | Should -BeFalse $proxy.IsBypassed([Uri]'http://localhost:8080/') | Should -BeTrue $proxy.IsBypassed([Uri]'https://corp.example.com/') | Should -BeTrue @@ -397,20 +407,56 @@ Describe 'proxy-agent-manager.ps1' { $proxy.UseDefaultCredentials | Should -BeTrue } + It 'bypasses every destination when NO_PROXY is an asterisk' { + $env:HTTPS_PROXY = 'http://proxy.example.com:3128' + $env:NO_PROXY = '*' + $manager = New-AgentDir + $null = Invoke-Manager $manager upgrade + $LASTEXITCODE | Should -Be 0 + $proxy = [System.Net.WebRequest]::DefaultWebProxy + $proxy.IsBypassed([Uri]'https://us-docker.pkg.dev/v2/tags/list') | Should -BeTrue + if ($PSVersionTable.PSVersion.Major -ge 7) { + [object]::ReferenceEquals($proxy, [System.Net.Http.HttpClient]::DefaultProxy) | Should -BeTrue + } + } + It 'keeps the system proxy, reports it and lets it use Windows credentials when no variables are set' { # Stands in for an Internet Options proxy, which .NET exposes as the default. $systemProxy = New-Object System.Net.WebProxy('http://system-proxy.example.com:8080') [System.Net.WebRequest]::DefaultWebProxy = $systemProxy + if ($PSVersionTable.PSVersion.Major -ge 7) { + [System.Net.Http.HttpClient]::DefaultProxy = $systemProxy + } $manager = New-AgentDir $output = Invoke-Manager $manager upgrade $LASTEXITCODE | Should -Be 0 [object]::ReferenceEquals($systemProxy, [System.Net.WebRequest]::DefaultWebProxy) | Should -BeTrue + if ($PSVersionTable.PSVersion.Major -ge 7) { + [object]::ReferenceEquals($systemProxy, [System.Net.Http.HttpClient]::DefaultProxy) | Should -BeTrue + } $systemProxy.Credentials | Should -Be ([System.Net.CredentialCache]::DefaultCredentials) $output | Should -Match 'Using system proxy http://system-proxy.example.com:8080' } + It 'preserves credentials already configured on the system proxy' { + $systemProxy = New-Object System.Net.WebProxy('http://system-proxy.example.com:8080') + $credentials = New-Object System.Net.NetworkCredential('proxy-user', 'proxy-password') + $systemProxy.Credentials = $credentials + [System.Net.WebRequest]::DefaultWebProxy = $systemProxy + if ($PSVersionTable.PSVersion.Major -ge 7) { + [System.Net.Http.HttpClient]::DefaultProxy = $systemProxy + } + $manager = New-AgentDir + $null = Invoke-Manager $manager upgrade + $LASTEXITCODE | Should -Be 0 + [object]::ReferenceEquals($credentials, $systemProxy.Credentials) | Should -BeTrue + } + It 'does not print credentials embedded in the system proxy URL' { [System.Net.WebRequest]::DefaultWebProxy = New-Object System.Net.WebProxy('http://user:secret@system-proxy.example.com:8080') + if ($PSVersionTable.PSVersion.Major -ge 7) { + [System.Net.Http.HttpClient]::DefaultProxy = [System.Net.WebRequest]::DefaultWebProxy + } $manager = New-AgentDir $output = Invoke-Manager $manager upgrade $LASTEXITCODE | Should -Be 0 @@ -420,6 +466,9 @@ Describe 'proxy-agent-manager.ps1' { It 'reports nothing when the system proxy is a direct connection' { [System.Net.WebRequest]::DefaultWebProxy = New-Object System.Net.WebProxy + if ($PSVersionTable.PSVersion.Major -ge 7) { + [System.Net.Http.HttpClient]::DefaultProxy = [System.Net.WebRequest]::DefaultWebProxy + } $manager = New-AgentDir $output = Invoke-Manager $manager upgrade $LASTEXITCODE | Should -Be 0