From c2a61aac2e53f52ca0e997f91f289a3fa5b19f32 Mon Sep 17 00:00:00 2001 From: Clemens Hoffmann Date: Fri, 23 Oct 2026 08:18:06 +0200 Subject: [PATCH 1/5] Implement support for running HAProxy with AWS-LC (FIPS) https://github.com/cloudfoundry/community/blob/main/toc/rfc/rfc-0061-haproxy-awslc.md --- .gitignore | 4 + README.md | 30 ++ acceptance-tests/README.md | 23 ++ .../acceptance_tests_suite_test.go | 26 +- acceptance-tests/bosh_helpers.go | 38 +++ acceptance-tests/config.go | 14 + acceptance-tests/lua_test.go | 4 +- acceptance-tests/run-local.sh | 4 +- acceptance-tests/ssl_variant_test.go | 64 +++++ acceptance-tests/strict_sni_test.go | 4 +- ci/README.md | 4 +- ci/pipeline.yml | 194 ++++++++++++- ci/scripts/autobump-dependencies.py | 224 +++++++++++++-- ci/scripts/functions-ci.sh | 2 + ci/scripts/requirements-test.txt | 2 + ci/scripts/shipit | 77 ++--- ci/scripts/test_autobump_dependencies.py | 270 ++++++++++++++++++ config/blobs.yml | 12 + docs/coredumps.md | 6 +- jobs/haproxy/spec | 16 +- jobs/haproxy/templates/haproxy_wrapper.erb | 27 +- jobs/haproxy/templates/pre-start.erb | 11 +- packages/README.md | 20 ++ packages/aws-lc-fips/packaging | 30 ++ packages/aws-lc-fips/spec | 8 + packages/aws-lc-src/packaging | 8 + packages/aws-lc-src/spec | 5 + packages/aws-lc/packaging | 21 ++ packages/aws-lc/spec | 7 + packages/cmake/packaging | 7 + packages/cmake/spec | 5 + packages/haproxy-awslc-fips-patched/packaging | 5 + packages/haproxy-awslc-fips-patched/spec | 11 + packages/haproxy-awslc-fips/packaging | 5 + packages/haproxy-awslc-fips/spec | 9 + packages/haproxy-awslc-patched/packaging | 5 + packages/haproxy-awslc-patched/spec | 11 + packages/haproxy-awslc/packaging | 5 + packages/haproxy-awslc/spec | 9 + packages/haproxy-deps/packaging | 36 +++ packages/{haproxy => haproxy-deps}/spec | 4 +- packages/haproxy-openssl-patched/packaging | 5 + packages/haproxy-openssl-patched/spec | 10 + packages/haproxy-openssl/packaging | 5 + packages/haproxy-openssl/spec | 8 + packages/haproxy/packaging | 72 ----- scripts/dev-build.sh | 113 ++++++++ .../haproxy/templates/haproxy_wrapper_spec.rb | 45 +++ src/apply-patches.sh | 17 ++ src/aws-lc-versions.sh | 14 + src/build-haproxy.sh | 34 +++ .../disable-rfc-6455-compliance.patch | 0 src/haproxy-versions.sh | 14 + 53 files changed, 1427 insertions(+), 177 deletions(-) create mode 100644 acceptance-tests/ssl_variant_test.go create mode 100644 ci/scripts/requirements-test.txt create mode 100644 ci/scripts/test_autobump_dependencies.py create mode 100644 packages/README.md create mode 100644 packages/aws-lc-fips/packaging create mode 100644 packages/aws-lc-fips/spec create mode 100644 packages/aws-lc-src/packaging create mode 100644 packages/aws-lc-src/spec create mode 100644 packages/aws-lc/packaging create mode 100644 packages/aws-lc/spec create mode 100644 packages/cmake/packaging create mode 100644 packages/cmake/spec create mode 100644 packages/haproxy-awslc-fips-patched/packaging create mode 100644 packages/haproxy-awslc-fips-patched/spec create mode 100644 packages/haproxy-awslc-fips/packaging create mode 100644 packages/haproxy-awslc-fips/spec create mode 100644 packages/haproxy-awslc-patched/packaging create mode 100644 packages/haproxy-awslc-patched/spec create mode 100644 packages/haproxy-awslc/packaging create mode 100644 packages/haproxy-awslc/spec create mode 100644 packages/haproxy-deps/packaging rename packages/{haproxy => haproxy-deps}/spec (74%) create mode 100644 packages/haproxy-openssl-patched/packaging create mode 100644 packages/haproxy-openssl-patched/spec create mode 100644 packages/haproxy-openssl/packaging create mode 100644 packages/haproxy-openssl/spec delete mode 100644 packages/haproxy/packaging create mode 100755 scripts/dev-build.sh create mode 100644 spec/haproxy/templates/haproxy_wrapper_spec.rb create mode 100644 src/apply-patches.sh create mode 100644 src/aws-lc-versions.sh create mode 100644 src/build-haproxy.sh rename {haproxy-patches => src/haproxy-patches}/disable-rfc-6455-compliance.patch (100%) create mode 100644 src/haproxy-versions.sh diff --git a/.gitignore b/.gitignore index 881ead696..3fcc0b20d 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ releases/**/*.tgz ci/scripts/stemcell/*.tgz ci/scripts/stemcell-jammy/*.tgz dev_releases +builds blobs/* .blobs .dev_builds @@ -21,6 +22,9 @@ blobs/* tmp my*.yml .bundle +.pytest_cache +__pycache__ +*.pyc vendor .vscode/launch.json .vscode/settings.json diff --git a/README.md b/README.md index 4da0389ab..445d98f4b 100644 --- a/README.md +++ b/README.md @@ -81,3 +81,33 @@ Unit/RSpec tests can also be debugged/stepped through when needed. See for examp ### Acceptance Tests See [acceptance-tests README](/acceptance-tests/README.md). + +### Local Dev Builds + +`scripts/dev-build.sh` builds the release tarball and uploads it to the targeted BOSH +director. Useful for iterating on packaging changes without going through CI. + +```bash +./scripts/dev-build.sh # build, version=dev +./scripts/dev-build.sh --version 1.0 # custom version tag +./scripts/dev-build.sh --upload-only # re-upload previously built tarballs +``` + +Tarballs are written to `./builds/` and uploaded with `bosh upload-release --fix`. + +### Packaging model + +`packages/` holds **every** package, and `bosh create-release` bundles all of them, so a plain +`bosh create-release` of the committed tree produces the one release — there is no build-time +assembly. The `haproxy` job lists all six `haproxy-*` binaries; the SSL backend is chosen at deploy +time via `ha_proxy.ssl_variant` (default `openssl`) and `ha_proxy.include_haproxy_patches` +(default `false`), handled in `jobs/haproxy/templates/haproxy_wrapper.erb`. + +Dependencies are one-directional: every `haproxy-*` binary depends on the shared `haproxy-deps` +(lua, pcre2, socat, hatop), compiled once. AWS-LC is compiled from a single shared `aws-lc-src` source +package used by both the non-FIPS `aws-lc` and the FIPS `aws-lc-fips` compile packages (FIPS differs +only by `-DFIPS=1` plus the Go toolchain for the integrity step). Because `aws-lc`/`aws-lc-fips` do not +bundle the HAProxy version file, their fingerprints are independent of HAProxy — a HAProxy bump reuses +the cached AWS-LC compile instead of rebuilding it. The HAProxy community patches (the optional +`-patched` builds) are plain source files under `src/haproxy-patches/`, applied at compile time by +`src/apply-patches.sh`. \ No newline at end of file diff --git a/acceptance-tests/README.md b/acceptance-tests/README.md index c8ff20c3c..aceff1ae8 100644 --- a/acceptance-tests/README.md +++ b/acceptance-tests/README.md @@ -12,6 +12,29 @@ cd acceptance-tests ./run-local.sh ``` +### TLS Backend Variant + +There is a single release that bundles every SSL variant. By default, the tests deploy it with the +system OpenSSL backend. To exercise a different TLS backend, set the `VARIANT` environment variable — +it drives the `ha_proxy.ssl_variant` deployment property. The HAProxy patches are toggled with +`INCLUDE_HAPROXY_PATCHES` (which drives `ha_proxy.include_haproxy_patches`). Examples: + +```shell +VARIANT=awslc ./run-local.sh # AWS-LC, non-FIPS +VARIANT=awslc-fips ./run-local.sh # AWS-LC FIPS +INCLUDE_HAPROXY_PATCHES=true ./run-local.sh # system OpenSSL + HAProxy patches +VARIANT=awslc INCLUDE_HAPROXY_PATCHES=true ./run-local.sh # AWS-LC + patches +VARIANT=awslc-fips INCLUDE_HAPROXY_PATCHES=true ./run-local.sh # AWS-LC FIPS + patches +``` + +`VARIANT` accepts `openssl` (default), `awslc`, or `awslc-fips`. + +A single invocation runs the suite against the chosen variant. + +Each run builds the release from scratch, compiling every bundled binary (AWS-LC and its FIPS toolchain included) inside the BOSH compilation VM. + +When using `-k` (keep BOSH running, see below) the cached state belongs to whichever variant ran last — switching `VARIANT` inside a kept container is not supported. Stop the container between variants, or run without `-k`. + ### Running on Docker for Mac Acceptance tests cannot be run on Mac with arm64 architecture: diff --git a/acceptance-tests/acceptance_tests_suite_test.go b/acceptance-tests/acceptance_tests_suite_test.go index ab937416b..fd285ce0a 100644 --- a/acceptance-tests/acceptance_tests_suite_test.go +++ b/acceptance-tests/acceptance_tests_suite_test.go @@ -226,8 +226,21 @@ func expectTLSCertificateRequiredErr(err error) { checkNetOpErr(err, "tls: certificate required") } -func expectTLSUnrecognizedNameErr(err error) { - checkNetOpErr(err, "tls: unrecognized name") +// expectStrictSNIRejectedErr asserts that a strict-sni handshake was rejected. +// - OpenSSL registers the SNI reject through the servername callback and sends +// unrecognized_name(112) => "tls: unrecognized name" +// - AWS-LC uses the select_certificate_cb path, which sends +// handshake_failure(40) => "tls: handshake failure" +func expectStrictSNIRejectedErr(err error) { + msg := netOpErrString(err) + if strings.HasPrefix(config.Variant, "awslc") { + Expect(msg).To(Or( + ContainSubstring("tls: handshake failure"), + ContainSubstring("tls: unrecognized name"), + )) + return + } + Expect(msg).To(ContainSubstring("tls: unrecognized name")) } func expectConnectionRefusedErr(err error) { @@ -239,11 +252,18 @@ func expectConnectionResetErr(err error) { } func checkNetOpErr(err error, expectString string) { + Expect(netOpErrString(err)).To(ContainSubstring(expectString)) +} + +// netOpErrString unwraps an http client error down to the underlying +// *net.OpError and returns its message, failing the test if the error is not +// shaped that way. +func netOpErrString(err error) string { Expect(err).To(HaveOccurred()) urlErr, ok := err.(*url.Error) Expect(ok).To(BeTrue()) tlsErr := urlErr.Unwrap() var opErr *net.OpError Expect(errors.As(tlsErr, &opErr)).To(BeTrue()) - Expect(opErr.Err.Error()).To(ContainSubstring(expectString)) + return opErr.Err.Error() } diff --git a/acceptance-tests/bosh_helpers.go b/acceptance-tests/bosh_helpers.go index 415ee0e4a..afaefa660 100644 --- a/acceptance-tests/bosh_helpers.go +++ b/acceptance-tests/bosh_helpers.go @@ -93,6 +93,21 @@ var opsfileConfigureSSH string = `--- var defaultOpsfiles = []string{opsfileChangeName, opsfileChangeVersion, opsfileConfigureSSH} var defaultSSHUser string = "ginkgo" +// The release bundles every SSL variant; this selects one at deploy time. +var opsfileSSLVariant string = `--- +# Select the SSL variant at deploy time (the release bundles all of them) +- type: replace + path: /instance_groups/name=haproxy/jobs/name=haproxy/properties/ha_proxy/ssl_variant? + value: ((ssl-variant)) +` + +// Orthogonal to the SSL variant: run the patched HAProxy build. +var opsfileIncludePatches string = `--- +- type: replace + path: /instance_groups/name=haproxy/jobs/name=haproxy/properties/ha_proxy/include_haproxy_patches? + value: ((include-haproxy-patches)) +` + func buildManifestVars(baseManifestVars baseManifestVars, customVars map[string]interface{}) map[string]interface{} { vars := map[string]interface{}{ "release-version": config.ReleaseVersion, @@ -146,6 +161,21 @@ func buildHAProxyInfo(baseManifestVars baseManifestVars, varsStoreReader varsSto func deployHAProxy(baseManifestVars baseManifestVars, customOpsfiles []string, customVars map[string]interface{}, expectSuccess bool) (haproxyInfo, varsStoreReader) { manifestVars := buildManifestVars(baseManifestVars, customVars) opsfiles := append(defaultOpsfiles, customOpsfiles...) + + // The release bundles every SSL variant; select the one under test at deploy time. + // config.Variant comes from the VARIANT env (default "openssl"); openssl is the job's + // own default, so we only inject the property for the non-default variants. + if config.Variant != "" && config.Variant != "openssl" { + opsfiles = append(opsfiles, opsfileSSLVariant) + manifestVars["ssl-variant"] = config.Variant + } + + // INCLUDE_HAPROXY_PATCHES selects the patched build (orthogonal to the SSL variant). + if config.IncludePatches { + opsfiles = append(opsfiles, opsfileIncludePatches) + manifestVars["include-haproxy-patches"] = config.IncludePatches + } + cmd, varsStoreReader := deployBaseManifestCmd(baseManifestVars.deploymentName, opsfiles, manifestVars) dumpCmd(cmd) @@ -332,3 +362,11 @@ func runHAProxySocketCommand(haproxyInfo haproxyInfo, command string) string { return strings.TrimSpace(stdout) } +// runningHAProxyVersionInfo returns `haproxy -vv` for the HAProxy binary that is +// actually running on the VM. +func runningHAProxyVersionInfo(haproxyInfo haproxyInfo) string { + cmd := `sudo sh -c 'exe=$(readlink -f /proc/$(pgrep -x -n haproxy)/exe); "$exe" -vv'` + stdout, _, err := runOnRemote(haproxyInfo.SSHUser, haproxyInfo.PublicIP, haproxyInfo.SSHPrivateKey, cmd) + Expect(err).NotTo(HaveOccurred()) + return strings.TrimSpace(stdout) +} diff --git a/acceptance-tests/config.go b/acceptance-tests/config.go index 5d3c6bce3..55809ad71 100644 --- a/acceptance-tests/config.go +++ b/acceptance-tests/config.go @@ -4,6 +4,7 @@ import ( "fmt" "os" "os/exec" + "strconv" ) var config Config @@ -18,6 +19,8 @@ type Config struct { BoshPath string `json:"boshPath"` BaseManifestPath string `json:"baseManifestPath"` HomePath string `json:"homePath"` + Variant string `json:"variant"` + IncludePatches bool `json:"includePatches"` } func loadConfig() (Config, error) { @@ -67,6 +70,15 @@ func loadConfig() (Config, error) { return Config{}, err } + // VARIANT selects the SSL library (openssl|awslc|awslc-fips); empty means openssl. + variant := os.Getenv("VARIANT") + if variant == "" { + variant = "openssl" + } + + // INCLUDE_HAPROXY_PATCHES toggles the patched build (orthogonal to VARIANT). + includePatches, _ := strconv.ParseBool(os.Getenv("INCLUDE_HAPROXY_PATCHES")) + return Config{ ReleaseRepoPath: releaseRepoPath, ReleaseVersion: releaseVersion, @@ -77,6 +89,8 @@ func loadConfig() (Config, error) { BoshPath: boshPath, BaseManifestPath: baseManifestPath, HomePath: homePath, + Variant: variant, + IncludePatches: includePatches, }, nil } diff --git a/acceptance-tests/lua_test.go b/acceptance-tests/lua_test.go index e690b41e2..c6deed0cf 100644 --- a/acceptance-tests/lua_test.go +++ b/acceptance-tests/lua_test.go @@ -16,7 +16,7 @@ var _ = Describe("Lua scripting", func() { # Enable Lua scripting - type: replace path: /instance_groups/name=haproxy/jobs/name=haproxy/properties/ha_proxy/lua_scripts?/- - value: "/var/vcap/packages/haproxy/lua_test.lua" + value: "/var/vcap/packages/haproxy-deps/lua_test.lua" - type: replace path: /instance_groups/name=haproxy/jobs/name=haproxy/properties/ha_proxy/frontend_config? value: "http-request use-service lua.lua_test if { path /lua_test }" @@ -53,7 +53,7 @@ core.register_service("lua_test", "http", lua_test) Expect(boshInstances(deploymentNameForTestNode())[0].ProcessState).To(Or(Equal("failing"), Equal("unresponsive agent"))) // upload Lua script file - uploadFile(haproxyInfo, strings.NewReader(replyLuaContent), "/var/vcap/packages/haproxy/lua_test.lua") + uploadFile(haproxyInfo, strings.NewReader(replyLuaContent), "/var/vcap/packages/haproxy-deps/lua_test.lua") closeLocalServer, localPort := startDefaultTestServer() defer closeLocalServer() diff --git a/acceptance-tests/run-local.sh b/acceptance-tests/run-local.sh index 1cb4b70aa..4c4c3fc65 100755 --- a/acceptance-tests/run-local.sh +++ b/acceptance-tests/run-local.sh @@ -94,9 +94,9 @@ if [ -n "$KEEP_RUNNING" ] ; then echo echo "*** KEEP_RUNNING enabled. Please clean up docker scratch after removing containers: ${DOCKER_SCRATCH}" echo - docker run --privileged -v "$REPO_DIR":/repo -v "${DOCKER_SCRATCH}":/scratch/docker -e REPO_ROOT=/repo -e FOCUS="${FOCUS}" -e PARALLELISM="${PARALLELISM}" -e KEEP_RUNNING="${KEEP_RUNNING}" haproxy-boshrelease-testflight bash -c "cd /repo/ci/scripts && ./acceptance-tests ; sleep infinity" + docker run --privileged -v "$REPO_DIR":/repo -v "${DOCKER_SCRATCH}":/scratch/docker -e REPO_ROOT=/repo -e FOCUS="${FOCUS}" -e PARALLELISM="${PARALLELISM}" -e KEEP_RUNNING="${KEEP_RUNNING}" -e VARIANT="${VARIANT:-}" haproxy-boshrelease-testflight bash -c "cd /repo/ci/scripts && ./acceptance-tests ; sleep infinity" else - docker run --rm --privileged -v "$REPO_DIR":/repo -v "${DOCKER_SCRATCH}":/scratch/docker -e REPO_ROOT=/repo -e KEEP_RUNNING="" -e PARALLELISM="${PARALLELISM}" haproxy-boshrelease-testflight bash -c "cd /repo/ci/scripts && ./acceptance-tests" + docker run --rm --privileged -v "$REPO_DIR":/repo -v "${DOCKER_SCRATCH}":/scratch/docker -e REPO_ROOT=/repo -e KEEP_RUNNING="" -e PARALLELISM="${PARALLELISM}" -e VARIANT="${VARIANT:-}" haproxy-boshrelease-testflight bash -c "cd /repo/ci/scripts && ./acceptance-tests" echo "Cleaning up docker scratch: ${DOCKER_SCRATCH}" sudo rm -rf "${DOCKER_SCRATCH}" fi diff --git a/acceptance-tests/ssl_variant_test.go b/acceptance-tests/ssl_variant_test.go new file mode 100644 index 000000000..be8a1500b --- /dev/null +++ b/acceptance-tests/ssl_variant_test.go @@ -0,0 +1,64 @@ +package acceptance_tests + +import ( + "fmt" + "regexp" + "strings" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// Verifies that the SSL backend actually running on the VM matches the variant +// the release was assembled with (config.Variant, from the VARIANT env var). +// +// The signal is the running binary's `haproxy -vv`, which reports two dedicated +// lines describing the library loaded at runtime (verified against real output +// from AWS-LC 4.0.0 / OpenSSL 3.0.13): +// +// variant "Running on SSL library version : ..." "SSL library FIPS mode : ..." +// openssl OpenSSL 3.0.13 30 Jan 2024 (line absent) +// awslc AWS-LC 4.0.0 no +// awslc-fips AWS-LC FIPS 4.0.0 yes +var ( + runningSSLLibRe = regexp.MustCompile(`Running on SSL library version\s*:\s*(.+)`) + sslFIPSModeOnRe = regexp.MustCompile(`SSL library FIPS mode\s*:\s*yes`) +) + +var _ = Describe("SSL variant", func() { + It("Runs the HAProxy binary for the configured SSL variant", func() { + haproxyInfo, _ := deployHAProxy(baseManifestVars{ + haproxyBackendPort: 12000, + haproxyBackendServers: []string{"127.0.0.1"}, + deploymentName: deploymentNameForTestNode(), + }, []string{}, map[string]interface{}{}, true) + + By("Reading the SSL library from the running HAProxy binary (haproxy -vv)") + versionInfo := runningHAProxyVersionInfo(haproxyInfo) + fmt.Fprintf(GinkgoWriter, "----- haproxy -vv (variant %q) -----\n%s\n", config.Variant, versionInfo) + + match := runningSSLLibRe.FindStringSubmatch(versionInfo) + Expect(match).NotTo(BeNil(), "no 'Running on SSL library version' line in haproxy -vv:\n%s", versionInfo) + runningSSL := strings.TrimSpace(match[1]) + + isAWSLC := strings.Contains(runningSSL, "AWS-LC") + fipsMode := sslFIPSModeOnRe.MatchString(versionInfo) + + switch { + case strings.HasPrefix(config.Variant, "awslc-fips"): + By("Expecting AWS-LC FIPS") + Expect(isAWSLC).To(BeTrue(), "expected AWS-LC, running SSL library was %q", runningSSL) + Expect(fipsMode).To(BeTrue(), "expected FIPS mode on; haproxy -vv was:\n%s", versionInfo) + case strings.HasPrefix(config.Variant, "awslc"): + By("Expecting AWS-LC (non-FIPS)") + Expect(isAWSLC).To(BeTrue(), "expected AWS-LC, running SSL library was %q", runningSSL) + Expect(fipsMode).To(BeFalse(), "expected FIPS mode off; haproxy -vv was:\n%s", versionInfo) + default: + // openssl (VARIANT unset/openssl); patches are an orthogonal axis and don't change the library. + By("Expecting system OpenSSL") + Expect(runningSSL).To(ContainSubstring("OpenSSL"), "expected OpenSSL, running SSL library was %q", runningSSL) + Expect(isAWSLC).To(BeFalse(), "expected OpenSSL but found AWS-LC, running SSL library was %q", runningSSL) + Expect(fipsMode).To(BeFalse(), "expected FIPS mode off; haproxy -vv was:\n%s", versionInfo) + } + }) +}) diff --git a/acceptance-tests/strict_sni_test.go b/acceptance-tests/strict_sni_test.go index 420814e84..afabbc755 100644 --- a/acceptance-tests/strict_sni_test.go +++ b/acceptance-tests/strict_sni_test.go @@ -129,11 +129,11 @@ var _ = Describe("Strict SNI", func() { By("Sending a request to HAProxy with no SNI is forbidden") _, err := httpClientNoSNI.Get("https://haproxy.internal") - expectTLSUnrecognizedNameErr(err) + expectStrictSNIRejectedErr(err) By("Sending a request to HAProxy with an incorrect SNI is forbidden") _, err = httpClientWrongSNI.Get("https://haproxy.internal") - expectTLSUnrecognizedNameErr(err) + expectStrictSNIRejectedErr(err) }) }) }) diff --git a/ci/README.md b/ci/README.md index a20983e1f..9a37e245d 100644 --- a/ci/README.md +++ b/ci/README.md @@ -135,6 +135,6 @@ For creating a new release please follow the versioning guide based on the [Sema Used for documentation updates, changes in the test suite or any updates in the testing frameworks (e.g. ginkgo). -Since releases `11.16.3` and `11.17.5` the build metadata has been included into the version number. The build metadata denotes the contained HAProxy version. As an example, `11.16.2+2.6.9` means that HAProxy 2.6.9 is used. + Since releases `11.16.3` and `11.17.5` the build metadata (everything after the `+`) has been included in the version number to denote the contained upstream versions. Since version 17.0.0, it records both the HAProxy and the AWS-LC version: `17.0.0+3.2.26-awslc-4.0.0` means HAProxy 3.2.26 and AWS-LC 4.0.0. -The `haproxy-boshrelease` also contains patches (see [haproxy-patches](../haproxy-patches)). The patched version is a part of the build metadata and is denoted by appending a hyphen and key word `patched`, like `11.17.4+2.6.9-patched`. + The release bundles every SSL variant and the HAProxy community patches (see [haproxy-patches](../src/haproxy-patches)). One build is selected at deploy time via the `ha_proxy.ssl_variant` (`openssl` | `awslc` | `awslc-fips`) and `ha_proxy.include_haproxy_patches` (`true` | `false`) job properties. diff --git a/ci/pipeline.yml b/ci/pipeline.yml index 42ebab1f8..20eb54e98 100644 --- a/ci/pipeline.yml +++ b/ci/pipeline.yml @@ -6,7 +6,11 @@ groups: - unit-tests - unit-tests-pr - acceptance-tests + - acceptance-tests-awslc + - acceptance-tests-awslc-fips - acceptance-tests-pr + - acceptance-tests-awslc-pr + - acceptance-tests-awslc-fips-pr - rc - shipit - patch @@ -151,6 +155,76 @@ jobs: icon_url: "((slack.icon))" text: "((slack.fail_url)) haproxy-boshrelease : acceptance tests failed" + - name: acceptance-tests-awslc + public: true + serial: true + serial_groups: [acceptance-tests] + plan: + - do: + - in_parallel: + - { get: git, trigger: true, passed: [acceptance-tests] } + - { get: stemcell } + - { get: stemcell-jammy } + - { get: haproxy-boshrelease-testflight, trigger: true, passed: [acceptance-tests] } + - task: acceptance-tests-awslc + privileged: true + timeout: 4h + image: haproxy-boshrelease-testflight + config: + platform: linux + inputs: + - { name: git } + - { name: stemcell } + - { name: stemcell-jammy } + run: + path: ./git/ci/scripts/acceptance-tests + args: [] + params: + REPO_ROOT: git + VARIANT: awslc + on_failure: + put: notify + params: + channel: "#haproxy-boshrelease" + username: ci-bot + icon_url: "((slack.icon))" + text: "((slack.fail_url)) haproxy-boshrelease : acceptance tests (AWS-LC) failed" + + - name: acceptance-tests-awslc-fips + public: true + serial: true + serial_groups: [acceptance-tests] + plan: + - do: + - in_parallel: + - { get: git, trigger: true, passed: [acceptance-tests-awslc] } + - { get: stemcell } + - { get: stemcell-jammy } + - { get: haproxy-boshrelease-testflight, trigger: true, passed: [acceptance-tests-awslc] } + - task: acceptance-tests-awslc-fips + privileged: true + timeout: 4h + image: haproxy-boshrelease-testflight + config: + platform: linux + inputs: + - { name: git } + - { name: stemcell } + - { name: stemcell-jammy } + run: + path: ./git/ci/scripts/acceptance-tests + args: [] + params: + REPO_ROOT: git + VARIANT: awslc-fips + on_failure: + put: notify + params: + channel: "#haproxy-boshrelease" + username: ci-bot + icon_url: "((slack.icon))" + text: "((slack.fail_url)) haproxy-boshrelease : acceptance tests (AWS-LC FIPS) failed" + - name: acceptance-tests-pr public: true serial: true @@ -197,12 +271,106 @@ jobs: status: failure context: acceptance-tests + - name: acceptance-tests-awslc-pr + public: true + serial: true + serial_groups: [acceptance-tests] + plan: + - do: + - in_parallel: + - { get: git-pull-requests-awslc, trigger: true, version: every } + - { get: stemcell } + - { get: stemcell-jammy } + - { get: haproxy-boshrelease-testflight-pr, passed: [acceptance-tests-pr] } + - put: git-pull-requests-awslc + params: + path: git-pull-requests-awslc + status: pending + context: acceptance-tests-awslc + get_params: + list_changed_files: true + - task: acceptance-tests-awslc + privileged: true + timeout: 4h + image: haproxy-boshrelease-testflight-pr + config: + platform: linux + inputs: + - { name: git-pull-requests-awslc } + - { name: stemcell } + - { name: stemcell-jammy } + run: + path: ./git-pull-requests-awslc/ci/scripts/acceptance-tests + args: [] + params: + REPO_ROOT: git-pull-requests-awslc + VARIANT: awslc + on_success: + put: git-pull-requests-awslc + params: + path: git-pull-requests-awslc + status: success + context: acceptance-tests-awslc + on_failure: + put: git-pull-requests-awslc + params: + path: git-pull-requests-awslc + status: failure + context: acceptance-tests-awslc + + - name: acceptance-tests-awslc-fips-pr + public: true + serial: true + serial_groups: [acceptance-tests] + plan: + - do: + - in_parallel: + - { get: git-pull-requests-awslc, trigger: true, version: every } + - { get: stemcell } + - { get: stemcell-jammy } + - { get: haproxy-boshrelease-testflight-pr, passed: [acceptance-tests-awslc-pr] } + - put: git-pull-requests-awslc + params: + path: git-pull-requests-awslc + status: pending + context: acceptance-tests-awslc-fips + get_params: + list_changed_files: true + - task: acceptance-tests-awslc-fips + privileged: true + timeout: 4h + image: haproxy-boshrelease-testflight-pr + config: + platform: linux + inputs: + - { name: git-pull-requests-awslc } + - { name: stemcell } + - { name: stemcell-jammy } + run: + path: ./git-pull-requests-awslc/ci/scripts/acceptance-tests + args: [] + params: + REPO_ROOT: git-pull-requests-awslc + VARIANT: awslc-fips + on_success: + put: git-pull-requests-awslc + params: + path: git-pull-requests-awslc + status: success + context: acceptance-tests-awslc-fips + on_failure: + put: git-pull-requests-awslc + params: + path: git-pull-requests-awslc + status: failure + context: acceptance-tests-awslc-fips + - name: rc public: true plan: - do: - in_parallel: - - { get: git, trigger: true, passed: [acceptance-tests] } + - { get: git, trigger: true, passed: [acceptance-tests, acceptance-tests-awslc, acceptance-tests-awslc-fips] } - { get: version, trigger: false, params: {pre: rc} } - put: version params: {file: version/number} @@ -264,7 +432,7 @@ jobs: - in_parallel: - { get: version, passed: [rc], params: {bump: final} } - { get: git, passed: [rc] } - - get: haproxy-boshrelease-testflight + - { get: haproxy-boshrelease-testflight, trigger: true, passed: [build-haproxy-testflight-image] } - task: release image: haproxy-boshrelease-testflight config: @@ -302,7 +470,7 @@ jobs: name: gh/name tag: gh/tag body: gh/notes.md - globs: [gh/artifacts/*, gh/artifacts-patched/*] + globs: [gh/artifacts/*] - put: notify params: channel: "#haproxy-boshrelease" @@ -325,7 +493,7 @@ jobs: - get: daily trigger: true - get: git - - get: haproxy-boshrelease-testflight + - { get: haproxy-boshrelease-testflight, trigger: true, passed: [build-haproxy-testflight-image] } - task: autobump-dependencies image: haproxy-boshrelease-testflight config: @@ -434,7 +602,21 @@ resources: access_token: ((github.access_token)) repository: cloudfoundry/haproxy-boshrelease base_branch: master - labels: [run-ci] + labels: [run-ci, run-ci-awslc] + required_review_approvals: 1 + trusted_teams: + - "wg-app-runtime-platform-networking-approvers" + trusted_users: + - "dependabot" + - "CFN-CI" + + - name: git-pull-requests-awslc + type: pull-request + source: + access_token: ((github.access_token)) + repository: cloudfoundry/haproxy-boshrelease + base_branch: master + labels: [run-ci-awslc] required_review_approvals: 1 trusted_teams: - "wg-app-runtime-platform-networking-approvers" @@ -479,7 +661,7 @@ resources: source: bucket: haproxy-boshrelease json_key: ((gcp.service_key)) - regexp: haproxy-([0-9\.+-]+).tgz + regexp: haproxy-([0-9a-z\.+-]+)\.tgz - name: daily type: time diff --git a/ci/scripts/autobump-dependencies.py b/ci/scripts/autobump-dependencies.py index 4a8c20d5c..2029ea2d4 100755 --- a/ci/scripts/autobump-dependencies.py +++ b/ci/scripts/autobump-dependencies.py @@ -24,18 +24,26 @@ LUA_VERSION = "5.4" PCRE_VERSION = "10" HATOP_VERSION = "0" - -# Required Environment Vars -BLOBSTORE_SECRET_ACCESS_KEY = os.environ["GCP_SERVICE_KEY"] -gh = github.Github(login_or_token=os.environ["GITHUB_COM_TOKEN"]) -PR_ORG = os.environ["PR_ORG"] -PR_BASE = os.environ["PR_BASE"] -PR_LABEL = os.environ["PR_LABEL"] +AWS_LC_VERSION = "4" # LTS line (v4.x); builds both FIPS and non-FIPS +CMAKE_VERSION = "4" +GOLANG_VERSION = "1.27" + +# Required environment variables — read lazily (via .get) so this module can be +# imported without them (e.g. by the offline resolution tests). main() validates +# that they are actually set before doing any real work. +BLOBSTORE_SECRET_ACCESS_KEY = os.environ.get("GCP_SERVICE_KEY") +_github_token = os.environ.get("GITHUB_COM_TOKEN") +gh = github.Github(auth=github.Auth.Token(_github_token)) if _github_token else github.Github() +PR_ORG = os.environ.get("PR_ORG") +PR_BASE = os.environ.get("PR_BASE") +PR_LABEL = os.environ.get("PR_LABEL") # if DRY_RUN is set, blobs will not be uploaded and no PR created (downloads and local changes are still performed) DRY_RUN = "DRY_RUN" in os.environ # Other Global Variables BLOBS_PATH = "config/blobs.yml" +VERSIONS_PATH = "src/haproxy-versions.sh" +AWS_LC_VERSIONS_PATH = "src/aws-lc-versions.sh" PACKAGING_PATH = "packages/{}/packaging" @@ -105,27 +113,44 @@ class Dependency: pinned_version: str root_url: str package: str = "haproxy" - remote_repo = gh.get_repo(f"{PR_ORG}/haproxy-boshrelease") + filename_suffix: str = ".tar.gz" + blob_version_prefix: str = "" + # Explicit path to the file holding this dependency's version constant. + # Defaults (below) to src/haproxy-versions.sh for package "haproxy"; the + # crypto deps override it to src/aws-lc-versions.sh. + versions_path: Optional[str] = None _latest_release: Optional[Release] = None _current_version: version.Version = None + @property + def remote_repo(self): + # Lazy so importing this module does not hit the GitHub API (keeps resolution testable offline). + return gh.get_repo(f"{PR_ORG}/haproxy-boshrelease") + @property def pr_branch(self): return f"{self.name}-auto-bump-{PR_BASE}" + @property + def versions_file(self) -> str: + if self.versions_path: + return self.versions_path + if self.package == "haproxy": + return VERSIONS_PATH + return PACKAGING_PATH.format(self.package) + @property def current_version(self) -> version.Version: """ - Fetches the current version of the release from the packaging file if not already known. + Fetches the current version of the release from the versions file if not already known. (Should always be identical to the version in blobs.yml) """ if self._current_version: return self._current_version - with open(PACKAGING_PATH.format(self.package), "r") as packaging_file: - for line in packaging_file.readlines(): + with open(self.versions_file, "r") as versions_file: + for line in versions_file.readlines(): if line.startswith(self.version_var_name): - # Regex: expecting e.g. "RELEASE_VERSION=1.2.3 # http://release.org/download". extracting Semver Group rgx = rf"{self.version_var_name}=((?:[0-9]+\.){{1,3}}[0-9]+)\s+#.*$" match = re.match(rgx, line) if match: @@ -155,8 +180,11 @@ def get_release_notes(self) -> str: """ raise NotImplementedError + def blob_filename(self, ver) -> str: + return f"{self.name}-{self.blob_version_prefix}{ver}{self.filename_suffix}" + def remove_current_blob(self): - current_blob_path = f"{self.package}/{self.name}-{self.current_version}.tar.gz" + current_blob_path = f"{self.package}/{self.blob_filename(self.current_version)}" if self._check_blob_exists(current_blob_path): BoshHelper.remove_blob(current_blob_path) else: @@ -172,17 +200,17 @@ def _check_blob_exists(self, blob_path) -> bool: def update_packaging_file(self): """ - Writes the new dependency version and download-url into packages/haproxy/packaging + Writes the new dependency version and download-url into the versions file. """ - with open(PACKAGING_PATH.format(self.package), "r") as packaging_file: + with open(self.versions_file, "r") as f: replacement = "" - for line in packaging_file.readlines(): + for line in f.readlines(): if line.startswith(self.version_var_name): line = f"{self.version_var_name}={self.latest_release.version} # {self.latest_release.url}\n" replacement += line - with open(PACKAGING_PATH.format(self.package), "w") as packaging_file_write: - packaging_file_write.write(replacement) + with open(self.versions_file, "w") as f: + f.write(replacement) def open_pr_exists(self) -> bool: prs_exist = False @@ -208,7 +236,7 @@ def create_pr(self): self._update_file( self.remote_repo, - PACKAGING_PATH.format(self.package), + self.versions_file, self.pr_branch, f"Bump {self.name} version to {self.latest_release.version}", ) @@ -254,10 +282,9 @@ def _update_file(self, repo, path, branch, message): class GithubDependency(Dependency): tagname_prefix: str = "" - filename_suffix: str = ".tar.gz" def fetch_latest_release(self) -> Release: - repo_org_and_name = self.root_url.lstrip("https://github.com/") + repo_org_and_name = self.root_url.removeprefix("https://github.com/") repo = gh.get_repo(repo_org_and_name) releases = repo.get_releases() @@ -274,14 +301,14 @@ def get_release_download_url(rel): for rel in releases: if rel.prerelease: continue - current_raw = rel.tag_name.lstrip(self.tagname_prefix) + current_raw = rel.tag_name.removeprefix(self.tagname_prefix) current_version = version.parse(current_raw) if latest_version < current_version and current_raw.startswith(self.pinned_version): latest_version = current_version latest_release = Release( rel.name, get_release_download_url(rel), - f"{self.name}-{str(current_version)}{self.filename_suffix}", + self.blob_filename(current_version), current_version, ) @@ -333,6 +360,94 @@ def get_release_notes(self) -> str: return f"Make sure to check the [CHANGELOG]({self.changelog_url}) for any breaking changes." +@dataclass +class GolangDependency(Dependency): + """ + Handles Go toolchain downloads from go.dev/dl/. + Go binaries are named go1.X.Y.linux-amd64.tar.gz but stored as golang-X.Y.Z.tar.gz in blobs. + """ + + def fetch_latest_release(self) -> Release: + data = requests.get(self.root_url) + html = BeautifulSoup(data.text, "html.parser") + + versions = [] + links = [link for link in html.select("a") if "href" in link.attrs] + + pattern = rf"go({self.pinned_version}(?:\.[0-9]+)*)\.linux-amd64\.tar\.gz" + + for link in links: + match = re.search(pattern, link.attrs["href"]) + if match: + ver = version.parse(match.group(1)) + url = requests.compat.urljoin(self.root_url, link.attrs["href"]) + versions.append( + Release( + f"golang-{match.group(1)}", + url, + self.blob_filename(ver), + ver, + ) + ) + + if versions: + return sorted(versions, key=lambda r: r.version, reverse=True)[0] + + raise Exception(f"Failed to get latest {self.name} version from {self.root_url}") + + def get_release_notes(self) -> str: + return f"Make sure to check the [CHANGELOG](https://go.dev/doc/devel/release) for any breaking changes." + + +@dataclass +class GithubArchiveDependency(Dependency): + """ + For GitHub repos where we resolve versions from git tags and download the + archive tarball, rather than from published Releases with attached assets. + AWS-LC LTS point releases (e.g. 4.0.1) are tagged but not always published + as GitHub Releases, so tag-based resolution avoids silently missing a bump. + """ + + tagname_prefix: str = "" + + def fetch_latest_release(self) -> Release: + repo_org_and_name = self.root_url.removeprefix("https://github.com/") + repo = gh.get_repo(repo_org_and_name) + tags = repo.get_tags() + + latest_release = None + latest_version = version.parse("0.0.0") + + for tag in tags: + current_raw = tag.name.removeprefix(self.tagname_prefix) + if not current_raw.startswith(self.pinned_version): + continue + try: + current_version = version.parse(current_raw) + except version.InvalidVersion: + # tags that aren't plain version numbers (e.g. FIPS branch tags) + continue + if current_version.is_prerelease: + continue + if latest_version < current_version: + latest_version = current_version + url = f"{self.root_url}/archive/refs/tags/{tag.name}.tar.gz" + latest_release = Release( + tag.name, + url, + self.blob_filename(current_version), + current_version, + ) + + if latest_version == version.parse("0.0.0") or latest_release is None: + raise Exception(f"No tag found for '{self.root_url}'") + + return latest_release + + def get_release_notes(self) -> str: + return f"Make sure to check the [CHANGELOG]({self.root_url}/releases) for any breaking changes." + + @dataclass class HaproxyDependency(Dependency): def __post_init__(self): @@ -456,8 +571,14 @@ def cleanup_local_changes(): local_git.clean("-fx") -def main() -> None: - dependencies: List[Dependency] = [ +def _require_env(): + missing = [v for v in ("GCP_SERVICE_KEY", "GITHUB_COM_TOKEN", "PR_ORG", "PR_BASE", "PR_LABEL") if not os.environ.get(v)] + if missing: + raise SystemExit("Missing required environment variables: " + ", ".join(missing)) + + +def build_dependencies() -> List[Dependency]: + return [ WebLinkDependency( "keepalived", "KEEPALIVED_VERSION", @@ -500,9 +621,48 @@ def main() -> None: tagname_prefix="v", filename_suffix="", ), + # AWS-LC LTS line (v4.x). One source blob builds both the non-FIPS and + # FIPS variants; FIPS is a build-time flag, not a separate release. + # LTS tags ship as archive tarballs (no release assets), so use + # GithubArchiveDependency; blob_version_prefix keeps the aws-lc-v* name. + GithubArchiveDependency( + "aws-lc", + "AWS_LC_VERSION", + AWS_LC_VERSION, + "https://github.com/aws/aws-lc", + tagname_prefix="v", + blob_version_prefix="v", + versions_path=AWS_LC_VERSIONS_PATH, + ), + GithubDependency( + "cmake", + "CMAKE_VERSION", + CMAKE_VERSION, + "https://github.com/Kitware/CMake", + tagname_prefix="v", + # We ship the prebuilt Linux binary tarball, not the source. This + # both selects the right release asset and produces a blob name + # (cmake-X.Y.Z-linux-x86_64.tar.gz) matching the packaging scripts. + filename_suffix="-linux-x86_64.tar.gz", + versions_path=AWS_LC_VERSIONS_PATH, + ), + GolangDependency( + "golang", + "GOLANG_VERSION", + GOLANG_VERSION, + "https://go.dev/dl/", + versions_path=AWS_LC_VERSIONS_PATH, + ), ] - write_private_yaml() + +def main() -> None: + _require_env() + dependencies = build_dependencies() + + # private.yml (blobstore creds) is only needed to upload blobs, which a dry run skips. + if not DRY_RUN: + write_private_yaml() for dependency in dependencies: current_version = dependency.current_version @@ -513,6 +673,15 @@ def main() -> None: print(f"[{dependency.name}] already on the latest version: {latest_version} " f"(pinned: {dependency.pinned_version}.*)") continue + if DRY_RUN: + # Read-only report: announce the bump but touch nothing — no PR-exists check, + # download, bosh blob op, file edit, upload, PR, or git reset/clean. + print( + f"[{dependency.name}] Version-Bump required: {current_version} --> {latest_version} " + f"(dry-run: would download {latest_release.url} -> blob {latest_release.file})" + ) + continue + if dependency.open_pr_exists(): print(f"[{dependency.name}] Open bump PR exists (for branch: {dependency.pr_branch})") continue @@ -521,8 +690,7 @@ def main() -> None: dependency.remove_current_blob() latest_release.add_blob(dependency.package) dependency.update_packaging_file() - if not DRY_RUN: - BoshHelper.upload_blobs() + BoshHelper.upload_blobs() dependency.create_pr() # clear the working directory for the next dependency bump. diff --git a/ci/scripts/functions-ci.sh b/ci/scripts/functions-ci.sh index fa8e5465d..5d68f567b 100755 --- a/ci/scripts/functions-ci.sh +++ b/ci/scripts/functions-ci.sh @@ -54,6 +54,8 @@ function bosh_release() { echo "----- Creating candidate BOSH release..." bosh -n reset-release # in case dev_releases/ is in repo accidentally + # Single release: the committed tree bundles every SSL variant; the backend is + # selected at deploy time via ha_proxy.ssl_variant (see the acceptance harness). bosh create-release --force bosh upload-release --rebase release_final_version=$(spruce json dev_releases/*/index.yml | jq -r ".builds[].version" | sed -e "s%+.*%%") diff --git a/ci/scripts/requirements-test.txt b/ci/scripts/requirements-test.txt new file mode 100644 index 000000000..26b77f68b --- /dev/null +++ b/ci/scripts/requirements-test.txt @@ -0,0 +1,2 @@ +-r requirements.txt +pytest diff --git a/ci/scripts/shipit b/ci/scripts/shipit index 636b7b51a..7daa130c5 100755 --- a/ci/scripts/shipit +++ b/ci/scripts/shipit @@ -64,11 +64,18 @@ git submodule update --init --recursive --force cd - version() { - # extract the version variable $1 from the packaging script $2 (default 'haproxy') pattern='s/VERSION=(.*)(\s?#.*)/\1/p' - package=${2:-haproxy} - # extract version and remove all spaces - sed -n -E "${pattern//VERSION/${1:?}}" "${REPO_ROOT}/packages/${package}/packaging" | sed 's/ *//g' + package=${2:-} + if [[ -n "${package}" ]]; then + source_file="${REPO_ROOT}/packages/${package}/packaging" + sed -n -E "${pattern//VERSION/${1:?}}" "${source_file}" | sed 's/ *//g' + else + # AWS_LC_VERSION / CMAKE_VERSION / GOLANG_VERSION live in aws-lc-versions.sh; + # the rest in haproxy-versions.sh. Each var appears in exactly one, so search both. + sed -n -E "${pattern//VERSION/${1:?}}" \ + "${REPO_ROOT}/src/haproxy-versions.sh" \ + "${REPO_ROOT}/src/aws-lc-versions.sh" | sed 's/ *//g' + fi } HAPROXY_VERSION=$(version HAPROXY_VERSION) @@ -76,11 +83,21 @@ LUA_VERSION=$(version LUA_VERSION) SOCAT_VERSION=$(version SOCAT_VERSION) PCRE_VERSION=$(version PCRE_VERSION) KEEPALIVED_VERSION=$(version KEEPALIVED_VERSION keepalived) +AWS_LC_VERSION=$(version AWS_LC_VERSION) +CMAKE_VERSION=$(version CMAKE_VERSION) +GOLANG_VERSION=$(version GOLANG_VERSION) -VERSION="${VERSION_TO_CREATE}+${HAPROXY_VERSION}" +VERSION="${VERSION_TO_CREATE}+${HAPROXY_VERSION}-awslc-${AWS_LC_VERSION}" cd "${REPO_ROOT}" header "Create final release..." + +# Single release: the committed tree bundles every build — SSL variant (openssl, awslc, +# awslc-fips), each with and without the HAProxy community patches. One is selected at deploy +# time via ha_proxy.ssl_variant + ha_proxy.include_haproxy_patches (default openssl, unpatched). +# A plain create-release of the tree produces it. +bosh upload-blobs + bosh -n create-release --final --version "${VERSION}" bosh -n create-release "releases/${RELEASE_NAME}/${RELEASE_NAME}-${VERSION}.yml" \ --tarball "releases/${RELEASE_NAME}/${RELEASE_NAME}-${VERSION}.tgz" @@ -95,8 +112,6 @@ echo "SHA1=${SHA1}" export SHA256=$(sha256sum "${RELEASE_TGZ}" | head -n1 | awk '{print $1}') echo "SHA256=${SHA256}" - - mkdir -p "${RELEASE_ROOT}/artifacts" echo "v${VERSION}" > "${RELEASE_ROOT}/tag" echo "v${VERSION}" > "${RELEASE_ROOT}/name" @@ -119,7 +134,7 @@ cat >> "${RELEASE_ROOT}/notes.md" < ### Versions @@ -133,6 +148,9 @@ The following versions of upstream components are included in this haproxy-boshr | Lua | \`${LUA_VERSION}\` | | PCRE | \`${PCRE_VERSION}\` | | socat | \`${SOCAT_VERSION}\` | +| AWS-LC (LTS) | \`${AWS_LC_VERSION}\` | +| cmake | \`${CMAKE_VERSION}\` | +| Go | \`${GOLANG_VERSION}\` | ### Deployment \`\`\`yaml @@ -144,6 +162,10 @@ releases: # for deployments with sha256, use the following line instead: # sha1: "sha256:${SHA256}" + +# The release bundles every SSL variant; select one at deploy time (defaults to openssl, stock): +# ha_proxy.ssl_variant: openssl | awslc | awslc-fips +# ha_proxy.include_haproxy_patches: false | true # append the bundled community patches \`\`\` EOF @@ -167,47 +189,8 @@ pushd "${REPO_ROOT}" git add -A git status git commit -m "release v${VERSION}" - - # After creating a final release we will also create a dev release patches from haproxy-patches directory - echo "- haproxy/patches.tar.gz" >> packages/haproxy/spec - tar -czvf haproxy-patches.tar.gz haproxy-patches - bosh add-blob haproxy-patches.tar.gz haproxy/patches.tar.gz - bosh upload-blobs - - bosh -n create-release --force --version "${VERSION}-patched" \ - --tarball "../${RELEASE_NAME}-${VERSION}-patched.tgz" - - # Undo changes to repo from creating dev release - git clean -df - git reset --hard popd -mkdir -p "${RELEASE_ROOT}/artifacts-patched" -mv "${RELEASE_NAME}-${VERSION}-patched.tgz" "${RELEASE_ROOT}/artifacts-patched" - -PATCHED_RELEASE_TGZ=${RELEASE_ROOT}/artifacts-patched/${RELEASE_NAME}-${VERSION}-patched.tgz -# shellcheck disable=SC2155 -export PATCHED_SHA1=$(sha1sum "${PATCHED_RELEASE_TGZ}" | head -n1 | awk '{print $1}') -echo "PATCHED_SHA1=${PATCHED_SHA1}" -# shellcheck disable=SC2155 -export PATCHED_SHA256=$(sha256sum "${PATCHED_RELEASE_TGZ}" | head -n1 | awk '{print $1}') -echo "PATCHED_SHA256=${PATCHED_SHA256}" - -cat >> "${RELEASE_ROOT}/notes.md" < skipped + FakeTag("v4.2.0-rc1"), # prerelease -> skipped + ]) + monkeypatch.setattr(autobump, "gh", FakeGithub(repo)) + + dep = autobump.GithubArchiveDependency( + name="aws-lc", version_var_name="AWS_LC_VERSION", pinned_version="4", + root_url="https://github.com/aws/aws-lc", tagname_prefix="v", blob_version_prefix="v", + ) + rel = dep.fetch_latest_release() + assert str(rel.version) == "4.1.0" + assert rel.file == "aws-lc-v4.1.0.tar.gz" + assert rel.url == "https://github.com/aws/aws-lc/archive/refs/tags/v4.1.0.tar.gz" + + +# --- GolangDependency (go.dev HTML scrape) ----------------------------------------- +def test_golang_dependency_scrapes_pinned_minor(monkeypatch): + html = """ + go1.27.0 + go1.27.1 + go1.28.0 + """ + monkeypatch.setattr(autobump.requests, "get", lambda *_a, **_k: FakeResponse(html)) + + dep = autobump.GolangDependency( + name="golang", version_var_name="GOLANG_VERSION", pinned_version="1.27", + root_url="https://go.dev/dl/", + ) + rel = dep.fetch_latest_release() + assert str(rel.version) == "1.27.1" + assert rel.file == "golang-1.27.1.tar.gz" + + +# --- WebLinkDependency (generic download-page scrape) ------------------------------ +def test_weblink_dependency_scrape(monkeypatch): + html = """ + socat-1.8.0.0 + socat-1.8.0.1 + """ + monkeypatch.setattr(autobump.requests, "get", lambda *_a, **_k: FakeResponse(html)) + + dep = autobump.WebLinkDependency( + name="socat", version_var_name="SOCAT_VERSION", pinned_version="1", + root_url="http://www.dest-unreach.org/socat/download/", + ) + rel = dep.fetch_latest_release() + assert str(rel.version) == "1.8.0.1" + assert rel.file == "socat-1.8.0.1.tar.gz" + + +# --- HaproxyDependency (releases.json) --------------------------------------------- +def test_haproxy_dependency_reads_releases_json(monkeypatch): + payload = '{"latest_release": "3.2.26", "releases": {"3.2.26": {"file": "haproxy-3.2.26.tar.gz"}}}' + monkeypatch.setattr(autobump.requests, "get", lambda *_a, **_k: FakeResponse(payload)) + + dep = autobump.HaproxyDependency( + name="haproxy", version_var_name="HAPROXY_VERSION", pinned_version="3.2", + root_url="https://www.haproxy.org/download/{}/src", + ) + rel = dep.fetch_latest_release() + assert str(rel.version) == "3.2.26" + assert rel.file == "haproxy-3.2.26.tar.gz" + assert rel.url.endswith("/download/3.2/src/haproxy-3.2.26.tar.gz") + + +# --- current_version parsing (from the versions file) ------------------------------ +def test_current_version_parsed_from_versions_file(tmp_path): + vfile = tmp_path / "versions.sh" + vfile.write_text("FOO_VERSION=1.2.3 # https://example.invalid/foo-1.2.3.tar.gz\n") + dep = autobump.GithubDependency( + name="foo", version_var_name="FOO_VERSION", pinned_version="1", + root_url="https://github.com/o/r", versions_path=str(vfile), + ) + assert dep.current_version == version.parse("1.2.3") + + +# --- no-bump decision uses >= on parsed versions (sanity on the comparison) -------- +def test_version_comparison_semantics(): + assert version.parse("3.2.26") > version.parse("3.2.9") # not lexical + assert version.parse("10.45") > version.parse("10.9") + + +# --- main() side-effect gating by DRY_RUN ------------------------------------------ +class _FakeDep: + """A dependency that is 'behind', so main() would try to bump it.""" + name = "cmake" + pinned_version = "4" + package = "haproxy" + pr_branch = "cmake-auto-bump-master" + + def __init__(self, latest): + self._latest = latest + self.calls = [] + + @property + def current_version(self): + return version.parse("4.4.3") + + @property + def latest_release(self): + return self._latest + + def open_pr_exists(self): + self.calls.append("open_pr_exists") + return False + + def remove_current_blob(self): + self.calls.append("remove_current_blob") + + def update_packaging_file(self): + self.calls.append("update_packaging_file") + + def create_pr(self): + self.calls.append("create_pr") + + +def _wire_main(monkeypatch, dry_run): + """Stub every side-effecting hook so main() can run offline; return the call log + dep.""" + module_calls = [] + monkeypatch.setattr(autobump, "DRY_RUN", dry_run) + monkeypatch.setattr(autobump, "_require_env", lambda: None) + monkeypatch.setattr(autobump, "write_private_yaml", lambda: module_calls.append("write_private_yaml")) + monkeypatch.setattr(autobump, "wget", lambda *a, **k: module_calls.append("wget")) + monkeypatch.setattr(autobump, "cleanup_local_changes", lambda: module_calls.append("cleanup")) + monkeypatch.setattr(autobump.BoshHelper, "add_blob", classmethod(lambda cls, *a: module_calls.append("add_blob"))) + monkeypatch.setattr(autobump.BoshHelper, "remove_blob", classmethod(lambda cls, *a: module_calls.append("remove_blob"))) + monkeypatch.setattr(autobump.BoshHelper, "upload_blobs", classmethod(lambda cls: module_calls.append("upload_blobs"))) + rel = autobump.Release("cmake-4.4.4", "https://example.invalid/cmake-4.4.4.tar.gz", "cmake-4.4.4.tar.gz", version.parse("4.4.4")) + dep = _FakeDep(rel) + monkeypatch.setattr(autobump, "build_dependencies", lambda: [dep]) + return module_calls, dep + + +def test_dry_run_is_side_effect_free(monkeypatch): + module_calls, dep = _wire_main(monkeypatch, dry_run=True) + autobump.main() + # A behind dependency is reported but NOTHING is touched. + assert module_calls == [] + assert dep.calls == [] + + +def test_real_run_performs_the_bump(monkeypatch): + module_calls, dep = _wire_main(monkeypatch, dry_run=False) + autobump.main() + # The full bump pipeline runs, including the destructive cleanup. + assert "write_private_yaml" in module_calls + assert "wget" in module_calls # Release.download + assert "add_blob" in module_calls # Release.add_blob -> BoshHelper + assert "upload_blobs" in module_calls + assert "cleanup" in module_calls + assert dep.calls == ["open_pr_exists", "remove_current_blob", "update_packaging_file", "create_pr"] diff --git a/config/blobs.yml b/config/blobs.yml index 0d2325780..64a2d634d 100644 --- a/config/blobs.yml +++ b/config/blobs.yml @@ -1,3 +1,15 @@ +haproxy/aws-lc-v4.0.0.tar.gz: + size: 130854647 + object_id: d911b2d3-6b54-4621-7d49-fb71b48d0968 + sha: sha256:dec49e5c38dd8d3b4c2700c692bbb457a149cd6d23ad99463ce3612720c23bb0 +haproxy/cmake-4.4.3-linux-x86_64.tar.gz: + size: 64872980 + object_id: b0d691d9-69ed-4964-51e8-cfaa7848976f + sha: sha256:d6c83076c575bc00b823522ac974bda66d0af05d6ddc30e739c12385cf32c6cc +haproxy/golang-1.27.1.tar.gz: + size: 70553950 + object_id: 3b678ee3-040f-4cde-7c9a-8a94ca41ff4c + sha: sha256:63d339f0da5ab53635a56f2490a7984dfe12dfcff22ad749f63edaf590168445 haproxy/haproxy-3.2.26.tar.gz: size: 5203667 object_id: b8a0ebf0-e6bc-4486-7873-12805ca22ba3 diff --git a/docs/coredumps.md b/docs/coredumps.md index 611ea1f56..22b278765 100644 --- a/docs/coredumps.md +++ b/docs/coredumps.md @@ -48,10 +48,12 @@ Add the `set-dumpable` directive to the `global` section in `haproxy.config.erb` Core dump files are written to `/var/vcap/data/haproxy/`. -To analyze a core dump, use `gdb` with the HAProxy binary and the core file: +To analyze a core dump, use `gdb` with the HAProxy binary and the core file. The binary lives in the +variant package (`haproxy-openssl`, `haproxy-awslc`, `haproxy-awslc-fips`), so resolve the running +one via its pid rather than hardcoding a package name: ```bash -gdb /var/vcap/packages/haproxy/bin/haproxy /var/vcap/data/haproxy/core... +gdb "$(readlink -f /proc/"$(pgrep -n haproxy)"/exe)" /var/vcap/data/haproxy/core... ``` Useful GDB commands once loaded: diff --git a/jobs/haproxy/spec b/jobs/haproxy/spec index 7da59dee4..6c49a6506 100644 --- a/jobs/haproxy/spec +++ b/jobs/haproxy/spec @@ -4,7 +4,13 @@ name: haproxy description: "The HAProxy server can be used to terminate SSL in front of the Routers. Each HAProxy instance should point to multiple Routers." packages: -- haproxy +- haproxy-deps +- haproxy-openssl +- haproxy-openssl-patched +- haproxy-awslc +- haproxy-awslc-patched +- haproxy-awslc-fips +- haproxy-awslc-fips-patched - ttar templates: @@ -42,6 +48,14 @@ consumes: optional: true properties: + ha_proxy.ssl_variant: + default: openssl + description: | + SSL library the HAProxy binary links against: openssl (default) | awslc | awslc-fips. + ha_proxy.include_haproxy_patches: + default: false + description: | + Run the HAProxy build that includes the bundled community patches ha_proxy.pre_start_script: description: | This script will be appended to the pre-start script and run before the job starts. diff --git a/jobs/haproxy/templates/haproxy_wrapper.erb b/jobs/haproxy/templates/haproxy_wrapper.erb index bb928f1d6..5e8db6463 100755 --- a/jobs/haproxy/templates/haproxy_wrapper.erb +++ b/jobs/haproxy/templates/haproxy_wrapper.erb @@ -3,7 +3,32 @@ set -e -export PATH=$PATH:/var/vcap/packages/haproxy/bin:/var/vcap/packages/ttar/bin +# --- Select the HAProxy binary package ------------------------------------------------ +# The release bundles every SSL variant; two orthogonal properties choose the binary, whose +# package name follows the convention haproxy-[-patched]: +# ha_proxy.ssl_variant openssl (default) | awslc | awslc-fips +# ha_proxy.include_haproxy_patches false (default) | true -> append "-patched" +<%- + base = p("ha_proxy.ssl_variant").to_s.strip + patches = p("ha_proxy.include_haproxy_patches") + want = "haproxy-#{base}" + (patches ? "-patched" : "") +-%> +<%- unless %w[openssl awslc awslc-fips].include?(base) -%> +echo "$(date): ERROR: ha_proxy.ssl_variant='<%= base %>' is not a valid SSL variant (openssl, awslc, awslc-fips)" >&2 +exit 1 +<%- end -%> + +HAPROXY_PKG="/var/vcap/packages/<%= want %>" +if [ ! -x "${HAPROXY_PKG}/bin/haproxy" ]; then + avail="" + for d in /var/vcap/packages/haproxy-*; do + [ -x "$d/bin/haproxy" ] && avail="${avail} $(basename "$d" | sed 's/^haproxy-//')" + done + echo "$(date): ERROR: '<%= want %>' is not bundled in this release; available:${avail}" >&2 + exit 1 +fi + +export PATH=$PATH:${HAPROXY_PKG}/bin:/var/vcap/packages/ttar/bin CONFIG=/var/vcap/jobs/haproxy/config/haproxy.config PID_FILE=/var/vcap/sys/run/haproxy/haproxy.pid DRAIN_LOCK=/var/vcap/sys/run/haproxy/drain.lock diff --git a/jobs/haproxy/templates/pre-start.erb b/jobs/haproxy/templates/pre-start.erb index b511c8c8e..67c52d96b 100644 --- a/jobs/haproxy/templates/pre-start.erb +++ b/jobs/haproxy/templates/pre-start.erb @@ -13,13 +13,10 @@ if [ ! -e /usr/bin/python ] && [ -e /usr/bin/python3 ]; then sudo ln -s /usr/bin/python3 /usr/bin/python fi -if [ ! -e /usr/local/bin/hatop ]; then - sudo ln -s /var/vcap/packages/haproxy/hatop-wrapper /usr/local/bin/hatop -fi - -if [ ! -e /usr/local/bin/socat ]; then - sudo ln -s /var/vcap/packages/haproxy/bin/socat /usr/local/bin/socat -fi +# socat and the hatop wrapper live in the shared haproxy-deps package. +DEPS_PKG="/var/vcap/packages/haproxy-deps" +sudo ln -sf ${DEPS_PKG}/hatop-wrapper /usr/local/bin/hatop +sudo ln -sf ${DEPS_PKG}/bin/socat /usr/local/bin/socat <%- if_p("ha_proxy.pre_start_script") do |script| -%> # ha_proxy.pre_start_script {{{ diff --git a/packages/README.md b/packages/README.md new file mode 100644 index 000000000..cc1428847 --- /dev/null +++ b/packages/README.md @@ -0,0 +1,20 @@ +# `packages/` — the complete release + +This directory holds **every** package in the release. `bosh create-release` bundles all of them, so a +plain `bosh create-release` of the committed tree produces the one and only release — there is no +build-time assembly. + +- `haproxy-deps`, `ttar`, `keepalived` — shared, compiled once +- `haproxy-openssl`, `haproxy-openssl-patched`, `haproxy-awslc`, `haproxy-awslc-patched`, + `haproxy-awslc-fips`, `haproxy-awslc-fips-patched` — the six HAProxy binaries: each SSL + variant (openssl, awslc, awslc-fips) built with and without the community patches +- `aws-lc-src` (shared source), `aws-lc`, `aws-lc-fips`, `cmake` — crypto build dependencies + +The SSL backend is chosen at **deploy time** via the `ha_proxy.ssl_variant` property (default +`openssl`); see `jobs/haproxy/templates/haproxy_wrapper.erb`. Every deploy compiles all bundled +binaries; switching backends later is a property change + redeploy with no recompile. + +Dependencies are one-directional: every `haproxy-*` binary depends on `haproxy-deps`; `aws-lc` and +`aws-lc-fips` build from the shared `aws-lc-src` + `cmake`. HAProxy patches for the `-patched` +variants are plain source files under [`../src/haproxy-patches/`](../src/haproxy-patches/), applied at +compile time by `src/apply-patches.sh`. diff --git a/packages/aws-lc-fips/packaging b/packages/aws-lc-fips/packaging new file mode 100644 index 000000000..b9c5eef0c --- /dev/null +++ b/packages/aws-lc-fips/packaging @@ -0,0 +1,30 @@ +set -euxo pipefail + +source aws-lc-versions.sh + +export PATH=/var/vcap/packages/cmake/bin:$PATH + +# Install Go toolchain (required for FIPS delocate/inject_hash) +tar xzf haproxy/golang-${GOLANG_VERSION}.tar.gz -C ${BOSH_INSTALL_TARGET} +export GOROOT=${BOSH_INSTALL_TARGET}/go +export GOPATH=${BOSH_INSTALL_TARGET}/gopath +export GOCACHE=${BOSH_INSTALL_TARGET}/gocache +export PATH=${GOROOT}/bin:$PATH + +# Build from the shared aws-lc-src package (source bundled once, not per variant). +cp -a /var/vcap/packages/aws-lc-src/aws-lc-${AWS_LC_VERSION} . +pushd aws-lc-${AWS_LC_VERSION} + mkdir -p build && cd build + cmake -DCMAKE_BUILD_TYPE=Release \ + -DFIPS=1 \ + -DBUILD_SHARED_LIBS=0 \ + -DBUILD_TESTING=0 \ + -DBUILD_TOOL=0 \ + -DCMAKE_INSTALL_PREFIX=${BOSH_INSTALL_TARGET} \ + .. + make -j$(nproc) + make install +popd + +# Clean up Go toolchain to save disk space +rm -rf ${BOSH_INSTALL_TARGET}/go ${BOSH_INSTALL_TARGET}/gopath ${BOSH_INSTALL_TARGET}/gocache diff --git a/packages/aws-lc-fips/spec b/packages/aws-lc-fips/spec new file mode 100644 index 000000000..125fd6eb8 --- /dev/null +++ b/packages/aws-lc-fips/spec @@ -0,0 +1,8 @@ +--- +name: aws-lc-fips +dependencies: +- cmake +- aws-lc-src +files: +- haproxy/golang-*.tar.gz +- aws-lc-versions.sh diff --git a/packages/aws-lc-src/packaging b/packages/aws-lc-src/packaging new file mode 100644 index 000000000..14b8ff584 --- /dev/null +++ b/packages/aws-lc-src/packaging @@ -0,0 +1,8 @@ +set -euxo pipefail + +source aws-lc-versions.sh + +# Unpack the AWS-LC source ONCE into this package. Both the non-FIPS (aws-lc) and +# FIPS (aws-lc-fips) compile packages depend on aws-lc-src and build from here, so +# the ~130 MB source blob is bundled in exactly one package instead of each. +tar xzf haproxy/aws-lc-v${AWS_LC_VERSION}.tar.gz -C ${BOSH_INSTALL_TARGET} diff --git a/packages/aws-lc-src/spec b/packages/aws-lc-src/spec new file mode 100644 index 000000000..53021b343 --- /dev/null +++ b/packages/aws-lc-src/spec @@ -0,0 +1,5 @@ +--- +name: aws-lc-src +files: +- haproxy/aws-lc-v*.tar.gz +- aws-lc-versions.sh diff --git a/packages/aws-lc/packaging b/packages/aws-lc/packaging new file mode 100644 index 000000000..c54b593f0 --- /dev/null +++ b/packages/aws-lc/packaging @@ -0,0 +1,21 @@ +set -euxo pipefail + +source aws-lc-versions.sh + +export PATH=/var/vcap/packages/cmake/bin:$PATH + +# Build from the shared aws-lc-src package (source bundled once, not per variant). +cp -a /var/vcap/packages/aws-lc-src/aws-lc-${AWS_LC_VERSION} . +pushd aws-lc-${AWS_LC_VERSION} + mkdir -p build && cd build + cmake -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_SHARED_LIBS=0 \ + -DDISABLE_GO=1 \ + -DDISABLE_PERL=1 \ + -DBUILD_TESTING=0 \ + -DBUILD_TOOL=0 \ + -DCMAKE_INSTALL_PREFIX=${BOSH_INSTALL_TARGET} \ + .. + make -j$(nproc) + make install +popd diff --git a/packages/aws-lc/spec b/packages/aws-lc/spec new file mode 100644 index 000000000..880794d1a --- /dev/null +++ b/packages/aws-lc/spec @@ -0,0 +1,7 @@ +--- +name: aws-lc +dependencies: +- cmake +- aws-lc-src +files: +- aws-lc-versions.sh diff --git a/packages/cmake/packaging b/packages/cmake/packaging new file mode 100644 index 000000000..54efb4838 --- /dev/null +++ b/packages/cmake/packaging @@ -0,0 +1,7 @@ +set -euxo pipefail + +source aws-lc-versions.sh + +# Install prebuilt cmake binary tarball — no compile step needed. +tar xzf haproxy/cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz +cp -r cmake-${CMAKE_VERSION}-linux-x86_64/. ${BOSH_INSTALL_TARGET}/ diff --git a/packages/cmake/spec b/packages/cmake/spec new file mode 100644 index 000000000..ac8dfc416 --- /dev/null +++ b/packages/cmake/spec @@ -0,0 +1,5 @@ +--- +name: cmake +files: +- haproxy/cmake-*.tar.gz +- aws-lc-versions.sh diff --git a/packages/haproxy-awslc-fips-patched/packaging b/packages/haproxy-awslc-fips-patched/packaging new file mode 100644 index 000000000..4fad01848 --- /dev/null +++ b/packages/haproxy-awslc-fips-patched/packaging @@ -0,0 +1,5 @@ +set -euxo pipefail + +source haproxy-versions.sh +SSL_MAKE_FLAGS="USE_OPENSSL_AWSLC=1 SSL_INC=/var/vcap/packages/aws-lc-fips/include SSL_LIB=/var/vcap/packages/aws-lc-fips/lib" +source build-haproxy.sh diff --git a/packages/haproxy-awslc-fips-patched/spec b/packages/haproxy-awslc-fips-patched/spec new file mode 100644 index 000000000..a94a011b5 --- /dev/null +++ b/packages/haproxy-awslc-fips-patched/spec @@ -0,0 +1,11 @@ +--- +name: haproxy-awslc-fips-patched +dependencies: +- haproxy-deps +- aws-lc-fips +files: +- haproxy/haproxy-*.tar.gz +- haproxy-patches/*.patch +- haproxy-versions.sh +- apply-patches.sh +- build-haproxy.sh diff --git a/packages/haproxy-awslc-fips/packaging b/packages/haproxy-awslc-fips/packaging new file mode 100644 index 000000000..4fad01848 --- /dev/null +++ b/packages/haproxy-awslc-fips/packaging @@ -0,0 +1,5 @@ +set -euxo pipefail + +source haproxy-versions.sh +SSL_MAKE_FLAGS="USE_OPENSSL_AWSLC=1 SSL_INC=/var/vcap/packages/aws-lc-fips/include SSL_LIB=/var/vcap/packages/aws-lc-fips/lib" +source build-haproxy.sh diff --git a/packages/haproxy-awslc-fips/spec b/packages/haproxy-awslc-fips/spec new file mode 100644 index 000000000..12aa6f42e --- /dev/null +++ b/packages/haproxy-awslc-fips/spec @@ -0,0 +1,9 @@ +--- +name: haproxy-awslc-fips +dependencies: +- haproxy-deps +- aws-lc-fips +files: +- haproxy/haproxy-*.tar.gz +- haproxy-versions.sh +- build-haproxy.sh diff --git a/packages/haproxy-awslc-patched/packaging b/packages/haproxy-awslc-patched/packaging new file mode 100644 index 000000000..640430ed0 --- /dev/null +++ b/packages/haproxy-awslc-patched/packaging @@ -0,0 +1,5 @@ +set -euxo pipefail + +source haproxy-versions.sh +SSL_MAKE_FLAGS="USE_OPENSSL_AWSLC=1 SSL_INC=/var/vcap/packages/aws-lc/include SSL_LIB=/var/vcap/packages/aws-lc/lib" +source build-haproxy.sh diff --git a/packages/haproxy-awslc-patched/spec b/packages/haproxy-awslc-patched/spec new file mode 100644 index 000000000..3e1653342 --- /dev/null +++ b/packages/haproxy-awslc-patched/spec @@ -0,0 +1,11 @@ +--- +name: haproxy-awslc-patched +dependencies: +- haproxy-deps +- aws-lc +files: +- haproxy/haproxy-*.tar.gz +- haproxy-patches/*.patch +- haproxy-versions.sh +- apply-patches.sh +- build-haproxy.sh diff --git a/packages/haproxy-awslc/packaging b/packages/haproxy-awslc/packaging new file mode 100644 index 000000000..640430ed0 --- /dev/null +++ b/packages/haproxy-awslc/packaging @@ -0,0 +1,5 @@ +set -euxo pipefail + +source haproxy-versions.sh +SSL_MAKE_FLAGS="USE_OPENSSL_AWSLC=1 SSL_INC=/var/vcap/packages/aws-lc/include SSL_LIB=/var/vcap/packages/aws-lc/lib" +source build-haproxy.sh diff --git a/packages/haproxy-awslc/spec b/packages/haproxy-awslc/spec new file mode 100644 index 000000000..9bbe8b5b0 --- /dev/null +++ b/packages/haproxy-awslc/spec @@ -0,0 +1,9 @@ +--- +name: haproxy-awslc +dependencies: +- haproxy-deps +- aws-lc +files: +- haproxy/haproxy-*.tar.gz +- haproxy-versions.sh +- build-haproxy.sh diff --git a/packages/haproxy-deps/packaging b/packages/haproxy-deps/packaging new file mode 100644 index 000000000..6c2ac8764 --- /dev/null +++ b/packages/haproxy-deps/packaging @@ -0,0 +1,36 @@ +set -euxo pipefail + +source haproxy-versions.sh + +mkdir ${BOSH_INSTALL_TARGET}/bin + +echo "Extracting lua..." +tar xzf haproxy/lua-${LUA_VERSION}.tar.gz +pushd lua-${LUA_VERSION} + make linux install INSTALL_TOP=${BOSH_INSTALL_TARGET} +popd + +echo "Extracting pcre..." +tar xzf haproxy/pcre2-${PCRE_VERSION}.tar.gz +pushd pcre2-${PCRE_VERSION} + ./configure \ + --enable-jit \ + --prefix ${BOSH_INSTALL_TARGET} + make + make install +popd + +echo "Installing socat..." +tar xzf haproxy/socat-${SOCAT_VERSION}.tar.gz +pushd socat-${SOCAT_VERSION} + ./configure + make + cp socat ${BOSH_INSTALL_TARGET}/bin + chmod 755 ${BOSH_INSTALL_TARGET}/bin/socat +popd + +echo "Installing hatop..." +cp haproxy/hatop-${HATOP_VERSION} ${BOSH_INSTALL_TARGET}/bin/hatop +chmod 755 ${BOSH_INSTALL_TARGET}/bin/hatop +cp hatop-wrapper ${BOSH_INSTALL_TARGET}/ +chmod 755 ${BOSH_INSTALL_TARGET}/hatop-wrapper diff --git a/packages/haproxy/spec b/packages/haproxy-deps/spec similarity index 74% rename from packages/haproxy/spec rename to packages/haproxy-deps/spec index 65c277991..b1bf0b687 100644 --- a/packages/haproxy/spec +++ b/packages/haproxy-deps/spec @@ -1,9 +1,9 @@ --- -name: haproxy +name: haproxy-deps files: -- haproxy/haproxy-*.tar.gz - haproxy/pcre2-*.tar.gz - haproxy/socat-*.tar.gz - haproxy/lua-*.tar.gz - haproxy/hatop-* - hatop-wrapper +- haproxy-versions.sh diff --git a/packages/haproxy-openssl-patched/packaging b/packages/haproxy-openssl-patched/packaging new file mode 100644 index 000000000..f403c9eaf --- /dev/null +++ b/packages/haproxy-openssl-patched/packaging @@ -0,0 +1,5 @@ +set -euxo pipefail + +source haproxy-versions.sh +SSL_MAKE_FLAGS="USE_OPENSSL=1" +source build-haproxy.sh diff --git a/packages/haproxy-openssl-patched/spec b/packages/haproxy-openssl-patched/spec new file mode 100644 index 000000000..0e29f7f54 --- /dev/null +++ b/packages/haproxy-openssl-patched/spec @@ -0,0 +1,10 @@ +--- +name: haproxy-openssl-patched +dependencies: +- haproxy-deps +files: +- haproxy/haproxy-*.tar.gz +- haproxy-patches/*.patch +- haproxy-versions.sh +- apply-patches.sh +- build-haproxy.sh diff --git a/packages/haproxy-openssl/packaging b/packages/haproxy-openssl/packaging new file mode 100644 index 000000000..f403c9eaf --- /dev/null +++ b/packages/haproxy-openssl/packaging @@ -0,0 +1,5 @@ +set -euxo pipefail + +source haproxy-versions.sh +SSL_MAKE_FLAGS="USE_OPENSSL=1" +source build-haproxy.sh diff --git a/packages/haproxy-openssl/spec b/packages/haproxy-openssl/spec new file mode 100644 index 000000000..1e01dec98 --- /dev/null +++ b/packages/haproxy-openssl/spec @@ -0,0 +1,8 @@ +--- +name: haproxy-openssl +dependencies: +- haproxy-deps +files: +- haproxy/haproxy-*.tar.gz +- haproxy-versions.sh +- build-haproxy.sh diff --git a/packages/haproxy/packaging b/packages/haproxy/packaging deleted file mode 100644 index 5724031b9..000000000 --- a/packages/haproxy/packaging +++ /dev/null @@ -1,72 +0,0 @@ -# abort script on failures -set -euxo pipefail - - -LUA_VERSION=5.4.9 # https://www.lua.org/ftp/lua-5.4.9.tar.gz - -PCRE_VERSION=10.49 # https://github.com/PCRE2Project/pcre2/releases/download/pcre2-10.49/pcre2-10.49.tar.gz - -SOCAT_VERSION=1.8.1.3 # http://www.dest-unreach.org/socat/download/socat-1.8.1.3.tar.gz - -HAPROXY_VERSION=3.2.26 # https://www.haproxy.org/download/3.2/src/haproxy-3.2.26.tar.gz - -HATOP_VERSION=0.8.2 # https://github.com/jhunt/hatop/releases/download/v0.8.2/hatop - -mkdir ${BOSH_INSTALL_TARGET}/bin - -echo "Extracting lua..." -tar xzf haproxy/lua-${LUA_VERSION}.tar.gz -pushd lua-${LUA_VERSION} - make linux install INSTALL_TOP=${BOSH_INSTALL_TARGET} -popd - -echo "Extracting pcre..." -tar xzf haproxy/pcre2-${PCRE_VERSION}.tar.gz -pushd pcre2-${PCRE_VERSION} - ./configure \ - --enable-jit \ - --prefix ${BOSH_INSTALL_TARGET} - make - make install -popd - -echo "Installing socat..." -tar xzf haproxy/socat-${SOCAT_VERSION}.tar.gz -pushd socat-${SOCAT_VERSION} - ./configure - make - cp socat ${BOSH_INSTALL_TARGET}/bin - chmod 755 ${BOSH_INSTALL_TARGET}/bin/socat -popd - -echo "Unpacking HAproxy..." -tar xf haproxy/haproxy-${HAPROXY_VERSION}.tar.gz -pushd haproxy-${HAPROXY_VERSION} - if [ -f ../haproxy/patches.tar.gz ]; then - mkdir -p ${BOSH_INSTALL_TARGET}/applied-patches - tar xf ../haproxy/patches.tar.gz - - for patchfile in haproxy-patches/*.patch; do - echo "Applying patch file ${patchfile}" - - # Conservatively limit patch fuzz factor to 0 to reduce chance of faulty patch - patch -F 0 -p0 < ${patchfile} - - # Save patches in install target for inspection later - cp ${patchfile} ${BOSH_INSTALL_TARGET}/applied-patches - done - - rm -r haproxy-patches - fi - - echo "Installing HAproxy..." - make TARGET=linux-glibc USE_PROMEX=1 USE_OPENSSL=1 USE_PCRE2=1 USE_PCRE2_JIT=yes USE_STATIC_PCRE2=1 USE_ZLIB=1 PCRE2DIR=${BOSH_INSTALL_TARGET} USE_LUA=1 LUA_LIB=${BOSH_INSTALL_TARGET}/lib LUA_INC=${BOSH_INSTALL_TARGET}/include - cp haproxy ${BOSH_INSTALL_TARGET}/bin/ - chmod 755 ${BOSH_INSTALL_TARGET}/bin/haproxy -popd - -echo "Installing hatop..." -cp haproxy/hatop-${HATOP_VERSION} ${BOSH_INSTALL_TARGET}/bin/hatop -chmod 755 ${BOSH_INSTALL_TARGET}/bin/hatop -cp hatop-wrapper ${BOSH_INSTALL_TARGET}/ -chmod 755 ${BOSH_INSTALL_TARGET}/hatop-wrapper diff --git a/scripts/dev-build.sh b/scripts/dev-build.sh new file mode 100755 index 000000000..a89901b75 --- /dev/null +++ b/scripts/dev-build.sh @@ -0,0 +1,113 @@ +#!/usr/bin/env bash +# +# dev-build.sh +# +# Builds the HAProxy release locally and uploads it to the BOSH director. +# +# Usage: ./scripts/dev-build.sh [--upload-only] [--version BASE] [--output-dir DIR] +## +# Versioning: +# Each invocation produces a release version of the form +dev-awslc-. +# e.g. "16.9.0+dev-awslc-4.0.0.1779286286". BASE defaults to the highest existing final release in +# releases/haproxy/ and can be overridden with --version. Each run uses a fresh timestamp. +# +# Examples: +# ./scripts/dev-build.sh # build and upload, BASE=highest final release +# ./scripts/dev-build.sh --version 17.0.0 # same, and override BASE +# ./scripts/dev-build.sh --upload-only # no build, only upload everything in builds/ + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "$SCRIPT_DIR/.." + +# Pull in AWS_LC_VERSION so it can be embedded in the dev version +# shellcheck disable=SC1091 +source src/aws-lc-versions.sh + +UPLOAD_ONLY=false +BASE_VERSION="" +OUTPUT_DIR="./builds" + +while [[ $# -gt 0 ]]; do + case "$1" in + --upload-only) + UPLOAD_ONLY=true + shift + ;; + --version) + BASE_VERSION="$2" + shift 2 + ;; + --output-dir) + OUTPUT_DIR="$2" + shift 2 + ;; + *) + echo "Unknown argument: $1" >&2 + exit 1 + ;; + esac +done + +# Derive base version from the highest existing final release if --version not given. +if [[ -z "$BASE_VERSION" ]]; then + BASE_VERSION=$( + ls releases/haproxy/haproxy-*.yml 2>/dev/null \ + | sed 's|.*/haproxy-||;s|\.yml$||;s|+.*||' \ + | sort -V \ + | tail -1 + ) + BASE_VERSION="${BASE_VERSION:-0.0.0}" +fi + +TIMESTAMP=$(date +%s) +mkdir -p "$OUTPUT_DIR" + +if [[ "$UPLOAD_ONLY" == false ]]; then + VERSION="${BASE_VERSION}+dev-awslc-${AWS_LC_VERSION}.${TIMESTAMP}" + TARBALL="$OUTPUT_DIR/haproxy-${VERSION}.tgz" + + echo "" + echo "========================================" + echo " Building: haproxy (version: $VERSION)" + echo "========================================" + echo "" + + bosh -n create-release --force \ + --name "haproxy" \ + --version "$VERSION" \ + --tarball "$TARBALL" + + echo " -> $TARBALL" +fi + +# --- Upload release(s) --- +echo "" +echo "========================================" +echo " Uploading releases to BOSH director" +echo "========================================" +echo "" + +if [[ "$UPLOAD_ONLY" == true ]]; then + upload_glob="$OUTPUT_DIR/haproxy-*.tgz" +else + upload_glob="$OUTPUT_DIR/haproxy-*.${TIMESTAMP}.tgz" +fi + +shopt -s nullglob +uploaded=0 +for tgz in $upload_glob; do + echo "Uploading: $tgz" + bosh upload-release "$tgz" --fix + uploaded=$((uploaded + 1)) +done +shopt -u nullglob + +if [[ $uploaded -eq 0 ]]; then + echo "No tarballs matching '$upload_glob' to upload." >&2 + exit 1 +fi + +echo "" +echo "Done." diff --git a/spec/haproxy/templates/haproxy_wrapper_spec.rb b/spec/haproxy/templates/haproxy_wrapper_spec.rb new file mode 100644 index 000000000..ae6baefb6 --- /dev/null +++ b/spec/haproxy/templates/haproxy_wrapper_spec.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +require 'rspec' + +# The wrapper composes the HAProxy binary package from two orthogonal properties: +# ha_proxy.ssl_variant openssl (default) | awslc | awslc-fips +# ha_proxy.include_haproxy_patches false (default) | true -> append "-patched" +# yielding /var/vcap/packages/haproxy-[-patched]. The package name is composed +# at render time, so these specs assert the rendered path + the 3-value validation. +describe 'bin/haproxy_wrapper' do + let(:template) { haproxy_job.template('bin/haproxy_wrapper') } + + def render(props) + template.render({ 'ha_proxy' => props }) + end + + it 'defaults to haproxy-openssl when nothing is set' do + expect(render({})).to include('HAPROXY_PKG="/var/vcap/packages/haproxy-openssl"') + end + + it 'selects the plain variant package' do + expect(render({ 'ssl_variant' => 'awslc' })).to include('HAPROXY_PKG="/var/vcap/packages/haproxy-awslc"') + expect(render({ 'ssl_variant' => 'awslc-fips' })).to include('HAPROXY_PKG="/var/vcap/packages/haproxy-awslc-fips"') + end + + it 'appends -patched when include_haproxy_patches is true' do + expect(render({ 'ssl_variant' => 'awslc', 'include_haproxy_patches' => true })) + .to include('HAPROXY_PKG="/var/vcap/packages/haproxy-awslc-patched"') + end + + it 'patches the openssl default when only include_haproxy_patches is set' do + expect(render({ 'include_haproxy_patches' => true })) + .to include('HAPROXY_PKG="/var/vcap/packages/haproxy-openssl-patched"') + end + + it 'rejects an invalid ssl_variant' do + out = render({ 'ssl_variant' => 'bogus' }) + expect(out).to include("ha_proxy.ssl_variant='bogus' is not a valid SSL variant") + expect(out).to include('exit 1') + end + + it 'exports the selected package onto PATH' do + expect(render({})).to include('export PATH=$PATH:${HAPROXY_PKG}/bin') + end +end diff --git a/src/apply-patches.sh b/src/apply-patches.sh new file mode 100644 index 000000000..06bbc3d9d --- /dev/null +++ b/src/apply-patches.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# Applies the HAProxy patches shipped as package source files (src/haproxy-patches/*.patch). +# +# Sourced (not executed) by the patched variant packaging scripts from within +# the unpacked HAProxy source tree, so patches apply to the current directory. +# Requires: +# PKG_ROOT - the package compile root (where haproxy-patches/ lives) +# BOSH_INSTALL_TARGET - set by BOSH; applied patches are copied here for inspection +mkdir -p ${BOSH_INSTALL_TARGET}/applied-patches + +for patchfile in ${PKG_ROOT}/haproxy-patches/*.patch; do + echo "Applying patch file ${patchfile}" + # Conservatively limit patch fuzz factor to 0 to reduce chance of faulty patch + patch -F 0 -p0 < ${patchfile} + # Save patches in install target for inspection later + cp ${patchfile} ${BOSH_INSTALL_TARGET}/applied-patches +done diff --git a/src/aws-lc-versions.sh b/src/aws-lc-versions.sh new file mode 100644 index 000000000..eb92d7574 --- /dev/null +++ b/src/aws-lc-versions.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# Version definitions for the AWS-LC / crypto packages (cmake, aws-lc, aws-lc-fips). +# +# Kept separate from haproxy-versions.sh on purpose: BOSH fingerprints each +# package by the contents of the files it bundles. If these versions lived in +# haproxy-versions.sh (which the crypto packages would then bundle), every +# HAProxy/lua/pcre/socat/hatop bump would change the crypto packages' +# fingerprints and force an unnecessary AWS-LC recompile With this +# split, the aws-lc/cmake fingerprints only change when a crypto version does, +# so the director reuses the cached compiled artifact across HAProxy bumps. +# Maintained automatically by ci/scripts/autobump-dependencies.py — do not edit by hand. +AWS_LC_VERSION=4.0.0 # https://github.com/aws/aws-lc/archive/refs/tags/v4.0.0.tar.gz (LTS line; builds both FIPS and non-FIPS) +CMAKE_VERSION=4.4.3 # https://github.com/Kitware/CMake/releases/download/v4.4.3/cmake-4.4.3-linux-x86_64.tar.gz +GOLANG_VERSION=1.27.1 # https://go.dev/dl/go1.27.1.linux-amd64.tar.gz diff --git a/src/build-haproxy.sh b/src/build-haproxy.sh new file mode 100644 index 000000000..7f02b07b8 --- /dev/null +++ b/src/build-haproxy.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Shared HAProxy compile, sourced by every haproxy- packaging script. +# +# The caller sets SSL_MAKE_FLAGS (the only thing that differs between SSL +# backends) before sourcing this. Bundled HAProxy patches are applied when the +# package ships patch files under haproxy-patches/ — that presence is the ONLY +# thing distinguishing a -patched package from its twin, so the -patched and base +# packaging scripts are identical and this logic lives in one place. +# +# Requires, with the package root as CWD: +# HAPROXY_VERSION from haproxy-versions.sh (sourced by the caller) +# SSL_MAKE_FLAGS e.g. "USE_OPENSSL=1" or +# "USE_OPENSSL_AWSLC=1 SSL_INC=/include SSL_LIB=/lib" +PKG_ROOT=$(pwd) +DEPS_DIR=/var/vcap/packages/haproxy-deps + +mkdir -p ${BOSH_INSTALL_TARGET}/bin + +echo "Unpacking HAproxy..." +tar xf haproxy/haproxy-${HAPROXY_VERSION}.tar.gz +pushd haproxy-${HAPROXY_VERSION} + # -patched packages ship patch files under haproxy-patches/; apply them when present. + if ls ${PKG_ROOT}/haproxy-patches/*.patch >/dev/null 2>&1; then + source ${PKG_ROOT}/apply-patches.sh + fi + + echo "Installing HAproxy..." + make TARGET=linux-glibc USE_PROMEX=1 ${SSL_MAKE_FLAGS} \ + USE_PCRE2=1 USE_PCRE2_JIT=yes USE_STATIC_PCRE2=1 USE_ZLIB=1 \ + PCRE2DIR=${DEPS_DIR} \ + USE_LUA=1 LUA_LIB=${DEPS_DIR}/lib LUA_INC=${DEPS_DIR}/include + cp haproxy ${BOSH_INSTALL_TARGET}/bin/ + chmod 755 ${BOSH_INSTALL_TARGET}/bin/haproxy +popd diff --git a/haproxy-patches/disable-rfc-6455-compliance.patch b/src/haproxy-patches/disable-rfc-6455-compliance.patch similarity index 100% rename from haproxy-patches/disable-rfc-6455-compliance.patch rename to src/haproxy-patches/disable-rfc-6455-compliance.patch diff --git a/src/haproxy-versions.sh b/src/haproxy-versions.sh new file mode 100644 index 000000000..ec90ebc60 --- /dev/null +++ b/src/haproxy-versions.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# Version definitions for HAProxy and its bundled dependencies (lua, pcre, +# socat, hatop). Sourced by the haproxy binary/deps packaging scripts. +# +# AWS-LC / cmake / golang versions live in src/aws-lc-versions.sh instead, so +# that bumping anything here does not re-fingerprint the crypto packages and +# trigger an unnecessary AWS-LC recompile. Consumers that need everything (the +# monolithic packages/haproxy/packaging and the CI scripts) source both files. +# Maintained automatically by ci/scripts/autobump-dependencies.py — do not edit by hand. +HAPROXY_VERSION=3.2.26 # https://www.haproxy.org/download/3.2/src/haproxy-3.2.26.tar.gz +LUA_VERSION=5.4.9 # https://www.lua.org/ftp/lua-5.4.9.tar.gz +PCRE_VERSION=10.49 # https://github.com/PCRE2Project/pcre2/releases/download/pcre2-10.49/pcre2-10.49.tar.gz +SOCAT_VERSION=1.8.1.3 # http://www.dest-unreach.org/socat/download/socat-1.8.1.3.tar.gz +HATOP_VERSION=0.8.2 # https://github.com/jhunt/hatop/releases/download/v0.8.2/hatop From 48ad09c74d76ba22e1d94b3f3de5b6e2dee27a6f Mon Sep 17 00:00:00 2001 From: Clemens Hoffmann Date: Fri, 23 Oct 2026 12:12:23 +0200 Subject: [PATCH 2/5] Fix missing update of hatop location in hatop-wrapper --- src/hatop-wrapper | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/hatop-wrapper b/src/hatop-wrapper index 5e1a5ccd0..7bda776ab 100644 --- a/src/hatop-wrapper +++ b/src/hatop-wrapper @@ -1,9 +1,9 @@ #!/bin/bash if [[ -z "${1:-}" ]]; then if [[ -e /var/vcap/sys/run/haproxy/stats.sock ]]; then - exec /var/vcap/packages/haproxy/bin/hatop -s /var/vcap/sys/run/haproxy/stats.sock + exec /var/vcap/packages/haproxy-deps/bin/hatop -s /var/vcap/sys/run/haproxy/stats.sock elif [[ -e /var/vcap/sys/run/haproxy/stats1.sock ]]; then - exec /var/vcap/packages/haproxy/bin/hatop -s /var/vcap/sys/run/haproxy/stats1.sock + exec /var/vcap/packages/haproxy-deps/bin/hatop -s /var/vcap/sys/run/haproxy/stats1.sock fi fi -exec /var/vcap/packages/haproxy/bin/hatop "$@" +exec /var/vcap/packages/haproxy-deps/bin/hatop "$@" From a49c9bae87fe9fe2c86302e118368b6e62aa9240 Mon Sep 17 00:00:00 2001 From: Clemens Hoffmann Date: Fri, 23 Oct 2026 12:28:04 +0200 Subject: [PATCH 3/5] Remove ttar submodule, move to haoproxy-boshrelease --- .gitmodules | 3 - ci/scripts/functions-ci.sh | 3 - ci/scripts/shipit | 2 - jobs/haproxy/spec | 1 - jobs/haproxy/templates/haproxy_wrapper.erb | 2 +- packages/README.md | 2 +- packages/haproxy-deps/packaging | 4 ++ packages/haproxy-deps/spec | 1 + packages/ttar/packaging | 11 --- packages/ttar/spec | 5 -- src/ttar | 1 - src/ttar/LICENSE | 21 ++++++ src/ttar/README.md | 80 ++++++++++++++++++++++ src/ttar/ttar | 34 +++++++++ 14 files changed, 142 insertions(+), 28 deletions(-) delete mode 100644 .gitmodules delete mode 100644 packages/ttar/packaging delete mode 100644 packages/ttar/spec delete mode 160000 src/ttar create mode 100644 src/ttar/LICENSE create mode 100644 src/ttar/README.md create mode 100755 src/ttar/ttar diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index 2f70e4059..000000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "src/ttar"] - path = src/ttar - url = https://github.com/jhunt/ttar.git diff --git a/ci/scripts/functions-ci.sh b/ci/scripts/functions-ci.sh index 5d68f567b..616ce77f2 100755 --- a/ci/scripts/functions-ci.sh +++ b/ci/scripts/functions-ci.sh @@ -44,10 +44,7 @@ function build_image() { function git_pull() { cd "${REPO_ROOT:?required}" - echo "----- Pulling in any git submodules..." git config --global --add safe.directory /repo - git config --global --add safe.directory /repo/src/ttar - git submodule update --init --recursive --force } function bosh_release() { diff --git a/ci/scripts/shipit b/ci/scripts/shipit index 7daa130c5..19286ef6e 100755 --- a/ci/scripts/shipit +++ b/ci/scripts/shipit @@ -59,8 +59,6 @@ blobstore: ${PADDED_GCP_SERVICE_KEY} YAML -header "Pulling in any git submodules..." -git submodule update --init --recursive --force cd - version() { diff --git a/jobs/haproxy/spec b/jobs/haproxy/spec index 6c49a6506..2afc912d3 100644 --- a/jobs/haproxy/spec +++ b/jobs/haproxy/spec @@ -11,7 +11,6 @@ packages: - haproxy-awslc-patched - haproxy-awslc-fips - haproxy-awslc-fips-patched -- ttar templates: haproxy_wrapper.erb: bin/haproxy_wrapper diff --git a/jobs/haproxy/templates/haproxy_wrapper.erb b/jobs/haproxy/templates/haproxy_wrapper.erb index 5e8db6463..fbbb22c49 100755 --- a/jobs/haproxy/templates/haproxy_wrapper.erb +++ b/jobs/haproxy/templates/haproxy_wrapper.erb @@ -28,7 +28,7 @@ if [ ! -x "${HAPROXY_PKG}/bin/haproxy" ]; then exit 1 fi -export PATH=$PATH:${HAPROXY_PKG}/bin:/var/vcap/packages/ttar/bin +export PATH=$PATH:${HAPROXY_PKG}/bin:/var/vcap/packages/haproxy-deps/bin CONFIG=/var/vcap/jobs/haproxy/config/haproxy.config PID_FILE=/var/vcap/sys/run/haproxy/haproxy.pid DRAIN_LOCK=/var/vcap/sys/run/haproxy/drain.lock diff --git a/packages/README.md b/packages/README.md index cc1428847..70b695cb5 100644 --- a/packages/README.md +++ b/packages/README.md @@ -4,7 +4,7 @@ This directory holds **every** package in the release. `bosh create-release` bun plain `bosh create-release` of the committed tree produces the one and only release — there is no build-time assembly. -- `haproxy-deps`, `ttar`, `keepalived` — shared, compiled once +- `haproxy-deps` (lua, pcre2, socat, hatop, and the `ttar` helper), `keepalived` — shared, compiled once - `haproxy-openssl`, `haproxy-openssl-patched`, `haproxy-awslc`, `haproxy-awslc-patched`, `haproxy-awslc-fips`, `haproxy-awslc-fips-patched` — the six HAProxy binaries: each SSL variant (openssl, awslc, awslc-fips) built with and without the community patches diff --git a/packages/haproxy-deps/packaging b/packages/haproxy-deps/packaging index 6c2ac8764..fafc8d72d 100644 --- a/packages/haproxy-deps/packaging +++ b/packages/haproxy-deps/packaging @@ -34,3 +34,7 @@ cp haproxy/hatop-${HATOP_VERSION} ${BOSH_INSTALL_TARGET}/bin/hatop chmod 755 ${BOSH_INSTALL_TARGET}/bin/hatop cp hatop-wrapper ${BOSH_INSTALL_TARGET}/ chmod 755 ${BOSH_INSTALL_TARGET}/hatop-wrapper + +echo "Installing ttar..." +cp ttar/ttar ${BOSH_INSTALL_TARGET}/bin/ttar +chmod 755 ${BOSH_INSTALL_TARGET}/bin/ttar diff --git a/packages/haproxy-deps/spec b/packages/haproxy-deps/spec index b1bf0b687..118d5394f 100644 --- a/packages/haproxy-deps/spec +++ b/packages/haproxy-deps/spec @@ -6,4 +6,5 @@ files: - haproxy/lua-*.tar.gz - haproxy/hatop-* - hatop-wrapper +- ttar/ttar - haproxy-versions.sh diff --git a/packages/ttar/packaging b/packages/ttar/packaging deleted file mode 100644 index 69b2608d9..000000000 --- a/packages/ttar/packaging +++ /dev/null @@ -1,11 +0,0 @@ -set -e # exit immediately if a simple command exits with a non-zero status -set -u # report the usage of uninitialized variables - - # Available variables -# $BOSH_COMPILE_TARGET - where this package & spec'd source files are available -# $BOSH_INSTALL_TARGET - where you copy/install files to be included in package -export HOME=/var/vcap - -mkdir -p ${BOSH_INSTALL_TARGET}/bin -cp ttar/ttar ${BOSH_INSTALL_TARGET}/bin -chmod 0755 ${BOSH_INSTALL_TARGET}/bin/ttar diff --git a/packages/ttar/spec b/packages/ttar/spec deleted file mode 100644 index 2610de3a3..000000000 --- a/packages/ttar/spec +++ /dev/null @@ -1,5 +0,0 @@ ---- -name: ttar -dependencies: [] -files: - - ttar/ttar diff --git a/src/ttar b/src/ttar deleted file mode 160000 index b7b382ed3..000000000 --- a/src/ttar +++ /dev/null @@ -1 +0,0 @@ -Subproject commit b7b382ed3476b0e5bb5f1284e9e9cd2ac566804c diff --git a/src/ttar/LICENSE b/src/ttar/LICENSE new file mode 100644 index 000000000..d23bf3842 --- /dev/null +++ b/src/ttar/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2017 James Hunt + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to +deal in the Software without restriction, including without limitation the +rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +sell copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +IN THE SOFTWARE. diff --git a/src/ttar/README.md b/src/ttar/README.md new file mode 100644 index 000000000..5b84ed82e --- /dev/null +++ b/src/ttar/README.md @@ -0,0 +1,80 @@ +ttar - Tiny Text Archives +========================= + +> **Attribution:** `ttar` was written by **James Hunt** and is vendored into +> haproxy-boshrelease from [github.com/jhunt/ttar](https://github.com/jhunt/ttar) +> (previously a git submodule). It is licensed under the **MIT License, +> Copyright (c) 2017 James Hunt** — see the bundled [LICENSE](./LICENSE). The script +> itself is byte-identical to upstream `master`. + +`ttar` is a small utility that reads specially crafted "text +archives" which contain the smashed together contents of other +files, at different paths. + +Each file is preceded by a file header that looks like this: + +``` +========================================= [] +``` + +At least three equal signs must be present. More can be provided +if it makes the file easier to read. + +The `` parameter is optional. If present, the file will be +chmoded to that (octal) mode. + +The `` parameter is required, and must be an absolute path +starting with '/'. + +But Why? +-------- + +Seriously. Why do this? + +I needed to be able to use a single ERB template in a +[BOSH][bosh] release, but somehow generate an arbitrary number of +files based on configuration. An oddly specific problem, that +demanded a small, oddly-specific program. + +Example +------- + +Here's a full example: + +``` +=================== /etc/named/10.in-addr.arpa +$TTL 1h +$ORIGIN 10.in-addr.arpa. +@ IN SOA foo.bl. hostmaster.foo.bl. ( + 2015082504 ; serial + 15m ; slaves should check back with us every 15min + 20 ; slaves should retry (on failure) after 20sec + 4h ; slaves stop being authoritative after 4hr without updates + 1h ; other NSes should cache our results for 1h + ) + + IN NS ns1.foo.bl. + IN NS ns2.foo.bl. + +11.0.0 IN PTR ns1.foo.bl. +12.0.0 IN PTR ns2.foo.bl. +42.0.0 IN PTR bar.foo.bl. +=================== /etc/named/foo.bl.db +$TTL 1h +$ORIGIN foo.bl. +@ IN SOA foo.bl. hostmaster.foo.bl. ( + 2015082505 ; serial + 15m ; slaves should check back with us every 15min + 20 ; slaves should retry (on failure) after 20sec + 4h ; slaves stop being authoritative after 4hr without updates + 1h ; other NSes should cache our results for 1h + ) + + IN NS ns1.foo.bl. + IN NS ns2.foo.bl. + + IN A 10.0.0.42 ; bar +bar IN A 10.0.0.42 +``` + +[bosh]: https://bosh.io diff --git a/src/ttar/ttar b/src/ttar/ttar new file mode 100755 index 000000000..1579b66e9 --- /dev/null +++ b/src/ttar/ttar @@ -0,0 +1,34 @@ +#!/usr/bin/perl +use strict; +use warnings; + +sub mkdir_p { + for my $path (map { (my $t = $_) =~ s|^/*||; $t } @_) { + my $parent = '/'; + for (split '/', $path) { + mkdir $parent unless -d $parent; + $parent = "${parent}${_}/"; + } + } +} + +my $fh; +while (<>) { + if (m|^={3,} (/.+)$|) { + close $fh if $fh; + mkdir_p("$1"); + print STDERR "$1\n"; + open $fh, ">", $1; + + } elsif (m|^={3,} ([0-7]{3,}) (/.+)$|) { + close $fh if $fh; + mkdir_p("$2"); + print STDERR "$2\n"; + open $fh, ">", $2; + chmod oct($1), $2; + + } elsif ($fh) { + print $fh $_; + } +} +close $fh if $fh; From 14aa7b5ca45ec359fb0b7591ac28cbe852b593e5 Mon Sep 17 00:00:00 2001 From: Clemens Hoffmann Date: Fri, 23 Oct 2026 13:27:48 +0200 Subject: [PATCH 4/5] Include package dependency tree, drop redundant packages readme --- README.md | 8 +++++++- packages/README.md | 20 -------------------- 2 files changed, 7 insertions(+), 21 deletions(-) delete mode 100644 packages/README.md diff --git a/README.md b/README.md index 445d98f4b..573a90233 100644 --- a/README.md +++ b/README.md @@ -110,4 +110,10 @@ only by `-DFIPS=1` plus the Go toolchain for the integrity step). Because `aws-l bundle the HAProxy version file, their fingerprints are independent of HAProxy — a HAProxy bump reuses the cached AWS-LC compile instead of rebuilding it. The HAProxy community patches (the optional `-patched` builds) are plain source files under `src/haproxy-patches/`, applied at compile time by -`src/apply-patches.sh`. \ No newline at end of file +`src/apply-patches.sh`. + +### Package dependency diagram + +![haproxy-boshrelease package dependency graph](docs/package-dependencies.png) + +Source: [`docs/package-dependencies.drawio`](docs/package-dependencies.drawio) (editable in [diagrams.net](https://app.diagrams.net/)). \ No newline at end of file diff --git a/packages/README.md b/packages/README.md deleted file mode 100644 index 70b695cb5..000000000 --- a/packages/README.md +++ /dev/null @@ -1,20 +0,0 @@ -# `packages/` — the complete release - -This directory holds **every** package in the release. `bosh create-release` bundles all of them, so a -plain `bosh create-release` of the committed tree produces the one and only release — there is no -build-time assembly. - -- `haproxy-deps` (lua, pcre2, socat, hatop, and the `ttar` helper), `keepalived` — shared, compiled once -- `haproxy-openssl`, `haproxy-openssl-patched`, `haproxy-awslc`, `haproxy-awslc-patched`, - `haproxy-awslc-fips`, `haproxy-awslc-fips-patched` — the six HAProxy binaries: each SSL - variant (openssl, awslc, awslc-fips) built with and without the community patches -- `aws-lc-src` (shared source), `aws-lc`, `aws-lc-fips`, `cmake` — crypto build dependencies - -The SSL backend is chosen at **deploy time** via the `ha_proxy.ssl_variant` property (default -`openssl`); see `jobs/haproxy/templates/haproxy_wrapper.erb`. Every deploy compiles all bundled -binaries; switching backends later is a property change + redeploy with no recompile. - -Dependencies are one-directional: every `haproxy-*` binary depends on `haproxy-deps`; `aws-lc` and -`aws-lc-fips` build from the shared `aws-lc-src` + `cmake`. HAProxy patches for the `-patched` -variants are plain source files under [`../src/haproxy-patches/`](../src/haproxy-patches/), applied at -compile time by `src/apply-patches.sh`. From 698b29bd089fcfed85b7bde3d010b891a413b82c Mon Sep 17 00:00:00 2001 From: Clemens Hoffmann Date: Fri, 23 Oct 2026 13:43:34 +0200 Subject: [PATCH 5/5] Fail early when invalid variant or patch parameter is provided --- jobs/haproxy/templates/haproxy.config.erb | 10 ++++++++++ .../global_and_default_options_spec.rb | 20 +++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/jobs/haproxy/templates/haproxy.config.erb b/jobs/haproxy/templates/haproxy.config.erb index 03d334e0c..121baf7cb 100644 --- a/jobs/haproxy/templates/haproxy.config.erb +++ b/jobs/haproxy/templates/haproxy.config.erb @@ -172,6 +172,16 @@ if ![true, false, "true", "false", "mtls_only"].include?(disable_domain_fronting abort("Unknown 'disable_domain_fronting' option: #{disable_domain_fronting}. Known options: true, false or 'mtls_only'") end +ssl_variant = p("ha_proxy.ssl_variant") +if !["openssl", "awslc", "awslc-fips"].include?(ssl_variant) + abort("Unknown 'ssl_variant' option: #{ssl_variant}. Known options: 'openssl', 'awslc', 'awslc-fips'") +end + +include_haproxy_patches = p("ha_proxy.include_haproxy_patches") +if ![true, false].include?(include_haproxy_patches) + abort("Unknown 'include_haproxy_patches' option: #{include_haproxy_patches}. Known options: true or false") +end + # }}} # IPv4 and IPv6 binding (v4v6) Option {{{ v4v6 = "" diff --git a/spec/haproxy/templates/haproxy_config/global_and_default_options_spec.rb b/spec/haproxy/templates/haproxy_config/global_and_default_options_spec.rb index a638ee33b..ae0506860 100644 --- a/spec/haproxy/templates/haproxy_config/global_and_default_options_spec.rb +++ b/spec/haproxy/templates/haproxy_config/global_and_default_options_spec.rb @@ -13,6 +13,26 @@ let(:properties) { {} } + context 'when ha_proxy.ssl_variant is an invalid value' do + let(:properties) { { 'ssl_variant' => 'openss' } } + + it 'aborts rendering with a meaningful error message' do + expect do + template.render({ 'ha_proxy' => properties }) + end.to raise_error(/Unknown 'ssl_variant' option: openss\. Known options: 'openssl', 'awslc', 'awslc-fips'/) + end + end + + context 'when ha_proxy.include_haproxy_patches is an invalid value' do + let(:properties) { { 'include_haproxy_patches' => 'nope' } } + + it 'aborts rendering with a meaningful error message' do + expect do + template.render({ 'ha_proxy' => properties }) + end.to raise_error(/Unknown 'include_haproxy_patches' option: nope\. Known options: true or false/) + end + end + it 'renders a valid haproxy template' do expect do HAProxy::Parser.new.parse(template.render({}))