Add AWS-LC support

.gitignore

@@ -6,6 +6,7 @@ releases/**/*.tgz
 ci/scripts/stemcell/*.tgz
 ci/scripts/stemcell-jammy/*.tgz
 dev_releases
+builds
 blobs/*
 .blobs
 .dev_builds
@@ -21,6 +22,9 @@ blobs/*
 tmp
 my*.yml
 .bundle
+.pytest_cache
+__pycache__
+*.pyc
 vendor
 .vscode/launch.json
 .vscode/settings.json

README.md

@@ -81,3 +81,39 @@ Unit/RSpec tests can also be debugged/stepped through when needed. See for examp
 ### Acceptance Tests
 
 See [acceptance-tests README](/acceptance-tests/README.md).
+
+### Local Dev Builds
+
+`scripts/dev-build.sh` builds the release tarball and uploads it to the targeted BOSH
+director. Useful for iterating on packaging changes without going through CI.
+
+```bash
+./scripts/dev-build.sh                       # build, version=dev
+./scripts/dev-build.sh --version 1.0         # custom version tag
+./scripts/dev-build.sh --upload-only         # re-upload previously built tarballs
+```
+
+Tarballs are written to `./builds/` and uploaded with `bosh upload-release --fix`.
+
+### Packaging model
+
+`packages/` holds **every** package, and `bosh create-release` bundles all of them, so a plain
+`bosh create-release` of the committed tree produces the one release — there is no build-time
+assembly. The `haproxy` job lists all six `haproxy-*` binaries; the SSL backend is chosen at deploy
+time via `ha_proxy.ssl_variant` (default `openssl`) and `ha_proxy.include_haproxy_patches` 
+(default `false`), handled in `jobs/haproxy/templates/haproxy_wrapper.erb`.
+
+Dependencies are one-directional: every `haproxy-*` binary depends on the shared `haproxy-deps`
+(lua, pcre2, socat, hatop), compiled once. AWS-LC is compiled from a single shared `aws-lc-src` source
+package used by both the non-FIPS `aws-lc` and the FIPS `aws-lc-fips` compile packages (FIPS differs
+only by `-DFIPS=1` plus the Go toolchain for the integrity step). Because `aws-lc`/`aws-lc-fips` do not
+bundle the HAProxy version file, their fingerprints are independent of HAProxy — a HAProxy bump reuses
+the cached AWS-LC compile instead of rebuilding it. The HAProxy community patches (the optional
+`-patched` builds) are plain source files under `src/haproxy-patches/`, applied at compile time by
+`src/apply-patches.sh`.
+
+### Package dependency diagram
+
+![haproxy-boshrelease package dependency graph](docs/package-dependencies.png)
+
+Source: [`docs/package-dependencies.drawio`](docs/package-dependencies.drawio) (editable in [diagrams.net](https://app.diagrams.net/)).

acceptance-tests/README.md

@@ -12,6 +12,29 @@ cd acceptance-tests
 ./run-local.sh
 ```
 
+### TLS Backend Variant
+
+There is a single release that bundles every SSL variant. By default, the tests deploy it with the
+system OpenSSL backend. To exercise a different TLS backend, set the `VARIANT` environment variable —
+it drives the `ha_proxy.ssl_variant` deployment property. The HAProxy patches are toggled with
+`INCLUDE_HAPROXY_PATCHES` (which drives `ha_proxy.include_haproxy_patches`). Examples:
+
+```shell
+VARIANT=awslc                              ./run-local.sh   # AWS-LC, non-FIPS
+VARIANT=awslc-fips                         ./run-local.sh   # AWS-LC FIPS
+INCLUDE_HAPROXY_PATCHES=true               ./run-local.sh   # system OpenSSL + HAProxy patches
+VARIANT=awslc      INCLUDE_HAPROXY_PATCHES=true ./run-local.sh   # AWS-LC + patches
+VARIANT=awslc-fips INCLUDE_HAPROXY_PATCHES=true ./run-local.sh   # AWS-LC FIPS + patches
+```
+
+`VARIANT` accepts `openssl` (default), `awslc`, or `awslc-fips`.
+
+A single invocation runs the suite against the chosen variant.
+
+Each run builds the release from scratch, compiling every bundled binary (AWS-LC and its FIPS toolchain included) inside the BOSH compilation VM.
+
+When using `-k` (keep BOSH running, see below) the cached state belongs to whichever variant ran last — switching `VARIANT` inside a kept container is not supported. Stop the container between variants, or run without `-k`.
+
 ### Running on Docker for Mac
 
 Acceptance tests cannot be run on Mac with arm64 architecture:

acceptance-tests/acceptance_tests_suite_test.go

@@ -226,8 +226,21 @@ func expectTLSCertificateRequiredErr(err error) {
 	checkNetOpErr(err, "tls: certificate required")
 }
 
-func expectTLSUnrecognizedNameErr(err error) {
-	checkNetOpErr(err, "tls: unrecognized name")
+// expectStrictSNIRejectedErr asserts that a strict-sni handshake was rejected.
+//   - OpenSSL registers the SNI reject through the servername callback and sends
+//     unrecognized_name(112)  => "tls: unrecognized name"
+//   - AWS-LC uses the select_certificate_cb path, which sends
+//     handshake_failure(40)    => "tls: handshake failure"
+func expectStrictSNIRejectedErr(err error) {
+	msg := netOpErrString(err)
+	if strings.HasPrefix(config.Variant, "awslc") {
+		Expect(msg).To(Or(
+			ContainSubstring("tls: handshake failure"),
+			ContainSubstring("tls: unrecognized name"),
+		))
+		return
+	}
+	Expect(msg).To(ContainSubstring("tls: unrecognized name"))
 }
 
 func expectConnectionRefusedErr(err error) {
@@ -239,11 +252,18 @@ func expectConnectionResetErr(err error) {
 }
 
 func checkNetOpErr(err error, expectString string) {
+	Expect(netOpErrString(err)).To(ContainSubstring(expectString))
+}
+
+// netOpErrString unwraps an http client error down to the underlying
+// *net.OpError and returns its message, failing the test if the error is not
+// shaped that way.
+func netOpErrString(err error) string {
 	Expect(err).To(HaveOccurred())
 	urlErr, ok := err.(*url.Error)
 	Expect(ok).To(BeTrue())
 	tlsErr := urlErr.Unwrap()
 	var opErr *net.OpError
 	Expect(errors.As(tlsErr, &opErr)).To(BeTrue())
-	Expect(opErr.Err.Error()).To(ContainSubstring(expectString))
+	return opErr.Err.Error()
 }

acceptance-tests/bosh_helpers.go

@@ -93,6 +93,21 @@ var opsfileConfigureSSH string = `---
 var defaultOpsfiles = []string{opsfileChangeName, opsfileChangeVersion, opsfileConfigureSSH}
 var defaultSSHUser string = "ginkgo"
 
+// The release bundles every SSL variant; this selects one at deploy time.
+var opsfileSSLVariant string = `---
+# Select the SSL variant at deploy time (the release bundles all of them)
+- type: replace
+  path: /instance_groups/name=haproxy/jobs/name=haproxy/properties/ha_proxy/ssl_variant?
+  value: ((ssl-variant))
+`
+
+// Orthogonal to the SSL variant: run the patched HAProxy build.
+var opsfileIncludePatches string = `---
+- type: replace
+  path: /instance_groups/name=haproxy/jobs/name=haproxy/properties/ha_proxy/include_haproxy_patches?
+  value: ((include-haproxy-patches))
+`
+
 func buildManifestVars(baseManifestVars baseManifestVars, customVars map[string]interface{}) map[string]interface{} {
 	vars := map[string]interface{}{
 		"release-version":         config.ReleaseVersion,
@@ -146,6 +161,21 @@ func buildHAProxyInfo(baseManifestVars baseManifestVars, varsStoreReader varsSto
 func deployHAProxy(baseManifestVars baseManifestVars, customOpsfiles []string, customVars map[string]interface{}, expectSuccess bool) (haproxyInfo, varsStoreReader) {
 	manifestVars := buildManifestVars(baseManifestVars, customVars)
 	opsfiles := append(defaultOpsfiles, customOpsfiles...)
+
+	// The release bundles every SSL variant; select the one under test at deploy time.
+	// config.Variant comes from the VARIANT env (default "openssl"); openssl is the job's
+	// own default, so we only inject the property for the non-default variants.
+	if config.Variant != "" && config.Variant != "openssl" {
+		opsfiles = append(opsfiles, opsfileSSLVariant)
+		manifestVars["ssl-variant"] = config.Variant
+	}
+
+	// INCLUDE_HAPROXY_PATCHES selects the patched build (orthogonal to the SSL variant).
+	if config.IncludePatches {
+		opsfiles = append(opsfiles, opsfileIncludePatches)
+		manifestVars["include-haproxy-patches"] = config.IncludePatches
+	}
+
 	cmd, varsStoreReader := deployBaseManifestCmd(baseManifestVars.deploymentName, opsfiles, manifestVars)
 
 	dumpCmd(cmd)
@@ -332,3 +362,11 @@ func runHAProxySocketCommand(haproxyInfo haproxyInfo, command string) string {
 	return strings.TrimSpace(stdout)
 }
 
+// runningHAProxyVersionInfo returns `haproxy -vv` for the HAProxy binary that is
+// actually running on the VM.
+func runningHAProxyVersionInfo(haproxyInfo haproxyInfo) string {
+	cmd := `sudo sh -c 'exe=$(readlink -f /proc/$(pgrep -x -n haproxy)/exe); "$exe" -vv'`
+	stdout, _, err := runOnRemote(haproxyInfo.SSHUser, haproxyInfo.PublicIP, haproxyInfo.SSHPrivateKey, cmd)
+	Expect(err).NotTo(HaveOccurred())
+	return strings.TrimSpace(stdout)
+}

acceptance-tests/config.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"os"
 	"os/exec"
+	"strconv"
 )
 
 var config Config
@@ -18,6 +19,8 @@ type Config struct {
 	BoshPath         string `json:"boshPath"`
 	BaseManifestPath string `json:"baseManifestPath"`
 	HomePath         string `json:"homePath"`
+	Variant          string `json:"variant"`
+	IncludePatches   bool   `json:"includePatches"`
 }
 
 func loadConfig() (Config, error) {
@@ -67,6 +70,15 @@ func loadConfig() (Config, error) {
 		return Config{}, err
 	}
 
+	// VARIANT selects the SSL library (openssl|awslc|awslc-fips); empty means openssl.
+	variant := os.Getenv("VARIANT")
+	if variant == "" {
+		variant = "openssl"
+	}
+
+	// INCLUDE_HAPROXY_PATCHES toggles the patched build (orthogonal to VARIANT).
+	includePatches, _ := strconv.ParseBool(os.Getenv("INCLUDE_HAPROXY_PATCHES"))
+
 	return Config{
 		ReleaseRepoPath:  releaseRepoPath,
 		ReleaseVersion:   releaseVersion,
@@ -77,6 +89,8 @@ func loadConfig() (Config, error) {
 		BoshPath:         boshPath,
 		BaseManifestPath: baseManifestPath,
 		HomePath:         homePath,
+		Variant:          variant,
+		IncludePatches:   includePatches,
 	}, nil
 }
 

acceptance-tests/lua_test.go

@@ -16,7 +16,7 @@ var _ = Describe("Lua scripting", func() {
 # Enable Lua scripting
 - type: replace
   path: /instance_groups/name=haproxy/jobs/name=haproxy/properties/ha_proxy/lua_scripts?/-
-  value: "/var/vcap/packages/haproxy/lua_test.lua"
+  value: "/var/vcap/packages/haproxy-deps/lua_test.lua"
 - type: replace
   path: /instance_groups/name=haproxy/jobs/name=haproxy/properties/ha_proxy/frontend_config?
   value: "http-request use-service lua.lua_test if { path /lua_test }"
@@ -53,7 +53,7 @@ core.register_service("lua_test", "http", lua_test)
 		Expect(boshInstances(deploymentNameForTestNode())[0].ProcessState).To(Or(Equal("failing"), Equal("unresponsive agent")))
 
 		// upload Lua script file
-		uploadFile(haproxyInfo, strings.NewReader(replyLuaContent), "/var/vcap/packages/haproxy/lua_test.lua")
+		uploadFile(haproxyInfo, strings.NewReader(replyLuaContent), "/var/vcap/packages/haproxy-deps/lua_test.lua")
 
 		closeLocalServer, localPort := startDefaultTestServer()
 		defer closeLocalServer()

acceptance-tests/run-local.sh

@@ -94,9 +94,9 @@ if [ -n "$KEEP_RUNNING" ] ; then
   echo
   echo "*** KEEP_RUNNING enabled. Please clean up docker scratch after removing containers: ${DOCKER_SCRATCH}"
   echo
-  docker run --privileged -v "$REPO_DIR":/repo -v "${DOCKER_SCRATCH}":/scratch/docker -e REPO_ROOT=/repo -e FOCUS="${FOCUS}" -e PARALLELISM="${PARALLELISM}" -e KEEP_RUNNING="${KEEP_RUNNING}" haproxy-boshrelease-testflight bash -c "cd /repo/ci/scripts && ./acceptance-tests ; sleep infinity"
+  docker run --privileged -v "$REPO_DIR":/repo -v "${DOCKER_SCRATCH}":/scratch/docker -e REPO_ROOT=/repo -e FOCUS="${FOCUS}" -e PARALLELISM="${PARALLELISM}" -e KEEP_RUNNING="${KEEP_RUNNING}" -e VARIANT="${VARIANT:-}" haproxy-boshrelease-testflight bash -c "cd /repo/ci/scripts && ./acceptance-tests ; sleep infinity"
 else
-  docker run --rm --privileged -v "$REPO_DIR":/repo -v "${DOCKER_SCRATCH}":/scratch/docker -e REPO_ROOT=/repo -e KEEP_RUNNING="" -e PARALLELISM="${PARALLELISM}" haproxy-boshrelease-testflight bash -c "cd /repo/ci/scripts && ./acceptance-tests"
+  docker run --rm --privileged -v "$REPO_DIR":/repo -v "${DOCKER_SCRATCH}":/scratch/docker -e REPO_ROOT=/repo -e KEEP_RUNNING="" -e PARALLELISM="${PARALLELISM}" -e VARIANT="${VARIANT:-}" haproxy-boshrelease-testflight bash -c "cd /repo/ci/scripts && ./acceptance-tests"
   echo "Cleaning up docker scratch: ${DOCKER_SCRATCH}"
   sudo rm -rf "${DOCKER_SCRATCH}"
 fi

acceptance-tests/ssl_variant_test.go

@@ -0,0 +1,64 @@
+package acceptance_tests
+
+import (
+	"fmt"
+	"regexp"
+	"strings"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+// Verifies that the SSL backend actually running on the VM matches the variant
+// the release was assembled with (config.Variant, from the VARIANT env var).
+//
+// The signal is the running binary's `haproxy -vv`, which reports two dedicated
+// lines describing the library loaded at runtime (verified against real output
+// from AWS-LC 4.0.0 / OpenSSL 3.0.13):
+//
+//	variant      "Running on SSL library version : ..."   "SSL library FIPS mode : ..."
+//	openssl      OpenSSL 3.0.13 30 Jan 2024               (line absent)
+//	awslc        AWS-LC 4.0.0                             no
+//	awslc-fips   AWS-LC FIPS 4.0.0                        yes
+var (
+	runningSSLLibRe = regexp.MustCompile(`Running on SSL library version\s*:\s*(.+)`)
+	sslFIPSModeOnRe = regexp.MustCompile(`SSL library FIPS mode\s*:\s*yes`)
+)
+
+var _ = Describe("SSL variant", func() {
+	It("Runs the HAProxy binary for the configured SSL variant", func() {
+		haproxyInfo, _ := deployHAProxy(baseManifestVars{
+			haproxyBackendPort:    12000,
+			haproxyBackendServers: []string{"127.0.0.1"},
+			deploymentName:        deploymentNameForTestNode(),
+		}, []string{}, map[string]interface{}{}, true)
+
+		By("Reading the SSL library from the running HAProxy binary (haproxy -vv)")
+		versionInfo := runningHAProxyVersionInfo(haproxyInfo)
+		fmt.Fprintf(GinkgoWriter, "----- haproxy -vv (variant %q) -----\n%s\n", config.Variant, versionInfo)
+
+		match := runningSSLLibRe.FindStringSubmatch(versionInfo)
+		Expect(match).NotTo(BeNil(), "no 'Running on SSL library version' line in haproxy -vv:\n%s", versionInfo)
+		runningSSL := strings.TrimSpace(match[1])
+
+		isAWSLC := strings.Contains(runningSSL, "AWS-LC")
+		fipsMode := sslFIPSModeOnRe.MatchString(versionInfo)
+
+		switch {
+		case strings.HasPrefix(config.Variant, "awslc-fips"):
+			By("Expecting AWS-LC FIPS")
+			Expect(isAWSLC).To(BeTrue(), "expected AWS-LC, running SSL library was %q", runningSSL)
+			Expect(fipsMode).To(BeTrue(), "expected FIPS mode on; haproxy -vv was:\n%s", versionInfo)
+		case strings.HasPrefix(config.Variant, "awslc"):
+			By("Expecting AWS-LC (non-FIPS)")
+			Expect(isAWSLC).To(BeTrue(), "expected AWS-LC, running SSL library was %q", runningSSL)
+			Expect(fipsMode).To(BeFalse(), "expected FIPS mode off; haproxy -vv was:\n%s", versionInfo)
+		default:
+			// openssl (VARIANT unset/openssl); patches are an orthogonal axis and don't change the library.
+			By("Expecting system OpenSSL")
+			Expect(runningSSL).To(ContainSubstring("OpenSSL"), "expected OpenSSL, running SSL library was %q", runningSSL)
+			Expect(isAWSLC).To(BeFalse(), "expected OpenSSL but found AWS-LC, running SSL library was %q", runningSSL)
+			Expect(fipsMode).To(BeFalse(), "expected FIPS mode off; haproxy -vv was:\n%s", versionInfo)
+		}
+	})
+})

acceptance-tests/strict_sni_test.go

@@ -129,11 +129,11 @@ var _ = Describe("Strict SNI", func() {
 
 			By("Sending a request to HAProxy with no SNI is forbidden")
 			_, err := httpClientNoSNI.Get("https://haproxy.internal")
-			expectTLSUnrecognizedNameErr(err)
+			expectStrictSNIRejectedErr(err)
 
 			By("Sending a request to HAProxy with an incorrect SNI is forbidden")
 			_, err = httpClientWrongSNI.Get("https://haproxy.internal")
-			expectTLSUnrecognizedNameErr(err)
+			expectStrictSNIRejectedErr(err)
 		})
 	})
 })

ci/README.md

@@ -135,6 +135,6 @@ For creating a new release please follow the versioning guide based on the [Sema
 
   Used for documentation updates, changes in the test suite or any updates in the testing frameworks (e.g. ginkgo).
 
-Since releases `11.16.3` and `11.17.5` the build metadata has been included into the version number. The build metadata denotes the contained HAProxy version. As an example, `11.16.2+2.6.9` means that HAProxy 2.6.9 is used.
+  Since releases `11.16.3` and `11.17.5` the build metadata (everything after the `+`) has been included in the version number to denote the contained upstream versions. Since version 17.0.0, it records both the HAProxy and the AWS-LC version: `17.0.0+3.2.26-awslc-4.0.0` means HAProxy 3.2.26 and AWS-LC 4.0.0.
 
-The `haproxy-boshrelease` also contains patches (see [haproxy-patches](../haproxy-patches)). The patched version is a part of the build metadata and is denoted by appending a hyphen and key word `patched`, like `11.17.4+2.6.9-patched`.
+  The release bundles every SSL variant and the HAProxy community patches (see [haproxy-patches](../src/haproxy-patches)). One build is selected at deploy time via the `ha_proxy.ssl_variant` (`openssl` | `awslc` | `awslc-fips`) and `ha_proxy.include_haproxy_patches` (`true` | `false`) job properties.