From bf3bcf75c01d99e2c5123b01f7b2b5a6bb8d5724 Mon Sep 17 00:00:00 2001 From: Stefan Date: Wed, 2 Sep 2026 17:08:50 +0200 Subject: [PATCH 01/11] feat(clients): executionGroups on endpoints + buildoor testing-build playbook Endpoints gain `executionGroups`, mapped to spamoor's `group(a,b)` rpchost prefix, so a scenario with `client_group` sends only to the endpoints in that group (e.g. buildoor's tx intake at /rpc) instead of spreading across every configured execution endpoint. Existing configs keep spamoor's default group. Adds playbooks/dev/buildoor-testing-build.yaml: switch a buildoor to its testing build source, run eoatx into the intake through the `builder` group, require buildoor blocks with N+ transactions, zero tx plan mismatches, no forks, then switch back. Claude-Session: https://claude.ai/code/session_01P3LSEorv3qsJrWb12ZKHNo --- docs/02-global-config.md | 7 ++ pkg/clients/clients.go | 6 ++ pkg/clients/execution/client.go | 1 + pkg/txmgr/spamoor.go | 4 + playbooks/dev/buildoor-testing-build.yaml | 119 ++++++++++++++++++++++ 5 files changed, 137 insertions(+) create mode 100644 playbooks/dev/buildoor-testing-build.yaml diff --git a/docs/02-global-config.md b/docs/02-global-config.md index 57a8859f..85861e75 100644 --- a/docs/02-global-config.md +++ b/docs/02-global-config.md @@ -25,6 +25,13 @@ endpoints: - name: "node-1" executionUrl: "http://127.0.0.1:8545" consensusUrl: "http://127.0.0.1:5052" + # An endpoint in a dedicated spamoor client group only receives transactions + # from scenarios that select that group (client_group). Use this to point a + # scenario at a builder's tx intake instead of the public txpool. + - name: "buildoor-intake" + executionUrl: "http://127.0.0.1:8085/rpc" + consensusUrl: "http://127.0.0.1:5052" + executionGroups: ["builder"] validatorNames: inventoryYaml: "./validator-names.yaml" diff --git a/pkg/clients/clients.go b/pkg/clients/clients.go index ab565b39..d2b5f968 100644 --- a/pkg/clients/clients.go +++ b/pkg/clients/clients.go @@ -40,6 +40,11 @@ type ClientConfig struct { ConsensusHeaders map[string]string `yaml:"consensusHeaders"` ExecutionURL string `yaml:"executionUrl"` ExecutionHeaders map[string]string `yaml:"executionHeaders"` + // ExecutionGroups are the spamoor client groups this execution endpoint + // belongs to. Scenarios select senders by group (client_group), so an + // endpoint in a dedicated group (e.g. a builder's tx intake) only receives + // transactions from scenarios that ask for it. Empty means "default". + ExecutionGroups []string `yaml:"executionGroups"` } func NewClientPool(logger logrus.FieldLogger) (*ClientPool, error) { @@ -95,6 +100,7 @@ func (pool *ClientPool) AddClient(config *ClientConfig) error { Name: config.Name, URL: config.ExecutionURL, Headers: config.ExecutionHeaders, + Groups: config.ExecutionGroups, }) if err != nil { return fmt.Errorf("could not init consensus client: %w", err) diff --git a/pkg/clients/execution/client.go b/pkg/clients/execution/client.go index 3114d36e..d5a1be9e 100644 --- a/pkg/clients/execution/client.go +++ b/pkg/clients/execution/client.go @@ -22,6 +22,7 @@ type ClientConfig struct { URL string Name string Headers map[string]string + Groups []string // spamoor client groups; empty = default } type Client struct { diff --git a/pkg/txmgr/spamoor.go b/pkg/txmgr/spamoor.go index e411b1d2..ed6d1d14 100644 --- a/pkg/txmgr/spamoor.go +++ b/pkg/txmgr/spamoor.go @@ -94,6 +94,10 @@ func (s *Spamoor) getClientOptions(client *execution.Client) *spamoor.ClientOpti rpcURL := client.GetEndpointConfig().URL rpcURL = fmt.Sprintf("name(%s)%s", client.GetName(), rpcURL) + if groups := client.GetEndpointConfig().Groups; len(groups) > 0 { + rpcURL = fmt.Sprintf("group(%s)%s", strings.Join(groups, ","), rpcURL) + } + if headers := client.GetEndpointConfig().Headers; len(headers) > 0 { headerParts := make([]string, 0, len(headers)) for key, value := range headers { diff --git a/playbooks/dev/buildoor-testing-build.yaml b/playbooks/dev/buildoor-testing-build.yaml new file mode 100644 index 00000000..024cf1e2 --- /dev/null +++ b/playbooks/dev/buildoor-testing-build.yaml @@ -0,0 +1,119 @@ +id: buildoor-testing-build +name: "buildoor testing build: planned transactions land on chain exactly" +description: | + Switches a buildoor to its testing build source (blocks built from the + builder's private tx intake through geth's testing_buildBlockV1), feeds the + intake with a spamoor scenario, and requires that every buildoor block after + the switch verified as an exact tx plan match: same transactions, same + order, same count. A single mismatch, missed payload or unknown block fails + the test. + + Endpoint setup: register the buildoor intake as an execution endpoint in a + dedicated client group, so only this scenario's transactions go there: + + endpoints: + - name: buildoor-intake + executionUrl: http://buildoor:8080/rpc + consensusUrl: http://beacon:5052 + executionGroups: [builder] + + The buildoor geth must serve the testing namespace (--http.api ...,testing). +version: 1.0.0 +tags: [buildoor, epbs, load, testing-build] +timeout: 30m +config: + buildoorUrl: "http://buildoor:8080" + spamoorPrivkey: "" + clientGroup: "builder" + minVerifiedBlocks: 6 + minTxsPerBlock: 20 + throughput: 300 + baseFeeGwei: 100 +tasks: +- name: check_clients_are_healthy + title: "Check if at least one client is ready" + timeout: 5m + config: + minClientCount: 1 + +- name: check_http_json + title: "Switch buildoor to the testing build source" + timeout: 1m + configVars: + url: ".buildoorUrl + \"/api/config/testing\"" + config: + method: POST + body: + source: testing + fill_gas_pct: 100 + policy: fifo + expectStatus: 200 + failOnCheckMiss: true + assertions: + - name: updated + query: ".status" + operator: eq + value: "updated" + +- name: run_task_background + title: "Feed the intake and verify buildoor's blocks" + config: + onBackgroundComplete: fail + backgroundTask: + name: run_spamoor_scenario + title: "spamoor eoatx into the builder intake" + configVars: + privateKey: "spamoorPrivkey" + scenarioYaml: ". as $v | {throughput: $v.throughput, max_pending: ($v.throughput * 10), max_wallets: 30, base_fee: $v.baseFeeGwei, tip_fee: 2, refill_amount: 50, refill_balance: 10, rebroadcast: 30, client_group: $v.clientGroup}" + config: + scenarioName: eoatx + foregroundTask: + name: run_tasks + title: "Verify" + config: + tasks: + - name: check_consensus_block_proposals + title: "buildoor proposes ${minVerifiedBlocks} blocks with >= ${minTxsPerBlock} transactions" + timeout: 20m + configVars: + blockCount: "minVerifiedBlocks" + minTransactionCount: "minTxsPerBlock" + config: + checkLookback: 64 + extraDataPattern: "^buildoor" + + - name: check_http_json + title: "Every included tx plan verified as an exact match" + timeout: 2m + configVars: + url: ".buildoorUrl + \"/api/buildoor/tx-queue\"" + # zero mismatches, zero unknown blocks, zero missed payloads, and + # at least minVerifiedBlocks exact matches + assertions: >- + . as $v | [ + {name: "no-mismatch", query: ".plan_checks.mismatch", operator: "eq", value: 0}, + {name: "no-missing-blocks", query: ".plan_checks.block_not_found", operator: "eq", value: 0}, + {name: "no-missed-payloads", query: ".plan_checks.missed", operator: "eq", value: 0}, + {name: "enough-matches", query: ".plan_checks.match", operator: "gte", value: $v.minVerifiedBlocks} + ] + config: + failOnCheckMiss: true + + - name: check_consensus_forks + title: "No forks" + timeout: 1m + config: + minCheckEpochCount: 1 + maxForkDistance: 1 + +cleanupTasks: +- name: check_http_json + title: "Switch buildoor back to the pool build source" + timeout: 1m + configVars: + url: ".buildoorUrl + \"/api/config/testing\"" + config: + method: POST + body: + source: pool + expectStatus: 200 From 7b9e0ad2b34b887cd247cbba5e65372e90978cbb Mon Sep 17 00:00:00 2001 From: Stefan Date: Wed, 2 Sep 2026 17:21:13 +0200 Subject: [PATCH 02/11] ci: pin staticcheck to v0.7.0 staticcheck v0.8.x requires Go 1.26; the check workflow runs Go 1.25.x with GOTOOLCHAIN=local, so `@latest` now fails at install time on every PR. Claude-Session: https://claude.ai/code/session_01P3LSEorv3qsJrWb12ZKHNo --- .github/workflows/_shared-check.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/_shared-check.yaml b/.github/workflows/_shared-check.yaml index 72a50a5b..5ad7f78d 100644 --- a/.github/workflows/_shared-check.yaml +++ b/.github/workflows/_shared-check.yaml @@ -30,7 +30,7 @@ jobs: run: if [ "$(gofmt -s -l . | wc -l)" -gt 0 ]; then exit 1; fi - name: Install staticcheck - run: go install honnef.co/go/tools/cmd/staticcheck@latest + run: go install honnef.co/go/tools/cmd/staticcheck@v0.7.0 - name: Run staticcheck run: staticcheck ./... From f1662d253b92f651b967eb68dd996257e0bc3db1 Mon Sep 17 00:00:00 2001 From: Stefan Date: Thu, 3 Sep 2026 10:49:22 +0200 Subject: [PATCH 03/11] fix: honest client-group semantics + a tally check that waits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review findings on the executionGroups field and the playbook: - The claim that an endpoint in a named group "only receives transactions from scenarios that ask for it" was false. spamoor seeds every client with the "default" group and group() only appends, and group-less selections (wallet funding, refills, deployments, scenarios without client_group) all resolve to "default" — so a named endpoint still receives that traffic. The field docs, the global-config example and the playbook now say so, and show "-default" as the way to actually reserve an endpoint. - verifyClientGroups fails at startup when the linked spamoor ignores a configured group entry, so a removal that is silently dropped can never degrade isolation unnoticed. Group removal needs spamoor > v1.2.2 (ethpandaops/spamoor#283). - The playbook's tx-queue check asserted the match count and the must-be-zero counters in one task with failOnCheckMiss, where an unsatisfied assertion is a failure rather than a wait — so it could fail on the first poll while buildoor was still tallying the last block. Split into a polled wait for the count and a fail-fast guard on the zero counters. Claude-Session: https://claude.ai/code/session_01P3LSEorv3qsJrWb12ZKHNo --- docs/02-global-config.md | 15 +++++++--- pkg/clients/clients.go | 16 +++++++++-- pkg/txmgr/spamoor.go | 33 ++++++++++++++++++++++ playbooks/dev/buildoor-testing-build.yaml | 34 +++++++++++++++++------ 4 files changed, 83 insertions(+), 15 deletions(-) diff --git a/docs/02-global-config.md b/docs/02-global-config.md index 85861e75..0d28895c 100644 --- a/docs/02-global-config.md +++ b/docs/02-global-config.md @@ -25,13 +25,20 @@ endpoints: - name: "node-1" executionUrl: "http://127.0.0.1:8545" consensusUrl: "http://127.0.0.1:5052" - # An endpoint in a dedicated spamoor client group only receives transactions - # from scenarios that select that group (client_group). Use this to point a - # scenario at a builder's tx intake instead of the public txpool. + # executionGroups puts an endpoint in spamoor client groups, so a scenario + # can direct its transactions at it with its client_group option. + # + # Membership is ADDITIVE: every endpoint is in "default", which is also the + # group used by every selection that names none - wallet funding and + # refills, contract deployments, and scenarios without client_group. Naming + # a group does not stop that traffic. Add "-default" to reserve an endpoint + # for the scenarios that ask for it, e.g. a builder's private tx intake that + # must hold only one scenario's transactions. Removal needs a spamoor + # version that supports it; assertoor fails at startup if it is ignored. - name: "buildoor-intake" executionUrl: "http://127.0.0.1:8085/rpc" consensusUrl: "http://127.0.0.1:5052" - executionGroups: ["builder"] + executionGroups: ["builder", "-default"] validatorNames: inventoryYaml: "./validator-names.yaml" diff --git a/pkg/clients/clients.go b/pkg/clients/clients.go index d2b5f968..926fc247 100644 --- a/pkg/clients/clients.go +++ b/pkg/clients/clients.go @@ -41,9 +41,19 @@ type ClientConfig struct { ExecutionURL string `yaml:"executionUrl"` ExecutionHeaders map[string]string `yaml:"executionHeaders"` // ExecutionGroups are the spamoor client groups this execution endpoint - // belongs to. Scenarios select senders by group (client_group), so an - // endpoint in a dedicated group (e.g. a builder's tx intake) only receives - // transactions from scenarios that ask for it. Empty means "default". + // belongs to, so a scenario can direct its transactions at specific + // endpoints via its client_group option. + // + // Membership is ADDITIVE: every client is in "default", which is also the + // group used by every selection that names none (wallet funding and + // refills, contract deployments, scenarios without client_group). Naming a + // group therefore does not stop that traffic. To reserve an endpoint — + // e.g. a builder's private tx intake that must hold only one scenario's + // transactions — also remove it from the default group with "-default". + // Removal requires a spamoor version that supports it; assertoor fails at + // startup if the linked one ignores the entry. + // + // executionGroups: ["builder", "-default"] ExecutionGroups []string `yaml:"executionGroups"` } diff --git a/pkg/txmgr/spamoor.go b/pkg/txmgr/spamoor.go index ed6d1d14..09fd4306 100644 --- a/pkg/txmgr/spamoor.go +++ b/pkg/txmgr/spamoor.go @@ -46,6 +46,10 @@ func NewSpamoor(ctx context.Context, logger logrus.FieldLogger, executionPool *e } for i, client := range clientPool.GetAllClients() { + if err := verifyClientGroups(endpoints[i].GetEndpointConfig(), client); err != nil { + return nil, err + } + s.clients[endpoints[i]] = client } @@ -273,3 +277,32 @@ func (s *Spamoor) NewWalletPoolByPrivkey(ctx context.Context, logger logrus.Fiel return walletPool, nil } + +// verifyClientGroups checks that spamoor applied the endpoint's configured +// client groups. Group membership decides which scenarios reach an endpoint, +// so a silently ignored entry would send transactions somewhere the operator +// deliberately excluded. A "-name" entry (removing a group, e.g. "-default" +// to keep an endpoint out of the group-less selections) needs a spamoor +// version that supports removal; older versions treat it as a literal group +// name, which this check turns into a startup error instead of a surprise. +func verifyClientGroups(config *execution.ClientConfig, client *spamoor.Client) error { + for _, group := range config.Groups { + if remove, found := strings.CutPrefix(group, "-"); found { + if client.HasGroup(remove) { + return fmt.Errorf( + "endpoint %q: executionGroups asked to remove client group %q but the client is still in it "+ + "(the linked spamoor does not support group removal)", + config.Name, remove) + } + + continue + } + + if !client.HasGroup(group) { + return fmt.Errorf("endpoint %q: executionGroups requested client group %q but the client is not in it", + config.Name, group) + } + } + + return nil +} diff --git a/playbooks/dev/buildoor-testing-build.yaml b/playbooks/dev/buildoor-testing-build.yaml index 024cf1e2..91ebcc03 100644 --- a/playbooks/dev/buildoor-testing-build.yaml +++ b/playbooks/dev/buildoor-testing-build.yaml @@ -9,13 +9,16 @@ description: | the test. Endpoint setup: register the buildoor intake as an execution endpoint in a - dedicated client group, so only this scenario's transactions go there: + dedicated client group so this scenario can address it. "-default" takes it + out of the group-less selections (wallet funding, deployments, scenarios + without a client_group) as well, so nothing else feeds the intake and the + built blocks hold only this scenario's transactions: endpoints: - name: buildoor-intake executionUrl: http://buildoor:8080/rpc consensusUrl: http://beacon:5052 - executionGroups: [builder] + executionGroups: [builder, "-default"] The buildoor geth must serve the testing namespace (--http.api ...,testing). version: 1.0.0 @@ -82,19 +85,34 @@ tasks: checkLookback: 64 extraDataPattern: "^buildoor" + # Two checks on purpose. The tally lags the proposals it counts (the + # verifier fetches the block from the EL after inclusion), and an + # unsatisfied assertion is a FAILURE rather than a wait under + # failOnCheckMiss - so the count is polled without it, and only the + # must-be-zero counters fail fast. - name: check_http_json - title: "Every included tx plan verified as an exact match" - timeout: 2m + title: "Wait for ${minVerifiedBlocks} verified tx plans" + timeout: 5m configVars: url: ".buildoorUrl + \"/api/buildoor/tx-queue\"" - # zero mismatches, zero unknown blocks, zero missed payloads, and - # at least minVerifiedBlocks exact matches assertions: >- . as $v | [ + {name: "enough-matches", query: ".plan_checks.match", operator: "gte", value: $v.minVerifiedBlocks} + ] + config: + pollInterval: 5s + + - name: check_http_json + title: "No tx plan was mismatched, missed or unverifiable" + timeout: 1m + configVars: + url: ".buildoorUrl + \"/api/buildoor/tx-queue\"" + # Cumulative counters, so this covers the whole run, not just now. + assertions: >- + [ {name: "no-mismatch", query: ".plan_checks.mismatch", operator: "eq", value: 0}, {name: "no-missing-blocks", query: ".plan_checks.block_not_found", operator: "eq", value: 0}, - {name: "no-missed-payloads", query: ".plan_checks.missed", operator: "eq", value: 0}, - {name: "enough-matches", query: ".plan_checks.match", operator: "gte", value: $v.minVerifiedBlocks} + {name: "no-missed-payloads", query: ".plan_checks.missed", operator: "eq", value: 0} ] config: failOnCheckMiss: true From c016864dee061db8fc962f66e206c5690d84ed2b Mon Sep 17 00:00:00 2001 From: Stefan Date: Thu, 3 Sep 2026 11:01:17 +0200 Subject: [PATCH 04/11] fix(lint): avoid shadowing err in the client group check Claude-Session: https://claude.ai/code/session_01P3LSEorv3qsJrWb12ZKHNo --- pkg/txmgr/spamoor.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/txmgr/spamoor.go b/pkg/txmgr/spamoor.go index 09fd4306..3c758fe2 100644 --- a/pkg/txmgr/spamoor.go +++ b/pkg/txmgr/spamoor.go @@ -46,8 +46,8 @@ func NewSpamoor(ctx context.Context, logger logrus.FieldLogger, executionPool *e } for i, client := range clientPool.GetAllClients() { - if err := verifyClientGroups(endpoints[i].GetEndpointConfig(), client); err != nil { - return nil, err + if groupErr := verifyClientGroups(endpoints[i].GetEndpointConfig(), client); groupErr != nil { + return nil, groupErr } s.clients[endpoints[i]] = client From f9f6c208ef18306dfc46fbb9187f8a149c6c3e50 Mon Sep 17 00:00:00 2001 From: Stefan Date: Thu, 3 Sep 2026 11:14:44 +0200 Subject: [PATCH 05/11] docs(playbook): the intake must be the only endpoint spamoor can pick MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit While buildoor runs the testing source its blocks are the only inclusion path, so spamoor's own wallet funding — which uses the group-less default selection, not the scenario's client_group — has to reach the intake too. Running the playbook with a second execution endpoint hung the scenario with one funding transaction pending in geth forever. Claude-Session: https://claude.ai/code/session_01P3LSEorv3qsJrWb12ZKHNo --- playbooks/dev/buildoor-testing-build.yaml | 30 ++++++++++++++++++----- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/playbooks/dev/buildoor-testing-build.yaml b/playbooks/dev/buildoor-testing-build.yaml index 91ebcc03..0e022cea 100644 --- a/playbooks/dev/buildoor-testing-build.yaml +++ b/playbooks/dev/buildoor-testing-build.yaml @@ -8,17 +8,35 @@ description: | order, same count. A single mismatch, missed payload or unknown block fails the test. - Endpoint setup: register the buildoor intake as an execution endpoint in a - dedicated client group so this scenario can address it. "-default" takes it - out of the group-less selections (wallet funding, deployments, scenarios - without a client_group) as well, so nothing else feeds the intake and the - built blocks hold only this scenario's transactions: + Endpoint setup. While the builder is on the testing source its blocks are + the ONLY way a transaction gets included: anything sent to a normal EL sits + in the public txpool forever, because buildoor builds solely from its + intake. That includes spamoor's own wallet funding and refills, which use + the group-less "default" selection rather than this scenario's client_group. + + So every endpoint spamoor may pick must be the intake. The simple form is to + give this playbook a config where the intake is the only execution endpoint + (it proxies all reads to the EL, so the client checks still work): + + endpoints: + - name: buildoor-intake + executionUrl: http://buildoor:8080/rpc + consensusUrl: http://beacon:5052 + + With several execution endpoints, keep the intake in "default" and take the + others out of it, so the scenario reaches every node it needs while funding + can only land somewhere that gets built (needs a spamoor with group removal, + ethpandaops/spamoor#283): endpoints: - name: buildoor-intake executionUrl: http://buildoor:8080/rpc consensusUrl: http://beacon:5052 - executionGroups: [builder, "-default"] + executionGroups: [builder] + - name: node-2 + executionUrl: http://node2:8545 + consensusUrl: http://beacon2:5052 + executionGroups: ["-default"] The buildoor geth must serve the testing namespace (--http.api ...,testing). version: 1.0.0 From b054f3237ed00cc7ed1f9ab12936ef6372ba0304 Mon Sep 17 00:00:00 2001 From: Stefan Date: Thu, 3 Sep 2026 11:18:19 +0200 Subject: [PATCH 06/11] fix(playbook): scenarioYaml refill amounts are wei, not ETH The --refill-amount / --refill-balance CLI flags take ETH and convert, the scenarioYaml fields are uint256 wei. Passing 50 funded every wallet with 50 wei: above the refill threshold, so spamoor reported 'no funding needed' and then every transaction was skipped for insufficient funds and the blocks came out empty. Now passed as explicit wei config values with a comment. Claude-Session: https://claude.ai/code/session_01P3LSEorv3qsJrWb12ZKHNo --- playbooks/dev/buildoor-testing-build.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/playbooks/dev/buildoor-testing-build.yaml b/playbooks/dev/buildoor-testing-build.yaml index 0e022cea..ac506e6e 100644 --- a/playbooks/dev/buildoor-testing-build.yaml +++ b/playbooks/dev/buildoor-testing-build.yaml @@ -50,6 +50,12 @@ config: minTxsPerBlock: 20 throughput: 300 baseFeeGwei: 100 + # spamoor's scenarioYaml takes these in WEI (the --refill-* CLI flags take + # ETH, the YAML fields do not). Too small and the wallets are "funded" with + # dust, pass the balance check, and every transaction is then skipped for + # insufficient funds. + refillAmountWei: 50000000000000000000 # 50 ETH + refillBalanceWei: 10000000000000000000 # 10 ETH tasks: - name: check_clients_are_healthy title: "Check if at least one client is ready" @@ -85,7 +91,7 @@ tasks: title: "spamoor eoatx into the builder intake" configVars: privateKey: "spamoorPrivkey" - scenarioYaml: ". as $v | {throughput: $v.throughput, max_pending: ($v.throughput * 10), max_wallets: 30, base_fee: $v.baseFeeGwei, tip_fee: 2, refill_amount: 50, refill_balance: 10, rebroadcast: 30, client_group: $v.clientGroup}" + scenarioYaml: ". as $v | {throughput: $v.throughput, max_pending: ($v.throughput * 10), max_wallets: 30, base_fee: $v.baseFeeGwei, tip_fee: 2, refill_amount: $v.refillAmountWei, refill_balance: $v.refillBalanceWei, rebroadcast: 30, client_group: $v.clientGroup}" config: scenarioName: eoatx foregroundTask: From e0c8b3d012a1293fe204b1705a6501f9d554ba8a Mon Sep 17 00:00:00 2001 From: Stefan Date: Thu, 3 Sep 2026 11:20:37 +0200 Subject: [PATCH 07/11] fix(playbook): quote the wei refill values configVars are evaluated as jq, whose numbers are float64, so an unquoted 50000000000000000000 reaches spamoor as "5e+19" and fails its uint256 parse. Claude-Session: https://claude.ai/code/session_01P3LSEorv3qsJrWb12ZKHNo --- playbooks/dev/buildoor-testing-build.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/playbooks/dev/buildoor-testing-build.yaml b/playbooks/dev/buildoor-testing-build.yaml index ac506e6e..163691b7 100644 --- a/playbooks/dev/buildoor-testing-build.yaml +++ b/playbooks/dev/buildoor-testing-build.yaml @@ -54,8 +54,10 @@ config: # ETH, the YAML fields do not). Too small and the wallets are "funded" with # dust, pass the balance check, and every transaction is then skipped for # insufficient funds. - refillAmountWei: 50000000000000000000 # 50 ETH - refillBalanceWei: 10000000000000000000 # 10 ETH + # Quoted: jq evaluates configVars as float64, so an unquoted 5e19 reaches + # spamoor as "5e+19" and fails its uint256 parse. + refillAmountWei: "50000000000000000000" # 50 ETH + refillBalanceWei: "10000000000000000000" # 10 ETH tasks: - name: check_clients_are_healthy title: "Check if at least one client is ready" From aae4eef6aed1afd45ceaac3169a8609fbb5d1b44 Mon Sep 17 00:00:00 2001 From: Stefan Date: Thu, 3 Sep 2026 11:21:27 +0200 Subject: [PATCH 08/11] fix(playbook): a configVars jq expression must start with a dot assertoor prepends "." to an expression that lacks one, so a bare [..] list became an index and the assertion set unmarshalled as a map. Claude-Session: https://claude.ai/code/session_01P3LSEorv3qsJrWb12ZKHNo --- playbooks/dev/buildoor-testing-build.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/playbooks/dev/buildoor-testing-build.yaml b/playbooks/dev/buildoor-testing-build.yaml index 163691b7..683b414f 100644 --- a/playbooks/dev/buildoor-testing-build.yaml +++ b/playbooks/dev/buildoor-testing-build.yaml @@ -134,8 +134,10 @@ tasks: configVars: url: ".buildoorUrl + \"/api/buildoor/tx-queue\"" # Cumulative counters, so this covers the whole run, not just now. + # Leading "." matters: assertoor prepends one to a configVars + # expression that lacks it, turning a bare [..] into an index. assertions: >- - [ + . | [ {name: "no-mismatch", query: ".plan_checks.mismatch", operator: "eq", value: 0}, {name: "no-missing-blocks", query: ".plan_checks.block_not_found", operator: "eq", value: 0}, {name: "no-missed-payloads", query: ".plan_checks.missed", operator: "eq", value: 0} From 7aabb420b8c8989946e7b2ed2e3625dab805f75d Mon Sep 17 00:00:00 2001 From: Stefan Date: Thu, 3 Sep 2026 11:29:44 +0200 Subject: [PATCH 09/11] fix: align endpoints with their spamoor clients; correct the simple form - InitClients appends only the options it could build, so pairing endpoints with GetAllClients() by index misaligns after a failure: every later endpoint gets another endpoint's client, wiring transactions to the wrong node and checking the wrong endpoint's groups. The count is now verified first; a client only fails to build on a malformed rpchost, so stopping is the right answer for what is a config error. - The playbook's "simple form" showed the intake with no executionGroups while the test's clientGroup defaults to "builder". A scenario selecting a group no client is in gets no client, so only the group-less wallet funding would have reached the intake and the proposal check could never pass. Claude-Session: https://claude.ai/code/session_01P3LSEorv3qsJrWb12ZKHNo --- pkg/txmgr/spamoor.go | 14 +++++++++++++- playbooks/dev/buildoor-testing-build.yaml | 5 ++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/pkg/txmgr/spamoor.go b/pkg/txmgr/spamoor.go index 3c758fe2..36ea612a 100644 --- a/pkg/txmgr/spamoor.go +++ b/pkg/txmgr/spamoor.go @@ -45,7 +45,19 @@ func NewSpamoor(ctx context.Context, logger logrus.FieldLogger, executionPool *e return nil, err } - for i, client := range clientPool.GetAllClients() { + // InitClients skips options it could not turn into a client, so a short + // list would silently pair every later endpoint with another endpoint's + // client — wiring transactions to the wrong node and checking the wrong + // endpoint's groups. A client only fails to build on a malformed rpchost, + // which is a config error worth stopping for. + spamoorClients := clientPool.GetAllClients() + if len(spamoorClients) != len(endpoints) { + return nil, fmt.Errorf("spamoor initialized %d of %d execution endpoints; "+ + "check the endpoint configuration (executionGroups, headers)", + len(spamoorClients), len(endpoints)) + } + + for i, client := range spamoorClients { if groupErr := verifyClientGroups(endpoints[i].GetEndpointConfig(), client); groupErr != nil { return nil, groupErr } diff --git a/playbooks/dev/buildoor-testing-build.yaml b/playbooks/dev/buildoor-testing-build.yaml index 683b414f..3606165b 100644 --- a/playbooks/dev/buildoor-testing-build.yaml +++ b/playbooks/dev/buildoor-testing-build.yaml @@ -16,12 +16,15 @@ description: | So every endpoint spamoor may pick must be the intake. The simple form is to give this playbook a config where the intake is the only execution endpoint - (it proxies all reads to the EL, so the client checks still work): + (it proxies all reads to the EL, so the client checks still work). It still + needs the group this test's clientGroup names, since a scenario selecting a + group no client is in gets no client at all: endpoints: - name: buildoor-intake executionUrl: http://buildoor:8080/rpc consensusUrl: http://beacon:5052 + executionGroups: [builder] With several execution endpoints, keep the intake in "default" and take the others out of it, so the scenario reaches every node it needs while funding From 8d74f068c32008e699cfbd6a76a4003224fe3643 Mon Sep 17 00:00:00 2001 From: Stefan Date: Thu, 3 Sep 2026 12:03:15 +0200 Subject: [PATCH 10/11] fix: trim client group names, as spamoor's parser does MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit spamoor runs TrimSpace on every group name; the check compared the raw config entry. A stray space therefore made the two disagree: an added group failed startup with a spurious "not in it", and — the case the guard exists for — a removal spamoor applied under a different name passed verification, leaving the endpoint in "default" while the config said otherwise. Claude-Session: https://claude.ai/code/session_01P3LSEorv3qsJrWb12ZKHNo --- pkg/txmgr/spamoor.go | 21 ++++++++++- pkg/txmgr/spamoor_groups_test.go | 64 ++++++++++++++++++++++++++++++++ 2 files changed, 83 insertions(+), 2 deletions(-) create mode 100644 pkg/txmgr/spamoor_groups_test.go diff --git a/pkg/txmgr/spamoor.go b/pkg/txmgr/spamoor.go index 36ea612a..f9c87f9c 100644 --- a/pkg/txmgr/spamoor.go +++ b/pkg/txmgr/spamoor.go @@ -110,7 +110,7 @@ func (s *Spamoor) getClientOptions(client *execution.Client) *spamoor.ClientOpti rpcURL := client.GetEndpointConfig().URL rpcURL = fmt.Sprintf("name(%s)%s", client.GetName(), rpcURL) - if groups := client.GetEndpointConfig().Groups; len(groups) > 0 { + if groups := normalizedGroups(client.GetEndpointConfig()); len(groups) > 0 { rpcURL = fmt.Sprintf("group(%s)%s", strings.Join(groups, ","), rpcURL) } @@ -298,7 +298,7 @@ func (s *Spamoor) NewWalletPoolByPrivkey(ctx context.Context, logger logrus.Fiel // version that supports removal; older versions treat it as a literal group // name, which this check turns into a startup error instead of a surprise. func verifyClientGroups(config *execution.ClientConfig, client *spamoor.Client) error { - for _, group := range config.Groups { + for _, group := range normalizedGroups(config) { if remove, found := strings.CutPrefix(group, "-"); found { if client.HasGroup(remove) { return fmt.Errorf( @@ -318,3 +318,20 @@ func verifyClientGroups(config *execution.ClientConfig, client *spamoor.Client) return nil } + +// normalizedGroups trims the endpoint's configured client groups and drops +// empty entries, matching what spamoor's own group(...) parser does. Without +// it a stray space makes the two disagree: spamoor applies the trimmed name +// while the check compares the raw one, which both fails startup spuriously +// on an added group and, worse, passes a removal that never took effect. +func normalizedGroups(config *execution.ClientConfig) []string { + groups := make([]string, 0, len(config.Groups)) + + for _, group := range config.Groups { + if group = strings.TrimSpace(group); group != "" { + groups = append(groups, group) + } + } + + return groups +} diff --git a/pkg/txmgr/spamoor_groups_test.go b/pkg/txmgr/spamoor_groups_test.go new file mode 100644 index 00000000..0dbb1c2c --- /dev/null +++ b/pkg/txmgr/spamoor_groups_test.go @@ -0,0 +1,64 @@ +package txmgr + +import ( + "slices" + "testing" + + "github.com/ethpandaops/assertoor/pkg/clients/execution" + "github.com/ethpandaops/spamoor/spamoor" +) + +// spamoor trims group names in its own parser, so the check must trim too: +// otherwise a stray space makes the two disagree and a removal that never +// took effect passes verification — the silent under-isolation this guard +// exists to prevent. +func TestNormalizedGroupsMatchesSpamoorParsing(t *testing.T) { + config := &execution.ClientConfig{ + Name: "intake", + URL: "http://localhost:8545", + Groups: []string{" builder ", "", " ", "-default "}, + } + + got := normalizedGroups(config) + if want := []string{"builder", "-default"}; !slices.Equal(got, want) { + t.Fatalf("normalizedGroups = %v, want %v", got, want) + } + + client, err := spamoor.NewClient(&spamoor.ClientOptions{ + RpcHost: "group(" + got[0] + ")" + config.URL, + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if !client.HasGroup("builder") { + t.Error("spamoor did not apply the trimmed group name") + } + + if err := verifyClientGroups(&execution.ClientConfig{Name: "intake", Groups: []string{" builder "}}, client); err != nil { + t.Errorf("verification must accept the same trimmed name spamoor applied: %v", err) + } +} + +// A configured group the linked spamoor did not apply must stop startup. +func TestVerifyClientGroupsRejectsAnUnappliedEntry(t *testing.T) { + client, err := spamoor.NewClient(&spamoor.ClientOptions{RpcHost: "group(builder)http://localhost:8545"}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if err := verifyClientGroups(&execution.ClientConfig{Name: "intake", Groups: []string{"builder"}}, client); err != nil { + t.Errorf("an applied group must verify: %v", err) + } + + // The pinned spamoor keeps the client in "default", so a removal that was + // ignored has to be an error rather than silent under-isolation. + err = verifyClientGroups(&execution.ClientConfig{Name: "intake", Groups: []string{"-default"}}, client) + if client.HasGroup("default") && err == nil { + t.Error("an ignored removal must fail verification") + } + + if err := verifyClientGroups(&execution.ClientConfig{Name: "intake", Groups: []string{"absent"}}, client); err == nil { + t.Error("a group the client is not in must fail verification") + } +} From 51518104a5dcd6c99ddf59afff03db44a25a5950 Mon Sep 17 00:00:00 2001 From: Stefan Date: Thu, 3 Sep 2026 12:03:47 +0200 Subject: [PATCH 11/11] fix(lint): avoid shadowing err in the group tests Claude-Session: https://claude.ai/code/session_01P3LSEorv3qsJrWb12ZKHNo --- pkg/txmgr/spamoor_groups_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/txmgr/spamoor_groups_test.go b/pkg/txmgr/spamoor_groups_test.go index 0dbb1c2c..dc2a6398 100644 --- a/pkg/txmgr/spamoor_groups_test.go +++ b/pkg/txmgr/spamoor_groups_test.go @@ -47,14 +47,14 @@ func TestVerifyClientGroupsRejectsAnUnappliedEntry(t *testing.T) { t.Fatalf("unexpected error: %v", err) } - if err := verifyClientGroups(&execution.ClientConfig{Name: "intake", Groups: []string{"builder"}}, client); err != nil { - t.Errorf("an applied group must verify: %v", err) + if applied := verifyClientGroups(&execution.ClientConfig{Name: "intake", Groups: []string{"builder"}}, client); applied != nil { + t.Errorf("an applied group must verify: %v", applied) } // The pinned spamoor keeps the client in "default", so a removal that was // ignored has to be an error rather than silent under-isolation. - err = verifyClientGroups(&execution.ClientConfig{Name: "intake", Groups: []string{"-default"}}, client) - if client.HasGroup("default") && err == nil { + ignored := verifyClientGroups(&execution.ClientConfig{Name: "intake", Groups: []string{"-default"}}, client) + if client.HasGroup("default") && ignored == nil { t.Error("an ignored removal must fail verification") }