Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
e345428
ci: add a Commit-Boost check to the kurtosis battery
MegaRedHand Oct 26, 2026
ad9ebea
docs: explain local images and the fork PR for the Commit-Boost check
MegaRedHand Oct 26, 2026
3a7bf07
ci: work around two upstream breakages in the Commit-Boost check
MegaRedHand Oct 26, 2026
6372671
ci: start spamoor in the Commit-Boost check
MegaRedHand Oct 27, 2026
ae289be
Merge main into ci/kurtosis-commit-boost
MegaRedHand Oct 27, 2026
a211f49
ci: run spamoor at a light load in the Commit-Boost check
MegaRedHand Oct 27, 2026
b259186
Merge main into ci/kurtosis-commit-boost
MegaRedHand Oct 27, 2026
1c9b9b7
Merge main into ci/kurtosis-commit-boost
MegaRedHand Oct 27, 2026
8be1731
Merge main into ci/kurtosis-commit-boost
MegaRedHand Nov 4, 2026
f45306b
Merge main into ci/kurtosis-commit-boost
MegaRedHand Nov 11, 2026
5831253
ci: pin the Commit-Boost job's actions to the lockfile's versions
MegaRedHand Nov 11, 2026
483f980
Merge branch 'main' into ci/kurtosis-commit-boost
MegaRedHand Nov 18, 2026
e742d1d
Merge main into ci/kurtosis-commit-boost
MegaRedHand Nov 18, 2026
21254a7
Merge main into ci/kurtosis-commit-boost
MegaRedHand Nov 18, 2026
20cf458
fix(ci): make the Commit-Boost check's metric reads and setup failure…
MegaRedHand Nov 18, 2026
d3308af
docs(ci): say what runs the base battery, and when notify must learn …
MegaRedHand Nov 18, 2026
db90e13
docs(ci): say how to bump the Commit-Boost image digests
MegaRedHand Nov 18, 2026
a3a3588
fix(ci): start the Commit-Boost check's validator clients with --init…
MegaRedHand Nov 18, 2026
396bb60
fix(ci): fail the Commit-Boost job's devnet step when kurtosis run fails
MegaRedHand Nov 18, 2026
56374f1
Merge main into ci/kurtosis-commit-boost
MegaRedHand Nov 18, 2026
00c6e22
Merge main into ci/kurtosis-commit-boost
MegaRedHand Nov 19, 2026
cfaee08
Merge main into ci/kurtosis-commit-boost
MegaRedHand Nov 20, 2026
ce77ea9
Merge branch 'main' into ci/kurtosis-commit-boost
MegaRedHand Nov 21, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
102 changes: 102 additions & 0 deletions .github/workflows/kurtosis_assertoor.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,15 @@ on:
description: "ethlambda image to test (default: ghcr.io/lambdaclass/ethlambda:unstable)"
required: false
type: string
battery:
description: "Which battery to run"
required: false
type: choice
options:
- assertoor
- commit-boost
- both
default: assertoor

permissions:
contents: read
Expand All @@ -34,6 +43,13 @@ env:
ETHEREUM_PACKAGE_REPO: https://github.com/lambdaclass/ethereum-package.git
ETHEREUM_PACKAGE_REF: 8378c1e3030ef4d41306aea9a0ba13d19372d125
ENCLAVE: ethlambda-assertoor
# The Commit-Boost check needs the fork's `--builder` / `--builder-proposals`
# wiring for ethlambda, which the commit above lacks. It has its own pin so
# the base battery stays where it is. Fork PR #22 (that wiring) is merged as
# a15e7df; this commit adds PR #23's fix for the builder's enode bootnodes on
# top. Re-pin to #23's merge commit once it lands; see the README.
COMMIT_BOOST_PACKAGE_REF: 7fa7d1225fe5d975300ecc5584bf28ca6d42d451
COMMIT_BOOST_ENCLAVE: ethlambda-commit-boost

jobs:
image:
Expand Down Expand Up @@ -79,6 +95,9 @@ jobs:
assertoor:
name: Assertoor
needs: image
# Every push and pull request, and every dispatch except one that asked for
# the commit-boost battery alone: `assertoor` and `both` both run this job.
if: github.event_name != 'workflow_dispatch' || inputs.battery != 'commit-boost'
runs-on: ubuntu-latest
timeout-minutes: 120
outputs:
Expand Down Expand Up @@ -168,6 +187,89 @@ jobs:
if: always()
run: kurtosis enclave rm -f "$ENCLAVE" || true

# Dispatch-only for now: main's image does not have the builder flags until
# the builder PR stack (#694 to #705) lands. Once it is on main, drop the
# event condition so the check also runs on push and on pull requests, and at
# the same time add `commit-boost` to `notify`'s `needs` and its failed.txt to
# the Slack text: `notify` reports pushes only, so until then it can never
# see this job.
commit-boost:
name: Commit-Boost
needs: image
if: github.event_name == 'workflow_dispatch' && inputs.battery != 'assertoor'
runs-on: ubuntu-latest
timeout-minutes: 90
steps:
- uses: actions/checkout@v6.1.0

- name: Free up disk space
uses: $/.github/actions/free-disk

- name: Install kurtosis
run: |
echo "deb [trusted=yes] https://sdk.kurtosis.com/kurtosis-cli-release-artifacts/ /" \
| sudo tee /etc/apt/sources.list.d/kurtosis.list
sudo apt-get update
sudo apt-get install -y "kurtosis-cli=$KURTOSIS_VERSION"
kurtosis analytics disable
kurtosis version

# A clone for the same reason as in the assertoor job.
- name: Fetch ethereum-package at the pinned commit
run: |
git init "$RUNNER_TEMP/ethereum-package"
cd "$RUNNER_TEMP/ethereum-package"
git fetch --depth 1 "$ETHEREUM_PACKAGE_REPO" "$COMMIT_BOOST_PACKAGE_REF"
git checkout --detach FETCH_HEAD
test "$(git rev-parse HEAD)" = "$COMMIT_BOOST_PACKAGE_REF"

- name: Point the args at the image under test
env:
IMAGE: ${{ needs.image.outputs.image }}
run: |
cp tooling/kurtosis-assertoor/commit_boost_params.yaml "$RUNNER_TEMP/args.yaml"
yq -i '
(.participants[] | select(.cl_type == "ethlambda") | .cl_image) = strenv(IMAGE) |
(.participants[] | select(.vc_type == "ethlambda") | .vc_image) = strenv(IMAGE)
' "$RUNNER_TEMP/args.yaml"
cat "$RUNNER_TEMP/args.yaml"

# pipefail, or the step takes tee's status: a Starlark error that rolled
# back a service would pass here and surface only in the next step.
- name: Start the devnet
run: |
set -o pipefail
kurtosis run "$RUNNER_TEMP/ethereum-package" \
--enclave "$COMMIT_BOOST_ENCLAVE" --args-file "$RUNNER_TEMP/args.yaml" \
| tee "$RUNNER_TEMP/kurtosis_output.txt"

- name: Wait for builder blocks
run: |
tooling/kurtosis-assertoor/scripts/check-builder-blocks.sh \
"$COMMIT_BOOST_ENCLAVE" 40 "$RUNNER_TEMP/builder-results"

- name: Dump the enclave
if: always()
run: |
mkdir -p "$RUNNER_TEMP/dump"
cp "$RUNNER_TEMP/args.yaml" "$RUNNER_TEMP/dump/args.yaml"
cp "$RUNNER_TEMP/kurtosis_output.txt" "$RUNNER_TEMP/dump/" || true
cp -r "$RUNNER_TEMP/builder-results" "$RUNNER_TEMP/dump/" || true
cd "$RUNNER_TEMP/dump"
kurtosis enclave dump "$COMMIT_BOOST_ENCLAVE" || true

- name: Upload the dump and results
if: always()
uses: actions/upload-artifact@v7.0.2
with:
name: commit-boost-enclave-dump-${{ github.run_id }}
path: ${{ runner.temp }}/dump
if-no-files-found: warn

- name: Remove the enclave
if: always()
run: kurtosis enclave rm -f "$COMMIT_BOOST_ENCLAVE" || true

notify:
name: Post failure to Slack
needs: [image, assertoor]
Expand Down
108 changes: 108 additions & 0 deletions tooling/kurtosis-assertoor/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -68,3 +68,111 @@ field by field and marks a node whose spec differs offline (`invalid node
specs: spec mismatch: GloasForkEpoch`); lighthouse reports the far-future
value and ethlambda omits the key. The chain itself is healthy (all nodes share
a head and finalize), so this is a spec-endpoint gap, not a harness problem.

## Commit-Boost check

Starts the same three participants with `mev_type: commit-boost`
(`commit_boost_params.yaml`): a flashbots mev-boost-relay, a reth-rbuilder
builder with its own lighthouse beacon node, and one
[Commit-Boost](https://github.com/Commit-Boost/commit-boost-client) PBS module
per participant, which stands in for MEV-Boost. It also starts spamoor at a
light load, since rbuilder submits no bid for a block without transactions and
a heavy load slows the relay past Commit-Boost's getHeader timeout (see the
comment in the params file). The package gives the ethlambda
beacon node `--builder <pbs url>` and the ethlambda validator client
`--builder-proposals`. There is no assertoor: the base battery's checks fail on
the spec-endpoint gap above, so `scripts/check-builder-blocks.sh` polls the
ethlambda services' Prometheus metrics instead, for up to the timeout it is given.

| Service | Metric | Shows |
|---|---|---|
| `cl-3-ethlambda-geth` | `lean_beacon_builder_registrations_total{outcome="forwarded"}` >= 1 | Registrations reach the builder through the beacon node |
| `cl-3-ethlambda-geth` | `lean_beacon_builder_payload_source_total{source="builder"}` >= 1 | The beacon node's own `produceBlockV3` took a builder bid |
| `vc-3-geth-ethlambda` | `ethlambda_validator_builder_registrations_sent_total` >= 1 | The validator client registers its validators |
| `vc-3-geth-ethlambda` | `ethlambda_validator_blinded_blocks_proposed_total` >= 1 | It signs and publishes blinded blocks through ethlambda's beacon node |
| `vc-2-geth-lighthouse-ethlambda` | `ethlambda_validator_blinded_blocks_proposed_total` >= 1 | It does the same through a third-party beacon node (lighthouse) |

Passing therefore covers the registration path, a builder block from ethlambda's
own beacon node, and ethlambda's validator client proposing blinded blocks
through lighthouse. The script writes every `lean_beacon_builder_*` and
`ethlambda_validator_*` blinded or registration series of the three services to
`<output-dir>/<service>.prom`, pass or fail, and rewrites `failed.txt` with the
unmet conditions after every evaluation, so a cancelled run still says what was
missing. It stops at once, with exit status 2, if a service has no `metrics`
port: the service names depend on ethereum-package's naming and on the
participant order, and a mismatch is a setup problem, not a slow builder.

The relay, builder, PBS and spamoor images are pinned by digest in
`commit_boost_params.yaml` (the package's defaults are floating tags).

### Run it locally

The image needs the builder support, so build one from a checkout of the
builder PR stack (#694 to #705). From that checkout's root, `make docker-build
DOCKER_TAG=<tag>` tags it `ghcr.io/lambdaclass/ethlambda:<tag>`; set that as the
`cl_image` and `vc_image` of the ethlambda entries in `commit_boost_params.yaml`.
Kurtosis uses an image that exists in the local Docker daemon and only pulls a
missing one (`kurtosis run --image-download`, default `missing`), so no setting
is needed; do not pass `--image-download always`. From the repository root:

```bash
git clone https://github.com/lambdaclass/ethereum-package.git /tmp/ethereum-package
git -C /tmp/ethereum-package checkout 7fa7d1225fe5d975300ecc5584bf28ca6d42d451

kurtosis run /tmp/ethereum-package --enclave ethlambda-commit-boost \
--args-file tooling/kurtosis-assertoor/commit_boost_params.yaml

tooling/kurtosis-assertoor/scripts/check-builder-blocks.sh \
ethlambda-commit-boost 40 /tmp/builder-results

kurtosis enclave rm -f ethlambda-commit-boost
```

### In CI

The `commit-boost` job of `.github/workflows/kurtosis_assertoor.yml` runs on
manual dispatch only. Pick `battery: commit-boost` (or `both`) and give `image`
the tag of an image built from the builder PR stack; the default `assertoor`
runs only the base battery, as before. Its pin is `COMMIT_BOOST_PACKAGE_REF`,
separate from `ETHEREUM_PACKAGE_REF`, and it uploads `commit-boost-enclave-dump-<run id>`.

It is dispatch-only because the image that pushes and pull requests test
(`sha-<7>` of `main`, or `unstable`) does not have `--builder` or
`--builder-proposals` until the builder PR stack is on `main`, so the check
would fail there. Once it lands, drop the event condition on the job to run it
on push and on pull requests.

### The fork pin

lambdaclass/ethereum-package#22 added ethlambda to `apply_external_builder_flags`
(`--builder=<url>` for the beacon node, `--builder-proposals` for the validator
client) and is merged (`a15e7df`). `COMMIT_BOOST_PACKAGE_REF` is
`7fa7d1225fe5d975300ecc5584bf28ca6d42d451`, from
lambdaclass/ethereum-package#23, which adds the builder's enode bootnode fix on
top of it. Re-pin to #23's merge commit once it lands: a squash-merge leaves the
current commit unreachable from the fork's default branch, so it may be garbage
collected.

### Upstream workarounds

A local run found two breakages in images and packages we do not control:

| Breakage | Worked around in |
|---|---|
| `ethpandaops/reth-rbuilder:develop` (newest image, 2026-07-17) parses only enode URLs as bootnodes, and the package has passed ENRs since its 08-05 refactor, so the builder's execution client refused them | fork PR #23, through the `COMMIT_BOOST_PACKAGE_REF` pin |
| Commit-Boost's PBS requires `SECONDS_PER_SLOT` in the chain config, and the genesis generator now writes only `SLOT_DURATION_MS` | `mev_params.commit_boost_config` in `commit_boost_params.yaml`, which gives the chain explicitly (`chain_id` has to be a hex string) |

Drop each once the upstream is fixed. The explicit chain must change with the
network parameters.

### Startup noise

The Commit-Boost PBS modules start after the consensus clients, so at startup
ethlambda's beacon node logs "Builder is not answering" and its first
registration batch fails. The validator client re-sends registrations every
epoch, so this clears within an epoch and is not a failure.

From epoch 4 until the chain first finalizes, the beacon node builds locally and
counts `lean_beacon_builder_fallback_total{reason="no_finality"}`: like
Lighthouse, it skips the builder when the proposal is more than 3 epochs past
the finalized checkpoint, and the genesis checkpoint counts as one.
123 changes: 123 additions & 0 deletions tooling/kurtosis-assertoor/commit_boost_params.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
# The Commit-Boost check for ethlambda: the same three participants as
# network_params.yaml, plus `mev_type: commit-boost`, so ethlambda's beacon node
# and validator client build blocks through a Commit-Boost PBS module. There is
# no assertoor: the check is scripts/check-builder-blocks.sh, which reads the
# builder metrics of every ethlambda service. See README.md ("Commit-Boost
# check") for the steps. The package comes from a clone of
# lambdaclass/ethereum-package at the commit pinned in
# .github/workflows/kurtosis_assertoor.yml (COMMIT_BOOST_PACKAGE_REF).
#
# Participants, in the order ethereum-package launches them (the order fixes the
# service names check-builder-blocks.sh reads):
# 1. lighthouse + geth, lighthouse validator. It has to come first: the first
# participant is the network's bootnode, and ethlambda does not report an
# ENR over the beacon API.
# 2. lighthouse + geth, validated by the ethlambda validator client, so the
# validator client proposes blinded blocks through a beacon node that is
# not ethlambda.
# 3. ethlambda beacon node + geth, validated by the ethlambda validator client.
#
# `mev_type: commit-boost` adds a flashbots mev-boost-relay, a reth-rbuilder
# builder (with its own lighthouse beacon node) and one Commit-Boost PBS module
# per participant. The package passes each ethlambda beacon node `--builder` and
# each ethlambda validator client `--builder-proposals`, which only an image
# built with the builder support understands. The workflow rewrites the
# `cl_image` and `vc_image` of the ethlambda entries (selected by type, not
# position) to the image under test; the values below are what a local run uses.
# If the battery ever changes `network_id`, the preset or the fork epochs, the
# explicit Commit-Boost chain under `mev_params` must change with it.
participants:
- el_type: geth
cl_type: lighthouse
- el_type: geth
cl_type: lighthouse
vc_type: ethlambda
vc_image: ghcr.io/lambdaclass/ethlambda:unstable
# Genesis keys in a fresh directory: nothing has ever signed with them, so
# registering them without history is safe. Without it the validator
# client refuses to start (no slashing-protection record), as in
# network_params.yaml.
vc_extra_params: ["--init-slashing-protection"]
- el_type: geth
cl_type: ethlambda
cl_image: ghcr.io/lambdaclass/ethlambda:unstable
vc_type: ethlambda
vc_image: ghcr.io/lambdaclass/ethlambda:unstable
vc_extra_params: ["--init-slashing-protection"]
network_params:
# ethlambda only supports the mainnet preset: 32-slot epochs of 12 s.
preset: mainnet
# Fulu is the last fork ethlambda processes; gloas stays unscheduled.
electra_fork_epoch: 0
fulu_fork_epoch: 0
mev_type: commit-boost
# rbuilder submits nothing for a block with no transactions, so without a
# spammer the relay never holds a bid and every getHeader answers 204. The load
# is kept light on purpose: under spamoor's defaults (a chain-load group plus
# 200 transfers and 6 blob transactions a slot) the relay's getHeader slowed
# past Commit-Boost's 950 ms timeout on a local run, so most proposals got no
# bid. The package also always adds 100 Uniswap swaps a slot, sent to the
# builder's own execution client; that is not configurable. The blob spammer
# stays so bids carry blob commitments.
additional_services:
- spamoor
spamoor_params:
# ethpandaops/spamoor:master on 2026-10-27
image: ethpandaops/spamoor@sha256:e890785f5814218bbcb544c495820ff7b02dd554e6d714f82849a0ac8de680c5
start_chainload: false
spammers:
- name: "EOA transfers"
description: "10 type-2 transfers a slot"
scenario: "eoatx"
config:
throughput: 10
max_pending: 20
max_wallets: 10
base_fee: 20
- name: "Blob transactions"
description: "1 type-4 transaction a slot with 1 sidecar"
scenario: "blob-combined"
config:
throughput: 1
sidecars: 1
max_pending: 2
max_wallets: 2
base_fee: 20
blob_fee: 20
# The package defaults are floating tags (`main`, `develop`, `latest`), so a
# run would change under us. These pin what the tags pointed to on 2026-10-26;
# bump them deliberately. To bump one, read the tag's current digest from the
# `Digest:` line of `docker buildx imagetools inspect <image>:<tag>`, and update
# the date here with it.
mev_params:
# ethpandaops/mev-boost-relay:main
mev_relay_image: ethpandaops/mev-boost-relay@sha256:3600f04fcaf0bbb231f4737470f094fdf2314db54b4853197fa369aea90f8b19
# ethpandaops/reth-rbuilder:develop
mev_builder_image: ethpandaops/reth-rbuilder@sha256:916ba84da2690e3e368f57da5b0c73bfd919b452de8b95f64895846e0f3a8308
# ghcr.io/commit-boost/pbs:latest
mev_boost_image: ghcr.io/commit-boost/pbs@sha256:8612c03681d390ab7d0154f5db1581584cd265563b723947da90cd00ba7ca415
# Commit-Boost reads the chain from the package's config.yaml by default,
# and its parser requires SECONDS_PER_SLOT, which the genesis generator no
# longer writes (it writes SLOT_DURATION_MS). So the chain is given
# explicitly: kurtosis's defaults for this network (12 s slots, genesis fork
# version 0x10000038, fulu from genesis, chain id 3151908 = 0x301824).
# chain_id must be a hex string; a bare integer does not parse. The rest is
# the package's own template (static_files/mev/commit-boost/cb-config.toml.tmpl).
commit_boost_config: |
chain = { genesis_time_secs = {{ .Timestamp }}, slot_time_secs = 12, genesis_fork_version = "0x10000038", fulu_fork_slot = 0, chain_id = "0x301824" }

[pbs]
host = "0.0.0.0"
port = {{ .Port }}

{{ range $index, $relay := .Relays }}
[[relays]]
id = "mev_relay_{{$index}}"
url = "{{ $relay }}"
{{- end }}

[logs.stdout]
level = "debug"

[logs.file]
enabled=false
Loading