From c45203e8a434da36bc2f27abd170d3f1df035afe Mon Sep 17 00:00:00 2001 From: MauroFab Date: Tue, 4 Aug 2026 20:56:11 -0300 Subject: [PATCH 1/2] perf(prover): default the cuda table scheduler to K = num_airs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `table_parallelism()`'s cuda arm scaled K by `available_parallelism()` (`cores * 2 / 3`). Measured over 881 runs on two RTX 5090 boxes, that is the wrong shape. All eight core-count curves fit `T(K) = S + max(Tmax, W/K)` within run-to-run noise, and the work K divides — W ≈ 5.3-8.0 s — is invariant to host core count over an 8x range, to CPU model, and to rayon pool width: cutting RAYON_NUM_THREADS 32 -> 4 leaves W alone and merely doubles S, with the best K still num_airs at every pool width. `available_parallelism()` sizes precisely that rayon pool, so it is the wrong quantity to scale K by. K is not a thread count; each table's work runs on the one global pool. Worst case against the best measured K, over four core counts on both boxes: cores/3 +30.2 % cores*2/3 +13.0 % (what this replaces) constant 12 +7.0 % num_airs +1.6 % (both non-zero cells inside noise, p = 0.88 / 0.80) `cores*2/3` fails where it was predicted to: low core counts, K=2 at 4 cores (+13.0 %) and K=5 at 8 cores (+8.1 %). Taking the ceiling rather than solving for an optimum is right in both regimes of the fit: if W/num_airs > Tmax more K strictly helps, and if W/num_airs < Tmax the extra drivers are floor-limited and cost nothing — the one staging slab is held 56 % of wall at K=31 and wall time still improves. The old doc comment's mechanism ("in-flight tables mostly sit in GPU waits") is not what happens — mean GPU utilisation never exceeded ~38 % at any K — so it is rewritten rather than re-tuned. What is meant to bound concurrency is memory admission rather than a count: that is what VramGate is for, and it never binds at the default budget. `table_parallelism` now takes `num_airs` and clamps to it, replacing the `.min(num_airs)` the call site applied. `auto_storage::decide` keeps a bounded figure through the new `storage_estimate_parallelism()`: `peak_bytes` sums the transient bytes of the top-k tables, so an unbounded k there sums every table — measured +27 % at 128 PAGE tables, +44 % at 512 — and would spill proofs to disk that fit in RAM. Its value is unchanged, so no storage decision moves. The CPU arm keeps `cores / 3`. The sweep ran only on cuda builds, where the parallelized work is device-bound; on a CPU-only build every table is pure host work and none of this evidence transfers. --- .github/workflows/benchmark-pr.yml | 3 +- crypto/stark/src/instruments.rs | 7 +- crypto/stark/src/prover.rs | 140 +++++++++++++++++++------ crypto/stark/src/tests/prover_tests.rs | 40 +++++++ prover/src/auto_storage.rs | 33 +++--- prover/src/tests/auto_storage_tests.rs | 40 +++++++ prover/tests/calibration.rs | 5 +- 7 files changed, 213 insertions(+), 55 deletions(-) diff --git a/.github/workflows/benchmark-pr.yml b/.github/workflows/benchmark-pr.yml index 91f5b02ac..956852588 100644 --- a/.github/workflows/benchmark-pr.yml +++ b/.github/workflows/benchmark-pr.yml @@ -273,7 +273,8 @@ jobs: # Optional table parallelism for the HEADLINE benchmark only (the memory # growth sweep always runs at default parallelism). `/bench k=N` overrides; - # otherwise default (cores/3). /bench-growth no longer forces k=1. + # otherwise the build's default (num_airs on cuda, cores/3 on CPU). + # /bench-growth no longer forces k=1. TABLE_K="" if [ "$EVENT_NAME" = "issue_comment" ]; then TABLE_K=$(echo "$COMMENT_BODY" | grep -o 'k=[0-9]*' | head -1 | cut -d= -f2) diff --git a/crypto/stark/src/instruments.rs b/crypto/stark/src/instruments.rs index 796aaf46f..0f68059f4 100644 --- a/crypto/stark/src/instruments.rs +++ b/crypto/stark/src/instruments.rs @@ -22,7 +22,7 @@ use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; // siblings overlap in wall time. Read them as per instance wall time. // - `scripts/profiling/phase_table.py` SUMS spans that share a label, so a // label used once per table reports the sum over all tables, which can -// exceed the enclosing phase's wall clock by up to `table_parallelism()`. +// exceed the enclosing phase's wall clock by up to the scheduler's `k`. // Give a per instance span its own label; never reuse a phase label for it. // // let _s = instruments::span("trace_build"); // RAII, stops on drop @@ -278,8 +278,9 @@ pub struct MultiProveTiming { /// root must be absorbed before the shared LogUp challenges are sampled. pub main_commits: Duration, /// Wall clock of the fused per-table region: aux build, aux commit and - /// rounds 2-4, which run as one task per table across `table_parallelism()` - /// drivers. There is no phase-level wall for the aux stages on their own + /// rounds 2-4, which run as one task per table across + /// `table_parallelism(num_airs)` drivers. There is no phase-level wall for + /// the aux stages on their own /// any more; their CPU time shows up in `round1_sub`. pub rounds_2_4: Duration, /// Sub-op breakdown for Round 1 (main + aux LDE vs Merkle). diff --git a/crypto/stark/src/prover.rs b/crypto/stark/src/prover.rs index 4047458bc..3ad0e00e6 100644 --- a/crypto/stark/src/prover.rs +++ b/crypto/stark/src/prover.rs @@ -267,8 +267,9 @@ where /// aux commit and rounds 2-4 into one task: /// - main: produced by the Round 1 main commit, which is a phase-wide barrier, /// so all N tables' main LDEs are live at once (O(N × main_cols × lde_size)). -/// - aux: produced and consumed inside the same fused task, so at most -/// `table_parallelism()` of them coexist (O(k × aux_cols × lde_size)). +/// - aux: produced and consumed inside the same fused task, so at most the +/// scheduler's `k` coexist (O(k × aux_cols × lde_size)) — which under `cuda` +/// is `num_airs`, so there they are all-N-live like the main ones. /// /// Under `debug-checks` the fused task is split around the cross-table bus /// balance check, so there the aux LDEs are all-N-live like the main ones. @@ -573,41 +574,112 @@ where (d, t) } -/// Number of tables to process concurrently in `multi_prove`. +/// Explicit `TABLE_PARALLELISM` override, honoured by both `k` values below so +/// setting it pins the scheduler and the storage estimate to the same number. +#[cfg(feature = "parallel")] +fn parallelism_override() -> Option { + std::env::var("TABLE_PARALLELISM") + .ok() + .and_then(|s| s.parse().ok()) +} + +#[cfg(feature = "parallel")] +fn host_cores() -> usize { + std::thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(4) +} + +/// Number of tables `multi_prove` proves concurrently, out of `num_airs` of +/// them. /// -/// Defaults: `num_cores / 3` on CPU builds (benchmarked optimal on both M3 Pro -/// and EPYC 9454P — every table there is pure host work), `num_cores * 2 / 3` -/// under `cuda`, where most in-flight tables sit in GPU waits so more of them -/// pay (swept flat at ~2/3 of the cores on a 16-core/RTX 5090 box). Both arms -/// are overridden by the `TABLE_PARALLELISM` env var. Without the `parallel` -/// feature this is hardcoded to 1 and the env var is ignored. +/// Defaults: **every table** under `cuda`, `num_cores / 3` on CPU builds +/// (benchmarked optimal on both M3 Pro and EPYC 9454P — every table there is +/// pure host work, so `k` genuinely competes for cores). Both arms are +/// overridden by the `TABLE_PARALLELISM` env var, and the result is clamped to +/// `1..=num_airs`. Without the `parallel` feature this is 1 and the env var is +/// ignored. /// -/// Not only the prover's `k`: `auto_storage::decide` feeds this into the -/// RAM-vs-Disk storage estimate, so the `cuda` arm also doubles that transient -/// term (see `peak_bytes`). -pub fn table_parallelism() -> usize { +/// # Why the `cuda` arm has no core term +/// +/// Measured over 881 runs on two RTX 5090 boxes +/// (`scripts/profiling/table-parallelism-sweep/`). All eight core-count curves +/// fit `T(k) = S + max(Tmax, W/k)` within run-to-run noise, and the divisible +/// work `W ≈ 5.3–8.0 s` is invariant to host core count over an 8× range, to CPU +/// model, and to rayon pool width: cutting `RAYON_NUM_THREADS` 32 → 4 moves `W` +/// by ~0 and merely doubles `S` (the serial continuation producer), with the +/// best `k` still `num_airs` at every pool width. `available_parallelism()` +/// sizes precisely that rayon pool, so it is the wrong quantity to scale `k` +/// by — `k` is not a thread count, it is a count of concurrent drivers whose +/// per-table work all runs on the one global pool. Worst-case cost against the +/// best measured `k`, over four core counts on both boxes: `cores/3` +30.2 %, +/// `cores*2/3` +13.0 %, constant 12 +7.0 %, `num_airs` +1.6 % (both of the +/// latter's non-zero cells inside noise, p = 0.88 / 0.80). +/// +/// Taking the ceiling rather than computing an optimum is right in both +/// regimes of the fit: if `W/num_airs > Tmax` more `k` strictly helps, and if +/// `W/num_airs < Tmax` the extra drivers are floor-limited and cost nothing — +/// measured, the one staging slab is held 56 % of wall at `k = 31` and wall +/// time still improves. What is meant to bound concurrency here is memory +/// admission rather than a count; that is `VramGate`'s job, and it never binds +/// at the default budget. +pub fn table_parallelism(num_airs: usize) -> usize { #[cfg(feature = "parallel")] { - std::env::var("TABLE_PARALLELISM") - .ok() - .and_then(|s| s.parse().ok()) - .unwrap_or_else(|| { - let cores = std::thread::available_parallelism() - .map(|n| n.get()) - .unwrap_or(4); - // GPU builds: with the admission scheduler most in-flight - // tables sit in GPU waits, so more of them pay (swept flat at - // ~2/3 of the cores on a 16-core/RTX 5090 box). CPU builds - // stay at cores/3 — every table is pure host work there. - #[cfg(feature = "cuda")] - { - (cores * 2 / 3).max(1) - } - #[cfg(not(feature = "cuda"))] - { - (cores / 3).max(1) - } - }) + let k = parallelism_override().unwrap_or_else(|| { + // GPU builds: run every table. The work `k` divides is device- and + // workload-bound, not core-bound — see the doc comment. + #[cfg(feature = "cuda")] + { + num_airs + } + // CPU builds: every table is pure host work, so `k` competes for + // the same cores the rayon pool wants. + #[cfg(not(feature = "cuda"))] + { + (host_cores() / 3).max(1) + } + }); + k.clamp(1, num_airs.max(1)) + } + #[cfg(not(feature = "parallel"))] + { + let _ = num_airs; + 1 + } +} + +/// How many tables' rounds 2-4 transients the *RAM* estimate assumes are alive +/// at once (`auto_storage::peak_bytes` sums the transient bytes of the top-k +/// tables, and `decide` turns that into RAM vs Disk). +/// +/// Deliberately not `table_parallelism(num_airs)`. That is a ceiling, not a +/// bound: on a `cuda` build what actually limits how many tables are in flight +/// is `VramGate`'s byte budget, which this host-side estimate cannot see. +/// Feeding an unbounded count in here would sum *every* table's transients — +/// on many-PAGE shapes that inflates the estimate by up to +44 % (512 PAGE +/// tables at blowup 4) and would spill proofs to disk that fit in RAM. On the +/// shapes that reach this path today (~21 tables, one PAGE table) the top-k sum +/// has all but saturated, so this value and `num_airs` agree to well under 1 %. +/// +/// Kept at exactly the value it had when the scheduler shared it, so splitting +/// the two does not move any storage decision. +/// +/// TODO: derive this from a byte budget rather than a table count, so it +/// tracks what `VramGate` admits instead of standing in for it. +pub fn storage_estimate_parallelism() -> usize { + #[cfg(feature = "parallel")] + { + parallelism_override().unwrap_or_else(|| { + #[cfg(feature = "cuda")] + { + (host_cores() * 2 / 3).max(1) + } + #[cfg(not(feature = "cuda"))] + { + (host_cores() / 3).max(1) + } + }) } #[cfg(not(feature = "parallel"))] { @@ -3075,7 +3147,7 @@ pub trait IsStarkProver< twiddle_caches.push(twiddles); } - let k = table_parallelism().min(num_airs).max(1); + let k = table_parallelism(num_airs); // VRAM budgeted admission. The budget caps the summed device working set // of the tables proved concurrently so large blocks don't exhaust VRAM. diff --git a/crypto/stark/src/tests/prover_tests.rs b/crypto/stark/src/tests/prover_tests.rs index ff4a0313c..480969a84 100644 --- a/crypto/stark/src/tests/prover_tests.rs +++ b/crypto/stark/src/tests/prover_tests.rs @@ -609,3 +609,43 @@ fn commit_rows_bit_reversed_matches_commit_bit_reversed() { } } } + +/// `k` is a count of concurrent table drivers — `run_admitted` spawns exactly +/// this many OS threads and indexes `order` with them — so it has to stay +/// inside `1..=num_airs` in every arm, including under a `TABLE_PARALLELISM` +/// override (CI's prover shard 1 sets one). +#[test] +fn table_parallelism_stays_within_one_and_num_airs() { + use crate::prover::table_parallelism; + + assert_eq!(table_parallelism(0), 1, "no tables still needs one driver"); + for n in [1usize, 2, 7, 31, 64, 1024] { + let k = table_parallelism(n); + assert!(k >= 1 && k <= n, "k={k} outside 1..={n}"); + } + + // Monotone in `num_airs` in every arm: cuda `n`, CPU `min(cores/3, n)`, + // override `min(override, n)`. + let mut prev = 0; + for n in 1..=64 { + let k = table_parallelism(n); + assert!(k >= prev, "k fell from {prev} to {k} at num_airs={n}"); + prev = k; + } +} + +/// The cuda default is every table: the sweep in `thoughts/k-sweep-877b/` found +/// no core count at which a smaller `k` wins, and `T(k) = S + max(Tmax, W/k)` +/// has no term that ever favours one. Skipped when the env var pins `k`. +#[cfg(all(feature = "cuda", feature = "parallel"))] +#[test] +fn cuda_table_parallelism_defaults_to_num_airs() { + use crate::prover::table_parallelism; + + if std::env::var("TABLE_PARALLELISM").is_ok() { + return; + } + for n in [1usize, 7, 31, 1024] { + assert_eq!(table_parallelism(n), n, "cuda k must be num_airs"); + } +} diff --git a/prover/src/auto_storage.rs b/prover/src/auto_storage.rs index 6b5ed8a5d..b4718974c 100644 --- a/prover/src/auto_storage.rs +++ b/prover/src/auto_storage.rs @@ -30,7 +30,7 @@ use crate::tables::register::{ }; use crate::tables::shift::{bus_interactions as shift_buses, cols::NUM_COLUMNS as SHIFT_COLS}; use crate::tables::trace_builder::TableLengths; -use stark::prover::table_parallelism; +use stark::prover::storage_estimate_parallelism; use stark::storage_mode::StorageMode; use sysinfo::System; @@ -222,7 +222,7 @@ pub fn decide(lengths: &TableLengths, blowup_factor: u8) -> StorageMode { log::info!("storage_mode: Disk (forced via FORCE_DISK_SPILL)"); return StorageMode::Disk; } - let estimated = peak_bytes(lengths, blowup_factor, table_parallelism()); + let estimated = peak_bytes(lengths, blowup_factor, storage_estimate_parallelism()); let mode = select_storage_mode(estimated, available_ram_bytes()); log::info!("estimated_peak_bytes: {estimated}, storage_mode: {mode:?}"); mode @@ -230,30 +230,33 @@ pub fn decide(lengths: &TableLengths, blowup_factor: u8) -> StorageMode { /// Peak RAM estimate in bytes for a proof whose trace shape matches `lengths`. /// -/// `table_parallelism` is the prover's `k` (`stark::prover::table_parallelism`), -/// and it is not only a prover knob: `decide` feeds it in here, so the `cuda` -/// arm's `cores * 2 / 3` doubles the transient term below versus the CPU arm's -/// `cores / 3` and makes `Disk` more likely. That direction is safe (it -/// over-estimates), but it means a change to `k` changes the storage decision. +/// `table_parallelism` is how many tables' rounds 2-4 transients this assumes +/// are alive at once. `decide` passes `storage_estimate_parallelism()`, which +/// is deliberately *not* the scheduler's `k` — that one is `num_airs` under +/// `cuda`, and summing every table's transients here inflates the estimate on +/// many-PAGE shapes (up to +44 %) and makes `Disk` more likely than the real +/// heap warrants. See that function for why the honest bound is a byte budget +/// rather than a count. pub fn peak_bytes(lengths: &TableLengths, blowup_factor: u8, table_parallelism: usize) -> u64 { let blowup = blowup_factor as u64; let k = table_parallelism.max(1); let specs = table_specs(lengths); // Persistent: every table's main LDE + Merkle really is alive at once (the - // Round 1 main commit is a phase-wide barrier). The aux LDE no longer is — - // it is produced and consumed inside one table's fused task, so at most k - // coexist — but it is still counted for every table here, which keeps this - // an over-estimate rather than making the bound unsound. + // Round 1 main commit is a phase-wide barrier). The aux LDE is produced and + // consumed inside one table's fused task, so only the scheduler's k coexist + // — exactly all of them on `cuda`, fewer on CPU builds. Counted for every + // table either way, which is exact on `cuda` and an over-estimate on CPU + // rather than an unsound bound. let persistent_total: u64 = specs .iter() .map(|s| persistent_per_table(*s, blowup)) .fold(0u64, u64::saturating_add); - // Transient: only k tables run the fused aux+rounds task at a time. The - // top-k tables by transient bytes bound it; with the scheduler's - // heaviest-first admission that top-k is also the set actually admitted - // first, so this is the realistic peak, not a worst case. + // Transient: k tables' fused aux+rounds tasks assumed in flight at once. + // The top-k tables by transient bytes bound that; with the scheduler's + // heaviest-first admission that top-k is also the set admitted first, so + // this is the realistic peak, not a worst case. let mut transient_per: Vec = specs .iter() .map(|s| transient_per_table(*s, blowup)) diff --git a/prover/src/tests/auto_storage_tests.rs b/prover/src/tests/auto_storage_tests.rs index 5d976f81b..e26674d27 100644 --- a/prover/src/tests/auto_storage_tests.rs +++ b/prover/src/tests/auto_storage_tests.rs @@ -95,3 +95,43 @@ fn unknown_available_defaults_to_disk() { let mode = select_storage_mode(peak_bytes(&empty_lengths(), 2, ALL_TABLES), None); assert_eq!(mode, StorageMode::Disk); } + +/// A shape with one PAGE table — everything the monolithic path proves today. +/// The top-k sum has saturated well before the table count, so the estimate is +/// insensitive to `k` in that range: this is why raising the *scheduler's* `k` +/// to `num_airs` does not move the storage decision on a normal workload. +#[test] +fn peak_bytes_is_k_saturated_on_single_page_shapes() { + let mut lengths = empty_lengths(); + lengths.cpu_padded_rows = 1 << 20; + lengths.memw_padded_rows = 1 << 20; + lengths.decode_rows = 1 << 16; + lengths.unique_page_count = 1; + + let bounded = peak_bytes(&lengths, 2, 12); + let unbounded = peak_bytes(&lengths, 2, ALL_TABLES); + assert!( + unbounded * 100 <= bounded * 101, + "estimate moved {bounded} -> {unbounded} on a one-page shape" + ); +} + +/// …and why `decide` must not simply be handed the scheduler's `k`. PAGE tables +/// are all the same size, so once there are many of them the top-k truncation +/// is doing real work: summing every table's transients inflates the estimate +/// by >20 % here, which spills proofs to disk that fit in RAM. +#[test] +fn unbounded_k_inflates_peak_bytes_on_many_page_shapes() { + let mut lengths = empty_lengths(); + lengths.cpu_padded_rows = 1 << 20; + lengths.memw_padded_rows = 1 << 20; + lengths.decode_rows = 1 << 16; + lengths.unique_page_count = 128; + + let bounded = peak_bytes(&lengths, 2, 21); + let unbounded = peak_bytes(&lengths, 2, ALL_TABLES); + assert!( + unbounded * 10 > bounded * 12, + "expected >20 % inflation, got {bounded} -> {unbounded}" + ); +} diff --git a/prover/tests/calibration.rs b/prover/tests/calibration.rs index ff11bcf4b..c7d4d66f5 100644 --- a/prover/tests/calibration.rs +++ b/prover/tests/calibration.rs @@ -11,7 +11,7 @@ use lambda_vm_prover::tables::MaxRowsConfig; use lambda_vm_prover::tables::trace_builder::count_table_lengths; use lambda_vm_prover::test_utils::{asm_elf_bytes, run_asm_elf}; use stark::proof::options::GoldilocksCubicProofOptions; -use stark::prover::table_parallelism; +use stark::prover::storage_estimate_parallelism; use std::sync::Arc; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::thread; @@ -36,7 +36,8 @@ fn peak_bytes_does_not_underestimate_measured_heap() { count_table_lengths(&elf, &logs, &max_rows, &[]).expect("count_table_lengths succeeds"); let opts = GoldilocksCubicProofOptions::with_blowup(2).expect("blowup=2 is valid"); - let predicted = peak_bytes(&lengths, opts.blowup_factor, table_parallelism()) as usize; + let predicted = + peak_bytes(&lengths, opts.blowup_factor, storage_estimate_parallelism()) as usize; drop(logs); From 975ef4efaf3933a4b2f6ff7fd21a960446800d4d Mon Sep 17 00:00:00 2001 From: MauroFab Date: Tue, 4 Aug 2026 20:56:23 -0300 Subject: [PATCH 2/2] docs(profiling): commit the TABLE_PARALLELISM sweep record MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The measurement behind the previous commit, under `scripts/profiling/table-parallelism-sweep/`: both write-ups, every timing CSV (881 runs over two RTX 5090 boxes, one row per run), the analysis scripts, the server-side sweep harnesses, the box/toolchain fingerprints, the frozen job orders and the diffs for the three experimental builds. `rules2.py` reproduces the rule-cost table from the committed CSVs; `amdahl.py` reproduces the fits. `thoughts/` is gitignored, so this lands next to the profiling tooling instead. Build logs, the per-run logs for the stages whose hypotheses were refuted, and the Stage 1a stack samples (gdb could not attach — the rented container dropped cap_sys_ptrace) are left out; the README says what is included and what is not. --- .../table-parallelism-sweep/README.md | 55 + .../table-parallelism-sweep/round1/README.md | 101 ++ .../table-parallelism-sweep/round1/analyze.py | 39 + .../round1/classify_stacks.py | 72 ++ .../table-parallelism-sweep/round1/confirm.sh | 32 + .../round1/confirm_analyze.py | 60 ++ .../round1/followup.sh | 38 + .../round1/followup2.sh | 28 + .../round1/followup_results.txt | 56 + .../table-parallelism-sweep/round1/gpuutil.sh | 34 + .../round1/results_c16.csv | 66 ++ .../round1/results_c32.csv | 66 ++ .../round1/results_c4.csv | 53 + .../round1/results_c8.csv | 53 + .../round1/results_confirm.csv | 109 ++ .../table-parallelism-sweep/round1/rules.py | 56 + .../round1/stacksample.sh | 31 + .../table-parallelism-sweep/round1/sweep.sh | 40 + .../table-parallelism-sweep/round2/NOTES.md | 981 ++++++++++++++++++ .../table-parallelism-sweep/round2/PLAN.md | 89 ++ .../table-parallelism-sweep/round2/amdahl.py | 86 ++ .../round2/analyze_ab.py | 81 ++ .../round2/analyze_rt.py | 73 ++ .../round2/data/box.txt | 93 ++ .../round2/data/derived_merged_c32.csv | 64 ++ .../round2/data/jobs_ab1ep.txt | 20 + .../round2/data/jobs_ab32.txt | 72 ++ .../round2/data/jobs_ab8.txt | 40 + .../round2/data/jobs_c16.txt | 36 + .../round2/data/jobs_c32.txt | 27 + .../round2/data/jobs_c32b.txt | 36 + .../round2/data/jobs_c4.txt | 36 + .../round2/data/jobs_c4b.txt | 40 + .../round2/data/jobs_c4c.txt | 32 + .../round2/data/jobs_c8.txt | 36 + .../round2/data/jobs_c8b.txt | 40 + .../round2/data/jobs_c8c.txt | 24 + .../round2/data/jobs_instr.txt | 36 + .../round2/data/jobs_instr2.txt | 18 + .../round2/data/jobs_old32.txt | 30 + .../round2/data/jobs_rt.txt | 72 ++ .../round2/data/log_c32_k1.txt | 8 + .../round2/data/log_c32_k12.txt | 8 + .../round2/data/log_c32_k16.txt | 8 + .../round2/data/log_c32_k2.txt | 8 + .../round2/data/log_c32_k21.txt | 8 + .../round2/data/log_c32_k31.txt | 8 + .../round2/data/log_c32_k4.txt | 8 + .../round2/data/log_c32_k6.txt | 8 + .../round2/data/log_c32_k8.txt | 8 + .../round2/data/log_c32b_k12.txt | 8 + .../round2/data/log_c32b_k16.txt | 8 + .../round2/data/log_c32b_k21.txt | 8 + .../round2/data/log_c32b_k31.txt | 8 + .../round2/data/log_c32b_k6.txt | 8 + .../round2/data/log_c32b_k8.txt | 8 + .../round2/data/log_c4_cli_stock_k1.txt | 8 + .../round2/data/log_c4_cli_stock_k12.txt | 8 + .../round2/data/log_c4_cli_stock_k16.txt | 8 + .../round2/data/log_c4_cli_stock_k2.txt | 8 + .../round2/data/log_c4_cli_stock_k21.txt | 8 + .../round2/data/log_c4_cli_stock_k31.txt | 8 + .../round2/data/log_c4_cli_stock_k4.txt | 8 + .../round2/data/log_c4_cli_stock_k6.txt | 8 + .../round2/data/log_c4_cli_stock_k8.txt | 8 + .../round2/data/log_c4b_cli_stock_k12.txt | 8 + .../round2/data/log_c4b_cli_stock_k16.txt | 8 + .../round2/data/log_c4b_cli_stock_k21.txt | 8 + .../round2/data/log_c4b_cli_stock_k31.txt | 8 + .../round2/data/log_c4b_cli_stock_k8.txt | 8 + .../round2/data/log_c4c_cli_stock_k1.txt | 8 + .../round2/data/log_c4c_cli_stock_k2.txt | 8 + .../round2/data/log_c4c_cli_stock_k4.txt | 8 + .../round2/data/log_c4c_cli_stock_k6.txt | 8 + .../round2/data/log_c8_cli_stock_k1.txt | 8 + .../round2/data/log_c8_cli_stock_k12.txt | 8 + .../round2/data/log_c8_cli_stock_k16.txt | 8 + .../round2/data/log_c8_cli_stock_k2.txt | 8 + .../round2/data/log_c8_cli_stock_k21.txt | 8 + .../round2/data/log_c8_cli_stock_k31.txt | 8 + .../round2/data/log_c8_cli_stock_k4.txt | 8 + .../round2/data/log_c8_cli_stock_k6.txt | 8 + .../round2/data/log_c8_cli_stock_k8.txt | 8 + .../round2/data/log_c8b_cli_stock_k12.txt | 8 + .../round2/data/log_c8b_cli_stock_k16.txt | 8 + .../round2/data/log_c8b_cli_stock_k21.txt | 8 + .../round2/data/log_c8b_cli_stock_k31.txt | 8 + .../round2/data/log_c8b_cli_stock_k8.txt | 8 + .../round2/data/log_c8c_cli_stock_k2.txt | 8 + .../round2/data/log_c8c_cli_stock_k4.txt | 8 + .../round2/data/log_c8c_cli_stock_k6.txt | 8 + .../round2/data/patch_instr.diff | 110 ++ .../round2/data/patch_instr2.diff | 145 +++ .../round2/data/patch_slots.diff | 37 + .../round2/data/progress.log | 215 ++++ .../round2/data/results_ab1ep.csv | 21 + .../round2/data/results_ab32.csv | 73 ++ .../round2/data/results_ab8.csv | 41 + .../round2/data/results_c16.csv | 37 + .../round2/data/results_c32.csv | 28 + .../round2/data/results_c32b.csv | 37 + .../round2/data/results_c4.csv | 37 + .../round2/data/results_c4b.csv | 41 + .../round2/data/results_c4c.csv | 33 + .../round2/data/results_c8.csv | 37 + .../round2/data/results_c8b.csv | 41 + .../round2/data/results_c8c.csv | 25 + .../round2/data/results_instr.csv | 37 + .../round2/data/results_instr2.csv | 19 + .../round2/data/results_old32.csv | 31 + .../round2/data/results_rt.csv | 73 ++ .../round2/data/slotproof.txt | 81 ++ .../round2/followup2.sh | 28 + .../table-parallelism-sweep/round2/pull.sh | 26 + .../table-parallelism-sweep/round2/rules2.py | 141 +++ .../round2/stacksample.sh | 31 + .../round2/sweep_ab.sh | 56 + .../round2/sweep_instr.sh | 62 ++ .../table-parallelism-sweep/round2/sweep_r.sh | 61 ++ .../round2/sweep_rayon.sh | 54 + 120 files changed, 5109 insertions(+) create mode 100644 scripts/profiling/table-parallelism-sweep/README.md create mode 100644 scripts/profiling/table-parallelism-sweep/round1/README.md create mode 100644 scripts/profiling/table-parallelism-sweep/round1/analyze.py create mode 100644 scripts/profiling/table-parallelism-sweep/round1/classify_stacks.py create mode 100644 scripts/profiling/table-parallelism-sweep/round1/confirm.sh create mode 100644 scripts/profiling/table-parallelism-sweep/round1/confirm_analyze.py create mode 100644 scripts/profiling/table-parallelism-sweep/round1/followup.sh create mode 100644 scripts/profiling/table-parallelism-sweep/round1/followup2.sh create mode 100644 scripts/profiling/table-parallelism-sweep/round1/followup_results.txt create mode 100644 scripts/profiling/table-parallelism-sweep/round1/gpuutil.sh create mode 100644 scripts/profiling/table-parallelism-sweep/round1/results_c16.csv create mode 100644 scripts/profiling/table-parallelism-sweep/round1/results_c32.csv create mode 100644 scripts/profiling/table-parallelism-sweep/round1/results_c4.csv create mode 100644 scripts/profiling/table-parallelism-sweep/round1/results_c8.csv create mode 100644 scripts/profiling/table-parallelism-sweep/round1/results_confirm.csv create mode 100644 scripts/profiling/table-parallelism-sweep/round1/rules.py create mode 100644 scripts/profiling/table-parallelism-sweep/round1/stacksample.sh create mode 100644 scripts/profiling/table-parallelism-sweep/round1/sweep.sh create mode 100644 scripts/profiling/table-parallelism-sweep/round2/NOTES.md create mode 100644 scripts/profiling/table-parallelism-sweep/round2/PLAN.md create mode 100644 scripts/profiling/table-parallelism-sweep/round2/amdahl.py create mode 100644 scripts/profiling/table-parallelism-sweep/round2/analyze_ab.py create mode 100644 scripts/profiling/table-parallelism-sweep/round2/analyze_rt.py create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/box.txt create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/derived_merged_c32.csv create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/jobs_ab1ep.txt create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/jobs_ab32.txt create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/jobs_ab8.txt create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/jobs_c16.txt create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/jobs_c32.txt create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/jobs_c32b.txt create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/jobs_c4.txt create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/jobs_c4b.txt create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/jobs_c4c.txt create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/jobs_c8.txt create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/jobs_c8b.txt create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/jobs_c8c.txt create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/jobs_instr.txt create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/jobs_instr2.txt create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/jobs_old32.txt create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/jobs_rt.txt create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/log_c32_k1.txt create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/log_c32_k12.txt create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/log_c32_k16.txt create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/log_c32_k2.txt create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/log_c32_k21.txt create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/log_c32_k31.txt create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/log_c32_k4.txt create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/log_c32_k6.txt create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/log_c32_k8.txt create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/log_c32b_k12.txt create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/log_c32b_k16.txt create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/log_c32b_k21.txt create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/log_c32b_k31.txt create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/log_c32b_k6.txt create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/log_c32b_k8.txt create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/log_c4_cli_stock_k1.txt create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/log_c4_cli_stock_k12.txt create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/log_c4_cli_stock_k16.txt create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/log_c4_cli_stock_k2.txt create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/log_c4_cli_stock_k21.txt create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/log_c4_cli_stock_k31.txt create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/log_c4_cli_stock_k4.txt create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/log_c4_cli_stock_k6.txt create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/log_c4_cli_stock_k8.txt create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/log_c4b_cli_stock_k12.txt create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/log_c4b_cli_stock_k16.txt create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/log_c4b_cli_stock_k21.txt create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/log_c4b_cli_stock_k31.txt create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/log_c4b_cli_stock_k8.txt create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/log_c4c_cli_stock_k1.txt create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/log_c4c_cli_stock_k2.txt create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/log_c4c_cli_stock_k4.txt create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/log_c4c_cli_stock_k6.txt create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/log_c8_cli_stock_k1.txt create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/log_c8_cli_stock_k12.txt create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/log_c8_cli_stock_k16.txt create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/log_c8_cli_stock_k2.txt create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/log_c8_cli_stock_k21.txt create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/log_c8_cli_stock_k31.txt create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/log_c8_cli_stock_k4.txt create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/log_c8_cli_stock_k6.txt create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/log_c8_cli_stock_k8.txt create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/log_c8b_cli_stock_k12.txt create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/log_c8b_cli_stock_k16.txt create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/log_c8b_cli_stock_k21.txt create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/log_c8b_cli_stock_k31.txt create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/log_c8b_cli_stock_k8.txt create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/log_c8c_cli_stock_k2.txt create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/log_c8c_cli_stock_k4.txt create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/log_c8c_cli_stock_k6.txt create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/patch_instr.diff create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/patch_instr2.diff create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/patch_slots.diff create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/progress.log create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/results_ab1ep.csv create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/results_ab32.csv create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/results_ab8.csv create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/results_c16.csv create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/results_c32.csv create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/results_c32b.csv create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/results_c4.csv create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/results_c4b.csv create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/results_c4c.csv create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/results_c8.csv create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/results_c8b.csv create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/results_c8c.csv create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/results_instr.csv create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/results_instr2.csv create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/results_old32.csv create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/results_rt.csv create mode 100644 scripts/profiling/table-parallelism-sweep/round2/data/slotproof.txt create mode 100755 scripts/profiling/table-parallelism-sweep/round2/followup2.sh create mode 100755 scripts/profiling/table-parallelism-sweep/round2/pull.sh create mode 100644 scripts/profiling/table-parallelism-sweep/round2/rules2.py create mode 100755 scripts/profiling/table-parallelism-sweep/round2/stacksample.sh create mode 100644 scripts/profiling/table-parallelism-sweep/round2/sweep_ab.sh create mode 100644 scripts/profiling/table-parallelism-sweep/round2/sweep_instr.sh create mode 100755 scripts/profiling/table-parallelism-sweep/round2/sweep_r.sh create mode 100644 scripts/profiling/table-parallelism-sweep/round2/sweep_rayon.sh diff --git a/scripts/profiling/table-parallelism-sweep/README.md b/scripts/profiling/table-parallelism-sweep/README.md new file mode 100644 index 000000000..865169b01 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/README.md @@ -0,0 +1,55 @@ +# `TABLE_PARALLELISM` (K) sweep + +The measurement record behind the `cuda` default in `stark::prover::table_parallelism` +(`K = num_airs`, no core term). 881 timed runs over two RTX 5090 boxes, both against +`7644043b` (`origin/main` with #875 and #877). + +Workload throughout, the repo's own GPU bench: + +``` +cli prove ethrex.elf --private-input ethrex_10_transfers.bin --continuations --epoch-size-log2 21 +``` + +6.807M cycles, 4 epochs, ~31 tables per epoch. + +- `round1/` — 2026-08-03, 342 runs, Ryzen 9 7950X**3D**. Established that the optimum + does not move with host core count. Its `README.md` recommends a constant 12; round 2 + supersedes that (see below), so read round 1 for the core-invariance evidence, not for + the recommendation. +- `round2/` — 2026-08-04, 539 runs, Ryzen 9 7950X (no 3D V-cache). Higher n at the low + core counts, plus the mechanism work. `NOTES.md` is the full write-up and its first + section is the summary. + +## Reproducing the headline numbers + +``` +cd round2/data +python3 ../rules2.py # rule-cost table, both boxes +python3 ../amdahl.py derived_merged_c32.csv # T(K) = S + max(Tmax, W/K) fit +python3 ../analyze_rt.py results_rt.csv # RAYON_NUM_THREADS decomposition +cd ../../round1 && python3 rules.py # round 1's own table +``` + +`rules2.py` prints the table the PR quotes: worst case over four core counts is +`cores/3` +23.2 % (+30.2 % on round 1), `cores*2/3` +13.0 %, constant 12 +7.0 %, +`num_airs` +1.6 %. + +Peak host heap versus K comes from the `Peak heap:` line in the per-run logs: + +``` +grep -h "Peak heap" round2/data/log_c32_k{1,8,16,31}.txt +``` + +## What is here and what is not + +Included: both write-ups, every timing CSV (one row per run, `tag,cpuspec,epoch,k,rep,seconds`), +the analysis scripts, the server-side sweep harnesses, the box/toolchain fingerprint +(`round2/data/box.txt`), the frozen job orders, the stage timeline, and the diffs for the +three experimental builds (`patch_*.diff` — instrumentation and the refuted per-driver-slot +change, none of it for merge). + +Left out to keep this reviewable: cargo build logs, the per-run logs for the stages whose +hypotheses were refuted (their CSVs are here), and the Stage 1a stack samples, which never +produced data — the rented container dropped `cap_sys_ptrace`, so gdb could not attach. + +Both boxes were hourly rentals and are gone; the `ssh` lines in the write-ups are dead. diff --git a/scripts/profiling/table-parallelism-sweep/round1/README.md b/scripts/profiling/table-parallelism-sweep/round1/README.md new file mode 100644 index 000000000..33f7839ce --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round1/README.md @@ -0,0 +1,101 @@ +# TABLE_PARALLELISM (K) sweep — 2026-08-03 + +Measured on branch `gpu-opt-table-scheduler` (PR #877, merged to main as `7644043b`), to test whether +#877's `TABLE_PARALLELISM` default of `(cores * 2 / 3)` on cuda builds is the right *form*. + +**Conclusion: optimal K is device/workload-bound, not core-bound.** The knee sits at K≈8 across an +8× range of core counts, so scaling K by host cores is the wrong shape. A constant ~12 is the best +worst-case rule of those tested. + +## Box + +RTX 5090 (32607 MiB, driver 595.71.05, nvcc 13.1) · AMD Ryzen 9 7950X3D, 16 physical / **32 logical** +(SMT on) · 93 GB RAM · Ubuntu 24.04. Rented, since expired. + +Note: cudarc's `cuda-12080` pin is a **floor**, not a version match — built clean on nvcc 13.1. + +## Workload + +The repo's own GPU-bench workload: +`cli prove ethrex.elf --private-input ethrex_10_transfers.bin --continuations --epoch-size-log2 21` +→ 6.807M cycles, 4 epochs, ~31 tables/epoch. Randomized-interleaved K order, warmup discarded. +GPU verified not thermally throttling throughout (47 °C, no clock-event reasons). + +## Results — knee is core-invariant + +| `available_parallelism` | knee @5% | best K | best time | +|---|---|---|---| +| 32 (full) | 8 | 31 | 9.80s | +| 16 (`taskset -c 0-15`) | 8 | 26 | 9.99s | +| 8 (`-c 0-7`) | 8 | 8 | 13.28s | +| 4 (`-c 0-3`) | 6 | 8 | 21.16s | + +`cores*2/3` would place K at 21 / 10 / 5 / 2 respectively. The knee does not move. + +Worst-case cost vs the best measured K in each config: + +| rule | c32 | c16 | c8 | c4 | worst | +|---|---|---|---|---|---| +| `cores/3` (CPU arm) | +4.5% | +14.3% | +19.1% | +30.2% | **+30.2%** | +| `cores*2/3` (merged) | +1.4% | +5.3% | +7.7% | +11.0% | **+11.0%** | +| constant 8 | +4.8% | +3.5% | 0% | 0% | +4.8% | +| **constant 12** | +3.8% | +2.8% | +3.8% | +2.0% | **+3.8%** | +| unbounded (`min(num_airs)`) | 0% | +2.5% | +5.9% | +4.3% | +5.9% | + +Anything in **K ∈ [8,16]** is within noise at every core count tested. The ratio's entire error is at +low core counts, where it starves the scheduler. + +High-n confirmation (n=9, Mann-Whitney vs best K): at 8 cores **nothing from K=8 upward is +distinguishable** (K=8 +1.4%, p=0.63). At 32 cores K=21 is +2.5% (p=0.07, ns). + +## Why the formula is wrong beyond the timings + +1. **K is not a thread count.** `run_admitted` spawns K OS driver threads off a shared atomic cursor, + but each table's work runs on the **global rayon pool**, already sized from + `available_parallelism()`. There is no `ThreadPoolBuilder` in production code. Core-scaling is + already handled by pool width; K is orthogonal to it. +2. **The formula doesn't reproduce its own source observation.** `available_parallelism()` returns + *logical* CPUs. On this box (likely the same spec as the author's) that is 32, so `cores*2/3` = **21**, + not the 10 the commit message reports sweeping. Toggling SMT changes the default 2× on identical + hardware. +3. **The stated mechanism does not hold.** Mean GPU utilization never exceeded ~38% at any K + (K=1 ≈20%, K=8 ≈38%, K=31 ≈33%). The premise that in-flight tables "mostly sit in GPU waits" is + false — the GPU is idle ~2/3 of the time even at max K. Consistent with #863's + CPU-bound-at-the-serial-producer finding. + +## Also measured + +- **`VramGate` never binds at the default budget**: default (~26 GB) 9.55s ≈ disabled (1 TB) 9.53s ≈ + 8 GB 9.53s; only 3 GB bites (10.78s, +13%). An 8 GB budget (3.3× cut) cost *nothing* while K=4 cost + 12% — so large K is not about running more *heavy* tables concurrently. +- **`num_airs` caps K**: K = 26/31/40/64/128/1024 all identical (9.50–9.98s, no trend). + +## Caveats + +- The knee is core-invariant, but the *top* of the flat region shows a real ~5% gain at 32 cores that + is absent at 8 — a weak core interaction in the tail, nowhere near the 4× the ratio asserts. +- The serial continuation producer is K-independent and grows as a share of wall time as cores + shrink, compressing relative differences at low core counts. That biases *toward* "K scales with + cores", so finding invariance despite it is conservative. +- **Untested:** the pinned-staging hypothesis — all K driver threads collide on + `pinned_staging` slot 0 because `worker_slot()` is `rayon::current_thread_index().unwrap_or(0)` and + they are not rayon workers. Consistent with everything measured (a single mutex saturates at a + contender count set by the staging/compute ratio, independent of cores and VRAM) but not + demonstrated. To test: build with `CARGO_PROFILE_RELEASE_DEBUG=2` for symbols, then sample + `gdb -p PID -batch -ex "thread apply all bt 14"` (~10×/run) and classify — see `stacksample.sh` / + `classify_stacks.py`. +- **#863 reconciliation is a hypothesis, not measured.** #863 found K saturating at 3, but measured the + *old* fixed-chunk scheduler where K was a chunk size with a barrier, so raising K raised straggler + cost. #877's work-stealing removes the barrier. Isolation point: `1e1e0f18`, the last commit with + `plan_table_chunks`. + +## Files + +`results_c{32,16,8,4}.csv` + `results_confirm.csv` — 347 timed runs, columns +`tag,cpuspec,epoch,k,rep,seconds`. `followup_results.txt` — VRAM-budget, K-cap, GPU-util and thermal +data verbatim. Harnesses: `sweep.sh`, `confirm.sh`, `followup.sh`, `followup2.sh` (RAYON_NUM_THREADS +decomposition — written, never ran), `gpuutil.sh`, `stacksample.sh`. Analyzers: `analyze.py`, +`rules.py`, `confirm_analyze.py`, `classify_stacks.py`. + +(`sim_ntt.py` / `sim_merkle.py` / `stage_hunks.py` in this directory are from the #875 review, not +this sweep.) diff --git a/scripts/profiling/table-parallelism-sweep/round1/analyze.py b/scripts/profiling/table-parallelism-sweep/round1/analyze.py new file mode 100644 index 000000000..c7ad8461f --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round1/analyze.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 +"""Summarize K sweep CSVs: median, min, max, IQR per K; knee = smallest K within +tol of the config's best median.""" +import csv, sys, statistics as st +from collections import defaultdict + +def load(path): + d = defaultdict(list) + cpus = epoch = None + with open(path) as f: + for r in csv.DictReader(f): + if r["seconds"] == "NA": + continue + d[int(r["k"])].append(float(r["seconds"])) + cpus, epoch = r["cpuspec"], r["epoch"] + return d, cpus, epoch + +def knee(meds, tol): + best = min(meds.values()) + for k in sorted(meds): + if meds[k] <= best * (1 + tol): + return k, best + return None, best + +for path in sys.argv[1:]: + d, cpus, epoch = load(path) + meds = {k: st.median(v) for k, v in d.items()} + best = min(meds.values()) + bestk = min(meds, key=lambda k: meds[k]) + print(f"\n=== {path} cpus={cpus} epoch=2^{epoch} ===") + print(f"{'K':>4} {'n':>3} {'median':>8} {'min':>8} {'max':>8} {'spread%':>8} {'vs best':>8}") + for k in sorted(d): + v = sorted(d[k]) + sp = (v[-1] - v[0]) / st.median(v) * 100 + print(f"{k:>4} {len(v):>3} {st.median(v):>8.2f} {v[0]:>8.2f} {v[-1]:>8.2f} " + f"{sp:>7.1f}% {meds[k]/best:>7.3f}x") + for tol in (0.02, 0.05): + k, b = knee(meds, tol) + print(f" knee@{int(tol*100)}%: K={k} (best K={bestk} @ {best:.2f}s)") diff --git a/scripts/profiling/table-parallelism-sweep/round1/classify_stacks.py b/scripts/profiling/table-parallelism-sweep/round1/classify_stacks.py new file mode 100644 index 000000000..864838817 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round1/classify_stacks.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +"""Classify gdb thread snapshots into blocking categories. + +Splits each snapshot into per-thread stacks, decides whether the thread is a +per-table scheduler DRIVER (contains run_admitted) or a rayon worker, and what +it is blocked on. Reports counts so we can tell a pinned-staging mutex stall +from a CUDA wait from a VramGate admission wait. +""" +import re, sys +from collections import Counter, defaultdict + + +def classify(stack: str): + s = stack + low = s.lower() + # Ordered: most specific first. A frame deeper in the stack tells us the + # *reason* we are in a lock/condvar, so check callers before generic waits. + if "vramgate" in low or "VramGate" in s: + return "VramGate admission wait" + if "pinned" in low and ("lock" in low or "mutex" in low): + return "pinned-staging mutex" + if re.search(r"async_dtoh_via|async_htod|htod_via|pinned_staging|pinned_hashes", s): + # inside a staging transfer; distinguish blocked-on-lock vs doing the copy + if re.search(r"lll_lock_wait|__futex|pthread_mutex_lock|futex_wait", s): + return "pinned-staging mutex" + return "staging transfer (running)" + if re.search(r"cuStreamSynchronize|cuEventSynchronize|cuCtxSynchronize|cuMemcpy|cuEventQuery", s): + return "CUDA sync / memcpy" + if re.search(r"cuMemHostAlloc|cuMemAllocHost|cuMemFreeHost", s): + return "pinned alloc" + if re.search(r"cuMemAllocAsync|cuMemAllocFromPoolAsync|cuMemFreeAsync|cuMemAlloc", s): + return "device alloc" + if "rayon_core" in s and re.search(r"sleep|idle|steal", low): + return "rayon idle" + if re.search(r"lll_lock_wait|pthread_mutex_lock|futex_wait", s): + return "other mutex/futex wait" + if re.search(r"pthread_cond_wait|condvar", low): + return "other condvar wait" + if "libcuda" in s: + return "in libcuda (running)" + return "running / other" + + +def role(stack: str): + if "run_admitted" in stack: + return "driver" + if "rayon_core" in stack or "rayon::" in stack: + return "rayon worker" + if "continuation" in stack and "execute" in stack: + return "producer" + return "other" + + +for path in sys.argv[1:]: + text = open(path, errors="replace").read() + samples = text.split("===== SAMPLE") + per_role = defaultdict(Counter) + nthreads = 0 + for samp in samples[1:]: + # gdb prints "Thread N (Thread 0x... (LWP ...)):" then frames + chunks = re.split(r"\nThread \d+ \(", samp) + for ch in chunks[1:]: + nthreads += 1 + per_role[role(ch)][classify(ch)] += 1 + print(f"\n=== {path} === thread-snapshots={nthreads}") + for r in ("driver", "rayon worker", "producer", "other"): + if not per_role[r]: + continue + tot = sum(per_role[r].values()) + print(f" {r} (n={tot})") + for cat, c in per_role[r].most_common(): + print(f" {c/tot*100:5.1f}% {c:5d} {cat}") diff --git a/scripts/profiling/table-parallelism-sweep/round1/confirm.sh b/scripts/profiling/table-parallelism-sweep/round1/confirm.sh new file mode 100644 index 000000000..377dd8352 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round1/confirm.sh @@ -0,0 +1,32 @@ +#!/bin/bash +# High-n head-to-head of the candidate rules at two core counts. +# At 32 cores: ratio rule -> K=21. At 8 cores: ratio rule -> K=5. +# Constant candidates: 8, 12, 16. Unbounded: 31. +set -u +CLI=/root/lambda_vm/target/release/cli +ELF=/root/ethrex.elf +INPUT=/root/lambda_vm/executor/tests/ethrex_10_transfers.bin +OUT=/root/results_confirm.csv +REPS=9 +echo "tag,cpuspec,epoch,k,rep,seconds" > "$OUT" + +# (cpuspec, K) pairs, all interleaved in one shuffled stream so drift hits every +# cell equally rather than accumulating within a config. +JOBS=$(for spec in none 0-7; do + for k in 5 8 12 16 21 31; do + for r in $(seq 1 $REPS); do echo "$spec $k $r"; done + done + done | shuf) + +TOTAL=$(echo "$JOBS" | wc -l); i=0 +echo "$JOBS" | while read -r spec k r; do + i=$((i+1)) + if [ "$spec" = "none" ]; then PREFIX=""; else PREFIX="taskset -c $spec"; fi + t=$(TABLE_PARALLELISM=$k $PREFIX "$CLI" prove "$ELF" --private-input "$INPUT" \ + -o /tmp/pc.bin --time --continuations --epoch-size-log2 21 2>&1 \ + | sed -n 's/^Proving time: \([0-9.]*\)s/\1/p') + [ -z "$t" ] && t="NA" + echo "confirm_$spec,$spec,21,$k,$r,$t" >> "$OUT" + echo "[$i/$TOTAL] cpus=$spec K=$k rep=$r -> ${t}s" +done +echo DONE_CONFIRM diff --git a/scripts/profiling/table-parallelism-sweep/round1/confirm_analyze.py b/scripts/profiling/table-parallelism-sweep/round1/confirm_analyze.py new file mode 100644 index 000000000..00e9e58b3 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round1/confirm_analyze.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +"""High-n confirm run: split by cpuspec, report median/IQR and a Mann-Whitney +test of each K against that config's best K (is the tail difference real?).""" +import csv, statistics as st, itertools +from collections import defaultdict + +d = defaultdict(lambda: defaultdict(list)) +with open("results_confirm.csv") as f: + for r in csv.DictReader(f): + if r["seconds"] != "NA": + d[r["cpuspec"]][int(r["k"])].append(float(r["seconds"])) + + +def mannwhitney_u(a, b): + """Exact-ish U + normal-approx two-sided p. Small n, no scipy.""" + n1, n2 = len(a), len(b) + allv = [(v, 0) for v in a] + [(v, 1) for v in b] + allv.sort() + ranks = {} + i = 0 + while i < len(allv): + j = i + while j + 1 < len(allv) and allv[j + 1][0] == allv[i][0]: + j += 1 + r = (i + j) / 2 + 1 + for k in range(i, j + 1): + ranks[k] = r + i = j + 1 + r1 = sum(ranks[i] for i, (_, g) in enumerate(allv) if g == 0) + u1 = r1 - n1 * (n1 + 1) / 2 + u = min(u1, n1 * n2 - u1) + mu = n1 * n2 / 2 + sd = (n1 * n2 * (n1 + n2 + 1) / 12) ** 0.5 + if sd == 0: + return u, 1.0 + z = (u - mu) / sd + # two-sided normal approx + from math import erf, sqrt + p = 2 * (0.5 * (1 + erf(-abs(z) / sqrt(2)))) + return u, p + + +LABEL = {"none": "32 cores (full)", "0-7": "8 cores (taskset 0-7)"} +for spec in ("none", "0-7"): + ks = d[spec] + if not ks: + continue + meds = {k: st.median(v) for k, v in ks.items()} + bk = min(meds, key=lambda k: meds[k]) + print(f"\n=== {LABEL.get(spec, spec)} === best K={bk} @ {meds[bk]:.2f}s") + print(f"{'K':>4} {'n':>3} {'median':>8} {'IQR':>14} {'vs best':>9} {'p vs best K':>12}") + for k in sorted(ks): + v = sorted(ks[k]) + q1 = st.median(v[: len(v) // 2]) + q3 = st.median(v[(len(v) + 1) // 2 :]) + _, p = mannwhitney_u(ks[k], ks[bk]) + star = "" if k == bk else (" <-- sig" if p < 0.05 else " ns") + print(f"{k:>4} {len(v):>3} {st.median(v):>8.2f} " + f"{q1:>6.2f}-{q3:<6.2f} {meds[k]/meds[bk]:>8.3f}x " + f"{'—' if k==bk else format(p,'.3f'):>12}{star}") diff --git a/scripts/profiling/table-parallelism-sweep/round1/followup.sh b/scripts/profiling/table-parallelism-sweep/round1/followup.sh new file mode 100644 index 000000000..9be887ea3 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round1/followup.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# Post-sweep experiments. Run only after the main K sweep is done. +set -u +CLI=/root/lambda_vm/target/release/cli +ELF=/root/ethrex.elf +INPUT=/root/lambda_vm/executor/tests/ethrex_10_transfers.bin + +run() { # $1=label $2=env assignments $3=K + local t + t=$(env $2 TABLE_PARALLELISM=$3 "$CLI" prove "$ELF" --private-input "$INPUT" \ + -o /tmp/f.bin --time --continuations --epoch-size-log2 21 2>&1 \ + | sed -n 's/^Proving time: \([0-9.]*\)s/\1/p') + echo "$1 K=$3 -> ${t}s" +} + +echo "===== 1. Does VramGate ever bind? (default budget = 80% of 32607MiB = ~26GB) =====" +for rep in 1 2 3; do + run "budget=default " "DUMMY=1" 31 + run "budget=1000000MB" "LAMBDA_VM_VRAM_BUDGET_MB=1000000" 31 + run "budget=8000MB " "LAMBDA_VM_VRAM_BUDGET_MB=8000" 31 + run "budget=3000MB " "LAMBDA_VM_VRAM_BUDGET_MB=3000" 31 +done + +echo +echo "===== 2. Is K capped by num_airs? (K>=num_airs should be identical) =====" +for rep in 1 2 3; do + for k in 26 31 40 64 128 1024; do run "cap" "DUMMY=1" $k; done +done + +echo +echo "===== 3. GPU utilization vs K =====" +for k in 1 3 8 16 31; do /root/gpuutil.sh $k 21 none; done + +echo +echo "===== 4. GPU utilization vs K at 8 cores =====" +for k in 1 3 8 16 31; do /root/gpuutil.sh $k 21 0-7; done + +echo FOLLOWUP_DONE diff --git a/scripts/profiling/table-parallelism-sweep/round1/followup2.sh b/scripts/profiling/table-parallelism-sweep/round1/followup2.sh new file mode 100644 index 000000000..3de7d98d6 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round1/followup2.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Decompose "cores" into rayon pool width vs machine width. +# taskset shrinks BOTH (pool + the K driver threads' CPUs). +# RAYON_NUM_THREADS shrinks ONLY the pool; K drivers still get real cores. +set -u +CLI=/root/lambda_vm/target/release/cli +ELF=/root/ethrex.elf +INPUT=/root/lambda_vm/executor/tests/ethrex_10_transfers.bin +OUT=/root/results_rayon.csv +echo "tag,cpuspec,epoch,k,rep,seconds" > "$OUT" + +JOBS=$(for rt in 4 8 32; do + for k in 1 2 3 4 6 8 12 16 21 31; do + for r in 1 2 3; do echo "$rt $k $r"; done + done + done | shuf) + +TOTAL=$(echo "$JOBS" | wc -l); i=0 +echo "$JOBS" | while read -r rt k r; do + i=$((i+1)) + t=$(RAYON_NUM_THREADS=$rt TABLE_PARALLELISM=$k "$CLI" prove "$ELF" \ + --private-input "$INPUT" -o /tmp/pr.bin --time --continuations \ + --epoch-size-log2 21 2>&1 | sed -n 's/^Proving time: \([0-9.]*\)s/\1/p') + [ -z "$t" ] && t="NA" + echo "rayon$rt,rayon=$rt,21,$k,$r,$t" >> "$OUT" + echo "[$i/$TOTAL] RAYON_NUM_THREADS=$rt K=$k rep=$r -> ${t}s" +done +echo DONE_RAYON diff --git a/scripts/profiling/table-parallelism-sweep/round1/followup_results.txt b/scripts/profiling/table-parallelism-sweep/round1/followup_results.txt new file mode 100644 index 000000000..caf530621 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round1/followup_results.txt @@ -0,0 +1,56 @@ +Verbatim followup.log from the box (79.161.122.162, RTX 5090, 32 logical cores), +PR #877 branch gpu-opt-table-scheduler @ d2359123, cuda build. +Workload: cli prove ethrex.elf --private-input ethrex_10_transfers.bin + --continuations --epoch-size-log2 21 (6.807M cycles, 4 epochs) + +===== 1. Does VramGate ever bind? (default budget = 80% of 32607MiB = ~26GB) ===== +budget=default K=31 -> 9.314s +budget=1000000MB K=31 -> 9.525s +budget=8000MB K=31 -> 9.347s +budget=3000MB K=31 -> 10.980s +budget=default K=31 -> 9.549s +budget=1000000MB K=31 -> 9.599s +budget=8000MB K=31 -> 9.530s +budget=3000MB K=31 -> 10.784s +budget=default K=31 -> 9.869s +budget=1000000MB K=31 -> 9.281s +budget=8000MB K=31 -> 9.717s +budget=3000MB K=31 -> 10.610s + medians: default 9.549 | disabled(1e6MB) 9.525 | 8000MB 9.530 | 3000MB 10.784 + +===== 2. Is K capped by num_airs? (K>=num_airs should be identical) ===== +cap K=26 -> 9.592s / 9.508s / 9.728s median 9.592 +cap K=31 -> 9.581s / 9.704s / 9.531s median 9.581 +cap K=40 -> 10.288s / 9.983s / 9.463s median 9.983 +cap K=64 -> 9.476s / 9.535s / 9.500s median 9.500 +cap K=128 -> 9.883s / 9.682s / 9.778s median 9.778 +cap K=1024 -> 10.011s / 9.405s / 9.613s median 9.613 + +===== 3. GPU utilization vs K (32 cores) ===== +(10Hz nvidia-smi sampling, LAMBDA_VM_MEMPOOL_RELEASE_MB=0 so memory.used tracks + working set not pool reservation; meanUtil averaged over samples with util>0) +K=1 time=17.463s samples=189 active=135 meanUtil=19.9% maxUtil=60% meanSMclk=2837MHz peakMem=7965MiB +K=3 time=11.905s samples=135 active=69 meanUtil=31.6% maxUtil=70% meanSMclk=2885MHz peakMem=9341MiB +K=8 time=10.435s samples=120 active=69 meanUtil=37.8% maxUtil=100% meanSMclk=2559MHz peakMem=13021MiB +K=16 time=10.086s samples=116 active=80 meanUtil=23.9% maxUtil=100% meanSMclk=2901MHz peakMem=17021MiB +K=31 time=9.750s samples=113 active=65 meanUtil=33.2% maxUtil=95% meanSMclk=2888MHz peakMem=16573MiB + +===== 4. GPU utilization vs K at 8 cores (taskset -c 0-7) ===== +K=1 time=20.340s samples=220 active=135 meanUtil=18.8% maxUtil=62% meanSMclk=2748MHz peakMem=7965MiB +K=3 time=14.910s samples=165 active=75 meanUtil=22.7% maxUtil=66% meanSMclk=2824MHz peakMem=9693MiB +K=8 time=13.817s samples=154 active=105 meanUtil=21.9% maxUtil=100% meanSMclk=2863MHz peakMem=13437MiB +K=16 time=13.571s samples=152 active=85 meanUtil=32.8% maxUtil=100% meanSMclk=2901MHz peakMem=15869MiB +K=31 time=13.038s samples=147 active=90 meanUtil=33.6% maxUtil=100% meanSMclk=2853MHz peakMem=15453MiB + +===== Thermal control (sampled mid-sweep) ===== +temperature.gpu=47C clocks.sm=2925MHz clocks.max.sm=3135MHz power.draw=129W +nvidia-smi -q -d PERFORMANCE: all Clocks Event Reasons "Not Active" +(no HW/SW thermal or power slowdown during the sweep) + +===== NOT RUN (box expired) ===== +- pinned-staging stack sampling (gdb thread-stack classification). Scripts were + written and uploaded (/root/stacksample.sh); the symbolized build + (CARGO_TARGET_DIR=/root/target_dbg CARGO_PROFILE_RELEASE_DEBUG=2) was the + command that died with the box. classify_stacks.py exists locally. +- followup2.sh: RAYON_NUM_THREADS decomposition (pool width vs machine width). +- old-scheduler (1e1e0f18) K curve for the #863-vs-#877 reconciliation. diff --git a/scripts/profiling/table-parallelism-sweep/round1/gpuutil.sh b/scripts/profiling/table-parallelism-sweep/round1/gpuutil.sh new file mode 100644 index 000000000..aa8f51a0a --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round1/gpuutil.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# Sample GPU utilization + memory at 10Hz across one prove at a given K. +# Usage: gpuutil.sh [cpuspec] +set -u +K="$1"; EPOCH="$2"; CPUSPEC="${3:-none}" +CLI=/root/lambda_vm/target/release/cli +ELF=/root/ethrex.elf +INPUT=/root/lambda_vm/executor/tests/ethrex_10_transfers.bin +SAMP=/root/gpusamp_k${K}_e${EPOCH}.csv + +if [ "$CPUSPEC" = "none" ]; then PREFIX=""; else PREFIX="taskset -c $CPUSPEC"; fi + +nvidia-smi --query-gpu=utilization.gpu,memory.used,clocks.sm,temperature.gpu \ + --format=csv,noheader,nounits -lms 100 > "$SAMP" 2>/dev/null & +SMIPID=$! +sleep 1 + +# LAMBDA_VM_MEMPOOL_RELEASE_MB=0 so memory.used tracks the working set, not the +# retained pool reservation (the pool's release threshold is otherwise u64::MAX). +T=$(LAMBDA_VM_MEMPOOL_RELEASE_MB=0 TABLE_PARALLELISM=$K $PREFIX "$CLI" prove "$ELF" \ + --private-input "$INPUT" -o /tmp/gp.bin --time --continuations \ + --epoch-size-log2 "$EPOCH" 2>&1 | sed -n 's/^Proving time: \([0-9.]*\)s/\1/p') + +kill $SMIPID 2>/dev/null; wait $SMIPID 2>/dev/null + +# Trim the leading/trailing idle samples (util==0 before start / after end). +awk -F',' -v k="$K" -v t="$T" -v cpus="$CPUSPEC" ' + { u=$1+0; m=$2+0; c=$3+0; + if (u>0) { active++; su+=u; if(u>mu) mu=u; sc+=c } + if (m>mm) mm=m; n++ } + END { + printf "K=%s cpus=%s time=%ss samples=%d active=%d meanUtil=%.1f%% maxUtil=%d%% meanSMclk=%.0fMHz peakMem=%dMiB\n", + k, cpus, t, n, active, (active?su/active:0), mu, (active?sc/active:0), mm + }' "$SAMP" diff --git a/scripts/profiling/table-parallelism-sweep/round1/results_c16.csv b/scripts/profiling/table-parallelism-sweep/round1/results_c16.csv new file mode 100644 index 000000000..260481d80 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round1/results_c16.csv @@ -0,0 +1,66 @@ +tag,cpuspec,epoch,k,rep,seconds +c16,0-15,21,21,2,10.441 +c16,0-15,21,4,5,11.036 +c16,0-15,21,2,5,13.309 +c16,0-15,21,13,3,10.357 +c16,0-15,21,1,1,17.886 +c16,0-15,21,10,3,10.427 +c16,0-15,21,26,5,9.710 +c16,0-15,21,6,4,11.376 +c16,0-15,21,6,3,11.425 +c16,0-15,21,16,3,10.185 +c16,0-15,21,10,1,10.516 +c16,0-15,21,31,4,10.218 +c16,0-15,21,26,3,10.340 +c16,0-15,21,8,5,10.336 +c16,0-15,21,31,5,10.267 +c16,0-15,21,2,4,12.788 +c16,0-15,21,3,2,12.201 +c16,0-15,21,21,5,10.225 +c16,0-15,21,1,3,17.372 +c16,0-15,21,21,1,10.064 +c16,0-15,21,3,5,12.308 +c16,0-15,21,13,5,10.188 +c16,0-15,21,6,1,10.916 +c16,0-15,21,2,2,12.968 +c16,0-15,21,13,1,10.238 +c16,0-15,21,4,4,11.369 +c16,0-15,21,13,4,10.265 +c16,0-15,21,3,4,12.235 +c16,0-15,21,3,1,12.055 +c16,0-15,21,6,5,11.421 +c16,0-15,21,1,5,17.484 +c16,0-15,21,31,1,10.235 +c16,0-15,21,16,2,10.195 +c16,0-15,21,8,2,11.025 +c16,0-15,21,16,5,10.127 +c16,0-15,21,10,2,10.429 +c16,0-15,21,4,2,10.943 +c16,0-15,21,3,3,11.925 +c16,0-15,21,31,2,10.102 +c16,0-15,21,2,1,13.071 +c16,0-15,21,21,3,10.052 +c16,0-15,21,10,4,10.596 +c16,0-15,21,5,5,11.434 +c16,0-15,21,8,4,10.122 +c16,0-15,21,21,4,9.764 +c16,0-15,21,8,1,10.184 +c16,0-15,21,4,3,11.280 +c16,0-15,21,13,2,10.331 +c16,0-15,21,5,4,11.499 +c16,0-15,21,5,2,10.994 +c16,0-15,21,5,3,11.412 +c16,0-15,21,10,5,10.879 +c16,0-15,21,1,2,17.324 +c16,0-15,21,4,1,11.468 +c16,0-15,21,16,1,10.302 +c16,0-15,21,26,4,10.028 +c16,0-15,21,16,4,10.567 +c16,0-15,21,5,1,11.115 +c16,0-15,21,6,2,10.777 +c16,0-15,21,2,3,13.093 +c16,0-15,21,1,4,17.509 +c16,0-15,21,8,3,10.740 +c16,0-15,21,31,3,10.409 +c16,0-15,21,26,1,9.758 +c16,0-15,21,26,2,9.987 diff --git a/scripts/profiling/table-parallelism-sweep/round1/results_c32.csv b/scripts/profiling/table-parallelism-sweep/round1/results_c32.csv new file mode 100644 index 000000000..285d67ca3 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round1/results_c32.csv @@ -0,0 +1,66 @@ +tag,cpuspec,epoch,k,rep,seconds +c32,none,21,3,2,11.722 +c32,none,21,3,4,11.451 +c32,none,21,5,4,11.159 +c32,none,21,10,3,10.599 +c32,none,21,10,1,9.650 +c32,none,21,3,3,11.727 +c32,none,21,1,2,17.675 +c32,none,21,1,5,17.542 +c32,none,21,8,4,10.386 +c32,none,21,21,3,10.096 +c32,none,21,21,5,10.589 +c32,none,21,16,4,9.841 +c32,none,21,10,5,10.659 +c32,none,21,5,1,11.155 +c32,none,21,13,3,10.144 +c32,none,21,31,4,9.694 +c32,none,21,16,3,10.155 +c32,none,21,26,4,9.867 +c32,none,21,13,1,10.260 +c32,none,21,31,5,9.939 +c32,none,21,26,1,10.520 +c32,none,21,10,2,10.239 +c32,none,21,8,3,10.303 +c32,none,21,26,5,9.706 +c32,none,21,21,4,9.931 +c32,none,21,31,1,10.036 +c32,none,21,6,5,10.545 +c32,none,21,4,5,11.593 +c32,none,21,2,2,13.037 +c32,none,21,3,5,11.632 +c32,none,21,10,4,10.234 +c32,none,21,6,3,10.266 +c32,none,21,4,1,10.978 +c32,none,21,8,2,10.274 +c32,none,21,13,4,9.993 +c32,none,21,21,2,9.823 +c32,none,21,5,2,10.635 +c32,none,21,1,4,17.246 +c32,none,21,21,1,9.942 +c32,none,21,16,2,10.131 +c32,none,21,8,5,10.225 +c32,none,21,31,3,9.800 +c32,none,21,2,5,12.778 +c32,none,21,1,1,17.053 +c32,none,21,13,5,10.285 +c32,none,21,5,5,10.901 +c32,none,21,31,2,9.703 +c32,none,21,4,2,10.953 +c32,none,21,6,4,10.528 +c32,none,21,2,4,12.831 +c32,none,21,4,4,11.007 +c32,none,21,26,3,9.536 +c32,none,21,3,1,11.736 +c32,none,21,13,2,10.168 +c32,none,21,1,3,16.905 +c32,none,21,4,3,10.652 +c32,none,21,26,2,10.065 +c32,none,21,8,1,10.044 +c32,none,21,2,3,12.517 +c32,none,21,2,1,13.051 +c32,none,21,16,5,9.888 +c32,none,21,16,1,10.202 +c32,none,21,6,2,10.077 +c32,none,21,6,1,10.406 +c32,none,21,5,3,10.825 diff --git a/scripts/profiling/table-parallelism-sweep/round1/results_c4.csv b/scripts/profiling/table-parallelism-sweep/round1/results_c4.csv new file mode 100644 index 000000000..ee8c47725 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round1/results_c4.csv @@ -0,0 +1,53 @@ +tag,cpuspec,epoch,k,rep,seconds +c4,0-3,21,21,3,21.016 +c4,0-3,21,26,3,21.449 +c4,0-3,21,2,1,22.932 +c4,0-3,21,2,2,23.732 +c4,0-3,21,10,1,22.525 +c4,0-3,21,1,2,28.235 +c4,0-3,21,5,2,20.917 +c4,0-3,21,3,3,22.812 +c4,0-3,21,10,3,21.542 +c4,0-3,21,2,3,24.195 +c4,0-3,21,13,3,21.921 +c4,0-3,21,1,3,26.859 +c4,0-3,21,31,4,22.160 +c4,0-3,21,6,2,22.138 +c4,0-3,21,31,2,21.972 +c4,0-3,21,31,1,21.460 +c4,0-3,21,16,3,22.065 +c4,0-3,21,3,4,22.331 +c4,0-3,21,21,1,23.534 +c4,0-3,21,8,1,21.012 +c4,0-3,21,2,4,23.264 +c4,0-3,21,16,4,21.603 +c4,0-3,21,10,2,23.255 +c4,0-3,21,3,2,22.428 +c4,0-3,21,8,3,21.229 +c4,0-3,21,26,2,20.291 +c4,0-3,21,1,1,25.657 +c4,0-3,21,13,4,21.176 +c4,0-3,21,6,4,21.628 +c4,0-3,21,8,4,21.879 +c4,0-3,21,21,2,22.593 +c4,0-3,21,4,3,23.519 +c4,0-3,21,16,1,22.520 +c4,0-3,21,4,1,22.936 +c4,0-3,21,8,2,21.100 +c4,0-3,21,4,2,22.140 +c4,0-3,21,5,4,23.796 +c4,0-3,21,6,3,23.203 +c4,0-3,21,5,1,23.475 +c4,0-3,21,13,2,21.502 +c4,0-3,21,16,2,22.401 +c4,0-3,21,5,3,21.547 +c4,0-3,21,1,4,28.766 +c4,0-3,21,10,4,21.464 +c4,0-3,21,21,4,21.071 +c4,0-3,21,3,1,23.685 +c4,0-3,21,26,1,21.848 +c4,0-3,21,31,3,22.361 +c4,0-3,21,4,4,22.206 +c4,0-3,21,13,1,21.672 +c4,0-3,21,6,1,21.718 +c4,0-3,21,26,4,22.178 diff --git a/scripts/profiling/table-parallelism-sweep/round1/results_c8.csv b/scripts/profiling/table-parallelism-sweep/round1/results_c8.csv new file mode 100644 index 000000000..94d328056 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round1/results_c8.csv @@ -0,0 +1,53 @@ +tag,cpuspec,epoch,k,rep,seconds +c8,0-7,21,6,4,14.161 +c8,0-7,21,8,1,12.616 +c8,0-7,21,1,2,20.157 +c8,0-7,21,1,1,20.119 +c8,0-7,21,13,4,13.655 +c8,0-7,21,21,2,13.953 +c8,0-7,21,4,3,14.795 +c8,0-7,21,13,1,14.640 +c8,0-7,21,10,2,14.286 +c8,0-7,21,1,4,19.958 +c8,0-7,21,3,3,15.160 +c8,0-7,21,8,3,13.859 +c8,0-7,21,16,1,14.202 +c8,0-7,21,5,1,14.336 +c8,0-7,21,6,2,14.366 +c8,0-7,21,4,4,14.886 +c8,0-7,21,2,3,15.848 +c8,0-7,21,16,4,13.911 +c8,0-7,21,3,1,15.967 +c8,0-7,21,10,4,13.967 +c8,0-7,21,31,2,14.092 +c8,0-7,21,2,1,15.803 +c8,0-7,21,2,4,15.620 +c8,0-7,21,3,2,14.692 +c8,0-7,21,21,1,13.484 +c8,0-7,21,16,2,13.932 +c8,0-7,21,31,3,13.282 +c8,0-7,21,31,1,14.612 +c8,0-7,21,6,3,13.969 +c8,0-7,21,6,1,14.082 +c8,0-7,21,26,4,13.201 +c8,0-7,21,8,2,14.580 +c8,0-7,21,13,2,13.796 +c8,0-7,21,31,4,14.035 +c8,0-7,21,8,4,12.705 +c8,0-7,21,2,2,16.362 +c8,0-7,21,13,3,13.771 +c8,0-7,21,21,3,13.805 +c8,0-7,21,26,3,14.361 +c8,0-7,21,4,1,15.378 +c8,0-7,21,26,2,13.781 +c8,0-7,21,10,3,13.732 +c8,0-7,21,5,2,14.129 +c8,0-7,21,10,1,12.855 +c8,0-7,21,26,1,12.643 +c8,0-7,21,21,4,13.827 +c8,0-7,21,4,2,13.453 +c8,0-7,21,1,3,19.766 +c8,0-7,21,16,3,13.193 +c8,0-7,21,5,4,14.580 +c8,0-7,21,5,3,14.282 +c8,0-7,21,3,4,16.122 diff --git a/scripts/profiling/table-parallelism-sweep/round1/results_confirm.csv b/scripts/profiling/table-parallelism-sweep/round1/results_confirm.csv new file mode 100644 index 000000000..8b56ef6df --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round1/results_confirm.csv @@ -0,0 +1,109 @@ +tag,cpuspec,epoch,k,rep,seconds +confirm_none,none,21,21,2,9.607 +confirm_0-7,0-7,21,21,6,14.245 +confirm_0-7,0-7,21,8,6,13.297 +confirm_0-7,0-7,21,5,2,14.946 +confirm_0-7,0-7,21,21,8,13.064 +confirm_none,none,21,16,5,9.976 +confirm_0-7,0-7,21,21,5,13.609 +confirm_0-7,0-7,21,21,3,13.648 +confirm_none,none,21,21,6,9.908 +confirm_0-7,0-7,21,16,9,13.182 +confirm_0-7,0-7,21,5,6,13.904 +confirm_none,none,21,31,6,9.851 +confirm_none,none,21,5,2,10.533 +confirm_none,none,21,5,9,10.502 +confirm_none,none,21,16,8,10.013 +confirm_none,none,21,16,9,10.124 +confirm_none,none,21,16,1,10.340 +confirm_0-7,0-7,21,31,5,12.997 +confirm_none,none,21,8,4,10.268 +confirm_0-7,0-7,21,5,7,13.980 +confirm_none,none,21,5,5,10.458 +confirm_0-7,0-7,21,16,8,13.586 +confirm_0-7,0-7,21,31,9,13.462 +confirm_none,none,21,31,1,9.731 +confirm_0-7,0-7,21,21,1,13.527 +confirm_none,none,21,12,2,10.251 +confirm_0-7,0-7,21,21,4,13.719 +confirm_0-7,0-7,21,21,9,14.016 +confirm_none,none,21,12,4,10.312 +confirm_none,none,21,8,8,10.495 +confirm_0-7,0-7,21,31,6,14.068 +confirm_none,none,21,31,3,9.466 +confirm_0-7,0-7,21,8,8,13.949 +confirm_none,none,21,21,1,10.264 +confirm_0-7,0-7,21,5,1,13.754 +confirm_none,none,21,12,6,10.200 +confirm_none,none,21,8,5,9.815 +confirm_none,none,21,12,7,9.691 +confirm_0-7,0-7,21,5,9,14.700 +confirm_none,none,21,16,7,9.908 +confirm_0-7,0-7,21,12,4,13.051 +confirm_none,none,21,16,3,9.651 +confirm_0-7,0-7,21,12,7,13.555 +confirm_0-7,0-7,21,16,7,13.536 +confirm_none,none,21,21,7,9.846 +confirm_none,none,21,31,7,9.465 +confirm_0-7,0-7,21,8,3,13.831 +confirm_none,none,21,31,8,9.885 +confirm_none,none,21,5,3,10.331 +confirm_0-7,0-7,21,8,4,13.789 +confirm_0-7,0-7,21,8,9,13.088 +confirm_0-7,0-7,21,31,8,14.707 +confirm_0-7,0-7,21,8,7,13.036 +confirm_0-7,0-7,21,12,1,12.936 +confirm_none,none,21,12,8,10.129 +confirm_0-7,0-7,21,5,8,14.775 +confirm_none,none,21,8,7,10.231 +confirm_none,none,21,8,1,10.099 +confirm_none,none,21,5,6,10.825 +confirm_0-7,0-7,21,16,5,13.820 +confirm_0-7,0-7,21,8,1,14.895 +confirm_0-7,0-7,21,31,4,14.015 +confirm_0-7,0-7,21,12,2,14.088 +confirm_0-7,0-7,21,12,6,13.162 +confirm_none,none,21,8,6,9.754 +confirm_none,none,21,21,8,9.663 +confirm_0-7,0-7,21,5,3,14.735 +confirm_0-7,0-7,21,16,1,13.981 +confirm_none,none,21,5,1,10.722 +confirm_0-7,0-7,21,16,4,14.034 +confirm_none,none,21,5,7,10.867 +confirm_0-7,0-7,21,31,2,14.363 +confirm_0-7,0-7,21,8,2,13.655 +confirm_none,none,21,8,2,10.113 +confirm_0-7,0-7,21,5,5,14.248 +confirm_none,none,21,8,9,9.909 +confirm_none,none,21,31,2,9.384 +confirm_none,none,21,12,1,10.196 +confirm_none,none,21,5,8,10.371 +confirm_0-7,0-7,21,31,3,13.108 +confirm_none,none,21,16,2,10.024 +confirm_none,none,21,12,3,10.001 +confirm_0-7,0-7,21,5,4,14.003 +confirm_none,none,21,31,5,9.328 +confirm_0-7,0-7,21,21,2,14.082 +confirm_0-7,0-7,21,12,5,14.131 +confirm_0-7,0-7,21,12,9,14.167 +confirm_0-7,0-7,21,16,3,13.824 +confirm_0-7,0-7,21,31,1,13.409 +confirm_none,none,21,16,6,9.810 +confirm_none,none,21,5,4,10.392 +confirm_0-7,0-7,21,16,6,13.359 +confirm_none,none,21,21,9,9.732 +confirm_0-7,0-7,21,31,7,13.460 +confirm_none,none,21,31,9,9.576 +confirm_none,none,21,16,4,10.284 +confirm_none,none,21,31,4,9.630 +confirm_none,none,21,12,9,9.893 +confirm_none,none,21,12,5,9.978 +confirm_none,none,21,8,3,9.813 +confirm_0-7,0-7,21,12,8,13.268 +confirm_none,none,21,21,3,9.813 +confirm_0-7,0-7,21,21,7,13.666 +confirm_0-7,0-7,21,16,2,13.287 +confirm_0-7,0-7,21,8,5,13.169 +confirm_none,none,21,21,5,9.444 +confirm_none,none,21,21,4,9.969 +confirm_0-7,0-7,21,12,3,13.817 diff --git a/scripts/profiling/table-parallelism-sweep/round1/rules.py b/scripts/profiling/table-parallelism-sweep/round1/rules.py new file mode 100644 index 000000000..3fcc9b189 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round1/rules.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 +"""For each core count, what does each candidate rule prescribe, and what does +that cost against the best measured K in that config?""" +import csv, statistics as st +from collections import defaultdict + +CONFIGS = [("results_c32.csv", 32), ("results_c16.csv", 16), + ("results_c8.csv", 8), ("results_c4.csv", 4)] + +def load(path): + d = defaultdict(list) + with open(path) as f: + for r in csv.DictReader(f): + if r["seconds"] != "NA": + d[int(r["k"])].append(float(r["seconds"])) + return {k: st.median(v) for k, v in d.items()} + +RULES = { + "cores/3 (CPU arm)": lambda c: max(c // 3, 1), + "cores*2/3 (PR #877)": lambda c: max(c * 2 // 3, 1), + "constant 8": lambda c: 8, + "constant 12": lambda c: 12, + "unbounded (=num_airs)": lambda c: 31, +} + +def nearest(meds, k): + return min(meds, key=lambda x: abs(x - k)) + +data = {c: load(p) for p, c in CONFIGS} + +print(f"{'rule':<24}", end="") +for _, c in CONFIGS: + print(f"{'c'+str(c):>18}", end="") +print(f"{'worst-case':>12}") +print("-" * (24 + 18 * 4 + 12)) + +for name, fn in RULES.items(): + print(f"{name:<24}", end="") + worst = 0.0 + for _, c in CONFIGS: + meds = data[c] + best = min(meds.values()) + k = fn(c) + kk = nearest(meds, k) + pen = (meds[kk] / best - 1) * 100 + worst = max(worst, pen) + tag = f"K={k}" + ("" if kk == k else f"~{kk}") + print(f"{tag+f' +{pen:.1f}%':>18}", end="") + print(f"{'+'+format(worst,'.1f')+'%':>12}") + +print("\nper-config best K and time:") +for _, c in CONFIGS: + meds = data[c] + bk = min(meds, key=lambda k: meds[k]) + print(f" c{c:<3} best K={bk:<3} {meds[bk]:.2f}s " + f"K=8 -> {meds[8]:.2f}s (+{(meds[8]/meds[bk]-1)*100:.1f}%)") diff --git a/scripts/profiling/table-parallelism-sweep/round1/stacksample.sh b/scripts/profiling/table-parallelism-sweep/round1/stacksample.sh new file mode 100644 index 000000000..540c72b5c --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round1/stacksample.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# Poor-man's off-CPU profiler: run a prove and repeatedly snapshot every thread's +# stack with gdb, so we can classify what the K driver threads actually block on +# (pinned-staging mutex vs CUDA sync vs VramGate condvar). +# Usage: stacksample.sh [cpuspec] +set -u +K="$1"; N="$2"; CPUSPEC="${3:-none}" +CLI=/root/target_dbg/release/cli # symbolized build +ELF=/root/ethrex.elf +INPUT=/root/lambda_vm/executor/tests/ethrex_10_transfers.bin +OUT=/root/stacks_k${K}_${CPUSPEC}.txt + +if [ "$CPUSPEC" = "none" ]; then PREFIX=""; else PREFIX="taskset -c $CPUSPEC"; fi +: > "$OUT" + +TABLE_PARALLELISM=$K $PREFIX "$CLI" prove "$ELF" --private-input "$INPUT" \ + -o /tmp/ss.bin --time --continuations --epoch-size-log2 21 > /root/ss_run_k${K}.log 2>&1 & +PID=$! + +# Let it get past ELF load / CUDA init / first epoch execute into proving. +sleep 4 +for i in $(seq 1 "$N"); do + kill -0 $PID 2>/dev/null || break + echo "===== SAMPLE $i =====" >> "$OUT" + gdb -p $PID -batch -ex "set pagination off" -ex "thread apply all bt 14" \ + >> "$OUT" 2>/dev/null + sleep 0.3 +done +wait $PID 2>/dev/null +echo "samples written to $OUT" +grep -c "^Thread" "$OUT" 2>/dev/null | sed 's/^/thread-snapshots: /' diff --git a/scripts/profiling/table-parallelism-sweep/round1/sweep.sh b/scripts/profiling/table-parallelism-sweep/round1/sweep.sh new file mode 100644 index 000000000..8fbbcc65e --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round1/sweep.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# K sweep harness. Randomized interleaved order, n reps per K. +# Usage: sweep.sh +set -u +TAG="$1"; CPUSPEC="$2"; REPS="$3"; EPOCH="$4"; KLIST="$5" + +CLI=/root/lambda_vm/target/release/cli +ELF=/root/ethrex.elf +INPUT=/root/lambda_vm/executor/tests/ethrex_10_transfers.bin +OUT=/root/results_${TAG}.csv + +if [ "$CPUSPEC" = "none" ]; then + PREFIX="" +else + PREFIX="taskset -c $CPUSPEC" +fi + +echo "tag,cpuspec,epoch,k,rep,seconds" > "$OUT" + +# Build the full randomized job list: every (K,rep) pair, shuffled. +JOBS=$(for k in $(echo "$KLIST" | tr ',' ' '); do + for r in $(seq 1 "$REPS"); do echo "$k $r"; done + done | shuf) + +# Warmup (not recorded): pulls the GPU out of idle clocks, warms page cache. +TABLE_PARALLELISM=4 $PREFIX "$CLI" prove "$ELF" --private-input "$INPUT" \ + -o /tmp/warm.bin --time --continuations --epoch-size-log2 "$EPOCH" >/dev/null 2>&1 + +TOTAL=$(echo "$JOBS" | wc -l) +i=0 +echo "$JOBS" | while read -r k r; do + i=$((i+1)) + t=$(TABLE_PARALLELISM=$k $PREFIX "$CLI" prove "$ELF" --private-input "$INPUT" \ + -o /tmp/p_${TAG}.bin --time --continuations --epoch-size-log2 "$EPOCH" 2>&1 \ + | sed -n 's/^Proving time: \([0-9.]*\)s/\1/p') + if [ -z "$t" ]; then t="NA"; fi + echo "$TAG,$CPUSPEC,$EPOCH,$k,$r,$t" >> "$OUT" + echo "[$i/$TOTAL] tag=$TAG cpus=$CPUSPEC K=$k rep=$r -> ${t}s" +done +echo "DONE $TAG -> $OUT" diff --git a/scripts/profiling/table-parallelism-sweep/round2/NOTES.md b/scripts/profiling/table-parallelism-sweep/round2/NOTES.md new file mode 100644 index 000000000..88d04e10c --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/NOTES.md @@ -0,0 +1,981 @@ +# K sweep round 2 — running notes + +Append-only. Each section is written at a stage boundary so a box death leaves prose, not just CSVs. +See `PLAN.md` for the brief and `../round1/README.md` for round 1. + +Raw data mirrors into `data/` from `root@79.161.122.162:/root/results/` every 60 s. + +--- + +## Summary (written last, kept first — every claim is expanded in the stage sections below) + +539 timed runs on an RTX 5090 / Ryzen 9 7950X box, code at `7644043b`. + +**On the PR argument (`cores*2/3`):** + +- The recommendation is **`K = num_airs`: drop the cap and the core term entirely**, not a better + constant. On the five high-n configurations across both boxes (round 1's own n=9 confirmation set plus + round 2's n=9/n=12 legs), **`num_airs` is the best rule in every one**, worst case **+1.6 %**, and both + of its non-zero cells are inside noise (p=0.88, p=0.80). Next best is constant 21 at +2.9 %. +- **Round 1's own n=9 confirmation data already said this and its README did not use it.** Recomputed: + best K = **31 at both** core counts round 1 re-measured, including c8, where its headline table + reported best K=8 from an n=4 sweep and charged unbounded K +5.9 %. At n=9 that same configuration puts + K=31 first, with nothing from K=8 up distinguishable from it. The one cell that ever argued against + unbounded K does not survive round 1's own repetition. +- **At matched n the two boxes agree at 32 cores**, cell by cell: `cores*2/3` +2.5 % vs +2.9 %, + constant 16 +4.6 % vs +3.3 %, constant 8 +5.5 % vs +6.7 %, constant 12 +5.8 % vs +7.0 %, `num_airs` + 0 % vs 0 %. My first draft called this a failed replication by comparing round 2's n=9 against round 1's + n=5 headline table; corrected. +- **Constant 12 (round 1's recommendation) is +7.0 % worst case** on round 2 and +5.8 % on round 1's own + n=9 c32 — it does not survive either box at high n. If a constant is required, 16–24; but there is no + measured reason to want one. +- Both core-scaled forms are the two worst rules everywhere: `cores*2/3` **+13.0 %** worst case, + `cores/3` **+23.2 %** (+30.2 % if round 1's low-n c4 leg is included; round 1 has no high-n c4). + `cores*2/3`'s error is exactly where round 1 located it — low core counts, K=2 at 4 cores (+13.0 %) and + K=5 at 8 cores (+8.1 %). **That is the part that replicates cleanly, and it is the whole case against + the merged formula.** At 32 cores the ratio is near-optimal on both boxes, which is why Stage 0's c32 + anchor was never the load-bearing leg. +- **The mechanism, measured:** K divides a quantity W ≈ 5.3–8.4 s that is **invariant** to host cores + (8× range), to CPU model, and to rayon pool width (Stage 2: 8× pool change moves W by ~0 and doubles + the rest). `available_parallelism()` sizes the rayon pool — provably the wrong quantity to scale K by. +- **Retire `knee@5%` as the decision metric.** The curve is `S + W/K`, so the tolerance-knee is a derived + quantity ≈ `W/(tol·T_best)` that reads 8 on one box and 16 on another at the same tolerance; there is no + saturating resource and therefore no knee. Round 1's headline ("knee K≈8, core-invariant") leaned on it. + The quantity the default actually has to minimise is **worst-case regret across core counts** — the + rule-cost table — which is what this round reports and which degrades gracefully as noise grows. +- Only measured cost of large K: host peak heap 9.25 → 11.0 GB (+19 %), **saturating by K≈16**. Raising K + also nudges `auto_storage::decide` toward `Disk`, since `peak_bytes` sums the transient bytes of the + **top-k** tables — but that term saturates structurally too (the extra tables are the smallest ones), + matching the measured heap curve. Worth a sanity check in the PR, not a blocker. + +**On the mechanism question (Stage 1), all three hypotheses in play came out negative:** + +- **Pinned-staging contention: refuted.** The premise is true and dramatic — aggregate driver wait on + slot 0 rises 0.19 s → 23.4 s as K goes 1 → 31 — and it costs *nothing*: wall time falls over the same + range. Removing the contention (per-driver slabs cut the wait 5 000×, to 4 ms) makes every K **6–12 % + slower**, p=0.002 at every K. The mutex serializes an already-serial resource (one staged D2H over one + PCIe link); the other drivers are running GPU kernels, not idling. +- **The author's counter-claim at `device.rs:513-517`: confirmed, with its scaling law.** The cost is + repeated pinned allocation, it is per-driver-cohort, and it therefore grows with epoch count — the + penalty collapses from +10.0 % to +1.0 % when the same work runs as 1 epoch instead of 4. +- **Round 1's #863 reconciliation: refuted.** Built at `1e1e0f18`, the old fixed-chunk scheduler also + wants K=31 (26 % faster than K=1, monotone). The barrier/straggler story does not explain "K saturates + at 3". + +**Incidental findings worth acting on:** + +- **`worker_slot`'s rayon arm is dead code on the `cuda` path.** `ray_n` = 0 in all 54 instrumented runs: + no rayon worker ever acquires a pinned-staging or pinned-hashes lock. The 32-slab per-rayon-worker + array is used at exactly one index. +- **#877's scheduler itself is worth −13.3 %** (10.42 s → 9.03 s best-to-best vs `1e1e0f18`), but it + **regressed K=1 by +12 %**. +- Stage 1a could not run at all: this container drops `cap_sys_ptrace` and `cap_perfmon`, so gdb and perf + are impossible. Future rentals should check `capsh --print | grep ptrace` at bring-up. + +**Box:** survived three unannounced reboots (17:54, 18:46, ~20:19); a cron-driven keepalive resumed the +chain each time with no data loss. All stages in PLAN.md were completed, plus two added ones +(0.5 Amdahl decomposition, 6 high-n confirmation) and one substituted (1a → in-process instrumentation). + +--- + +## Bring-up (2026-08-04 17:47–17:59 UTC) — done + +**Box** (`data/box.txt`): RTX 5090 32607 MiB, driver 595.71.05, PCIe gen4 x16 · nvcc 13.1.115 · +AMD Ryzen 9 **7950X** (16C/32T, L3 64 MiB in 2 instances — no 3D V-cache, as expected) · 93 GB RAM · +62 GB free · Ubuntu 24.04.4, kernel 6.8.0. + +Round 1's box was a 7950X**3D** (128 MiB L3). That is the one intended difference; everything in +Stage 0 exists to check whether the anchor curve survives it. + +**Code under test:** `7644043b` (`origin/main`, contains #875 and #877), clean worktree. +Verified `crypto/math-cuda/src/device.rs:518` still reads +`let idx = rayon::current_thread_index().unwrap_or(0);` — the Stage 1b target is present as +described. (✓ verified by reading the checked-out file, not just the plan.) + +**Repo transfer.** Bundled the two commits the plan needs (`7644043b`, `1e1e0f18`) locally with +`git bundle` (6.3 MB, `bundle verify` reports a complete history) and cloned from the bundle on the +server, to avoid putting credentials on a rented box. **This was unnecessary: the repo is public** +(`https://github.com/yetanotherco/lambda_vm.git`, per `infra/provision.sh` §9) — a plain HTTPS clone +would have been credential-free too. Harmless either way, and the bundle also carried `1e1e0f18` +without a second fetch. + +**Toolchain — two corrections to what I first reported.** + +1. I wrote "no cargo on the box, installed rustup stable". **Wrong.** `~/.cargo/bin/cargo` is dated + 17:29:22, ~18 minutes *before* my first connection, and `~/.rustup/toolchains/` already contained + **1.94.0 and nightly-2026-02-01** (17:29:29 / 17:29:55) — i.e. the box was already provisioned with + exactly what `infra/provision.sh` §6 prescribes. My `which cargo` check ran over a + non-interactive, non-login ssh shell, which sources neither `.bashrc` nor `.profile`, and those are + what put `$HOME/.cargo/bin` on `PATH`. A `which` miss in that shell is not evidence of absence. + My rustup run added only the `stable` toolchain on top and rewrote `.cargo/env`. +2. Consequently the toolchain that built every binary is **not** the 1.97.1 stable I reported, but + **rustc 1.94.0 (4a4ef493e)** — `rust-toolchain.toml` pins `channel = "1.94.0"`, so cargo used it for + every in-repo build regardless of what the default was. Verified by running `rustc --version` inside + `/root/lambda_vm` (recorded in `data/box.txt`). This is the intended pin, it applied uniformly to all + five binaries and all arms, so no measurement is affected — but the earlier claim was wrong. + +Neither the rv64im sysroot (`provision.sh` §8) nor LLVM 21 (§3) was needed: those build *guest* +programs, and this round used the prebuilt `ethrex.elf`. No build step ever demanded them. + +**Build.** `cargo build --release -p cli --features jemalloc-stats,prover/cuda` — the feature set the +repo's own GPU bench path uses (`BENCH_FEATURES` in `.github/workflows/benchmark-gpu.yml`, consumed by +`scripts/bench_abba.sh`). **The plan's assumption that builds dominate the wall +clock is wrong on this box: the full build is ~20 s of compile** (104 crates) after ~40 s of crate +downloads. `nvcc` produced real cubins +(`target/release/build/math-cuda-*/out/{arith,ntt,keccak,barycentric,deep}.cubin`), and the GPU path +is live — no `math-cuda: GPU backend unavailable` warning, and a smoke prove at K=8 took **9.447 s**, +already inside round 1's best-time band. So the extra builds (symbolized, patched, old-scheduler) are +cheap and the stage ordering is not build-constrained. + +`n_slots = rayon::current_num_threads().max(1)` (`device.rs:321`) ⇒ **32** unrestricted on this box. +Under `taskset -c 0-7` it is 8, so K=16 drivers alias in pairs; under `RAYON_NUM_THREADS=4` it is 4. +Recorded per stage below. + +### Operational finding: this box reboots + +At **17:54 UTC the box rebooted on its own** (`uptime` = 1 min, tmux server gone, `/var/log/onstart.log` +re-ran), killing the Stage 0 sweep after 2 recorded runs. `/root` survived, so the CSV rows did. +tmux alone is therefore *not* enough resilience here — a reboot kills the tmux server. + +Added `/root/keepalive.sh`, driven by `crontab` (`@reboot` + every minute): if +`/root/results/CURRENT_CMD` is non-empty and tmux session `job` is absent, it relaunches that command. +`cron` on this image runs under supervisor, so it comes back after a reboot. The single fixed session +name also enforces the never-two-proves-at-once rule. Reboots are logged to +`data/keepalive.log`, so the run count between reboots is auditable. + +Two harness deviations, both forced by the reboot: + +1. `sweep_r.sh` now warms up on **every** start, not only cold ones. keepalive resumes the sweep + after a reboot, and a resume that skips warmup would drop one cold-GPU-clock run at a random K + into the data. (Local copy updated to match.) +2. `stacksample.sh` and `followup2.sh` wrote to `/root/…` instead of `/root/results/…`, which is the + only mirrored directory — a reboot would have lost their output. Round-2 copies with corrected + paths live in this directory; round 1's originals are untouched. + +The box went on to reboot **twice more** (18:46, 20:19). `data/box.txt` now carries a +"DATA-INTEGRITY EVENTS" section recording each boot time and exactly which CSV rows predate it — a +mid-sweep reboot resets clock and page-cache state, and clock state is precisely what the +7950X-vs-7950X3D comparison turns on, so a later reader needs to see it. Summary: only 2 rows in the +whole dataset (both in `results_c32.csv`) predate reboot 1; no timed row spans reboot 2; 39 of +`results_c8b.csv`'s 40 rows predate reboot 3 and 1 postdates it. No A/B pairing can be skewed by a +reboot, because each A/B sweep interleaves its arms in one frozen randomized order. + +### CPU topology of the taskset ladder (measured, `lscpu -p`) + +CPUs 0–15 are physical cores 0–15; 16–31 are their SMT siblings. L3 domain 0 = cores 0–7, +L3 domain 1 = cores 8–15 (32 MiB each). So the ladder is: + +| config | what it actually is | L3 available | `n_slots` | +|---|---|---|---| +| full (32) | 16 cores + 16 SMT siblings, both CCDs | 64 MiB | 32 | +| `-c 0-15` | 16 physical cores, no SMT, both CCDs | 64 MiB | 16 | +| `-c 0-7` | 8 physical cores, **CCD0 only** | 32 MiB | 8 | +| `-c 0-3` | 4 physical cores, CCD0 only | 32 MiB | 4 | + +Worth flagging for the two-box comparison: on a 7950X**3D** one CCD carries the 64 MiB V-cache stack +(128 MiB L3 total on that die) and the other does not, so round 1's `-c 0-7` and `-c 0-3` legs very +likely ran with ~4× the L3 that round 2's do. **The low-core legs are the cache-confounded ones**, and +they are exactly where round 1 located the ratio rule's error. Round 2's low-core legs are the +no-V-cache version of the same experiment; treat a divergence there as informative about cache, not +only about K. + +**Still unmeasured at this point:** everything. Stage 0 is in flight. + +--- + +## Stage 0 — anchor curve at 32 cores (18:04–18:13 UTC) — baseline **comparable, offset explained** + +> **Corrected 20:40 UTC.** As first written, this section compared round 2's n=9 curve against round 1's +> low-n *headline table* and concluded the anchor "did not reproduce". That comparison was wrong: round 1 +> also has an n=9 confirmation dataset (`../round1/results_confirm.csv`) and against *that*, the two +> boxes agree closely. See "Correction" below, which supersedes the two struck claims in this section. +> The gate verdict is **comparable, absolute offset explained by clocks** — not "not met". + +`data/results_c32.csv` (n=3, K∈{1,2,4,6,8,12,16,21,31}) + `data/results_c32b.csv` (n=6, plateau only, +added because n=3 could not separate a shape change from a threshold artifact). Merged, stock build, +full cores, epoch 2^21, `n_slots`=32: + +| K | n | median s | sd | min–max | vs best | +|---|---|---|---|---|---| +| 1 | 3 | 15.88 | 0.32 | 15.43–16.03 | 1.760× | +| 2 | 3 | 11.89 | 0.06 | 11.88–11.99 | 1.317× | +| 4 | 3 | 10.40 | 0.21 | 10.14–10.55 | 1.152× | +| 6 | 9 | 9.81 | 0.21 | 9.44–10.07 | 1.087× | +| 8 | 9 | **9.63** | 0.15 | 9.34–9.81 | 1.067× | +| 12 | 9 | 9.66 | 0.17 | 9.29–9.89 | 1.070× | +| 16 | 9 | 9.33 | 0.13 | 9.16–9.53 | 1.033× | +| 21 | 9 | 9.29 | 0.18 | 8.97–9.59 | 1.029× | +| 31 | 9 | **9.03** | 0.19 | 8.76–9.40 | 1.000× | + +knee@5% = **K=16**; knee@2% = K=31; best K=31 at **9.03 s**. + +### What reproduced and what did not (measured) + +Reproduced: the *shape* is round 1's curve almost exactly on the steep side — K=1 is 1.76× best on +both boxes (1.760 vs 1.760), K=2 is 1.32 vs 1.31, K=4 1.15 vs 1.12 — and the same +long flat-ish tail from K≈6 onward. Same workload (4 epochs, 6.807 M cycles), same epoch size. + +Differences, and what they do and don't mean: + +- **Best time is 9.03 s, not ~9.8 s** — this box is ~8 % faster. Expected direction for a non-3D + 7950X: the workload is CPU-bound at the serial continuation producer (#863), and the 7950X's CCDs + both clock to 5.7 GHz whereas the 7950X3D's V-cache CCD is capped near 5.25 GHz. Not a problem for + comparability — the curves are compared as ratios. +- **The gain past K=8 is real here, not noise.** Against K=31: K=8 is **+6.7 % (p=0.0001)**, K=12 + +7.0 %, K=16 +3.3 % (p=0.0008), K=21 +2.9 %. Mann–Whitney, exact, two-sided, n=9 per arm. Round 1's + headline table put K=8 at +4.8 % (n=5) and called K∈[8,16] "within noise" — **but its own n=9 + confirmation puts K=8 at +5.5 %, p=0.0012, i.e. not within noise either.** Round 1 also explicitly + caveated this: *"the top of the flat region shows a real ~5 % gain at 32 cores that is absent at 8."* + So this is a **documented caveat strengthening slightly**, not a failed replication. +- `knee@5%` reads 16 here vs 8 in round 1's headline. That is a threshold artifact, see below. + +Two consequences: + +1. **`knee@5%` is not a robust statistic** for this curve, and should be retired as the decision metric. + The curve is flat to within ~7 % from K=6 to K=31, so a 5 % tolerance line lands anywhere in [8,31] + on ±1.5 pp of noise. Round 1 leaned on it for its headline ("knee K≈8, core-invariant"). The metric + the default actually has to minimise is **worst-case regret across core counts** — the rule-cost + table — which degrades gracefully as noise grows. That is the headline used from here on. +2. **At 32 cores `cores*2/3` prescribes K=21 = +2.9 %, better than constant 12 (+7.0 %) or constant 8 + (+6.7 %).** ~~That is the reverse of round 1's ranking at c32.~~ **Wrong — struck.** Round 1's c32 + ranking was the same: ratio +1.4 %, const-12 +3.8 %, const-8 +4.8 %, i.e. the ratio already beat both + constants at 32 cores there too. Nothing is reversed; the magnitudes roughly double and const-8 / + const-12 swap by 0.3 pp, which is inside this data's noise. And round 2's +2.9 % ~~differs from round + 1's +1.4 %~~ **matches round 1's n=9 confirmation value of +2.5 % (p=0.077)** — the +1.4 % came from + round 1's n=5 table, and comparing an n=9 result against an n=5 one is what produced the phantom + discrepancy. + +### Deviation from PLAN.md, and why + +The plan says: if the anchor does not reproduce, report that as the headline instead of building on a +non-comparable baseline. The baseline is **comparable, offset explained** — same shape, same workload, +ratio differences of 1–3 pp, absolute offset predicted by the clock difference between the 3D and non-3D +parts. So I reordered rather than stopped: Stage 4's core-count legs before Stage 1. + +**The better reason for that reorder** (lead's framing, and it is the correct one): round 1's case +against `cores*2/3` never rested on c32 at all — at 32 cores the ratio prescribes K=21, which is close +to optimal on both boxes. The case rested entirely on the **low-core legs** (round 1: c4 +11.0 %, +c8 +7.7 %). Stage 0 re-measured the one leg that was never load-bearing, so it left the conclusion +*untested*, not in question. That also sharpens the order — **c4 is the single decisive datapoint** +(where the ratio prescribes K=2 and starves hardest, and where round 1's worst case lives), then c8, +then c16, which is nearly uninformative by comparison. + +(I ran them c8 → c4 → c16, having reordered before that guidance arrived. All three completed, so the +ordering cost nothing; had the box died after the first leg I would have had the less decisive one.) + +**Unmeasured at this point:** every core count except 32 on this box; the whole of Stage 1 (mechanism), +2 (rayon decomposition) and 3 (old scheduler). + +### Correction (20:40 UTC) — compare like n with like n, and round 1 already had the answer + +Triggered by the lead catching two bad comparisons above. Chasing them down turned up something bigger: +**round 1's own n=9 confirmation dataset (`results_confirm.csv`, 108 runs at c32 and c8) was +under-used in its README, and it does not say what round 1's headline says.** Recomputed here: + +Round 1, `results_confirm.csv`, **n=9 per cell**, penalty vs the best measured K in that config: + +| config | best K | K=5 | K=8 | K=12 | K=16 | K=21 | K=31 | +|---|---|---|---|---|---|---|---| +| c32 (`none`) | **31** @ 9.58 s | +9.7 % p<0.0001 | +5.5 % p=0.0012 | +5.8 % p=0.0003 | +4.6 % p=0.0008 | +2.5 % p=0.077 | — | +| c8 (`0-7`) | **31** @ 13.46 s | +5.8 % p=0.063 | +1.4 % p=0.67 | +0.7 % p=0.73 | +0.9 % p=0.80 | +1.5 % p=0.67 | — | + +Two things follow, and both strengthen round 2's conclusion using round 1's own data: + +1. **Round 1's high-n data has best K = 31 at *both* core counts it re-measured** — including c8, where + its headline table reported best K=8 @ 13.28 s from an n=4 sweep and charged unbounded K +5.9 %. At + n=9 the same configuration puts K=31 *first*, with nothing from K=8 up distinguishable from it. + So the single cell that ever argued against unbounded K does not survive round 1's own repetition. +2. **At matched n the two boxes agree at c32**, cell by cell: + +| rule | r1 c32 n=9 | r2 c32 n=9 | +|---|---|---| +| `cores*2/3` (K=21) | +2.5 % | +2.9 % | +| constant 16 | +4.6 % | +3.3 % | +| constant 8 | +5.5 % | +6.7 % | +| constant 12 | +5.8 % | +7.0 % | +| `num_airs` (K=31) | 0.0 % | 0.0 % | + +That is a replication, not a divergence. My original "did not reproduce" verdict came from comparing +round 2's n=9 against round 1's n=5 headline table. Corrected. + +### Two-box rule table, high-n cells only (supersedes the low-n version in Stage 4/6) + +`~` = the rule's K was not measured, nearest measured K used. + +| rule | r1 c32 (n=9) | r1 c8 (n=9) | r2 c32 (n=9) | r2 c8 (n=12) | r2 c4 (n=12) | **worst** | +|---|---|---|---|---|---|---| +| `cores/3` | +5.5 %~ | +5.8 %~ | +6.7 %~ | +19.7 % | +23.2 % | **+23.2 %** | +| `cores*2/3` (#877) | +2.5 % | +5.8 % | +2.9 % | +8.1 %~ | +13.0 % | **+13.0 %** | +| constant 8 | +5.5 % | +1.4 % | +6.7 % | +4.7 % | +1.1 % | +6.7 % | +| constant 12 | +5.8 % | +0.7 % | +7.0 % | +2.0 % | 0.0 % | +7.0 % | +| constant 16 | +4.6 % | +0.9 % | +3.3 % | +3.2 % | +0.4 % | +4.6 % | +| constant 21 | +2.5 % | +1.5 % | +2.9 % | 0.0 % | 0.0 % | +2.9 % | +| **`num_airs`** | **0.0 %** | **0.0 %** | **0.0 %** | **+1.6 %** | **+0.6 %** | **+1.6 %** | + +**`num_airs` is the best rule in every high-n config on both boxes, worst case +1.6 %, and its two +non-zero cells are inside noise (p=0.88 and p=0.80).** Next best is constant 21 at +2.9 %. Round 1's +recommended constant 12 is +7.0 %; the merged `cores*2/3` is +13.0 %; the old `cores/3` is +23.2 %. + +Coverage caveat: round 1's confirmation set only swept K∈{5,8,12,16,21,31} at two core counts, so its +`cores/3` cells use a nearest-K substitute and it has no c4 or c16 leg. Round 2 supplies c4 and c16. + +--- + +## Stage 0.5 — the curve is Amdahl, and that explains the core-invariance without any lock + +Derived from data already in hand (round 2's c32 plus round 1's four CSVs), no extra runs. Written up +here because it reframes what Stage 1 is even looking for. `amdahl.py` fits + + T(K) = S + max(Tmax, W/K) + +S = the part K cannot touch (continuation producer, execute, trace build, commit glue), +W = the table-proving work K divides, Tmax = a longest-single-table floor. This is the null hypothesis +the pinned-staging mutex has to beat: `run_admitted` (verified at `crypto/stark/src/prover.rs:689`) is +a work-stealing list scheduler over ~31 very unequal tables, and a list schedule over unequal work +*already* produces a 1/K curve that flattens — no saturating resource needed. + +| curve | fit | rms | noise (sd/K) | +|---|---|---|---| +| r2 c32 (n=9) | `8.82 + max(0.00, 6.85/K)` | 0.19 s | 0.13–0.21 s | +| r1 c32 (n=5) | `9.11 + max(0.91, 8.00/K)` | 0.16 s | — | +| r1 c16 | `9.67 + max(0.43, 7.62/K)` | 0.22 s | — | +| r1 c8 | `12.89 + max(0.88, 7.03/K)` | 0.27 s | — | +| r1 c4 | `20.79 + max(1.00, 6.50/K)` | 0.34 s | — | + +**Every curve on both boxes is fit to within run-to-run noise by three parameters, and round 2's c32 +wants no floor at all (Tmax = 0) — i.e. nothing has saturated by K=31.** A mutex that serialized +staging would show up precisely as a floor, so within K ≤ 31 the curves carry no saturation signature. +(Not a disproof of contention; it says contention is not needed to explain the shape, and that any +contention present is not yet the binding constraint.) + +### The model-free version of the same point + +No fitting required — just `T(K=1) − T(best K)`, which is W to within W/K_max: + +| config | T(1) | T(best) | **W ≈** | S ≈ | W as % of T(best) | +|---|---|---|---|---|---| +| r1 c32 | 17.25 | 9.80 | **7.45** | 9.56 | 76 % | +| r1 c16 | 17.48 | 9.99 | **7.50** | 9.75 | 75 % | +| r1 c8 | 20.04 | 13.28 | **6.76** | 13.06 | 51 % | +| r1 c4 | 27.55 | 21.16 | **6.38** | 20.96 | 30 % | +| r2 c32 | 15.88 | 9.03 | **6.86** | 8.81 | 76 % | + +**W is 6.4–7.5 s everywhere — across an 8× range of host cores and across two different CPUs — while S +grows 2.2× as cores shrink 8×.** That is the mechanism behind round 1's headline, stated without any +reference to locks: *the work K parallelizes barely speeds up when you add host cores.* Per-table +proving is GPU work plus host work that the rayon pool does not shorten much, so K is the only lever +on it. An optimum that divides a core-invariant quantity is core-invariant. Nothing about +`available_parallelism()` belongs in the formula. + +Three consequences that bear directly on the PR: + +1. **There is no "knee", so no knee value should be quoted.** T is monotone decreasing in K up to the + `num_airs` cap. The tolerance-knee is a derived quantity, K\* ≈ W/(tol·T_best), which is why it read + 8 on one box and 16 on another at the same tolerance: T_best differs by 8 %. It moves with the + *tolerance you picked*, not with anything in the machine. +2. **The correct rule is "as large as `num_airs` allows".** The model says the penalty of any rule is + `W·(1/K_rule − 1/K_max)/T_best`, monotone in 1/K_rule, with no term that ever favours a smaller K. + Round 1 already measured the two things that could have created such a term and found neither: + K = 26/31/40/64/128/1024 all identical (num_airs caps it anyway) and `VramGate` never binds at the + default budget. +3. **The one measured cost of large K is host RAM, and it is small and saturating.** From the + `jemalloc-stats` peak-heap line in the per-K run logs (`data/log_c32*_k*.txt`, `data/log_c8_*`): + 9.25 GB at K=1 → 10.6 GB at K=8 → **11.3 GB at K=16–21 → 11.0 GB at K=31**. So +2.1 GB (+22 %) for + the whole K range, saturating by K≈16 rather than scaling with K. Worth stating in the PR because + `auto_storage::decide` feeds `table_parallelism()` into the RAM-vs-disk decision (`peak_bytes`, + `prover/src/auto_storage.rs:238`), so raising K pushes that estimate up and makes `Disk` more likely. + Read the formula before assuming it over-reacts: its K term is the sum of the transient bytes of the + **top-k** tables, so it saturates structurally as well — the tables added past k≈16 are the smallest + ones. The direction of change is real but modest, matching the measured heap curve. A PR raising K + should sanity-check the resulting storage decision on a large block, not treat it as free. + +Caveats, explicitly: the fit is phenomenological, W's core-invariance is *inferred from* T(1)−T(best) +rather than instrumented per phase, and both boxes are one GPU model. Stage 2 is a direct test — +`RAYON_NUM_THREADS` shrinks only the rayon pool, so under this model it should inflate S and leave W +roughly alone. + +--- + +## Stage 4 — core-count legs (all three complete; see also Stage 6's high-n redo of c8/c4) + +### c8 leg done 18:23 UTC + +`data/results_c8.csv`, `taskset -c 0-7` = 8 physical cores on CCD0 only, `n_slots`=8, n=4, stock build. + +| K | 1 | 2 | 4 | 6 | 8 | 12 | 16 | 21 | 31 | +|---|---|---|---|---|---|---|---|---|---| +| median s | 19.03 | 14.86 | 13.87 | 13.43 | 13.01 | 12.67 | 12.87 | **12.36** | 12.83 | +| vs best | 1.54× | 1.20× | 1.12× | 1.09× | 1.05× | 1.02× | 1.04× | 1.00× | 1.04× | + +Best K = **21** (12.36 s); knee@5% = 12. Round 1's c8: best K=**8** (13.28 s), knee@5% = 8. + +- **The optimum did not move down with cores — it stayed high.** At 8 cores the best measured K is 21, + i.e. 2.6× the number of cores. `cores*2/3` prescribes **K=5** here; the nearest measured K=4 is + **+12.2 %** (round 1's c8 penalty for the same rule was +7.7 %). So the ratio rule is *worse* on this + box at 8 cores, not better. +- **Amdahl again, same W.** Fit: `12.08 + max(0.67, 6.72/K)`, rms 0.228 s. W=6.72 s versus 6.85 at + c32 on this box and 6.4–7.5 across all of round 1. Fifth independent curve, same K-parallelizable + work. Model-free: T(1) − T(best) = 19.03 − 12.36 = 6.67 s. +- Caveat on this leg specifically: `-c 0-7` is one CCD, so 32 MiB of L3 versus round 1's likely 128 MiB + (V-cache die). S here is 12.08 s vs round 1's 12.89 s — this box is still faster despite ¼ the L3, + so the cache difference is not dominating the low-core legs. + +### c4 leg done 18:37 UTC + +`data/results_c4.csv`, `taskset -c 0-3` = 4 physical cores on CCD0, `n_slots`=4, n=4. + +| K | 1 | 2 | 4 | 6 | 8 | 12 | 16 | 21 | 31 | +|---|---|---|---|---|---|---|---|---|---| +| median s | 24.92 | 22.85 | 21.39 | 21.13 | 20.56 | **20.24** | 20.77 | 20.49 | 20.97 | +| vs best | 1.23× | 1.13× | 1.06× | 1.04× | 1.02× | 1.00× | 1.03× | 1.01× | 1.04× | + +Best K = **12** (20.24 s); round 1's c4 best was K=8 (21.16 s). Fit `20.08 + max(0.54, 5.34/K)`, +rms 0.206 s. W=5.34 s here — the low end of the 5.3–8.0 band but the same order; K's leverage is +smallest at 4 cores because S = 20.1 s dominates (W is only 26 % of T_best). + +**`cores*2/3` prescribes K=2 at 4 cores: +12.9 %** (round 1: +11.0 %). At 8 cores it prescribes K=5: +**+8.6 %** (round 1: +7.7 %). So round 1's central claim — *the ratio's error is at low core counts, +where it prescribes a K far below the optimum* — **replicates cleanly on a second, different CPU**, and +slightly larger. That part of round 1 stands. + +### Two-box rule table (worst case over all measured configs on each box) + +| rule | round 2 (7950X) | round 1 (7950X3D) | **worst of both** | +|---|---|---|---| +| `cores/3` (CPU arm) | +23.1 % | +30.2 % | **+30.2 %** | +| `cores*2/3` (#877 merged) | +12.9 % | +11.0 % | **+12.9 %** | +| constant 8 | +6.7 % | +4.8 % | +6.7 % | +| constant 12 (round 1's pick) | +7.0 % | +3.8 % | **+7.0 %** | +| constant 16 | +4.1 % | +5.0 % | +5.0 % | +| **constant 21** | +2.9 % | +4.0 % | **+4.0 %** | +| unbounded (`min(num_airs)`) | +3.8 % | +5.9 % | +5.9 % | + +(round 2's c16 column excluded — incomplete when this table was computed; see the c16 section.) + +**Round 1's recommendation of "constant 12" does not survive the second box: +3.8 % → +7.0 %.** The +two-box worst-case winner is a **constant in the high teens to low twenties** (21 → +4.0 %, 16 → +5.0 %), +with plain `num_airs` at +5.9 %. Round 1 picked 12 because its c32 leg said K=8–16 was flat; on this box +that band costs 4–7 % at 32 cores. Anyone quoting "constant 12" from round 1 should quote **~16–24** +instead — and the difference between 16, 21 and `num_airs` is inside the noise of this data, so the +honest statement is *"a constant of roughly 16–24, or just `num_airs`"*, not a specific integer. + +What is robust across both boxes and all eight measured curves: **the rule must not be a function of +host core count.** Both core-scaled forms are the two worst rows in the table on both boxes. + +### c16 leg done 18:43 UTC — Stage 4 complete + +`data/results_c16.csv`, `taskset -c 0-15` = 16 physical cores, no SMT, both CCDs, `n_slots`=16, n=4. + +| K | 1 | 2 | 4 | 6 | 8 | 12 | 16 | 21 | 31 | +|---|---|---|---|---|---|---|---|---|---| +| median s | 17.03 | 12.46 | 10.74 | 10.27 | 9.91 | 9.64 | 9.52 | 9.62 | **9.12** | +| vs best | 1.87× | 1.37× | 1.18× | 1.13× | 1.09× | 1.06× | 1.04× | 1.05× | 1.00× | + +Best K=31 (9.12 s). Fit `8.84 + max(0.58, 8.01/K)`, rms 0.201 s, W=8.01 s. + +Round 2's four fitted W values: 6.85 (c32), 8.01 (c16), 6.72 (c8), 5.34 (c4). Round 1's four: 8.00, +7.62, 7.03, 6.50. **Eight curves, two CPUs, 8× core range: W ∈ [5.3, 8.0] s with no monotone core +trend** (only a mild fall at 4 cores). Every one of the eight is fit to within its own noise by +`S + max(Tmax, W/K)`. + +### Stage 4 verdict — measured + +Complete round-2 rule table, penalty vs the best measured K in each config: + +| rule | c32 | c16 | c8 | c4 | worst (r2) | worst (r1) | **worst of both** | +|---|---|---|---|---|---|---|---| +| `cores/3` | K=10 +6.7 % | K=5 +12.6 % | K=2 +20.2 % | K=1 +23.1 % | +23.1 % | +30.2 % | **+30.2 %** | +| `cores*2/3` (#877) | K=21 +2.9 % | K=10 +8.6 % | K=5 +8.6 % | K=2 +12.9 % | +12.9 % | +11.0 % | **+12.9 %** | +| constant 8 | +6.7 % | +8.6 % | +5.2 % | +1.6 % | +8.6 % | +4.8 % | +8.6 % | +| constant 12 | +7.0 % | +5.7 % | +2.4 % | 0.0 % | +7.0 % | +3.8 % | +7.0 % | +| constant 16 | +3.3 % | +4.4 % | +4.1 % | +2.6 % | +4.4 % | +5.0 % | **+5.0 %** | +| constant 21 | +2.9 % | +5.4 % | 0.0 % | +1.2 % | +5.4 % | +4.0 % | +5.4 % | +| **unbounded = `num_airs`** | 0.0 % | 0.0 % | +3.8 % | +3.6 % | +3.8 % | +5.9 % | +5.9 % | + +And the significance tests that actually settle it (exact Mann–Whitney, two-sided, on both boxes' +raw runs): + +| test | round 2 | round 1 | +|---|---|---| +| c32: K=8 vs K=31 | +6.7 %, **p=0.0001** | +4.8 %, **p=0.008** | +| c32: K=16 vs K=31 | +4.9 %, **p=0.002** | +3.4 %, p=0.095 | +| c16: K=16 vs K=31 | +4.4 %, p=0.114 | +1.0 %, p=0.548 | +| c8: best small K vs K=31 | −3.6 %, p=0.486 | −5.6 %, p=0.343 | +| c4: best small K vs K=31 | −3.5 %, p=0.057 | −4.1 %, p=0.057 | + +**The conclusion, stated as strongly as the data supports it:** + +1. **No K below `num_airs` is significantly better than `num_airs` anywhere, on either box, at any core + count** (every "small K wins" cell above has p ≥ 0.057 and is a *median* difference of ≤ 4 % on n=4). + Round 1's table charged unbounded K a +5.9 % worst case; that cell is its c8 leg, where the same + comparison has p=0.34. It is noise. +2. **`num_airs` is significantly better than the mid-range constants at 32 cores** (K=16 +4.9 %, + p=0.002; K=8 +6.7 %, p=0.0001 on round 2; K=8 +4.8 %, p=0.008 on round 1). +3. Therefore the right change is not a better constant — it is **no cap and no core term: K = + `num_airs`.** That is also what the Amdahl form predicts a priori (T monotone in 1/K, no term + favouring smaller K), and it is consistent with round 1's separate finding that K = 26/31/40/64/128/1024 + all measure identically because `num_airs` caps K anyway. +4. If a constant is wanted for the RAM estimate's sake, **16–24**, not round 1's 12: constant 12 is + +7.0 % worst-case on this box versus +3.8 % on round 1's, and constant 8 is +8.6 %. +5. Both core-scaled forms remain the two worst rules on both boxes. `cores*2/3`'s error is where round 1 + said it was — low core counts, where it prescribes K=2 at c4 (+12.9 %) and K=5 at c8 (+8.6 %) — + and that part replicates cleanly. + +The one measured cost of `num_airs`: host peak heap 9.25 GB → 11.0 GB (+19 %), saturating by K≈16 +(§Stage 0.5). Not VRAM — round 1 showed `VramGate` never binds at the default budget. + +**Unmeasured at this point:** all of Stage 1 (1a in flight since 18:43), 2, 3. + +--- + +## Stage 1a — **cannot be run on this box.** gdb/perf are unavailable in the container + +Every `gdb -p` attach returns: + +``` +Could not attach to process. If your uid matches the uid of the target +process, check the setting of /proc/sys/kernel/yama/ptrace_scope +``` + +Diagnosed, not guessed (`/proc/self/status`, `capsh --print`): the container's bounding set **drops +`cap_sys_ptrace`, `cap_perfmon` and `cap_bpf`**, `yama/ptrace_scope` is 1, and `/proc/sys` is mounted +read-only so it cannot be lowered. Seccomp is active (mode 2, 1 filter). So gdb *and* perf are both +structurally impossible here, whatever the sampling script does. `data/stacks_k*_none.txt` contain +22/21/18/18/… attach failures and nothing else — they are kept as evidence, not data. + +This is a box property, not a plan error: round 1's box was presumably a privileged instance. Any +future round that wants stack sampling must rent with `--cap-add=SYS_PTRACE` (or check +`capsh --print | grep ptrace` during bring-up, which is worth adding to `box.txt`). + +### Substitute: measure the mutex from inside the process instead + +Better than sampling anyway — quantitative rather than statistical. Patch (`patch_instr.py`, +diff kept at `data/patch_instr.diff`) adds to `device.rs`: + +- counters for **nanoseconds spent waiting to acquire** the staging lock and the number of + acquisitions, split by caller kind (`rayon::current_thread_index().is_none()` ⇒ a scheduler driver + thread), plus the worst single wait; +- the two and only two lock sites — `htod_via` (device.rs:679) and `async_dtoh_via` (device.rs:763) — + routed through the timing wrapper; +- **per-driver slots as a run-time switch** (`LAMBDA_VM_PER_DRIVER_SLOTS=1`), so the Stage 1b + intervention can be A/B'd inside a single binary with no build difference between arms. + +Wait time is exactly the quantity the planned stack classification would have estimated ("driver-thread +time in pinned-staging mutex"), measured directly. + +### Why the hypothesis is mechanically plausible (verified by reading, before measuring) + +- `run_admitted` (`crypto/stark/src/prover.rs:689`) spawns `min(K, num_tables)` **`std::thread`** workers + off a shared atomic cursor and calls `task(idx)` *on those threads*. They are not rayon workers, so + `rayon::current_thread_index()` is `None` for every GPU call made directly from them. +- `worker_slot` maps `None` → slot 0 (`device.rs:518`), so all K of them share one slab and one mutex. +- The critical section is **long**: `PendingD2H` (device.rs:623-637) holds the `MutexGuard` until it is + dropped, and its `Drop` waits on the copy's event. The file's own header comment says the LDE call + "holds the lock across the D2H + memcpy-to-user-Vecs window". So the lock spans a device→host DMA + plus a host-side memcpy, not a few instructions. + +### The Stage 1b intervention verifiably took effect + +`data/slotproof.txt`: with `LAMBDA_VM_SLOT_DEBUG=1` at K=8 the patched binary reports each non-rayon +thread taking its own distinct raw slot (e.g. one `run_admitted` round's eight threads got raw slots +63, 61, 68, 64, 65, 67, 66, 69 → mod 32 = 31, 29, 4, 0, 1, 3, 2, 5 — eight distinct slabs), and the +stock binary prints nothing at all under the same env (negative control for the probe itself). So a +null result from 1b is a real null, not an intervention that silently didn't apply. + +**Unmeasured at this point:** 1b timings (in flight), 1a-substitute counters, 1c, 2, 3. + +--- + +## Stage 1b — **the pinned-staging hypothesis is REFUTED.** Per-driver slots are strictly worse + +`data/results_ab32.csv`, 72 runs, stock and patched **interleaved** in one randomized order, full cores, +`n_slots`=32, n=6 per cell, exact Mann–Whitney: + +| K | stock median s | patched (per-driver slots) s | patched cost | p | +|---|---|---|---|---| +| 1 | 15.96 (±0.46) | 16.95 (±0.55) | **+6.2 %** | 0.002 | +| 4 | 10.22 (±0.85) | 10.97 (±0.65) | **+7.3 %** | 0.002 | +| 8 | 9.58 (±0.46) | 10.54 (±0.61) | **+10.0 %** | 0.002 | +| 16 | 9.47 (±0.34) | 10.29 (±0.69) | **+8.7 %** | 0.002 | +| 21 | 9.31 (±0.28) | 10.17 (±0.72) | **+9.2 %** | 0.002 | +| 31 | 8.98 (±0.28) | 10.09 (±0.66) | **+12.4 %** | 0.002 | + +(± is max−min over the 6 runs. p=0.002 is the floor of the exact test at n=6/6, i.e. complete +separation, at *every* K.) + +**Giving the K driver threads their own pinned slabs never helps at any K — it costs 6–12 %, and the +cost grows with K rather than shrinking.** If a saturating mutex on slot 0 were the binding constraint +at large K, relieving it would have to help most exactly where the constraint bites hardest. The +opposite happened. + +### The K=1 arm is the decisive control + +At K=1 there is one driver thread and therefore **zero contention to remove**, yet per-driver slots +still cost **+6.2 %** (p=0.002). So most of the penalty is not about contention at all: it is the cost +of spreading staging across many cold, grow-only slabs instead of reusing one hot one. The incremental +penalty attributable to *more slots in flight* is only the part above that floor (+1 pp at K=4, ~+6 pp +at K=31). + +### This confirms the author's counter-claim, which round 1 flagged as unpublished + +`device.rs:513-517` asserts that per-driver slots "cost more in repeated pinned allocation than the +shared mutex does". Round 1 listed that as an untested assertion and the pinned-staging contention +story as its leading hypothesis. **Measured: the comment is right and the hypothesis is wrong.** + +### Honest limits of this experiment + +- The patch assigns slots from a **monotonic** counter `% n_slots`, exactly as PLAN.md specified. But + `run_admitted` is called 3× per epoch × 4 epochs, spawning fresh OS threads each time, so successive + cohorts land on *different* slabs and each of the 32 slabs gets grown at least once. That is the + pessimal form of per-driver slots. A pool/free-list variant (slots 0..K−1 reused every cohort) would + pay the growth cost once and isolate contention better. So this refutes *this* intervention + decisively and the contention hypothesis strongly, but not the last 6 pp of nuance. +- Which is why the confound-free measurement is the 1a-substitute counters (next section): if driver + threads spend only milliseconds waiting on that lock, no intervention design can rescue the + hypothesis. +- Variance is consistently higher in the patched arm (max−min 0.55–0.72 s vs 0.28–0.46 s for stock), + consistent with allocation noise rather than lock-queue noise. + +**Unmeasured at this point:** 1a-substitute counters (next), 1c single-epoch, Stage 2, Stage 3, 1b's +8-core leg. + +--- + +## Stage 1a-substitute — the mutex is *massively* contended and the contention is **free** + +`data/results_instr.csv`, 36 runs, one instrumented binary, slot policy switched at run time so the +two arms differ in nothing else. Medians of n=3 (n=2 for one cell). + +| K | slots | wall s | aggregate driver **wait** | ms per acquisition | worst single wait | rayon acquisitions | +|---|---|---|---|---|---|---| +| 1 | off | **16.02** | 187 ms | 0.134 | 92 ms | **0** | +| 1 | on | 17.06 | 2.0 ms | 0.001 | 0.0 ms | 0 | +| 2 | off | **11.69** | 1 722 ms | 1.235 | 184 ms | 0 | +| 2 | on | 12.68 | 2.1 ms | 0.002 | 0.0 ms | 0 | +| 4 | off | **10.23** | 6 117 ms | 4.388 | 420 ms | 0 | +| 4 | on | 10.87 | 2.2 ms | 0.002 | 0.0 ms | 0 | +| 8 | off | **9.68** | 13 205 ms | 9.473 | 497 ms | 0 | +| 8 | on | 10.74 | 2.3 ms | 0.002 | 0.1 ms | 0 | +| 16 | off | **9.57** | 21 222 ms | 15.224 | 378 ms | 0 | +| 16 | on | 10.40 | 4.0 ms | 0.003 | 1.6 ms | 0 | +| 31 | off | **9.14** | 23 411 ms | 16.794 | 475 ms | 0 | +| 31 | on | 10.06 | 755 ms | 0.541 | 384 ms | 0 | + +Sanity check that the instrument is sound: **the acquisition count is exactly 1394 in all 36 runs**, +independent of K and of slot policy — the number of staged transfers is a property of the workload, as +it should be. + +### Four measured findings + +1. **The hypothesis' premise is true, in spades.** Stock aggregate driver wait on the shared slot-0 + mutex rises monotonically with K: 0.19 s → 1.7 s → 6.1 s → 13.2 s → 21.2 s → **23.4 s**, i.e. from + 0.13 ms to 16.8 ms per staged transfer, with single waits up to half a second. Round 1 was right + that the K drivers pile onto one mutex. +2. **And it costs nothing.** Over the same range, wall time *falls* 16.02 s → 9.14 s. Between K=8 and + K=31, aggregate wait nearly doubles (13.2 s → 23.4 s) while wall time *improves* 5.6 %. Lock-wait + time here is overlapped queueing, not lost time: a driver blocked on the slab has nothing else to do, + because what the holder is doing (a D2H DMA plus a host memcpy) is the very resource it is waiting for. +3. **Removing the contention entirely does not help — it hurts.** Per-driver slots cut the wait by up to + **5 000×** (21 222 ms → 4.0 ms at K=16) and every single arm is *slower* by 0.6–1.1 s. Combined with + Stage 1b's independent build-level A/B (+6–12 %, p=0.002 at every K), the pinned-staging contention + hypothesis is **refuted from both directions**: the intervention that provably removes the contention + provably does not pay. +4. **The K=1 arm prices the alternative cost exactly.** At K=1 there is essentially no contention to + remove (187 ms → 2 ms), yet per-driver slots cost **1.04 s**. `run_admitted` is called 3× per epoch + over 4 epochs, so 12 successive driver cohorts land on 12 *different* grow-only slabs and each is + grown from scratch, instead of one hot slab reused 12 times. ~12 pinned re-allocations of a + multi-hundred-MB buffer at ~100 ms each ≈ the 1.0–1.2 s observed. That is precisely the mechanism + `device.rs:513-517` claims, now measured. + +### A code-level finding that fell out of this: the rayon arm of `worker_slot` is dead here + +**`ray_n` = 0 in all 36 runs.** No rayon worker ever acquires a pinned-staging or pinned-hashes lock in +this workload — every one of the 1394 acquisitions comes from a non-rayon thread. So on the `cuda` +path the whole *per-rayon-worker* slab design (`Vec>` sized +`rayon::current_num_threads()` = 32) is exercised at exactly **one** index, and `worker_slot`'s +`Some(i)` arm never fires for staging. 31 of 32 slabs are allocated-but-unused bookkeeping. +That is a cheap simplification opportunity, and it also means the "per-worker split" rationale in the +doc comment describes a case that does not occur on this path. (Measured on this workload only — +a CPU-side path that calls `htod_via`/`async_dtoh_via` from inside a rayon region would change it.) + +### What this does and does not license + +- It does **not** say the staging path is fast or well-designed. It says its *mutex* is not the K curve's + limiter, and that the specific fix in PLAN.md is net negative. +- The untested design that could still win: a **pre-grown slab pool** (K slabs allocated once at max + size and recycled across cohorts) rather than monotonic per-thread slots. That separates "remove + queueing" from "pay for cold slabs", which neither of my two interventions does. Cost ceiling for + such a fix is bounded by the lock's *occupancy* — measured next. + +### Hold time closes the accounting (`data/results_instr2.csv`, 18 runs, n=3) + +Second instrumented build adds the time each staging slot is *held*. In stock mode all drivers use slot +0, so holds cannot overlap and aggregate hold time **is** serialized occupancy of that one slab. + +| K | slots | wall s | aggregate wait | aggregate **hold** | ms held per transfer | hold ÷ wall | +|---|---|---|---|---|---|---| +| 1 | off | 15.90 | 208 ms | 3 295 ms | 2.36 | **0.21×** | +| 1 | on | 17.11 | 2 ms | 4 306 ms | 3.09 | 0.25× | +| 8 | off | 9.73 | 13 189 ms | 4 191 ms | 3.01 | **0.43×** | +| 8 | on | 10.50 | 2 ms | 17 543 ms | 12.58 | 1.67× | +| 31 | off | 8.97 | 21 075 ms | 5 021 ms | 3.60 | **0.56×** | +| 31 | on | 10.35 | 1 006 ms | 33 572 ms | 24.08 | 3.24× | + +1. **The K=1 row closes the causal loop on the regression.** With one slab live at a time there is no + overlap and no queueing in either arm, so the hold delta is pure allocation: **+1 011 ms** + (3 295 → 4 306). The measured wall-time regression at K=1 is **+1 210 ms**. The repeated + `cuMemHostAlloc` accounts for essentially all of it. +2. **Per-driver slots inflate the critical section 4–7×**: 3.01 → 12.58 ms per transfer at K=8, + 3.60 → 24.08 ms at K=31, and the inflation *grows* with the number of distinct slabs — consistent + with the code's own note that per-call driver-level allocation "convoys the driver lock under load". +3. **In stock mode the critical-section length is essentially K-independent** (2.36 / 3.01 / 3.60 ms at + K = 1 / 8 / 31). Nothing inside the section degrades under load; it is a fixed per-transfer cost. +4. **Slot 0 is held 43 % of wall time at K=8 and 56 % at K=31 — and wall time still improves.** That is + the cleanest statement of why the hypothesis fails: what the mutex serializes is a *genuinely serial + resource* (a staged D2H over one PCIe gen4 ×16 link plus a host memcpy), so letting more copies be + in flight cannot multiply the bandwidth behind them. The other drivers are not idle while one holds + the slab — they are running GPU kernels. The mutex marks the serialization; it does not cause it. + (The PCIe-bandwidth framing is the natural explanation for the pattern, not something I measured — + I did not instrument bytes transferred.) + +**Revised view of the slab-pool idea:** weaker than it looked. A pool would keep the 3 ms section and +allow K-way overlap, which relative to the *on* arm saves the allocation inflation — but the *off* arm +already achieves stock timing with a fully serialized slab, and giving staging K-way overlap (which the +*on* arm did achieve, wait → 2 ms) produced no gain at all. So the expected upside of a pool is around +zero to ~1 s at K=8 with wide error bars, and the honest statement is that it is unresolved rather than +promising. It is not where the next optimisation should start. + +--- + +## Stage 1c — the author's counter-claim, reconciled: the penalty scales with epoch count + +`data/results_ab1ep.csv`, 20 runs, interleaved, `--epoch-size-log2 23` so the whole 6.807 M-cycle +workload fits in **one** epoch (confirmed: `Epochs: 1` in both arms' logs). One epoch ⇒ 3 +`run_admitted` cohorts instead of 12, so ~4× fewer fresh slabs to grow. + +| K | epochs | stock s | patched s | patched cost | p | +|---|---|---|---|---|---| +| 8 | 4 (2^21) | 9.58 | 10.54 | +10.0 % | 0.002 | +| 8 | **1** (2^23) | 7.44 | 7.51 | **+1.0 %** | 0.151 (ns) | +| 21 | 4 (2^21) | 9.31 | 10.17 | +9.2 % | 0.002 | +| 21 | **1** (2^23) | 7.22 | 7.71 | **+6.4 %** | 0.008 | + +**The per-driver-slot penalty collapses from +10.0 % to +1.0 % at K=8 when the run has one epoch instead +of four** — and at one epoch, K=8, it is no longer distinguishable from stock at all. That is the +predicted fingerprint of the repeated-allocation mechanism and nothing else: the cost tracks *how many +driver cohorts are spawned*, hence how many cold grow-only slabs must be grown, not how much contention +is relieved (contention relief is identical in both epoch configurations). + +So `device.rs:513-517` — "per-driver slots cost more in repeated pinned allocation than the shared mutex +does" — is **confirmed, with the scaling law made explicit**: the cost is per-cohort, so it grows with +epoch count, which is exactly the direction that matters, since real blocks run tens to hundreds of +epochs, not four. + +Note this is the *opposite* of the reconciliation PLAN.md anticipated ("1b wins overall but loses on +epoch 1"). Per-driver slots lose everywhere; they merely lose *least* where there are fewest slab +growths. Same underlying mechanism, inverted sign. + +Incidental (not the question, but recorded): one 2^23 epoch proves the same 6.807 M cycles in 7.22 s +versus 8.98–9.14 s for four 2^21 epochs — ~20 % of this workload's wall time is per-epoch continuation +overhead at 2^21. + +--- + +## Stage 2 — `RAYON_NUM_THREADS` decomposition: **the rayon pool width does not touch W** + +`data/results_rt.csv`. `taskset` shrinks the rayon pool *and* the K driver threads' CPUs; +`RAYON_NUM_THREADS` shrinks **only** the pool, leaving the drivers all 32 CPUs. Under the Amdahl +reading, S is rayon-parallel host work and W is the work K divides — so if W is genuinely not +rayon-bound, shrinking the pool should inflate S and leave W alone. + +Complete, 72 runs, n=3 per cell: + +| `RAYON_NUM_THREADS` | T(K=1) | T(best) | best K | **W ≈ T1−Tb** | fitted S | fitted W | rms | +|---|---|---|---|---|---|---|---| +| 32 | 16.05 | 8.82 | **31** | **7.23** | 8.20 | 7.78 | 0.26 | +| 8 | 19.14 | 10.77 | **31** | **8.37** | 11.27 | 7.79 | 0.37 | +| 4 | 24.33 | 17.36 | **31** | **6.97** | 16.02 | 8.32 | 0.51 | + +**Shrinking the rayon pool 8× (32 → 4 threads) leaves W at 6.97–8.37 s with no monotone trend, while S +roughly doubles (8.2 → 16.0 s). Best K is 31 at every pool width.** The fitted W is 7.78–8.32 s across +all three — flatter still than the model-free estimate. + +This is the most direct refutation of the merged formula's *shape* that these experiments can produce. +`table_parallelism()` scales K by `available_parallelism()`, which is exactly the quantity that sizes the +rayon pool — and the rayon pool width provably has **no effect on the work K parallelizes**. It only +changes the part K cannot touch. A knob whose optimum divides a quantity that is invariant to `cores` +must not be a function of `cores`. + +Contrast with `taskset`, which restricts the drivers' CPUs as well: + +| config | pool width | driver CPUs | W | S | +|---|---|---|---|---| +| full | 32 | 32 | 6.85 | 8.82 | +| `RAYON_NUM_THREADS=4` | 4 | **32** | **6.97** | 16.02 | +| `taskset -c 0-3` | 4 | **4** | **5.34** | 20.08 | + +Same pool width, and W is 6.97 when the drivers have real CPUs versus 5.34 when they are confined to 4. +So what little core-sensitivity K's benefit has comes from the **driver threads' own CPU availability**, +not from the pool the formula is derived from — and even at 4 CPUs, K=12–31 were statistically +indistinguishable (Stage 4). + +Confound recorded, not removed: `RAYON_NUM_THREADS` also shrinks `n_slots` +(= `rayon::current_num_threads()`), so the staging slab array shrinks with it. For the staging path that +confound is **inert**, because the instrumented runs showed rayon workers never acquire those locks at +all (`ray_n` = 0) and stock drivers all use slot 0 regardless of how many slots exist. + +--- + +## Stage 1b, 8-core leg — same verdict, weaker power + +`data/results_ab8.csv`, 40 runs interleaved, `taskset -c 0-7`, **`n_slots`=8** so patched K=16 drivers +alias in pairs and K=31 in ~quads (fewer distinct slabs to grow than at 32 slots, i.e. the allocation +penalty should be *smaller* here). n=4. + +| K | stock s | patched s | patched cost | p | +|---|---|---|---|---| +| 1 | 18.97 | 19.98 | +5.3 % | 0.029 | +| 4 | 13.82 | 13.70 | −0.9 % | 0.486 (ns) | +| 8 | 12.66 | 13.25 | +4.7 % | 0.057 | +| 16 | 12.78 | 13.14 | +2.8 % | 0.343 (ns) | +| 31 | 12.20 | 13.07 | +7.1 % | 0.057 | + +Same direction at every K but 4 reps instead of 6, so only K=1 clears p<0.05. **The single cell where +per-driver slots are nominally faster (K=4, −0.9 %) is squarely inside noise (p=0.49).** Best K is 31 for +both binaries. Nothing here rescues the hypothesis; the 32-core leg (p=0.002 at every K) remains the +load-bearing evidence. + +--- + +## Stage 3 — **the #863 reconciliation hypothesis is also refuted.** The old scheduler wanted large K too + +`data/results_old32.csv`, built from **`1e1e0f18`** — the last commit with `plan_table_chunks`, i.e. the +fixed-chunk scheduler #877 replaced. Same workload, full cores, n=3. + +| K | 1 | 2 | 3 | 4 | 6 | 8 | 12 | 16 | 21 | 31 | +|---|---|---|---|---|---|---|---|---|---|---| +| median s | 14.15 | 13.56 | 13.30 | 13.09 | 12.13 | 11.80 | 11.23 | 11.34 | 11.14 | **10.42** | +| vs best | 1.36× | 1.30× | 1.28× | 1.26× | 1.17× | 1.13× | 1.08× | 1.09× | 1.07× | 1.00× | + +knee@5 % = **31**. Monotone decreasing all the way; **K=31 is 26 % faster than K=1 and 12 % faster than +K=8 on the old scheduler.** + +Round 1's stated hypothesis was that #863's "K saturates at 3" happened because the old scheduler used +fixed chunks with a barrier, so raising K raised straggler cost. **Measured on the actual old-scheduler +commit, there is no saturation at 3 — or anywhere below 31.** So the barrier/straggler story does not +explain #863's observation; whatever produced "K saturates at 3" there was something else (different +epoch size, workload, or branch state), and it should not be cited as reconciled. Marking this as a +refuted hypothesis rather than a resolved one. + +Two incidental measurements worth keeping: + +- **#877's scheduler is a real win at usable K**: best-to-best, 10.42 s (old) → 9.03 s (new) = **−13.3 %** + at K=31. The PR's core change is well justified; it is only its *default-K formula* that these + experiments dispute. +- **…but it regressed K=1**: 14.15 s (old) → 15.88 s (new), i.e. **+12 %** with a single driver. Nobody + runs K=1 by default, so this is a curiosity rather than a bug report, but it means the two schedulers + cross over around K≈2–4 and the new one's advantage is entirely in its concurrency. + +--- + +## Stage 6 — powering up the load-bearing null. `num_airs` is not beaten anywhere + +Stage 4's central claim rested on n=4 cells with p ∈ [0.057, 1.0] — underpowered nulls, exactly what +adversarial verification should refuse to accept. Re-measured the plateau at the two low core counts +with **n=8** (`data/results_c4b.csv`, `data/results_c8b.csv`, 80 runs). + +**4 cores** (`taskset -c 0-3`, n=8): + +| K | 8 | 12 | 16 | 21 | 31 | +|---|---|---|---|---|---| +| median s | 20.19 | 20.14 | 20.18 | 20.23 | **20.03** | +| vs K=31 | +0.8 % | +0.5 % | +0.7 % | +1.0 % | — | +| p vs K=31 | 0.72 | 0.52 | 0.38 | 0.80 | — | + +**Everything from K=8 to K=31 is within 1.0 % and indistinguishable, with K=31 nominally best.** The n=4 +result ("best K=12, K=31 costs +3.6 %") was noise. + +**8 cores** (`taskset -c 0-7`, n=8): + +| K | 8 | 12 | 16 | 21 | 31 | +|---|---|---|---|---|---| +| median s | 12.91 | 12.75 | 12.76 | **12.42** | 12.50 | +| vs K=31 | +3.3 % | +2.0 % | +2.1 % | −0.6 % | — | +| p vs K=31 | 0.10 | 0.19 | 0.23 | 0.88 | — | + +K=21 and K=31 are equivalent (0.6 % apart, p=0.88). K=8 is +3.3 %, the same direction as everywhere else, +but does not clear significance even at n=8. + +### Rule table, recomputed with the high-n legs merged in + +| rule | c32 | c16 | c8 | c4 | worst (r2) | worst (r1) | +|---|---|---|---|---|---|---| +| `cores/3` | +6.7 % | +12.6 % | +19.7 % | +23.2 % | +23.2 % | +30.2 % | +| `cores*2/3` (#877) | +2.9 % | +8.6 % | +8.1 % | +13.0 % | **+13.0 %** | +11.0 % | +| constant 8 | +6.7 % | +8.6 % | +4.7 % | +1.1 % | +8.6 % | +4.8 % | +| constant 12 | +7.0 % | +5.7 % | +2.0 % | 0.0 % | +7.0 % | +3.8 % | +| constant 16 | +3.3 % | +4.4 % | +3.2 % | +0.4 % | +4.4 % | +5.0 % | +| constant 21 | +2.9 % | +5.4 % | 0.0 % | 0.0 % | +5.4 % | +4.0 % | +| **unbounded = `num_airs`** | **0.0 %** | **0.0 %** | **+1.6 %** | **+0.6 %** | **+1.6 %** | +5.9 % | + +**With the low-core legs at n=8, `num_airs` is the best rule by a clear margin: +1.6 % worst case, and +both of its nominal deficits (c8 +1.6 %, c4 +0.6 %) are far inside noise (p=0.88 and p=0.80 against the +respective best K).** The only number that ever argued against unbounded K was round 1's c8 cell +(+5.9 %), and that comparison is p=0.34 at n=4; round 2's n=8 measurement of the same configuration puts +it at +1.6 %. + +Final recommendation, and it is also the simplest possible change: **`table_parallelism()` should not +consult `available_parallelism()` at all — K should be `num_airs`**, which `multi_prove` already clamps +to via `.min(num_airs)`. The only measured cost is +19 % host peak heap, saturating by K≈16. + cross over somewhere around K≈2–4 and the new one's advantage is entirely in its concurrency, not its + per-table path. + +--- + +## Files + +**Data** (`data/`, mirrored from the box; 539 timed runs across 14 `results_*.csv`): + +| file | what | +|---|---| +| `box.txt` | GPU/CPU/nvcc/commit/build-features fingerprint | +| `results_c32.csv`, `results_c32b.csv` | Stage 0 anchor (n=3) + plateau pass (n=6), 32 cores | +| `results_c16.csv`, `results_c8.csv`, `results_c4.csv` | Stage 4 core-count legs (n=4) | +| `results_c8b.csv`, `results_c4b.csv` | Stage 6 high-n plateau at 8 and 4 cores (n=8) | +| `results_ab32.csv`, `results_ab8.csv` | Stage 1b stock-vs-per-driver-slots, interleaved | +| `results_ab1ep.csv` | Stage 1c single-epoch (2^23) A/B | +| `results_instr.csv` | staging-mutex **wait** time vs K, both slot policies | +| `results_instr2.csv` | staging-mutex **hold** time vs K, both slot policies | +| `results_rt.csv` | Stage 2 `RAYON_NUM_THREADS` × K | +| `results_old32.csv` | Stage 3 old fixed-chunk scheduler (`1e1e0f18`) | +| `patch_slots.diff`, `patch_instr.diff`, `patch_instr2.diff` | exact source of each experimental build | +| `slotproof.txt` | proof the Stage 1b intervention took effect + negative control | +| `stacks_k*_none.txt`, `stackclass.txt` | Stage 1a's gdb attach failures, kept as evidence | +| `log__*_k.txt` | full CLI output (incl. `Peak heap`) for rep 1 of each cell | +| `progress.log`, `keepalive.log` | stage timeline and the three reboot/resume events | +| `derived_merged_c32.csv` | c32 + c32b concatenated, for reproducing the c32 Amdahl fit | + +**Harnesses** (server-side, all resumable, one fsync'd CSV row per run, frozen job order): +`sweep_r.sh` (single-binary K sweep) · `sweep_ab.sh` (interleaved multi-binary A/B, records `n_slots`) · +`sweep_rayon.sh` (`RAYON_NUM_THREADS` × K; replaces round 1's `followup2.sh`, which truncated its CSV on +every start and reshuffled on resume — fatal with an auto-resuming keepalive) · `sweep_instr.sh` +(instrumented runs, parses the staging counters) · `stacksample.sh` (round 1's, output paths corrected; +unusable on this box). + +**Analysis** (laptop-side): `analyze_ab.py` (A/B medians + exact Mann–Whitney, no scipy) · +`rules2.py` (two-box rule table) · `amdahl.py` (`S + max(Tmax, W/K)` fit) · `analyze_rt.py` (per-pool-width +decomposition). Round 1's `analyze.py` / `rules.py` / `classify_stacks.py` reused unchanged. + +**Not committed** (server-side only, recreatable from the diffs above): `patch_slots.py`, +`patch_instr.py`, `patch_instr2.py`, `keepalive.sh`, `runall*.sh`, `stage*.sh`. + +## Reproducing + +``` +ssh -p 63821 root@79.161.122.162 # while it lives +/root/results/CURRENT_CMD # empty = idle; write a stage script path to restart +cat /root/results/progress.log # what ran, when +``` +Binaries on the box: `/root/bin/cli_stock` (7644043b as-is), `cli_slots` (per-driver slots), +`cli_instr` (wait counters + runtime slot switch), `cli_instr2` (+ hold counters), +`cli_oldsched` (`1e1e0f18`). Worktrees: `/root/lambda_vm{,_slots,_instr,_instr2,_old}`. diff --git a/scripts/profiling/table-parallelism-sweep/round2/PLAN.md b/scripts/profiling/table-parallelism-sweep/round2/PLAN.md new file mode 100644 index 000000000..0ce33a671 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/PLAN.md @@ -0,0 +1,89 @@ +# K sweep, round 2 — the three things the expired box never measured + +Continuation of `../round1/` (347 runs, 2026-08-03). That round established the *what*: +optimal `TABLE_PARALLELISM` (K) is core-invariant, knee K≈8 across 4→32 cores, so #877's merged +`cores*2/3` is the wrong form. This round goes after the *why*, plus the two decompositions the box +died before running. + +## Box (round 2) + +RTX 5090 (32607 MiB, driver 595.71.05, nvcc **13.1**) · AMD Ryzen 9 **7950X** 16 physical / 32 logical +· 93 GB RAM · 62 GB free disk · gdb/tmux/python3/rsync present, **no cargo — install rustup first**. +`ssh -p 63821 root@79.161.122.162` (vast.ai, expected to expire without warning). + +Differs from round 1 in one respect: 7950X, **not** 7950X3D — no 3D V-cache. That is why Stage 0 +re-anchors the curve before anything is built on top of it. + +Code under test: `origin/main` @ **`7644043b`** (contains #875 `5749a956` and #877). +Workload held identical to round 1 for comparability: `cli prove ethrex.elf --private-input +ethrex_10_transfers.bin --continuations --epoch-size-log2 21` → 6.807M cycles, 4 epochs, ~31 +tables/epoch. One timed run ≈ 10s at full cores, so runs are cheap and **build time dominates**. + +## Stages, ordered value-first + +Each stage answers a question on its own, so a box death between stages still leaves a publishable +result. Within a stage, ordering is randomized-interleaved and resumable (see Checkpointing). + +**Stage 0 — bring-up + anchor.** rustup, clone at `7644043b`, build `release` + `cuda`; then a second +symbolized build (`CARGO_PROFILE_RELEASE_DEBUG=2`) into a separate target dir for Stage 1a. Anchor +curve at full cores, K ∈ {1,2,4,6,8,12,16,21,31}, n=3. **Gate:** the knee must land at ~8 and best +time near ~9.8s. If it doesn't reproduce on the non-3D part, say so loudly — every later stage is +interpreted relative to this curve. + +**Stage 1 — pinned-staging slot-0 contention.** The leading mechanistic explanation for a +core-invariant knee: a single mutex saturates at a contender count set by the staging/compute ratio, +which depends on the device and the workload but not on host cores. Three probes: + +- **1a observational.** `stacksample.sh` + `classify_stacks.py` (both in `../round1/`) against + the symbolized build at K ∈ {1,4,8,16,31}, ≥15 samples/run. Prediction if the hypothesis holds: + driver-thread time in `pinned-staging mutex` climbs with K and saturates right around the knee. + Falsified if drivers are dominated by `CUDA sync / memcpy` at every K. +- **1b interventional — the decisive one.** Give non-rayon callers distinct slots and re-run the + Stage 0 curve. In `crypto/math-cuda/src/device.rs`, `worker_slot()` currently returns + `rayon::current_thread_index().unwrap_or(0)`. Change *only* the `None` arm to hand each + non-rayon thread a stable distinct index (thread-local id from a monotonic `AtomicUsize`, `% len`); + keep the rayon arm and the clamp untouched. It covers both `pinned_staging` and `pinned_hashes`, + since both call it. **Watch the slot count:** `n_slots = rayon::current_num_threads()`, so 32 on + this box — per-driver slots fit up to K=31 with no resize, but under `taskset -c 0-7` there are + only 8 slots and K=16 drivers *alias in pairs*. Record `n_slots` alongside every row; do not + compare across configs that differ in it. +- **1c the author's counter-claim.** device.rs:513-517 asserts per-driver slots "cost more in + repeated pinned allocation than the shared mutex does" — unpublished. Slabs grow lazily and are + reused across all 4 epochs, so repeated allocation can only bite on epoch 1. Get the per-epoch + split (the `--time` phase output, kept per K in the run logs) and the `pinned alloc` category from + 1a. If 1b wins overall but loses on epoch 1, that reconciles both observations and is the finding. + +**Stage 2 — `RAYON_NUM_THREADS` decomposition.** `../round1/followup2.sh`, written and never +run. `taskset` shrinks pool width *and* the drivers' CPUs; `RAYON_NUM_THREADS` shrinks only the +pool. Tests claim #1 (K is not a thread count) directly. Confound to record, not remove: it also +shrinks `n_slots`. Run on the **stock** build so it stays comparable to round 1. + +**Stage 3 — old-scheduler K curve.** Build `1e1e0f18` (last commit with `plan_table_chunks`) and +sweep the same K list. Reconciles #863's "K saturates at 3" with this round's 8-31: there K was a +fixed chunk size with a barrier, so raising it raised straggler cost. Last because it needs a whole +second build for a historical question. + +**Stage 4 — rule data, if the box survives.** Re-run c4/c8/c16/c32 on the winning build so the PR +replacing `cores*2/3` carries two-box evidence rather than one. + +## Checkpointing — the box will die mid-run + +1. Everything runs under **tmux** on the server; ssh drops don't kill a sweep. +2. `sweep_r.sh` appends and `sync`s **one CSV row per timed run**. Nothing is buffered to the end. +3. `pull.sh` runs on the laptop and mirrors `root@…:/root/results/` → `./data/` every 60s, so the + local copy is never more than a minute stale. Started before the sweep, independent of the agent. +4. **Resumable without reshuffling.** The randomized order is generated once into + `jobs_.txt` and reused on restart; already-recorded `(k,rep)` pairs are skipped. Round 1's + harness reshuffled on restart, which would have silently broken the interleaving that the + randomization exists to provide. +5. `box.txt` fingerprints GPU/CPU/nvcc/commit/`n_slots` at bring-up. A *different* rented box on a + later resume would otherwise silently mix into the same CSVs. +6. `NOTES.md` here is updated at every stage boundary with the conclusion so far — a death mid-Stage-2 + still leaves Stage 0 and 1 written up, not just raw CSVs. + +## Files + +`sweep_r.sh` (resumable K sweep, server-side) · `pull.sh` (local mirror loop) · `data/` (CSVs + +per-K run logs, pulled) · `NOTES.md` (running conclusions). Harnesses reused unchanged from +`../round1/`: `stacksample.sh`, `classify_stacks.py`, `followup2.sh`, `analyze.py`, `rules.py`, +`gpuutil.sh`. diff --git a/scripts/profiling/table-parallelism-sweep/round2/amdahl.py b/scripts/profiling/table-parallelism-sweep/round2/amdahl.py new file mode 100644 index 000000000..aefc00d22 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/amdahl.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +"""Third framing of the K curve: is it just Amdahl + a longest-table floor? + +The pinned-staging mutex is one candidate mechanism for a core-invariant knee. +A more mundane one needs checking first, because `run_admitted` is a work-stealing +list scheduler over ~31 very unequal tables (round 1: ~43% near-empty): + + T(K) = S + max(Tmax, W/K) + +S = the K-independent serial part (continuation producer, execute, trace build), +W = total table-proving work, Tmax = the longest single table. That form has a +knee at K = W/Tmax and it is core-invariant *in shape* for any host, because +neither S, W nor Tmax depends on how many drivers you spawn. If it fits the +measured curve, no lock-contention story is needed to explain a knee — the knee +is just where the list schedule stops being able to use another driver. + +Grid-searches (S, W, Tmax) on the median curve and reports the residual. +Usage: amdahl.py [ ...] (any sweep CSV with k,seconds columns) +""" +import csv, sys, statistics as st +from collections import defaultdict + + +def fit(meds): + ks = sorted(meds) + best = None + t1 = meds[ks[0]] + for si in range(0, 1201): + S = si * 0.01 + if S >= t1: + break + for wi in range(1, 2001): + W = wi * 0.01 + for ti in range(0, 801): + Tmax = ti * 0.01 + err = 0.0 + for k in ks: + pred = S + max(Tmax, W / k) + err += (pred - meds[k]) ** 2 + if best is None or err < best[0]: + best = (err, S, W, Tmax) + return best + + +def fit_coarse(meds): + """Two-pass: coarse grid then refine, so this runs in seconds not hours.""" + ks = sorted(meds) + t1 = meds[ks[0]] + + def score(S, W, Tmax): + return sum((S + max(Tmax, W / k) - meds[k]) ** 2 for k in ks) + + best = None + for S in [i * 0.1 for i in range(0, int(t1 * 10))]: + for W in [i * 0.1 for i in range(1, 300)]: + for Tmax in [i * 0.1 for i in range(0, 120)]: + e = score(S, W, Tmax) + if best is None or e < best[0]: + best = (e, S, W, Tmax) + _, S0, W0, T0 = best + for S in [S0 + i * 0.01 for i in range(-12, 13)]: + for W in [W0 + i * 0.01 for i in range(-12, 13)]: + for Tmax in [T0 + i * 0.01 for i in range(-12, 13)]: + if S < 0 or W <= 0 or Tmax < 0: + continue + e = score(S, W, Tmax) + if e < best[0]: + best = (e, S, W, Tmax) + return best + + +for path in sys.argv[1:]: + d = defaultdict(list) + for r in csv.DictReader(open(path)): + if r["seconds"] != "NA": + d[int(r["k"])].append(float(r["seconds"])) + meds = {k: st.median(v) for k, v in d.items()} + e, S, W, Tmax = fit_coarse(meds) + n = len(meds) + print(f"\n=== {path} === n_K={n}") + print(f" fit T(K) = {S:.2f} + max({Tmax:.2f}, {W:.2f}/K) rms={((e/n)**0.5):.3f}s") + print(f" implied knee K* = W/Tmax = {W/Tmax:.1f}" if Tmax > 0 else " Tmax=0 (pure Amdahl)") + print(f" {'K':>4} {'measured':>9} {'pred':>8} {'resid':>7}") + for k in sorted(meds): + pred = S + max(Tmax, W / k) + print(f" {k:>4} {meds[k]:>9.2f} {pred:>8.2f} {pred-meds[k]:>+7.2f}") diff --git a/scripts/profiling/table-parallelism-sweep/round2/analyze_ab.py b/scripts/profiling/table-parallelism-sweep/round2/analyze_ab.py new file mode 100644 index 000000000..f5cc5bed7 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/analyze_ab.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +"""Summarize an interleaved A/B K sweep (sweep_ab.sh CSV). + +Per K: median per binary, the A/B delta, and a Mann-Whitney U two-sided p-value +computed exactly from the rank sum (no scipy on the rented box). Small n, so the +p-values are there to stop us reading noise as signal, not to prove anything. +""" +import csv, sys, statistics as st +from collections import defaultdict +from itertools import permutations +from math import comb + + +def mannwhitney_p(a, b): + """Two-sided exact Mann-Whitney U p-value. Falls back to None if too big.""" + na, nb = len(a), len(b) + if na == 0 or nb == 0: + return None + if comb(na + nb, na) > 200000: + return None + u_obs = sum(1 for x in a for y in b if x < y) + 0.5 * sum(1 for x in a for y in b if x == y) + # Exact null distribution over all label assignments of the pooled ranks. + pooled = sorted(a + b) + ranks = {v: i for i, v in enumerate(pooled)} + del ranks # unused; kept explicit that we enumerate values, not ranks + from itertools import combinations + cnt = tot = 0 + mean_u = na * nb / 2 + for idx in combinations(range(na + nb), na): + s = set(idx) + xa = [pooled[i] for i in s] + xb = [pooled[i] for i in range(na + nb) if i not in s] + u = sum(1 for x in xa for y in xb if x < y) + 0.5 * sum(1 for x in xa for y in xb if x == y) + tot += 1 + if abs(u - mean_u) >= abs(u_obs - mean_u) - 1e-9: + cnt += 1 + return cnt / tot + + +def main(paths): + for path in paths: + d = defaultdict(lambda: defaultdict(list)) + nslots = set() + with open(path) as f: + for r in csv.DictReader(f): + if r["seconds"] == "NA": + continue + d[int(r["k"])][r["cli"]].append(float(r["seconds"])) + nslots.add(r.get("n_slots", "?")) + bins = sorted({b for k in d for b in d[k]}) + print(f"\n=== {path} === n_slots={','.join(sorted(nslots))} bins={bins}") + hdr = f"{'K':>4}" + for b in bins: + hdr += f"{b+' med':>18}{'n':>4}" + if len(bins) == 2: + hdr += f"{'delta':>10}{'p':>8}" + print(hdr) + for k in sorted(d): + line = f"{k:>4}" + meds = {} + for b in bins: + v = sorted(d[k].get(b, [])) + meds[b] = st.median(v) if v else float("nan") + sp = (v[-1] - v[0]) if len(v) > 1 else 0.0 + line += f"{meds[b]:>13.2f}±{sp:>3.2f}{len(v):>4}" + if len(bins) == 2: + a, b = bins + delta = (meds[b] / meds[a] - 1) * 100 + p = mannwhitney_p(d[k].get(a, []), d[k].get(b, [])) + line += f"{delta:>+9.1f}%" + (f"{p:>8.3f}" if p is not None else f"{'-':>8}") + print(line) + # overall best per binary + print(" best per binary:") + for b in bins: + m = {k: st.median(d[k][b]) for k in d if d[k].get(b)} + bk = min(m, key=lambda k: m[k]) + print(f" {b:<12} best K={bk:<3} {m[bk]:.2f}s") + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/scripts/profiling/table-parallelism-sweep/round2/analyze_rt.py b/scripts/profiling/table-parallelism-sweep/round2/analyze_rt.py new file mode 100644 index 000000000..487443d44 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/analyze_rt.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +"""Stage 2: decompose the K curve per rayon-pool width. + +`taskset` shrinks the rayon pool AND the K driver threads' CPUs. +`RAYON_NUM_THREADS` shrinks only the pool — the drivers still get real cores. +Under the Amdahl reading (NOTES 'Stage 0.5'), S is rayon-parallel host work and W +is the K-parallelizable table work; if W really is not rayon-pool-bound, then +shrinking the pool should inflate S and leave W roughly alone. + +Confound recorded, not removed: RAYON_NUM_THREADS also shrinks +n_slots = rayon::current_num_threads(), though the staging measurements showed +rayon workers never touch those slabs anyway (ray_n = 0), so for the staging path +that confound is inert. + +Usage: analyze_rt.py results_rt.csv [and optionally taskset CSVs to compare] +""" +import csv, sys, statistics as st +from collections import defaultdict + + +def fit(meds): + ks = sorted(meds) + + def score(S, W, T): + return sum((S + max(T, W / k) - meds[k]) ** 2 for k in ks) + + t1 = meds[ks[0]] + best = None + for S in [i * 0.1 for i in range(0, int(t1 * 10))]: + for W in [i * 0.1 for i in range(1, 300)]: + for T in [i * 0.1 for i in range(0, 120)]: + e = score(S, W, T) + if best is None or e < best[0]: + best = (e, S, W, T) + _, S0, W0, T0 = best + for S in [S0 + i * 0.01 for i in range(-12, 13)]: + for W in [W0 + i * 0.01 for i in range(-12, 13)]: + for T in [T0 + i * 0.01 for i in range(-12, 13)]: + if S < 0 or W <= 0 or T < 0: + continue + e = score(S, W, T) + if e < best[0]: + best = (e, S, W, T) + return best + + +groups = defaultdict(lambda: defaultdict(list)) +for path in sys.argv[1:]: + for r in csv.DictReader(open(path)): + if r["seconds"] == "NA": + continue + key = r.get("rayon") or r.get("cpuspec") or "?" + groups[f"{path.split('/')[-1]}:{key}"][int(r["k"])].append(float(r["seconds"])) + +print(f"{'group':<26} {'K=1':>7} {'best':>7} {'bestK':>6} {'W~=T1-Tb':>9} " + f"{'fitS':>7} {'fitW':>7} {'fitTmax':>8} {'rms':>6}") +for g in sorted(groups): + meds = {k: st.median(v) for k, v in groups[g].items()} + if 1 not in meds or len(meds) < 4: + print(f"{g:<26} (insufficient K coverage: {sorted(meds)})") + continue + t1, tb = meds[1], min(meds.values()) + bk = min(meds, key=lambda k: meds[k]) + e, S, W, T = fit(meds) + print(f"{g:<26} {t1:>7.2f} {tb:>7.2f} {bk:>6} {t1-tb:>9.2f} " + f"{S:>7.2f} {W:>7.2f} {T:>8.2f} {(e/len(meds))**0.5:>6.3f}") + +print() +for g in sorted(groups): + meds = {k: st.median(v) for k, v in groups[g].items()} + tb = min(meds.values()) + row = " ".join(f"K{k}={meds[k]:.2f}({meds[k]/tb:.2f}x)" for k in sorted(meds)) + print(f"{g}:\n {row}") diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/box.txt b/scripts/profiling/table-parallelism-sweep/round2/data/box.txt new file mode 100644 index 000000000..1585a259d --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/box.txt @@ -0,0 +1,93 @@ +=== k-sweep-877b box fingerprint === +Tue Aug 4 17:48:11 UTC 2026 + +--- nvidia-smi --- +name, memory.total [MiB], driver_version, pcie.link.gen.max, pcie.link.width.max +NVIDIA GeForce RTX 5090, 32607 MiB, 595.71.05, 4, 16 + +--- nvcc --- +Built on Tue_Dec_16_07:23:41_PM_PST_2025 +Cuda compilation tools, release 13.1, V13.1.115 +Build cuda_13.1.r13.1/compiler.37061995_0 + +--- nproc --- +32 + +--- lscpu --- +CPU(s): 32 +Model name: AMD Ryzen 9 7950X 16-Core Processor +Thread(s) per core: 2 +Core(s) per socket: 16 +Socket(s): 1 +CPU(s) scaling MHz: 30% +CPU max MHz: 5881.0000 +CPU min MHz: 545.0000 +Flags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good amd_lbr_v2 nopl nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba perfmon_v2 ibrs ibpb stibp ibrs_enhanced vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local user_shstk avx512_bf16 clzero irperf xsaveerptr rdpru wbnoinvd cppc amd_ibpb_ret arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic vgif x2avic v_spec_ctrl vnmi avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg avx512_vpopcntdq rdpid overflow_recov succor smca fsrm flush_l1d ibpb_exit_to_user +L3 cache: 64 MiB (2 instances) + +--- mem/disk --- + total used free shared buff/cache available +Mem: 93 3 74 0 16 90 +overlay 64G 2.9G 62G 5% / + +--- os --- +PRETTY_NAME="Ubuntu 24.04.4 LTS" +NAME="Ubuntu" +Linux 19210dca8023 6.8.0-117-generic #117~22.04.1-Ubuntu SMP PREEMPT_DYNAMIC Thu May 7 22:17:46 UTC x86_64 x86_64 x86_64 GNU/Linux + +--- code under test --- +7644043 perf(prover): per-table scheduler with VRAM admission for multi_prove (#877) +cargo: cargo 1.94.0 (85eff7c80 2026-01-15) +build features: jemalloc-stats,prover/cuda (per .github/workflows/benchmark-gpu.yml BENCH_FEATURES) + +=== DATA-INTEGRITY EVENTS: unannounced host reboots === + +This box rebooted three times on its own during the session. A mid-sweep reboot +resets GPU/CPU clock state and the page cache, so it is a data-integrity event, not +just an availability one — and clock state is exactly what the 7950X vs 7950X3D +comparison turns on. Boot times below are derived from /proc/uptime at the +keepalive relaunch (see keepalive.log); the "rows affected" mapping is derived from +the sweep logs' warmup/SKIP markers. + +REBOOT 1 — boot ~2026-08-04T17:54:04Z + In flight: Stage 0 sweep `c32`, 2 runs recorded. + Rows PREDATING it: results_c32.csv rows (K=8 rep=2, 9.761s) and (K=6 rep=1, 9.553s) + — 2 of that file's 27. Every other timed row in the whole dataset POSTDATES it. + Note: these 2 rows were recorded before the harness was changed to warm up on + every start, so they are the only rows in the dataset whose warmup provenance + differs. Both sit mid-distribution for their K (K=8 spans 9.34-9.81 over n=9), + so they are not outliers; not excluded. + +REBOOT 2 — boot ~2026-08-04T18:46:24Z + In flight: Stage 1b's slot-policy proof run (not a timed measurement). + NO timed CSV row was lost or spans it. Stage 4 had completed at 18:43:52, so all + of results_c32/c32b/c8/c4/c16 predate it; results_ab32 and everything after + postdate it. + +REBOOT 3 — boot ~2026-08-04T20:18:56Z + In flight: Stage 6 leg `c8b`. + results_c4b.csv: all 40 rows predate it. + results_c8b.csv: 39 of 40 rows predate it; exactly 1 postdates it. (The log shows + 38 completed-run lines then a resume with 39 SKIPs — the 39th row was fsync'd to + the CSV but its tee'd log line was lost to the reboot. Working as designed: the + checkpointing contract is one fsync'd CSV row per run, and the log is best-effort.) + Also: with the sweep complete but CURRENT_CMD still set, cron relaunched the + finished script once a minute from 20:20 to 20:27. Each relaunch ran one + UNRECORDED warmup prove and then skipped every job. Eight stray warmup proves, + no effect on recorded data; CURRENT_CMD was then cleared. + +Reboots did not interrupt any A/B pairing: every A/B sweep interleaves its arms in +one frozen randomized order, so a reboot cannot land systematically on one arm. + +=== TOOLCHAIN (verified in-repo, not just the rustup default) === +rust-toolchain.toml pins channel=1.94.0 profile=default, so rustup fetched it on the +first cargo invocation and ALL builds used it. Confirmed inside /root/lambda_vm: +rustc 1.94.0 (4a4ef493e 2026-03-02) +cargo 1.94.0 (85eff7c80 2026-01-15) +This matches infra/provision.sh section 6. Installed toolchains: +stable-x86_64-unknown-linux-gnu (default) +nightly-2026-02-01-x86_64-unknown-linux-gnu +1.94.0-x86_64-unknown-linux-gnu (active) + +Repo is PUBLIC: https://github.com/yetanotherco/lambda_vm.git — the git-bundle transfer +used at bring-up was unnecessary but harmless (and still credential-free). diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/derived_merged_c32.csv b/scripts/profiling/table-parallelism-sweep/round2/data/derived_merged_c32.csv new file mode 100644 index 000000000..2f94a733e --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/derived_merged_c32.csv @@ -0,0 +1,64 @@ +tag,cpuspec,epoch,k,rep,seconds,cli +c32,none,21,8,2,9.761,cli_stock +c32,none,21,6,1,9.553,cli_stock +c32,none,21,4,3,10.136,cli_stock +c32,none,21,8,3,9.501,cli_stock +c32,none,21,6,3,9.614,cli_stock +c32,none,21,2,2,11.888,cli_stock +c32,none,21,16,1,9.308,cli_stock +c32,none,21,2,3,11.988,cli_stock +c32,none,21,1,2,16.029,cli_stock +c32,none,21,21,1,9.045,cli_stock +c32,none,21,12,3,9.686,cli_stock +c32,none,21,12,1,9.664,cli_stock +c32,none,21,8,1,9.632,cli_stock +c32,none,21,12,2,9.553,cli_stock +c32,none,21,21,3,9.319,cli_stock +c32,none,21,1,1,15.426,cli_stock +c32,none,21,1,3,15.885,cli_stock +c32,none,21,4,1,10.396,cli_stock +c32,none,21,2,1,11.877,cli_stock +c32,none,21,31,3,9.399,cli_stock +c32,none,21,31,2,9.106,cli_stock +c32,none,21,6,2,9.731,cli_stock +c32,none,21,31,1,8.951,cli_stock +c32,none,21,16,3,9.263,cli_stock +c32,none,21,16,2,9.160,cli_stock +c32,none,21,21,2,8.970,cli_stock +c32,none,21,4,2,10.551,cli_stock +c32b,none,21,21,5,9.256,cli_stock +c32b,none,21,31,6,9.022,cli_stock +c32b,none,21,12,3,9.290,cli_stock +c32b,none,21,31,5,8.804,cli_stock +c32b,none,21,16,3,9.528,cli_stock +c32b,none,21,21,4,9.290,cli_stock +c32b,none,21,16,4,9.504,cli_stock +c32b,none,21,31,1,8.758,cli_stock +c32b,none,21,12,4,9.651,cli_stock +c32b,none,21,8,4,9.459,cli_stock +c32b,none,21,16,6,9.494,cli_stock +c32b,none,21,6,6,9.814,cli_stock +c32b,none,21,31,4,9.057,cli_stock +c32b,none,21,16,2,9.434,cli_stock +c32b,none,21,16,1,9.261,cli_stock +c32b,none,21,21,2,9.128,cli_stock +c32b,none,21,21,1,9.586,cli_stock +c32b,none,21,16,5,9.326,cli_stock +c32b,none,21,6,4,9.435,cli_stock +c32b,none,21,12,2,9.656,cli_stock +c32b,none,21,31,2,9.027,cli_stock +c32b,none,21,8,6,9.572,cli_stock +c32b,none,21,12,6,9.886,cli_stock +c32b,none,21,8,5,9.807,cli_stock +c32b,none,21,6,2,9.975,cli_stock +c32b,none,21,8,2,9.639,cli_stock +c32b,none,21,21,6,9.322,cli_stock +c32b,none,21,12,5,9.838,cli_stock +c32b,none,21,31,3,9.157,cli_stock +c32b,none,21,6,1,10.075,cli_stock +c32b,none,21,21,3,9.336,cli_stock +c32b,none,21,6,3,9.904,cli_stock +c32b,none,21,12,1,9.630,cli_stock +c32b,none,21,8,1,9.337,cli_stock +c32b,none,21,6,5,9.880,cli_stock +c32b,none,21,8,3,9.700,cli_stock diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/jobs_ab1ep.txt b/scripts/profiling/table-parallelism-sweep/round2/data/jobs_ab1ep.txt new file mode 100644 index 000000000..21995e4e4 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/jobs_ab1ep.txt @@ -0,0 +1,20 @@ +/root/bin/cli_stock 8 4 +/root/bin/cli_stock 21 1 +/root/bin/cli_stock 21 5 +/root/bin/cli_stock 8 1 +/root/bin/cli_stock 8 2 +/root/bin/cli_slots 21 3 +/root/bin/cli_stock 8 5 +/root/bin/cli_stock 8 3 +/root/bin/cli_slots 8 3 +/root/bin/cli_stock 21 2 +/root/bin/cli_slots 21 4 +/root/bin/cli_stock 21 4 +/root/bin/cli_stock 21 3 +/root/bin/cli_slots 8 4 +/root/bin/cli_slots 8 2 +/root/bin/cli_slots 21 1 +/root/bin/cli_slots 8 1 +/root/bin/cli_slots 21 5 +/root/bin/cli_slots 8 5 +/root/bin/cli_slots 21 2 diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/jobs_ab32.txt b/scripts/profiling/table-parallelism-sweep/round2/data/jobs_ab32.txt new file mode 100644 index 000000000..34b03bcd8 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/jobs_ab32.txt @@ -0,0 +1,72 @@ +/root/bin/cli_stock 4 2 +/root/bin/cli_slots 31 2 +/root/bin/cli_stock 8 1 +/root/bin/cli_slots 16 2 +/root/bin/cli_stock 4 5 +/root/bin/cli_stock 31 1 +/root/bin/cli_stock 16 3 +/root/bin/cli_slots 1 5 +/root/bin/cli_slots 4 6 +/root/bin/cli_stock 1 2 +/root/bin/cli_stock 8 3 +/root/bin/cli_slots 8 4 +/root/bin/cli_slots 21 2 +/root/bin/cli_slots 16 5 +/root/bin/cli_slots 8 6 +/root/bin/cli_stock 21 3 +/root/bin/cli_slots 1 2 +/root/bin/cli_slots 4 3 +/root/bin/cli_stock 1 5 +/root/bin/cli_slots 21 4 +/root/bin/cli_stock 31 5 +/root/bin/cli_slots 4 4 +/root/bin/cli_slots 31 3 +/root/bin/cli_stock 31 6 +/root/bin/cli_slots 8 3 +/root/bin/cli_slots 21 1 +/root/bin/cli_stock 8 5 +/root/bin/cli_slots 31 1 +/root/bin/cli_stock 4 3 +/root/bin/cli_stock 1 6 +/root/bin/cli_slots 4 2 +/root/bin/cli_stock 21 2 +/root/bin/cli_stock 21 4 +/root/bin/cli_slots 21 6 +/root/bin/cli_slots 4 1 +/root/bin/cli_slots 21 5 +/root/bin/cli_slots 8 5 +/root/bin/cli_slots 31 4 +/root/bin/cli_stock 16 5 +/root/bin/cli_stock 16 6 +/root/bin/cli_slots 16 1 +/root/bin/cli_stock 1 3 +/root/bin/cli_stock 8 2 +/root/bin/cli_stock 8 6 +/root/bin/cli_stock 4 1 +/root/bin/cli_stock 16 1 +/root/bin/cli_slots 8 1 +/root/bin/cli_stock 31 3 +/root/bin/cli_stock 1 1 +/root/bin/cli_stock 16 4 +/root/bin/cli_stock 31 2 +/root/bin/cli_slots 1 6 +/root/bin/cli_slots 4 5 +/root/bin/cli_slots 1 3 +/root/bin/cli_slots 31 6 +/root/bin/cli_slots 8 2 +/root/bin/cli_stock 16 2 +/root/bin/cli_slots 21 3 +/root/bin/cli_slots 16 3 +/root/bin/cli_stock 21 5 +/root/bin/cli_slots 16 4 +/root/bin/cli_stock 1 4 +/root/bin/cli_stock 8 4 +/root/bin/cli_slots 1 1 +/root/bin/cli_slots 1 4 +/root/bin/cli_stock 21 6 +/root/bin/cli_stock 4 6 +/root/bin/cli_stock 31 4 +/root/bin/cli_stock 21 1 +/root/bin/cli_slots 31 5 +/root/bin/cli_stock 4 4 +/root/bin/cli_slots 16 6 diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/jobs_ab8.txt b/scripts/profiling/table-parallelism-sweep/round2/data/jobs_ab8.txt new file mode 100644 index 000000000..afbb80ab5 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/jobs_ab8.txt @@ -0,0 +1,40 @@ +/root/bin/cli_slots 31 2 +/root/bin/cli_stock 1 3 +/root/bin/cli_slots 31 3 +/root/bin/cli_slots 16 1 +/root/bin/cli_slots 1 4 +/root/bin/cli_stock 16 2 +/root/bin/cli_slots 8 1 +/root/bin/cli_stock 31 1 +/root/bin/cli_slots 31 1 +/root/bin/cli_slots 1 1 +/root/bin/cli_slots 16 2 +/root/bin/cli_slots 1 2 +/root/bin/cli_stock 16 1 +/root/bin/cli_slots 4 1 +/root/bin/cli_slots 4 4 +/root/bin/cli_slots 4 3 +/root/bin/cli_stock 1 4 +/root/bin/cli_slots 1 3 +/root/bin/cli_stock 31 4 +/root/bin/cli_slots 4 2 +/root/bin/cli_stock 4 3 +/root/bin/cli_slots 16 3 +/root/bin/cli_slots 31 4 +/root/bin/cli_slots 16 4 +/root/bin/cli_stock 1 1 +/root/bin/cli_stock 4 1 +/root/bin/cli_slots 8 3 +/root/bin/cli_stock 31 3 +/root/bin/cli_stock 16 3 +/root/bin/cli_slots 8 4 +/root/bin/cli_stock 8 3 +/root/bin/cli_stock 4 2 +/root/bin/cli_stock 8 1 +/root/bin/cli_stock 1 2 +/root/bin/cli_stock 31 2 +/root/bin/cli_stock 8 2 +/root/bin/cli_stock 16 4 +/root/bin/cli_slots 8 2 +/root/bin/cli_stock 4 4 +/root/bin/cli_stock 8 4 diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/jobs_c16.txt b/scripts/profiling/table-parallelism-sweep/round2/data/jobs_c16.txt new file mode 100644 index 000000000..d320a5272 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/jobs_c16.txt @@ -0,0 +1,36 @@ +/root/bin/cli_stock 31 3 +/root/bin/cli_stock 6 3 +/root/bin/cli_stock 16 4 +/root/bin/cli_stock 4 3 +/root/bin/cli_stock 8 4 +/root/bin/cli_stock 4 1 +/root/bin/cli_stock 8 2 +/root/bin/cli_stock 4 2 +/root/bin/cli_stock 6 4 +/root/bin/cli_stock 16 1 +/root/bin/cli_stock 31 2 +/root/bin/cli_stock 8 1 +/root/bin/cli_stock 2 4 +/root/bin/cli_stock 12 1 +/root/bin/cli_stock 12 2 +/root/bin/cli_stock 16 3 +/root/bin/cli_stock 1 3 +/root/bin/cli_stock 6 1 +/root/bin/cli_stock 2 3 +/root/bin/cli_stock 2 1 +/root/bin/cli_stock 4 4 +/root/bin/cli_stock 2 2 +/root/bin/cli_stock 12 3 +/root/bin/cli_stock 21 2 +/root/bin/cli_stock 1 4 +/root/bin/cli_stock 21 3 +/root/bin/cli_stock 12 4 +/root/bin/cli_stock 31 1 +/root/bin/cli_stock 8 3 +/root/bin/cli_stock 21 1 +/root/bin/cli_stock 16 2 +/root/bin/cli_stock 1 2 +/root/bin/cli_stock 1 1 +/root/bin/cli_stock 6 2 +/root/bin/cli_stock 31 4 +/root/bin/cli_stock 21 4 diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/jobs_c32.txt b/scripts/profiling/table-parallelism-sweep/round2/data/jobs_c32.txt new file mode 100644 index 000000000..4bfa4953e --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/jobs_c32.txt @@ -0,0 +1,27 @@ +8 2 +6 1 +4 3 +8 3 +6 3 +2 2 +16 1 +2 3 +1 2 +21 1 +12 3 +12 1 +8 1 +12 2 +21 3 +1 1 +1 3 +4 1 +2 1 +31 3 +31 2 +6 2 +31 1 +16 3 +16 2 +21 2 +4 2 diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/jobs_c32b.txt b/scripts/profiling/table-parallelism-sweep/round2/data/jobs_c32b.txt new file mode 100644 index 000000000..08a7e4b3b --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/jobs_c32b.txt @@ -0,0 +1,36 @@ +21 5 +31 6 +12 3 +31 5 +16 3 +21 4 +16 4 +31 1 +12 4 +8 4 +16 6 +6 6 +31 4 +16 2 +16 1 +21 2 +21 1 +16 5 +6 4 +12 2 +31 2 +8 6 +12 6 +8 5 +6 2 +8 2 +21 6 +12 5 +31 3 +6 1 +21 3 +6 3 +12 1 +8 1 +6 5 +8 3 diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/jobs_c4.txt b/scripts/profiling/table-parallelism-sweep/round2/data/jobs_c4.txt new file mode 100644 index 000000000..a3e35a3cc --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/jobs_c4.txt @@ -0,0 +1,36 @@ +/root/bin/cli_stock 31 3 +/root/bin/cli_stock 2 2 +/root/bin/cli_stock 31 4 +/root/bin/cli_stock 21 3 +/root/bin/cli_stock 12 3 +/root/bin/cli_stock 16 1 +/root/bin/cli_stock 4 4 +/root/bin/cli_stock 31 2 +/root/bin/cli_stock 2 4 +/root/bin/cli_stock 8 1 +/root/bin/cli_stock 12 4 +/root/bin/cli_stock 4 1 +/root/bin/cli_stock 6 3 +/root/bin/cli_stock 21 2 +/root/bin/cli_stock 21 4 +/root/bin/cli_stock 2 3 +/root/bin/cli_stock 1 4 +/root/bin/cli_stock 8 2 +/root/bin/cli_stock 1 3 +/root/bin/cli_stock 2 1 +/root/bin/cli_stock 12 2 +/root/bin/cli_stock 16 3 +/root/bin/cli_stock 6 1 +/root/bin/cli_stock 31 1 +/root/bin/cli_stock 21 1 +/root/bin/cli_stock 8 3 +/root/bin/cli_stock 16 4 +/root/bin/cli_stock 8 4 +/root/bin/cli_stock 4 3 +/root/bin/cli_stock 12 1 +/root/bin/cli_stock 4 2 +/root/bin/cli_stock 16 2 +/root/bin/cli_stock 6 2 +/root/bin/cli_stock 6 4 +/root/bin/cli_stock 1 1 +/root/bin/cli_stock 1 2 diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/jobs_c4b.txt b/scripts/profiling/table-parallelism-sweep/round2/data/jobs_c4b.txt new file mode 100644 index 000000000..4e0ebb98f --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/jobs_c4b.txt @@ -0,0 +1,40 @@ +/root/bin/cli_stock 16 7 +/root/bin/cli_stock 8 6 +/root/bin/cli_stock 16 2 +/root/bin/cli_stock 12 8 +/root/bin/cli_stock 12 6 +/root/bin/cli_stock 16 3 +/root/bin/cli_stock 21 1 +/root/bin/cli_stock 12 2 +/root/bin/cli_stock 8 3 +/root/bin/cli_stock 8 5 +/root/bin/cli_stock 12 7 +/root/bin/cli_stock 16 4 +/root/bin/cli_stock 16 8 +/root/bin/cli_stock 8 7 +/root/bin/cli_stock 21 8 +/root/bin/cli_stock 31 8 +/root/bin/cli_stock 31 5 +/root/bin/cli_stock 31 4 +/root/bin/cli_stock 31 6 +/root/bin/cli_stock 8 2 +/root/bin/cli_stock 8 4 +/root/bin/cli_stock 12 1 +/root/bin/cli_stock 21 3 +/root/bin/cli_stock 12 5 +/root/bin/cli_stock 16 1 +/root/bin/cli_stock 21 5 +/root/bin/cli_stock 12 4 +/root/bin/cli_stock 8 1 +/root/bin/cli_stock 21 7 +/root/bin/cli_stock 8 8 +/root/bin/cli_stock 12 3 +/root/bin/cli_stock 31 1 +/root/bin/cli_stock 31 2 +/root/bin/cli_stock 16 6 +/root/bin/cli_stock 21 6 +/root/bin/cli_stock 31 7 +/root/bin/cli_stock 21 4 +/root/bin/cli_stock 21 2 +/root/bin/cli_stock 16 5 +/root/bin/cli_stock 31 3 diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/jobs_c4c.txt b/scripts/profiling/table-parallelism-sweep/round2/data/jobs_c4c.txt new file mode 100644 index 000000000..8333cbef7 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/jobs_c4c.txt @@ -0,0 +1,32 @@ +/root/bin/cli_stock 4 4 +/root/bin/cli_stock 2 4 +/root/bin/cli_stock 4 5 +/root/bin/cli_stock 6 7 +/root/bin/cli_stock 6 6 +/root/bin/cli_stock 4 3 +/root/bin/cli_stock 1 1 +/root/bin/cli_stock 1 3 +/root/bin/cli_stock 1 7 +/root/bin/cli_stock 1 6 +/root/bin/cli_stock 1 8 +/root/bin/cli_stock 1 5 +/root/bin/cli_stock 1 4 +/root/bin/cli_stock 4 6 +/root/bin/cli_stock 6 4 +/root/bin/cli_stock 2 6 +/root/bin/cli_stock 6 5 +/root/bin/cli_stock 2 2 +/root/bin/cli_stock 2 7 +/root/bin/cli_stock 4 7 +/root/bin/cli_stock 4 2 +/root/bin/cli_stock 4 1 +/root/bin/cli_stock 4 8 +/root/bin/cli_stock 1 2 +/root/bin/cli_stock 6 2 +/root/bin/cli_stock 2 3 +/root/bin/cli_stock 6 3 +/root/bin/cli_stock 2 1 +/root/bin/cli_stock 2 8 +/root/bin/cli_stock 6 1 +/root/bin/cli_stock 2 5 +/root/bin/cli_stock 6 8 diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/jobs_c8.txt b/scripts/profiling/table-parallelism-sweep/round2/data/jobs_c8.txt new file mode 100644 index 000000000..3ef533ae4 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/jobs_c8.txt @@ -0,0 +1,36 @@ +/root/bin/cli_stock 12 3 +/root/bin/cli_stock 2 1 +/root/bin/cli_stock 16 2 +/root/bin/cli_stock 12 1 +/root/bin/cli_stock 16 4 +/root/bin/cli_stock 12 4 +/root/bin/cli_stock 21 1 +/root/bin/cli_stock 8 1 +/root/bin/cli_stock 2 2 +/root/bin/cli_stock 6 4 +/root/bin/cli_stock 4 1 +/root/bin/cli_stock 21 3 +/root/bin/cli_stock 4 4 +/root/bin/cli_stock 31 1 +/root/bin/cli_stock 16 1 +/root/bin/cli_stock 8 4 +/root/bin/cli_stock 6 1 +/root/bin/cli_stock 21 2 +/root/bin/cli_stock 1 2 +/root/bin/cli_stock 1 4 +/root/bin/cli_stock 1 1 +/root/bin/cli_stock 31 2 +/root/bin/cli_stock 16 3 +/root/bin/cli_stock 8 3 +/root/bin/cli_stock 2 3 +/root/bin/cli_stock 6 2 +/root/bin/cli_stock 31 3 +/root/bin/cli_stock 31 4 +/root/bin/cli_stock 4 2 +/root/bin/cli_stock 4 3 +/root/bin/cli_stock 2 4 +/root/bin/cli_stock 12 2 +/root/bin/cli_stock 8 2 +/root/bin/cli_stock 1 3 +/root/bin/cli_stock 6 3 +/root/bin/cli_stock 21 4 diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/jobs_c8b.txt b/scripts/profiling/table-parallelism-sweep/round2/data/jobs_c8b.txt new file mode 100644 index 000000000..62fcb722a --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/jobs_c8b.txt @@ -0,0 +1,40 @@ +/root/bin/cli_stock 12 7 +/root/bin/cli_stock 21 6 +/root/bin/cli_stock 8 4 +/root/bin/cli_stock 21 8 +/root/bin/cli_stock 12 8 +/root/bin/cli_stock 31 3 +/root/bin/cli_stock 16 4 +/root/bin/cli_stock 12 2 +/root/bin/cli_stock 21 5 +/root/bin/cli_stock 16 2 +/root/bin/cli_stock 31 1 +/root/bin/cli_stock 21 7 +/root/bin/cli_stock 21 3 +/root/bin/cli_stock 16 3 +/root/bin/cli_stock 12 3 +/root/bin/cli_stock 8 3 +/root/bin/cli_stock 8 6 +/root/bin/cli_stock 31 4 +/root/bin/cli_stock 16 6 +/root/bin/cli_stock 31 7 +/root/bin/cli_stock 12 4 +/root/bin/cli_stock 8 7 +/root/bin/cli_stock 31 6 +/root/bin/cli_stock 31 5 +/root/bin/cli_stock 16 5 +/root/bin/cli_stock 31 2 +/root/bin/cli_stock 21 1 +/root/bin/cli_stock 8 8 +/root/bin/cli_stock 21 2 +/root/bin/cli_stock 12 6 +/root/bin/cli_stock 16 1 +/root/bin/cli_stock 16 8 +/root/bin/cli_stock 8 1 +/root/bin/cli_stock 31 8 +/root/bin/cli_stock 12 1 +/root/bin/cli_stock 8 2 +/root/bin/cli_stock 12 5 +/root/bin/cli_stock 16 7 +/root/bin/cli_stock 8 5 +/root/bin/cli_stock 21 4 diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/jobs_c8c.txt b/scripts/profiling/table-parallelism-sweep/round2/data/jobs_c8c.txt new file mode 100644 index 000000000..08383e5e6 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/jobs_c8c.txt @@ -0,0 +1,24 @@ +/root/bin/cli_stock 4 3 +/root/bin/cli_stock 4 6 +/root/bin/cli_stock 4 7 +/root/bin/cli_stock 6 4 +/root/bin/cli_stock 2 7 +/root/bin/cli_stock 6 5 +/root/bin/cli_stock 2 2 +/root/bin/cli_stock 2 4 +/root/bin/cli_stock 2 6 +/root/bin/cli_stock 6 3 +/root/bin/cli_stock 6 8 +/root/bin/cli_stock 6 6 +/root/bin/cli_stock 4 1 +/root/bin/cli_stock 6 1 +/root/bin/cli_stock 4 8 +/root/bin/cli_stock 4 4 +/root/bin/cli_stock 4 5 +/root/bin/cli_stock 6 7 +/root/bin/cli_stock 2 8 +/root/bin/cli_stock 4 2 +/root/bin/cli_stock 2 1 +/root/bin/cli_stock 2 3 +/root/bin/cli_stock 2 5 +/root/bin/cli_stock 6 2 diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/jobs_instr.txt b/scripts/profiling/table-parallelism-sweep/round2/data/jobs_instr.txt new file mode 100644 index 000000000..f46a87856 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/jobs_instr.txt @@ -0,0 +1,36 @@ +off 1 2 +off 16 2 +off 16 1 +off 2 1 +on 1 3 +off 4 2 +off 8 1 +on 2 2 +off 4 1 +on 16 1 +on 16 3 +off 16 3 +off 31 2 +off 1 3 +off 1 1 +off 8 2 +on 1 1 +off 31 1 +on 31 3 +on 31 2 +on 4 1 +on 1 2 +on 2 1 +off 8 3 +on 8 1 +on 31 1 +on 8 3 +off 31 3 +off 2 2 +off 4 3 +on 16 2 +on 4 3 +on 8 2 +on 2 3 +off 2 3 +on 4 2 diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/jobs_instr2.txt b/scripts/profiling/table-parallelism-sweep/round2/data/jobs_instr2.txt new file mode 100644 index 000000000..1db064f50 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/jobs_instr2.txt @@ -0,0 +1,18 @@ +off 1 3 +off 31 2 +on 8 2 +off 8 2 +on 8 3 +off 1 2 +on 31 1 +off 8 3 +on 1 1 +off 31 3 +on 1 3 +on 1 2 +on 8 1 +on 31 3 +off 31 1 +on 31 2 +off 1 1 +off 8 1 diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/jobs_old32.txt b/scripts/profiling/table-parallelism-sweep/round2/data/jobs_old32.txt new file mode 100644 index 000000000..06385dec9 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/jobs_old32.txt @@ -0,0 +1,30 @@ +31 2 +12 3 +3 1 +4 2 +16 2 +16 3 +8 3 +2 2 +4 1 +1 2 +21 3 +3 2 +6 2 +2 3 +4 3 +6 3 +21 1 +6 1 +1 1 +21 2 +12 2 +8 1 +3 3 +8 2 +31 3 +1 3 +31 1 +2 1 +16 1 +12 1 diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/jobs_rt.txt b/scripts/profiling/table-parallelism-sweep/round2/data/jobs_rt.txt new file mode 100644 index 000000000..802cde687 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/jobs_rt.txt @@ -0,0 +1,72 @@ +4 1 2 +8 16 2 +4 21 1 +32 12 3 +8 1 2 +4 21 3 +4 8 3 +4 4 3 +4 2 3 +32 21 3 +4 4 1 +32 31 2 +32 16 2 +4 2 1 +32 31 1 +8 21 1 +32 12 2 +8 4 3 +32 2 3 +4 31 3 +4 1 3 +32 1 1 +8 2 1 +32 4 1 +32 31 3 +8 21 2 +4 31 2 +8 31 2 +32 21 1 +32 8 1 +4 16 1 +4 12 3 +8 16 1 +32 16 3 +32 8 2 +4 12 1 +4 4 2 +4 16 3 +4 8 2 +4 8 1 +8 1 3 +8 8 3 +8 2 2 +8 12 1 +8 4 1 +32 4 3 +32 1 2 +32 16 1 +8 8 2 +4 2 2 +8 4 2 +4 21 2 +32 12 1 +32 1 3 +8 12 2 +32 2 2 +4 12 2 +8 1 1 +4 1 1 +8 31 3 +32 2 1 +32 21 2 +8 31 1 +32 4 2 +4 16 2 +8 21 3 +8 8 1 +4 31 1 +8 16 3 +8 12 3 +8 2 3 +32 8 3 diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/log_c32_k1.txt b/scripts/profiling/table-parallelism-sweep/round2/data/log_c32_k1.txt new file mode 100644 index 000000000..c16c2f5d0 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/log_c32_k1.txt @@ -0,0 +1,8 @@ +Reading ELF file... +Reading private input file... +Generating continuation proof (blowup=2, epoch_size_log2=21, epoch_size=2097152)... +Writing proof... +Proof written to "/tmp/p_c32.bin" +Epochs: 4 +Proving time: 15.426s +Peak heap: 9248 MB diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/log_c32_k12.txt b/scripts/profiling/table-parallelism-sweep/round2/data/log_c32_k12.txt new file mode 100644 index 000000000..da375f456 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/log_c32_k12.txt @@ -0,0 +1,8 @@ +Reading ELF file... +Reading private input file... +Generating continuation proof (blowup=2, epoch_size_log2=21, epoch_size=2097152)... +Writing proof... +Proof written to "/tmp/p_c32.bin" +Epochs: 4 +Proving time: 9.664s +Peak heap: 10768 MB diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/log_c32_k16.txt b/scripts/profiling/table-parallelism-sweep/round2/data/log_c32_k16.txt new file mode 100644 index 000000000..ad3d84dad --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/log_c32_k16.txt @@ -0,0 +1,8 @@ +Reading ELF file... +Reading private input file... +Generating continuation proof (blowup=2, epoch_size_log2=21, epoch_size=2097152)... +Writing proof... +Proof written to "/tmp/p_c32.bin" +Epochs: 4 +Proving time: 9.308s +Peak heap: 11317 MB diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/log_c32_k2.txt b/scripts/profiling/table-parallelism-sweep/round2/data/log_c32_k2.txt new file mode 100644 index 000000000..049b979c3 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/log_c32_k2.txt @@ -0,0 +1,8 @@ +Reading ELF file... +Reading private input file... +Generating continuation proof (blowup=2, epoch_size_log2=21, epoch_size=2097152)... +Writing proof... +Proof written to "/tmp/p_c32.bin" +Epochs: 4 +Proving time: 11.877s +Peak heap: 9459 MB diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/log_c32_k21.txt b/scripts/profiling/table-parallelism-sweep/round2/data/log_c32_k21.txt new file mode 100644 index 000000000..105ddfe5a --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/log_c32_k21.txt @@ -0,0 +1,8 @@ +Reading ELF file... +Reading private input file... +Generating continuation proof (blowup=2, epoch_size_log2=21, epoch_size=2097152)... +Writing proof... +Proof written to "/tmp/p_c32.bin" +Epochs: 4 +Proving time: 9.045s +Peak heap: 11334 MB diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/log_c32_k31.txt b/scripts/profiling/table-parallelism-sweep/round2/data/log_c32_k31.txt new file mode 100644 index 000000000..03d1b6d18 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/log_c32_k31.txt @@ -0,0 +1,8 @@ +Reading ELF file... +Reading private input file... +Generating continuation proof (blowup=2, epoch_size_log2=21, epoch_size=2097152)... +Writing proof... +Proof written to "/tmp/p_c32.bin" +Epochs: 4 +Proving time: 8.951s +Peak heap: 11010 MB diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/log_c32_k4.txt b/scripts/profiling/table-parallelism-sweep/round2/data/log_c32_k4.txt new file mode 100644 index 000000000..e7d48cf12 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/log_c32_k4.txt @@ -0,0 +1,8 @@ +Reading ELF file... +Reading private input file... +Generating continuation proof (blowup=2, epoch_size_log2=21, epoch_size=2097152)... +Writing proof... +Proof written to "/tmp/p_c32.bin" +Epochs: 4 +Proving time: 10.396s +Peak heap: 10028 MB diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/log_c32_k6.txt b/scripts/profiling/table-parallelism-sweep/round2/data/log_c32_k6.txt new file mode 100644 index 000000000..c73ae5ecc --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/log_c32_k6.txt @@ -0,0 +1,8 @@ +Reading ELF file... +Reading private input file... +Generating continuation proof (blowup=2, epoch_size_log2=21, epoch_size=2097152)... +Writing proof... +Proof written to "/tmp/p_c32.bin" +Epochs: 4 +Proving time: 9.553s +Peak heap: 10347 MB diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/log_c32_k8.txt b/scripts/profiling/table-parallelism-sweep/round2/data/log_c32_k8.txt new file mode 100644 index 000000000..f26981cb6 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/log_c32_k8.txt @@ -0,0 +1,8 @@ +Reading ELF file... +Reading private input file... +Generating continuation proof (blowup=2, epoch_size_log2=21, epoch_size=2097152)... +Writing proof... +Proof written to "/tmp/p_c32.bin" +Epochs: 4 +Proving time: 9.632s +Peak heap: 10566 MB diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/log_c32b_k12.txt b/scripts/profiling/table-parallelism-sweep/round2/data/log_c32b_k12.txt new file mode 100644 index 000000000..6af6d20d1 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/log_c32b_k12.txt @@ -0,0 +1,8 @@ +Reading ELF file... +Reading private input file... +Generating continuation proof (blowup=2, epoch_size_log2=21, epoch_size=2097152)... +Writing proof... +Proof written to "/tmp/p_c32b.bin" +Epochs: 4 +Proving time: 9.630s +Peak heap: 11257 MB diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/log_c32b_k16.txt b/scripts/profiling/table-parallelism-sweep/round2/data/log_c32b_k16.txt new file mode 100644 index 000000000..496bbc8e4 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/log_c32b_k16.txt @@ -0,0 +1,8 @@ +Reading ELF file... +Reading private input file... +Generating continuation proof (blowup=2, epoch_size_log2=21, epoch_size=2097152)... +Writing proof... +Proof written to "/tmp/p_c32b.bin" +Epochs: 4 +Proving time: 9.261s +Peak heap: 11360 MB diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/log_c32b_k21.txt b/scripts/profiling/table-parallelism-sweep/round2/data/log_c32b_k21.txt new file mode 100644 index 000000000..a508078f7 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/log_c32b_k21.txt @@ -0,0 +1,8 @@ +Reading ELF file... +Reading private input file... +Generating continuation proof (blowup=2, epoch_size_log2=21, epoch_size=2097152)... +Writing proof... +Proof written to "/tmp/p_c32b.bin" +Epochs: 4 +Proving time: 9.586s +Peak heap: 11068 MB diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/log_c32b_k31.txt b/scripts/profiling/table-parallelism-sweep/round2/data/log_c32b_k31.txt new file mode 100644 index 000000000..aafbdff97 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/log_c32b_k31.txt @@ -0,0 +1,8 @@ +Reading ELF file... +Reading private input file... +Generating continuation proof (blowup=2, epoch_size_log2=21, epoch_size=2097152)... +Writing proof... +Proof written to "/tmp/p_c32b.bin" +Epochs: 4 +Proving time: 8.758s +Peak heap: 10798 MB diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/log_c32b_k6.txt b/scripts/profiling/table-parallelism-sweep/round2/data/log_c32b_k6.txt new file mode 100644 index 000000000..2d5a72724 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/log_c32b_k6.txt @@ -0,0 +1,8 @@ +Reading ELF file... +Reading private input file... +Generating continuation proof (blowup=2, epoch_size_log2=21, epoch_size=2097152)... +Writing proof... +Proof written to "/tmp/p_c32b.bin" +Epochs: 4 +Proving time: 10.075s +Peak heap: 10208 MB diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/log_c32b_k8.txt b/scripts/profiling/table-parallelism-sweep/round2/data/log_c32b_k8.txt new file mode 100644 index 000000000..35f5fb6a5 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/log_c32b_k8.txt @@ -0,0 +1,8 @@ +Reading ELF file... +Reading private input file... +Generating continuation proof (blowup=2, epoch_size_log2=21, epoch_size=2097152)... +Writing proof... +Proof written to "/tmp/p_c32b.bin" +Epochs: 4 +Proving time: 9.337s +Peak heap: 10928 MB diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/log_c4_cli_stock_k1.txt b/scripts/profiling/table-parallelism-sweep/round2/data/log_c4_cli_stock_k1.txt new file mode 100644 index 000000000..911d8eecf --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/log_c4_cli_stock_k1.txt @@ -0,0 +1,8 @@ +Reading ELF file... +Reading private input file... +Generating continuation proof (blowup=2, epoch_size_log2=21, epoch_size=2097152)... +Writing proof... +Proof written to "/tmp/p_c4.bin" +Epochs: 4 +Proving time: 25.098s +Peak heap: 9251 MB diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/log_c4_cli_stock_k12.txt b/scripts/profiling/table-parallelism-sweep/round2/data/log_c4_cli_stock_k12.txt new file mode 100644 index 000000000..fd68b3995 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/log_c4_cli_stock_k12.txt @@ -0,0 +1,8 @@ +Reading ELF file... +Reading private input file... +Generating continuation proof (blowup=2, epoch_size_log2=21, epoch_size=2097152)... +Writing proof... +Proof written to "/tmp/p_c4.bin" +Epochs: 4 +Proving time: 20.286s +Peak heap: 10703 MB diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/log_c4_cli_stock_k16.txt b/scripts/profiling/table-parallelism-sweep/round2/data/log_c4_cli_stock_k16.txt new file mode 100644 index 000000000..1167d84d3 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/log_c4_cli_stock_k16.txt @@ -0,0 +1,8 @@ +Reading ELF file... +Reading private input file... +Generating continuation proof (blowup=2, epoch_size_log2=21, epoch_size=2097152)... +Writing proof... +Proof written to "/tmp/p_c4.bin" +Epochs: 4 +Proving time: 18.791s +Peak heap: 10467 MB diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/log_c4_cli_stock_k2.txt b/scripts/profiling/table-parallelism-sweep/round2/data/log_c4_cli_stock_k2.txt new file mode 100644 index 000000000..1f5ee442f --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/log_c4_cli_stock_k2.txt @@ -0,0 +1,8 @@ +Reading ELF file... +Reading private input file... +Generating continuation proof (blowup=2, epoch_size_log2=21, epoch_size=2097152)... +Writing proof... +Proof written to "/tmp/p_c4.bin" +Epochs: 4 +Proving time: 23.348s +Peak heap: 9691 MB diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/log_c4_cli_stock_k21.txt b/scripts/profiling/table-parallelism-sweep/round2/data/log_c4_cli_stock_k21.txt new file mode 100644 index 000000000..7e6e64f98 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/log_c4_cli_stock_k21.txt @@ -0,0 +1,8 @@ +Reading ELF file... +Reading private input file... +Generating continuation proof (blowup=2, epoch_size_log2=21, epoch_size=2097152)... +Writing proof... +Proof written to "/tmp/p_c4.bin" +Epochs: 4 +Proving time: 19.517s +Peak heap: 11014 MB diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/log_c4_cli_stock_k31.txt b/scripts/profiling/table-parallelism-sweep/round2/data/log_c4_cli_stock_k31.txt new file mode 100644 index 000000000..993a0fd4b --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/log_c4_cli_stock_k31.txt @@ -0,0 +1,8 @@ +Reading ELF file... +Reading private input file... +Generating continuation proof (blowup=2, epoch_size_log2=21, epoch_size=2097152)... +Writing proof... +Proof written to "/tmp/p_c4.bin" +Epochs: 4 +Proving time: 21.334s +Peak heap: 10715 MB diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/log_c4_cli_stock_k4.txt b/scripts/profiling/table-parallelism-sweep/round2/data/log_c4_cli_stock_k4.txt new file mode 100644 index 000000000..a657dd740 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/log_c4_cli_stock_k4.txt @@ -0,0 +1,8 @@ +Reading ELF file... +Reading private input file... +Generating continuation proof (blowup=2, epoch_size_log2=21, epoch_size=2097152)... +Writing proof... +Proof written to "/tmp/p_c4.bin" +Epochs: 4 +Proving time: 21.437s +Peak heap: 10021 MB diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/log_c4_cli_stock_k6.txt b/scripts/profiling/table-parallelism-sweep/round2/data/log_c4_cli_stock_k6.txt new file mode 100644 index 000000000..d290847e5 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/log_c4_cli_stock_k6.txt @@ -0,0 +1,8 @@ +Reading ELF file... +Reading private input file... +Generating continuation proof (blowup=2, epoch_size_log2=21, epoch_size=2097152)... +Writing proof... +Proof written to "/tmp/p_c4.bin" +Epochs: 4 +Proving time: 21.413s +Peak heap: 10137 MB diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/log_c4_cli_stock_k8.txt b/scripts/profiling/table-parallelism-sweep/round2/data/log_c4_cli_stock_k8.txt new file mode 100644 index 000000000..fc0e71589 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/log_c4_cli_stock_k8.txt @@ -0,0 +1,8 @@ +Reading ELF file... +Reading private input file... +Generating continuation proof (blowup=2, epoch_size_log2=21, epoch_size=2097152)... +Writing proof... +Proof written to "/tmp/p_c4.bin" +Epochs: 4 +Proving time: 20.158s +Peak heap: 10200 MB diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/log_c4b_cli_stock_k12.txt b/scripts/profiling/table-parallelism-sweep/round2/data/log_c4b_cli_stock_k12.txt new file mode 100644 index 000000000..c9365deac --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/log_c4b_cli_stock_k12.txt @@ -0,0 +1,8 @@ +Reading ELF file... +Reading private input file... +Generating continuation proof (blowup=2, epoch_size_log2=21, epoch_size=2097152)... +Writing proof... +Proof written to "/tmp/p_c4b.bin" +Epochs: 4 +Proving time: 19.987s +Peak heap: 10326 MB diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/log_c4b_cli_stock_k16.txt b/scripts/profiling/table-parallelism-sweep/round2/data/log_c4b_cli_stock_k16.txt new file mode 100644 index 000000000..385d38784 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/log_c4b_cli_stock_k16.txt @@ -0,0 +1,8 @@ +Reading ELF file... +Reading private input file... +Generating continuation proof (blowup=2, epoch_size_log2=21, epoch_size=2097152)... +Writing proof... +Proof written to "/tmp/p_c4b.bin" +Epochs: 4 +Proving time: 19.650s +Peak heap: 10758 MB diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/log_c4b_cli_stock_k21.txt b/scripts/profiling/table-parallelism-sweep/round2/data/log_c4b_cli_stock_k21.txt new file mode 100644 index 000000000..1c05587c4 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/log_c4b_cli_stock_k21.txt @@ -0,0 +1,8 @@ +Reading ELF file... +Reading private input file... +Generating continuation proof (blowup=2, epoch_size_log2=21, epoch_size=2097152)... +Writing proof... +Proof written to "/tmp/p_c4b.bin" +Epochs: 4 +Proving time: 20.728s +Peak heap: 11210 MB diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/log_c4b_cli_stock_k31.txt b/scripts/profiling/table-parallelism-sweep/round2/data/log_c4b_cli_stock_k31.txt new file mode 100644 index 000000000..7578f2557 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/log_c4b_cli_stock_k31.txt @@ -0,0 +1,8 @@ +Reading ELF file... +Reading private input file... +Generating continuation proof (blowup=2, epoch_size_log2=21, epoch_size=2097152)... +Writing proof... +Proof written to "/tmp/p_c4b.bin" +Epochs: 4 +Proving time: 20.009s +Peak heap: 10761 MB diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/log_c4b_cli_stock_k8.txt b/scripts/profiling/table-parallelism-sweep/round2/data/log_c4b_cli_stock_k8.txt new file mode 100644 index 000000000..d98493785 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/log_c4b_cli_stock_k8.txt @@ -0,0 +1,8 @@ +Reading ELF file... +Reading private input file... +Generating continuation proof (blowup=2, epoch_size_log2=21, epoch_size=2097152)... +Writing proof... +Proof written to "/tmp/p_c4b.bin" +Epochs: 4 +Proving time: 20.898s +Peak heap: 10637 MB diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/log_c4c_cli_stock_k1.txt b/scripts/profiling/table-parallelism-sweep/round2/data/log_c4c_cli_stock_k1.txt new file mode 100644 index 000000000..df320066b --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/log_c4c_cli_stock_k1.txt @@ -0,0 +1,8 @@ +Reading ELF file... +Reading private input file... +Generating continuation proof (blowup=2, epoch_size_log2=21, epoch_size=2097152)... +Writing proof... +Proof written to "/tmp/p_c4c.bin" +Epochs: 4 +Proving time: 23.807s +Peak heap: 9251 MB diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/log_c4c_cli_stock_k2.txt b/scripts/profiling/table-parallelism-sweep/round2/data/log_c4c_cli_stock_k2.txt new file mode 100644 index 000000000..dadf98222 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/log_c4c_cli_stock_k2.txt @@ -0,0 +1,8 @@ +Reading ELF file... +Reading private input file... +Generating continuation proof (blowup=2, epoch_size_log2=21, epoch_size=2097152)... +Writing proof... +Proof written to "/tmp/p_c4c.bin" +Epochs: 4 +Proving time: 21.051s +Peak heap: 9830 MB diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/log_c4c_cli_stock_k4.txt b/scripts/profiling/table-parallelism-sweep/round2/data/log_c4c_cli_stock_k4.txt new file mode 100644 index 000000000..1af8285d1 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/log_c4c_cli_stock_k4.txt @@ -0,0 +1,8 @@ +Reading ELF file... +Reading private input file... +Generating continuation proof (blowup=2, epoch_size_log2=21, epoch_size=2097152)... +Writing proof... +Proof written to "/tmp/p_c4c.bin" +Epochs: 4 +Proving time: 21.318s +Peak heap: 9963 MB diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/log_c4c_cli_stock_k6.txt b/scripts/profiling/table-parallelism-sweep/round2/data/log_c4c_cli_stock_k6.txt new file mode 100644 index 000000000..21acec6ae --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/log_c4c_cli_stock_k6.txt @@ -0,0 +1,8 @@ +Reading ELF file... +Reading private input file... +Generating continuation proof (blowup=2, epoch_size_log2=21, epoch_size=2097152)... +Writing proof... +Proof written to "/tmp/p_c4c.bin" +Epochs: 4 +Proving time: 20.107s +Peak heap: 10316 MB diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/log_c8_cli_stock_k1.txt b/scripts/profiling/table-parallelism-sweep/round2/data/log_c8_cli_stock_k1.txt new file mode 100644 index 000000000..0bc14c6a6 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/log_c8_cli_stock_k1.txt @@ -0,0 +1,8 @@ +Reading ELF file... +Reading private input file... +Generating continuation proof (blowup=2, epoch_size_log2=21, epoch_size=2097152)... +Writing proof... +Proof written to "/tmp/p_c8.bin" +Epochs: 4 +Proving time: 19.079s +Peak heap: 9253 MB diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/log_c8_cli_stock_k12.txt b/scripts/profiling/table-parallelism-sweep/round2/data/log_c8_cli_stock_k12.txt new file mode 100644 index 000000000..d2cd77c09 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/log_c8_cli_stock_k12.txt @@ -0,0 +1,8 @@ +Reading ELF file... +Reading private input file... +Generating continuation proof (blowup=2, epoch_size_log2=21, epoch_size=2097152)... +Writing proof... +Proof written to "/tmp/p_c8.bin" +Epochs: 4 +Proving time: 13.194s +Peak heap: 10683 MB diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/log_c8_cli_stock_k16.txt b/scripts/profiling/table-parallelism-sweep/round2/data/log_c8_cli_stock_k16.txt new file mode 100644 index 000000000..dc0081273 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/log_c8_cli_stock_k16.txt @@ -0,0 +1,8 @@ +Reading ELF file... +Reading private input file... +Generating continuation proof (blowup=2, epoch_size_log2=21, epoch_size=2097152)... +Writing proof... +Proof written to "/tmp/p_c8.bin" +Epochs: 4 +Proving time: 12.880s +Peak heap: 10383 MB diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/log_c8_cli_stock_k2.txt b/scripts/profiling/table-parallelism-sweep/round2/data/log_c8_cli_stock_k2.txt new file mode 100644 index 000000000..7f11a687f --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/log_c8_cli_stock_k2.txt @@ -0,0 +1,8 @@ +Reading ELF file... +Reading private input file... +Generating continuation proof (blowup=2, epoch_size_log2=21, epoch_size=2097152)... +Writing proof... +Proof written to "/tmp/p_c8.bin" +Epochs: 4 +Proving time: 15.990s +Peak heap: 9739 MB diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/log_c8_cli_stock_k21.txt b/scripts/profiling/table-parallelism-sweep/round2/data/log_c8_cli_stock_k21.txt new file mode 100644 index 000000000..6439484bb --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/log_c8_cli_stock_k21.txt @@ -0,0 +1,8 @@ +Reading ELF file... +Reading private input file... +Generating continuation proof (blowup=2, epoch_size_log2=21, epoch_size=2097152)... +Writing proof... +Proof written to "/tmp/p_c8.bin" +Epochs: 4 +Proving time: 12.058s +Peak heap: 11373 MB diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/log_c8_cli_stock_k31.txt b/scripts/profiling/table-parallelism-sweep/round2/data/log_c8_cli_stock_k31.txt new file mode 100644 index 000000000..a3ec43cad --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/log_c8_cli_stock_k31.txt @@ -0,0 +1,8 @@ +Reading ELF file... +Reading private input file... +Generating continuation proof (blowup=2, epoch_size_log2=21, epoch_size=2097152)... +Writing proof... +Proof written to "/tmp/p_c8.bin" +Epochs: 4 +Proving time: 12.695s +Peak heap: 11337 MB diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/log_c8_cli_stock_k4.txt b/scripts/profiling/table-parallelism-sweep/round2/data/log_c8_cli_stock_k4.txt new file mode 100644 index 000000000..a58a443e1 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/log_c8_cli_stock_k4.txt @@ -0,0 +1,8 @@ +Reading ELF file... +Reading private input file... +Generating continuation proof (blowup=2, epoch_size_log2=21, epoch_size=2097152)... +Writing proof... +Proof written to "/tmp/p_c8.bin" +Epochs: 4 +Proving time: 13.491s +Peak heap: 9987 MB diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/log_c8_cli_stock_k6.txt b/scripts/profiling/table-parallelism-sweep/round2/data/log_c8_cli_stock_k6.txt new file mode 100644 index 000000000..9ec25ca07 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/log_c8_cli_stock_k6.txt @@ -0,0 +1,8 @@ +Reading ELF file... +Reading private input file... +Generating continuation proof (blowup=2, epoch_size_log2=21, epoch_size=2097152)... +Writing proof... +Proof written to "/tmp/p_c8.bin" +Epochs: 4 +Proving time: 13.349s +Peak heap: 10153 MB diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/log_c8_cli_stock_k8.txt b/scripts/profiling/table-parallelism-sweep/round2/data/log_c8_cli_stock_k8.txt new file mode 100644 index 000000000..f740010e9 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/log_c8_cli_stock_k8.txt @@ -0,0 +1,8 @@ +Reading ELF file... +Reading private input file... +Generating continuation proof (blowup=2, epoch_size_log2=21, epoch_size=2097152)... +Writing proof... +Proof written to "/tmp/p_c8.bin" +Epochs: 4 +Proving time: 13.062s +Peak heap: 10265 MB diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/log_c8b_cli_stock_k12.txt b/scripts/profiling/table-parallelism-sweep/round2/data/log_c8b_cli_stock_k12.txt new file mode 100644 index 000000000..1f5bca3a7 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/log_c8b_cli_stock_k12.txt @@ -0,0 +1,8 @@ +Reading ELF file... +Reading private input file... +Generating continuation proof (blowup=2, epoch_size_log2=21, epoch_size=2097152)... +Writing proof... +Proof written to "/tmp/p_c8b.bin" +Epochs: 4 +Proving time: 12.619s +Peak heap: 11322 MB diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/log_c8b_cli_stock_k16.txt b/scripts/profiling/table-parallelism-sweep/round2/data/log_c8b_cli_stock_k16.txt new file mode 100644 index 000000000..07106777e --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/log_c8b_cli_stock_k16.txt @@ -0,0 +1,8 @@ +Reading ELF file... +Reading private input file... +Generating continuation proof (blowup=2, epoch_size_log2=21, epoch_size=2097152)... +Writing proof... +Proof written to "/tmp/p_c8b.bin" +Epochs: 4 +Proving time: 12.247s +Peak heap: 11253 MB diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/log_c8b_cli_stock_k21.txt b/scripts/profiling/table-parallelism-sweep/round2/data/log_c8b_cli_stock_k21.txt new file mode 100644 index 000000000..92814d928 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/log_c8b_cli_stock_k21.txt @@ -0,0 +1,8 @@ +Reading ELF file... +Reading private input file... +Generating continuation proof (blowup=2, epoch_size_log2=21, epoch_size=2097152)... +Writing proof... +Proof written to "/tmp/p_c8b.bin" +Epochs: 4 +Proving time: 12.104s +Peak heap: 11585 MB diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/log_c8b_cli_stock_k31.txt b/scripts/profiling/table-parallelism-sweep/round2/data/log_c8b_cli_stock_k31.txt new file mode 100644 index 000000000..de55d731a --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/log_c8b_cli_stock_k31.txt @@ -0,0 +1,8 @@ +Reading ELF file... +Reading private input file... +Generating continuation proof (blowup=2, epoch_size_log2=21, epoch_size=2097152)... +Writing proof... +Proof written to "/tmp/p_c8b.bin" +Epochs: 4 +Proving time: 12.464s +Peak heap: 11441 MB diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/log_c8b_cli_stock_k8.txt b/scripts/profiling/table-parallelism-sweep/round2/data/log_c8b_cli_stock_k8.txt new file mode 100644 index 000000000..e0b52c37c --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/log_c8b_cli_stock_k8.txt @@ -0,0 +1,8 @@ +Reading ELF file... +Reading private input file... +Generating continuation proof (blowup=2, epoch_size_log2=21, epoch_size=2097152)... +Writing proof... +Proof written to "/tmp/p_c8b.bin" +Epochs: 4 +Proving time: 12.323s +Peak heap: 10093 MB diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/log_c8c_cli_stock_k2.txt b/scripts/profiling/table-parallelism-sweep/round2/data/log_c8c_cli_stock_k2.txt new file mode 100644 index 000000000..d2472c6e0 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/log_c8c_cli_stock_k2.txt @@ -0,0 +1,8 @@ +Reading ELF file... +Reading private input file... +Generating continuation proof (blowup=2, epoch_size_log2=21, epoch_size=2097152)... +Writing proof... +Proof written to "/tmp/p_c8c.bin" +Epochs: 4 +Proving time: 15.058s +Peak heap: 9707 MB diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/log_c8c_cli_stock_k4.txt b/scripts/profiling/table-parallelism-sweep/round2/data/log_c8c_cli_stock_k4.txt new file mode 100644 index 000000000..34591a7f1 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/log_c8c_cli_stock_k4.txt @@ -0,0 +1,8 @@ +Reading ELF file... +Reading private input file... +Generating continuation proof (blowup=2, epoch_size_log2=21, epoch_size=2097152)... +Writing proof... +Proof written to "/tmp/p_c8c.bin" +Epochs: 4 +Proving time: 14.634s +Peak heap: 10122 MB diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/log_c8c_cli_stock_k6.txt b/scripts/profiling/table-parallelism-sweep/round2/data/log_c8c_cli_stock_k6.txt new file mode 100644 index 000000000..16311c84f --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/log_c8c_cli_stock_k6.txt @@ -0,0 +1,8 @@ +Reading ELF file... +Reading private input file... +Generating continuation proof (blowup=2, epoch_size_log2=21, epoch_size=2097152)... +Writing proof... +Proof written to "/tmp/p_c8c.bin" +Epochs: 4 +Proving time: 12.992s +Peak heap: 10490 MB diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/patch_instr.diff b/scripts/profiling/table-parallelism-sweep/round2/data/patch_instr.diff new file mode 100644 index 000000000..620f9c645 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/patch_instr.diff @@ -0,0 +1,110 @@ +diff --git a/crypto/math-cuda/src/device.rs b/crypto/math-cuda/src/device.rs +index a7c129c..239c33b 100644 +--- a/crypto/math-cuda/src/device.rs ++++ b/crypto/math-cuda/src/device.rs +@@ -15,6 +15,67 @@ use math::field::traits::IsFFTField; + use crate::Result; + use crate::ntt::{twiddles_forward, twiddles_inverse}; + ++// ===== k-sweep-877b Stage 1a/1b instrumentation (NOT for merge) ===== ++// gdb/perf are unavailable in this container (no cap_sys_ptrace / cap_perfmon), ++// so measure the pinned-staging mutex directly. Wait time is the contention ++// metric: it is exactly what the planned stack sampling would have estimated. ++pub static SW_WAIT_DRV: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); ++pub static SW_WAIT_RAY: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); ++pub static SW_N_DRV: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); ++pub static SW_N_RAY: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); ++pub static SW_MAX_DRV_NS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); ++ ++/// Per-driver staging slots as a run-time switch, so one binary A/Bs the Stage 1b ++/// intervention with no build difference between arms. ++fn per_driver_slots() -> bool { ++ static ON: OnceLock = OnceLock::new(); ++ *ON.get_or_init(|| std::env::var_os("LAMBDA_VM_PER_DRIVER_SLOTS").is_some()) ++} ++ ++/// Periodic stderr dump of the counters; the last line printed before exit is ++/// the run's tally. A daemon thread is the simplest reporter that needs no new ++/// dependency and no hook in the cli crate. ++fn staging_reporter() { ++ static ONCE: std::sync::Once = std::sync::Once::new(); ++ ONCE.call_once(|| { ++ if std::env::var_os("LAMBDA_VM_STAGING_STATS").is_none() { ++ return; ++ } ++ std::thread::spawn(|| { ++ loop { ++ std::thread::sleep(std::time::Duration::from_millis(150)); ++ eprintln!( ++ "staging-stats: drv_wait_ms={:.1} drv_n={} drv_max_us={} ray_wait_ms={:.1} ray_n={} per_driver_slots={}", ++ SW_WAIT_DRV.load(Ordering::Relaxed) as f64 / 1e6, ++ SW_N_DRV.load(Ordering::Relaxed), ++ SW_MAX_DRV_NS.load(Ordering::Relaxed) / 1000, ++ SW_WAIT_RAY.load(Ordering::Relaxed) as f64 / 1e6, ++ SW_N_RAY.load(Ordering::Relaxed), ++ per_driver_slots() as u8, ++ ); ++ } ++ }); ++ }); ++} ++ ++fn lock_staging(slot: &Mutex) -> std::sync::MutexGuard<'_, PinnedStaging> { ++ staging_reporter(); ++ let drv = rayon::current_thread_index().is_none(); ++ let t0 = std::time::Instant::now(); ++ let g = slot.lock().unwrap(); ++ let ns = t0.elapsed().as_nanos() as u64; ++ if drv { ++ SW_WAIT_DRV.fetch_add(ns, Ordering::Relaxed); ++ SW_N_DRV.fetch_add(1, Ordering::Relaxed); ++ SW_MAX_DRV_NS.fetch_max(ns, Ordering::Relaxed); ++ } else { ++ SW_WAIT_RAY.fetch_add(ns, Ordering::Relaxed); ++ SW_N_RAY.fetch_add(1, Ordering::Relaxed); ++ } ++ g ++} ++// ===== end instrumentation ===== ++ + /// Reusable pinned host staging buffer. Shared across all streams via a + /// `Mutex` (see `Backend::pinned_staging`); the LDE call holds the lock + /// across the D2H + memcpy-to-user-Vecs window. +@@ -516,7 +577,18 @@ impl Backend { + /// the shared mutex does — the staged transfers are already hidden by + /// cross-table overlap. + fn worker_slot(&self, len: usize) -> usize { +- let idx = rayon::current_thread_index().unwrap_or(0); ++ let idx = match rayon::current_thread_index() { ++ Some(i) => i, ++ // k-sweep-877b: run-time-switchable per-driver slots. ++ None if per_driver_slots() => { ++ static NEXT_SLOT: AtomicUsize = AtomicUsize::new(0); ++ thread_local! { ++ static MY_SLOT: usize = NEXT_SLOT.fetch_add(1, Ordering::Relaxed); ++ } ++ MY_SLOT.with(|s| *s) % len.max(1) ++ } ++ None => 0, ++ }; + // Should be unreachable with rayon's fixed default pool, but if a + // larger custom pool sneaks in we still want safety: Fall back to + // slot 0 (correctness preserved, just contention). +@@ -676,7 +748,7 @@ pub fn htod_via( + // Chunk in whole elements so a `T` never straddles a chunk boundary. + let chunk_elems = (HTOD_CHUNK_BYTES / elem_size.max(1)).max(1); + +- let mut staging = slot.lock().unwrap(); ++ let mut staging = lock_staging(slot); + // Only ask for a chunk's worth of pinned memory (or the whole copy when + // smaller). If another path (`async_dtoh_via` on the host-retaining flow) + // already grew this slot larger, it stays larger — grow-only — and we just +@@ -760,7 +832,7 @@ pub fn async_dtoh_via<'a, T: cudarc::driver::DeviceRepr>( + assert!(n_elems <= src.len()); + let n_bytes = n_elems * std::mem::size_of::(); + let u64_len = n_bytes.div_ceil(8); +- let mut staging = slot.lock().unwrap(); ++ let mut staging = lock_staging(slot); + staging.ensure_capacity(u64_len, ctx)?; + ctx.bind_to_thread()?; + // SAFETY: dst is this slot's pinned allocation — stable address (only diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/patch_instr2.diff b/scripts/profiling/table-parallelism-sweep/round2/data/patch_instr2.diff new file mode 100644 index 000000000..b4b993551 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/patch_instr2.diff @@ -0,0 +1,145 @@ +diff --git a/crypto/math-cuda/src/device.rs b/crypto/math-cuda/src/device.rs +index a7c129c..8e95a2b 100644 +--- a/crypto/math-cuda/src/device.rs ++++ b/crypto/math-cuda/src/device.rs +@@ -15,6 +15,79 @@ use math::field::traits::IsFFTField; + use crate::Result; + use crate::ntt::{twiddles_forward, twiddles_inverse}; + ++// ===== k-sweep-877b Stage 1a/1b instrumentation (NOT for merge) ===== ++// gdb/perf are unavailable in this container (no cap_sys_ptrace / cap_perfmon), ++// so measure the pinned-staging mutex directly. Wait time is the contention ++// metric: it is exactly what the planned stack sampling would have estimated. ++pub static SW_WAIT_DRV: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); ++pub static SW_WAIT_RAY: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); ++pub static SW_N_DRV: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); ++pub static SW_N_RAY: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); ++pub static SW_MAX_DRV_NS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); ++ ++/// Per-driver staging slots as a run-time switch, so one binary A/Bs the Stage 1b ++/// intervention with no build difference between arms. ++fn per_driver_slots() -> bool { ++ static ON: OnceLock = OnceLock::new(); ++ *ON.get_or_init(|| std::env::var_os("LAMBDA_VM_PER_DRIVER_SLOTS").is_some()) ++} ++ ++/// Periodic stderr dump of the counters; the last line printed before exit is ++/// the run's tally. A daemon thread is the simplest reporter that needs no new ++/// dependency and no hook in the cli crate. ++fn staging_reporter() { ++ static ONCE: std::sync::Once = std::sync::Once::new(); ++ ONCE.call_once(|| { ++ if std::env::var_os("LAMBDA_VM_STAGING_STATS").is_none() { ++ return; ++ } ++ std::thread::spawn(|| { ++ loop { ++ std::thread::sleep(std::time::Duration::from_millis(150)); ++ eprintln!( ++ "staging-stats: drv_wait_ms={:.1} drv_n={} drv_max_us={} ray_wait_ms={:.1} ray_n={} hold_ms={:.1} per_driver_slots={}", ++ SW_WAIT_DRV.load(Ordering::Relaxed) as f64 / 1e6, ++ SW_N_DRV.load(Ordering::Relaxed), ++ SW_MAX_DRV_NS.load(Ordering::Relaxed) / 1000, ++ SW_WAIT_RAY.load(Ordering::Relaxed) as f64 / 1e6, ++ SW_N_RAY.load(Ordering::Relaxed), ++ SW_HOLD_NS.load(Ordering::Relaxed) as f64 / 1e6, ++ per_driver_slots() as u8, ++ ); ++ } ++ }); ++ }); ++} ++ ++fn lock_staging(slot: &Mutex) -> std::sync::MutexGuard<'_, PinnedStaging> { ++ staging_reporter(); ++ let drv = rayon::current_thread_index().is_none(); ++ let t0 = std::time::Instant::now(); ++ let g = slot.lock().unwrap(); ++ let ns = t0.elapsed().as_nanos() as u64; ++ if drv { ++ SW_WAIT_DRV.fetch_add(ns, Ordering::Relaxed); ++ SW_N_DRV.fetch_add(1, Ordering::Relaxed); ++ SW_MAX_DRV_NS.fetch_max(ns, Ordering::Relaxed); ++ } else { ++ SW_WAIT_RAY.fetch_add(ns, Ordering::Relaxed); ++ SW_N_RAY.fetch_add(1, Ordering::Relaxed); ++ } ++ g ++} ++pub static SW_HOLD_NS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); ++ ++/// Accumulates the time a staging slot is held. Total held time bounds what any ++/// staging-path fix could recover: a critical section that is occupied H seconds ++/// cannot be costing more than H seconds of serialization. ++pub struct HoldTimer(pub std::time::Instant); ++impl Drop for HoldTimer { ++ fn drop(&mut self) { ++ SW_HOLD_NS.fetch_add(self.0.elapsed().as_nanos() as u64, Ordering::Relaxed); ++ } ++} ++// ===== end instrumentation ===== ++ + /// Reusable pinned host staging buffer. Shared across all streams via a + /// `Mutex` (see `Backend::pinned_staging`); the LDE call holds the lock + /// across the D2H + memcpy-to-user-Vecs window. +@@ -516,7 +589,18 @@ impl Backend { + /// the shared mutex does — the staged transfers are already hidden by + /// cross-table overlap. + fn worker_slot(&self, len: usize) -> usize { +- let idx = rayon::current_thread_index().unwrap_or(0); ++ let idx = match rayon::current_thread_index() { ++ Some(i) => i, ++ // k-sweep-877b: run-time-switchable per-driver slots. ++ None if per_driver_slots() => { ++ static NEXT_SLOT: AtomicUsize = AtomicUsize::new(0); ++ thread_local! { ++ static MY_SLOT: usize = NEXT_SLOT.fetch_add(1, Ordering::Relaxed); ++ } ++ MY_SLOT.with(|s| *s) % len.max(1) ++ } ++ None => 0, ++ }; + // Should be unreachable with rayon's fixed default pool, but if a + // larger custom pool sneaks in we still want safety: Fall back to + // slot 0 (correctness preserved, just contention). +@@ -621,6 +705,7 @@ pub fn backend() -> Result<&'static Backend> { + /// `htod_via`/`async_dtoh_via` on the same slot from the thread holding a + /// live `PendingD2H` — the non-reentrant slot mutex self-deadlocks. + pub struct PendingD2H<'a> { ++ hold: HoldTimer, + staging: std::sync::MutexGuard<'a, PinnedStaging>, + n_bytes: usize, + } +@@ -676,7 +761,8 @@ pub fn htod_via( + // Chunk in whole elements so a `T` never straddles a chunk boundary. + let chunk_elems = (HTOD_CHUNK_BYTES / elem_size.max(1)).max(1); + +- let mut staging = slot.lock().unwrap(); ++ let mut staging = lock_staging(slot); ++ let _hold = HoldTimer(std::time::Instant::now()); + // Only ask for a chunk's worth of pinned memory (or the whole copy when + // smaller). If another path (`async_dtoh_via` on the host-retaining flow) + // already grew this slot larger, it stays larger — grow-only — and we just +@@ -760,7 +846,8 @@ pub fn async_dtoh_via<'a, T: cudarc::driver::DeviceRepr>( + assert!(n_elems <= src.len()); + let n_bytes = n_elems * std::mem::size_of::(); + let u64_len = n_bytes.div_ceil(8); +- let mut staging = slot.lock().unwrap(); ++ let mut staging = lock_staging(slot); ++ let _hold_start = std::time::Instant::now(); + staging.ensure_capacity(u64_len, ctx)?; + ctx.bind_to_thread()?; + // SAFETY: dst is this slot's pinned allocation — stable address (only +@@ -785,7 +872,11 @@ pub fn async_dtoh_via<'a, T: cudarc::driver::DeviceRepr>( + let _ = stream.synchronize(); + return Err(e); + } +- Ok(PendingD2H { staging, n_bytes }) ++ Ok(PendingD2H { ++ hold: HoldTimer(_hold_start), ++ staging, ++ n_bytes, ++ }) + } + + /// Best-effort stream drain on error paths: while `armed`, dropping this guard diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/patch_slots.diff b/scripts/profiling/table-parallelism-sweep/round2/data/patch_slots.diff new file mode 100644 index 000000000..a1e119fa7 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/patch_slots.diff @@ -0,0 +1,37 @@ +diff --git a/crypto/math-cuda/src/device.rs b/crypto/math-cuda/src/device.rs +index a7c129c..4a2143f 100644 +--- a/crypto/math-cuda/src/device.rs ++++ b/crypto/math-cuda/src/device.rs +@@ -516,7 +516,31 @@ impl Backend { + /// the shared mutex does — the staged transfers are already hidden by + /// cross-table overlap. + fn worker_slot(&self, len: usize) -> usize { +- let idx = rayon::current_thread_index().unwrap_or(0); ++ let idx = match rayon::current_thread_index() { ++ Some(i) => i, ++ // k-sweep-877b Stage 1b EXPERIMENT (not for merge). Non-rayon ++ // callers are the per-table scheduler's K driver threads; stock ++ // code funnels them all onto slot 0, so they share one pinned slab ++ // behind one mutex. Hand each non-rayon thread its own stable slot ++ // instead. The rayon arm and the clamp below are untouched. ++ None => { ++ static NEXT_SLOT: std::sync::atomic::AtomicUsize = ++ std::sync::atomic::AtomicUsize::new(0); ++ thread_local! { ++ static MY_SLOT: usize = { ++ let s = NEXT_SLOT.fetch_add(1, std::sync::atomic::Ordering::Relaxed); ++ if std::env::var_os("LAMBDA_VM_SLOT_DEBUG").is_some() { ++ eprintln!( ++ "slot-debug: non-rayon thread {:?} -> raw slot {s}", ++ std::thread::current().id() ++ ); ++ } ++ s ++ }; ++ } ++ MY_SLOT.with(|s| *s) % len.max(1) ++ } ++ }; + // Should be unreachable with rayon's fixed default pool, but if a + // larger custom pool sneaks in we still want safety: Fall back to + // slot 0 (correctness preserved, just contention). diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/progress.log b/scripts/profiling/table-parallelism-sweep/round2/data/progress.log new file mode 100644 index 000000000..720a2b32e --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/progress.log @@ -0,0 +1,215 @@ +2026-08-04T18:13:29Z runall start (uptime 1165s) +2026-08-04T18:13:29Z build dbg (symbolized) +2026-08-04T18:13:50Z build slots (patched worker_slot) +2026-08-04T18:14:06Z build oldsched (1e1e0f18) +2026-08-04T18:14:22Z builds done +total 27676 +drwxrwxr-x 2 root root 60 Aug 4 18:14 . +drwx------ 1 root root 4096 Aug 4 18:14 .. +-rwxrwxr-x 1 root root 9596464 Aug 4 18:14 cli_oldsched +-rwxrwxr-x 1 root root 9377144 Aug 4 18:14 cli_slots +-rwxrwxr-x 1 root root 9374344 Aug 4 17:51 cli_stock +2026-08-04T18:14:22Z stage4 c8 (taskset -c 0-7) +2026-08-04T18:23:16Z stage4 c4 (taskset -c 0-3) +2026-08-04T18:36:47Z stage4 c16 (taskset -c 0-15) +2026-08-04T18:43:52Z stage4 done +2026-08-04T18:43:52Z stage1a sampling K=1 +2026-08-04T18:44:09Z stage1a sampling K=4 +2026-08-04T18:44:20Z stage1a sampling K=8 +2026-08-04T18:44:30Z stage1a sampling K=16 +2026-08-04T18:44:40Z stage1a sampling K=31 +2026-08-04T18:44:50Z stage1a done +2026-08-04T18:44:50Z stage1b slot proof +2026-08-04T18:47:01Z runall start (uptime 37s) +2026-08-04T18:47:01Z builds done +total 27676 +drwxrwxr-x 2 root root 60 Aug 4 18:14 . +drwx------ 1 root root 4096 Aug 4 18:14 .. +-rwxrwxr-x 1 root root 9596464 Aug 4 18:14 cli_oldsched +-rwxrwxr-x 1 root root 9377144 Aug 4 18:14 cli_slots +-rwxrwxr-x 1 root root 9374344 Aug 4 17:51 cli_stock +2026-08-04T18:47:01Z stage4 c8 already done +2026-08-04T18:47:01Z stage4 c4 already done +2026-08-04T18:47:01Z stage4 c16 already done +2026-08-04T18:47:01Z stage4 done +2026-08-04T18:47:01Z stage1a K=1 has 22 samples, skip +2026-08-04T18:47:01Z stage1a K=4 has 21 samples, skip +2026-08-04T18:47:01Z stage1a sampling K=8 +2026-08-04T18:47:11Z stage1a sampling K=16 +2026-08-04T18:47:22Z stage1a sampling K=31 +2026-08-04T18:47:31Z stage1a done +2026-08-04T18:47:31Z stage1b slot proof +2026-08-04T18:47:53Z stage1b A/B at 32 cores +2026-08-04T18:52:52Z runall start (uptime 388s) +2026-08-04T18:52:52Z builds done +total 27676 +drwxrwxr-x 2 root root 60 Aug 4 18:14 . +drwx------ 1 root root 4096 Aug 4 18:51 .. +-rwxrwxr-x 1 root root 9596464 Aug 4 18:14 cli_oldsched +-rwxrwxr-x 1 root root 9377144 Aug 4 18:14 cli_slots +-rwxrwxr-x 1 root root 9374344 Aug 4 17:51 cli_stock +2026-08-04T18:52:52Z stage4 c8 already done +2026-08-04T18:52:52Z stage4 c4 already done +2026-08-04T18:52:52Z stage4 c16 already done +2026-08-04T18:52:52Z stage4 done +2026-08-04T18:52:52Z stage1a K=1 has 22 samples, skip +2026-08-04T18:52:52Z stage1a K=4 has 21 samples, skip +2026-08-04T18:52:52Z stage1a K=8 has 18 samples, skip +2026-08-04T18:52:52Z stage1a K=16 has 18 samples, skip +2026-08-04T18:52:52Z stage1a sampling K=31 +2026-08-04T18:53:01Z stage1a done +2026-08-04T18:53:01Z stage1b A/B at 32 cores +2026-08-04T18:53:16Z runall2 start (uptime 413s) +2026-08-04T18:53:16Z 1b A/B 32 cores +2026-08-04T19:02:59Z build instr +2026-08-04T19:03:16Z 1a-substitute staging-mutex measurement +2026-08-04T19:07:48Z runall3 start (uptime 1284s) +2026-08-04T19:07:48Z 1a-substitute staging-mutex wait measurement +2026-08-04T19:11:15Z build instr2 (hold time) +2026-08-04T19:11:31Z staging hold-time measurement +2026-08-04T19:15:35Z 1c single-epoch A/B (epoch 2^23) +2026-08-04T19:18:25Z stage2 rayon decomposition +2026-08-04T19:36:33Z 1b A/B 8 cores +2026-08-04T19:46:49Z stage3 old scheduler K curve +2026-08-04T19:53:38Z ALL_DONE +2026-08-04T19:54:01Z runall3 start (uptime 4057s) +2026-08-04T19:54:01Z ALL_DONE +2026-08-04T19:54:46Z stage6 start (uptime 4103s) +2026-08-04T19:54:46Z stage6 c4b +2026-08-04T20:08:55Z stage6 c8b +2026-08-04T20:20:01Z stage6 start (uptime 65s) +2026-08-04T20:20:01Z stage6 c4b done +2026-08-04T20:20:01Z stage6 c8b +2026-08-04T20:20:29Z STAGE6_ALL_DONE +2026-08-04T20:21:01Z stage6 start (uptime 125s) +2026-08-04T20:21:01Z stage6 c4b done +2026-08-04T20:21:01Z stage6 c8b done +2026-08-04T20:21:01Z STAGE6_ALL_DONE +2026-08-04T20:22:01Z stage6 start (uptime 185s) +2026-08-04T20:22:01Z stage6 c4b done +2026-08-04T20:22:01Z stage6 c8b done +2026-08-04T20:22:01Z STAGE6_ALL_DONE +2026-08-04T20:23:01Z stage6 start (uptime 245s) +2026-08-04T20:23:01Z stage6 c4b done +2026-08-04T20:23:01Z stage6 c8b done +2026-08-04T20:23:01Z STAGE6_ALL_DONE +2026-08-04T20:24:01Z stage6 start (uptime 305s) +2026-08-04T20:24:01Z stage6 c4b done +2026-08-04T20:24:01Z stage6 c8b done +2026-08-04T20:24:01Z STAGE6_ALL_DONE +2026-08-04T20:25:01Z stage6 start (uptime 365s) +2026-08-04T20:25:01Z stage6 c4b done +2026-08-04T20:25:01Z stage6 c8b done +2026-08-04T20:25:01Z STAGE6_ALL_DONE +2026-08-04T20:26:01Z stage6 start (uptime 425s) +2026-08-04T20:26:01Z stage6 c4b done +2026-08-04T20:26:01Z stage6 c8b done +2026-08-04T20:26:01Z STAGE6_ALL_DONE +2026-08-04T20:27:01Z stage6 start (uptime 485s) +2026-08-04T20:27:01Z stage6 c4b done +2026-08-04T20:27:01Z stage6 c8b done +2026-08-04T20:27:01Z STAGE6_ALL_DONE +2026-08-04T20:34:42Z stage7 start (uptime 946s) +2026-08-04T20:34:42Z stage7 c4c low-K +2026-08-04T20:47:04Z stage7 c8c low-K +2026-08-04T20:53:02Z STAGE7_ALL_DONE +2026-08-04T20:54:01Z stage7 start (uptime 2105s) +2026-08-04T20:54:01Z STAGE7_ALL_DONE +2026-08-04T20:55:01Z stage7 start (uptime 2165s) +2026-08-04T20:55:01Z STAGE7_ALL_DONE +2026-08-04T20:56:01Z stage7 start (uptime 2225s) +2026-08-04T20:56:01Z STAGE7_ALL_DONE +2026-08-04T20:57:01Z stage7 start (uptime 2285s) +2026-08-04T20:57:01Z STAGE7_ALL_DONE +2026-08-04T20:58:01Z stage7 start (uptime 2345s) +2026-08-04T20:58:01Z STAGE7_ALL_DONE +2026-08-04T20:59:01Z stage7 start (uptime 2405s) +2026-08-04T20:59:01Z STAGE7_ALL_DONE +2026-08-04T21:00:01Z stage7 start (uptime 2465s) +2026-08-04T21:00:01Z STAGE7_ALL_DONE +2026-08-04T21:01:02Z stage7 start (uptime 2526s) +2026-08-04T21:01:02Z STAGE7_ALL_DONE +2026-08-04T21:02:01Z stage7 start (uptime 2585s) +2026-08-04T21:02:01Z STAGE7_ALL_DONE +2026-08-04T21:03:01Z stage7 start (uptime 2645s) +2026-08-04T21:03:01Z STAGE7_ALL_DONE +2026-08-04T21:04:01Z stage7 start (uptime 2705s) +2026-08-04T21:04:01Z STAGE7_ALL_DONE +2026-08-04T21:05:01Z stage7 start (uptime 2765s) +2026-08-04T21:05:01Z STAGE7_ALL_DONE +2026-08-04T21:06:01Z stage7 start (uptime 2825s) +2026-08-04T21:06:01Z STAGE7_ALL_DONE +2026-08-04T21:07:01Z stage7 start (uptime 2885s) +2026-08-04T21:07:01Z STAGE7_ALL_DONE +2026-08-04T21:08:01Z stage7 start (uptime 2945s) +2026-08-04T21:08:01Z STAGE7_ALL_DONE +2026-08-04T21:09:01Z stage7 start (uptime 3005s) +2026-08-04T21:09:01Z STAGE7_ALL_DONE +2026-08-04T21:10:01Z stage7 start (uptime 3065s) +2026-08-04T21:10:01Z STAGE7_ALL_DONE +2026-08-04T21:11:01Z stage7 start (uptime 3125s) +2026-08-04T21:11:01Z STAGE7_ALL_DONE +2026-08-04T21:12:01Z stage7 start (uptime 3185s) +2026-08-04T21:12:01Z STAGE7_ALL_DONE +2026-08-04T21:13:01Z stage7 start (uptime 3245s) +2026-08-04T21:13:01Z STAGE7_ALL_DONE +2026-08-04T21:14:01Z stage7 start (uptime 3305s) +2026-08-04T21:14:01Z STAGE7_ALL_DONE +2026-08-04T21:15:01Z stage7 start (uptime 3365s) +2026-08-04T21:15:01Z STAGE7_ALL_DONE +2026-08-04T21:16:01Z stage7 start (uptime 3425s) +2026-08-04T21:16:01Z STAGE7_ALL_DONE +2026-08-04T21:17:01Z stage7 start (uptime 3485s) +2026-08-04T21:17:01Z STAGE7_ALL_DONE +2026-08-04T21:18:01Z stage7 start (uptime 3545s) +2026-08-04T21:18:01Z STAGE7_ALL_DONE +2026-08-04T21:19:01Z stage7 start (uptime 3605s) +2026-08-04T21:19:01Z STAGE7_ALL_DONE +2026-08-04T21:20:01Z stage7 start (uptime 3665s) +2026-08-04T21:20:01Z STAGE7_ALL_DONE +2026-08-04T21:21:01Z stage7 start (uptime 3725s) +2026-08-04T21:21:01Z STAGE7_ALL_DONE +2026-08-04T21:22:01Z stage7 start (uptime 3785s) +2026-08-04T21:22:01Z STAGE7_ALL_DONE +2026-08-04T21:23:01Z stage7 start (uptime 3845s) +2026-08-04T21:23:01Z STAGE7_ALL_DONE +2026-08-04T21:24:02Z stage7 start (uptime 3906s) +2026-08-04T21:24:02Z STAGE7_ALL_DONE +2026-08-04T21:25:01Z stage7 start (uptime 3965s) +2026-08-04T21:25:01Z STAGE7_ALL_DONE +2026-08-04T21:26:01Z stage7 start (uptime 4025s) +2026-08-04T21:26:01Z STAGE7_ALL_DONE +2026-08-04T21:27:01Z stage7 start (uptime 4085s) +2026-08-04T21:27:01Z STAGE7_ALL_DONE +2026-08-04T21:28:01Z stage7 start (uptime 4145s) +2026-08-04T21:28:01Z STAGE7_ALL_DONE +2026-08-04T21:29:01Z stage7 start (uptime 4205s) +2026-08-04T21:29:01Z STAGE7_ALL_DONE +2026-08-04T21:30:01Z stage7 start (uptime 4265s) +2026-08-04T21:30:01Z STAGE7_ALL_DONE +2026-08-04T21:31:01Z stage7 start (uptime 4325s) +2026-08-04T21:31:01Z STAGE7_ALL_DONE +2026-08-04T21:32:01Z stage7 start (uptime 4385s) +2026-08-04T21:32:01Z STAGE7_ALL_DONE +2026-08-04T21:33:01Z stage7 start (uptime 4445s) +2026-08-04T21:33:01Z STAGE7_ALL_DONE +2026-08-04T21:34:01Z stage7 start (uptime 4505s) +2026-08-04T21:34:01Z STAGE7_ALL_DONE +2026-08-04T21:35:01Z stage7 start (uptime 4565s) +2026-08-04T21:35:01Z STAGE7_ALL_DONE +2026-08-04T21:36:01Z stage7 start (uptime 4625s) +2026-08-04T21:36:01Z STAGE7_ALL_DONE +2026-08-04T21:37:01Z stage7 start (uptime 4685s) +2026-08-04T21:37:01Z STAGE7_ALL_DONE +2026-08-04T21:38:01Z stage7 start (uptime 4745s) +2026-08-04T21:38:01Z STAGE7_ALL_DONE +2026-08-04T21:39:01Z stage7 start (uptime 4805s) +2026-08-04T21:39:01Z STAGE7_ALL_DONE +2026-08-04T21:40:01Z stage7 start (uptime 4865s) +2026-08-04T21:40:01Z STAGE7_ALL_DONE +2026-08-04T21:41:01Z stage7 start (uptime 4925s) +2026-08-04T21:41:01Z STAGE7_ALL_DONE +2026-08-04T21:42:01Z stage7 start (uptime 4985s) +2026-08-04T21:42:01Z STAGE7_ALL_DONE +2026-08-04T21:43:01Z stage7 start (uptime 5045s) +2026-08-04T21:43:01Z STAGE7_ALL_DONE diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/results_ab1ep.csv b/scripts/profiling/table-parallelism-sweep/round2/data/results_ab1ep.csv new file mode 100644 index 000000000..abf166c8f --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/results_ab1ep.csv @@ -0,0 +1,21 @@ +tag,cli,cpuspec,epoch,k,rep,seconds,n_slots +ab1ep,cli_stock,none,23,8,4,7.568,32 +ab1ep,cli_stock,none,23,21,1,7.169,32 +ab1ep,cli_stock,none,23,21,5,7.357,32 +ab1ep,cli_stock,none,23,8,1,7.462,32 +ab1ep,cli_stock,none,23,8,2,7.436,32 +ab1ep,cli_slots,none,23,21,3,7.769,32 +ab1ep,cli_stock,none,23,8,5,7.369,32 +ab1ep,cli_stock,none,23,8,3,7.285,32 +ab1ep,cli_slots,none,23,8,3,7.380,32 +ab1ep,cli_stock,none,23,21,2,7.217,32 +ab1ep,cli_slots,none,23,21,4,7.658,32 +ab1ep,cli_stock,none,23,21,4,7.335,32 +ab1ep,cli_stock,none,23,21,3,7.141,32 +ab1ep,cli_slots,none,23,8,4,7.693,32 +ab1ep,cli_slots,none,23,8,2,7.659,32 +ab1ep,cli_slots,none,23,21,1,7.712,32 +ab1ep,cli_slots,none,23,8,1,7.470,32 +ab1ep,cli_slots,none,23,21,5,7.831,32 +ab1ep,cli_slots,none,23,8,5,7.512,32 +ab1ep,cli_slots,none,23,21,2,7.701,32 diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/results_ab32.csv b/scripts/profiling/table-parallelism-sweep/round2/data/results_ab32.csv new file mode 100644 index 000000000..15edae27d --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/results_ab32.csv @@ -0,0 +1,73 @@ +tag,cli,cpuspec,epoch,k,rep,seconds,n_slots +ab32,cli_stock,none,21,4,2,9.895,32 +ab32,cli_slots,none,21,31,2,9.916,32 +ab32,cli_stock,none,21,8,1,9.436,32 +ab32,cli_slots,none,21,16,2,10.282,32 +ab32,cli_stock,none,21,4,5,10.170,32 +ab32,cli_stock,none,21,31,1,8.752,32 +ab32,cli_stock,none,21,16,3,9.536,32 +ab32,cli_slots,none,21,1,5,16.674,32 +ab32,cli_slots,none,21,4,6,11.009,32 +ab32,cli_stock,none,21,1,2,16.070,32 +ab32,cli_stock,none,21,8,3,9.510,32 +ab32,cli_slots,none,21,8,4,10.467,32 +ab32,cli_slots,none,21,21,2,10.079,32 +ab32,cli_slots,none,21,16,5,10.499,32 +ab32,cli_slots,none,21,8,6,10.221,32 +ab32,cli_stock,none,21,21,3,9.493,32 +ab32,cli_slots,none,21,1,2,17.220,32 +ab32,cli_slots,none,21,4,3,10.836,32 +ab32,cli_stock,none,21,1,5,15.874,32 +ab32,cli_slots,none,21,21,4,10.000,32 +ab32,cli_stock,none,21,31,5,9.022,32 +ab32,cli_slots,none,21,4,4,11.486,32 +ab32,cli_slots,none,21,31,3,10.132,32 +ab32,cli_stock,none,21,31,6,8.811,32 +ab32,cli_slots,none,21,8,3,10.827,32 +ab32,cli_slots,none,21,21,1,10.270,32 +ab32,cli_stock,none,21,8,5,9.641,32 +ab32,cli_slots,none,21,31,1,10.348,32 +ab32,cli_stock,none,21,4,3,10.232,32 +ab32,cli_stock,none,21,1,6,15.742,32 +ab32,cli_slots,none,21,4,2,10.936,32 +ab32,cli_stock,none,21,21,2,9.218,32 +ab32,cli_stock,none,21,21,4,9.472,32 +ab32,cli_slots,none,21,21,6,10.710,32 +ab32,cli_slots,none,21,4,1,10.843,32 +ab32,cli_slots,none,21,21,5,9.988,32 +ab32,cli_slots,none,21,8,5,10.612,32 +ab32,cli_slots,none,21,31,4,9.686,32 +ab32,cli_stock,none,21,16,5,9.438,32 +ab32,cli_stock,none,21,16,6,9.198,32 +ab32,cli_slots,none,21,16,1,10.625,32 +ab32,cli_stock,none,21,1,3,16.012,32 +ab32,cli_stock,none,21,8,2,9.873,32 +ab32,cli_stock,none,21,8,6,9.413,32 +ab32,cli_stock,none,21,4,1,10.212,32 +ab32,cli_stock,none,21,16,1,9.499,32 +ab32,cli_slots,none,21,8,1,10.450,32 +ab32,cli_stock,none,21,31,3,9.028,32 +ab32,cli_stock,none,21,1,1,16.200,32 +ab32,cli_stock,none,21,16,4,9.524,32 +ab32,cli_stock,none,21,31,2,8.981,32 +ab32,cli_slots,none,21,1,6,17.047,32 +ab32,cli_slots,none,21,4,5,11.282,32 +ab32,cli_slots,none,21,1,3,16.920,32 +ab32,cli_slots,none,21,31,6,10.159,32 +ab32,cli_slots,none,21,8,2,10.638,32 +ab32,cli_stock,none,21,16,2,9.279,32 +ab32,cli_slots,none,21,21,3,10.441,32 +ab32,cli_slots,none,21,16,3,10.301,32 +ab32,cli_stock,none,21,21,5,9.280,32 +ab32,cli_slots,none,21,16,4,9.939,32 +ab32,cli_stock,none,21,1,4,15.912,32 +ab32,cli_stock,none,21,8,4,9.653,32 +ab32,cli_slots,none,21,1,1,16.890,32 +ab32,cli_slots,none,21,1,4,16.988,32 +ab32,cli_stock,none,21,21,6,9.251,32 +ab32,cli_stock,none,21,4,6,10.372,32 +ab32,cli_stock,none,21,31,4,8.981,32 +ab32,cli_stock,none,21,21,1,9.345,32 +ab32,cli_slots,none,21,31,5,10.042,32 +ab32,cli_stock,none,21,4,4,10.749,32 +ab32,cli_slots,none,21,16,6,10.190,32 diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/results_ab8.csv b/scripts/profiling/table-parallelism-sweep/round2/data/results_ab8.csv new file mode 100644 index 000000000..d14b519b3 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/results_ab8.csv @@ -0,0 +1,41 @@ +tag,cli,cpuspec,epoch,k,rep,seconds,n_slots +ab8,cli_slots,0-7,21,31,2,12.997,8 +ab8,cli_stock,0-7,21,1,3,18.949,8 +ab8,cli_slots,0-7,21,31,3,13.308,8 +ab8,cli_slots,0-7,21,16,1,12.687,8 +ab8,cli_slots,0-7,21,1,4,20.402,8 +ab8,cli_stock,0-7,21,16,2,12.761,8 +ab8,cli_slots,0-7,21,8,1,13.038,8 +ab8,cli_stock,0-7,21,31,1,12.699,8 +ab8,cli_slots,0-7,21,31,1,12.534,8 +ab8,cli_slots,0-7,21,1,1,19.619,8 +ab8,cli_slots,0-7,21,16,2,13.246,8 +ab8,cli_slots,0-7,21,1,2,19.965,8 +ab8,cli_stock,0-7,21,16,1,12.791,8 +ab8,cli_slots,0-7,21,4,1,13.688,8 +ab8,cli_slots,0-7,21,4,4,13.716,8 +ab8,cli_slots,0-7,21,4,3,13.412,8 +ab8,cli_stock,0-7,21,1,4,19.427,8 +ab8,cli_slots,0-7,21,1,3,19.987,8 +ab8,cli_stock,0-7,21,31,4,12.123,8 +ab8,cli_slots,0-7,21,4,2,14.011,8 +ab8,cli_stock,0-7,21,4,3,13.797,8 +ab8,cli_slots,0-7,21,16,3,13.314,8 +ab8,cli_slots,0-7,21,31,4,13.141,8 +ab8,cli_slots,0-7,21,16,4,13.041,8 +ab8,cli_stock,0-7,21,1,1,18.424,8 +ab8,cli_stock,0-7,21,4,1,13.494,8 +ab8,cli_slots,0-7,21,8,3,13.622,8 +ab8,cli_stock,0-7,21,31,3,12.026,8 +ab8,cli_stock,0-7,21,16,3,12.724,8 +ab8,cli_slots,0-7,21,8,4,13.369,8 +ab8,cli_stock,0-7,21,8,3,13.085,8 +ab8,cli_stock,0-7,21,4,2,13.836,8 +ab8,cli_stock,0-7,21,8,1,12.478,8 +ab8,cli_stock,0-7,21,1,2,18.987,8 +ab8,cli_stock,0-7,21,31,2,12.280,8 +ab8,cli_stock,0-7,21,8,2,12.841,8 +ab8,cli_stock,0-7,21,16,4,12.796,8 +ab8,cli_slots,0-7,21,8,2,13.129,8 +ab8,cli_stock,0-7,21,4,4,14.131,8 +ab8,cli_stock,0-7,21,8,4,12.307,8 diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/results_c16.csv b/scripts/profiling/table-parallelism-sweep/round2/data/results_c16.csv new file mode 100644 index 000000000..cc0de2469 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/results_c16.csv @@ -0,0 +1,37 @@ +tag,cli,cpuspec,epoch,k,rep,seconds,n_slots +c16,cli_stock,0-15,21,31,3,9.557,16 +c16,cli_stock,0-15,21,6,3,10.124,16 +c16,cli_stock,0-15,21,16,4,9.980,16 +c16,cli_stock,0-15,21,4,3,10.738,16 +c16,cli_stock,0-15,21,8,4,10.066,16 +c16,cli_stock,0-15,21,4,1,10.769,16 +c16,cli_stock,0-15,21,8,2,9.758,16 +c16,cli_stock,0-15,21,4,2,10.702,16 +c16,cli_stock,0-15,21,6,4,10.425,16 +c16,cli_stock,0-15,21,16,1,9.332,16 +c16,cli_stock,0-15,21,31,2,9.087,16 +c16,cli_stock,0-15,21,8,1,9.716,16 +c16,cli_stock,0-15,21,2,4,12.694,16 +c16,cli_stock,0-15,21,12,1,9.542,16 +c16,cli_stock,0-15,21,12,2,9.675,16 +c16,cli_stock,0-15,21,16,3,9.567,16 +c16,cli_stock,0-15,21,1,3,17.129,16 +c16,cli_stock,0-15,21,6,1,10.113,16 +c16,cli_stock,0-15,21,2,3,12.334,16 +c16,cli_stock,0-15,21,2,1,12.351,16 +c16,cli_stock,0-15,21,4,4,10.735,16 +c16,cli_stock,0-15,21,2,2,12.567,16 +c16,cli_stock,0-15,21,12,3,9.671,16 +c16,cli_stock,0-15,21,21,2,9.620,16 +c16,cli_stock,0-15,21,1,4,17.467,16 +c16,cli_stock,0-15,21,21,3,9.618,16 +c16,cli_stock,0-15,21,12,4,9.616,16 +c16,cli_stock,0-15,21,31,1,9.062,16 +c16,cli_stock,0-15,21,8,3,10.184,16 +c16,cli_stock,0-15,21,21,1,9.538,16 +c16,cli_stock,0-15,21,16,2,9.475,16 +c16,cli_stock,0-15,21,1,2,16.929,16 +c16,cli_stock,0-15,21,1,1,16.637,16 +c16,cli_stock,0-15,21,6,2,10.477,16 +c16,cli_stock,0-15,21,31,4,9.161,16 +c16,cli_stock,0-15,21,21,4,9.980,16 diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/results_c32.csv b/scripts/profiling/table-parallelism-sweep/round2/data/results_c32.csv new file mode 100644 index 000000000..d540449ed --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/results_c32.csv @@ -0,0 +1,28 @@ +tag,cpuspec,epoch,k,rep,seconds,cli +c32,none,21,8,2,9.761,cli_stock +c32,none,21,6,1,9.553,cli_stock +c32,none,21,4,3,10.136,cli_stock +c32,none,21,8,3,9.501,cli_stock +c32,none,21,6,3,9.614,cli_stock +c32,none,21,2,2,11.888,cli_stock +c32,none,21,16,1,9.308,cli_stock +c32,none,21,2,3,11.988,cli_stock +c32,none,21,1,2,16.029,cli_stock +c32,none,21,21,1,9.045,cli_stock +c32,none,21,12,3,9.686,cli_stock +c32,none,21,12,1,9.664,cli_stock +c32,none,21,8,1,9.632,cli_stock +c32,none,21,12,2,9.553,cli_stock +c32,none,21,21,3,9.319,cli_stock +c32,none,21,1,1,15.426,cli_stock +c32,none,21,1,3,15.885,cli_stock +c32,none,21,4,1,10.396,cli_stock +c32,none,21,2,1,11.877,cli_stock +c32,none,21,31,3,9.399,cli_stock +c32,none,21,31,2,9.106,cli_stock +c32,none,21,6,2,9.731,cli_stock +c32,none,21,31,1,8.951,cli_stock +c32,none,21,16,3,9.263,cli_stock +c32,none,21,16,2,9.160,cli_stock +c32,none,21,21,2,8.970,cli_stock +c32,none,21,4,2,10.551,cli_stock diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/results_c32b.csv b/scripts/profiling/table-parallelism-sweep/round2/data/results_c32b.csv new file mode 100644 index 000000000..398b88495 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/results_c32b.csv @@ -0,0 +1,37 @@ +tag,cpuspec,epoch,k,rep,seconds,cli +c32b,none,21,21,5,9.256,cli_stock +c32b,none,21,31,6,9.022,cli_stock +c32b,none,21,12,3,9.290,cli_stock +c32b,none,21,31,5,8.804,cli_stock +c32b,none,21,16,3,9.528,cli_stock +c32b,none,21,21,4,9.290,cli_stock +c32b,none,21,16,4,9.504,cli_stock +c32b,none,21,31,1,8.758,cli_stock +c32b,none,21,12,4,9.651,cli_stock +c32b,none,21,8,4,9.459,cli_stock +c32b,none,21,16,6,9.494,cli_stock +c32b,none,21,6,6,9.814,cli_stock +c32b,none,21,31,4,9.057,cli_stock +c32b,none,21,16,2,9.434,cli_stock +c32b,none,21,16,1,9.261,cli_stock +c32b,none,21,21,2,9.128,cli_stock +c32b,none,21,21,1,9.586,cli_stock +c32b,none,21,16,5,9.326,cli_stock +c32b,none,21,6,4,9.435,cli_stock +c32b,none,21,12,2,9.656,cli_stock +c32b,none,21,31,2,9.027,cli_stock +c32b,none,21,8,6,9.572,cli_stock +c32b,none,21,12,6,9.886,cli_stock +c32b,none,21,8,5,9.807,cli_stock +c32b,none,21,6,2,9.975,cli_stock +c32b,none,21,8,2,9.639,cli_stock +c32b,none,21,21,6,9.322,cli_stock +c32b,none,21,12,5,9.838,cli_stock +c32b,none,21,31,3,9.157,cli_stock +c32b,none,21,6,1,10.075,cli_stock +c32b,none,21,21,3,9.336,cli_stock +c32b,none,21,6,3,9.904,cli_stock +c32b,none,21,12,1,9.630,cli_stock +c32b,none,21,8,1,9.337,cli_stock +c32b,none,21,6,5,9.880,cli_stock +c32b,none,21,8,3,9.700,cli_stock diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/results_c4.csv b/scripts/profiling/table-parallelism-sweep/round2/data/results_c4.csv new file mode 100644 index 000000000..be42add07 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/results_c4.csv @@ -0,0 +1,37 @@ +tag,cli,cpuspec,epoch,k,rep,seconds,n_slots +c4,cli_stock,0-3,21,31,3,20.293,4 +c4,cli_stock,0-3,21,2,2,22.585,4 +c4,cli_stock,0-3,21,31,4,21.406,4 +c4,cli_stock,0-3,21,21,3,21.034,4 +c4,cli_stock,0-3,21,12,3,20.186,4 +c4,cli_stock,0-3,21,16,1,18.791,4 +c4,cli_stock,0-3,21,4,4,20.181,4 +c4,cli_stock,0-3,21,31,2,20.611,4 +c4,cli_stock,0-3,21,2,4,20.971,4 +c4,cli_stock,0-3,21,8,1,20.158,4 +c4,cli_stock,0-3,21,12,4,20.309,4 +c4,cli_stock,0-3,21,4,1,21.437,4 +c4,cli_stock,0-3,21,6,3,21.066,4 +c4,cli_stock,0-3,21,21,2,19.942,4 +c4,cli_stock,0-3,21,21,4,21.391,4 +c4,cli_stock,0-3,21,2,3,23.120,4 +c4,cli_stock,0-3,21,1,4,24.740,4 +c4,cli_stock,0-3,21,8,2,20.515,4 +c4,cli_stock,0-3,21,1,3,25.994,4 +c4,cli_stock,0-3,21,2,1,23.348,4 +c4,cli_stock,0-3,21,12,2,19.964,4 +c4,cli_stock,0-3,21,16,3,21.374,4 +c4,cli_stock,0-3,21,6,1,21.413,4 +c4,cli_stock,0-3,21,31,1,21.334,4 +c4,cli_stock,0-3,21,21,1,19.517,4 +c4,cli_stock,0-3,21,8,3,20.613,4 +c4,cli_stock,0-3,21,16,4,20.408,4 +c4,cli_stock,0-3,21,8,4,20.823,4 +c4,cli_stock,0-3,21,4,3,21.350,4 +c4,cli_stock,0-3,21,12,1,20.286,4 +c4,cli_stock,0-3,21,4,2,21.855,4 +c4,cli_stock,0-3,21,16,2,21.127,4 +c4,cli_stock,0-3,21,6,2,21.186,4 +c4,cli_stock,0-3,21,6,4,20.307,4 +c4,cli_stock,0-3,21,1,1,25.098,4 +c4,cli_stock,0-3,21,1,2,23.839,4 diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/results_c4b.csv b/scripts/profiling/table-parallelism-sweep/round2/data/results_c4b.csv new file mode 100644 index 000000000..32c6e02fb --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/results_c4b.csv @@ -0,0 +1,41 @@ +tag,cli,cpuspec,epoch,k,rep,seconds,n_slots +c4b,cli_stock,0-3,21,16,7,19.936,4 +c4b,cli_stock,0-3,21,8,6,20.357,4 +c4b,cli_stock,0-3,21,16,2,20.201,4 +c4b,cli_stock,0-3,21,12,8,20.688,4 +c4b,cli_stock,0-3,21,12,6,20.258,4 +c4b,cli_stock,0-3,21,16,3,19.997,4 +c4b,cli_stock,0-3,21,21,1,20.728,4 +c4b,cli_stock,0-3,21,12,2,20.018,4 +c4b,cli_stock,0-3,21,8,3,19.154,4 +c4b,cli_stock,0-3,21,8,5,21.265,4 +c4b,cli_stock,0-3,21,12,7,19.470,4 +c4b,cli_stock,0-3,21,16,4,20.164,4 +c4b,cli_stock,0-3,21,16,8,20.964,4 +c4b,cli_stock,0-3,21,8,7,20.023,4 +c4b,cli_stock,0-3,21,21,8,20.393,4 +c4b,cli_stock,0-3,21,31,8,20.665,4 +c4b,cli_stock,0-3,21,31,5,19.819,4 +c4b,cli_stock,0-3,21,31,4,20.417,4 +c4b,cli_stock,0-3,21,31,6,18.893,4 +c4b,cli_stock,0-3,21,8,2,19.688,4 +c4b,cli_stock,0-3,21,8,4,19.429,4 +c4b,cli_stock,0-3,21,12,1,19.987,4 +c4b,cli_stock,0-3,21,21,3,20.300,4 +c4b,cli_stock,0-3,21,12,5,20.417,4 +c4b,cli_stock,0-3,21,16,1,19.650,4 +c4b,cli_stock,0-3,21,21,5,18.846,4 +c4b,cli_stock,0-3,21,12,4,19.947,4 +c4b,cli_stock,0-3,21,8,1,20.898,4 +c4b,cli_stock,0-3,21,21,7,18.576,4 +c4b,cli_stock,0-3,21,8,8,21.619,4 +c4b,cli_stock,0-3,21,12,3,21.194,4 +c4b,cli_stock,0-3,21,31,1,20.009,4 +c4b,cli_stock,0-3,21,31,2,20.398,4 +c4b,cli_stock,0-3,21,16,6,21.261,4 +c4b,cli_stock,0-3,21,21,6,20.229,4 +c4b,cli_stock,0-3,21,31,7,19.689,4 +c4b,cli_stock,0-3,21,21,4,19.686,4 +c4b,cli_stock,0-3,21,21,2,20.223,4 +c4b,cli_stock,0-3,21,16,5,20.684,4 +c4b,cli_stock,0-3,21,31,3,20.060,4 diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/results_c4c.csv b/scripts/profiling/table-parallelism-sweep/round2/data/results_c4c.csv new file mode 100644 index 000000000..85008909b --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/results_c4c.csv @@ -0,0 +1,33 @@ +tag,cli,cpuspec,epoch,k,rep,seconds,n_slots +c4c,cli_stock,0-3,21,4,4,20.602,4 +c4c,cli_stock,0-3,21,2,4,22.630,4 +c4c,cli_stock,0-3,21,4,5,20.675,4 +c4c,cli_stock,0-3,21,6,7,21.425,4 +c4c,cli_stock,0-3,21,6,6,20.124,4 +c4c,cli_stock,0-3,21,4,3,20.999,4 +c4c,cli_stock,0-3,21,1,1,23.807,4 +c4c,cli_stock,0-3,21,1,3,25.484,4 +c4c,cli_stock,0-3,21,1,7,24.357,4 +c4c,cli_stock,0-3,21,1,6,22.723,4 +c4c,cli_stock,0-3,21,1,8,25.054,4 +c4c,cli_stock,0-3,21,1,5,24.099,4 +c4c,cli_stock,0-3,21,1,4,24.749,4 +c4c,cli_stock,0-3,21,4,6,21.316,4 +c4c,cli_stock,0-3,21,6,4,22.145,4 +c4c,cli_stock,0-3,21,2,6,21.502,4 +c4c,cli_stock,0-3,21,6,5,21.182,4 +c4c,cli_stock,0-3,21,2,2,20.616,4 +c4c,cli_stock,0-3,21,2,7,21.366,4 +c4c,cli_stock,0-3,21,4,7,20.680,4 +c4c,cli_stock,0-3,21,4,2,21.147,4 +c4c,cli_stock,0-3,21,4,1,21.318,4 +c4c,cli_stock,0-3,21,4,8,21.264,4 +c4c,cli_stock,0-3,21,1,2,24.265,4 +c4c,cli_stock,0-3,21,6,2,20.185,4 +c4c,cli_stock,0-3,21,2,3,21.846,4 +c4c,cli_stock,0-3,21,6,3,21.621,4 +c4c,cli_stock,0-3,21,2,1,21.051,4 +c4c,cli_stock,0-3,21,2,8,21.504,4 +c4c,cli_stock,0-3,21,6,1,20.107,4 +c4c,cli_stock,0-3,21,2,5,22.515,4 +c4c,cli_stock,0-3,21,6,8,20.855,4 diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/results_c8.csv b/scripts/profiling/table-parallelism-sweep/round2/data/results_c8.csv new file mode 100644 index 000000000..a76cdb88b --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/results_c8.csv @@ -0,0 +1,37 @@ +tag,cli,cpuspec,epoch,k,rep,seconds,n_slots +c8,cli_stock,0-7,21,12,3,12.709,8 +c8,cli_stock,0-7,21,2,1,15.990,8 +c8,cli_stock,0-7,21,16,2,11.955,8 +c8,cli_stock,0-7,21,12,1,13.194,8 +c8,cli_stock,0-7,21,16,4,12.855,8 +c8,cli_stock,0-7,21,12,4,12.622,8 +c8,cli_stock,0-7,21,21,1,12.058,8 +c8,cli_stock,0-7,21,8,1,13.062,8 +c8,cli_stock,0-7,21,2,2,14.428,8 +c8,cli_stock,0-7,21,6,4,12.833,8 +c8,cli_stock,0-7,21,4,1,13.491,8 +c8,cli_stock,0-7,21,21,3,13.085,8 +c8,cli_stock,0-7,21,4,4,14.006,8 +c8,cli_stock,0-7,21,31,1,12.695,8 +c8,cli_stock,0-7,21,16,1,12.880,8 +c8,cli_stock,0-7,21,8,4,12.964,8 +c8,cli_stock,0-7,21,6,1,13.349,8 +c8,cli_stock,0-7,21,21,2,12.640,8 +c8,cli_stock,0-7,21,1,2,19.403,8 +c8,cli_stock,0-7,21,1,4,18.290,8 +c8,cli_stock,0-7,21,1,1,19.079,8 +c8,cli_stock,0-7,21,31,2,12.961,8 +c8,cli_stock,0-7,21,16,3,13.419,8 +c8,cli_stock,0-7,21,8,3,12.954,8 +c8,cli_stock,0-7,21,2,3,15.294,8 +c8,cli_stock,0-7,21,6,2,13.504,8 +c8,cli_stock,0-7,21,31,3,12.261,8 +c8,cli_stock,0-7,21,31,4,13.004,8 +c8,cli_stock,0-7,21,4,2,14.232,8 +c8,cli_stock,0-7,21,4,3,13.736,8 +c8,cli_stock,0-7,21,2,4,14.333,8 +c8,cli_stock,0-7,21,12,2,12.589,8 +c8,cli_stock,0-7,21,8,2,13.082,8 +c8,cli_stock,0-7,21,1,3,18.978,8 +c8,cli_stock,0-7,21,6,3,13.710,8 +c8,cli_stock,0-7,21,21,4,12.088,8 diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/results_c8b.csv b/scripts/profiling/table-parallelism-sweep/round2/data/results_c8b.csv new file mode 100644 index 000000000..ab9d3078e --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/results_c8b.csv @@ -0,0 +1,41 @@ +tag,cli,cpuspec,epoch,k,rep,seconds,n_slots +c8b,cli_stock,0-7,21,12,7,12.231,8 +c8b,cli_stock,0-7,21,21,6,12.362,8 +c8b,cli_stock,0-7,21,8,4,13.049,8 +c8b,cli_stock,0-7,21,21,8,12.477,8 +c8b,cli_stock,0-7,21,12,8,13.194,8 +c8b,cli_stock,0-7,21,31,3,11.561,8 +c8b,cli_stock,0-7,21,16,4,12.726,8 +c8b,cli_stock,0-7,21,12,2,12.401,8 +c8b,cli_stock,0-7,21,21,5,13.170,8 +c8b,cli_stock,0-7,21,16,2,12.403,8 +c8b,cli_stock,0-7,21,31,1,12.464,8 +c8b,cli_stock,0-7,21,21,7,11.907,8 +c8b,cli_stock,0-7,21,21,3,13.357,8 +c8b,cli_stock,0-7,21,16,3,12.327,8 +c8b,cli_stock,0-7,21,12,3,12.874,8 +c8b,cli_stock,0-7,21,8,3,13.666,8 +c8b,cli_stock,0-7,21,8,6,11.569,8 +c8b,cli_stock,0-7,21,31,4,11.894,8 +c8b,cli_stock,0-7,21,16,6,12.960,8 +c8b,cli_stock,0-7,21,31,7,12.321,8 +c8b,cli_stock,0-7,21,12,4,13.036,8 +c8b,cli_stock,0-7,21,8,7,12.664,8 +c8b,cli_stock,0-7,21,31,6,12.753,8 +c8b,cli_stock,0-7,21,31,5,12.750,8 +c8b,cli_stock,0-7,21,16,5,12.989,8 +c8b,cli_stock,0-7,21,31,2,12.533,8 +c8b,cli_stock,0-7,21,21,1,12.104,8 +c8b,cli_stock,0-7,21,8,8,12.779,8 +c8b,cli_stock,0-7,21,21,2,12.140,8 +c8b,cli_stock,0-7,21,12,6,12.465,8 +c8b,cli_stock,0-7,21,16,1,12.247,8 +c8b,cli_stock,0-7,21,16,8,13.378,8 +c8b,cli_stock,0-7,21,8,1,12.323,8 +c8b,cli_stock,0-7,21,31,8,12.942,8 +c8b,cli_stock,0-7,21,12,1,12.619,8 +c8b,cli_stock,0-7,21,8,2,13.487,8 +c8b,cli_stock,0-7,21,12,5,13.151,8 +c8b,cli_stock,0-7,21,16,7,12.784,8 +c8b,cli_stock,0-7,21,8,5,13.111,8 +c8b,cli_stock,0-7,21,21,4,12.600,8 diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/results_c8c.csv b/scripts/profiling/table-parallelism-sweep/round2/data/results_c8c.csv new file mode 100644 index 000000000..7fed68cc6 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/results_c8c.csv @@ -0,0 +1,25 @@ +tag,cli,cpuspec,epoch,k,rep,seconds,n_slots +c8c,cli_stock,0-7,21,4,3,13.365,8 +c8c,cli_stock,0-7,21,4,6,13.064,8 +c8c,cli_stock,0-7,21,4,7,13.746,8 +c8c,cli_stock,0-7,21,6,4,13.294,8 +c8c,cli_stock,0-7,21,2,7,14.670,8 +c8c,cli_stock,0-7,21,6,5,12.994,8 +c8c,cli_stock,0-7,21,2,2,14.656,8 +c8c,cli_stock,0-7,21,2,4,14.725,8 +c8c,cli_stock,0-7,21,2,6,14.661,8 +c8c,cli_stock,0-7,21,6,3,13.161,8 +c8c,cli_stock,0-7,21,6,8,13.408,8 +c8c,cli_stock,0-7,21,6,6,12.560,8 +c8c,cli_stock,0-7,21,4,1,14.634,8 +c8c,cli_stock,0-7,21,6,1,12.992,8 +c8c,cli_stock,0-7,21,4,8,13.232,8 +c8c,cli_stock,0-7,21,4,4,13.224,8 +c8c,cli_stock,0-7,21,4,5,12.773,8 +c8c,cli_stock,0-7,21,6,7,12.311,8 +c8c,cli_stock,0-7,21,2,8,15.693,8 +c8c,cli_stock,0-7,21,4,2,13.226,8 +c8c,cli_stock,0-7,21,2,1,15.058,8 +c8c,cli_stock,0-7,21,2,3,15.502,8 +c8c,cli_stock,0-7,21,2,5,14.581,8 +c8c,cli_stock,0-7,21,6,2,13.033,8 diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/results_instr.csv b/scripts/profiling/table-parallelism-sweep/round2/data/results_instr.csv new file mode 100644 index 000000000..5142afb56 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/results_instr.csv @@ -0,0 +1,37 @@ +tag,cli,cpuspec,epoch,k,rep,seconds,n_slots,slots_mode,drv_wait_ms,drv_n,drv_max_us,ray_wait_ms,ray_n +instr,cli_instr,none,21,1,2,15.738,32,off,225.3,1394,92496,0.0,0 +instr,cli_instr,none,21,16,2,9.570,32,off,17271.4,1394,303365,0.0,0 +instr,cli_instr,none,21,16,1,9.747,32,off,23135.6,1394,488538,0.0,0 +instr,cli_instr,none,21,2,1,12.357,32,off,1722.0,1394,183809,0.0,0 +instr,cli_instr,none,21,1,3,17.060,32,on,2.0,1394,13,0.0,0 +instr,cli_instr,none,21,4,2,10.417,32,off,6117.3,1394,420310,0.0,0 +instr,cli_instr,none,21,8,1,9.894,32,off,13204.8,1394,509665,0.0,0 +instr,cli_instr,none,21,2,2,12.721,32,on,2.2,1394,36,0.0,0 +instr,cli_instr,none,21,4,1,10.233,32,off,5874.0,1394,398799,0.0,0 +instr,cli_instr,none,21,16,1,10.622,32,on,4.0,1394,1575,0.0,0 +instr,cli_instr,none,21,16,3,10.403,32,on,9.2,1394,3730,0.0,0 +instr,cli_instr,none,21,16,3,9.344,32,off,21222.2,1394,378241,0.0,0 +instr,cli_instr,none,21,31,2,9.140,32,off,23410.5,1394,474643,0.0,0 +instr,cli_instr,none,21,1,3,16.021,32,off,176.6,1394,98994,0.0,0 +instr,cli_instr,none,21,1,1,16.093,32,off,187.2,1394,89176,0.0,0 +instr,cli_instr,none,21,8,2,9.675,32,off,15471.2,1394,497046,0.0,0 +instr,cli_instr,none,21,1,1,17.152,32,on,2.0,1394,10,0.0,0 +instr,cli_instr,none,21,31,1,9.054,32,off,24961.1,1394,526484,0.0,0 +instr,cli_instr,none,21,31,3,10.017,32,on,790.0,1394,498653,0.0,0 +instr,cli_instr,none,21,31,2,10.059,32,on,754.8,1394,358521,0.0,0 +instr,cli_instr,none,21,4,1,11.087,32,on,2.2,1394,11,0.0,0 +instr,cli_instr,none,21,1,2,17.000,32,on,2.0,1394,21,0.0,0 +instr,cli_instr,none,21,2,1,12.439,32,on,2.1,1394,16,0.0,0 +instr,cli_instr,none,21,8,3,9.585,32,off,12643.9,1394,474919,0.0,0 +instr,cli_instr,none,21,8,1,10.971,32,on,5.4,1394,3005,0.0,0 +instr,cli_instr,none,21,31,1,10.416,32,on,494.9,1394,384396,0.0,0 +instr,cli_instr,none,21,8,3,10.740,32,on,2.3,1394,10,0.0,0 +instr,cli_instr,none,21,31,3,9.208,32,off,20747.6,1394,435934,0.0,0 +instr,cli_instr,none,21,2,2,11.562,32,off,1845.7,1394,235621,0.0,0 +instr,cli_instr,none,21,4,3,10.123,32,off,6500.5,1394,428124,0.0,0 +instr,cli_instr,none,21,16,2,10.350,32,on,2.4,1394,35,0.0,0 +instr,cli_instr,none,21,4,3,10.652,32,on,2.2,1394,10,0.0,0 +instr,cli_instr,none,21,8,2,10.440,32,on,2.3,1394,71,0.0,0 +instr,cli_instr,none,21,2,3,12.675,32,on,2.1,1394,16,0.0,0 +instr,cli_instr,none,21,2,3,11.690,32,off,1636.1,1394,159457,0.0,0 +instr,cli_instr,none,21,4,2,11.355,32,on,2.2,1394,22,0.0,0 diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/results_instr2.csv b/scripts/profiling/table-parallelism-sweep/round2/data/results_instr2.csv new file mode 100644 index 000000000..22a590182 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/results_instr2.csv @@ -0,0 +1,19 @@ +tag,cli,cpuspec,epoch,k,rep,seconds,n_slots,slots_mode,drv_wait_ms,drv_n,drv_max_us,ray_wait_ms,ray_n,hold_ms +instr2,cli_instr2,none,21,1,3,15.897,32,off,208.2,1394,88635,0.0,0,3294.6 +instr2,cli_instr2,none,21,31,2,8.797,32,off,19728.0,1394,410982,0.0,0,4667.1 +instr2,cli_instr2,none,21,8,2,10.698,32,on,2.4,1394,35,0.0,0,17587.0 +instr2,cli_instr2,none,21,8,2,10.005,32,off,11603.1,1394,520517,0.0,0,4212.5 +instr2,cli_instr2,none,21,8,3,10.500,32,on,2.3,1394,18,0.0,0,17289.1 +instr2,cli_instr2,none,21,1,2,16.106,32,off,182.8,1394,89409,0.0,0,3418.3 +instr2,cli_instr2,none,21,31,1,10.350,32,on,522.5,1394,280777,0.0,0,34901.3 +instr2,cli_instr2,none,21,8,3,9.452,32,off,14774.7,1394,660588,0.0,0,4169.6 +instr2,cli_instr2,none,21,1,1,17.091,32,on,2.0,1394,13,0.0,0,4467.7 +instr2,cli_instr2,none,21,31,3,8.966,32,off,21570.0,1394,348347,0.0,0,5021.3 +instr2,cli_instr2,none,21,1,3,17.234,32,on,2.0,1394,14,0.0,0,4299.3 +instr2,cli_instr2,none,21,1,2,17.115,32,on,2.0,1394,10,0.0,0,4305.8 +instr2,cli_instr2,none,21,8,1,10.315,32,on,5.3,1394,2969,0.0,0,17542.8 +instr2,cli_instr2,none,21,31,3,10.079,32,on,1063.1,1394,387528,0.0,0,30622.4 +instr2,cli_instr2,none,21,31,1,9.257,32,off,21075.0,1394,342867,0.0,0,5046.2 +instr2,cli_instr2,none,21,31,2,10.393,32,on,1006.4,1394,635037,0.0,0,33572.3 +instr2,cli_instr2,none,21,1,1,15.889,32,off,212.5,1394,98256,0.0,0,3244.8 +instr2,cli_instr2,none,21,8,1,9.465,32,off,14679.0,1394,487275,0.0,0,4264.6 diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/results_old32.csv b/scripts/profiling/table-parallelism-sweep/round2/data/results_old32.csv new file mode 100644 index 000000000..7c228ad5b --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/results_old32.csv @@ -0,0 +1,31 @@ +tag,cpuspec,epoch,k,rep,seconds,cli +old32,none,21,31,2,10.266,cli_oldsched +old32,none,21,12,3,11.233,cli_oldsched +old32,none,21,3,1,14.118,cli_oldsched +old32,none,21,4,2,13.168,cli_oldsched +old32,none,21,16,2,11.143,cli_oldsched +old32,none,21,16,3,11.780,cli_oldsched +old32,none,21,8,3,11.419,cli_oldsched +old32,none,21,2,2,13.701,cli_oldsched +old32,none,21,4,1,12.258,cli_oldsched +old32,none,21,1,2,14.147,cli_oldsched +old32,none,21,21,3,11.137,cli_oldsched +old32,none,21,3,2,13.298,cli_oldsched +old32,none,21,6,2,11.819,cli_oldsched +old32,none,21,2,3,13.136,cli_oldsched +old32,none,21,4,3,13.086,cli_oldsched +old32,none,21,6,3,12.134,cli_oldsched +old32,none,21,21,1,11.336,cli_oldsched +old32,none,21,6,1,12.340,cli_oldsched +old32,none,21,1,1,13.975,cli_oldsched +old32,none,21,21,2,10.790,cli_oldsched +old32,none,21,12,2,11.524,cli_oldsched +old32,none,21,8,1,11.903,cli_oldsched +old32,none,21,3,3,13.285,cli_oldsched +old32,none,21,8,2,11.799,cli_oldsched +old32,none,21,31,3,10.419,cli_oldsched +old32,none,21,1,3,14.669,cli_oldsched +old32,none,21,31,1,10.439,cli_oldsched +old32,none,21,2,1,13.560,cli_oldsched +old32,none,21,16,1,11.335,cli_oldsched +old32,none,21,12,1,11.204,cli_oldsched diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/results_rt.csv b/scripts/profiling/table-parallelism-sweep/round2/data/results_rt.csv new file mode 100644 index 000000000..f480a38a4 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/results_rt.csv @@ -0,0 +1,73 @@ +tag,cli,cpuspec,epoch,k,rep,seconds,n_slots,rayon +rt,cli_stock,rayon=4,21,1,2,23.746,4,4 +rt,cli_stock,rayon=8,21,16,2,12.394,8,8 +rt,cli_stock,rayon=4,21,21,1,18.803,4,4 +rt,cli_stock,rayon=32,21,12,3,9.734,32,32 +rt,cli_stock,rayon=8,21,1,2,19.416,8,8 +rt,cli_stock,rayon=4,21,21,3,17.029,4,4 +rt,cli_stock,rayon=4,21,8,3,19.159,4,4 +rt,cli_stock,rayon=4,21,4,3,18.632,4,4 +rt,cli_stock,rayon=4,21,2,3,20.201,4,4 +rt,cli_stock,rayon=32,21,21,3,9.623,32,32 +rt,cli_stock,rayon=4,21,4,1,17.952,4,4 +rt,cli_stock,rayon=32,21,31,2,8.800,32,32 +rt,cli_stock,rayon=32,21,16,2,9.563,32,32 +rt,cli_stock,rayon=4,21,2,1,21.404,4,4 +rt,cli_stock,rayon=32,21,31,1,8.820,32,32 +rt,cli_stock,rayon=8,21,21,1,12.393,8,8 +rt,cli_stock,rayon=32,21,12,2,9.948,32,32 +rt,cli_stock,rayon=8,21,4,3,12.878,8,8 +rt,cli_stock,rayon=32,21,2,3,11.731,32,32 +rt,cli_stock,rayon=4,21,31,3,16.606,4,4 +rt,cli_stock,rayon=4,21,1,3,24.329,4,4 +rt,cli_stock,rayon=32,21,1,1,16.036,32,32 +rt,cli_stock,rayon=8,21,2,1,14.709,8,8 +rt,cli_stock,rayon=32,21,4,1,10.182,32,32 +rt,cli_stock,rayon=32,21,31,3,9.132,32,32 +rt,cli_stock,rayon=8,21,21,2,11.926,8,8 +rt,cli_stock,rayon=4,21,31,2,17.355,4,4 +rt,cli_stock,rayon=8,21,31,2,11.532,8,8 +rt,cli_stock,rayon=32,21,21,1,9.512,32,32 +rt,cli_stock,rayon=32,21,8,1,9.631,32,32 +rt,cli_stock,rayon=4,21,16,1,17.667,4,4 +rt,cli_stock,rayon=4,21,12,3,17.643,4,4 +rt,cli_stock,rayon=8,21,16,1,12.121,8,8 +rt,cli_stock,rayon=32,21,16,3,9.356,32,32 +rt,cli_stock,rayon=32,21,8,2,9.468,32,32 +rt,cli_stock,rayon=4,21,12,1,18.355,4,4 +rt,cli_stock,rayon=4,21,4,2,18.085,4,4 +rt,cli_stock,rayon=4,21,16,3,17.993,4,4 +rt,cli_stock,rayon=4,21,8,2,18.717,4,4 +rt,cli_stock,rayon=4,21,8,1,18.990,4,4 +rt,cli_stock,rayon=8,21,1,3,18.946,8,8 +rt,cli_stock,rayon=8,21,8,3,11.909,8,8 +rt,cli_stock,rayon=8,21,2,2,15.139,8,8 +rt,cli_stock,rayon=8,21,12,1,12.743,8,8 +rt,cli_stock,rayon=8,21,4,1,13.330,8,8 +rt,cli_stock,rayon=32,21,4,3,10.277,32,32 +rt,cli_stock,rayon=32,21,1,2,16.059,32,32 +rt,cli_stock,rayon=32,21,16,1,9.397,32,32 +rt,cli_stock,rayon=8,21,8,2,12.820,8,8 +rt,cli_stock,rayon=4,21,2,2,20.166,4,4 +rt,cli_stock,rayon=8,21,4,2,13.343,8,8 +rt,cli_stock,rayon=4,21,21,2,18.657,4,4 +rt,cli_stock,rayon=32,21,12,1,9.659,32,32 +rt,cli_stock,rayon=32,21,1,3,16.049,32,32 +rt,cli_stock,rayon=8,21,12,2,11.540,8,8 +rt,cli_stock,rayon=32,21,2,2,12.056,32,32 +rt,cli_stock,rayon=4,21,12,2,17.188,4,4 +rt,cli_stock,rayon=8,21,1,1,19.140,8,8 +rt,cli_stock,rayon=4,21,1,1,25.762,4,4 +rt,cli_stock,rayon=8,21,31,3,10.682,8,8 +rt,cli_stock,rayon=32,21,2,1,11.885,32,32 +rt,cli_stock,rayon=32,21,21,2,9.528,32,32 +rt,cli_stock,rayon=8,21,31,1,10.766,8,8 +rt,cli_stock,rayon=32,21,4,2,10.453,32,32 +rt,cli_stock,rayon=4,21,16,2,16.962,4,4 +rt,cli_stock,rayon=8,21,21,3,11.609,8,8 +rt,cli_stock,rayon=8,21,8,1,11.973,8,8 +rt,cli_stock,rayon=4,21,31,1,19.276,4,4 +rt,cli_stock,rayon=8,21,16,3,11.746,8,8 +rt,cli_stock,rayon=8,21,12,3,12.337,8,8 +rt,cli_stock,rayon=8,21,2,3,14.580,8,8 +rt,cli_stock,rayon=32,21,8,3,9.774,32,32 diff --git a/scripts/profiling/table-parallelism-sweep/round2/data/slotproof.txt b/scripts/profiling/table-parallelism-sweep/round2/data/slotproof.txt new file mode 100644 index 000000000..8d3ea1865 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/data/slotproof.txt @@ -0,0 +1,81 @@ +--- patched, TABLE_PARALLELISM=8, LAMBDA_VM_SLOT_DEBUG=1 (expect several distinct raw slots) --- +slot-debug: non-rayon thread ThreadId(100) -> raw slot 63 +slot-debug: non-rayon thread ThreadId(101) -> raw slot 61 +slot-debug: non-rayon thread ThreadId(102) -> raw slot 68 +slot-debug: non-rayon thread ThreadId(103) -> raw slot 64 +slot-debug: non-rayon thread ThreadId(104) -> raw slot 65 +slot-debug: non-rayon thread ThreadId(105) -> raw slot 67 +slot-debug: non-rayon thread ThreadId(108) -> raw slot 66 +slot-debug: non-rayon thread ThreadId(109) -> raw slot 69 +slot-debug: non-rayon thread ThreadId(110) -> raw slot 74 +slot-debug: non-rayon thread ThreadId(111) -> raw slot 76 +slot-debug: non-rayon thread ThreadId(112) -> raw slot 70 +slot-debug: non-rayon thread ThreadId(113) -> raw slot 73 +slot-debug: non-rayon thread ThreadId(114) -> raw slot 71 +slot-debug: non-rayon thread ThreadId(115) -> raw slot 75 +slot-debug: non-rayon thread ThreadId(116) -> raw slot 72 +slot-debug: non-rayon thread ThreadId(117) -> raw slot 77 +slot-debug: non-rayon thread ThreadId(38) -> raw slot 1 +slot-debug: non-rayon thread ThreadId(39) -> raw slot 3 +slot-debug: non-rayon thread ThreadId(40) -> raw slot 0 +slot-debug: non-rayon thread ThreadId(41) -> raw slot 4 +slot-debug: non-rayon thread ThreadId(42) -> raw slot 2 +slot-debug: non-rayon thread ThreadId(43) -> raw slot 5 +slot-debug: non-rayon thread ThreadId(44) -> raw slot 7 +slot-debug: non-rayon thread ThreadId(45) -> raw slot 6 +slot-debug: non-rayon thread ThreadId(46) -> raw slot 16 +slot-debug: non-rayon thread ThreadId(47) -> raw slot 17 +slot-debug: non-rayon thread ThreadId(48) -> raw slot 22 +slot-debug: non-rayon thread ThreadId(49) -> raw slot 20 +slot-debug: non-rayon thread ThreadId(50) -> raw slot 23 +slot-debug: non-rayon thread ThreadId(51) -> raw slot 19 +slot-debug: non-rayon thread ThreadId(52) -> raw slot 21 +slot-debug: non-rayon thread ThreadId(53) -> raw slot 18 +slot-debug: non-rayon thread ThreadId(54) -> raw slot 15 +slot-debug: non-rayon thread ThreadId(55) -> raw slot 8 +slot-debug: non-rayon thread ThreadId(56) -> raw slot 10 +slot-debug: non-rayon thread ThreadId(57) -> raw slot 13 +slot-debug: non-rayon thread ThreadId(58) -> raw slot 12 +slot-debug: non-rayon thread ThreadId(59) -> raw slot 9 +slot-debug: non-rayon thread ThreadId(60) -> raw slot 11 +slot-debug: non-rayon thread ThreadId(61) -> raw slot 14 +slot-debug: non-rayon thread ThreadId(62) -> raw slot 31 +slot-debug: non-rayon thread ThreadId(63) -> raw slot 27 +slot-debug: non-rayon thread ThreadId(64) -> raw slot 24 +slot-debug: non-rayon thread ThreadId(65) -> raw slot 25 +slot-debug: non-rayon thread ThreadId(66) -> raw slot 28 +slot-debug: non-rayon thread ThreadId(67) -> raw slot 26 +slot-debug: non-rayon thread ThreadId(68) -> raw slot 29 +slot-debug: non-rayon thread ThreadId(69) -> raw slot 30 +slot-debug: non-rayon thread ThreadId(70) -> raw slot 37 +slot-debug: non-rayon thread ThreadId(71) -> raw slot 33 +slot-debug: non-rayon thread ThreadId(72) -> raw slot 32 +slot-debug: non-rayon thread ThreadId(73) -> raw slot 38 +slot-debug: non-rayon thread ThreadId(74) -> raw slot 35 +slot-debug: non-rayon thread ThreadId(75) -> raw slot 34 +slot-debug: non-rayon thread ThreadId(76) -> raw slot 36 +slot-debug: non-rayon thread ThreadId(77) -> raw slot 39 +slot-debug: non-rayon thread ThreadId(78) -> raw slot 40 +slot-debug: non-rayon thread ThreadId(79) -> raw slot 42 +slot-debug: non-rayon thread ThreadId(80) -> raw slot 43 +slot-debug: non-rayon thread ThreadId(81) -> raw slot 41 +slot-debug: non-rayon thread ThreadId(82) -> raw slot 47 +slot-debug: non-rayon thread ThreadId(83) -> raw slot 46 +slot-debug: non-rayon thread ThreadId(84) -> raw slot 45 +slot-debug: non-rayon thread ThreadId(85) -> raw slot 44 +slot-debug: non-rayon thread ThreadId(86) -> raw slot 52 +slot-debug: non-rayon thread ThreadId(87) -> raw slot 49 +slot-debug: non-rayon thread ThreadId(88) -> raw slot 55 +slot-debug: non-rayon thread ThreadId(89) -> raw slot 48 +slot-debug: non-rayon thread ThreadId(90) -> raw slot 50 +slot-debug: non-rayon thread ThreadId(91) -> raw slot 51 +slot-debug: non-rayon thread ThreadId(92) -> raw slot 53 +slot-debug: non-rayon thread ThreadId(93) -> raw slot 54 +slot-debug: non-rayon thread ThreadId(94) -> raw slot 56 +slot-debug: non-rayon thread ThreadId(95) -> raw slot 59 +slot-debug: non-rayon thread ThreadId(96) -> raw slot 58 +slot-debug: non-rayon thread ThreadId(97) -> raw slot 60 +slot-debug: non-rayon thread ThreadId(98) -> raw slot 62 +slot-debug: non-rayon thread ThreadId(99) -> raw slot 57 +--- stock, same env (negative control, expect 0) --- +0 diff --git a/scripts/profiling/table-parallelism-sweep/round2/followup2.sh b/scripts/profiling/table-parallelism-sweep/round2/followup2.sh new file mode 100755 index 000000000..f0ee468c4 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/followup2.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Decompose "cores" into rayon pool width vs machine width. +# taskset shrinks BOTH (pool + the K driver threads' CPUs). +# RAYON_NUM_THREADS shrinks ONLY the pool; K drivers still get real cores. +set -u +CLI=/root/lambda_vm/target/release/cli +ELF=/root/ethrex.elf +INPUT=/root/lambda_vm/executor/tests/ethrex_10_transfers.bin +OUT=/root/results/results_rayon.csv +echo "tag,cpuspec,epoch,k,rep,seconds" > "$OUT" + +JOBS=$(for rt in 4 8 32; do + for k in 1 2 3 4 6 8 12 16 21 31; do + for r in 1 2 3; do echo "$rt $k $r"; done + done + done | shuf) + +TOTAL=$(echo "$JOBS" | wc -l); i=0 +echo "$JOBS" | while read -r rt k r; do + i=$((i+1)) + t=$(RAYON_NUM_THREADS=$rt TABLE_PARALLELISM=$k "$CLI" prove "$ELF" \ + --private-input "$INPUT" -o /tmp/pr.bin --time --continuations \ + --epoch-size-log2 21 2>&1 | sed -n 's/^Proving time: \([0-9.]*\)s/\1/p') + [ -z "$t" ] && t="NA" + echo "rayon$rt,rayon=$rt,21,$k,$r,$t" >> "$OUT" + echo "[$i/$TOTAL] RAYON_NUM_THREADS=$rt K=$k rep=$r -> ${t}s" +done +echo DONE_RAYON diff --git a/scripts/profiling/table-parallelism-sweep/round2/pull.sh b/scripts/profiling/table-parallelism-sweep/round2/pull.sh new file mode 100755 index 000000000..0162c03ce --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/pull.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# Local mirror loop: pull the server's results dir every INTERVAL seconds so the +# laptop copy is never more than a minute behind. Runs independent of whoever is +# driving the sweep; the rented box is expected to vanish without warning. +# +# Usage: pull.sh [interval_seconds] +set -u +INTERVAL="${1:-60}" +REMOTE="root@79.161.122.162" +PORT=63821 +DEST="$(cd "$(dirname "$0")" && pwd)/data" +mkdir -p "$DEST" + +echo "mirroring $REMOTE:/root/results/ -> $DEST every ${INTERVAL}s" +while true; do + # Tolerate the dir not existing yet, and the box being gone: never exit on + # rsync failure, just report and retry. + if rsync -az -e "ssh -p $PORT -o ConnectTimeout=10 -o BatchMode=yes" \ + "$REMOTE:/root/results/" "$DEST/" 2>/dev/null; then + n=$(find "$DEST" -name '*.csv' -exec cat {} + 2>/dev/null | grep -cv '^tag,' || true) + echo "[$(date -u +%H:%M:%SZ)] ok — $n timed rows mirrored" + else + echo "[$(date -u +%H:%M:%SZ)] pull failed (box down or dir absent) — retrying" + fi + sleep "$INTERVAL" +done diff --git a/scripts/profiling/table-parallelism-sweep/round2/rules2.py b/scripts/profiling/table-parallelism-sweep/round2/rules2.py new file mode 100644 index 000000000..16d540bb2 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/rules2.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +"""Round-2 rule comparison, run from the mirrored data/ dir. + +Differences from round 1's rules.py: + * merges the c32 anchor with the higher-n plateau pass (c32b); + * reports each rule's penalty against the best MEASURED K in that config, and + also against the best K the rule could have picked from the swept list, so a + rule is not charged for K values nobody measured; + * carries n_slots through, because per-driver slots alias once K > n_slots; + * prints round 1's ratios beside round 2's so the two boxes are comparable. +""" +import csv, statistics as st, sys, os +from collections import defaultdict + +HERE = os.path.dirname(os.path.abspath(__file__)) +R1 = os.path.join(HERE, "..", "round1") + +CONFIGS = [(["results_c32.csv", "results_c32b.csv"], 32), + (["results_c16.csv"], 16), + (["results_c8.csv", "results_c8b.csv"], 8), + (["results_c4.csv", "results_c4b.csv"], 4)] + +RULES = { + "cores/3 (old CPU default)": lambda c: max(c // 3, 1), + "cores*2/3 (#877 merged)": lambda c: max(c * 2 // 3, 1), + "constant 8": lambda c: 8, + "constant 12": lambda c: 12, + "constant 16": lambda c: 16, + "constant 21": lambda c: 21, + "unbounded (min num_airs)": lambda c: 31, +} + + +def load(paths, base="."): + d = defaultdict(list) + slots = set() + found = False + for p in paths: + fp = os.path.join(base, p) + if not os.path.exists(fp): + continue + found = True + for r in csv.DictReader(open(fp)): + if r["seconds"] == "NA": + continue + d[int(r["k"])].append(float(r["seconds"])) + if r.get("n_slots"): + slots.add(r["n_slots"]) + if not found: + return None, None + # sweep_r.sh (used for the c32 anchor) predates the n_slots column; on this + # box an unrestricted run has n_slots = 32. + return {k: st.median(v) for k, v in d.items()}, slots + + +def nearest(meds, k): + return min(meds, key=lambda x: abs(x - k)) + + +data, slotinfo, ns = {}, {}, {} +for paths, c in CONFIGS: + m, s = load(paths) + if m: + data[c] = m + ns[c] = ",".join(sorted(s)) or str(c) + +r1 = {} +for p, c in [("results_c32.csv", 32), ("results_c16.csv", 16), + ("results_c8.csv", 8), ("results_c4.csv", 4)]: + m, _ = load([p], R1) + if m: + r1[c] = m + +cores = sorted(data, reverse=True) +print("round 2 — median seconds, and x best measured K in that config") +print(f"{'K':>5}", end="") +for c in cores: + print(f"{'c'+str(c):>16}", end="") +print() +allk = sorted({k for c in cores for k in data[c]}) +for k in allk: + print(f"{k:>5}", end="") + for c in cores: + if k in data[c]: + best = min(data[c].values()) + print(f"{data[c][k]:>10.2f} {data[c][k]/best:>4.2f}x", end="") + else: + print(f"{'-':>16}", end="") + print() +print(f"{'nslots':>5}", end="") +for c in cores: + print(f"{ns.get(c,'?'):>16}", end="") +print() + +print("\nbest K per config (round 2 | round 1):") +for c in cores: + bk = min(data[c], key=lambda k: data[c][k]) + r1s = "" + if c in r1: + b1 = min(r1[c], key=lambda k: r1[c][k]) + r1s = f" | r1 best K={b1} {r1[c][b1]:.2f}s" + print(f" c{c:<3} best K={bk:<3} {data[c][bk]:.2f}s{r1s}") + +print("\nrule penalty vs best measured K in each config:") +print(f"{'rule':<26}", end="") +for c in cores: + print(f"{'c'+str(c):>16}", end="") +print(f"{'worst':>9}") +print("-" * (26 + 16 * len(cores) + 9)) +for name, fn in RULES.items(): + print(f"{name:<26}", end="") + worst = 0.0 + for c in cores: + meds = data[c] + best = min(meds.values()) + k = fn(c) + kk = nearest(meds, k) + pen = (meds[kk] / best - 1) * 100 + worst = max(worst, pen) + tag = f"K={k}" + ("" if kk == k else f"~{kk}") + print(f"{tag+f' +{pen:.1f}%':>16}", end="") + print(f"{'+'+format(worst,'.1f')+'%':>9}") + +if r1: + print("\nsame table on round 1's box (7950X3D), for comparison:") + r1cores = sorted(r1, reverse=True) + print(f"{'rule':<26}", end="") + for c in r1cores: + print(f"{'c'+str(c):>16}", end="") + print(f"{'worst':>9}") + for name, fn in RULES.items(): + print(f"{name:<26}", end="") + worst = 0.0 + for c in r1cores: + meds = r1[c] + best = min(meds.values()) + kk = nearest(meds, fn(c)) + pen = (meds[kk] / best - 1) * 100 + worst = max(worst, pen) + print(f"{'K='+str(fn(c))+f' +{pen:.1f}%':>16}", end="") + print(f"{'+'+format(worst,'.1f')+'%':>9}") diff --git a/scripts/profiling/table-parallelism-sweep/round2/stacksample.sh b/scripts/profiling/table-parallelism-sweep/round2/stacksample.sh new file mode 100755 index 000000000..f4c406971 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/stacksample.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# Poor-man's off-CPU profiler: run a prove and repeatedly snapshot every thread's +# stack with gdb, so we can classify what the K driver threads actually block on +# (pinned-staging mutex vs CUDA sync vs VramGate condvar). +# Usage: stacksample.sh [cpuspec] +set -u +K="$1"; N="$2"; CPUSPEC="${3:-none}" +CLI=/root/target_dbg/release/cli # symbolized build +ELF=/root/ethrex.elf +INPUT=/root/lambda_vm/executor/tests/ethrex_10_transfers.bin +OUT=/root/results/stacks_k${K}_${CPUSPEC}.txt + +if [ "$CPUSPEC" = "none" ]; then PREFIX=""; else PREFIX="taskset -c $CPUSPEC"; fi +: > "$OUT" + +TABLE_PARALLELISM=$K $PREFIX "$CLI" prove "$ELF" --private-input "$INPUT" \ + -o /tmp/ss.bin --time --continuations --epoch-size-log2 21 > /root/results/ss_run_k${K}.log 2>&1 & +PID=$! + +# Let it get past ELF load / CUDA init / first epoch execute into proving. +sleep 4 +for i in $(seq 1 "$N"); do + kill -0 $PID 2>/dev/null || break + echo "===== SAMPLE $i =====" >> "$OUT" + gdb -p $PID -batch -ex "set pagination off" -ex "thread apply all bt 14" \ + >> "$OUT" 2>/dev/null + sleep 0.3 +done +wait $PID 2>/dev/null +echo "samples written to $OUT" +grep -c "^Thread" "$OUT" 2>/dev/null | sed 's/^/thread-snapshots: /' diff --git a/scripts/profiling/table-parallelism-sweep/round2/sweep_ab.sh b/scripts/profiling/table-parallelism-sweep/round2/sweep_ab.sh new file mode 100644 index 000000000..07ef10ff0 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/sweep_ab.sh @@ -0,0 +1,56 @@ +#!/bin/bash +# Interleaved A/B K sweep over two or more cli binaries. Server-side. +# +# Usage: sweep_ab.sh +# +# Same checkpointing contract as sweep_r.sh (one fsync'd row per run, frozen job +# order, resume skips recorded rows) but the randomized order spans the binaries +# too. Round 2's box reboots, so a design that ran all of A then all of B would +# put any thermal/clock drift straight into the A-vs-B difference. +set -u +TAG="$1"; CPUSPEC="$2"; REPS="$3"; EPOCH="$4"; KLIST="$5"; BINS="$6" +ELF=/root/ethrex.elf +INPUT=/root/lambda_vm/executor/tests/ethrex_10_transfers.bin +DIR=/root/results +OUT="$DIR/results_${TAG}.csv" +JOBS="$DIR/jobs_${TAG}.txt" +mkdir -p "$DIR" + +if [ "$CPUSPEC" = "none" ]; then PREFIX=""; else PREFIX="taskset -c $CPUSPEC"; fi +# n_slots = rayon::current_num_threads() = available_parallelism under the same +# affinity mask (device.rs:321). Recorded per row: per-driver slots alias when +# K > n_slots, so rows with different n_slots are not comparable. +NSLOTS=$($PREFIX nproc) + +[ -f "$OUT" ] || echo "tag,cli,cpuspec,epoch,k,rep,seconds,n_slots" > "$OUT" + +if [ ! -f "$JOBS" ]; then + for b in $(echo "$BINS" | tr ',' ' '); do + for k in $(echo "$KLIST" | tr ',' ' '); do + for r in $(seq 1 "$REPS"); do echo "$b $k $r"; done + done + done | shuf > "$JOBS" +fi + +echo "warmup run (not recorded)" +first_bin=$(echo "$BINS" | cut -d, -f1) +TABLE_PARALLELISM=4 $PREFIX "$first_bin" prove "$ELF" --private-input "$INPUT" \ + -o "/tmp/warm_${TAG}.bin" --time --continuations --epoch-size-log2 "$EPOCH" >/dev/null 2>&1 + +TOTAL=$(wc -l < "$JOBS"); i=0 +while read -r b k r; do + i=$((i+1)) + bn=$(basename "$b") + if grep -q "^${TAG},${bn},${CPUSPEC},${EPOCH},${k},${r}," "$OUT"; then + echo "[$i/$TOTAL] $bn K=$k rep=$r -> SKIP (recorded)"; continue + fi + raw=$(TABLE_PARALLELISM=$k $PREFIX "$b" prove "$ELF" --private-input "$INPUT" \ + -o "/tmp/p_${TAG}.bin" --time --continuations --epoch-size-log2 "$EPOCH" 2>&1) + [ "$r" = "1" ] && printf '%s\n' "$raw" > "$DIR/log_${TAG}_${bn}_k${k}.txt" + t=$(printf '%s\n' "$raw" | sed -n 's/^Proving time: \([0-9.]*\)s/\1/p') + [ -z "$t" ] && t="NA" + echo "$TAG,$bn,$CPUSPEC,$EPOCH,$k,$r,$t,$NSLOTS" >> "$OUT" + sync -f "$OUT" 2>/dev/null || sync + echo "[$i/$TOTAL] tag=$TAG $bn cpus=$CPUSPEC nslots=$NSLOTS K=$k rep=$r -> ${t}s" +done < "$JOBS" +echo "DONE $TAG -> $OUT" diff --git a/scripts/profiling/table-parallelism-sweep/round2/sweep_instr.sh b/scripts/profiling/table-parallelism-sweep/round2/sweep_instr.sh new file mode 100644 index 000000000..8e9f2eb8d --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/sweep_instr.sh @@ -0,0 +1,62 @@ +#!/bin/bash +# Stage 1a substitute: how much time do the K driver threads actually spend +# blocked on the pinned-staging mutex, and does giving them per-driver slots +# remove it? Both arms are the SAME binary (run-time switch), so nothing but the +# slot policy differs. +# +# Usage: sweep_instr.sh [cli_path] +# Resumable + fsync'd per row, like the other harnesses. +set -u +TAG="$1"; CPUSPEC="$2"; REPS="$3"; EPOCH="$4"; KLIST="$5" +CLI="${6:-/root/bin/cli_instr}" +BN=$(basename "$CLI") +ELF=/root/ethrex.elf +INPUT=/root/lambda_vm/executor/tests/ethrex_10_transfers.bin +DIR=/root/results +OUT="$DIR/results_${TAG}.csv" +JOBS="$DIR/jobs_${TAG}.txt" +mkdir -p "$DIR" +if [ "$CPUSPEC" = "none" ]; then PREFIX=""; else PREFIX="taskset -c $CPUSPEC"; fi +NSLOTS=$($PREFIX nproc) + +[ -f "$OUT" ] || echo "tag,cli,cpuspec,epoch,k,rep,seconds,n_slots,slots_mode,drv_wait_ms,drv_n,drv_max_us,ray_wait_ms,ray_n,hold_ms" > "$OUT" + +if [ ! -f "$JOBS" ]; then + for mode in off on; do + for k in $(echo "$KLIST" | tr ',' ' '); do + for r in $(seq 1 "$REPS"); do echo "$mode $k $r"; done + done + done | shuf > "$JOBS" +fi + +echo "warmup run (not recorded)" +TABLE_PARALLELISM=4 $PREFIX "$CLI" prove "$ELF" --private-input "$INPUT" \ + -o "/tmp/warm_${TAG}.bin" --time --continuations --epoch-size-log2 "$EPOCH" >/dev/null 2>&1 + +TOTAL=$(wc -l < "$JOBS"); i=0 +while read -r mode k r; do + i=$((i+1)) + if grep -q "^${TAG},${BN},${CPUSPEC},${EPOCH},${k},${r},.*,${mode}," "$OUT"; then + echo "[$i/$TOTAL] slots=$mode K=$k rep=$r -> SKIP (recorded)"; continue + fi + if [ "$mode" = "on" ]; then export LAMBDA_VM_PER_DRIVER_SLOTS=1; else unset LAMBDA_VM_PER_DRIVER_SLOTS; fi + raw=$(LAMBDA_VM_STAGING_STATS=1 TABLE_PARALLELISM=$k $PREFIX "$CLI" prove "$ELF" \ + --private-input "$INPUT" -o "/tmp/p_${TAG}.bin" --time --continuations \ + --epoch-size-log2 "$EPOCH" 2>&1) + [ "$r" = "1" ] && printf '%s\n' "$raw" | grep -v "^staging-stats" > "$DIR/log_${TAG}_${mode}_k${k}.txt" + t=$(printf '%s\n' "$raw" | sed -n 's/^Proving time: \([0-9.]*\)s/\1/p') + # last reporter line = the run's tally + st=$(printf '%s\n' "$raw" | grep "^staging-stats" | tail -1) + dw=$(printf '%s\n' "$st" | sed -n 's/.*drv_wait_ms=\([0-9.]*\).*/\1/p') + dn=$(printf '%s\n' "$st" | sed -n 's/.*drv_n=\([0-9]*\).*/\1/p') + dm=$(printf '%s\n' "$st" | sed -n 's/.*drv_max_us=\([0-9]*\).*/\1/p') + rw=$(printf '%s\n' "$st" | sed -n 's/.*ray_wait_ms=\([0-9.]*\).*/\1/p') + rn=$(printf '%s\n' "$st" | sed -n 's/.*ray_n=\([0-9]*\).*/\1/p') + hm=$(printf '%s\n' "$st" | sed -n 's/.*hold_ms=\([0-9.]*\).*/\1/p') + [ -z "$t" ] && t="NA" + echo "$TAG,$BN,$CPUSPEC,$EPOCH,$k,$r,$t,$NSLOTS,$mode,${dw:-NA},${dn:-NA},${dm:-NA},${rw:-NA},${rn:-NA},${hm:-NA}" >> "$OUT" + sync -f "$OUT" 2>/dev/null || sync + echo "[$i/$TOTAL] slots=$mode K=$k rep=$r -> ${t}s drv_wait=${dw}ms/${dn} max=${dm}us ray_wait=${rw}ms/${rn} hold=${hm}ms" +done < "$JOBS" +unset LAMBDA_VM_PER_DRIVER_SLOTS +echo "DONE $TAG -> $OUT" diff --git a/scripts/profiling/table-parallelism-sweep/round2/sweep_r.sh b/scripts/profiling/table-parallelism-sweep/round2/sweep_r.sh new file mode 100755 index 000000000..382eec2bd --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/sweep_r.sh @@ -0,0 +1,61 @@ +#!/bin/bash +# Resumable, checkpointing K sweep. Server-side. +# +# Usage: sweep_r.sh [cli_path] +# +# Two properties round 1's harness lacked: +# * one fsync'd CSV row per timed run, so a box that dies loses at most the +# run in flight; +# * resume that does NOT reshuffle. The randomized (k,rep) order is generated +# once into jobs_.txt and reused; rows already in the CSV are skipped. +# Reshuffling on restart would quietly destroy the bias control that the +# interleaving exists to provide. +set -u +TAG="$1"; CPUSPEC="$2"; REPS="$3"; EPOCH="$4"; KLIST="$5" +CLI="${6:-/root/lambda_vm/target/release/cli}" +ELF=/root/ethrex.elf +INPUT=/root/lambda_vm/executor/tests/ethrex_10_transfers.bin +DIR=/root/results +OUT="$DIR/results_${TAG}.csv" +JOBS="$DIR/jobs_${TAG}.txt" +mkdir -p "$DIR" + +if [ "$CPUSPEC" = "none" ]; then PREFIX=""; else PREFIX="taskset -c $CPUSPEC"; fi + +[ -f "$OUT" ] || echo "tag,cpuspec,epoch,k,rep,seconds,cli" > "$OUT" + +# Job order: generated once, then frozen for all resumes. +if [ ! -f "$JOBS" ]; then + for k in $(echo "$KLIST" | tr ',' ' '); do + for r in $(seq 1 "$REPS"); do echo "$k $r"; done + done | shuf > "$JOBS" +fi + +# Warmup on EVERY start, not just cold ones: pulls the GPU off idle clocks and +# warms the page cache. Round 2's box reboots without warning (first one killed +# Stage 0 after 2 runs) and cron/keepalive.sh resumes the sweep, so a +# resume-without-warmup would drop a cold-clock run at a random K into the data. +if true; then + echo "warmup run (not recorded)" + TABLE_PARALLELISM=4 $PREFIX "$CLI" prove "$ELF" --private-input "$INPUT" \ + -o "/tmp/warm_${TAG}.bin" --time --continuations --epoch-size-log2 "$EPOCH" >/dev/null 2>&1 +fi + +TOTAL=$(wc -l < "$JOBS"); i=0 +while read -r k r; do + i=$((i+1)) + if grep -q ",${EPOCH},${k},${r}," "$OUT"; then + echo "[$i/$TOTAL] K=$k rep=$r -> SKIP (recorded)"; continue + fi + raw=$(TABLE_PARALLELISM=$k $PREFIX "$CLI" prove "$ELF" --private-input "$INPUT" \ + -o "/tmp/p_${TAG}.bin" --time --continuations --epoch-size-log2 "$EPOCH" 2>&1) + # Keep rep 1's full output per K: the per-phase/per-epoch breakdown is what + # Stage 1c needs, and it is gone once we reduce to a single number. + [ "$r" = "1" ] && printf '%s\n' "$raw" > "$DIR/log_${TAG}_k${k}.txt" + t=$(printf '%s\n' "$raw" | sed -n 's/^Proving time: \([0-9.]*\)s/\1/p') + [ -z "$t" ] && t="NA" + echo "$TAG,$CPUSPEC,$EPOCH,$k,$r,$t,$(basename "$CLI")" >> "$OUT" + sync -f "$OUT" 2>/dev/null || sync + echo "[$i/$TOTAL] tag=$TAG cpus=$CPUSPEC K=$k rep=$r -> ${t}s" +done < "$JOBS" +echo "DONE $TAG -> $OUT" diff --git a/scripts/profiling/table-parallelism-sweep/round2/sweep_rayon.sh b/scripts/profiling/table-parallelism-sweep/round2/sweep_rayon.sh new file mode 100644 index 000000000..4d5e99ad1 --- /dev/null +++ b/scripts/profiling/table-parallelism-sweep/round2/sweep_rayon.sh @@ -0,0 +1,54 @@ +#!/bin/bash +# Resumable RAYON_NUM_THREADS x K sweep — the decomposition round 1 wrote as +# followup2.sh but never ran. Rewritten here because followup2.sh truncates its +# CSV on every start and reshuffles its job order, and round 2's box reboots and +# auto-resumes: that combination would have wiped the data on the first reboot. +# +# taskset shrinks the rayon pool AND the K driver threads' CPUs. +# RAYON_NUM_THREADS shrinks only the pool; the drivers still get real cores. +# Confound recorded, not removed: n_slots = rayon::current_num_threads(), so it +# shrinks with the pool too (device.rs:321). +# +# Usage: sweep_rayon.sh [cli_path] +set -u +TAG="$1"; REPS="$2"; EPOCH="$3"; KLIST="$4"; RTLIST="$5" +CLI="${6:-/root/bin/cli_stock}" +ELF=/root/ethrex.elf +INPUT=/root/lambda_vm/executor/tests/ethrex_10_transfers.bin +DIR=/root/results +OUT="$DIR/results_${TAG}.csv" +JOBS="$DIR/jobs_${TAG}.txt" +mkdir -p "$DIR" + +[ -f "$OUT" ] || echo "tag,cli,cpuspec,epoch,k,rep,seconds,n_slots,rayon" > "$OUT" + +if [ ! -f "$JOBS" ]; then + for rt in $(echo "$RTLIST" | tr ',' ' '); do + for k in $(echo "$KLIST" | tr ',' ' '); do + for r in $(seq 1 "$REPS"); do echo "$rt $k $r"; done + done + done | shuf > "$JOBS" +fi + +echo "warmup run (not recorded)" +TABLE_PARALLELISM=4 "$CLI" prove "$ELF" --private-input "$INPUT" \ + -o "/tmp/warm_${TAG}.bin" --time --continuations --epoch-size-log2 "$EPOCH" >/dev/null 2>&1 + +bn=$(basename "$CLI") +TOTAL=$(wc -l < "$JOBS"); i=0 +while read -r rt k r; do + i=$((i+1)) + if grep -q "^${TAG},${bn},rayon=${rt},${EPOCH},${k},${r}," "$OUT"; then + echo "[$i/$TOTAL] RAYON=$rt K=$k rep=$r -> SKIP (recorded)"; continue + fi + raw=$(RAYON_NUM_THREADS=$rt TABLE_PARALLELISM=$k "$CLI" prove "$ELF" \ + --private-input "$INPUT" -o "/tmp/p_${TAG}.bin" --time --continuations \ + --epoch-size-log2 "$EPOCH" 2>&1) + [ "$r" = "1" ] && printf '%s\n' "$raw" > "$DIR/log_${TAG}_rt${rt}_k${k}.txt" + t=$(printf '%s\n' "$raw" | sed -n 's/^Proving time: \([0-9.]*\)s/\1/p') + [ -z "$t" ] && t="NA" + echo "$TAG,$bn,rayon=$rt,$EPOCH,$k,$r,$t,$rt,$rt" >> "$OUT" + sync -f "$OUT" 2>/dev/null || sync + echo "[$i/$TOTAL] tag=$TAG RAYON_NUM_THREADS=$rt K=$k rep=$r -> ${t}s" +done < "$JOBS" +echo "DONE $TAG -> $OUT"