f8175c8c69f80dd21609ffe92b417e942f074d49 / alderbench/src/inproc.rs · 20602 bytes · raw
//! In-process workloads: c2c-latency (cache-line ping-pong between core
//! pairs) and dram-latency (pointer-chase through the memory hierarchy).
//! Zero-dep: CPU pinning goes through a raw `sched_setaffinity` syscall
//! (x86_64 nr 203); timing uses `std::time::Instant` (clock_gettime via
//! the vDSO — ns resolution, no TSC calibration needed).
//!
//! Both workloads measure while pinning threads, so they double as
//! coherency/memory stability checks: the ping-pong verifies every
//! cache-line handoff carries the exact expected value (a coherence or
//! clock failure shows up as a skipped/duplicated sequence number =
//! Verdict::Failed), and the pointer-chase verifies the chase walks one
//! full cycle of the randomized chain (a flipped address bit breaks the
//! cycle). Verdict is the stability signal; the latencies are the score.
use std::io;
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::{Arc, Mutex};
use std::thread;
use std::time::{Duration, Instant};
use crate::workload::{C2cPair, DramLevel, Score, Verdict, Workload};
/// Parse a stress-ng `--taskset`-style cpu list ("0-11", "12-15",
/// "0-15", "3,7", ""/all = every logical cpu). Out-of-range entries are
/// clamped to the CPU count; invalid tokens are skipped. Sorted + deduped.
pub fn parse_cpu_list(cores: &str) -> Vec<usize> {
let nproc = thread::available_parallelism()
.map(|n| n.get())
.unwrap_or(1);
let trimmed = cores.trim();
if trimmed.is_empty() || trimmed.eq_ignore_ascii_case("all") {
return (0..nproc).collect();
}
let mut cpus = Vec::new();
for token in trimmed.split(',') {
let token = token.trim();
if let Some((a, b)) = token.split_once('-') {
if let (Ok(a), Ok(b)) = (a.trim().parse::<usize>(), b.trim().parse::<usize>()) {
for c in a..=b {
if c < nproc {
cpus.push(c);
}
}
}
} else if let Ok(c) = token.parse::<usize>() {
if c < nproc {
cpus.push(c);
}
}
}
cpus.sort_unstable();
cpus.dedup();
cpus
}
/// Pin the CALLING thread to one logical cpu. Raw syscall — std doesn't
/// expose sched_setaffinity and we don't want a libc dep. Returns false
/// if pinning isn't available (non-x86_64) or the kernel rejected it.
fn pin_self(cpu: usize) -> bool {
let words = cpu / 64 + 1;
let mut mask = vec![0u64; words];
mask[cpu / 64] |= 1u64 << (cpu % 64);
let ret =
unsafe { sched_setaffinity_syscall(0, words * std::mem::size_of::<u64>(), mask.as_ptr()) };
ret == 0
}
#[cfg(target_arch = "x86_64")]
unsafe fn sched_setaffinity_syscall(pid: i32, len: usize, mask: *const u64) -> isize {
const SYS_SCHED_SETAFFINITY: isize = 203;
let ret: isize;
core::arch::asm!(
"syscall",
inlateout("rax") SYS_SCHED_SETAFFINITY => ret,
in("rdi") pid,
in("rsi") len,
in("rdx") mask,
lateout("rcx") _,
lateout("r11") _,
options(nostack)
);
ret
}
#[cfg(not(target_arch = "x86_64"))]
unsafe fn sched_setaffinity_syscall(_pid: i32, _len: usize, _mask: *const u64) -> isize {
-1
}
// ---------------------------------------------------------------------------
// Shared in-process runner skeleton
// ---------------------------------------------------------------------------
/// Measurement body: runs to completion, honors `cancel` between units of
/// work, returns (verdict, score). `Stops early → (Stopped, None)`.
type Worker = fn(duration_secs: u64, cpus: &[usize], cancel: &AtomicBool) -> (Verdict, Score);
/// One struct for both in-process workloads: the trait plumbing (start /
/// is_running / stop / wait / score) is identical; only the worker fn
/// pointer + name differ.
struct InProc {
name: &'static str,
duration_secs: u64,
cores: String,
worker: Worker,
handle: Option<thread::JoinHandle<()>>,
cancel: Arc<AtomicBool>,
done: Arc<AtomicBool>,
result: Arc<Mutex<Option<(Verdict, Score)>>>,
}
impl InProc {
fn new(name: &'static str, duration_secs: u64, cores: String, worker: Worker) -> Self {
InProc {
name,
duration_secs,
cores,
worker,
handle: None,
cancel: Arc::new(AtomicBool::new(false)),
done: Arc::new(AtomicBool::new(false)),
result: Arc::new(Mutex::new(None)),
}
}
}
impl Workload for InProc {
fn name(&self) -> &str {
self.name
}
fn params(&self) -> String {
format!("~{}s cores={}", self.duration_secs, self.cores)
}
fn start(&mut self) -> io::Result<()> {
self.cancel.store(false, Ordering::Relaxed);
self.done.store(false, Ordering::Relaxed);
*self.result.lock().unwrap() = None;
let duration = self.duration_secs;
let cores = self.cores.clone();
let cancel = Arc::clone(&self.cancel);
let done = Arc::clone(&self.done);
let result = Arc::clone(&self.result);
let worker = self.worker;
self.handle = Some(thread::spawn(move || {
let cpus = parse_cpu_list(&cores);
let out = worker(duration, &cpus, &cancel);
*result.lock().unwrap() = Some(out);
done.store(true, Ordering::Release);
}));
Ok(())
}
fn is_running(&mut self) -> bool {
!self.done.load(Ordering::Acquire)
}
fn stop(&mut self) -> io::Result<()> {
self.cancel.store(true, Ordering::Relaxed);
Ok(())
}
fn wait(&mut self) -> Verdict {
match self.handle.take() {
Some(handle) => {
if handle.join().is_err() {
return Verdict::Error("measurement thread panicked".to_string());
}
let guard = self.result.lock().unwrap();
match &*guard {
Some((v, _)) => v.clone(),
None => Verdict::Stopped,
}
}
None => Verdict::Stopped,
}
}
fn score(&mut self) -> Score {
let guard = self.result.lock().unwrap();
match &*guard {
Some((_, s)) => s.clone(),
None => Score::None,
}
}
}
/// Public constructors used by workload::from_spec.
pub fn c2c_latency(duration_secs: u64, cores: String) -> Box<dyn Workload> {
Box::new(InProc::new(
"c2c-latency",
duration_secs,
cores,
measure_c2c,
))
}
pub fn dram_latency(duration_secs: u64, cores: String) -> Box<dyn Workload> {
Box::new(InProc::new(
"dram-latency",
duration_secs,
cores,
measure_dram,
))
}
// ---------------------------------------------------------------------------
// c2c-latency: cache-line ping-pong between pinned core pairs
// ---------------------------------------------------------------------------
/// Round trips per pair. ~2 ms of transfers per pair on this chip; the
/// full upper triangle of 16 logical cpus (120 pairs) lands well under
/// the 30 s default duration.
const C2C_ROUND_TRIPS: u64 = 20_000;
fn measure_c2c(_duration: u64, cpus: &[usize], cancel: &AtomicBool) -> (Verdict, Score) {
if cpus.len() < 2 {
return (
Verdict::Error("c2c-latency needs at least 2 logical cpus".to_string()),
Score::None,
);
}
let mut pairs = Vec::new();
for (ia, &a) in cpus.iter().enumerate() {
for &b in &cpus[ia + 1..] {
if cancel.load(Ordering::Relaxed) {
return (Verdict::Stopped, Score::None);
}
match ping_pong(a, b) {
Ok(ns_per_hop) => pairs.push(C2cPair {
cpu_a: a,
cpu_b: b,
ns_per_hop,
}),
Err(v) => return (v, Score::None),
}
}
}
(Verdict::Clean, Score::C2c { pairs })
}
/// Ping-pong a cache line (one AtomicU64) between cpu `a` and cpu `b`
/// for C2C_ROUND_TRIPS round trips. Each sequence value has exactly one
/// writer, so a plain spin-load that sees a value past its expectation
/// means the line was corrupted or the other side double-wrote — a
/// coherency/clock failure, reported as Verdict::Failed.
fn ping_pong(a: usize, b: usize) -> Result<f64, Verdict> {
let ball = Arc::new(AtomicU64::new(0));
let corrupt = Arc::new(AtomicBool::new(false));
let pin_fail = Arc::new(AtomicBool::new(false));
let ball_a = Arc::clone(&ball);
let ball_b = Arc::clone(&ball);
let corrupt_a = Arc::clone(&corrupt);
let corrupt_b = Arc::clone(&corrupt);
let pf_a = Arc::clone(&pin_fail);
let pf_b = Arc::clone(&pin_fail);
// Initiator: A. Stores 1, waits for 2, stores 3, … timing brackets
// the full 0 → 2N transfer sequence.
let tx = thread::spawn(move || {
if !pin_self(a) {
pf_a.store(true, Ordering::Relaxed);
}
let t0 = Instant::now();
for i in 0..C2C_ROUND_TRIPS {
let expect = 2 * i;
loop {
let v = ball_a.load(Ordering::Relaxed);
if v == expect {
break;
}
if v > expect {
corrupt_a.store(true, Ordering::Relaxed);
return Duration::ZERO;
}
std::hint::spin_loop();
}
ball_a.store(expect + 1, Ordering::Relaxed);
}
// Wait for B's final store so the timing covers 2N transfers.
let final_expect = 2 * C2C_ROUND_TRIPS;
loop {
let v = ball_a.load(Ordering::Relaxed);
if v == final_expect {
break;
}
if v > final_expect {
corrupt_a.store(true, Ordering::Relaxed);
return Duration::ZERO;
}
std::hint::spin_loop();
}
t0.elapsed()
});
// Responder: B. Waits for 1, stores 2, waits for 3, …
let rx = thread::spawn(move || {
if !pin_self(b) {
pf_b.store(true, Ordering::Relaxed);
}
for i in 0..C2C_ROUND_TRIPS {
let expect = 2 * i + 1;
loop {
let v = ball_b.load(Ordering::Relaxed);
if v == expect {
break;
}
if v > expect {
corrupt_b.store(true, Ordering::Relaxed);
return;
}
std::hint::spin_loop();
}
ball_b.store(expect + 1, Ordering::Relaxed);
}
});
let elapsed = match (tx.join(), rx.join()) {
(Ok(e), Ok(())) => e,
_ => return Err(Verdict::Error("ping-pong thread panicked".to_string())),
};
if pin_fail.load(Ordering::Relaxed) {
return Err(Verdict::Error(format!(
"can't pin cpus {a}/{b} (sched_setaffinity failed)"
)));
}
if corrupt.load(Ordering::Relaxed) {
return Err(Verdict::Failed(format!(
"cache-line handoff {a}<->{b} corrupted"
)));
}
let transfers = 2.0 * C2C_ROUND_TRIPS as f64;
Ok(elapsed.as_nanos() as f64 / transfers)
}
// ---------------------------------------------------------------------------
// dram-latency: pointer-chase through the cache hierarchy
// ---------------------------------------------------------------------------
/// Chase buffers, bytes. Sizes straddle the 12600KF hierarchy: L1D 48K
/// per P-core, L2 2MB per P-core, L3 20MB shared. 32K fits L1, 512K fits
/// L2, 8M fits L3, 64M falls through to DRAM.
const DRAM_SIZES: [u64; 4] = [32 * 1024, 512 * 1024, 8 * 1024 * 1024, 64 * 1024 * 1024];
/// Minimum time per size so the slow levels (DRAM) get enough samples.
const DRAM_MIN_PER_SIZE: Duration = Duration::from_millis(200);
fn measure_dram(_duration: u64, cpus: &[usize], cancel: &AtomicBool) -> (Verdict, Score) {
let Some(&cpu) = cpus.first() else {
return (
Verdict::Error("dram-latency needs at least 1 cpu".to_string()),
Score::None,
);
};
// Runs on the InProc worker thread itself — pin it (the thread exits
// with the measurement, so no affinity restore needed).
if !pin_self(cpu) {
return (
Verdict::Error(format!("can't pin cpu {cpu} (sched_setaffinity failed)")),
Score::None,
);
}
let mut levels = Vec::new();
for &size in &DRAM_SIZES {
if cancel.load(Ordering::Relaxed) {
return (Verdict::Stopped, Score::None);
}
let lines = (size / 64) as usize;
let table = build_chase_table(lines);
match chase(&table, lines, cancel) {
Ok(ns) if ns.is_nan() => return (Verdict::Stopped, Score::None), // cancelled mid-size
Ok(ns) => levels.push(DramLevel {
size_kb: size / 1024,
ns_per_access: ns,
}),
Err(v) => return (v, Score::None),
}
}
(Verdict::Clean, Score::Dram { levels })
}
/// Random single cycle over `lines` cache lines, laid out in a u64 table
/// with one u64 payload per 64-byte line. `table[line*8]` = next line.
/// A cycle of length exactly `lines` starting at line 0.
fn build_chase_table(lines: usize) -> Vec<u64> {
let mut order: Vec<u32> = (0..lines as u32).collect();
// Fisher-Yates with a xorshift RNG (deterministic per run — fine for
// a latency measurement; the chase only needs to be irregular).
let mut state: u64 = 0x9E3779B97F4A7C15 ^ lines as u64;
for i in (1..lines).rev() {
state ^= state << 13;
state ^= state >> 7;
state ^= state << 17;
let j = (state % (i as u64 + 1)) as usize;
order.swap(i, j);
}
// order[0] is the head (chase start). link[order[i]] = order[i+1].
let mut table = vec![0u64; lines * 8];
for i in 0..lines {
let line = order[i] as usize;
table[line * 8] = order[(i + 1) % lines] as u64;
}
table
}
/// Chase full laps of the cycle until DRAM_MIN_PER_SIZE elapsed (or
/// cancel). Returns ns/access. The table is ONE cycle covering all
/// `lines` nodes, so starting anywhere, exactly `lines` dependent loads
/// must land back on the start — if a stored address is ever wrong
/// (flipped bit), the lap check fails: that's the memory-stability
/// self-check.
fn chase(table: &[u64], lines: usize, cancel: &AtomicBool) -> Result<f64, Verdict> {
let mut idx = 0u64;
let mut laps = 0u64;
let t0 = Instant::now();
loop {
for _ in 0..lines {
idx = std::hint::black_box(table[idx as usize * 8]);
}
laps += 1;
if idx != 0 {
return Err(Verdict::Failed(
"pointer-chase left the cycle (memory fault?)".to_string(),
));
}
if t0.elapsed() >= DRAM_MIN_PER_SIZE || laps > 100_000_000 {
break;
}
if cancel.load(Ordering::Relaxed) {
return Ok(f64::NAN); // ignored — caller checks cancel separately
}
}
let accesses = laps as f64 * lines as f64;
Ok(t0.elapsed().as_nanos() as f64 / accesses)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn parse_cpu_list_forms() {
let n = thread::available_parallelism()
.map(|n| n.get())
.unwrap_or(1);
assert_eq!(parse_cpu_list("").len(), n);
assert_eq!(parse_cpu_list("all").len(), n);
assert_eq!(parse_cpu_list("2-4"), vec![2, 3, 4]);
assert_eq!(parse_cpu_list("3,1,2-2"), vec![1, 2, 3]); // sort + dedup
assert_eq!(parse_cpu_list("junk"), Vec::<usize>::new());
// Out-of-range clamped away (cpu id >= nproc dropped).
assert_eq!(parse_cpu_list("0-999").len(), n);
}
#[test]
fn chase_table_is_one_cycle() {
let lines = 4096;
let table = build_chase_table(lines);
let mut idx = 0u64;
for _ in 0..lines {
idx = table[idx as usize * 8];
assert!(idx < lines as u64);
}
assert_eq!(
idx, 0,
"single cycle must return to the start after `lines` hops"
);
}
#[test]
fn chase_small_table_measures_ns() {
let lines = 1024;
let table = build_chase_table(lines);
let cancel = AtomicBool::new(false);
let ns = chase(&table, lines, &cancel).unwrap();
// Only a floor + deadlock cap: absolute ns bounds are meaningless
// when the suite runs on a box under load (the dev machine's
// benchmarks), and the hierarchy assertions live in dram_e2e.
assert!(ns > 0.0 && ns < 1e6, "sanity range, got {ns}");
}
#[test]
fn chase_fails_on_desynced_lap() {
let lines = 1024;
let mut table = build_chase_table(lines);
// Rewire one edge to skip a node: the cycle splits, so a
// `lines`-hop lap from 0 no longer lands back on 0.
let a = table[0];
let b = table[a as usize * 8];
table[a as usize * 8] = table[b as usize * 8];
let cancel = AtomicBool::new(false);
let r = chase(&table, lines, &cancel);
assert!(matches!(r, Err(Verdict::Failed(_))), "got {r:?}");
}
#[test]
#[cfg_attr(not(target_arch = "x86_64"), ignore)] // pinning is x86_64-only here
fn ping_pong_between_two_cpus() {
let cpus = parse_cpu_list("0-15");
if cpus.len() < 2 {
return; // single-cpu box (container?) — nothing to test
}
let ns = ping_pong(cpus[0], cpus[1]).expect("clean handoff");
// Floor + deadlock cap only — a preempted responder (this test
// runs on a box that may be mid-benchmark) legitimately pushes
// handoffs way past the tens-of-ns idle range.
assert!(ns > 0.0 && ns < 1e7, "sanity range, got {ns}");
}
#[test]
#[cfg_attr(not(target_arch = "x86_64"), ignore)]
fn c2c_e2e_via_trait() {
use crate::config::WorkloadSpec;
let cpus = parse_cpu_list("0-15");
if cpus.len() < 2 {
return;
}
let spec = WorkloadSpec {
name: "c2c-latency".to_string(),
duration_secs: 1,
cores: "0-3".to_string(),
};
let mut w = crate::workload::from_spec(&spec, std::path::Path::new("bin")).unwrap();
assert_eq!(w.name(), "c2c-latency");
w.start().unwrap();
while w.is_running() {
thread::sleep(Duration::from_millis(25));
}
assert_eq!(w.wait(), Verdict::Clean);
match w.score() {
Score::C2c { pairs } => {
// 4 cpus → 6 upper-triangle pairs.
assert_eq!(pairs.len(), 6);
assert!(pairs.iter().all(|p| p.ns_per_hop > 0.0));
}
other => panic!("expected C2c score, got {other:?}"),
}
}
#[test]
#[cfg_attr(not(target_arch = "x86_64"), ignore)]
fn dram_e2e_via_trait() {
use crate::config::WorkloadSpec;
let spec = WorkloadSpec {
name: "dram-latency".to_string(),
duration_secs: 1,
cores: "0".to_string(),
};
let mut w = crate::workload::from_spec(&spec, std::path::Path::new("bin")).unwrap();
w.start().unwrap();
while w.is_running() {
thread::sleep(Duration::from_millis(50));
}
assert_eq!(w.wait(), Verdict::Clean);
match w.score() {
Score::Dram { levels } => {
assert_eq!(levels.len(), 4);
// DRAM (64 MiB) must be far slower than the L1-sized set.
assert!(levels[0].ns_per_access < levels[3].ns_per_access);
assert!(
levels[3].ns_per_access > 30.0,
"DRAM tier too fast: {levels:?}"
);
}
other => panic!("expected Dram score, got {other:?}"),
}
}
#[test]
fn stop_mid_run_yields_stopped() {
let cpus = parse_cpu_list("0-15");
if cpus.len() < 2 {
return;
}
let mut w = c2c_latency(3600, "0-15".to_string());
w.start().unwrap();
w.stop().unwrap();
while w.is_running() {
thread::sleep(Duration::from_millis(25));
}
assert_eq!(w.wait(), Verdict::Stopped);
}
}