ryan/libdpf
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions
libdpf/test/profile/cost_matrix.py
Ryan Henry 0d22946a0e Checkpoint the party/runtime stack before share-program and malicious-mode work. 
Ship the TLS mesh, composer, Beaver/Yao/leaf MPC, prep/online paths, apps, and docs so the tree is pushable before elevating share_expr, security_mode, and prep resume.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-09-28 05:59:19 -06:00

505 lines
17 KiB
Python
Executable file
Raw Blame History

#!/usr/bin/env python3
"""Cost matrix for libdpf profile harnesses.
Each cell is one named case. Cells are grouped into slices so an optimization
pass can rerun only the code it touched. A label is a directory of results.
Rerunning a slice under the same label replaces those rows and leaves the rest.
Examples:
test/profile/cost_matrix.py --list
test/profile/cost_matrix.py --label baseline
test/profile/cost_matrix.py --label baseline --slice interval
test/profile/cost_matrix.py --label baseline --slice memoizer --slice bits
test/profile/cost_matrix.py --label baseline --family grotto --slice horner
test/profile/cost_matrix.py --label baseline --case interval_u32_L4096
test/profile/cost_matrix.py --label after-horner --slice horner
test/profile/cost_matrix.py --compare baseline after-horner
Tiers:
std default. Benchable party flows, plus every in-process cell.
heavy wildcard_single_leaf, long beaver streams, full-domain DCFs,
and party flows whose names end in _domain.
all std and heavy.
smoke party flows that are negative tests, not cost points.
Default repeats are per slice (party 3, tables 20, walks 12). --repeat
overrides every selected cell.
Columns (blank when not instrumented):
prg_evals, bytes_sent, bytes_recv_from_p2, bytes_recv_from_peer,
rounds (channel exchange barriers; not frames), preprocess_bytes,
alloc_bytes, logical_bytes, layout_waste (alloc-logical),
payload_sent/recv, wire_overhead (framed bytes minus payload).
For p2, bytes_*_from_peer / bytes_*_to_peer sum the p0 and p1 links.
"""
from __future__ import annotations
import argparse
import csv
import os
import subprocess
import sys
from collections import defaultdict
from pathlib import Path
LONG_FIELDS = [
"family",
"slice",
"tier",
"case",
"role",
"items",
"repeat",
"warmup",
"avg_ns",
"min_ns",
"max_ns",
"avg_cycles",
"per_item_ns",
"out_bytes",
"prg_evals",
"preprocess_bytes",
"alloc_bytes",
"logical_bytes",
"layout_waste",
"bytes_sent",
"bytes_recv",
"bytes_recv_from_p2",
"bytes_recv_from_peer",
"bytes_sent_to_p2",
"bytes_sent_to_peer",
"payload_sent",
"payload_recv",
"wire_overhead",
"rounds",
"frames_sent",
"frames_recv",
"avg_bytes_sent",
"avg_frames_sent",
"wall_ms",
"rc",
"sink",
]
MATRIX_FIELDS = [
"family",
"slice",
"tier",
"case",
"role",
"avg_ns",
"per_item_ns",
"avg_cycles",
"out_bytes",
"prg_evals",
"preprocess_bytes",
"alloc_bytes",
"logical_bytes",
"layout_waste",
"bytes_sent",
"bytes_recv",
"bytes_recv_from_p2",
"bytes_recv_from_peer",
"rounds",
"wire_overhead",
"frames_sent",
"frames_recv",
"wall_ms",
"repeat",
]
COMPARE_NUM = [
"avg_ns",
"prg_evals",
"bytes_sent",
"bytes_recv_from_p2",
"bytes_recv_from_peer",
"rounds",
"preprocess_bytes",
"alloc_bytes",
"wire_overhead",
]
def default_bin_dir() -> Path:
here = Path(__file__).resolve().parent
candidate = here.parents[1] / "build" / "test" / "bin"
return candidate
def timing_for(family: str, slice_name: str, tier: str) -> tuple[int, int]:
if tier == "heavy":
return 1, 0
if family == "party":
return 3, 1
if slice_name in {"window", "principal", "closed", "reduced", "exact"}:
return 20, 2
if slice_name == "keygen":
return 8, 1
if slice_name in {"memoizer", "bits", "inner-product", "helpers"}:
return 12, 2
return 12, 2
def run_capture(cmd: list[str]) -> tuple[int, str, str]:
proc = subprocess.run(cmd, text=True, capture_output=True)
return proc.returncode, proc.stdout, proc.stderr
def parse_list(text: str) -> list[dict[str, str]]:
rows = []
reader = csv.DictReader(text.splitlines(), delimiter="\t")
for row in reader:
if not row.get("case"):
continue
rows.append(row)
return rows
def discover(bin_dir: Path) -> list[dict[str, str]]:
cells: list[dict[str, str]] = []
for binary in ("profile_eval", "profile_grotto", "profile_party"):
path = bin_dir / binary
if not path.is_file():
raise SystemExit(f"missing {path}; build the profile targets first")
cmd = [str(path), "--list"]
if binary == "profile_party":
cmd.extend(["--tier", "all"])
rc, out, err = run_capture(cmd)
if rc != 0:
raise SystemExit(err or out or f"{binary} --list failed")
if err.strip():
print(err, file=sys.stderr, end="" if err.endswith("\n") else "\n")
for row in parse_list(out):
row["driver"] = binary
row.setdefault("tier", "std")
cells.append(row)
heavy_party = [
"wildcard_single_leaf",
"beaver_stream_n512",
"beaver_stream_n2048",
]
for row in cells:
if row["family"] == "party" and (
row["case"] in heavy_party or row["case"].startswith("dcf_full_")
):
row["tier"] = "heavy"
return cells
def select_cells(cells, args) -> list[dict[str, str]]:
chosen = []
for row in cells:
if args.family and row["family"] not in args.family:
continue
if args.slice and row["slice"] not in args.slice:
continue
if args.case and row["case"] not in args.case:
continue
tier = row.get("tier") or "std"
if args.tier == "std" and tier != "std":
continue
if args.tier == "heavy" and tier != "heavy":
continue
if args.tier == "smoke" and tier != "smoke":
continue
if args.tier == "all" and tier == "smoke":
continue
chosen.append(row)
return chosen
def blank_cost_fields() -> dict[str, str]:
return {
"prg_evals": "",
"preprocess_bytes": "",
"alloc_bytes": "",
"logical_bytes": "",
"layout_waste": "",
"bytes_sent": "",
"bytes_recv": "",
"bytes_recv_from_p2": "",
"bytes_recv_from_peer": "",
"bytes_sent_to_p2": "",
"bytes_sent_to_peer": "",
"payload_sent": "",
"payload_recv": "",
"wire_overhead": "",
"rounds": "",
"frames_sent": "",
"frames_recv": "",
"avg_bytes_sent": "",
"avg_frames_sent": "",
"wall_ms": "",
}
def parse_inprocess(text: str, tier_of: dict[tuple[str, str], str]) -> list[dict[str, str]]:
rows = []
reader = csv.DictReader(text.splitlines(), delimiter="\t")
for row in reader:
if not row or row.get("family") == "sink" or not row.get("case"):
continue
# Skip the trailing sink summary line if DictReader mis-parses it.
if row.get("family") == "sink":
continue
out = blank_cost_fields()
out.update({
"family": row.get("family") or "",
"slice": row.get("slice") or "",
"tier": tier_of.get((row.get("family") or "", row.get("case") or ""), "std"),
"case": row.get("case") or "",
"role": "-",
"items": row.get("items") or "",
"repeat": row.get("repeat") or "",
"warmup": row.get("warmup") or "",
"avg_ns": row.get("avg_ns") or "",
"min_ns": row.get("min_ns") or "",
"max_ns": row.get("max_ns") or "",
"avg_cycles": row.get("avg_cycles") or "",
"per_item_ns": row.get("per_item_ns") or "",
"out_bytes": row.get("out_bytes") or "",
"prg_evals": row.get("prg_evals") or "",
"preprocess_bytes": row.get("preprocess_bytes") or "",
"alloc_bytes": row.get("alloc_bytes") or "",
"logical_bytes": row.get("logical_bytes") or "",
"layout_waste": row.get("layout_waste") or "",
"rc": "0",
"sink": row.get("sink") or "",
})
rows.append(out)
return rows
def parse_party(text: str) -> list[dict[str, str]]:
rows = []
reader = csv.DictReader(text.splitlines(), delimiter="\t")
for row in reader:
if not row.get("flow"):
continue
out = blank_cost_fields()
out.update({
"family": row.get("family") or "party",
"slice": row.get("slice") or "",
"tier": row.get("tier") or "std",
"case": row["flow"],
"role": row.get("role") or "-",
"items": "1",
"repeat": "",
"warmup": "",
"avg_ns": row.get("avg_ns") or "",
"min_ns": row.get("min_ns") or "",
"max_ns": row.get("max_ns") or "",
"avg_cycles": "",
"per_item_ns": row.get("avg_ns") or "",
"out_bytes": "",
"prg_evals": row.get("prg_evals") or "",
"bytes_sent": row.get("bytes_sent") or "",
"bytes_recv": row.get("bytes_recv") or "",
"bytes_recv_from_p2": row.get("bytes_recv_from_p2") or "",
"bytes_recv_from_peer": row.get("bytes_recv_from_peer") or "",
"bytes_sent_to_p2": row.get("bytes_sent_to_p2") or "",
"bytes_sent_to_peer": row.get("bytes_sent_to_peer") or "",
"payload_sent": row.get("payload_sent") or "",
"payload_recv": row.get("payload_recv") or "",
"wire_overhead": row.get("wire_overhead") or "",
"rounds": row.get("rounds") or "",
"frames_sent": row.get("frames_sent") or "",
"frames_recv": row.get("frames_recv") or "",
"avg_bytes_sent": row.get("avg_bytes_sent") or "",
"avg_frames_sent": row.get("avg_frames_sent") or "",
"wall_ms": row.get("wall_ms") or "",
"rc": row.get("rc") or "",
"sink": "",
})
rows.append(out)
return rows
def fill_timing(rows: list[dict[str, str]], repeat: int, warmup: int) -> None:
for row in rows:
if not row["repeat"]:
row["repeat"] = str(repeat)
if not row["warmup"]:
row["warmup"] = str(warmup)
def invoke_group(bin_dir: Path, driver: str, cases: list[dict[str, str]],
repeat: int, warmup: int) -> tuple[int, list[dict[str, str]], str]:
cmd = [str(bin_dir / driver), "--repeat", str(repeat), "--warmup", str(warmup)]
for case in cases:
cmd.extend(["--case", case["case"]])
rc, out, err = run_capture(cmd)
tier_of = {(c["family"], c["case"]): c.get("tier") or "std" for c in cases}
if driver == "profile_party":
parsed = parse_party(out)
else:
parsed = parse_inprocess(out, tier_of)
fill_timing(parsed, repeat, warmup)
return rc, parsed, err
def write_tsv(path: Path, fields: list[str], rows: list[dict[str, str]]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
with path.open("w", newline="") as fh:
writer = csv.DictWriter(fh, fieldnames=fields, delimiter="\t", extrasaction="ignore")
writer.writeheader()
for row in rows:
writer.writerow(row)
def read_tsv(path: Path) -> list[dict[str, str]]:
if not path.is_file():
return []
with path.open(newline="") as fh:
return list(csv.DictReader(fh, delimiter="\t"))
def merge_rows(old: list[dict[str, str]], new: list[dict[str, str]]) -> list[dict[str, str]]:
replaced = {(row["family"], row["slice"], row["case"], row["role"]) for row in new}
kept = [
row for row in old
if (row.get("family"), row.get("slice"), row.get("case"), row.get("role")) not in replaced
]
# Older long.tsv rows may lack new columns; normalize on merge.
merged = []
for row in kept + new:
full = blank_cost_fields()
full.update({k: "" for k in LONG_FIELDS})
full.update({k: v for k, v in row.items() if v is not None})
merged.append(full)
return merged
def matrix_view(rows: list[dict[str, str]]) -> list[dict[str, str]]:
view = []
for row in rows:
view.append({key: row.get(key, "") for key in MATRIX_FIELDS})
view.sort(key=lambda r: (r["family"], r["slice"], r["case"], r["role"]))
return view
def print_catalog(cells: list[dict[str, str]]) -> None:
groups: dict[tuple[str, str, str], int] = defaultdict(int)
for row in cells:
groups[(row["family"], row["slice"], row.get("tier") or "std")] += 1
print("family\tslice\ttier\tcases")
for (family, slice_name, tier), count in sorted(groups.items()):
print(f"{family}\t{slice_name}\t{tier}\t{count}")
print(f"# {len(cells)} cases in {len(groups)} slices")
def compare_labels(out_root: Path, left: str, right: str) -> int:
a_rows = read_tsv(out_root / left / "matrix.tsv")
b_rows = read_tsv(out_root / right / "matrix.tsv")
if not a_rows or not b_rows:
raise SystemExit(f"need matrix.tsv in both {left} and {right}")
b_index = {
(row["family"], row["slice"], row["case"], row["role"]): row
for row in b_rows
}
header = ["family", "slice", "case", "role"]
for col in COMPARE_NUM:
header.extend([f"{col}_a", f"{col}_b", f"delta_{col}"])
print("\t".join(header))
missing = 0
for row in a_rows:
key = (row["family"], row["slice"], row["case"], row["role"])
other = b_index.get(key)
if other is None:
missing += 1
continue
parts = [row["family"], row["slice"], row["case"], row["role"]]
for col in COMPARE_NUM:
av_s = row.get(col, "") or ""
bv_s = other.get(col, "") or ""
try:
av = float(av_s)
bv = float(bv_s)
delta = f"{bv - av:.0f}"
except ValueError:
delta = ""
parts.extend([av_s, bv_s, delta])
print("\t".join(parts))
if missing:
print(f"# {missing} rows in {left} have no match in {right}", file=sys.stderr)
return 0
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("--bin-dir", type=Path, default=default_bin_dir())
parser.add_argument("--out", type=Path, default=None,
help="directory that holds one subdirectory per label")
parser.add_argument("--label", default="baseline")
parser.add_argument("--family", action="append", default=[])
parser.add_argument("--slice", action="append", default=[])
parser.add_argument("--case", action="append", default=[])
parser.add_argument("--tier", choices=("std", "heavy", "all", "smoke"), default="std")
parser.add_argument("--repeat", type=int, default=None)
parser.add_argument("--warmup", type=int, default=None)
parser.add_argument("--list", action="store_true")
parser.add_argument("--compare", nargs=2, metavar=("LABEL_A", "LABEL_B"))
args = parser.parse_args()
out_root = args.out
if out_root is None:
out_root = args.bin_dir.parent / "cost-matrix"
if args.compare:
return compare_labels(out_root, args.compare[0], args.compare[1])
cells = discover(args.bin_dir)
chosen = select_cells(cells, args)
if args.list:
print_catalog(chosen)
return 0
if not chosen:
print("no cells match", file=sys.stderr)
return 2
by_driver: dict[str, list[dict[str, str]]] = defaultdict(list)
for row in chosen:
by_driver[row["driver"]].append(row)
fresh: list[dict[str, str]] = []
failures = 0
for driver, rows in by_driver.items():
buckets: dict[tuple[str, str, str], list[dict[str, str]]] = defaultdict(list)
for row in rows:
buckets[(row["family"], row["slice"], row.get("tier") or "std")].append(row)
for (family, slice_name, tier), group in buckets.items():
repeat, warmup = timing_for(family, slice_name, tier)
if args.repeat is not None:
repeat = args.repeat
if args.warmup is not None:
warmup = args.warmup
print(f"# {driver} {family}/{slice_name} tier={tier} "
f"cases={len(group)} repeat={repeat} warmup={warmup}",
file=sys.stderr)
rc, parsed, err = invoke_group(args.bin_dir, driver, group, repeat, warmup)
if err.strip():
print(err, file=sys.stderr, end="" if err.endswith("\n") else "\n")
got = {(row["case"], row["role"]) for row in parsed}
expected = {row["case"] for row in group}
have_cases = {case for case, _role in got}
missing = sorted(expected - have_cases)
if missing:
print(f"# missing results: {', '.join(missing)}", file=sys.stderr)
failures += len(missing)
if rc != 0:
failures += 1
fresh.extend(parsed)
label_dir = out_root / args.label
merged = merge_rows(read_tsv(label_dir / "long.tsv"), fresh)
write_tsv(label_dir / "long.tsv", LONG_FIELDS, merged)
write_tsv(label_dir / "matrix.tsv", MATRIX_FIELDS, matrix_view(merged))
print(f"# wrote {label_dir / 'matrix.tsv'} ({len(merged)} rows)", file=sys.stderr)
return 1 if failures else 0
if __name__ == "__main__":
sys.exit(main())
Reference in a new issue View git blame Copy permalink