#!/usr/bin/env python3 """Cost matrix for libdpf profile harnesses. Each cell is one named case. Cells are grouped into slices so an optimization pass can rerun only the code it touched. A label is a directory of results. Rerunning a slice under the same label replaces those rows and leaves the rest. Examples: test/profile/cost_matrix.py --list test/profile/cost_matrix.py --label baseline test/profile/cost_matrix.py --label baseline --slice interval test/profile/cost_matrix.py --label baseline --slice memoizer --slice bits test/profile/cost_matrix.py --label baseline --family grotto --slice horner test/profile/cost_matrix.py --label baseline --case interval_u32_L4096 test/profile/cost_matrix.py --label after-horner --slice horner test/profile/cost_matrix.py --compare baseline after-horner Tiers: std default. Benchable party flows, plus every in-process cell. heavy wildcard_single_leaf, long beaver streams, full-domain DCFs, and party flows whose names end in _domain. all std and heavy. smoke party flows that are negative tests, not cost points. Default repeats are per slice (party 3, tables 20, walks 12). --repeat overrides every selected cell. Columns (blank when not instrumented): prg_evals, bytes_sent, bytes_recv_from_p2, bytes_recv_from_peer, rounds (channel exchange barriers; not frames), preprocess_bytes, alloc_bytes, logical_bytes, layout_waste (alloc-logical), payload_sent/recv, wire_overhead (framed bytes minus payload). For p2, bytes_*_from_peer / bytes_*_to_peer sum the p0 and p1 links. """ from __future__ import annotations import argparse import csv import os import subprocess import sys from collections import defaultdict from pathlib import Path LONG_FIELDS = [ "family", "slice", "tier", "case", "role", "items", "repeat", "warmup", "avg_ns", "min_ns", "max_ns", "avg_cycles", "per_item_ns", "out_bytes", "prg_evals", "preprocess_bytes", "alloc_bytes", "logical_bytes", "layout_waste", "bytes_sent", "bytes_recv", "bytes_recv_from_p2", "bytes_recv_from_peer", "bytes_sent_to_p2", "bytes_sent_to_peer", "payload_sent", "payload_recv", "wire_overhead", "rounds", "frames_sent", "frames_recv", "avg_bytes_sent", "avg_frames_sent", "wall_ms", "rc", "sink", ] MATRIX_FIELDS = [ "family", "slice", "tier", "case", "role", "avg_ns", "per_item_ns", "avg_cycles", "out_bytes", "prg_evals", "preprocess_bytes", "alloc_bytes", "logical_bytes", "layout_waste", "bytes_sent", "bytes_recv", "bytes_recv_from_p2", "bytes_recv_from_peer", "rounds", "wire_overhead", "frames_sent", "frames_recv", "wall_ms", "repeat", ] COMPARE_NUM = [ "avg_ns", "prg_evals", "bytes_sent", "bytes_recv_from_p2", "bytes_recv_from_peer", "rounds", "preprocess_bytes", "alloc_bytes", "wire_overhead", ] def default_bin_dir() -> Path: here = Path(__file__).resolve().parent candidate = here.parents[1] / "build" / "test" / "bin" return candidate def timing_for(family: str, slice_name: str, tier: str) -> tuple[int, int]: if tier == "heavy": return 1, 0 if family == "party": return 3, 1 if slice_name in {"window", "principal", "closed", "reduced", "exact"}: return 20, 2 if slice_name == "keygen": return 8, 1 if slice_name in {"memoizer", "bits", "inner-product", "helpers"}: return 12, 2 return 12, 2 def run_capture(cmd: list[str]) -> tuple[int, str, str]: proc = subprocess.run(cmd, text=True, capture_output=True) return proc.returncode, proc.stdout, proc.stderr def parse_list(text: str) -> list[dict[str, str]]: rows = [] reader = csv.DictReader(text.splitlines(), delimiter="\t") for row in reader: if not row.get("case"): continue rows.append(row) return rows def discover(bin_dir: Path) -> list[dict[str, str]]: cells: list[dict[str, str]] = [] for binary in ("profile_eval", "profile_grotto", "profile_party"): path = bin_dir / binary if not path.is_file(): raise SystemExit(f"missing {path}; build the profile targets first") cmd = [str(path), "--list"] if binary == "profile_party": cmd.extend(["--tier", "all"]) rc, out, err = run_capture(cmd) if rc != 0: raise SystemExit(err or out or f"{binary} --list failed") if err.strip(): print(err, file=sys.stderr, end="" if err.endswith("\n") else "\n") for row in parse_list(out): row["driver"] = binary row.setdefault("tier", "std") cells.append(row) heavy_party = [ "wildcard_single_leaf", "beaver_stream_n512", "beaver_stream_n2048", ] for row in cells: if row["family"] == "party" and ( row["case"] in heavy_party or row["case"].startswith("dcf_full_") ): row["tier"] = "heavy" return cells def select_cells(cells, args) -> list[dict[str, str]]: chosen = [] for row in cells: if args.family and row["family"] not in args.family: continue if args.slice and row["slice"] not in args.slice: continue if args.case and row["case"] not in args.case: continue tier = row.get("tier") or "std" if args.tier == "std" and tier != "std": continue if args.tier == "heavy" and tier != "heavy": continue if args.tier == "smoke" and tier != "smoke": continue if args.tier == "all" and tier == "smoke": continue chosen.append(row) return chosen def blank_cost_fields() -> dict[str, str]: return { "prg_evals": "", "preprocess_bytes": "", "alloc_bytes": "", "logical_bytes": "", "layout_waste": "", "bytes_sent": "", "bytes_recv": "", "bytes_recv_from_p2": "", "bytes_recv_from_peer": "", "bytes_sent_to_p2": "", "bytes_sent_to_peer": "", "payload_sent": "", "payload_recv": "", "wire_overhead": "", "rounds": "", "frames_sent": "", "frames_recv": "", "avg_bytes_sent": "", "avg_frames_sent": "", "wall_ms": "", } def parse_inprocess(text: str, tier_of: dict[tuple[str, str], str]) -> list[dict[str, str]]: rows = [] reader = csv.DictReader(text.splitlines(), delimiter="\t") for row in reader: if not row or row.get("family") == "sink" or not row.get("case"): continue # Skip the trailing sink summary line if DictReader mis-parses it. if row.get("family") == "sink": continue out = blank_cost_fields() out.update({ "family": row.get("family") or "", "slice": row.get("slice") or "", "tier": tier_of.get((row.get("family") or "", row.get("case") or ""), "std"), "case": row.get("case") or "", "role": "-", "items": row.get("items") or "", "repeat": row.get("repeat") or "", "warmup": row.get("warmup") or "", "avg_ns": row.get("avg_ns") or "", "min_ns": row.get("min_ns") or "", "max_ns": row.get("max_ns") or "", "avg_cycles": row.get("avg_cycles") or "", "per_item_ns": row.get("per_item_ns") or "", "out_bytes": row.get("out_bytes") or "", "prg_evals": row.get("prg_evals") or "", "preprocess_bytes": row.get("preprocess_bytes") or "", "alloc_bytes": row.get("alloc_bytes") or "", "logical_bytes": row.get("logical_bytes") or "", "layout_waste": row.get("layout_waste") or "", "rc": "0", "sink": row.get("sink") or "", }) rows.append(out) return rows def parse_party(text: str) -> list[dict[str, str]]: rows = [] reader = csv.DictReader(text.splitlines(), delimiter="\t") for row in reader: if not row.get("flow"): continue out = blank_cost_fields() out.update({ "family": row.get("family") or "party", "slice": row.get("slice") or "", "tier": row.get("tier") or "std", "case": row["flow"], "role": row.get("role") or "-", "items": "1", "repeat": "", "warmup": "", "avg_ns": row.get("avg_ns") or "", "min_ns": row.get("min_ns") or "", "max_ns": row.get("max_ns") or "", "avg_cycles": "", "per_item_ns": row.get("avg_ns") or "", "out_bytes": "", "prg_evals": row.get("prg_evals") or "", "bytes_sent": row.get("bytes_sent") or "", "bytes_recv": row.get("bytes_recv") or "", "bytes_recv_from_p2": row.get("bytes_recv_from_p2") or "", "bytes_recv_from_peer": row.get("bytes_recv_from_peer") or "", "bytes_sent_to_p2": row.get("bytes_sent_to_p2") or "", "bytes_sent_to_peer": row.get("bytes_sent_to_peer") or "", "payload_sent": row.get("payload_sent") or "", "payload_recv": row.get("payload_recv") or "", "wire_overhead": row.get("wire_overhead") or "", "rounds": row.get("rounds") or "", "frames_sent": row.get("frames_sent") or "", "frames_recv": row.get("frames_recv") or "", "avg_bytes_sent": row.get("avg_bytes_sent") or "", "avg_frames_sent": row.get("avg_frames_sent") or "", "wall_ms": row.get("wall_ms") or "", "rc": row.get("rc") or "", "sink": "", }) rows.append(out) return rows def fill_timing(rows: list[dict[str, str]], repeat: int, warmup: int) -> None: for row in rows: if not row["repeat"]: row["repeat"] = str(repeat) if not row["warmup"]: row["warmup"] = str(warmup) def invoke_group(bin_dir: Path, driver: str, cases: list[dict[str, str]], repeat: int, warmup: int) -> tuple[int, list[dict[str, str]], str]: cmd = [str(bin_dir / driver), "--repeat", str(repeat), "--warmup", str(warmup)] for case in cases: cmd.extend(["--case", case["case"]]) rc, out, err = run_capture(cmd) tier_of = {(c["family"], c["case"]): c.get("tier") or "std" for c in cases} if driver == "profile_party": parsed = parse_party(out) else: parsed = parse_inprocess(out, tier_of) fill_timing(parsed, repeat, warmup) return rc, parsed, err def write_tsv(path: Path, fields: list[str], rows: list[dict[str, str]]) -> None: path.parent.mkdir(parents=True, exist_ok=True) with path.open("w", newline="") as fh: writer = csv.DictWriter(fh, fieldnames=fields, delimiter="\t", extrasaction="ignore") writer.writeheader() for row in rows: writer.writerow(row) def read_tsv(path: Path) -> list[dict[str, str]]: if not path.is_file(): return [] with path.open(newline="") as fh: return list(csv.DictReader(fh, delimiter="\t")) def merge_rows(old: list[dict[str, str]], new: list[dict[str, str]]) -> list[dict[str, str]]: replaced = {(row["family"], row["slice"], row["case"], row["role"]) for row in new} kept = [ row for row in old if (row.get("family"), row.get("slice"), row.get("case"), row.get("role")) not in replaced ] # Older long.tsv rows may lack new columns; normalize on merge. merged = [] for row in kept + new: full = blank_cost_fields() full.update({k: "" for k in LONG_FIELDS}) full.update({k: v for k, v in row.items() if v is not None}) merged.append(full) return merged def matrix_view(rows: list[dict[str, str]]) -> list[dict[str, str]]: view = [] for row in rows: view.append({key: row.get(key, "") for key in MATRIX_FIELDS}) view.sort(key=lambda r: (r["family"], r["slice"], r["case"], r["role"])) return view def print_catalog(cells: list[dict[str, str]]) -> None: groups: dict[tuple[str, str, str], int] = defaultdict(int) for row in cells: groups[(row["family"], row["slice"], row.get("tier") or "std")] += 1 print("family\tslice\ttier\tcases") for (family, slice_name, tier), count in sorted(groups.items()): print(f"{family}\t{slice_name}\t{tier}\t{count}") print(f"# {len(cells)} cases in {len(groups)} slices") def compare_labels(out_root: Path, left: str, right: str) -> int: a_rows = read_tsv(out_root / left / "matrix.tsv") b_rows = read_tsv(out_root / right / "matrix.tsv") if not a_rows or not b_rows: raise SystemExit(f"need matrix.tsv in both {left} and {right}") b_index = { (row["family"], row["slice"], row["case"], row["role"]): row for row in b_rows } header = ["family", "slice", "case", "role"] for col in COMPARE_NUM: header.extend([f"{col}_a", f"{col}_b", f"delta_{col}"]) print("\t".join(header)) missing = 0 for row in a_rows: key = (row["family"], row["slice"], row["case"], row["role"]) other = b_index.get(key) if other is None: missing += 1 continue parts = [row["family"], row["slice"], row["case"], row["role"]] for col in COMPARE_NUM: av_s = row.get(col, "") or "" bv_s = other.get(col, "") or "" try: av = float(av_s) bv = float(bv_s) delta = f"{bv - av:.0f}" except ValueError: delta = "" parts.extend([av_s, bv_s, delta]) print("\t".join(parts)) if missing: print(f"# {missing} rows in {left} have no match in {right}", file=sys.stderr) return 0 def main() -> int: parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument("--bin-dir", type=Path, default=default_bin_dir()) parser.add_argument("--out", type=Path, default=None, help="directory that holds one subdirectory per label") parser.add_argument("--label", default="baseline") parser.add_argument("--family", action="append", default=[]) parser.add_argument("--slice", action="append", default=[]) parser.add_argument("--case", action="append", default=[]) parser.add_argument("--tier", choices=("std", "heavy", "all", "smoke"), default="std") parser.add_argument("--repeat", type=int, default=None) parser.add_argument("--warmup", type=int, default=None) parser.add_argument("--list", action="store_true") parser.add_argument("--compare", nargs=2, metavar=("LABEL_A", "LABEL_B")) args = parser.parse_args() out_root = args.out if out_root is None: out_root = args.bin_dir.parent / "cost-matrix" if args.compare: return compare_labels(out_root, args.compare[0], args.compare[1]) cells = discover(args.bin_dir) chosen = select_cells(cells, args) if args.list: print_catalog(chosen) return 0 if not chosen: print("no cells match", file=sys.stderr) return 2 by_driver: dict[str, list[dict[str, str]]] = defaultdict(list) for row in chosen: by_driver[row["driver"]].append(row) fresh: list[dict[str, str]] = [] failures = 0 for driver, rows in by_driver.items(): buckets: dict[tuple[str, str, str], list[dict[str, str]]] = defaultdict(list) for row in rows: buckets[(row["family"], row["slice"], row.get("tier") or "std")].append(row) for (family, slice_name, tier), group in buckets.items(): repeat, warmup = timing_for(family, slice_name, tier) if args.repeat is not None: repeat = args.repeat if args.warmup is not None: warmup = args.warmup print(f"# {driver} {family}/{slice_name} tier={tier} " f"cases={len(group)} repeat={repeat} warmup={warmup}", file=sys.stderr) rc, parsed, err = invoke_group(args.bin_dir, driver, group, repeat, warmup) if err.strip(): print(err, file=sys.stderr, end="" if err.endswith("\n") else "\n") got = {(row["case"], row["role"]) for row in parsed} expected = {row["case"] for row in group} have_cases = {case for case, _role in got} missing = sorted(expected - have_cases) if missing: print(f"# missing results: {', '.join(missing)}", file=sys.stderr) failures += len(missing) if rc != 0: failures += 1 fresh.extend(parsed) label_dir = out_root / args.label merged = merge_rows(read_tsv(label_dir / "long.tsv"), fresh) write_tsv(label_dir / "long.tsv", LONG_FIELDS, merged) write_tsv(label_dir / "matrix.tsv", MATRIX_FIELDS, matrix_view(merged)) print(f"# wrote {label_dir / 'matrix.tsv'} ({len(merged)} rows)", file=sys.stderr) return 1 if failures else 0 if __name__ == "__main__": sys.exit(main())