506 lines
17 KiB
Python
506 lines
17 KiB
Python
|
#!/usr/bin/env python3
|
||
|
|
"""Cost matrix for libdpf profile harnesses.
|
||
|
|
|
||
|
|
Each cell is one named case. Cells are grouped into slices so an optimization
|
||
|
|
pass can rerun only the code it touched. A label is a directory of results.
|
||
|
|
Rerunning a slice under the same label replaces those rows and leaves the rest.
|
||
|
|
|
||
|
|
Examples:
|
||
|
|
test/profile/cost_matrix.py --list
|
||
|
|
test/profile/cost_matrix.py --label baseline
|
||
|
|
test/profile/cost_matrix.py --label baseline --slice interval
|
||
|
|
test/profile/cost_matrix.py --label baseline --slice memoizer --slice bits
|
||
|
|
test/profile/cost_matrix.py --label baseline --family grotto --slice horner
|
||
|
|
test/profile/cost_matrix.py --label baseline --case interval_u32_L4096
|
||
|
|
test/profile/cost_matrix.py --label after-horner --slice horner
|
||
|
|
test/profile/cost_matrix.py --compare baseline after-horner
|
||
|
|
|
||
|
|
Tiers:
|
||
|
|
std default. Benchable party flows, plus every in-process cell.
|
||
|
|
heavy wildcard_single_leaf, long beaver streams, full-domain DCFs,
|
||
|
|
and party flows whose names end in _domain.
|
||
|
|
all std and heavy.
|
||
|
|
smoke party flows that are negative tests, not cost points.
|
||
|
|
|
||
|
|
Default repeats are per slice (party 3, tables 20, walks 12). --repeat
|
||
|
|
overrides every selected cell.
|
||
|
|
|
||
|
|
Columns (blank when not instrumented):
|
||
|
|
prg_evals, bytes_sent, bytes_recv_from_p2, bytes_recv_from_peer,
|
||
|
|
rounds (channel exchange barriers; not frames), preprocess_bytes,
|
||
|
|
alloc_bytes, logical_bytes, layout_waste (alloc-logical),
|
||
|
|
payload_sent/recv, wire_overhead (framed bytes minus payload).
|
||
|
|
For p2, bytes_*_from_peer / bytes_*_to_peer sum the p0 and p1 links.
|
||
|
|
"""
|
||
|
|
|
||
|
|
from __future__ import annotations
|
||
|
|
|
||
|
|
import argparse
|
||
|
|
import csv
|
||
|
|
import os
|
||
|
|
import subprocess
|
||
|
|
import sys
|
||
|
|
from collections import defaultdict
|
||
|
|
from pathlib import Path
|
||
|
|
|
||
|
|
LONG_FIELDS = [
|
||
|
|
"family",
|
||
|
|
"slice",
|
||
|
|
"tier",
|
||
|
|
"case",
|
||
|
|
"role",
|
||
|
|
"items",
|
||
|
|
"repeat",
|
||
|
|
"warmup",
|
||
|
|
"avg_ns",
|
||
|
|
"min_ns",
|
||
|
|
"max_ns",
|
||
|
|
"avg_cycles",
|
||
|
|
"per_item_ns",
|
||
|
|
"out_bytes",
|
||
|
|
"prg_evals",
|
||
|
|
"preprocess_bytes",
|
||
|
|
"alloc_bytes",
|
||
|
|
"logical_bytes",
|
||
|
|
"layout_waste",
|
||
|
|
"bytes_sent",
|
||
|
|
"bytes_recv",
|
||
|
|
"bytes_recv_from_p2",
|
||
|
|
"bytes_recv_from_peer",
|
||
|
|
"bytes_sent_to_p2",
|
||
|
|
"bytes_sent_to_peer",
|
||
|
|
"payload_sent",
|
||
|
|
"payload_recv",
|
||
|
|
"wire_overhead",
|
||
|
|
"rounds",
|
||
|
|
"frames_sent",
|
||
|
|
"frames_recv",
|
||
|
|
"avg_bytes_sent",
|
||
|
|
"avg_frames_sent",
|
||
|
|
"wall_ms",
|
||
|
|
"rc",
|
||
|
|
"sink",
|
||
|
|
]
|
||
|
|
|
||
|
|
MATRIX_FIELDS = [
|
||
|
|
"family",
|
||
|
|
"slice",
|
||
|
|
"tier",
|
||
|
|
"case",
|
||
|
|
"role",
|
||
|
|
"avg_ns",
|
||
|
|
"per_item_ns",
|
||
|
|
"avg_cycles",
|
||
|
|
"out_bytes",
|
||
|
|
"prg_evals",
|
||
|
|
"preprocess_bytes",
|
||
|
|
"alloc_bytes",
|
||
|
|
"logical_bytes",
|
||
|
|
"layout_waste",
|
||
|
|
"bytes_sent",
|
||
|
|
"bytes_recv",
|
||
|
|
"bytes_recv_from_p2",
|
||
|
|
"bytes_recv_from_peer",
|
||
|
|
"rounds",
|
||
|
|
"wire_overhead",
|
||
|
|
"frames_sent",
|
||
|
|
"frames_recv",
|
||
|
|
"wall_ms",
|
||
|
|
"repeat",
|
||
|
|
]
|
||
|
|
|
||
|
|
COMPARE_NUM = [
|
||
|
|
"avg_ns",
|
||
|
|
"prg_evals",
|
||
|
|
"bytes_sent",
|
||
|
|
"bytes_recv_from_p2",
|
||
|
|
"bytes_recv_from_peer",
|
||
|
|
"rounds",
|
||
|
|
"preprocess_bytes",
|
||
|
|
"alloc_bytes",
|
||
|
|
"wire_overhead",
|
||
|
|
]
|
||
|
|
|
||
|
|
|
||
|
|
def default_bin_dir() -> Path:
|
||
|
|
here = Path(__file__).resolve().parent
|
||
|
|
candidate = here.parents[1] / "build" / "test" / "bin"
|
||
|
|
return candidate
|
||
|
|
|
||
|
|
|
||
|
|
def timing_for(family: str, slice_name: str, tier: str) -> tuple[int, int]:
|
||
|
|
if tier == "heavy":
|
||
|
|
return 1, 0
|
||
|
|
if family == "party":
|
||
|
|
return 3, 1
|
||
|
|
if slice_name in {"window", "principal", "closed", "reduced", "exact"}:
|
||
|
|
return 20, 2
|
||
|
|
if slice_name == "keygen":
|
||
|
|
return 8, 1
|
||
|
|
if slice_name in {"memoizer", "bits", "inner-product", "helpers"}:
|
||
|
|
return 12, 2
|
||
|
|
return 12, 2
|
||
|
|
|
||
|
|
|
||
|
|
def run_capture(cmd: list[str]) -> tuple[int, str, str]:
|
||
|
|
proc = subprocess.run(cmd, text=True, capture_output=True)
|
||
|
|
return proc.returncode, proc.stdout, proc.stderr
|
||
|
|
|
||
|
|
|
||
|
|
def parse_list(text: str) -> list[dict[str, str]]:
|
||
|
|
rows = []
|
||
|
|
reader = csv.DictReader(text.splitlines(), delimiter="\t")
|
||
|
|
for row in reader:
|
||
|
|
if not row.get("case"):
|
||
|
|
continue
|
||
|
|
rows.append(row)
|
||
|
|
return rows
|
||
|
|
|
||
|
|
|
||
|
|
def discover(bin_dir: Path) -> list[dict[str, str]]:
|
||
|
|
cells: list[dict[str, str]] = []
|
||
|
|
for binary in ("profile_eval", "profile_grotto", "profile_party"):
|
||
|
|
path = bin_dir / binary
|
||
|
|
if not path.is_file():
|
||
|
|
raise SystemExit(f"missing {path}; build the profile targets first")
|
||
|
|
cmd = [str(path), "--list"]
|
||
|
|
if binary == "profile_party":
|
||
|
|
cmd.extend(["--tier", "all"])
|
||
|
|
rc, out, err = run_capture(cmd)
|
||
|
|
if rc != 0:
|
||
|
|
raise SystemExit(err or out or f"{binary} --list failed")
|
||
|
|
if err.strip():
|
||
|
|
print(err, file=sys.stderr, end="" if err.endswith("\n") else "\n")
|
||
|
|
for row in parse_list(out):
|
||
|
|
row["driver"] = binary
|
||
|
|
row.setdefault("tier", "std")
|
||
|
|
cells.append(row)
|
||
|
|
heavy_party = [
|
||
|
|
"wildcard_single_leaf",
|
||
|
|
"beaver_stream_n512",
|
||
|
|
"beaver_stream_n2048",
|
||
|
|
]
|
||
|
|
for row in cells:
|
||
|
|
if row["family"] == "party" and (
|
||
|
|
row["case"] in heavy_party or row["case"].startswith("dcf_full_")
|
||
|
|
):
|
||
|
|
row["tier"] = "heavy"
|
||
|
|
return cells
|
||
|
|
|
||
|
|
|
||
|
|
def select_cells(cells, args) -> list[dict[str, str]]:
|
||
|
|
chosen = []
|
||
|
|
for row in cells:
|
||
|
|
if args.family and row["family"] not in args.family:
|
||
|
|
continue
|
||
|
|
if args.slice and row["slice"] not in args.slice:
|
||
|
|
continue
|
||
|
|
if args.case and row["case"] not in args.case:
|
||
|
|
continue
|
||
|
|
tier = row.get("tier") or "std"
|
||
|
|
if args.tier == "std" and tier != "std":
|
||
|
|
continue
|
||
|
|
if args.tier == "heavy" and tier != "heavy":
|
||
|
|
continue
|
||
|
|
if args.tier == "smoke" and tier != "smoke":
|
||
|
|
continue
|
||
|
|
if args.tier == "all" and tier == "smoke":
|
||
|
|
continue
|
||
|
|
chosen.append(row)
|
||
|
|
return chosen
|
||
|
|
|
||
|
|
|
||
|
|
def blank_cost_fields() -> dict[str, str]:
|
||
|
|
return {
|
||
|
|
"prg_evals": "",
|
||
|
|
"preprocess_bytes": "",
|
||
|
|
"alloc_bytes": "",
|
||
|
|
"logical_bytes": "",
|
||
|
|
"layout_waste": "",
|
||
|
|
"bytes_sent": "",
|
||
|
|
"bytes_recv": "",
|
||
|
|
"bytes_recv_from_p2": "",
|
||
|
|
"bytes_recv_from_peer": "",
|
||
|
|
"bytes_sent_to_p2": "",
|
||
|
|
"bytes_sent_to_peer": "",
|
||
|
|
"payload_sent": "",
|
||
|
|
"payload_recv": "",
|
||
|
|
"wire_overhead": "",
|
||
|
|
"rounds": "",
|
||
|
|
"frames_sent": "",
|
||
|
|
"frames_recv": "",
|
||
|
|
"avg_bytes_sent": "",
|
||
|
|
"avg_frames_sent": "",
|
||
|
|
"wall_ms": "",
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
def parse_inprocess(text: str, tier_of: dict[tuple[str, str], str]) -> list[dict[str, str]]:
|
||
|
|
rows = []
|
||
|
|
reader = csv.DictReader(text.splitlines(), delimiter="\t")
|
||
|
|
for row in reader:
|
||
|
|
if not row or row.get("family") == "sink" or not row.get("case"):
|
||
|
|
continue
|
||
|
|
# Skip the trailing sink summary line if DictReader mis-parses it.
|
||
|
|
if row.get("family") == "sink":
|
||
|
|
continue
|
||
|
|
out = blank_cost_fields()
|
||
|
|
out.update({
|
||
|
|
"family": row.get("family") or "",
|
||
|
|
"slice": row.get("slice") or "",
|
||
|
|
"tier": tier_of.get((row.get("family") or "", row.get("case") or ""), "std"),
|
||
|
|
"case": row.get("case") or "",
|
||
|
|
"role": "-",
|
||
|
|
"items": row.get("items") or "",
|
||
|
|
"repeat": row.get("repeat") or "",
|
||
|
|
"warmup": row.get("warmup") or "",
|
||
|
|
"avg_ns": row.get("avg_ns") or "",
|
||
|
|
"min_ns": row.get("min_ns") or "",
|
||
|
|
"max_ns": row.get("max_ns") or "",
|
||
|
|
"avg_cycles": row.get("avg_cycles") or "",
|
||
|
|
"per_item_ns": row.get("per_item_ns") or "",
|
||
|
|
"out_bytes": row.get("out_bytes") or "",
|
||
|
|
"prg_evals": row.get("prg_evals") or "",
|
||
|
|
"preprocess_bytes": row.get("preprocess_bytes") or "",
|
||
|
|
"alloc_bytes": row.get("alloc_bytes") or "",
|
||
|
|
"logical_bytes": row.get("logical_bytes") or "",
|
||
|
|
"layout_waste": row.get("layout_waste") or "",
|
||
|
|
"rc": "0",
|
||
|
|
"sink": row.get("sink") or "",
|
||
|
|
})
|
||
|
|
rows.append(out)
|
||
|
|
return rows
|
||
|
|
|
||
|
|
|
||
|
|
def parse_party(text: str) -> list[dict[str, str]]:
|
||
|
|
rows = []
|
||
|
|
reader = csv.DictReader(text.splitlines(), delimiter="\t")
|
||
|
|
for row in reader:
|
||
|
|
if not row.get("flow"):
|
||
|
|
continue
|
||
|
|
out = blank_cost_fields()
|
||
|
|
out.update({
|
||
|
|
"family": row.get("family") or "party",
|
||
|
|
"slice": row.get("slice") or "",
|
||
|
|
"tier": row.get("tier") or "std",
|
||
|
|
"case": row["flow"],
|
||
|
|
"role": row.get("role") or "-",
|
||
|
|
"items": "1",
|
||
|
|
"repeat": "",
|
||
|
|
"warmup": "",
|
||
|
|
"avg_ns": row.get("avg_ns") or "",
|
||
|
|
"min_ns": row.get("min_ns") or "",
|
||
|
|
"max_ns": row.get("max_ns") or "",
|
||
|
|
"avg_cycles": "",
|
||
|
|
"per_item_ns": row.get("avg_ns") or "",
|
||
|
|
"out_bytes": "",
|
||
|
|
"prg_evals": row.get("prg_evals") or "",
|
||
|
|
"bytes_sent": row.get("bytes_sent") or "",
|
||
|
|
"bytes_recv": row.get("bytes_recv") or "",
|
||
|
|
"bytes_recv_from_p2": row.get("bytes_recv_from_p2") or "",
|
||
|
|
"bytes_recv_from_peer": row.get("bytes_recv_from_peer") or "",
|
||
|
|
"bytes_sent_to_p2": row.get("bytes_sent_to_p2") or "",
|
||
|
|
"bytes_sent_to_peer": row.get("bytes_sent_to_peer") or "",
|
||
|
|
"payload_sent": row.get("payload_sent") or "",
|
||
|
|
"payload_recv": row.get("payload_recv") or "",
|
||
|
|
"wire_overhead": row.get("wire_overhead") or "",
|
||
|
|
"rounds": row.get("rounds") or "",
|
||
|
|
"frames_sent": row.get("frames_sent") or "",
|
||
|
|
"frames_recv": row.get("frames_recv") or "",
|
||
|
|
"avg_bytes_sent": row.get("avg_bytes_sent") or "",
|
||
|
|
"avg_frames_sent": row.get("avg_frames_sent") or "",
|
||
|
|
"wall_ms": row.get("wall_ms") or "",
|
||
|
|
"rc": row.get("rc") or "",
|
||
|
|
"sink": "",
|
||
|
|
})
|
||
|
|
rows.append(out)
|
||
|
|
return rows
|
||
|
|
|
||
|
|
|
||
|
|
def fill_timing(rows: list[dict[str, str]], repeat: int, warmup: int) -> None:
|
||
|
|
for row in rows:
|
||
|
|
if not row["repeat"]:
|
||
|
|
row["repeat"] = str(repeat)
|
||
|
|
if not row["warmup"]:
|
||
|
|
row["warmup"] = str(warmup)
|
||
|
|
|
||
|
|
|
||
|
|
def invoke_group(bin_dir: Path, driver: str, cases: list[dict[str, str]],
|
||
|
|
repeat: int, warmup: int) -> tuple[int, list[dict[str, str]], str]:
|
||
|
|
cmd = [str(bin_dir / driver), "--repeat", str(repeat), "--warmup", str(warmup)]
|
||
|
|
for case in cases:
|
||
|
|
cmd.extend(["--case", case["case"]])
|
||
|
|
rc, out, err = run_capture(cmd)
|
||
|
|
tier_of = {(c["family"], c["case"]): c.get("tier") or "std" for c in cases}
|
||
|
|
if driver == "profile_party":
|
||
|
|
parsed = parse_party(out)
|
||
|
|
else:
|
||
|
|
parsed = parse_inprocess(out, tier_of)
|
||
|
|
fill_timing(parsed, repeat, warmup)
|
||
|
|
return rc, parsed, err
|
||
|
|
|
||
|
|
|
||
|
|
def write_tsv(path: Path, fields: list[str], rows: list[dict[str, str]]) -> None:
|
||
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||
|
|
with path.open("w", newline="") as fh:
|
||
|
|
writer = csv.DictWriter(fh, fieldnames=fields, delimiter="\t", extrasaction="ignore")
|
||
|
|
writer.writeheader()
|
||
|
|
for row in rows:
|
||
|
|
writer.writerow(row)
|
||
|
|
|
||
|
|
|
||
|
|
def read_tsv(path: Path) -> list[dict[str, str]]:
|
||
|
|
if not path.is_file():
|
||
|
|
return []
|
||
|
|
with path.open(newline="") as fh:
|
||
|
|
return list(csv.DictReader(fh, delimiter="\t"))
|
||
|
|
|
||
|
|
|
||
|
|
def merge_rows(old: list[dict[str, str]], new: list[dict[str, str]]) -> list[dict[str, str]]:
|
||
|
|
replaced = {(row["family"], row["slice"], row["case"], row["role"]) for row in new}
|
||
|
|
kept = [
|
||
|
|
row for row in old
|
||
|
|
if (row.get("family"), row.get("slice"), row.get("case"), row.get("role")) not in replaced
|
||
|
|
]
|
||
|
|
# Older long.tsv rows may lack new columns; normalize on merge.
|
||
|
|
merged = []
|
||
|
|
for row in kept + new:
|
||
|
|
full = blank_cost_fields()
|
||
|
|
full.update({k: "" for k in LONG_FIELDS})
|
||
|
|
full.update({k: v for k, v in row.items() if v is not None})
|
||
|
|
merged.append(full)
|
||
|
|
return merged
|
||
|
|
|
||
|
|
|
||
|
|
def matrix_view(rows: list[dict[str, str]]) -> list[dict[str, str]]:
|
||
|
|
view = []
|
||
|
|
for row in rows:
|
||
|
|
view.append({key: row.get(key, "") for key in MATRIX_FIELDS})
|
||
|
|
view.sort(key=lambda r: (r["family"], r["slice"], r["case"], r["role"]))
|
||
|
|
return view
|
||
|
|
|
||
|
|
|
||
|
|
def print_catalog(cells: list[dict[str, str]]) -> None:
|
||
|
|
groups: dict[tuple[str, str, str], int] = defaultdict(int)
|
||
|
|
for row in cells:
|
||
|
|
groups[(row["family"], row["slice"], row.get("tier") or "std")] += 1
|
||
|
|
print("family\tslice\ttier\tcases")
|
||
|
|
for (family, slice_name, tier), count in sorted(groups.items()):
|
||
|
|
print(f"{family}\t{slice_name}\t{tier}\t{count}")
|
||
|
|
print(f"# {len(cells)} cases in {len(groups)} slices")
|
||
|
|
|
||
|
|
|
||
|
|
def compare_labels(out_root: Path, left: str, right: str) -> int:
|
||
|
|
a_rows = read_tsv(out_root / left / "matrix.tsv")
|
||
|
|
b_rows = read_tsv(out_root / right / "matrix.tsv")
|
||
|
|
if not a_rows or not b_rows:
|
||
|
|
raise SystemExit(f"need matrix.tsv in both {left} and {right}")
|
||
|
|
b_index = {
|
||
|
|
(row["family"], row["slice"], row["case"], row["role"]): row
|
||
|
|
for row in b_rows
|
||
|
|
}
|
||
|
|
header = ["family", "slice", "case", "role"]
|
||
|
|
for col in COMPARE_NUM:
|
||
|
|
header.extend([f"{col}_a", f"{col}_b", f"delta_{col}"])
|
||
|
|
print("\t".join(header))
|
||
|
|
missing = 0
|
||
|
|
for row in a_rows:
|
||
|
|
key = (row["family"], row["slice"], row["case"], row["role"])
|
||
|
|
other = b_index.get(key)
|
||
|
|
if other is None:
|
||
|
|
missing += 1
|
||
|
|
continue
|
||
|
|
parts = [row["family"], row["slice"], row["case"], row["role"]]
|
||
|
|
for col in COMPARE_NUM:
|
||
|
|
av_s = row.get(col, "") or ""
|
||
|
|
bv_s = other.get(col, "") or ""
|
||
|
|
try:
|
||
|
|
av = float(av_s)
|
||
|
|
bv = float(bv_s)
|
||
|
|
delta = f"{bv - av:.0f}"
|
||
|
|
except ValueError:
|
||
|
|
delta = ""
|
||
|
|
parts.extend([av_s, bv_s, delta])
|
||
|
|
print("\t".join(parts))
|
||
|
|
if missing:
|
||
|
|
print(f"# {missing} rows in {left} have no match in {right}", file=sys.stderr)
|
||
|
|
return 0
|
||
|
|
|
||
|
|
|
||
|
|
def main() -> int:
|
||
|
|
parser = argparse.ArgumentParser(description=__doc__,
|
||
|
|
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||
|
|
parser.add_argument("--bin-dir", type=Path, default=default_bin_dir())
|
||
|
|
parser.add_argument("--out", type=Path, default=None,
|
||
|
|
help="directory that holds one subdirectory per label")
|
||
|
|
parser.add_argument("--label", default="baseline")
|
||
|
|
parser.add_argument("--family", action="append", default=[])
|
||
|
|
parser.add_argument("--slice", action="append", default=[])
|
||
|
|
parser.add_argument("--case", action="append", default=[])
|
||
|
|
parser.add_argument("--tier", choices=("std", "heavy", "all", "smoke"), default="std")
|
||
|
|
parser.add_argument("--repeat", type=int, default=None)
|
||
|
|
parser.add_argument("--warmup", type=int, default=None)
|
||
|
|
parser.add_argument("--list", action="store_true")
|
||
|
|
parser.add_argument("--compare", nargs=2, metavar=("LABEL_A", "LABEL_B"))
|
||
|
|
args = parser.parse_args()
|
||
|
|
|
||
|
|
out_root = args.out
|
||
|
|
if out_root is None:
|
||
|
|
out_root = args.bin_dir.parent / "cost-matrix"
|
||
|
|
|
||
|
|
if args.compare:
|
||
|
|
return compare_labels(out_root, args.compare[0], args.compare[1])
|
||
|
|
|
||
|
|
cells = discover(args.bin_dir)
|
||
|
|
chosen = select_cells(cells, args)
|
||
|
|
if args.list:
|
||
|
|
print_catalog(chosen)
|
||
|
|
return 0
|
||
|
|
if not chosen:
|
||
|
|
print("no cells match", file=sys.stderr)
|
||
|
|
return 2
|
||
|
|
|
||
|
|
by_driver: dict[str, list[dict[str, str]]] = defaultdict(list)
|
||
|
|
for row in chosen:
|
||
|
|
by_driver[row["driver"]].append(row)
|
||
|
|
|
||
|
|
fresh: list[dict[str, str]] = []
|
||
|
|
failures = 0
|
||
|
|
for driver, rows in by_driver.items():
|
||
|
|
buckets: dict[tuple[str, str, str], list[dict[str, str]]] = defaultdict(list)
|
||
|
|
for row in rows:
|
||
|
|
buckets[(row["family"], row["slice"], row.get("tier") or "std")].append(row)
|
||
|
|
for (family, slice_name, tier), group in buckets.items():
|
||
|
|
repeat, warmup = timing_for(family, slice_name, tier)
|
||
|
|
if args.repeat is not None:
|
||
|
|
repeat = args.repeat
|
||
|
|
if args.warmup is not None:
|
||
|
|
warmup = args.warmup
|
||
|
|
print(f"# {driver} {family}/{slice_name} tier={tier} "
|
||
|
|
f"cases={len(group)} repeat={repeat} warmup={warmup}",
|
||
|
|
file=sys.stderr)
|
||
|
|
rc, parsed, err = invoke_group(args.bin_dir, driver, group, repeat, warmup)
|
||
|
|
if err.strip():
|
||
|
|
print(err, file=sys.stderr, end="" if err.endswith("\n") else "\n")
|
||
|
|
got = {(row["case"], row["role"]) for row in parsed}
|
||
|
|
expected = {row["case"] for row in group}
|
||
|
|
have_cases = {case for case, _role in got}
|
||
|
|
missing = sorted(expected - have_cases)
|
||
|
|
if missing:
|
||
|
|
print(f"# missing results: {', '.join(missing)}", file=sys.stderr)
|
||
|
|
failures += len(missing)
|
||
|
|
if rc != 0:
|
||
|
|
failures += 1
|
||
|
|
fresh.extend(parsed)
|
||
|
|
|
||
|
|
label_dir = out_root / args.label
|
||
|
|
merged = merge_rows(read_tsv(label_dir / "long.tsv"), fresh)
|
||
|
|
write_tsv(label_dir / "long.tsv", LONG_FIELDS, merged)
|
||
|
|
write_tsv(label_dir / "matrix.tsv", MATRIX_FIELDS, matrix_view(merged))
|
||
|
|
print(f"# wrote {label_dir / 'matrix.tsv'} ({len(merged)} rows)", file=sys.stderr)
|
||
|
|
return 1 if failures else 0
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
sys.exit(main())
|