/// @file test/profile/grotto_profile.cpp /// @brief Workload for grotto evaluation: tables, offset walks, prefix parity. /// /// Key generation is a separate case from the eval that consumes those keys. /// Table cases probe the domain once, then time only inputs that evaluate. /// `items` is the number of table evaluations inside one sample, or 1 for a /// single offset / prefix walk (both parties). /// /// profile_grotto --list /// profile_grotto --case window_gelu_p16 --repeat 100 /// /// Profile-guided build, separate from COVERAGE: /// cmake -S test -B build-pgo -DLIBDPF_PGO=generate -DCMAKE_BUILD_TYPE=Release /// cmake --build build-pgo --target profile_grotto /// build-pgo/bin/profile_grotto --repeat 40 --warmup 2 /// cmake -S test -B build-pgo -DLIBDPF_PGO=use -DCMAKE_BUILD_TYPE=Release /// cmake --build build-pgo --target profile_grotto #include "harness.hpp" #include "dpf.hpp" #include "grotto/closed_form.hpp" #include "grotto/exact_steps.hpp" #include "grotto/offset_horner.hpp" #include "grotto/offset_jet.hpp" #include "grotto/offset_poly.hpp" #include "grotto/offset_repr.hpp" #include "grotto/offset_twist.hpp" #include "grotto/prefix_parity.hpp" #include "grotto/range_lut.hpp" #include "grotto/window_lut.hpp" #include #include #include #include #include #include #include namespace { using profile::sample; using profile::touch_arr; using profile::touch_vec; using profile::touch_word; using profile::work; const char kUsage[] = "profile_grotto [--list] [--family grotto] [--slice S] [--case NAME]\n" " [--repeat N=30] [--warmup W=2]\n" "Slices: window, principal, closed, reduced, exact, horner, poly, jet,\n" " twist, repr, prefix, keygen.\n" "Times grotto table evals and offset / prefix walks.\n"; inline work cell(const char * slice, std::string name, std::uint64_t items, std::function fn) { return profile::make_work("grotto", slice, std::move(name), items, std::move(fn)); } template std::vector probe(Enum which, unsigned bits, Fn fn, const std::vector & cand) { std::vector ok; ok.reserve(cand.size()); for (const std::int64_t raw : cand) { try { (void)fn(which, bits, raw); ok.push_back(raw); } catch (const std::exception &) { } } return ok; } std::vector around_one(unsigned bits) { const std::int64_t one = std::int64_t{1} << bits; return { -4 * one, -2 * one, -one, -one / 2, -1, 0, 1, one / 20, one / 5, one / 3, one / 2, one, (3 * one) / 2, 2 * one, 4 * one }; } template void add_table(std::vector & works, const char * slice, std::string name, Fn fn, std::vector samples) { if (samples.empty()) { std::cerr << name << " skipped: no in-domain sample\n"; return; } auto held = std::make_shared>(std::move(samples)); const std::uint64_t items = static_cast(held->size()) * 32u; std::function body = [held, fn]() -> sample { std::uint64_t h = 0x84222325cbf29ce4ull; for (int rep = 0; rep < 32; ++rep) { for (const std::int64_t raw : *held) { // Multiply-add, not a plain xor: an even number of identical // xors cancels, and the compiler will delete the evals. h = (h * 0x100000001b3ull) ^ static_cast(fn(raw)) ^ static_cast(rep); } } return touch_word(h, held->size() * sizeof(std::int64_t)); }; works.push_back(cell(slice, std::move(name), items, std::move(body))); } template std::vector knots_of(std::size_t n, T gap) { std::vector knots(n); for (std::size_t i = 0; i < n; ++i) knots[i] = static_cast(1 + gap * static_cast(i)); return knots; } template std::vector> horner_rows(std::size_t n) { std::vector> rows(n); for (std::size_t i = 0; i < n; ++i) for (std::size_t m = 0; m <= Degree; ++m) rows[i][m] = (i + 1) * (m + 3); return rows; } std::vector> poly_rows(std::size_t n, std::size_t degree) { std::vector> rows(n, std::vector(degree + 1)); for (std::size_t i = 0; i < n; ++i) for (std::size_t m = 0; m <= degree; ++m) rows[i][m] = (i + 1) * (m + 3); return rows; } std::vector coeff_col(std::size_t degree) { std::vector c(degree + 1); for (std::size_t m = 0; m <= degree; ++m) c[m] = m + 5; return c; } template std::array ends_of(T start, T step) { std::array ends{}; for (std::size_t i = 0; i < N; ++i) ends[i] = static_cast(start + step * static_cast(i)); return ends; } } // namespace template void add_horner_degree(std::vector & works) { using in_t = std::uint32_t; const in_t center = 50; const in_t eta = 7; works.push_back(cell("keygen", "horner_keygen_d" + std::to_string(Degree), 1, [] { auto built = grotto::make_offset_horner_keys(in_t{50}); return touch_word(built.wrap_share[0][0] ^ built.wrap_share[Degree][1], sizeof(built.wrap_share)); })); using mat_t = decltype(grotto::make_offset_horner_keys(center)); auto mat = std::make_shared(grotto::make_offset_horner_keys(center)); for (const std::size_t pieces : std::initializer_list{1, 4, 16, 64}) { auto knots = std::make_shared>(knots_of(pieces, 1000)); auto rows = std::make_shared>>( horner_rows(pieces)); const auto name = "horner_eval_d" + std::to_string(Degree) + "_p" + std::to_string(pieces); works.push_back(cell("horner", name, 1, [mat, knots, rows, eta] { const auto a = grotto::offset_horner_eval<0, Degree>(*mat, *knots, *rows, eta); const auto b = grotto::offset_horner_eval<1, Degree>(*mat, *knots, *rows, eta); return touch_word(a ^ b, 16); })); } } template void add_prefix_n(std::vector & works, const char * width, Input step, const BitPtr & bits, const GtPtr & gts) { const auto ends = ends_of(Input{1}, step); const auto ntag = std::to_string(N); works.push_back(cell("prefix", std::string("prefix_") + width + "_e" + ntag, N, [bits, ends] { const auto a = grotto::prefix_parities(std::get<0>(*bits), ends); const auto b = grotto::prefix_parities(std::get<1>(*bits), ends); return touch_arr(std::get<0>(a)) + touch_arr(std::get<0>(b)); })); works.push_back(cell("prefix", std::string("signed_prefix_") + width + "_e" + ntag, N, [gts, ends] { const auto a = grotto::signed_prefix_parities(std::get<0>(*gts), ends); const auto b = grotto::signed_prefix_parities(std::get<1>(*gts), ends); return touch_arr(a) + touch_arr(b); })); } template void add_prefix_width(std::vector & works, const char * width, Input step) { const auto bit_keys = dpf::make_dpf(Input{1000}, dpf::bit{1}); const auto gt_keys = dpf::make_dpf(Input{1000}, dpf::gt(std::uint64_t{1})); using bit_ptr = std::shared_ptr>; using gt_ptr = std::shared_ptr>; auto bits = std::make_shared>(bit_keys); auto gts = std::make_shared>(gt_keys); add_prefix_n(works, width, step, bits, gts); add_prefix_n(works, width, step, bits, gts); add_prefix_n(works, width, step, bits, gts); add_prefix_n(works, width, step, bits, gts); } int main(int argc, char ** argv) { const auto opt = profile::parse_args(argc, argv, 30, 2, kUsage); std::vector works; const unsigned precisions[] = {8u, 16u, 24u, 32u}; const auto add_window = [&](const char * stem, grotto::window which, unsigned bits) { auto samples = probe(which, bits, [](grotto::window w, unsigned b, std::int64_t raw) { return grotto::eval_window(w, b, raw); }, around_one(bits)); add_table(works, "window", std::string("window_") + stem + "_p" + std::to_string(bits), [which, bits](std::int64_t raw) { return grotto::eval_window(which, bits, raw); }, std::move(samples)); }; const auto add_principal = [&](const char * stem, grotto::principal which, unsigned bits) { auto samples = probe(which, bits, [](grotto::principal w, unsigned b, std::int64_t raw) { return grotto::eval_principal(w, b, raw); }, around_one(bits)); add_table(works, "principal", std::string("principal_") + stem + "_p" + std::to_string(bits), [which, bits](std::int64_t raw) { return grotto::eval_principal(which, bits, raw); }, std::move(samples)); }; const auto add_closed = [&](const char * stem, grotto::closed which, unsigned bits) { auto samples = probe(which, bits, [](grotto::closed w, unsigned b, std::int64_t raw) { return grotto::eval_closed(w, b, raw); }, around_one(bits)); add_table(works, "closed", std::string("closed_") + stem + "_p" + std::to_string(bits), [which, bits](std::int64_t raw) { return grotto::eval_closed(which, bits, raw); }, std::move(samples)); }; const auto add_reduced = [&](const char * stem, grotto::reduced which, unsigned bits) { auto samples = probe(which, bits, [](grotto::reduced w, unsigned b, std::int64_t raw) { return grotto::eval_reduced(w, b, raw); }, around_one(bits)); add_table(works, "reduced", std::string("reduced_") + stem + "_p" + std::to_string(bits), [which, bits](std::int64_t raw) { return grotto::eval_reduced(which, bits, raw); }, std::move(samples)); }; const struct { const char * stem; grotto::window which; } windows[] = { {"gelu", grotto::window::gelu}, {"sigmoid", grotto::window::sigmoid}, {"tanh", grotto::window::tanh}, {"erf", grotto::window::erf}, {"erfc", grotto::window::erfc}, {"silu", grotto::window::silu}, {"mish", grotto::window::mish}, {"softplus", grotto::window::softplus}, {"probit", grotto::window::probit}, {"smoothstep", grotto::window::smoothstep}, {"asin", grotto::window::asin}, {"acos", grotto::window::acos}, {"hardelish", grotto::window::hardelish}, {"lecun_tanh", grotto::window::lecun_tanh}, }; for (const auto & spec : windows) for (const unsigned bits : precisions) add_window(spec.stem, spec.which, bits); const struct { const char * stem; grotto::principal which; } principals[] = { {"ln", grotto::principal::ln}, {"exp", grotto::principal::exp}, {"sin", grotto::principal::sin}, {"sqrt", grotto::principal::sqrt}, {"sinh", grotto::principal::sinh}, {"inv", grotto::principal::inv}, }; for (const auto & spec : principals) for (const unsigned bits : precisions) add_principal(spec.stem, spec.which, bits); const struct { const char * stem; grotto::closed which; } closeds[] = { {"atan", grotto::closed::atan}, {"cbrt", grotto::closed::cbrt}, {"sinc", grotto::closed::sinc}, {"softsign", grotto::closed::softsign}, {"logistic", grotto::closed::logistic}, }; for (const auto & spec : closeds) for (const unsigned bits : precisions) add_closed(spec.stem, spec.which, bits); const struct { const char * stem; grotto::reduced which; } reduceds[] = { {"ln", grotto::reduced::ln}, {"exp", grotto::reduced::exp}, {"sin", grotto::reduced::sin}, {"expm1", grotto::reduced::expm1}, {"log1p", grotto::reduced::log1p}, {"sqrt", grotto::reduced::sqrt}, }; for (const auto & spec : reduceds) for (const unsigned bits : precisions) add_reduced(spec.stem, spec.which, bits); const auto add_exact = [&](const std::string & name, unsigned bits, auto fn) { std::vector samples; for (const std::int64_t raw : around_one(bits)) { try { (void)fn(raw); samples.push_back(raw); } catch (const std::exception &) { } } add_table(works, "exact", name, fn, std::move(samples)); }; for (const unsigned bits : {8u, 16u, 32u}) { const auto tag = "_p" + std::to_string(bits); add_exact("exact_dec_floor" + tag, bits, [bits](std::int64_t raw) { return grotto::eval_dec_floor(raw, bits); }); add_exact("exact_bit_width" + tag, bits, [bits](std::int64_t raw) { return grotto::eval_bit_width(raw, bits); }); add_exact("exact_dec_width" + tag, bits, [bits](std::int64_t raw) { return grotto::eval_dec_width(raw, bits); }); add_exact("exact_ilog256" + tag, bits, [bits](std::int64_t raw) { return grotto::eval_ilog256(raw, bits); }); } add_horner_degree<1>(works); add_horner_degree<3>(works); using in_t = std::uint32_t; const in_t center = 50; const in_t eta = 7; for (const std::size_t degree : std::initializer_list{1, 4, 8, 16}) { works.push_back(cell("keygen", "poly_keygen_d" + std::to_string(degree), 1, [degree] { auto built = grotto::make_offset_poly_keys(in_t{50}, degree); return touch_word(built.wrap_share[0][0], built.wrap_share.size() * sizeof(built.wrap_share[0])); })); using mat_t = decltype(grotto::make_offset_poly_keys(center, degree)); auto mat = std::make_shared(grotto::make_offset_poly_keys(center, degree)); for (const std::size_t pieces : std::initializer_list{4, 8, 16}) { if (degree == 16 && pieces == 16) continue; auto knots = std::make_shared>(knots_of(pieces, 800)); auto rows = std::make_shared>>( poly_rows(pieces, degree)); const auto name = "poly_eval_d" + std::to_string(degree) + "_p" + std::to_string(pieces); works.push_back(cell("poly", name, 1, [mat, knots, rows] { const auto a = grotto::offset_poly_eval<0>(*mat, *knots, *rows, eta, nullptr); const auto b = grotto::offset_poly_eval<1>(*mat, *knots, *rows, eta, nullptr); return touch_word(a ^ b, 16); })); } } for (const std::size_t degree : std::initializer_list{1, 4, 8, 16}) { works.push_back(cell("keygen", "jet_keygen_d" + std::to_string(degree), 1, [degree] { auto built = grotto::make_offset_jet_keys(in_t{50}, degree); return touch_word(built.wrap_share[0][0], built.wrap_share.size() * sizeof(built.wrap_share[0])); })); using mat_t = decltype(grotto::make_offset_jet_keys(center, degree)); auto mat = std::make_shared(grotto::make_offset_jet_keys(center, degree)); for (const std::size_t pieces : std::initializer_list{4, 16}) { auto knots = std::make_shared>(knots_of(pieces, 800)); auto col = std::make_shared>(coeff_col(degree)); const auto name = "jet_eval_d" + std::to_string(degree) + "_p" + std::to_string(pieces); works.push_back(cell("jet", name, 1, [mat, knots, col] { const auto a = grotto::offset_jet_eval<0>(*mat, *knots, *col, eta); const auto b = grotto::offset_jet_eval<1>(*mat, *knots, *col, eta); return touch_word(a ^ b, 16); })); } } for (const std::size_t degree : std::initializer_list{1, 4, 8}) { auto knots = std::make_shared>(knots_of(8, 900)); auto col = std::make_shared>(coeff_col(degree)); using odd_t = decltype(grotto::make_offset_twist_keys(center, degree, std::uint64_t{3})); using half_t = decltype(grotto::make_offset_twist_keys(center, degree, grotto::twist_half)); auto odd = std::make_shared( grotto::make_offset_twist_keys(center, degree, std::uint64_t{3})); auto half = std::make_shared( grotto::make_offset_twist_keys(center, degree, grotto::twist_half)); const auto dtag = std::to_string(degree); works.push_back(cell("keygen", "twist_keygen_odd_d" + dtag, 1, [degree] { auto built = grotto::make_offset_twist_keys(in_t{50}, degree, std::uint64_t{3}); return touch_word(built.wrap_share[0][0], built.wrap_share.size() * sizeof(built.wrap_share[0])); })); works.push_back(cell("twist", "twist_eval_odd_d" + dtag + "_p8", 1, [odd, knots, col] { const auto a = grotto::offset_twist_eval<0>(*odd, *knots, *col, eta); const auto b = grotto::offset_twist_eval<1>(*odd, *knots, *col, eta); return touch_word(a ^ b, 16); })); works.push_back(cell("twist", "twist_eval_half_d" + dtag + "_p8", 1, [half, knots, col] { const auto a = grotto::offset_twist_eval<0>(*half, *knots, *col, eta); const auto b = grotto::offset_twist_eval<1>(*half, *knots, *col, eta); return touch_word(a ^ b, 16); })); } { const std::vector fib_state{1, 0}; const std::vector wide_state{1, 2, 3, 4}; using fib_t = decltype(grotto::make_offset_repr_keys(center, fib_state)); using wide_t = decltype(grotto::make_offset_repr_keys(center, wide_state)); auto fib = std::make_shared(grotto::make_offset_repr_keys(center, fib_state)); auto wide = std::make_shared(grotto::make_offset_repr_keys(center, wide_state)); auto fib_m = std::make_shared>>( grotto::offset_repr_fibonacci_matrix()); auto tri_m = std::make_shared>>( std::vector>{ {1, 1, 0, 0}, {0, 1, 1, 0}, {0, 0, 1, 1}, {0, 0, 0, 1}, }); works.push_back(cell("keygen", "repr_keygen_dim2", 1, [] { auto built = grotto::make_offset_repr_keys(in_t{50}, std::vector{1, 0}); return touch_word(built.wrap_share[0][0], built.wrap_share.size() * sizeof(built.wrap_share[0])); })); works.push_back(cell("keygen", "repr_keygen_dim4", 1, [] { auto built = grotto::make_offset_repr_keys(in_t{50}, std::vector{1, 2, 3, 4}); return touch_word(built.wrap_share[0][0], built.wrap_share.size() * sizeof(built.wrap_share[0])); })); for (const std::size_t pieces : std::initializer_list{4, 8, 16}) { auto knots = std::make_shared>(knots_of(pieces, 700)); const auto tag = "_p" + std::to_string(pieces); works.push_back(cell("repr", "repr_eval_fib" + tag, 1, [fib, fib_m, knots] { const auto a = grotto::offset_repr_eval<0>(*fib, *fib_m, *knots, eta); const auto b = grotto::offset_repr_eval<1>(*fib, *fib_m, *knots, eta); return touch_vec(a) + touch_vec(b); })); works.push_back(cell("repr", "repr_eval_dim4" + tag, 1, [wide, tri_m, knots] { const auto a = grotto::offset_repr_eval<0>(*wide, *tri_m, *knots, eta); const auto b = grotto::offset_repr_eval<1>(*wide, *tri_m, *knots, eta); return touch_vec(a) + touch_vec(b); })); } works.push_back(cell("repr", "repr_crc32_jump", 64, [] { std::uint32_t state = 0x12345678u; for (unsigned i = 0; i < 64; ++i) state = grotto::offset_repr_crc32_jump(state, 1ull << (i % 17)); return touch_word(state, 64u * 32u * sizeof(std::uint32_t)); })); } add_prefix_width(works, "u16", 400); add_prefix_width(works, "u32", 100000); return profile::run_works(opt, works); }