Record Grotto half-ulp tables and comparison geneval, and factor shared beaver terms before the quotient.
Horner and window evaluation need those tables in the tree. Comparison geneval opens the same value words as a Doerner–Shelat key. A factor common to every polynomial term is multiplied first so that preprocessing stays smaller. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
parent
3f10e05176
commit
875f09fec1
14 changed files with 42668 additions and 184 deletions
|
|
@ -5,7 +5,10 @@
|
|||
/// cubic. `erfc`, `softminus`, `logsigmoid`, and `acos` are integer
|
||||
/// rewrites of `erf`, `softplus`, and `asin`. `asin` on `(1/2, 1]`
|
||||
/// uses `π/2 − 2 asin(sqrt((1−x)/2))` with the principal square-root
|
||||
/// table. `probit` is stored on `(0, 1/2]` and mirrored.
|
||||
/// table. `probit` is stored on `(0, 1/2]` and mirrored. The tail
|
||||
/// below 1/20 is a cubic in `ln(p)` (knots at scale `2^{k+10}`),
|
||||
/// because a cubic in `p` cannot meet half an ulp on the first
|
||||
/// input step once `k` is large.
|
||||
|
||||
#ifndef LIBDPF_INCLUDE_GROTTO_WINDOW_LUT_HPP__
|
||||
#define LIBDPF_INCLUDE_GROTTO_WINDOW_LUT_HPP__
|
||||
|
|
@ -128,38 +131,153 @@ inline std::int64_t round_half_away_i128(__int128 number, unsigned shift)
|
|||
return static_cast<std::int64_t>(neg ? -out : out);
|
||||
}
|
||||
|
||||
/// `round(sqrt(v / 2^{k+1}) * 2^k)`, `v > 0`.
|
||||
inline std::int64_t sqrt_half_scale(unsigned fractional_bits, std::int64_t magnitude)
|
||||
inline unsigned __int128 isqrt_floor(unsigned __int128 n)
|
||||
{
|
||||
const int log = 63 - __builtin_clzll(static_cast<unsigned long long>(magnitude));
|
||||
const std::int64_t mant = magnitude << (fractional_bits - static_cast<unsigned>(log + 1));
|
||||
const std::int64_t root = eval_principal(principal::sqrt, fractional_bits, mant);
|
||||
const int exp2 = log - static_cast<int>(fractional_bits);
|
||||
if ((exp2 & 1) == 0)
|
||||
return round_half_away_i128(root, static_cast<unsigned>(-exp2) / 2u);
|
||||
const unsigned t = static_cast<unsigned>(-exp2 - 1) / 2u;
|
||||
// sqrt(2) rounded onto 62 fractional bits.
|
||||
constexpr __int128 sqrt2_62 = 6521908912666391106LL;
|
||||
return round_half_away_i128(__int128(root) * sqrt2_62, 62u + t + 1u);
|
||||
if (n == 0)
|
||||
return 0;
|
||||
const unsigned bits = (n >> 64) != 0
|
||||
? 128u - static_cast<unsigned>(__builtin_clzll(static_cast<unsigned long long>(n >> 64)))
|
||||
: 64u - static_cast<unsigned>(__builtin_clzll(static_cast<unsigned long long>(n)));
|
||||
unsigned __int128 x = static_cast<unsigned __int128>(1) << ((bits + 1u) / 2u);
|
||||
for (;;)
|
||||
{
|
||||
const unsigned __int128 y = (x + n / x) >> 1;
|
||||
if (y >= x)
|
||||
break;
|
||||
x = y;
|
||||
}
|
||||
while (x > 0 && x > n / x)
|
||||
--x;
|
||||
return x;
|
||||
}
|
||||
|
||||
/// `round(sqrt(v / 2^{k+1}) * 2^{k+extra})`, `v > 0`. Eight extra bits so the
|
||||
/// half-angle identity can absorb the square root before the final rounding.
|
||||
inline std::int64_t sqrt_half_scale_fine(unsigned fractional_bits, std::int64_t magnitude)
|
||||
{
|
||||
constexpr unsigned extra = 8;
|
||||
const unsigned shift = fractional_bits + 2u * extra;
|
||||
const unsigned __int128 radicand =
|
||||
static_cast<unsigned __int128>(static_cast<std::uint64_t>(magnitude)) << shift;
|
||||
const unsigned __int128 root = isqrt_floor(radicand);
|
||||
// sqrt(gap << (k+2*extra)) / sqrt(2) = sqrt(gap / 2^{k+1}) * 2^{k+extra}
|
||||
static constexpr unsigned __int128 sqrt2_64 =
|
||||
(static_cast<unsigned __int128>(1) << 64) | static_cast<unsigned __int128>(7640891576956012809ULL);
|
||||
const unsigned __int128 scaled = (root * sqrt2_64 + (static_cast<unsigned __int128>(1) << 64)) >> 65;
|
||||
return static_cast<std::int64_t>(scaled);
|
||||
}
|
||||
|
||||
inline int piece_of_scaled(const window_table & table, std::int64_t raw, unsigned extra)
|
||||
{
|
||||
int lo = 0;
|
||||
int hi = static_cast<int>(table.nparts);
|
||||
while (hi - lo > 1)
|
||||
{
|
||||
const int mid = (lo + hi) / 2;
|
||||
if ((table.knots[mid] << extra) <= raw)
|
||||
lo = mid;
|
||||
else
|
||||
hi = mid;
|
||||
}
|
||||
return lo;
|
||||
}
|
||||
|
||||
inline std::int64_t eval_asin_abs(unsigned fractional_bits, std::int64_t magnitude)
|
||||
{
|
||||
constexpr unsigned extra = 8;
|
||||
const auto half = std::int64_t{1} << (fractional_bits - 1);
|
||||
const window_table & table = at(ASIN, fractional_bits);
|
||||
if (magnitude <= half)
|
||||
return eval_table(table, fractional_bits, magnitude);
|
||||
const std::int64_t one = std::int64_t{1} << fractional_bits;
|
||||
if (magnitude >= one)
|
||||
return HALF_PI_RAW[slot_of(fractional_bits)];
|
||||
const std::int64_t gap = one - magnitude;
|
||||
const auto pi = HALF_PI_RAW[slot_of(fractional_bits)];
|
||||
if (gap <= 0)
|
||||
return pi;
|
||||
std::int64_t reduced = sqrt_half_scale(fractional_bits, gap);
|
||||
if (reduced > half)
|
||||
reduced = half;
|
||||
const std::int64_t inner = eval_table(table, fractional_bits, reduced);
|
||||
const std::int64_t lifted = pi - 2 * inner;
|
||||
return lifted < 0 ? 0 : lifted;
|
||||
std::int64_t reduced = sqrt_half_scale_fine(fractional_bits, gap);
|
||||
const std::int64_t half_fine = half << extra;
|
||||
if (reduced > half_fine)
|
||||
reduced = half_fine;
|
||||
const unsigned scale = fractional_bits + extra;
|
||||
const std::int64_t inner = horner(
|
||||
table.pieces[piece_of_scaled(table, reduced, extra)], table.q, reduced, scale);
|
||||
// pi/2 at 64 fractional bits, then onto scale k+extra in one rounding.
|
||||
static constexpr unsigned __int128 half_pi_64 =
|
||||
(static_cast<unsigned __int128>(1) << 64) | static_cast<unsigned __int128>(10529333758598939754ULL);
|
||||
const __int128 pi_fine = round_half_away_i128(
|
||||
static_cast<__int128>(half_pi_64), 64u - scale);
|
||||
const __int128 lifted = pi_fine - 2 * static_cast<__int128>(inner);
|
||||
const auto out = round_half_away_i128(lifted, extra);
|
||||
return out < 0 ? 0 : out;
|
||||
}
|
||||
|
||||
/// Surplus fractional bits on probit-tail knots. `u = ln(p)` is stored as
|
||||
/// `round(u * 2^{k+probit_tail_extra})`.
|
||||
inline constexpr unsigned probit_tail_extra = 10;
|
||||
|
||||
// ln((32+i)/64) * 2^64, stored as a positive magnitude. Every anchor is in (0, ln 2].
|
||||
static constexpr std::uint64_t probit_ln2_64 = 12786308645202655660ull;
|
||||
static constexpr std::uint64_t probit_ln_anchor_mag[32] = {
|
||||
12786308645202655660ull, 12218671733053503897ull, 11667981761989453435ull, 11133256087961349648ull,
|
||||
10613595130224743362ull, 10108173265494422292ull, 9616230936675340827ull, 9137067786804269247ull,
|
||||
8670036662410753619ull, 8214538357444912273ull, 7770016990662967709ull, 7335955927010031419ull,
|
||||
6911874167941132216ull, 6497323147432841322ull, 6091883880171659064ull, 5695164416463867605ull,
|
||||
5306797565112371681ull, 4926438851101192057ull, 4553764679618851579ull, 4188470681899169456ull,
|
||||
3830270221691897566ull, 3478893044001375095ull, 3134084050134459383ull, 2795602185149230175ull,
|
||||
2463219425550596028ull, 2136719856585056848ull, 1815898829783402670ull, 1500562192519310430ull,
|
||||
1190525582320469641ull, 885613779509420443ull, 585660112482476600ull, 290505910572683730ull,
|
||||
};
|
||||
|
||||
/// `round_half_away(ln(probability / 2^k) * 2^{k+10})`.
|
||||
inline std::int64_t probit_ln_argument(unsigned fractional_bits, std::int64_t probability)
|
||||
{
|
||||
const auto bits = static_cast<unsigned long long>(probability);
|
||||
const int e = 63 - __builtin_clzll(bits);
|
||||
const unsigned shift_in = static_cast<unsigned>(e + 1);
|
||||
const auto wide = static_cast<unsigned __int128>(bits) << (64u - shift_in);
|
||||
const auto m64 = static_cast<std::uint64_t>(wide);
|
||||
const unsigned idx = static_cast<unsigned>((m64 - (1ull << 63)) >> 58);
|
||||
const unsigned b_num = 32u + idx;
|
||||
const unsigned __int128 t_scaled = (static_cast<unsigned __int128>(m64) * 64u) / b_num;
|
||||
__int128 t = static_cast<__int128>(t_scaled - (static_cast<unsigned __int128>(1) << 64));
|
||||
__int128 p = t;
|
||||
__int128 acc = 0;
|
||||
for (int n = 1; n <= 14; ++n)
|
||||
{
|
||||
const __int128 term = p / n;
|
||||
acc += (n & 1) ? term : -term;
|
||||
p = (p * t) >> 64;
|
||||
}
|
||||
const __int128 ln_m = -static_cast<__int128>(probit_ln_anchor_mag[idx]) + acc;
|
||||
const int exp_fix = e + 1 - static_cast<int>(fractional_bits);
|
||||
const __int128 ln_x = ln_m + static_cast<__int128>(exp_fix) * static_cast<__int128>(probit_ln2_64);
|
||||
return round_half_away_i128(ln_x, 54u - fractional_bits);
|
||||
}
|
||||
|
||||
/// Horner, then one extra right shift so a tail argument at scale `k+10` rounds onto scale `k`.
|
||||
inline std::int64_t eval_cubic_extra(
|
||||
const cubic_bits & piece, unsigned q, std::int64_t raw,
|
||||
unsigned fractional_bits, unsigned extra_shift)
|
||||
{
|
||||
using namespace principal_detail;
|
||||
__int128 coeff[4];
|
||||
for (int i = 0; i < 4; ++i)
|
||||
coeff[i] = unpack_coeff(piece.hi[i], piece.lo[i]);
|
||||
const int q_use = static_cast<int>(q) < static_cast<int>(fractional_bits) + 16
|
||||
? static_cast<int>(q)
|
||||
: static_cast<int>(fractional_bits) + 16;
|
||||
const int drop = static_cast<int>(q) - q_use;
|
||||
for (int i = 0; i < 4; ++i)
|
||||
coeff[i] = rshift_ties_even(coeff[i], drop);
|
||||
|
||||
w256 acc = w_from_i128(coeff[3]);
|
||||
for (int i = 2; i >= 0; --i)
|
||||
{
|
||||
acc = w_mul_i64(acc, raw);
|
||||
w256 term = w_shl(w_from_i128(coeff[i]), fractional_bits * static_cast<unsigned>(3 - i));
|
||||
acc = w_add(acc, term);
|
||||
}
|
||||
const unsigned denom_shift = static_cast<unsigned>(q_use) + 2u * fractional_bits + extra_shift;
|
||||
return round_half_away_pow2(acc, denom_shift);
|
||||
}
|
||||
|
||||
inline std::int64_t eval_probit_abs(unsigned fractional_bits, std::int64_t probability)
|
||||
|
|
@ -167,7 +285,14 @@ inline std::int64_t eval_probit_abs(unsigned fractional_bits, std::int64_t proba
|
|||
const window_table & mid = at(PROBIT_MID, fractional_bits);
|
||||
if (probability >= mid.knots[0])
|
||||
return eval_table(mid, fractional_bits, probability);
|
||||
return eval_table(at(PROBIT_TAIL, fractional_bits), fractional_bits, probability);
|
||||
const window_table & tail = at(PROBIT_TAIL, fractional_bits);
|
||||
std::int64_t u = probit_ln_argument(fractional_bits, probability);
|
||||
if (u < tail.knots[0])
|
||||
u = tail.knots[0];
|
||||
if (u > tail.knots[tail.nparts])
|
||||
u = tail.knots[tail.nparts];
|
||||
const unsigned scale = fractional_bits + probit_tail_extra;
|
||||
return eval_cubic_extra(tail.pieces[piece_of(tail, u)], tail.q, u, scale, probit_tail_extra);
|
||||
}
|
||||
|
||||
inline std::int64_t eval_smoothstep(unsigned fractional_bits, std::int64_t raw)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue