libdpf/include/grotto/range_lut.hpp

677 lines
23 KiB
C++
Raw Normal View History

/// @file grotto/range_lut.hpp
/// @brief Full-domain maps built from the principal-domain cubics.
/// @details Each reduced map is one of the elementary range reductions, and
/// the polynomial it evaluates is the matching principal table:
/// `ln` / `lg` / `log10` share the mantissa logarithm;
/// `exp` / `exp2` / `exp10` share the `2^{-13}` exponential;
/// `sin` / `cos` share the quarter-turn sine;
/// `tan` / `cot` share `tanf` and `tang`;
/// `sec` / `csc` share `sec` and `gsec`;
/// `sinh` / `cosh` / `tanh` / `sech` share the hyperbolic addition;
/// `coth` and `csch` use their principal small-argument tables;
/// `sqrt` / `inv` / `rsqrt` / `invsq` are dyadic lifts of `[1/2, 1]`.
/// @copyright Copyright (c) 2019-2026 Ryan Henry and [others](@ref authors)
/// @license Released under a GNU General Public v2.0 (GPLv2) license.
#ifndef LIBDPF_INCLUDE_GROTTO_RANGE_LUT_HPP__
#define LIBDPF_INCLUDE_GROTTO_RANGE_LUT_HPP__
#include <cstdint>
#include <stdexcept>
#include "hedley/hedley.h"
#include "grotto/principal_lut.hpp"
namespace grotto
{
enum class reduced : unsigned
{
ln = 0,
lg,
log10,
exp,
exp2,
exp10,
sin,
cos,
tan,
cot,
sec,
csc,
sinh,
cosh,
tanh,
coth,
sech,
csch,
sqrt,
inv,
rsqrt,
invsq,
};
HEDLEY_WARN_UNUSED_RESULT
inline std::int64_t eval_reduced(reduced which, unsigned fractional_bits, std::int64_t raw);
namespace range_detail
{
using u128 = unsigned __int128;
constexpr u128 words(std::uint64_t hi, std::uint64_t lo)
{
return (u128{hi} << 64) | lo;
}
/// `value * 2^64`, rounded half away from zero. Values above `2^64` keep the
/// high limb so the constant is not truncated.
inline constexpr u128 ln2_64 = words(0, 12786308645202655660ULL);
inline constexpr u128 inv_ln2_64 = words(1, 8166282121979093367ULL);
inline constexpr u128 log10_2_64 = words(0, 5553023288523357132ULL);
inline constexpr u128 ln10_64 = words(2, 5581709770980765788ULL);
inline constexpr u128 inv_ln10_64 = words(0, 8011319160293570763ULL);
inline constexpr u128 sqrt2_64 = words(1, 7640891576956012809ULL);
inline constexpr u128 rsqrt2_64 = words(0, 13043817825332782212ULL);
inline constexpr u128 two_over_pi_64 = words(0, 11743562013128004906ULL);
inline constexpr u128 four_over_pi_64 = words(1, 5040379952546458196ULL);
inline constexpr u128 pi_over_4_64 = words(0, 14488038916154245685ULL);
/// `exp(2^{i-13}) * 2^64`.
inline constexpr u128 exp_chunk_64[13] = {
words(1, 2251937258231296ULL),
words(1, 4504149427926357ULL),
words(1, 9009398635954180ULL),
words(1, 18023197466514910ULL),
words(1, 36064004308734226ULL),
words(1, 72198514957318099ULL),
words(1, 144679606912572172ULL),
words(1, 290493950045950331ULL),
words(1, 585562514163419534ULL),
words(1, 1189712777830127574ULL),
words(1, 2456155437534072733ULL),
words(1, 5239344172067481206ULL),
words(1, 11966795255776918679ULL),
};
inline std::int64_t round_mag(u128 mag, unsigned shift, bool neg)
{
if (shift >= 128)
return 0;
if (shift > 0)
{
mag += u128{1} << (shift - 1);
mag >>= shift;
}
if (mag > static_cast<u128>(INT64_MAX))
throw std::overflow_error("range lut: value does not fit int64");
const auto out = static_cast<std::int64_t>(mag);
return neg ? -out : out;
}
inline std::int64_t round_i128(__int128 value, unsigned shift)
{
const bool neg = value < 0;
const auto mag = static_cast<u128>(neg ? -value : value);
return round_mag(mag, shift, neg);
}
inline std::int64_t scale_unit(u128 mag64, unsigned fractional_bits)
{
return round_mag(mag64, 64u - fractional_bits, false);
}
inline std::int64_t mul_raw(std::int64_t lhs, std::int64_t rhs, unsigned fractional_bits)
{
return round_i128(static_cast<__int128>(lhs) * rhs, fractional_bits);
}
inline std::int64_t div_raw(std::int64_t num, std::int64_t den, unsigned fractional_bits)
{
if (den == 0)
throw std::domain_error("range lut: division by zero");
const bool neg = (num < 0) != (den < 0);
auto n = static_cast<u128>(num < 0 ? -static_cast<__int128>(num) : num);
auto d = static_cast<u128>(den < 0 ? -static_cast<__int128>(den) : den);
n <<= fractional_bits;
const u128 quot = (n + d / 2) / d;
return round_mag(quot, 0, neg);
}
inline std::int64_t shift_pow2(std::int64_t value, int places)
{
if (places == 0 || value == 0)
return value;
if (places > 0)
{
if (places >= 62)
throw std::overflow_error("range lut: exponent overflow");
const __int128 wide = static_cast<__int128>(value) << places;
if (wide > INT64_MAX || wide < INT64_MIN)
throw std::overflow_error("range lut: exponent overflow");
return static_cast<std::int64_t>(wide);
}
return round_i128(value, static_cast<unsigned>(-places));
}
HEDLEY_CONST
HEDLEY_NO_THROW
constexpr std::int64_t one_raw(unsigned fractional_bits) noexcept
{
return std::int64_t{1} << fractional_bits;
}
HEDLEY_CONST
HEDLEY_NO_THROW
constexpr u128 magnitude_of(std::int64_t raw) noexcept
{
if (raw >= 0)
return static_cast<u128>(raw);
return static_cast<u128>(-static_cast<__int128>(raw));
}
inline std::int64_t abs_raw(std::int64_t raw)
{
const u128 mag = magnitude_of(raw);
if (mag > static_cast<u128>(INT64_MAX))
throw std::overflow_error("range lut: magnitude does not fit int64");
return static_cast<std::int64_t>(mag);
}
struct dyadic
{
std::int64_t mantissa_raw;
int power;
};
inline dyadic split_positive(std::int64_t raw, unsigned fractional_bits)
{
if (raw <= 0)
throw std::domain_error("range lut: reduction requires a positive input");
const auto mag = static_cast<unsigned long long>(raw);
const int floor_log = 63 - __builtin_clzll(mag);
const int shift = static_cast<int>(fractional_bits) - floor_log - 1;
std::int64_t mantissa = shift >= 0
? raw << shift
: round_i128(raw, static_cast<unsigned>(-shift));
int power = floor_log + 1 - static_cast<int>(fractional_bits);
const std::int64_t one = one_raw(fractional_bits);
const std::int64_t half = one >> 1;
if (mantissa >= one)
{
mantissa >>= 1;
++power;
}
if (mantissa < half)
mantissa = half;
return dyadic{mantissa, power};
}
inline std::int64_t ln2_raw(unsigned fractional_bits)
{
return scale_unit(ln2_64, fractional_bits);
}
inline std::int64_t eval_ln_positive(unsigned fractional_bits, std::int64_t raw)
{
const dyadic part = split_positive(raw, fractional_bits);
const std::int64_t ln_m = eval_principal(principal::ln, fractional_bits, part.mantissa_raw);
return ln_m + static_cast<std::int64_t>(part.power) * ln2_raw(fractional_bits);
}
inline std::int64_t eval_exp_at_scale(unsigned fractional_bits, std::int64_t raw)
{
if (fractional_bits < 13)
{
const int lift = static_cast<int>(16u - fractional_bits);
const __int128 lifted_arg = static_cast<__int128>(raw) << lift;
if (lifted_arg > INT64_MAX || lifted_arg < INT64_MIN)
throw std::overflow_error("range lut: exponent overflow");
const std::int64_t lifted = eval_exp_at_scale(16, static_cast<std::int64_t>(lifted_arg));
return round_i128(lifted, static_cast<unsigned>(lift));
}
const std::int64_t ln2 = ln2_raw(fractional_bits);
if (ln2 <= 0)
throw std::logic_error("range lut: ln 2 constant");
std::int64_t n_bin = raw / ln2;
std::int64_t remainder = raw - n_bin * ln2;
if (remainder < 0)
{
remainder += ln2;
--n_bin;
}
while (remainder >= ln2)
{
remainder -= ln2;
++n_bin;
}
const std::int64_t step = std::int64_t{1} << (fractional_bits - 13);
const std::int64_t chunks = remainder / step;
const std::int64_t tiny = remainder - chunks * step;
std::int64_t table_raw = tiny << 13;
const std::int64_t one = one_raw(fractional_bits);
if (table_raw > one)
table_raw = one;
std::int64_t exp_s = eval_principal(principal::exp, fractional_bits, table_raw);
for (unsigned bit = 0; bit < 13; ++bit)
{
if ((static_cast<unsigned long long>(chunks) & (1ull << bit)) == 0)
continue;
// `chunk` is `exp(2^{i-13}) * 2^64`, so the product's high limb is the raw product.
const u128 prod = static_cast<u128>(exp_s) * exp_chunk_64[bit];
exp_s = round_mag(prod, 64, false);
}
return shift_pow2(exp_s, static_cast<int>(n_bin));
}
inline std::int64_t fractional_raw(std::int64_t raw, unsigned fractional_bits, std::int64_t & whole)
{
const std::int64_t one = one_raw(fractional_bits);
std::int64_t q = raw / one;
std::int64_t f = raw - q * one;
if (f < 0)
{
f += one;
--q;
}
whole = q;
return f;
}
inline std::int64_t pow10_raw(int exponent, unsigned fractional_bits)
{
const std::int64_t one = one_raw(fractional_bits);
if (exponent == 0)
return one;
if (exponent < 0)
return div_raw(one, pow10_raw(-exponent, fractional_bits), fractional_bits);
u128 acc = static_cast<u128>(one);
for (int i = 0; i < exponent; ++i)
{
if (acc > static_cast<u128>(INT64_MAX) / 10)
throw std::overflow_error("range lut: exponent overflow");
acc *= 10;
}
return static_cast<std::int64_t>(acc);
}
struct angle
{
unsigned index;
std::int64_t frac_raw;
};
/// `{ |x| * multiplier }` at this precision, with the integer part reduced
/// only as far as the low bits the quadrant logic reads.
inline angle reduce_positive(unsigned fractional_bits, std::int64_t raw, u128 multiplier_64)
{
const u128 scaled = magnitude_of(raw) * multiplier_64;
const u128 rounded = (scaled + (u128{1} << 63)) >> 64;
const u128 one = u128{1} << fractional_bits;
return angle{
static_cast<unsigned>(rounded >> fractional_bits),
static_cast<std::int64_t>(rounded & (one - 1)),
};
}
inline std::int64_t principal_sin_fraction(unsigned fractional_bits, std::int64_t fraction_raw, bool complement)
{
const std::int64_t one = one_raw(fractional_bits);
std::int64_t argument = complement ? one - fraction_raw : fraction_raw;
if (argument < 0)
argument = 0;
if (argument > one)
argument = one;
return eval_principal(principal::sin, fractional_bits, argument);
}
inline std::int64_t sin_from_angle(unsigned fractional_bits, const angle & turned, int sign)
{
const unsigned which = turned.index & 3u;
const bool complement = which == 1 || which == 3;
const int quadrant_sign = (which == 2 || which == 3) ? -1 : 1;
const std::int64_t magnitude = principal_sin_fraction(
fractional_bits, turned.frac_raw, complement);
return magnitude * quadrant_sign * sign;
}
inline std::int64_t cos_from_angle(unsigned fractional_bits, const angle & turned)
{
angle shifted = turned;
shifted.index += 1;
return sin_from_angle(fractional_bits, shifted, 1);
}
inline std::int64_t pi_over_4_raw(unsigned fractional_bits)
{
return scale_unit(pi_over_4_64, fractional_bits);
}
inline std::int64_t tan_positive(unsigned fractional_bits, std::int64_t magnitude, int quarter_shift)
{
const angle turned = reduce_positive(fractional_bits, magnitude, four_over_pi_64);
const unsigned q = (turned.index + static_cast<unsigned>(quarter_shift)) & 3u;
const std::int64_t one = one_raw(fractional_bits);
std::int64_t t = (q == 0 || q == 2) ? turned.frac_raw : one - turned.frac_raw;
if (t < 0)
t = 0;
if (t > one)
t = one;
const std::int64_t z = mul_raw(t, pi_over_4_raw(fractional_bits), fractional_bits);
if (q == 0 || q == 3)
{
const std::int64_t tanf = eval_principal(principal::tanf, fractional_bits, t);
const std::int64_t y = mul_raw(z, tanf, fractional_bits);
return q == 3 ? -y : y;
}
if (z == 0)
throw std::domain_error("range lut: tan pole");
const std::int64_t tang = eval_principal(principal::tang, fractional_bits, t);
const std::int64_t y = div_raw(one, z, fractional_bits) + tang;
return q == 2 ? -y : y;
}
inline std::int64_t sec_positive(unsigned fractional_bits, std::int64_t magnitude, int octant_shift)
{
const angle turned = reduce_positive(fractional_bits, magnitude, four_over_pi_64);
const unsigned q8 = (turned.index + static_cast<unsigned>(octant_shift)) & 7u;
const unsigned q = q8 & 3u;
const int sigma = (q8 & 4u) == 0 ? 1 : -1;
const std::int64_t one = one_raw(fractional_bits);
std::int64_t t = (q == 0 || q == 2) ? turned.frac_raw : one - turned.frac_raw;
if (t < 0)
t = 0;
if (t > one)
t = one;
if (q == 0 || q == 3)
{
const std::int64_t sec = eval_principal(principal::sec, fractional_bits, t);
const int sign = (q == 3 ? -1 : 1) * sigma;
return sec * sign;
}
const std::int64_t z = mul_raw(t, pi_over_4_raw(fractional_bits), fractional_bits);
if (z == 0)
throw std::domain_error("range lut: sec pole");
const std::int64_t gsec = eval_principal(principal::gsec, fractional_bits, t);
std::int64_t y = div_raw(one, z, fractional_bits) + gsec;
if (q == 2)
y = -y;
return y * sigma;
}
inline void quotient_2_13(unsigned fractional_bits, std::int64_t magnitude,
std::int64_t & quotient, std::int64_t & remainder)
{
if (fractional_bits >= 13)
{
const unsigned shift = fractional_bits - 13;
quotient = magnitude >> shift;
const std::int64_t mask = shift >= 63 ? INT64_MAX : (std::int64_t{1} << shift) - 1;
remainder = shift == 0 ? 0 : magnitude & mask;
return;
}
const int lift = static_cast<int>(13u - fractional_bits);
const __int128 wide = static_cast<__int128>(magnitude) << lift;
if (wide > INT64_MAX)
throw std::overflow_error("range lut: exponent overflow");
quotient = static_cast<std::int64_t>(wide);
remainder = 0;
}
inline std::int64_t exp_of_quotient(unsigned fractional_bits, std::int64_t quotient, std::int64_t magnitude)
{
if (quotient == 0)
return one_raw(fractional_bits);
__int128 argument;
if (fractional_bits >= 13)
argument = static_cast<__int128>(quotient) << (fractional_bits - 13);
else
argument = magnitude;
if (argument > INT64_MAX)
throw std::overflow_error("range lut: exponent overflow");
return eval_exp_at_scale(fractional_bits, static_cast<std::int64_t>(argument));
}
struct hyp
{
std::int64_t sinh_raw;
std::int64_t cosh_raw;
};
inline hyp sinh_cosh(unsigned fractional_bits, std::int64_t raw)
{
const bool neg = raw < 0;
const auto mag_wide = magnitude_of(raw);
if (mag_wide > static_cast<u128>(INT64_MAX))
throw std::overflow_error("range lut: exponent overflow");
const std::int64_t mag = static_cast<std::int64_t>(mag_wide);
std::int64_t quotient = 0;
std::int64_t remainder = 0;
quotient_2_13(fractional_bits, mag, quotient, remainder);
const std::int64_t one = one_raw(fractional_bits);
std::int64_t table = 0;
if (fractional_bits >= 13 && remainder != 0)
{
const __int128 lifted = static_cast<__int128>(remainder) << 13;
table = lifted > one ? one : static_cast<std::int64_t>(lifted);
}
const std::int64_t sr = eval_principal(principal::sinh, fractional_bits, table);
const std::int64_t cr = eval_principal(principal::cosh, fractional_bits, table);
std::int64_t sh = sr;
std::int64_t ch = cr;
if (quotient != 0)
{
const std::int64_t grown = exp_of_quotient(fractional_bits, quotient, mag);
std::int64_t inv = 0;
if (grown != 0)
inv = div_raw(one, grown, fractional_bits);
const std::int64_t sq = round_i128(static_cast<__int128>(grown) - inv, 1);
const std::int64_t cq = round_i128(static_cast<__int128>(grown) + inv, 1);
const __int128 sinh_sum = static_cast<__int128>(sq) * cr + static_cast<__int128>(cq) * sr;
const __int128 cosh_sum = static_cast<__int128>(cq) * cr + static_cast<__int128>(sq) * sr;
sh = round_i128(sinh_sum, fractional_bits);
ch = round_i128(cosh_sum, fractional_bits);
}
if (neg)
sh = -sh;
return hyp{sh, ch};
}
/// `ln(2^{k+1} ± 1) / 2`, the saturation threshold used by `tanh` and `coth`.
inline std::int64_t beta_raw(unsigned fractional_bits, bool plus)
{
u128 ln = u128{fractional_bits + 1} * ln2_64;
const u128 eps = u128{1} << (63u - fractional_bits);
if (plus)
ln += eps;
else
ln -= eps;
return round_mag(ln, 65u - fractional_bits, false);
}
inline int half_pow_of(int power)
{
return (power & 1) != 0 ? (power - 1) / 2 : power / 2;
}
} // namespace range_detail
HEDLEY_WARN_UNUSED_RESULT
inline std::int64_t eval_reduced(reduced which, unsigned fractional_bits, std::int64_t raw)
{
using namespace range_detail;
if (!principal_precision(fractional_bits))
throw std::invalid_argument("range lut: precision must be 8, 12, ..., 32");
const std::int64_t one = one_raw(fractional_bits);
switch (which)
{
case reduced::ln:
return eval_ln_positive(fractional_bits, raw);
case reduced::lg:
{
const dyadic part = split_positive(raw, fractional_bits);
const std::int64_t ln_m = eval_principal(principal::ln, fractional_bits, part.mantissa_raw);
const std::int64_t lg_m = mul_raw(
ln_m, scale_unit(inv_ln2_64, fractional_bits), fractional_bits);
return lg_m + (static_cast<std::int64_t>(part.power) << fractional_bits);
}
case reduced::log10:
{
const dyadic part = split_positive(raw, fractional_bits);
const std::int64_t ln_m = eval_principal(principal::ln, fractional_bits, part.mantissa_raw);
const std::int64_t mantissa = mul_raw(
ln_m, scale_unit(inv_ln10_64, fractional_bits), fractional_bits);
const std::int64_t lift = static_cast<std::int64_t>(part.power)
* scale_unit(log10_2_64, fractional_bits);
return mantissa + lift;
}
case reduced::exp:
return eval_exp_at_scale(fractional_bits, raw);
case reduced::exp2:
{
std::int64_t whole = 0;
const std::int64_t frac = fractional_raw(raw, fractional_bits, whole);
const std::int64_t natural = mul_raw(frac, ln2_raw(fractional_bits), fractional_bits);
return shift_pow2(eval_exp_at_scale(fractional_bits, natural), static_cast<int>(whole));
}
case reduced::exp10:
{
std::int64_t whole = 0;
const std::int64_t frac = fractional_raw(raw, fractional_bits, whole);
const std::int64_t natural = mul_raw(
frac, scale_unit(ln10_64, fractional_bits), fractional_bits);
if (whole > 18 || whole < -18)
throw std::overflow_error("range lut: exponent overflow");
return mul_raw(
eval_exp_at_scale(fractional_bits, natural),
pow10_raw(static_cast<int>(whole), fractional_bits),
fractional_bits);
}
case reduced::sin:
return sin_from_angle(
fractional_bits,
reduce_positive(fractional_bits, raw, two_over_pi_64),
raw < 0 ? -1 : 1);
case reduced::cos:
return cos_from_angle(
fractional_bits, reduce_positive(fractional_bits, raw, two_over_pi_64));
case reduced::tan:
{
const std::int64_t y = tan_positive(fractional_bits, abs_raw(raw), 0);
return raw < 0 ? -y : y;
}
case reduced::cot:
{
if (raw == 0)
throw std::domain_error("range lut: cot pole");
const std::int64_t y = -tan_positive(fractional_bits, abs_raw(raw), 2);
return raw < 0 ? -y : y;
}
case reduced::sec:
return sec_positive(fractional_bits, abs_raw(raw), 0);
case reduced::csc:
{
if (raw == 0)
throw std::domain_error("range lut: csc pole");
const std::int64_t y = sec_positive(fractional_bits, abs_raw(raw), -2);
return raw < 0 ? -y : y;
}
case reduced::sinh:
return sinh_cosh(fractional_bits, raw).sinh_raw;
case reduced::cosh:
return sinh_cosh(fractional_bits, raw).cosh_raw;
case reduced::tanh:
{
if (raw == 0)
return 0;
const std::int64_t limit = beta_raw(fractional_bits, false);
const std::int64_t mag = abs_raw(raw);
if (mag >= limit)
return raw < 0 ? -one : one;
const hyp pair = sinh_cosh(fractional_bits, raw);
return div_raw(pair.sinh_raw, pair.cosh_raw, fractional_bits);
}
case reduced::coth:
{
if (raw == 0)
throw std::domain_error("range lut: coth pole");
const std::int64_t limit = beta_raw(fractional_bits, true);
const std::int64_t mag = abs_raw(raw);
std::int64_t y;
if (mag >= limit)
y = one;
else
{
const std::int64_t t = div_raw(mag, limit, fractional_bits);
const std::int64_t argument = t > one ? one : t;
const std::int64_t removed = eval_principal(principal::coth, fractional_bits, argument);
y = removed + div_raw(one, mag, fractional_bits);
}
return raw < 0 ? -y : y;
}
case reduced::sech:
{
const std::int64_t ch = sinh_cosh(fractional_bits, raw).cosh_raw;
return div_raw(one, ch, fractional_bits);
}
case reduced::csch:
{
if (raw == 0)
throw std::domain_error("range lut: csch pole");
const bool neg = raw < 0;
const std::int64_t mag = abs_raw(raw);
std::int64_t y;
if (mag <= one)
{
const std::int64_t removed = eval_principal(principal::csch, fractional_bits, mag);
y = removed + div_raw(one, mag, fractional_bits);
}
else
{
y = div_raw(one, sinh_cosh(fractional_bits, mag).sinh_raw, fractional_bits);
}
return neg ? -y : y;
}
case reduced::sqrt:
{
if (raw == 0)
return 0;
const dyadic part = split_positive(raw, fractional_bits);
std::int64_t root = eval_principal(principal::sqrt, fractional_bits, part.mantissa_raw);
if ((part.power & 1) != 0)
root = mul_raw(root, scale_unit(sqrt2_64, fractional_bits), fractional_bits);
return shift_pow2(root, half_pow_of(part.power));
}
case reduced::inv:
{
const dyadic part = split_positive(raw, fractional_bits);
const std::int64_t reciprocal = eval_principal(
principal::inv, fractional_bits, part.mantissa_raw);
return shift_pow2(reciprocal, -part.power);
}
case reduced::rsqrt:
{
const dyadic part = split_positive(raw, fractional_bits);
std::int64_t root = eval_principal(principal::rsqrt, fractional_bits, part.mantissa_raw);
if ((part.power & 1) != 0)
root = mul_raw(root, scale_unit(rsqrt2_64, fractional_bits), fractional_bits);
return shift_pow2(root, -half_pow_of(part.power));
}
case reduced::invsq:
{
const dyadic part = split_positive(raw, fractional_bits);
const std::int64_t square = eval_principal(
principal::invsq, fractional_bits, part.mantissa_raw);
return shift_pow2(square, -2 * part.power);
}
}
throw std::invalid_argument("range lut: unknown map");
}
} // namespace grotto
#endif // LIBDPF_INCLUDE_GROTTO_RANGE_LUT_HPP__