libdpf/include/dpf/cmp_group.hpp

/// @file dpf/cmp_group.hpp
/// @brief Comparison-payload group for types that do not fit in a masked `uint64_t`.
/// @details Payloads of at most 64 bits that already convert to `uint64_t`
///          stay on that path. Everything else — wider integers, `modint`,
///          `fixedpoint`, `xor_wrapper`, `bitstring`, and `dpf::vec` — is a
///          little-endian limb vector. Lanes of a `vec` add (or XOR) apart,
///          with no carry from one lane into the next. A PRG stretch fills a
///          group element from one GGM node, so the element is uniform even
///          when it is wider than 64 bits.
/// @copyright Copyright (c) 2019-2026 Ryan Henry and [others](@ref authors)
/// @license Released under a GNU General Public v2.0 (GPLv2) license.

#ifndef LIBDPF_INCLUDE_DPF_CMP_GROUP_HPP__
#define LIBDPF_INCLUDE_DPF_CMP_GROUP_HPP__

#include <cstddef>
#include <cstdint>
#include <cstring>
#include <stdexcept>
#include <type_traits>

#include "hedley/hedley.h"

#include "dpf/twiddle.hpp"
#include "dpf/utils.hpp"
#include "dpf/wildcard.hpp"

namespace dpf
{
namespace detail
{

struct group_elem
{
    static constexpr std::size_t cap = 4;
    uint64_t limb[cap]{};
    std::uint16_t lane_bits = 64;
    std::uint16_t lanes = 1;
    bool xor_group = false;
};

template <typename T, typename = void>
struct is_modint_tag : std::false_type {};
template <typename T>
struct is_modint_tag<T, std::void_t<decltype(T::dpf_modint)>>
  : std::bool_constant<T::dpf_modint> {};

template <typename T, typename = void>
struct is_bitstring_tag : std::false_type {};
template <typename T>
struct is_bitstring_tag<T, std::void_t<decltype(T::dpf_bitstring)>>
  : std::bool_constant<T::dpf_bitstring> {};

template <typename T, typename = void>
struct is_vec_tag : std::false_type {};
template <typename T>
struct is_vec_tag<T, std::void_t<decltype(T::dpf_vec)>>
  : std::bool_constant<T::dpf_vec> {};

template <typename T, typename = void>
struct has_from_seed : std::false_type {};
template <typename T>
struct has_from_seed<T, std::void_t<decltype(
    T::from_seed(static_cast<const void *>(nullptr), std::size_t{0}))>>
  : std::true_type {};

template <typename T, typename = void>
struct has_integral_representation : std::false_type {};
template <typename T>
struct has_integral_representation<T, std::void_t<decltype(
    std::declval<const T &>().integral_representation())>>
  : std::true_type {};

template <typename T, bool IsVec>
struct cmp_lane_of { using type = T; };
template <typename T>
struct cmp_lane_of<T, true> { using type = typename T::lane_type; };

template <typename Beta>
struct cmp_group_info
{
    using type = concrete_type_t<std::decay_t<Beta>>;
    static constexpr bool is_vec = is_vec_tag<type>::value;
    using lane = typename cmp_lane_of<type, is_vec>::type;
    static constexpr bool lane_xor =
        utils::is_xor_wrapper_v<lane> || is_bitstring_tag<lane>::value;
    static constexpr std::size_t lanes = []() constexpr {
        if constexpr (is_vec)
            return type::lane_count;
        else
            return std::size_t{1};
    }();
    static constexpr std::size_t lane_bits = utils::bitlength_of_v<lane>;
    static constexpr std::size_t total_bits = lanes * lane_bits;
    /// @brief `uint64_t` ring, including a fixed-point value whose raw word fits.
    static constexpr bool narrow_ring =
        !is_vec && !lane_xor && lane_bits <= 64;
    static constexpr bool custom = !narrow_ring;

    static_assert(!custom || total_bits <= 256,
        "comparison payload exceeds 256 bits");
    static_assert(!custom || total_bits > 0,
        "comparison payload has no bits");
};

template <typename Concrete>
HEDLEY_ALWAYS_INLINE
group_elem group_layout()
{
    using info = cmp_group_info<Concrete>;
    group_elem g;
    g.lane_bits = static_cast<std::uint16_t>(info::lane_bits);
    g.lanes = static_cast<std::uint16_t>(info::lanes);
    g.xor_group = info::lane_xor;
    return g;
}

HEDLEY_ALWAYS_INLINE
group_elem group_zero(const group_elem & layout)
{
    group_elem g;
    g.lane_bits = layout.lane_bits;
    g.lanes = layout.lanes;
    g.xor_group = layout.xor_group;
    return g;
}

inline bool bit_at(const uint64_t limb[4], std::size_t bit) noexcept
{
    return ((limb[bit / 64u] >> (bit % 64u)) & 1ull) != 0;
}

inline void set_bit(uint64_t limb[4], std::size_t bit, bool on) noexcept
{
    const std::size_t i = bit / 64u;
    const uint64_t m = 1ull << (bit % 64u);
    if (on)
        limb[i] |= m;
    else
        limb[i] &= ~m;
}

inline void mask_limbs(uint64_t limb[4], std::size_t bits) noexcept
{
    if (bits >= 256)
        return;
    for (std::size_t b = bits; b < 256; ++b)
        set_bit(limb, b, false);
}

inline void read_lane(const group_elem & g, std::size_t lane, uint64_t out[4]) noexcept
{
    std::memset(out, 0, 4 * sizeof(uint64_t));
    const std::size_t base = lane * g.lane_bits;
    for (std::size_t b = 0; b < g.lane_bits; ++b)
        set_bit(out, b, bit_at(g.limb, base + b));
}

inline void write_lane(group_elem & g, std::size_t lane, const uint64_t in[4]) noexcept
{
    const std::size_t base = lane * g.lane_bits;
    for (std::size_t b = 0; b < g.lane_bits; ++b)
        set_bit(g.limb, base + b, bit_at(in, b));
}

inline void add_lane(uint64_t a[4], const uint64_t b[4], std::size_t bits) noexcept
{
    unsigned carry = 0;
    const std::size_t n = (bits + 63u) / 64u;
    for (std::size_t i = 0; i < n; ++i)
    {
        const unsigned __int128 sum =
            static_cast<unsigned __int128>(a[i]) + b[i] + carry;
        a[i] = static_cast<uint64_t>(sum);
        carry = static_cast<unsigned>(sum >> 64);
    }
    mask_limbs(a, bits);
}

inline void neg_lane(uint64_t a[4], std::size_t bits) noexcept
{
    uint64_t one[4] = {1, 0, 0, 0};
    for (std::size_t i = 0; i < 4; ++i)
        a[i] = ~a[i];
    mask_limbs(a, bits);
    add_lane(a, one, bits);
}

/// @brief Bit-serial product. Used when a wildcard coefficient scales δ, and when
/// interval containment multiplies δ by a small public integer.
/// @param a the `a`
/// @param b the `b`
/// @param bits the packed bits
/// @param out the output buffer
inline void mul_lane(const uint64_t a[4], const uint64_t b[4], std::size_t bits,
    uint64_t out[4]) noexcept
{
    std::memset(out, 0, 4 * sizeof(uint64_t));
    for (std::size_t bit = 0; bit < bits; ++bit)
    {
        if (!bit_at(b, bit))
            continue;
        uint64_t shifted[4]{};
        for (std::size_t s = 0; s < bits; ++s)
        {
            if (s + bit < bits && bit_at(a, s))
                set_bit(shifted, s + bit, true);
        }
        add_lane(out, shifted, bits);
    }
}

inline group_elem group_apply(const group_elem & a, const group_elem & b,
    void (*lane_op)(uint64_t *, const uint64_t *, std::size_t))
{
    group_elem out = group_zero(a);
    for (std::size_t i = 0; i < a.lanes; ++i)
    {
        uint64_t la[4]{}, lb[4]{};
        read_lane(a, i, la);
        read_lane(b, i, lb);
        if (a.xor_group)
        {
            for (std::size_t k = 0; k < 4; ++k)
                la[k] ^= lb[k];
            mask_limbs(la, a.lane_bits);
        }
        else
        {
            lane_op(la, lb, a.lane_bits);
        }
        write_lane(out, i, la);
    }
    return out;
}

inline group_elem group_add(const group_elem & a, const group_elem & b)
{
    return group_apply(a, b, add_lane);
}

inline group_elem group_neg(const group_elem & a)
{
    group_elem out = group_zero(a);
    if (a.xor_group)
        return a;
    for (std::size_t i = 0; i < a.lanes; ++i)
    {
        uint64_t la[4]{};
        read_lane(a, i, la);
        neg_lane(la, a.lane_bits);
        write_lane(out, i, la);
    }
    return out;
}

inline group_elem group_sub(const group_elem & a, const group_elem & b)
{
    if (a.xor_group)
        return group_add(a, b);
    return group_add(a, group_neg(b));
}

inline group_elem group_mul(const group_elem & a, const group_elem & b)
{
    group_elem out = group_zero(a);
    for (std::size_t i = 0; i < a.lanes; ++i)
    {
        uint64_t la[4]{}, lb[4]{}, lc[4]{};
        read_lane(a, i, la);
        read_lane(b, i, lb);
        if (a.xor_group)
        {
            for (std::size_t k = 0; k < 4; ++k)
                lc[k] = la[k] & lb[k];
            mask_limbs(lc, a.lane_bits);
        }
        else
        {
            mul_lane(la, lb, a.lane_bits, lc);
        }
        write_lane(out, i, lc);
    }
    return out;
}

inline group_elem group_sgn(bool t, const group_elem & a)
{
    return t ? group_neg(a) : a;
}

/// @brief Multiplicative identity: `1` in each additive lane, all-ones in an XOR lane.
/// @param layout the `layout`
/// @return Multiplicative identity: `1` in each additive lane, all-ones in an XOR lane
inline group_elem group_one(const group_elem & layout)
{
    group_elem g = group_zero(layout);
    for (std::size_t i = 0; i < g.lanes; ++i)
    {
        uint64_t lane[4]{};
        if (g.xor_group)
        {
            for (std::size_t b = 0; b < g.lane_bits; ++b)
                set_bit(lane, b, true);
        }
        else
        {
            lane[0] = 1;
        }
        write_lane(g, i, lane);
    }
    return g;
}

/// @brief Integer `s` in every lane. Negative `s` is the group negation of `|s|`.
/// @param s the `s`
/// @param layout the `layout`
/// @return Integer `s` in every lane
inline group_elem group_scalar(int s, const group_elem & layout)
{
    if (layout.xor_group)
    {
        if ((s & 1) == 0)
            return group_zero(layout);
        return group_one(layout);
    }
    const bool neg = s < 0;
    const auto mag = static_cast<unsigned>(neg ? -s : s);
    group_elem g = group_zero(layout);
    for (std::size_t i = 0; i < g.lanes; ++i)
    {
        uint64_t lane[4] = {mag, 0, 0, 0};
        mask_limbs(lane, g.lane_bits);
        if (neg)
            neg_lane(lane, g.lane_bits);
        write_lane(g, i, lane);
    }
    return g;
}

inline group_elem group_from_bytes(const unsigned char * bytes, std::size_t nbytes,
    const group_elem & layout)
{
    group_elem g = group_zero(layout);
    const std::size_t need = (static_cast<std::size_t>(layout.lanes) * layout.lane_bits
        + 7u) / 8u;
    if (nbytes < need)
        throw std::invalid_argument("comparison group stretch was short");
    std::size_t bit = 0;
    for (std::size_t lane = 0; lane < layout.lanes; ++lane)
    {
        uint64_t raw[4]{};
        for (std::size_t b = 0; b < layout.lane_bits; ++b, ++bit)
        {
            const unsigned char byte = bytes[bit / 8u];
            const bool on = ((byte >> (bit % 8u)) & 1u) != 0;
            set_bit(raw, b, on);
        }
        write_lane(g, lane, raw);
    }
    return g;
}

template <typename PRG, typename Node>
group_elem group_from_node(Node node, const group_elem & layout)
{
    auto seed = dpf::unset_lo_2bits(node);
    auto kids = PRG::eval01(seed);
    unsigned char bytes[64]{};
    constexpr std::size_t nb = sizeof(kids[0]);
    static_assert(nb <= 32, "comparison stretch expects a 128- or 256-bit block");
    std::memcpy(bytes, &kids[0], nb);
    std::memcpy(bytes + nb, &kids[1], nb);
    return group_from_bytes(bytes, nb * 2, layout);
}

template <typename Word>
Word group_to_word(const group_elem & g)
{
    Word w{};
    static_assert(sizeof(Word) <= sizeof(g.limb),
        "comparison word is wider than 256 bits");
    std::memcpy(&w, g.limb, sizeof(Word));
    return w;
}

template <typename Word>
group_elem group_from_word(const Word & w, const group_elem & layout)
{
    group_elem g = group_zero(layout);
    std::memcpy(g.limb, &w, sizeof(Word) < sizeof(g.limb) ? sizeof(Word) : sizeof(g.limb));
    mask_limbs(g.limb, static_cast<std::size_t>(layout.lanes) * layout.lane_bits);
    return g;
}

template <typename T>
void store_raw_integer(group_elem & g, std::size_t lane, const T & value)
{
    uint64_t tmp[4]{};
    if constexpr (has_integral_representation<T>::value)
    {
        auto raw = value.integral_representation();
        std::memcpy(tmp, &raw, sizeof(raw) < sizeof(tmp) ? sizeof(raw) : sizeof(tmp));
    }
    else if constexpr (is_modint_tag<T>::value)
    {
        auto raw = static_cast<typename T::integral_type>(value);
        std::memcpy(tmp, &raw, sizeof(raw) < sizeof(tmp) ? sizeof(raw) : sizeof(tmp));
    }
    else if constexpr (is_bitstring_tag<T>::value)
    {
        auto raw = utils::to_integral_type<T>{}(value);
        std::memcpy(tmp, &raw, sizeof(raw) < sizeof(tmp) ? sizeof(raw) : sizeof(tmp));
    }
    else if constexpr (utils::is_xor_wrapper_v<T>)
    {
        auto raw = value.data();
        std::memcpy(tmp, &raw, sizeof(raw) < sizeof(tmp) ? sizeof(raw) : sizeof(tmp));
    }
    else
    {
        static_assert(std::is_trivially_copyable_v<T>,
            "comparison payload must be trivially copyable");
        static_assert(sizeof(T) <= sizeof(tmp),
            "comparison payload exceeds 256 bits");
        std::memcpy(tmp, &value, sizeof(T));
    }
    mask_limbs(tmp, g.lane_bits);
    write_lane(g, lane, tmp);
}

template <typename Beta>
group_elem group_from_beta(const Beta & value)
{
    using C = std::decay_t<Beta>;
    group_elem g = group_layout<C>();
    if constexpr (is_vec_tag<C>::value)
    {
        for (std::size_t i = 0; i < C::lane_count; ++i)
        {
            auto lane = group_from_beta(value.lanes[i]);
            uint64_t raw[4]{};
            read_lane(lane, 0, raw);
            write_lane(g, i, raw);
        }
    }
    else
    {
        store_raw_integer(g, 0, value);
    }
    return g;
}

template <typename Beta>
Beta group_to_beta(const group_elem & g)
{
    using C = std::decay_t<Beta>;
    if constexpr (is_vec_tag<C>::value)
    {
        C out{};
        for (std::size_t i = 0; i < C::lane_count; ++i)
        {
            group_elem lane = group_zero(g);
            lane.lanes = 1;
            lane.lane_bits = g.lane_bits;
            lane.xor_group = g.xor_group;
            uint64_t raw[4]{};
            read_lane(g, i, raw);
            write_lane(lane, 0, raw);
            out.lanes[i] = group_to_beta<typename C::lane_type>(lane);
        }
        return out;
    }
    else
    {
        uint64_t tmp[4]{};
        read_lane(g, 0, tmp);
        if constexpr (has_integral_representation<C>::value)
        {
            using integral = typename C::integral_type;
            integral raw{};
            std::memcpy(&raw, tmp, sizeof(raw) < sizeof(tmp) ? sizeof(raw) : sizeof(tmp));
            return C::from_raw(raw);
        }
        else if constexpr (is_modint_tag<C>::value)
        {
            using integral = typename C::integral_type;
            integral raw{};
            std::memcpy(&raw, tmp, sizeof(raw) < sizeof(tmp) ? sizeof(raw) : sizeof(tmp));
            return C{raw};
        }
        else if constexpr (is_bitstring_tag<C>::value)
        {
            using integral = typename utils::make_from_integral_value<C>::integral_type;
            integral raw{};
            std::memcpy(&raw, tmp, sizeof(raw) < sizeof(tmp) ? sizeof(raw) : sizeof(tmp));
            return utils::make_from_integral_value<C>{}(raw);
        }
        else if constexpr (utils::is_xor_wrapper_v<C>)
        {
            using raw_type = typename C::value_type;
            raw_type raw{};
            std::memcpy(&raw, tmp, sizeof(raw) < sizeof(tmp) ? sizeof(raw) : sizeof(tmp));
            return C{raw};
        }
        else
        {
            C raw{};
            std::memcpy(&raw, tmp, sizeof(C) < sizeof(tmp) ? sizeof(C) : sizeof(tmp));
            return raw;
        }
    }
}

template <typename PRG>
group_elem group_value_cw(const typename PRG::block_type & c0L,
    const typename PRG::block_type & c0R,
    const typename PRG::block_type & c1L,
    const typename PRG::block_type & c1R,
    uint8_t t0, uint8_t t1, int ai, group_elem & Va, const group_elem & beta)
{
    (void)t0;
    const group_elem v0L = group_from_node<PRG>(c0L, Va);
    const group_elem v0R = group_from_node<PRG>(c0R, Va);
    const group_elem v1L = group_from_node<PRG>(c1L, Va);
    const group_elem v1R = group_from_node<PRG>(c1R, Va);
    const group_elem & v0K = ai == 0 ? v0L : v0R;
    const group_elem & v1K = ai == 0 ? v1L : v1R;
    const group_elem & v0Lo = ai == 0 ? v0R : v0L;
    const group_elem & v1Lo = ai == 0 ? v1R : v1L;
    group_elem vcw = group_sgn(t1 != 0,
        group_add(group_add(v1Lo, group_neg(v0Lo)), group_neg(Va)));
    // Lose-left plants β. A planted recipe passes the plant in `beta` for both directions.
    if (ai == 1)
        vcw = group_add(vcw, group_sgn(t1 != 0, beta));
    Va = group_add(group_add(group_add(Va, group_neg(v1K)), v0K),
        group_sgn(t1 != 0, vcw));
    return vcw;
}

template <typename PRG>
group_elem group_final_cw(const typename PRG::block_type & s0,
    const typename PRG::block_type & s1, uint8_t t1, const group_elem & Va,
    const group_elem & on_path)
{
    const group_elem c0 = group_from_node<PRG>(s0, Va);
    const group_elem c1 = group_from_node<PRG>(s1, Va);
    return group_sgn(t1 != 0,
        group_add(group_add(group_add(c1, group_neg(c0)), group_neg(Va)), on_path));
}

/// @brief Type-erased group for a comparison payload that supplies `from_seed`,
/// `operator+`, and unary `operator-`. The element is at most 256 bytes.
struct payload_ops
{
    static constexpr std::size_t cap = 256;
    std::size_t size = 0;
    void (*add)(unsigned char *, const unsigned char *, const unsigned char *) = nullptr;
    void (*neg)(unsigned char *, const unsigned char *) = nullptr;
    void (*from_node)(unsigned char *, const void *, std::size_t) = nullptr;
    void (*scale)(unsigned char *, const unsigned char *, std::int64_t) = nullptr;
};

template <typename T>
void payload_add(unsigned char * dst, const unsigned char * a, const unsigned char * b)
{
    T x{}, y{};
    std::memcpy(&x, a, sizeof(T));
    std::memcpy(&y, b, sizeof(T));
    const T z = x + y;
    std::memset(dst, 0, payload_ops::cap);
    std::memcpy(dst, &z, sizeof(T));
}

template <typename T>
void payload_neg(unsigned char * dst, const unsigned char * a)
{
    T x{};
    std::memcpy(&x, a, sizeof(T));
    const T z = -x;
    std::memset(dst, 0, payload_ops::cap);
    std::memcpy(dst, &z, sizeof(T));
}

template <typename T>
void payload_from_node(unsigned char * dst, const void * node, std::size_t n)
{
    const T z = T::from_seed(node, n);
    std::memset(dst, 0, payload_ops::cap);
    std::memcpy(dst, &z, sizeof(T));
}

template <typename T>
void payload_scale(unsigned char * dst, const unsigned char * a, std::int64_t k)
{
    T g{};
    std::memcpy(&g, a, sizeof(T));
    T r{};
    if (k < 0)
    {
        g = -g;
        k = -k;
    }
    auto m = static_cast<std::uint64_t>(k);
    while (m != 0)
    {
        if ((m & 1u) != 0)
            r = r + g;
        m >>= 1;
        if (m != 0)
            g = g + g;
    }
    std::memset(dst, 0, payload_ops::cap);
    std::memcpy(dst, &r, sizeof(T));
}

template <typename T>
payload_ops make_payload_ops()
{
    static_assert(sizeof(T) <= payload_ops::cap,
        "comparison payload exceeds 256 bytes");
    static_assert(has_from_seed<T>::value,
        "comparison payload needs from_seed");
    payload_ops ops;
    ops.size = sizeof(T);
    ops.add = &payload_add<T>;
    ops.neg = &payload_neg<T>;
    ops.from_node = &payload_from_node<T>;
    ops.scale = &payload_scale<T>;
    return ops;
}

inline void payload_copy(unsigned char * dst, const unsigned char * src, std::size_t n)
{
    std::memset(dst, 0, payload_ops::cap);
    if (n != 0)
        std::memcpy(dst, src, n);
}

inline void payload_sgn(const payload_ops & ops, unsigned char * dst,
    const unsigned char * src, bool neg)
{
    if (!neg)
        payload_copy(dst, src, ops.size);
    else
        ops.neg(dst, src);
}

/// @brief One comparison-level correction in `ops`'s group.
/// @details Mirrors `group_value_cw`. When `beta` is null the correction is split
/// into a group element independent of δ and an integer coefficient of δ.
inline void payload_value_cw(const payload_ops & ops,
    const void * n0l, const void * n0r, const void * n1l, const void * n1r,
    std::size_t node_len, std::uint8_t t1, int ai,
    unsigned char * va, std::int64_t & va_c,
    const unsigned char * beta, unsigned char * vcw, std::int64_t * coeff)
{
    unsigned char v0l[payload_ops::cap]{}, v0r[payload_ops::cap]{};
    unsigned char v1l[payload_ops::cap]{}, v1r[payload_ops::cap]{};
    ops.from_node(v0l, n0l, node_len);
    ops.from_node(v0r, n0r, node_len);
    ops.from_node(v1l, n1l, node_len);
    ops.from_node(v1r, n1r, node_len);
    const unsigned char * v0k = ai == 0 ? v0l : v0r;
    const unsigned char * v1k = ai == 0 ? v1l : v1r;
    const unsigned char * v0lo = ai == 0 ? v0r : v0l;
    const unsigned char * v1lo = ai == 0 ? v1r : v1l;
    unsigned char neg_v0[payload_ops::cap]{}, neg_va[payload_ops::cap]{};
    ops.neg(neg_v0, v0lo);
    ops.neg(neg_va, va);
    unsigned char sum[payload_ops::cap]{}, inner[payload_ops::cap]{};
    ops.add(sum, v1lo, neg_v0);
    ops.add(inner, sum, neg_va);
    std::int64_t inner_c = -va_c;
    if (ai == 1)
    {
        if (beta != nullptr)
            ops.add(inner, inner, beta);
        else
            inner_c += 1;
    }
    payload_sgn(ops, vcw, inner, t1 != 0);
    const std::int64_t vcw_c = (t1 != 0) ? -inner_c : inner_c;
    unsigned char sgn_vcw[payload_ops::cap]{};
    payload_sgn(ops, sgn_vcw, vcw, t1 != 0);
    unsigned char neg_v1k[payload_ops::cap]{}, acc[payload_ops::cap]{};
    ops.neg(neg_v1k, v1k);
    ops.add(acc, va, neg_v1k);
    ops.add(acc, acc, v0k);
    ops.add(va, acc, sgn_vcw);
    va_c += (t1 != 0) ? -vcw_c : vcw_c;
    if (coeff != nullptr)
        *coeff = vcw_c;
    if (beta != nullptr && vcw_c != 0)
    {
        unsigned char extra[payload_ops::cap]{};
        ops.scale(extra, beta, vcw_c);
        ops.add(vcw, vcw, extra);
    }
}

inline void payload_final_cw(const payload_ops & ops,
    const void * s0, const void * s1, std::size_t node_len, std::uint8_t t1,
    const unsigned char * va, std::int64_t va_c,
    const unsigned char * on_path, int on_c,
    unsigned char * out, std::int64_t * coeff)
{
    unsigned char c0[payload_ops::cap]{}, c1[payload_ops::cap]{};
    ops.from_node(c0, s0, node_len);
    ops.from_node(c1, s1, node_len);
    unsigned char neg_c0[payload_ops::cap]{}, neg_va[payload_ops::cap]{};
    ops.neg(neg_c0, c0);
    ops.neg(neg_va, va);
    unsigned char sum[payload_ops::cap]{}, inner[payload_ops::cap]{};
    ops.add(sum, c1, neg_c0);
    ops.add(inner, sum, neg_va);
    if (on_path != nullptr)
        ops.add(inner, inner, on_path);
    std::int64_t inner_c = -va_c + on_c;
    payload_sgn(ops, out, inner, t1 != 0);
    const std::int64_t out_c = (t1 != 0) ? -inner_c : inner_c;
    if (coeff != nullptr)
        *coeff = out_c;
}

template <typename Word>
Word payload_to_word(const unsigned char * bytes, std::size_t n)
{
    Word w{};
    std::memcpy(&w, bytes, n < sizeof(Word) ? n : sizeof(Word));
    return w;
}

template <typename T, typename = void>
struct payload_has_canonicalize : std::false_type {};

template <typename T>
struct payload_has_canonicalize<T,
    std::void_t<decltype(T::canonicalize(T{}))>> : std::true_type {};

template <typename T, typename Word>
T payload_from_word(const Word & w)
{
    T t{};
    utils::raw_memcpy(&t, &w, sizeof(T) < sizeof(Word) ? sizeof(T) : sizeof(Word));
    if constexpr (payload_has_canonicalize<T>::value)
        return T::canonicalize(t);
    return t;
}

template <typename Word>
std::int64_t payload_coeff_of(const Word & w)
{
    std::int64_t k = 0;
    std::memcpy(&k, &w, sizeof(k) < sizeof(Word) ? sizeof(k) : sizeof(Word));
    return k;
}

template <typename Word>
Word payload_coeff_word(std::int64_t k)
{
    Word w{};
    std::memcpy(&w, &k, sizeof(k) < sizeof(Word) ? sizeof(k) : sizeof(Word));
    return w;
}

}  // namespace detail
}  // namespace dpf

#endif  // LIBDPF_INCLUDE_DPF_CMP_GROUP_HPP__